<commit_before>package web

import (
	"flag"
	"fmt"
	"github.com/zenoss/go-json-rest"
	"net/http"
	"os"
	"path"
	"runtime"
)

var webroot string

func init() {
	webrootDefault := ""
	servicedHome := os.Getenv("SERVICED_HOME")
	if len(servicedHome) > 0 {
		webrootDefault = servicedHome + "/share/web/static"
	}
	flag.StringVar(&webroot, "webroot", webrootDefault, "static director for web content, defaults to GO runtime path of src")
}

/*******************************************************************************
 *
 * Data Structures
 *
 ******************************************************************************/

type simpleResponse struct {
	Detail string
	links  []link
}

type link struct {
	Name   string
	Method string
	URL    string
}

type login struct {
	Username string
	Password string
}

const createlink = "Create"
const updatelink = "Update"
const retrievelink = "Retrieve"
const deletelink = "Delete"

/*******************************************************************************
 *
 * Public Functions
 *
 ******************************************************************************/

/*
 * Inform the user that a login is required
 */
func restUnauthorized(w *rest.ResponseWriter) {
	writeJSON(w, &simpleResponse{"Not authorized", loginLink()}, http.StatusUnauthorized)
	return
}

/*
 * Provide a generic response for an oopsie.
 */
func restServerError(w *rest.ResponseWriter) {
	writeJSON(w, &simpleResponse{"Internal Server Error", homeLink()}, http.StatusInternalServerError)
	return
}

/*
 * The user sent us junk, or we were incapabale of decoding what they sent.
 */
func restBadRequest(w *rest.ResponseWriter) {
	writeJSON(w, &simpleResponse{"Bad Request", homeLink()}, http.StatusBadRequest)
	return
}

/*
 * Write 200 success
 */
func restSuccess(w *rest.ResponseWriter) {
	w.WriteHeader(200)
	return
}

// WriteJSON struct as JSON with specified HTTP status code
func writeJSON(w *rest.ResponseWriter, v interface{}, code int) {
	w.Header().Set("content-type", "application/json")
	w.WriteHeader(code)
	err := w.WriteJson(v)
	if err != nil {
		panic(err)
	}
}

/*
 * Provides content for root /
 */
func mainPage(w *rest.ResponseWriter, r *rest.Request) {
	noCache(w)
	http.ServeFile(
		w.ResponseWriter,
		r.Request,
		staticRoot()+"/index.html")
}

/*
 * Provides content for /test
 */
func testPage(w *rest.ResponseWriter, r *rest.Request) {
	noCache(w)
	http.ServeFile(
		w.ResponseWriter,
		r.Request,
		staticRoot()+"/test/index.html")
}

/*
 * Provides content for /favicon.ico
 */
func favIcon(w *rest.ResponseWriter, r *rest.Request) {
	http.ServeFile(
		w.ResponseWriter,
		r.Request,
		staticRoot()+"/ico/zenoss-o.png")
}

/*
 * Serves content from static/
 */
func staticData(w *rest.ResponseWriter, r *rest.Request) {
	fileToServe := path.Join(staticRoot(), r.PathParam("resource"))
	http.ServeFile(
		w.ResponseWriter,
		r.Request,
		fileToServe)
}

/*******************************************************************************
 *
 * Private helper functions
 *
 ******************************************************************************/

/*
 * Provide a list of login related API calls
 */
func loginLink() []link {
	return []link{
		link{createlink, "POST", "/login"},
		link{deletelink, "DELETE", "/login"},
	}
}

/*
 * Provide a basic link to the index
 */
func homeLink() []link {
	return []link{link{retrievelink, "GET", "/"}}
}

/*
 * Provide a list of host related API calls
 */
func hostsLinks() []link {
	return []link{
		link{retrievelink, "GET", "/hosts"},
		link{createlink, "POST", "/hosts/add"},
	}
}

func hostLinks(hostID string) []link {
	hostURI := fmt.Sprintf("/hosts/%s", hostID)
	return []link{
		link{retrievelink, "GET", hostURI},
		link{updatelink, "PUT", hostURI},
		link{deletelink, "DELETE", hostURI},
	}
}

/*
 * Provide a list of pool related API calls
 */
func poolsLinks() []link {
	return []link{
		link{retrievelink, "GET", "/pools"},
		link{createlink, "POST", "/pools/add"},
	}
}

func poolLinks(poolID string) []link {
	poolURI := fmt.Sprintf("/pools/%s", poolID)
	return []link{
		link{retrievelink, "GET", poolURI},
		link{"RetrieveHosts", "GET", poolURI + "/hosts"},
		link{updatelink, "PUT", poolURI},
		link{deletelink, "DELETE", poolURI},
	}
}

func servicesLinks() []link {
	return []link{
		link{retrievelink, "GET", servicesURI},
		link{createlink, "POST", servicesURI + "/add"},
	}
}

/*
 * Provide a list of service related API calls
 */
func serviceLinks(serviceID string) []link {
	serviceURI := fmt.Sprintf("/services/%s", serviceID)
	return []link{
		link{retrievelink, "GET", serviceURI},
		link{"ServiceLogs", "GET", serviceURI + "/logs"},
		link{updatelink, "PUT", serviceURI},
		link{deletelink, "DELETE", serviceURI},
	}
}

/*
 * Provide a list of template related API calls.
 */
func templatesLink() []link {
	return []link{
		link{retrievelink, "GET", "/templates"},
		link{createlink, "POST", "/templates/add"},
		link{"Deploy", "POST", "/templates/deploy"},
	}
}

func templateLinks(templateID string) []link {
	templateURI := fmt.Sprintf("/templates/%s", templateID)
	return []link{
		link{retrievelink, "GET", templateURI},
		link{updatelink, "PUT", templateURI},
		link{deletelink, "DELETE", templateURI},
	}
}

/*
 * Inform browsers that this call should not be cached. Ever.
 */
func noCache(w *rest.ResponseWriter) {
	headers := w.ResponseWriter.Header()
	headers.Add("Cache-Control", "no-cache, no-store, must-revalidate")
	headers.Add("Pragma", "no-cache")
	headers.Add("Expires", "0")
}

/*
 * Hack to get us the location on the filesystem of our static files.
 */
func staticRoot() string {
	if len(webroot) == 0 {
		_, filename, _, _ := runtime.Caller(1)
		return path.Join(path.Dir(filename), "static")
	}
	return webroot
}

const servicesURI = "/services"
const hostsURI = "/hosts"
const templatesURI = "/templates"
const poolsURI = "/pools"
<commit_msg>Links in simpleResponse struct needs to be public so it gets marshalled into JSON. Also changed URL -> Url because that's what it was before<commit_after>package web

import (
	"flag"
	"fmt"
	"github.com/zenoss/go-json-rest"
	"net/http"
	"os"
	"path"
	"runtime"
)

var webroot string

func init() {
	webrootDefault := ""
	servicedHome := os.Getenv("SERVICED_HOME")
	if len(servicedHome) > 0 {
		webrootDefault = servicedHome + "/share/web/static"
	}
	flag.StringVar(&webroot, "webroot", webrootDefault, "static director for web content, defaults to GO runtime path of src")
}

/*******************************************************************************
 *
 * Data Structures
 *
 ******************************************************************************/

type simpleResponse struct {
	Detail string
	Links  []link
}

type link struct {
	Name   string
	Method string
	Url    string
}

type login struct {
	Username string
	Password string
}

const createlink = "Create"
const updatelink = "Update"
const retrievelink = "Retrieve"
const deletelink = "Delete"

/*******************************************************************************
 *
 * Public Functions
 *
 ******************************************************************************/

/*
 * Inform the user that a login is required
 */
func restUnauthorized(w *rest.ResponseWriter) {
	writeJSON(w, &simpleResponse{"Not authorized", loginLink()}, http.StatusUnauthorized)
	return
}

/*
 * Provide a generic response for an oopsie.
 */
func restServerError(w *rest.ResponseWriter) {
	writeJSON(w, &simpleResponse{"Internal Server Error", homeLink()}, http.StatusInternalServerError)
	return
}

/*
 * The user sent us junk, or we were incapabale of decoding what they sent.
 */
func restBadRequest(w *rest.ResponseWriter) {
	writeJSON(w, &simpleResponse{"Bad Request", homeLink()}, http.StatusBadRequest)
	return
}

/*
 * Write 200 success
 */
func restSuccess(w *rest.ResponseWriter) {
	w.WriteHeader(200)
	return
}

// WriteJSON struct as JSON with specified HTTP status code
func writeJSON(w *rest.ResponseWriter, v interface{}, code int) {
	w.Header().Set("content-type", "application/json")
	w.WriteHeader(code)
	err := w.WriteJson(v)
	if err != nil {
		panic(err)
	}
}

/*
 * Provides content for root /
 */
func mainPage(w *rest.ResponseWriter, r *rest.Request) {
	noCache(w)
	http.ServeFile(
		w.ResponseWriter,
		r.Request,
		staticRoot()+"/index.html")
}

/*
 * Provides content for /test
 */
func testPage(w *rest.ResponseWriter, r *rest.Request) {
	noCache(w)
	http.ServeFile(
		w.ResponseWriter,
		r.Request,
		staticRoot()+"/test/index.html")
}

/*
 * Provides content for /favicon.ico
 */
func favIcon(w *rest.ResponseWriter, r *rest.Request) {
	http.ServeFile(
		w.ResponseWriter,
		r.Request,
		staticRoot()+"/ico/zenoss-o.png")
}

/*
 * Serves content from static/
 */
func staticData(w *rest.ResponseWriter, r *rest.Request) {
	fileToServe := path.Join(staticRoot(), r.PathParam("resource"))
	http.ServeFile(
		w.ResponseWriter,
		r.Request,
		fileToServe)
}

/*******************************************************************************
 *
 * Private helper functions
 *
 ******************************************************************************/

/*
 * Provide a list of login related API calls
 */
func loginLink() []link {
	return []link{
		link{createlink, "POST", "/login"},
		link{deletelink, "DELETE", "/login"},
	}
}

/*
 * Provide a basic link to the index
 */
func homeLink() []link {
	return []link{link{retrievelink, "GET", "/"}}
}

/*
 * Provide a list of host related API calls
 */
func hostsLinks() []link {
	return []link{
		link{retrievelink, "GET", "/hosts"},
		link{createlink, "POST", "/hosts/add"},
	}
}

func hostLinks(hostID string) []link {
	hostURI := fmt.Sprintf("/hosts/%s", hostID)
	return []link{
		link{retrievelink, "GET", hostURI},
		link{updatelink, "PUT", hostURI},
		link{deletelink, "DELETE", hostURI},
	}
}

/*
 * Provide a list of pool related API calls
 */
func poolsLinks() []link {
	return []link{
		link{retrievelink, "GET", "/pools"},
		link{createlink, "POST", "/pools/add"},
	}
}

func poolLinks(poolID string) []link {
	poolURI := fmt.Sprintf("/pools/%s", poolID)
	return []link{
		link{retrievelink, "GET", poolURI},
		link{"RetrieveHosts", "GET", poolURI + "/hosts"},
		link{updatelink, "PUT", poolURI},
		link{deletelink, "DELETE", poolURI},
	}
}

func servicesLinks() []link {
	return []link{
		link{retrievelink, "GET", servicesURI},
		link{createlink, "POST", servicesURI + "/add"},
	}
}

/*
 * Provide a list of service related API calls
 */
func serviceLinks(serviceID string) []link {
	serviceURI := fmt.Sprintf("/services/%s", serviceID)
	return []link{
		link{retrievelink, "GET", serviceURI},
		link{"ServiceLogs", "GET", serviceURI + "/logs"},
		link{updatelink, "PUT", serviceURI},
		link{deletelink, "DELETE", serviceURI},
	}
}

/*
 * Provide a list of template related API calls.
 */
func templatesLink() []link {
	return []link{
		link{retrievelink, "GET", "/templates"},
		link{createlink, "POST", "/templates/add"},
		link{"Deploy", "POST", "/templates/deploy"},
	}
}

func templateLinks(templateID string) []link {
	templateURI := fmt.Sprintf("/templates/%s", templateID)
	return []link{
		link{retrievelink, "GET", templateURI},
		link{updatelink, "PUT", templateURI},
		link{deletelink, "DELETE", templateURI},
	}
}

/*
 * Inform browsers that this call should not be cached. Ever.
 */
func noCache(w *rest.ResponseWriter) {
	headers := w.ResponseWriter.Header()
	headers.Add("Cache-Control", "no-cache, no-store, must-revalidate")
	headers.Add("Pragma", "no-cache")
	headers.Add("Expires", "0")
}

/*
 * Hack to get us the location on the filesystem of our static files.
 */
func staticRoot() string {
	if len(webroot) == 0 {
		_, filename, _, _ := runtime.Caller(1)
		return path.Join(path.Dir(filename), "static")
	}
	return webroot
}

const servicesURI = "/services"
const hostsURI = "/hosts"
const templatesURI = "/templates"
const poolsURI = "/pools"
<|endoftext|>


<commit_before>package gonvim

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"
	"unsafe"

	"github.com/dzhou121/neovim-fzf-shim/rplugin/go/fzf"
	"github.com/dzhou121/ui"
	"github.com/neovim/go-client/nvim"
)

var editor *Editor

// Highlight is
type Highlight struct {
	foreground *RGBA
	background *RGBA
}

// Char is
type Char struct {
	char      string
	highlight Highlight
}

// Editor is the editor
type Editor struct {
	nvim         *nvim.Nvim
	nvimAttached bool
	mode         string
	font         *Font
	rows         int
	cols         int
	cursor       *CursorHandler
	Foreground   RGBA
	Background   RGBA
	window       *ui.Window
	area         *ui.Area
	areaHandler  *AreaHandler
	close        chan bool
	popup        *PopupMenu
	finder       *Finder
	width        int
	height       int
	selectedBg   *RGBA
	matchFg      *RGBA
}

func initWindow(box *ui.Box, width, height int) *ui.Window {
	window := ui.NewWindow("Gonvim", width, height, false)
	window.SetChild(box)
	window.OnClosing(func(*ui.Window) bool {
		ui.Quit()
		return true
	})
	window.OnContentSizeChanged(func(w *ui.Window, data unsafe.Pointer) bool {
		if editor == nil {
			return true
		}
		width, height = window.ContentSize()
		if width == editor.width && height == editor.height {
			return true
		}
		editor.width = width
		editor.height = height
		editor.area.SetSize(width, height)
		editor.resize()
		editor.finder.rePosition()
		return true
	})
	window.Show()
	return window
}

// InitEditor inits the editor
func InitEditor() error {
	if editor != nil {
		return nil
	}
	width := 800
	height := 600
	ah := initArea()
	cursor := &CursorHandler{}
	cursorArea := ui.NewArea(cursor)
	cursor.area = cursorArea

	popupMenu := initPopupmenu()
	finder := initFinder()

	box := ui.NewHorizontalBox()
	box.Append(ah.area, false)
	box.Append(cursor.area, false)
	box.Append(popupMenu.box, false)
	box.Append(finder.box, false)

	ah.area.SetSize(width, height)
	// ah.area.SetPosition(100, 100)
	window := initWindow(box, width, height)

	neovim, err := nvim.NewEmbedded(&nvim.EmbedOptions{
		Args: os.Args[1:],
	})
	if err != nil {
		return err
	}

	font := initFont("", 14, 0)

	editor = &Editor{
		nvim:         neovim,
		nvimAttached: false,
		window:       window,
		area:         ah.area,
		areaHandler:  ah,
		mode:         "normal",
		close:        make(chan bool),
		cursor:       cursor,
		popup:        popupMenu,
		finder:       finder,
		width:        width,
		height:       height,
		font:         font,
		cols:         0,
		rows:         0,
		selectedBg:   newRGBA(81, 154, 186, 0.6),
		matchFg:      newRGBA(81, 154, 186, 1),
	}

	editor.resize()
	editor.handleNotification()
	editor.finder.rePosition()
	go func() {
		neovim.Serve()
		editor.close <- true
	}()

	o := make(map[string]interface{})
	o["rgb"] = true
	o["popupmenu_external"] = true
	editor.nvim.AttachUI(editor.cols, editor.rows, o)
	editor.nvim.Subscribe("Gui")
	editor.nvim.Command("runtime plugin/nvim_gui_shim.vim")
	editor.nvim.Command("runtime! ginit.vim")
	fzf.RegisterPlugin(editor.nvim)

	go func() {
		<-editor.close
		ui.Quit()
	}()

	return nil
}

func (e *Editor) handleNotification() {
	ah := e.areaHandler
	e.nvim.RegisterHandler("Gui", func(updates ...interface{}) {
		event := updates[0].(string)
		switch event {
		case "Font":
			e.guiFont(updates[1:])
		case "Linespace":
			e.guiLinespace(updates[1:])
		case "finder_pattern":
			e.finder.showPattern(updates[1:])
		case "finder_pattern_pos":
			e.finder.cursorPos(updates[1:])
		case "finder_show_result":
			e.finder.showResult(updates[1:])
		case "finder_show":
			e.finder.show()
		case "finder_hide":
			e.finder.hide()
		case "finder_select":
			e.finder.selectResult(updates[1:])
		default:
			fmt.Println("unhandled Gui event", event)
		}
	})
	mutex := &sync.Mutex{}
	e.nvim.RegisterHandler("redraw", func(updates ...[]interface{}) {
		mutex.Lock()
		for _, update := range updates {
			event := update[0].(string)
			args := update[1:]
			switch event {
			case "update_fg":
				args := update[1].([]interface{})
				editor.Foreground = calcColor(reflectToInt(args[0]))
			case "update_bg":
				args := update[1].([]interface{})
				bg := calcColor(reflectToInt(args[0]))
				editor.Background = bg
			case "cursor_goto":
				ah.cursorGoto(args)
			case "put":
				ah.put(args)
			case "eol_clear":
				ah.eolClear(args)
			case "clear":
				ah.clear(args)
			case "resize":
				ah.resize(args)
			case "highlight_set":
				ah.highlightSet(args)
			case "set_scroll_region":
				ah.setScrollRegion(args)
			case "scroll":
				ah.scroll(args)
			case "mode_change":
				ah.modeChange(args)
			case "popupmenu_show":
				editor.popup.show(args)
			case "popupmenu_hide":
				editor.popup.hide(args)
			case "popupmenu_select":
				editor.popup.selectItem(args)
			default:
				fmt.Println("Unhandle event", event)
			}
		}
		mutex.Unlock()
		if !e.nvimAttached {
			e.nvimAttached = true
		}
		drawCursor()
	})
}

func (e *Editor) guiFont(args ...interface{}) {
	fontArg := args[0].([]interface{})
	parts := strings.Split(fontArg[0].(string), ":")
	if len(parts) < 1 {
		return
	}

	height := 14
	for _, p := range parts[1:] {
		if strings.HasPrefix(p, "h") {
			var err error
			height, err = strconv.Atoi(p[1:])
			if err != nil {
				return
			}
		}
	}

	e.font.change(parts[0], height)
	e.resize()
}

func (e *Editor) guiLinespace(args ...interface{}) {
	fontArg := args[0].([]interface{})
	lineSpace, err := strconv.Atoi(fontArg[0].(string))
	if err != nil {
		return
	}
	e.font.changeLineSpace(lineSpace)
	e.resize()
}

func (e *Editor) resize() {
	width := e.width
	height := e.height
	cols := width / editor.font.width
	rows := height / editor.font.lineHeight
	oldCols := editor.cols
	oldRows := editor.rows
	editor.cols = cols
	editor.rows = rows
	if oldCols > 0 && oldRows > 0 {
		editor.nvim.TryResizeUI(cols, rows)
	}
}

func drawCursor() {
	row := editor.areaHandler.cursor[0]
	col := editor.areaHandler.cursor[1]
	ui.QueueMain(func() {
		editor.cursor.area.SetPosition(col*editor.font.width, row*editor.font.lineHeight)
	})

	mode := editor.mode
	if mode == "normal" {
		ui.QueueMain(func() {
			editor.cursor.area.SetSize(editor.font.width, editor.font.lineHeight)
			editor.cursor.bg = newRGBA(255, 255, 255, 0.5)
		})
	} else if mode == "insert" {
		ui.QueueMain(func() {
			editor.cursor.area.SetSize(1, editor.font.lineHeight)
			editor.cursor.bg = newRGBA(255, 255, 255, 0.9)
		})
	}
}

func (hl *Highlight) copy() Highlight {
	highlight := Highlight{}
	if hl.foreground != nil {
		highlight.foreground = hl.foreground.copy()
	}
	if hl.background != nil {
		highlight.background = hl.background.copy()
	}
	return highlight
}
<commit_msg>change selected bg back<commit_after>package gonvim

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"sync"
	"unsafe"

	"github.com/dzhou121/neovim-fzf-shim/rplugin/go/fzf"
	"github.com/dzhou121/ui"
	"github.com/neovim/go-client/nvim"
)

var editor *Editor

// Highlight is
type Highlight struct {
	foreground *RGBA
	background *RGBA
}

// Char is
type Char struct {
	char      string
	highlight Highlight
}

// Editor is the editor
type Editor struct {
	nvim         *nvim.Nvim
	nvimAttached bool
	mode         string
	font         *Font
	rows         int
	cols         int
	cursor       *CursorHandler
	Foreground   RGBA
	Background   RGBA
	window       *ui.Window
	area         *ui.Area
	areaHandler  *AreaHandler
	close        chan bool
	popup        *PopupMenu
	finder       *Finder
	width        int
	height       int
	selectedBg   *RGBA
	matchFg      *RGBA
}

func initWindow(box *ui.Box, width, height int) *ui.Window {
	window := ui.NewWindow("Gonvim", width, height, false)
	window.SetChild(box)
	window.OnClosing(func(*ui.Window) bool {
		ui.Quit()
		return true
	})
	window.OnContentSizeChanged(func(w *ui.Window, data unsafe.Pointer) bool {
		if editor == nil {
			return true
		}
		width, height = window.ContentSize()
		if width == editor.width && height == editor.height {
			return true
		}
		editor.width = width
		editor.height = height
		editor.area.SetSize(width, height)
		editor.resize()
		editor.finder.rePosition()
		return true
	})
	window.Show()
	return window
}

// InitEditor inits the editor
func InitEditor() error {
	if editor != nil {
		return nil
	}
	width := 800
	height := 600
	ah := initArea()
	cursor := &CursorHandler{}
	cursorArea := ui.NewArea(cursor)
	cursor.area = cursorArea

	popupMenu := initPopupmenu()
	finder := initFinder()

	box := ui.NewHorizontalBox()
	box.Append(ah.area, false)
	box.Append(cursor.area, false)
	box.Append(popupMenu.box, false)
	box.Append(finder.box, false)

	ah.area.SetSize(width, height)
	// ah.area.SetPosition(100, 100)
	window := initWindow(box, width, height)

	neovim, err := nvim.NewEmbedded(&nvim.EmbedOptions{
		Args: os.Args[1:],
	})
	if err != nil {
		return err
	}

	font := initFont("", 14, 0)

	editor = &Editor{
		nvim:         neovim,
		nvimAttached: false,
		window:       window,
		area:         ah.area,
		areaHandler:  ah,
		mode:         "normal",
		close:        make(chan bool),
		cursor:       cursor,
		popup:        popupMenu,
		finder:       finder,
		width:        width,
		height:       height,
		font:         font,
		cols:         0,
		rows:         0,
		selectedBg:   newRGBA(81, 154, 186, 0.5),
		matchFg:      newRGBA(81, 154, 186, 1),
	}

	editor.resize()
	editor.handleNotification()
	editor.finder.rePosition()
	go func() {
		neovim.Serve()
		editor.close <- true
	}()

	o := make(map[string]interface{})
	o["rgb"] = true
	o["popupmenu_external"] = true
	editor.nvim.AttachUI(editor.cols, editor.rows, o)
	editor.nvim.Subscribe("Gui")
	editor.nvim.Command("runtime plugin/nvim_gui_shim.vim")
	editor.nvim.Command("runtime! ginit.vim")
	fzf.RegisterPlugin(editor.nvim)

	go func() {
		<-editor.close
		ui.Quit()
	}()

	return nil
}

func (e *Editor) handleNotification() {
	ah := e.areaHandler
	e.nvim.RegisterHandler("Gui", func(updates ...interface{}) {
		event := updates[0].(string)
		switch event {
		case "Font":
			e.guiFont(updates[1:])
		case "Linespace":
			e.guiLinespace(updates[1:])
		case "finder_pattern":
			e.finder.showPattern(updates[1:])
		case "finder_pattern_pos":
			e.finder.cursorPos(updates[1:])
		case "finder_show_result":
			e.finder.showResult(updates[1:])
		case "finder_show":
			e.finder.show()
		case "finder_hide":
			e.finder.hide()
		case "finder_select":
			e.finder.selectResult(updates[1:])
		default:
			fmt.Println("unhandled Gui event", event)
		}
	})
	mutex := &sync.Mutex{}
	e.nvim.RegisterHandler("redraw", func(updates ...[]interface{}) {
		mutex.Lock()
		for _, update := range updates {
			event := update[0].(string)
			args := update[1:]
			switch event {
			case "update_fg":
				args := update[1].([]interface{})
				editor.Foreground = calcColor(reflectToInt(args[0]))
			case "update_bg":
				args := update[1].([]interface{})
				bg := calcColor(reflectToInt(args[0]))
				editor.Background = bg
			case "cursor_goto":
				ah.cursorGoto(args)
			case "put":
				ah.put(args)
			case "eol_clear":
				ah.eolClear(args)
			case "clear":
				ah.clear(args)
			case "resize":
				ah.resize(args)
			case "highlight_set":
				ah.highlightSet(args)
			case "set_scroll_region":
				ah.setScrollRegion(args)
			case "scroll":
				ah.scroll(args)
			case "mode_change":
				ah.modeChange(args)
			case "popupmenu_show":
				editor.popup.show(args)
			case "popupmenu_hide":
				editor.popup.hide(args)
			case "popupmenu_select":
				editor.popup.selectItem(args)
			default:
				fmt.Println("Unhandle event", event)
			}
		}
		mutex.Unlock()
		if !e.nvimAttached {
			e.nvimAttached = true
		}
		drawCursor()
	})
}

func (e *Editor) guiFont(args ...interface{}) {
	fontArg := args[0].([]interface{})
	parts := strings.Split(fontArg[0].(string), ":")
	if len(parts) < 1 {
		return
	}

	height := 14
	for _, p := range parts[1:] {
		if strings.HasPrefix(p, "h") {
			var err error
			height, err = strconv.Atoi(p[1:])
			if err != nil {
				return
			}
		}
	}

	e.font.change(parts[0], height)
	e.resize()
}

func (e *Editor) guiLinespace(args ...interface{}) {
	fontArg := args[0].([]interface{})
	lineSpace, err := strconv.Atoi(fontArg[0].(string))
	if err != nil {
		return
	}
	e.font.changeLineSpace(lineSpace)
	e.resize()
}

func (e *Editor) resize() {
	width := e.width
	height := e.height
	cols := width / editor.font.width
	rows := height / editor.font.lineHeight
	oldCols := editor.cols
	oldRows := editor.rows
	editor.cols = cols
	editor.rows = rows
	if oldCols > 0 && oldRows > 0 {
		editor.nvim.TryResizeUI(cols, rows)
	}
}

func drawCursor() {
	row := editor.areaHandler.cursor[0]
	col := editor.areaHandler.cursor[1]
	ui.QueueMain(func() {
		editor.cursor.area.SetPosition(col*editor.font.width, row*editor.font.lineHeight)
	})

	mode := editor.mode
	if mode == "normal" {
		ui.QueueMain(func() {
			editor.cursor.area.SetSize(editor.font.width, editor.font.lineHeight)
			editor.cursor.bg = newRGBA(255, 255, 255, 0.5)
		})
	} else if mode == "insert" {
		ui.QueueMain(func() {
			editor.cursor.area.SetSize(1, editor.font.lineHeight)
			editor.cursor.bg = newRGBA(255, 255, 255, 0.9)
		})
	}
}

func (hl *Highlight) copy() Highlight {
	highlight := Highlight{}
	if hl.foreground != nil {
		highlight.foreground = hl.foreground.copy()
	}
	if hl.background != nil {
		highlight.background = hl.background.copy()
	}
	return highlight
}
<|endoftext|>


<commit_before>package registry

import (
	"fmt"
	"os"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage"
	"github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/factory"
	"github.com/docker/libtrust"
	"github.com/spf13/cobra"
)

func emit(format string, a ...interface{}) {
	if dryRun {
		fmt.Printf(format+"\n", a...)
	}
}

func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error {

	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
	if !ok {
		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
	}

	// mark
	markSet := make(map[digest.Digest]struct{})
	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
		emit(repoName)

		var err error
		named, err := reference.ParseNamed(repoName)
		if err != nil {
			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
		}
		repository, err := registry.Repository(ctx, named)
		if err != nil {
			return fmt.Errorf("failed to construct repository: %v", err)
		}

		manifestService, err := repository.Manifests(ctx)
		if err != nil {
			return fmt.Errorf("failed to construct manifest service: %v", err)
		}

		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
		if !ok {
			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
		}

		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
			// Mark the manifest's blob
			emit("%s: marking manifest %s ", repoName, dgst)
			markSet[dgst] = struct{}{}

			manifest, err := manifestService.Get(ctx, dgst)
			if err != nil {
				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
			}

			descriptors := manifest.References()
			for _, descriptor := range descriptors {
				markSet[descriptor.Digest] = struct{}{}
				emit("%s: marking blob %s", repoName, descriptor.Digest)
			}

			switch manifest.(type) {
			case *schema1.SignedManifest:
				signaturesGetter, ok := manifestService.(distribution.SignaturesGetter)
				if !ok {
					return fmt.Errorf("unable to convert ManifestService into SignaturesGetter")
				}
				signatures, err := signaturesGetter.GetSignatures(ctx, dgst)
				if err != nil {
					return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
				}
				for _, signatureDigest := range signatures {
					emit("%s: marking signature %s", repoName, signatureDigest)
					markSet[signatureDigest] = struct{}{}
				}
				break
			case *schema2.DeserializedManifest:
				config := manifest.(*schema2.DeserializedManifest).Config
				emit("%s: marking configuration %s", repoName, config.Digest)
				markSet[config.Digest] = struct{}{}
				break
			}

			return nil
		})

		return err
	})

	if err != nil {
		return fmt.Errorf("failed to mark: %v\n", err)
	}

	// sweep
	blobService := registry.Blobs()
	deleteSet := make(map[digest.Digest]struct{})
	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
		// check if digest is in markSet. If not, delete it!
		if _, ok := markSet[dgst]; !ok {
			deleteSet[dgst] = struct{}{}
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error enumerating blobs: %v", err)
	}

	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
	// Construct vacuum
	vacuum := storage.NewVacuum(ctx, storageDriver)
	for dgst := range deleteSet {
		emit("blob eligible for deletion: %s", dgst)
		if dryRun {
			continue
		}
		err = vacuum.RemoveBlob(string(dgst))
		if err != nil {
			return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
		}
	}

	return err
}

func init() {
	GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything except remove the blobs")
}

var dryRun bool

// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
var GCCmd = &cobra.Command{
	Use:   "garbage-collect <config>",
	Short: "`garbage-collect` deletes layers not referenced by any manifests",
	Long:  "`garbage-collect` deletes layers not referenced by any manifests",
	Run: func(cmd *cobra.Command, args []string) {
		config, err := resolveConfiguration(args)
		if err != nil {
			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
			cmd.Usage()
			os.Exit(1)
		}

		driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
			os.Exit(1)
		}

		ctx := context.Background()
		ctx, err = configureLogging(ctx, config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
			os.Exit(1)
		}

		k, err := libtrust.GenerateECP256PrivateKey()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			os.Exit(1)
		}

		registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
			os.Exit(1)
		}

		err = markAndSweep(ctx, driver, registry)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
			os.Exit(1)
		}
	},
}
<commit_msg>Ensure GC continues marking if _manifests is nonexistent<commit_after>package registry

import (
	"fmt"
	"os"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/manifest/schema1"
	"github.com/docker/distribution/manifest/schema2"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage"
	"github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/factory"
	"github.com/docker/libtrust"
	"github.com/spf13/cobra"
)

func emit(format string, a ...interface{}) {
	if dryRun {
		fmt.Printf(format+"\n", a...)
	}
}

func markAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace) error {

	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
	if !ok {
		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
	}

	// mark
	markSet := make(map[digest.Digest]struct{})
	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
		emit(repoName)

		var err error
		named, err := reference.ParseNamed(repoName)
		if err != nil {
			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
		}
		repository, err := registry.Repository(ctx, named)
		if err != nil {
			return fmt.Errorf("failed to construct repository: %v", err)
		}

		manifestService, err := repository.Manifests(ctx)
		if err != nil {
			return fmt.Errorf("failed to construct manifest service: %v", err)
		}

		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
		if !ok {
			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
		}

		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
			// Mark the manifest's blob
			emit("%s: marking manifest %s ", repoName, dgst)
			markSet[dgst] = struct{}{}

			manifest, err := manifestService.Get(ctx, dgst)
			if err != nil {
				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
			}

			descriptors := manifest.References()
			for _, descriptor := range descriptors {
				markSet[descriptor.Digest] = struct{}{}
				emit("%s: marking blob %s", repoName, descriptor.Digest)
			}

			switch manifest.(type) {
			case *schema1.SignedManifest:
				signaturesGetter, ok := manifestService.(distribution.SignaturesGetter)
				if !ok {
					return fmt.Errorf("unable to convert ManifestService into SignaturesGetter")
				}
				signatures, err := signaturesGetter.GetSignatures(ctx, dgst)
				if err != nil {
					return fmt.Errorf("failed to get signatures for signed manifest: %v", err)
				}
				for _, signatureDigest := range signatures {
					emit("%s: marking signature %s", repoName, signatureDigest)
					markSet[signatureDigest] = struct{}{}
				}
				break
			case *schema2.DeserializedManifest:
				config := manifest.(*schema2.DeserializedManifest).Config
				emit("%s: marking configuration %s", repoName, config.Digest)
				markSet[config.Digest] = struct{}{}
				break
			}

			return nil
		})

		if err != nil {
			// In certain situations such as unfinished uploads, deleting all
			// tags in S3 or removing the _manifests folder manually, this
			// error may be of type PathNotFound.
			//
			// In these cases we can continue marking other manifests safely.
			if _, ok := err.(driver.PathNotFoundError); ok {
				return nil
			}
		}

		return err
	})

	if err != nil {
		return fmt.Errorf("failed to mark: %v\n", err)
	}

	// sweep
	blobService := registry.Blobs()
	deleteSet := make(map[digest.Digest]struct{})
	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
		// check if digest is in markSet. If not, delete it!
		if _, ok := markSet[dgst]; !ok {
			deleteSet[dgst] = struct{}{}
		}
		return nil
	})
	if err != nil {
		return fmt.Errorf("error enumerating blobs: %v", err)
	}

	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
	// Construct vacuum
	vacuum := storage.NewVacuum(ctx, storageDriver)
	for dgst := range deleteSet {
		emit("blob eligible for deletion: %s", dgst)
		if dryRun {
			continue
		}
		err = vacuum.RemoveBlob(string(dgst))
		if err != nil {
			return fmt.Errorf("failed to delete blob %s: %v\n", dgst, err)
		}
	}

	return err
}

func init() {
	GCCmd.Flags().BoolVarP(&dryRun, "dry-run", "d", false, "do everything except remove the blobs")
}

var dryRun bool

// GCCmd is the cobra command that corresponds to the garbage-collect subcommand
var GCCmd = &cobra.Command{
	Use:   "garbage-collect <config>",
	Short: "`garbage-collect` deletes layers not referenced by any manifests",
	Long:  "`garbage-collect` deletes layers not referenced by any manifests",
	Run: func(cmd *cobra.Command, args []string) {
		config, err := resolveConfiguration(args)
		if err != nil {
			fmt.Fprintf(os.Stderr, "configuration error: %v\n", err)
			cmd.Usage()
			os.Exit(1)
		}

		driver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct %s driver: %v", config.Storage.Type(), err)
			os.Exit(1)
		}

		ctx := context.Background()
		ctx, err = configureLogging(ctx, config)
		if err != nil {
			fmt.Fprintf(os.Stderr, "unable to configure logging with config: %s", err)
			os.Exit(1)
		}

		k, err := libtrust.GenerateECP256PrivateKey()
		if err != nil {
			fmt.Fprint(os.Stderr, err)
			os.Exit(1)
		}

		registry, err := storage.NewRegistry(ctx, driver, storage.DisableSchema1Signatures, storage.Schema1SigningKey(k))
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to construct registry: %v", err)
			os.Exit(1)
		}

		err = markAndSweep(ctx, driver, registry)
		if err != nil {
			fmt.Fprintf(os.Stderr, "failed to garbage collect: %v", err)
			os.Exit(1)
		}
	},
}
<|endoftext|>


<commit_before>package webhook

import (
	"encoding/json"
	"net/http"
	"time"

	"github.com/levenlabs/go-llog"

	"github.com/levenlabs/postmaster/config"
	"github.com/levenlabs/postmaster/db"
	"gopkg.in/validator.v2"
)

type WebhookEvent db.StatsJob

func init() {
	if config.WebhookAddr == "" {
		return
	}

	go func() {
		s := &http.Server{
			Addr:           config.WebhookAddr,
			Handler:        http.HandlerFunc(hookHandler),
			ReadTimeout:    10 * time.Second,
			WriteTimeout:   10 * time.Second,
			MaxHeaderBytes: 1 << 20,
		}
		llog.Info("listening for webhook", llog.KV{"addr": config.WebhookAddr})
		err := s.ListenAndServe()
		llog.Fatal("error listening for webhoook", llog.KV{"addr": config.WebhookAddr, "err": err})
	}()
}

func hookHandler(w http.ResponseWriter, r *http.Request) {
	kv := llog.KV{"ip": r.RemoteAddr}
	llog.Info("webhook request", kv)

	if r.Method != "POST" {
		kv["method"] = r.Method
		llog.Warn("webhook invalid http method", kv)
		http.Error(w, "Invalid HTTP Method", http.StatusMethodNotAllowed)
		return
	}

	if config.WebhookPassword != "" {
		_, password, authOk := r.BasicAuth()
		if !authOk || password != config.WebhookPassword {
			llog.Warn("webhook authorization failed", kv)
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}
	}

	decoder := json.NewDecoder(r.Body)
	var events []WebhookEvent
	err := decoder.Decode(&events)
	if err != nil || len(events) == 0 {
		kv["err"] = err
		llog.Warn("webhook failed to parse body", kv)
		http.Error(w, "Invalid POST Body", http.StatusBadRequest)
		return
	}

	for _, event := range events {
		kv["event"] = event
		llog.Info("webhook processing event", kv)

		if err := validator.Validate(event); err != nil {
			kv["err"] = err
			llog.Warn("webhook event failed validation", kv)
			return
		}

		contents, err := json.Marshal(event)
		if err != nil {
			kv["err"] = err
			llog.Error("webhook couldn't marshal event", kv)
			delete(kv, "err")
			continue
		}
		if err = db.StoreStatsJob(string(contents)); err != nil {
			kv["err"] = err
			llog.Error("webhook couldn't store stats job", kv)
			delete(kv, "err")
		}
	}
}
<commit_msg>Don't be as verbose with logging webhooks<commit_after>package webhook

import (
	"encoding/json"
	"net/http"
	"time"

	"github.com/levenlabs/go-llog"

	"github.com/levenlabs/postmaster/config"
	"github.com/levenlabs/postmaster/db"
	"gopkg.in/validator.v2"
)

type WebhookEvent db.StatsJob

func init() {
	if config.WebhookAddr == "" {
		return
	}

	go func() {
		s := &http.Server{
			Addr:           config.WebhookAddr,
			Handler:        http.HandlerFunc(hookHandler),
			ReadTimeout:    10 * time.Second,
			WriteTimeout:   10 * time.Second,
			MaxHeaderBytes: 1 << 20,
		}
		llog.Info("listening for webhook", llog.KV{"addr": config.WebhookAddr})
		err := s.ListenAndServe()
		llog.Fatal("error listening for webhoook", llog.KV{"addr": config.WebhookAddr, "err": err})
	}()
}

func hookHandler(w http.ResponseWriter, r *http.Request) {
	kv := llog.KV{"ip": r.RemoteAddr}
	llog.Debug("webhook request", kv)

	if r.Method != "POST" {
		kv["method"] = r.Method
		llog.Warn("webhook invalid http method", kv)
		http.Error(w, "Invalid HTTP Method", http.StatusMethodNotAllowed)
		return
	}

	if config.WebhookPassword != "" {
		_, password, authOk := r.BasicAuth()
		if !authOk || password != config.WebhookPassword {
			llog.Warn("webhook authorization failed", kv)
			http.Error(w, "Unauthorized", http.StatusUnauthorized)
			return
		}
	}

	decoder := json.NewDecoder(r.Body)
	var events []WebhookEvent
	err := decoder.Decode(&events)
	if err != nil || len(events) == 0 {
		kv["err"] = err
		llog.Warn("webhook failed to parse body", kv)
		http.Error(w, "Invalid POST Body", http.StatusBadRequest)
		return
	}

	for _, event := range events {
		kv["event"] = event
		llog.Debug("webhook processing event", kv)

		if err := validator.Validate(event); err != nil {
			kv["err"] = err
			llog.Warn("webhook event failed validation", kv)
			return
		}

		contents, err := json.Marshal(event)
		if err != nil {
			kv["err"] = err
			llog.Error("webhook couldn't marshal event", kv)
			delete(kv, "err")
			continue
		}
		if err = db.StoreStatsJob(string(contents)); err != nil {
			kv["err"] = err
			llog.Error("webhook couldn't store stats job", kv)
			delete(kv, "err")
		}
	}
}
<|endoftext|>


<commit_before>package ekanite

import (
	"expvar"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ekanite/ekanite/input"
)

const (
	DefaultNumShards       = 16
	DefaultIndexDuration   = 24 * time.Hour
	DefaultRetentionPeriod = 24 * time.Hour

	RetentionCheckInterval = time.Hour
)

var (
	stats = expvar.NewMap("engine")
)

type EventIndexer interface {
	Index(events []*Event) error
}

// Batcher accepts "input events", and once it has a certain number, or a certain amount
// of time has passed, sends those as indexable Events to an Indexer. It also supports a
// maximum number of unprocessed Events it will keep pending. Once this limit is reached,
// it will not accept anymore until outstanding Events are processed.
type Batcher struct {
	indexer  EventIndexer
	size     int
	duration time.Duration

	c chan *input.Event
}

// NewBatcher returns a Batcher for EventIndexer e, a batching size of sz, a maximum duration
// of dur, and a maximum outstanding count of max.
func NewBatcher(e EventIndexer, sz int, dur time.Duration, max int) *Batcher {
	return &Batcher{
		indexer:  e,
		size:     sz,
		duration: dur,
		c:        make(chan *input.Event, max),
	}
}

// Start starts the batching process.
func (b *Batcher) Start(errChan chan<- error) error {
	go func() {
		batch := make([]*Event, 0, b.size)
		timer := time.NewTimer(b.duration)
		timer.Stop() // Stop any first firing.

		send := func() {
			err := b.indexer.Index(batch)
			if err != nil {
				stats.Add("batchIndexedError", 1)
				return
			}
			stats.Add("batchIndexed", 1)
			stats.Add("eventsIndexed", int64(len(batch)))
			if errChan != nil {
				errChan <- err
			}
			batch = make([]*Event, 0, b.size)
		}

		for {
			select {
			case event := <-b.c:
				idxEvent := &Event{
					event,
				}
				batch = append(batch, idxEvent)
				if len(batch) == 1 {
					timer.Reset(b.duration)
				}
				if len(batch) == b.size {
					timer.Stop()
					send()
				}
			case <-timer.C:
				stats.Add("batchTimeout", 1)
				send()
			}
		}
	}()

	return nil
}

// C returns the channel on the batcher to which events should be sent.
func (b *Batcher) C() chan<- *input.Event {
	return b.c
}

// Engine is the component that performs all indexing.
type Engine struct {
	path            string        // Path to all indexed data
	NumShards       int           // Number of shards to use when creating an index.
	IndexDuration   time.Duration // Duration of created indexes.
	RetentionPeriod time.Duration // How long after Index end-time to hang onto data.

	mu      sync.RWMutex
	indexes Indexes

	open bool
	done chan struct{}
	wg   sync.WaitGroup

	Logger *log.Logger
}

// NewEngine returns a new indexing engine, which will use any data located at path.
func NewEngine(path string) *Engine {
	return &Engine{
		path:            path,
		NumShards:       DefaultNumShards,
		IndexDuration:   DefaultIndexDuration,
		RetentionPeriod: DefaultRetentionPeriod,
		done:            make(chan struct{}),
		Logger:          log.New(os.Stderr, "[engine] ", log.LstdFlags),
	}
}

// Open opens the engine.
func (e *Engine) Open() error {
	if err := os.MkdirAll(e.path, 0755); err != nil {
		return err
	}
	d, err := os.Open(e.path)
	if err != nil {
		return fmt.Errorf("failed to open engine: %s", err.Error())
	}

	fis, err := d.Readdir(0)
	if err != nil {
		return err
	}

	// Open all indexes.
	for _, fi := range fis {
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		indexPath := filepath.Join(e.path, fi.Name())
		i, err := OpenIndex(indexPath)
		if err != nil {
			log.Printf("engine failed to open at index %s: %s", indexPath, err.Error())
			return err
		}
		log.Printf("engine opened index with %d shard(s) at %s", len(i.Shards), indexPath)
		e.indexes = append(e.indexes, i)
	}

	e.wg.Add(1)
	go e.runRetentionEnforcement()

	e.open = true
	return nil
}

// Close closes the engine.
func (e *Engine) Close() error {
	if !e.open {
		return nil
	}

	for _, i := range e.indexes {
		if err := i.Close(); err != nil {
			return err
		}
	}

	close(e.done)
	e.wg.Wait()

	e.open = false
	return nil
}

// Total returns the total number of documents indexed.
func (e *Engine) Total() (uint64, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()

	var total uint64
	for _, i := range e.indexes {
		t, err := i.Total()
		if err != nil {
			return 0, err
		}
		total += t
	}
	return total, nil
}

// runRetentionEnforcement periodically run retention enforcement.
func (e *Engine) runRetentionEnforcement() {
	defer e.wg.Done()
	for {
		select {
		case <-e.done:
			return

		case <-time.After(RetentionCheckInterval):
			e.Logger.Print("retention enforcement commencing")
			stats.Add("retentionEnforcementRun", 1)
			e.enforceRetention()
		}
	}
}

// enforceRetention removes indexes which have aged out.
func (e *Engine) enforceRetention() {
	e.mu.Lock()
	defer e.mu.Unlock()

	filtered := e.indexes[:0]
	for _, i := range e.indexes {
		if i.Expired(time.Now().UTC(), e.RetentionPeriod) {
			if err := DeleteIndex(i); err != nil {
				e.Logger.Printf("retention enforcement failed to delete index %s: %s", i.path, err.Error())
			} else {
				e.Logger.Printf("retention enforcement deleted index %s", i.path)
				stats.Add("retentionEnforcementDeletions", 1)
			}
		} else {
			filtered = append(filtered, i)
		}
	}
	e.indexes = filtered
	return
}

// indexForReferenceTime returns an index suitable for indexing an event
// for the given reference time. Must be called under RLock.
func (e *Engine) indexForReferenceTime(t time.Time) *Index {
	for _, i := range e.indexes {
		if i.Contains(t) {
			return i
		}
	}
	return nil
}

// createIndex creates an index with a given start and end time and adds the
// created index to the Engine's store. It must be called under lock.
func (e *Engine) createIndex(startTime, endTime time.Time) (*Index, error) {
	// There cannot be two indexes with the same start time, since this would mean
	// two indexes with the same path. So if an index already exists with the requested
	// start time, use that index's end time as the start time.
	var idx *Index
	for _, i := range e.indexes {
		if i.startTime == startTime {
			idx = i
			break
		}
	}
	if idx != nil {
		startTime = idx.endTime // XXX This could still align with another start time! Needs some sort of loop.
		assert(!startTime.After(endTime), "new start time after end time")
	}

	i, err := NewIndex(e.path, startTime, endTime, e.NumShards)
	if err != nil {
		return nil, err
	}
	e.indexes = append(e.indexes, i)
	sort.Sort(e.indexes)

	e.Logger.Printf("index %s created with %d shards, start time: %s, end time: %s",
		i.Path(), e.NumShards, i.StartTime(), i.EndTime())
	return i, nil
}

// createIndexForReferenceTime creates an index suitable for indexing an event at the given
// reference time.
func (e *Engine) createIndexForReferenceTime(rt time.Time) (*Index, error) {
	start := rt.Truncate(e.IndexDuration).UTC()
	end := start.Add(e.IndexDuration).UTC()
	return e.createIndex(start, end)
}

// Index indexes a batch of Events. It blocks until all processing has completed.
func (e *Engine) Index(events []*Event) error {
	e.mu.RLock()
	defer e.mu.RUnlock()

	var wg sync.WaitGroup

	// De-multiplex the batch into sub-batches, one sub-batch for each Index.
	subBatches := make(map[*Index][]Document, 0)

	for _, ev := range events {
		index := e.indexForReferenceTime(ev.ReferenceTime())
		if index == nil {
			func() {
				// Take a RWLock, check again, and create a new index if necessary.
				// Doing this in a function makes lock management foolproof.
				e.mu.RUnlock()
				defer e.mu.RLock()
				e.mu.Lock()
				defer e.mu.Unlock()

				index = e.indexForReferenceTime(ev.ReferenceTime())
				if index == nil {
					var err error
					index, err = e.createIndexForReferenceTime(ev.ReferenceTime())
					if err != nil || index == nil {
						panic(fmt.Sprintf("failed to create index for %s", ev.ReferenceTime()))
					}
				}
			}()
		}

		if _, ok := subBatches[index]; !ok {
			subBatches[index] = make([]Document, 0)
		}
		subBatches[index] = append(subBatches[index], ev)
	}

	// Index each batch in parallel.
	for index, subBatch := range subBatches {
		wg.Add(1)
		go func(i *Index, b []Document) {
			defer wg.Done()
			i.Index(b)
		}(index, subBatch)
	}
	wg.Wait()
	return nil
}

// Search performs a search.
func (e *Engine) Search(query string) (<-chan string, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()
	stats.Add("queriesRx", 1)

	// Buffer channel to control how many docs are sent back. XXX Will this allow
	// the client to control? Possibly.
	c := make(chan string, 1)

	go func() {
		// Sequentially search each index, starting with the earliest in time.
		// This could be done in parallel but more sorting would be required.

		for i := len(e.indexes) - 1; i >= 0; i-- {
			e.Logger.Printf("searching index %s", e.indexes[i].Path())
			ids, err := e.indexes[i].Search(query)
			if err != nil {
				e.Logger.Println("error performing search:", err.Error())
				break
			}
			for _, id := range ids {
				b, err := e.indexes[i].Document(id)
				if err != nil {
					e.Logger.Println("error getting document:", err.Error())
					break
				}
				stats.Add("docsIDsRetrived", 1)
				c <- string(b) // There is excessive byte-slice-to-strings here.
			}
		}
		close(c)
	}()

	return c, nil
}

// Path returns the path to the indexed data directory.
func (e *Engine) Path() string {
	return e.path
}

// assert will panic with a given formatted message if the given condition is false.
func assert(condition bool, msg string, v ...interface{}) {
	if !condition {
		panic(fmt.Sprintf("assert failed: "+msg, v...))
	}
}
<commit_msg>Better panic message when index creation fails<commit_after>package ekanite

import (
	"expvar"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ekanite/ekanite/input"
)

const (
	DefaultNumShards       = 16
	DefaultIndexDuration   = 24 * time.Hour
	DefaultRetentionPeriod = 24 * time.Hour

	RetentionCheckInterval = time.Hour
)

var (
	stats = expvar.NewMap("engine")
)

type EventIndexer interface {
	Index(events []*Event) error
}

// Batcher accepts "input events", and once it has a certain number, or a certain amount
// of time has passed, sends those as indexable Events to an Indexer. It also supports a
// maximum number of unprocessed Events it will keep pending. Once this limit is reached,
// it will not accept anymore until outstanding Events are processed.
type Batcher struct {
	indexer  EventIndexer
	size     int
	duration time.Duration

	c chan *input.Event
}

// NewBatcher returns a Batcher for EventIndexer e, a batching size of sz, a maximum duration
// of dur, and a maximum outstanding count of max.
func NewBatcher(e EventIndexer, sz int, dur time.Duration, max int) *Batcher {
	return &Batcher{
		indexer:  e,
		size:     sz,
		duration: dur,
		c:        make(chan *input.Event, max),
	}
}

// Start starts the batching process.
func (b *Batcher) Start(errChan chan<- error) error {
	go func() {
		batch := make([]*Event, 0, b.size)
		timer := time.NewTimer(b.duration)
		timer.Stop() // Stop any first firing.

		send := func() {
			err := b.indexer.Index(batch)
			if err != nil {
				stats.Add("batchIndexedError", 1)
				return
			}
			stats.Add("batchIndexed", 1)
			stats.Add("eventsIndexed", int64(len(batch)))
			if errChan != nil {
				errChan <- err
			}
			batch = make([]*Event, 0, b.size)
		}

		for {
			select {
			case event := <-b.c:
				idxEvent := &Event{
					event,
				}
				batch = append(batch, idxEvent)
				if len(batch) == 1 {
					timer.Reset(b.duration)
				}
				if len(batch) == b.size {
					timer.Stop()
					send()
				}
			case <-timer.C:
				stats.Add("batchTimeout", 1)
				send()
			}
		}
	}()

	return nil
}

// C returns the channel on the batcher to which events should be sent.
func (b *Batcher) C() chan<- *input.Event {
	return b.c
}

// Engine is the component that performs all indexing.
type Engine struct {
	path            string        // Path to all indexed data
	NumShards       int           // Number of shards to use when creating an index.
	IndexDuration   time.Duration // Duration of created indexes.
	RetentionPeriod time.Duration // How long after Index end-time to hang onto data.

	mu      sync.RWMutex
	indexes Indexes

	open bool
	done chan struct{}
	wg   sync.WaitGroup

	Logger *log.Logger
}

// NewEngine returns a new indexing engine, which will use any data located at path.
func NewEngine(path string) *Engine {
	return &Engine{
		path:            path,
		NumShards:       DefaultNumShards,
		IndexDuration:   DefaultIndexDuration,
		RetentionPeriod: DefaultRetentionPeriod,
		done:            make(chan struct{}),
		Logger:          log.New(os.Stderr, "[engine] ", log.LstdFlags),
	}
}

// Open opens the engine.
func (e *Engine) Open() error {
	if err := os.MkdirAll(e.path, 0755); err != nil {
		return err
	}
	d, err := os.Open(e.path)
	if err != nil {
		return fmt.Errorf("failed to open engine: %s", err.Error())
	}

	fis, err := d.Readdir(0)
	if err != nil {
		return err
	}

	// Open all indexes.
	for _, fi := range fis {
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		indexPath := filepath.Join(e.path, fi.Name())
		i, err := OpenIndex(indexPath)
		if err != nil {
			log.Printf("engine failed to open at index %s: %s", indexPath, err.Error())
			return err
		}
		log.Printf("engine opened index with %d shard(s) at %s", len(i.Shards), indexPath)
		e.indexes = append(e.indexes, i)
	}

	e.wg.Add(1)
	go e.runRetentionEnforcement()

	e.open = true
	return nil
}

// Close closes the engine.
func (e *Engine) Close() error {
	if !e.open {
		return nil
	}

	for _, i := range e.indexes {
		if err := i.Close(); err != nil {
			return err
		}
	}

	close(e.done)
	e.wg.Wait()

	e.open = false
	return nil
}

// Total returns the total number of documents indexed.
func (e *Engine) Total() (uint64, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()

	var total uint64
	for _, i := range e.indexes {
		t, err := i.Total()
		if err != nil {
			return 0, err
		}
		total += t
	}
	return total, nil
}

// runRetentionEnforcement periodically run retention enforcement.
func (e *Engine) runRetentionEnforcement() {
	defer e.wg.Done()
	for {
		select {
		case <-e.done:
			return

		case <-time.After(RetentionCheckInterval):
			e.Logger.Print("retention enforcement commencing")
			stats.Add("retentionEnforcementRun", 1)
			e.enforceRetention()
		}
	}
}

// enforceRetention removes indexes which have aged out.
func (e *Engine) enforceRetention() {
	e.mu.Lock()
	defer e.mu.Unlock()

	filtered := e.indexes[:0]
	for _, i := range e.indexes {
		if i.Expired(time.Now().UTC(), e.RetentionPeriod) {
			if err := DeleteIndex(i); err != nil {
				e.Logger.Printf("retention enforcement failed to delete index %s: %s", i.path, err.Error())
			} else {
				e.Logger.Printf("retention enforcement deleted index %s", i.path)
				stats.Add("retentionEnforcementDeletions", 1)
			}
		} else {
			filtered = append(filtered, i)
		}
	}
	e.indexes = filtered
	return
}

// indexForReferenceTime returns an index suitable for indexing an event
// for the given reference time. Must be called under RLock.
func (e *Engine) indexForReferenceTime(t time.Time) *Index {
	for _, i := range e.indexes {
		if i.Contains(t) {
			return i
		}
	}
	return nil
}

// createIndex creates an index with a given start and end time and adds the
// created index to the Engine's store. It must be called under lock.
func (e *Engine) createIndex(startTime, endTime time.Time) (*Index, error) {
	// There cannot be two indexes with the same start time, since this would mean
	// two indexes with the same path. So if an index already exists with the requested
	// start time, use that index's end time as the start time.
	var idx *Index
	for _, i := range e.indexes {
		if i.startTime == startTime {
			idx = i
			break
		}
	}
	if idx != nil {
		startTime = idx.endTime // XXX This could still align with another start time! Needs some sort of loop.
		assert(!startTime.After(endTime), "new start time after end time")
	}

	i, err := NewIndex(e.path, startTime, endTime, e.NumShards)
	if err != nil {
		return nil, err
	}
	e.indexes = append(e.indexes, i)
	sort.Sort(e.indexes)

	e.Logger.Printf("index %s created with %d shards, start time: %s, end time: %s",
		i.Path(), e.NumShards, i.StartTime(), i.EndTime())
	return i, nil
}

// createIndexForReferenceTime creates an index suitable for indexing an event at the given
// reference time.
func (e *Engine) createIndexForReferenceTime(rt time.Time) (*Index, error) {
	start := rt.Truncate(e.IndexDuration).UTC()
	end := start.Add(e.IndexDuration).UTC()
	return e.createIndex(start, end)
}

// Index indexes a batch of Events. It blocks until all processing has completed.
func (e *Engine) Index(events []*Event) error {
	e.mu.RLock()
	defer e.mu.RUnlock()

	var wg sync.WaitGroup

	// De-multiplex the batch into sub-batches, one sub-batch for each Index.
	subBatches := make(map[*Index][]Document, 0)

	for _, ev := range events {
		index := e.indexForReferenceTime(ev.ReferenceTime())
		if index == nil {
			func() {
				// Take a RWLock, check again, and create a new index if necessary.
				// Doing this in a function makes lock management foolproof.
				e.mu.RUnlock()
				defer e.mu.RLock()
				e.mu.Lock()
				defer e.mu.Unlock()

				index = e.indexForReferenceTime(ev.ReferenceTime())
				if index == nil {
					var err error
					index, err = e.createIndexForReferenceTime(ev.ReferenceTime())
					if err != nil || index == nil {
						panic(fmt.Sprintf("failed to create index for %s: %s", ev.ReferenceTime(), err))
					}
				}
			}()
		}

		if _, ok := subBatches[index]; !ok {
			subBatches[index] = make([]Document, 0)
		}
		subBatches[index] = append(subBatches[index], ev)
	}

	// Index each batch in parallel.
	for index, subBatch := range subBatches {
		wg.Add(1)
		go func(i *Index, b []Document) {
			defer wg.Done()
			i.Index(b)
		}(index, subBatch)
	}
	wg.Wait()
	return nil
}

// Search performs a search.
func (e *Engine) Search(query string) (<-chan string, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()
	stats.Add("queriesRx", 1)

	// Buffer channel to control how many docs are sent back. XXX Will this allow
	// the client to control? Possibly.
	c := make(chan string, 1)

	go func() {
		// Sequentially search each index, starting with the earliest in time.
		// This could be done in parallel but more sorting would be required.

		for i := len(e.indexes) - 1; i >= 0; i-- {
			e.Logger.Printf("searching index %s", e.indexes[i].Path())
			ids, err := e.indexes[i].Search(query)
			if err != nil {
				e.Logger.Println("error performing search:", err.Error())
				break
			}
			for _, id := range ids {
				b, err := e.indexes[i].Document(id)
				if err != nil {
					e.Logger.Println("error getting document:", err.Error())
					break
				}
				stats.Add("docsIDsRetrived", 1)
				c <- string(b) // There is excessive byte-slice-to-strings here.
			}
		}
		close(c)
	}()

	return c, nil
}

// Path returns the path to the indexed data directory.
func (e *Engine) Path() string {
	return e.path
}

// assert will panic with a given formatted message if the given condition is false.
func assert(condition bool, msg string, v ...interface{}) {
	if !condition {
		panic(fmt.Sprintf("assert failed: "+msg, v...))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"

	// "database/sql"

	_ "github.com/lib/pq"

	"github.com/DimShadoWWW/power-pg/proxy"
	"github.com/parnurzeal/gorequest"
)

var (
	localHost     = flag.String("l", ":5432", "puerto local para escuchar")
	dbHostname    = flag.String("h", "localhost", "hostname del servidor PostgreSQL")
	dbPort        = flag.String("r", "5432", "puerto del servidor PostgreSQL")
	dbName        = flag.String("d", "dbname", "nombre de la base de datos")
	dbUsername    = flag.String("u", "username", "usuario para acceder al servidor PostgreSQL")
	dbPassword    = flag.String("p", "password", "password para acceder al servidor PostgreSQL")
	remoteService = flag.String("s", "", "http://localhost:8080/query")
	// messages      = []string{}
)

type msgStruct struct {
	Type    string
	Content string
}

var (
	db *sql.DB
)

func main() {
	flag.Parse()
	// dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable",
	// 	*dbUsername, *dbPassword, *dbName)
	// db, err := sql.Open("postgres", dbinfo)

	msgs := make(chan string)
	msgCh := make(chan proxy.Pkg)
	msgOut := make(chan msgStruct)
	if *remoteService != "" {
		go func() {
			time.Sleep(time.Second * 3)
			inFile, _ := os.Open("canales_list.txt")
			defer inFile.Close()
			scanner := bufio.NewScanner(inFile)
			scanner.Split(bufio.ScanLines)

			for scanner.Scan() {
				time.Sleep(time.Second * 1)
				// messages = []string{}
				// fmt.Println(scanner.Text())
				// msgOut <- fmt.Sprintf("# %s\n", scanner.Text())
				msgOut <- msgStruct{Type: "C", Content: scanner.Text()}
				_, _, errs := gorequest.New().Get(fmt.Sprintf("%s%s", *remoteService, scanner.Text())).End()
				if errs != nil {
					log.Fatalf("log failed: %v", errs)
				}
			}
			log.Println("done")
			os.Exit(0)
		}()
	}

	go func() {
		f, err := os.OpenFile("/all.txt", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		for msg := range msgs {
			// fmt.Println(msg)
			_, err := f.WriteString(fmt.Sprintf("%s\n", msg))
			if err != nil {
				log.Fatalf("log failed: %v", err)
			}
		}
	}()

	go func() {
		f, err := os.OpenFile("/reports/report.md", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
		// c := 0
		spaces := regexp.MustCompile("[\t]+")
		multipleSpaces := regexp.MustCompile("    ")
		for {
			// select {
			// case msg1 := <-msgOut:
			msg := <-msgOut
			if msg.Type == "C" {
				// c = 0
				f.Close()
				f, err = os.OpenFile(fmt.Sprintf("/reports/report-%s.md", msg.Content), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				// c = 0
				if err != nil {
					panic(err)
				}
				_, err := f.WriteString(fmt.Sprintf("# %s\n", msg.Content))
				if err != nil {
					log.Fatalf("log failed: %v", err)
				}
			} else {
				// case msg2 := <-msgOut:
				// c = c + 1
				m := spaces.ReplaceAll([]byte(msg.Content), []byte{' '})
				m = multipleSpaces.ReplaceAll(m, []byte{' '})
				_, err := f.WriteString(fmt.Sprintf("\n```sql\n%s\n```\n", string(m)))
				if err != nil {
					log.Fatalf("log failed: %v", err)
				}
			}
		}
	}()

	go func() {
		temp := ""
		for msg := range msgCh {
			if msg.Type == 'P' {
				if strings.Contains(string(msg.Content), "$1") {
					var newMsg proxy.ReadBuf
					newMsg = msg.Content
					_ = newMsg.Int32()

					// The name of the destination portal (an empty string selects the unnamed portal).
					p := bytes.Index(newMsg, []byte{0})
					// remove first string
					msgs <- fmt.Sprintf("msg ---->%#v\n", newMsg)
					msgs <- fmt.Sprintf("msg ---->%s\n", string(newMsg))
					msgs <- fmt.Sprintf("first string ---->%#v\n", newMsg[:p+1])
					msgs <- fmt.Sprintf("first string ---->%s\n", string(newMsg[:p+1]))
					newMsg = newMsg[p+1:]
					fmt.Printf("0 newMsg   ----->%#v\n", newMsg)

					// The name of the source prepared statement (an empty string selects the unnamed prepared statement).
					p = bytes.Index(newMsg, []byte{0})
					// remove second string
					msgs <- fmt.Sprintf("second string: message ---->%#v\n", newMsg[:p])
					temp = string(newMsg[:p])
					msgs <- fmt.Sprintf("second string: message temp ---->%s\n", temp)

				} else {
					temp = ""
					selectIdx := strings.Index(string(msg.Content), string([]byte{83, 69, 76, 69, 67, 84, 32}))
					if selectIdx == -1 {
						selectIdx = 0
					}
					sepIdx := strings.Index(string(msg.Content), string([]byte{0, 0, 1, 0, 0}))
					if sepIdx == -1 || sepIdx+5 > len(msg.Content) {
						sepIdx := strings.Index(string(msg.Content), string([]byte{0, 1, 0, 0}))
						if sepIdx == -1 || sepIdx+4 > len(msg.Content) {
							sepIdx = len(msg.Content) - 4
						} else {
							sepIdx = len(msg.Content) - 1
						}
					}
					if sepIdx == -1 {
						sepIdx = len(msg.Content)
					}
					if selectIdx == -1 {
						selectIdx = 0
					}

					fmt.Printf("SEP index ----->%v\n", sepIdx)
					fmt.Printf("SEP len   ----->%v\n", len(msg.Content))
					fmt.Printf("SEP CONT  ----->%v\n", msg.Content)
					// messages = append(messages, string(bytes.Trim(msg.Content[selectIdx:sepIdx], "\x00")))
					msgOut <- msgStruct{Type: "M", Content: string(bytes.Trim(msg.Content[selectIdx:sepIdx], "\x00"))}
				}
			} else {
				if msg.Type == 'B' && temp != "" && len(msg.Content) > 28 {
					var newMsg proxy.ReadBuf
					newMsg = msg.Content

					// The name of the destination portal (an empty string selects the unnamed portal).
					p := bytes.Index(newMsg, []byte{0})
					// remove first string
					msgs <- fmt.Sprintf("msg ---->%#v\n", newMsg)
					msgs <- fmt.Sprintf("first string ---->%#v\n", newMsg[:p+1])
					newMsg = newMsg[p:]
					msgs <- fmt.Sprintf("0 newMsg   ----->%#v\n", newMsg)

					// The name of the source prepared statement (an empty string selects the unnamed prepared statement).
					p = bytes.Index(newMsg, []byte{0})
					// remove second string
					msgs <- fmt.Sprintf("second string ---->%#v\n", newMsg[:p+1])
					newMsg = newMsg[p+1:]
					msgs <- fmt.Sprintf("1 newMsg   ----->%#v\n", newMsg)

					t := newMsg.Int16()
					msgs <- fmt.Sprintf("vars types numbers ---->%#v\n", t)
					for i := 0; i < t; i++ {
						t = newMsg.Int16()
						msgs <- fmt.Sprintf("22 newMsg   ----->%#v\n", newMsg)
					}

					msgs <- fmt.Sprintf("23 newMsg   ----->%#v\n", newMsg)
					totalVar := newMsg.Int16()
					vars := make(map[int]string)
					var varsIdx []int
					if (totalVar == 0 && len(newMsg) > 4) || totalVar > len(newMsg) {
						for totalVar := 0; totalVar != 0 && totalVar < len(newMsg); {
							totalVar = newMsg.Int16()
							msgs <- fmt.Sprintf("24 newMsg   ----->%#v\n", newMsg)
							msgs <- fmt.Sprintf("1 totalVar  ----->%d\n", totalVar)
						}
					}
					//
					// if totalVar == 0 && len(newMsg) > 4 {
					// 	totalVar = newMsg.Int16()
					// }
					// if totalVar == 0 && len(newMsg) > 4 {
					// 	totalVar = newMsg.Int32()
					// }
					msgs <- fmt.Sprintf("totalVar   ----->%d\n", totalVar)
					for i := 0; i < totalVar; i++ {
						msgs <- fmt.Sprintf("2 newMsg   ----->%#v\n", newMsg)
						varLen := newMsg.Int32()
						// var1 := newMsg.Next(4)
						// // fmt.Printf("aa   -----> %#v\n", aa)
						// // fmt.Printf("aa bits ----->%8b\n", aa[len(aa)-1])
						// varLen := int(binary.BigEndian.Uint32(var1))
						// if varLen > len(newMsg) {
						// 	varLen = int(binary.BigEndian.Uint16(var1[:2]))
						// }
						msgs <- fmt.Sprintf("varLen ----->%v\n", varLen)
						msgs <- fmt.Sprintf("newMsg   ----->%#v\n", newMsg)
						if varLen > len(newMsg) {
							varLen = len(newMsg) - 4
							msgs <- fmt.Sprintf("1 varLen ----->%v\n", varLen)
							msgs <- fmt.Sprintf("1 newMsg   ----->%#v\n", newMsg)
						}
						vars[i] = string(newMsg.Next(varLen))
						msgs <- fmt.Sprintf("vars   ----->%#v\n", vars)
						varsIdx = append(varsIdx, i)
						msgs <- fmt.Sprintf("varIdx  ----->%#v\n", varsIdx)
					}

					sort.Sort(sort.Reverse(sort.IntSlice(varsIdx)))
					for _, k := range varsIdx {
						// messages = append(messages, strings.Replace(temp, fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", string(newMsg[k+1])), -1))
						temp = strings.Replace(temp, fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", string(vars[k])), -1)
						msgs <- fmt.Sprintf("message subst k ----->%v\n", k)
						msgs <- fmt.Sprintf("message subst newMsg ----->%#v\n", newMsg)
						msgs <- fmt.Sprintf("message subst msg ----->%v\n", vars[k+1])
						msgs <- fmt.Sprintf("message subst temp ----->%v\n", temp)
						msgs <- fmt.Sprintf("message subst param %s ----->%v\n", fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", vars[k]))
					}
					msgs <- fmt.Sprintf("end message  ----->%v\n", temp)
					// msgOut <- msgStruct{Type: "M", Content: temp}
					// } else {
				}
				msgOut <- msgStruct{Type: "M", Content: temp}
				temp = ""
			}
		}
	}()

	proxy.Start(localHost, dbHostname, dbPort, getQueryModificada, msgs, msgCh)
}

func getQueryModificada(queryOriginal string) string {
	// log.Println("aa")
	// if queryOriginal[:5] != "power" {
	// 	return queryOriginal
	// }

	// log.Println(queryOriginal)
	fmt.Println(queryOriginal)
	return queryOriginal
}
<commit_msg>Update<commit_after>package main

import (
	"bufio"
	"bytes"
	"database/sql"
	"flag"
	"fmt"
	"log"
	"os"
	"regexp"
	"sort"
	"strings"
	"time"

	// "database/sql"

	_ "github.com/lib/pq"

	"github.com/DimShadoWWW/power-pg/proxy"
	"github.com/parnurzeal/gorequest"
)

var (
	localHost     = flag.String("l", ":5432", "puerto local para escuchar")
	dbHostname    = flag.String("h", "localhost", "hostname del servidor PostgreSQL")
	dbPort        = flag.String("r", "5432", "puerto del servidor PostgreSQL")
	dbName        = flag.String("d", "dbname", "nombre de la base de datos")
	dbUsername    = flag.String("u", "username", "usuario para acceder al servidor PostgreSQL")
	dbPassword    = flag.String("p", "password", "password para acceder al servidor PostgreSQL")
	remoteService = flag.String("s", "", "http://localhost:8080/query")
	// messages      = []string{}
)

type msgStruct struct {
	Type    string
	Content string
}

var (
	db *sql.DB
)

func main() {
	flag.Parse()
	// dbinfo := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=disable",
	// 	*dbUsername, *dbPassword, *dbName)
	// db, err := sql.Open("postgres", dbinfo)

	msgs := make(chan string)
	msgCh := make(chan proxy.Pkg)
	msgOut := make(chan msgStruct)
	if *remoteService != "" {
		go func() {
			time.Sleep(time.Second * 3)
			inFile, _ := os.Open("canales_list.txt")
			defer inFile.Close()
			scanner := bufio.NewScanner(inFile)
			scanner.Split(bufio.ScanLines)

			for scanner.Scan() {
				time.Sleep(time.Second * 1)
				// messages = []string{}
				// fmt.Println(scanner.Text())
				// msgOut <- fmt.Sprintf("# %s\n", scanner.Text())
				msgOut <- msgStruct{Type: "C", Content: scanner.Text()}
				_, _, errs := gorequest.New().Get(fmt.Sprintf("%s%s", *remoteService, scanner.Text())).End()
				if errs != nil {
					log.Fatalf("log failed: %v", errs)
				}
			}
			log.Println("done")
			os.Exit(0)
		}()
	}

	go func() {
		f, err := os.OpenFile("/all.txt", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
		if err != nil {
			panic(err)
		}
		defer f.Close()
		for msg := range msgs {
			// fmt.Println(msg)
			_, err := f.WriteString(fmt.Sprintf("%s\n", msg))
			if err != nil {
				log.Fatalf("log failed: %v", err)
			}
		}
	}()

	go func() {
		f, err := os.OpenFile("/reports/report.md", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
		// c := 0
		spaces := regexp.MustCompile("[\t]+")
		multipleSpaces := regexp.MustCompile("    ")
		for {
			// select {
			// case msg1 := <-msgOut:
			msg := <-msgOut
			if msg.Type == "C" {
				// c = 0
				f.Close()
				f, err = os.OpenFile(fmt.Sprintf("/reports/report-%s.md", msg.Content), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
				// c = 0
				if err != nil {
					panic(err)
				}
				_, err := f.WriteString(fmt.Sprintf("# %s\n", msg.Content))
				if err != nil {
					log.Fatalf("log failed: %v", err)
				}
			} else {
				// case msg2 := <-msgOut:
				// c = c + 1
				m := spaces.ReplaceAll([]byte(msg.Content), []byte{' '})
				m = multipleSpaces.ReplaceAll(m, []byte{' '})
				_, err := f.WriteString(fmt.Sprintf("\n```sql\n%s\n```\n", string(m)))
				if err != nil {
					log.Fatalf("log failed: %v", err)
				}
			}
		}
	}()

	go func() {
		temp := ""
		for msg := range msgCh {
			if msg.Type == 'P' {
				if strings.Contains(string(msg.Content), "$1") {
					var newMsg proxy.ReadBuf
					newMsg = msg.Content
					_ = newMsg.Int32()

					// The name of the destination portal (an empty string selects the unnamed portal).
					p := bytes.Index(newMsg, []byte{0})
					// remove first string
					msgs <- fmt.Sprintf("msg ---->%#v\n", newMsg)
					msgs <- fmt.Sprintf("msg ---->%s\n", string(newMsg))
					msgs <- fmt.Sprintf("first string ---->%#v\n", newMsg[:p+1])
					msgs <- fmt.Sprintf("first string ---->%s\n", string(newMsg[:p+1]))
					newMsg = newMsg[p+1:]
					fmt.Printf("0 newMsg   ----->%#v\n", newMsg)

					// The name of the source prepared statement (an empty string selects the unnamed prepared statement).
					p = bytes.Index(newMsg, []byte{0})
					// remove second string
					msgs <- fmt.Sprintf("second string: message ---->%#v\n", newMsg[:p])
					temp = string(newMsg[:p])
					msgs <- fmt.Sprintf("second string: message temp ---->%s\n", temp)

				} else {
					temp = ""
					selectIdx := strings.Index(string(msg.Content), string([]byte{83, 69, 76, 69, 67, 84, 32}))
					if selectIdx == -1 {
						selectIdx = 0
					}
					sepIdx := strings.Index(string(msg.Content), string([]byte{0, 0, 1, 0, 0}))
					if sepIdx == -1 || sepIdx+5 > len(msg.Content) {
						sepIdx := strings.Index(string(msg.Content), string([]byte{0, 1, 0, 0}))
						if sepIdx == -1 || sepIdx+4 > len(msg.Content) {
							sepIdx = len(msg.Content) - 4
						} else {
							sepIdx = len(msg.Content) - 1
						}
					}
					if sepIdx == -1 {
						sepIdx = len(msg.Content)
					}
					if selectIdx == -1 {
						selectIdx = 0
					}

					fmt.Printf("SEP index ----->%v\n", sepIdx)
					fmt.Printf("SEP len   ----->%v\n", len(msg.Content))
					fmt.Printf("SEP CONT  ----->%v\n", msg.Content)
					// messages = append(messages, string(bytes.Trim(msg.Content[selectIdx:sepIdx], "\x00")))
					msgOut <- msgStruct{Type: "M", Content: string(bytes.Trim(msg.Content[selectIdx:sepIdx], "\x00"))}
				}
			} else {
				if msg.Type == 'B' && temp != "" && len(msg.Content) > 28 {
					var newMsg proxy.ReadBuf
					newMsg = msg.Content

					// The name of the destination portal (an empty string selects the unnamed portal).
					p := bytes.Index(newMsg, []byte{0})
					// remove first string
					msgs <- fmt.Sprintf("msg ---->%#v\n", newMsg)
					msgs <- fmt.Sprintf("first string ---->%#v\n", newMsg[:p+1])
					newMsg = newMsg[p+1:]
					msgs <- fmt.Sprintf("0 newMsg   ----->%#v\n", newMsg)

					// The name of the source prepared statement (an empty string selects the unnamed prepared statement).
					p = bytes.Index(newMsg, []byte{0})
					// remove second string
					msgs <- fmt.Sprintf("second string ---->%#v\n", newMsg[:p+1])
					newMsg = newMsg[p+1:]
					msgs <- fmt.Sprintf("1 newMsg   ----->%#v\n", newMsg)

					t := newMsg.Int16()
					msgs <- fmt.Sprintf("vars types numbers ---->%#v\n", t)
					for i := 0; i < t; i++ {
						t = newMsg.Int16()
						msgs <- fmt.Sprintf("22 newMsg   ----->%#v\n", newMsg)
					}

					msgs <- fmt.Sprintf("23 newMsg   ----->%#v\n", newMsg)
					totalVar := newMsg.Int16()
					vars := make(map[int]string)
					var varsIdx []int
					if (totalVar == 0 && len(newMsg) > 4) || totalVar > len(newMsg) {
						for totalVar := 0; totalVar != 0 && totalVar < len(newMsg); {
							totalVar = newMsg.Int16()
							msgs <- fmt.Sprintf("24 newMsg   ----->%#v\n", newMsg)
							msgs <- fmt.Sprintf("1 totalVar  ----->%d\n", totalVar)
						}
					}
					//
					// if totalVar == 0 && len(newMsg) > 4 {
					// 	totalVar = newMsg.Int16()
					// }
					// if totalVar == 0 && len(newMsg) > 4 {
					// 	totalVar = newMsg.Int32()
					// }
					msgs <- fmt.Sprintf("totalVar   ----->%d\n", totalVar)
					for i := 0; i < totalVar; i++ {
						msgs <- fmt.Sprintf("2 newMsg   ----->%#v\n", newMsg)
						varLen := newMsg.Int32()
						// var1 := newMsg.Next(4)
						// // fmt.Printf("aa   -----> %#v\n", aa)
						// // fmt.Printf("aa bits ----->%8b\n", aa[len(aa)-1])
						// varLen := int(binary.BigEndian.Uint32(var1))
						// if varLen > len(newMsg) {
						// 	varLen = int(binary.BigEndian.Uint16(var1[:2]))
						// }
						msgs <- fmt.Sprintf("varLen ----->%v\n", varLen)
						msgs <- fmt.Sprintf("newMsg   ----->%#v\n", newMsg)
						if varLen > len(newMsg) {
							varLen = len(newMsg) - 4
							msgs <- fmt.Sprintf("1 varLen ----->%v\n", varLen)
							msgs <- fmt.Sprintf("1 newMsg   ----->%#v\n", newMsg)
						}
						vars[i] = string(newMsg.Next(varLen))
						msgs <- fmt.Sprintf("vars   ----->%#v\n", vars)
						varsIdx = append(varsIdx, i)
						msgs <- fmt.Sprintf("varIdx  ----->%#v\n", varsIdx)
					}

					sort.Sort(sort.Reverse(sort.IntSlice(varsIdx)))
					for _, k := range varsIdx {
						// messages = append(messages, strings.Replace(temp, fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", string(newMsg[k+1])), -1))
						temp = strings.Replace(temp, fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", string(vars[k])), -1)
						msgs <- fmt.Sprintf("message subst k ----->%v\n", k)
						msgs <- fmt.Sprintf("message subst newMsg ----->%#v\n", newMsg)
						msgs <- fmt.Sprintf("message subst msg ----->%v\n", vars[k+1])
						msgs <- fmt.Sprintf("message subst temp ----->%v\n", temp)
						msgs <- fmt.Sprintf("message subst param %s ----->%v\n", fmt.Sprintf("$%d", k+1), fmt.Sprintf("'%s'", vars[k]))
					}
					msgs <- fmt.Sprintf("end message  ----->%v\n", temp)
					// msgOut <- msgStruct{Type: "M", Content: temp}
					// } else {
				}
				msgOut <- msgStruct{Type: "M", Content: temp}
				temp = ""
			}
		}
	}()

	proxy.Start(localHost, dbHostname, dbPort, getQueryModificada, msgs, msgCh)
}

func getQueryModificada(queryOriginal string) string {
	// log.Println("aa")
	// if queryOriginal[:5] != "power" {
	// 	return queryOriginal
	// }

	// log.Println(queryOriginal)
	fmt.Println(queryOriginal)
	return queryOriginal
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"

	"github.com/wallaceicy06/muni-sign/admin/config"
)

var templates = template.Must(template.ParseFiles("templates/index.html"))

var configFilePath = flag.String("config_file", "", "the path to the file that stores the configuration for the sign")
var port = flag.Int("port", 8080, "the port to serve this webserver")

type Server struct {
	cfg config.SignConfig
}

func main() {
	flag.Parse()

	if *configFilePath == "" {
		fmt.Fprintln(os.Stderr, "A config file path is required.")
		os.Exit(1)
	}

	sc := config.NewFileSignConfig(*configFilePath)
	server := &Server{
		cfg: sc,
	}
	server.serve()
}

func (s *Server) serve() {
	http.HandleFunc("/", s.rootHandler)
	if err := http.ListenAndServe(fmt.Sprintf(":%d", *port), nil); err != nil {
		log.Fatalf("Error starting web server on port %d: %v", port, err)
	}
}

func (s *Server) rootHandler(w http.ResponseWriter, r *http.Request) {
	c, err := s.cfg.Get()
	if err != nil {
		http.Error(w, fmt.Sprintf("Internal error: %v", err), http.StatusInternalServerError)
		return
	}
	if err := templates.ExecuteTemplate(w, "index.html", c); err != nil {
		http.Error(w, fmt.Sprintf("Internal error: %v", err), http.StatusInternalServerError)
		return
	}
}
<commit_msg>Refactor the server for better testability.<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/signal"

	"github.com/wallaceicy06/muni-sign/admin/config"
)

var templates = template.Must(template.ParseFiles("templates/index.html"))

var configFilePath = flag.String("config_file", "", "the path to the file that stores the configuration for the sign")
var port = flag.Int("port", 8080, "the port to serve this webserver")

type server struct {
	cfg  config.SignConfig
	port int
}

func main() {
	flag.Parse()

	if *configFilePath == "" {
		fmt.Fprintln(os.Stderr, "A config file path is required.")
		os.Exit(1)
	}

	srv := newServer(*port, config.NewFileSignConfig(*configFilePath)).serve()

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt)
	<-sigs
	srv.Shutdown(context.Background())
	os.Exit(0)
}

func newServer(port int, cfg config.SignConfig) *server {
	return &server{
		port: port,
		cfg:  cfg,
	}
}

func (s *server) serve() *http.Server {
	srv := &http.Server{
		Addr: fmt.Sprintf(":%d", s.port),
	}

	http.HandleFunc("/", s.rootHandler)

	go func() {
		if err := srv.ListenAndServe(); err != nil {
			log.Printf("Error serving: %v", err)
		}
	}()

	return srv
}

func (s *server) rootHandler(w http.ResponseWriter, r *http.Request) {
	c, err := s.cfg.Get()
	if err != nil {
		http.Error(w, fmt.Sprintf("Internal error: %v", err), http.StatusInternalServerError)
		return
	}
	if err := templates.ExecuteTemplate(w, "index.html", c); err != nil {
		http.Error(w, fmt.Sprintf("Internal error: %v", err), http.StatusInternalServerError)
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/signal"

	"github.com/wallaceicy06/muni-sign/admin/config"
)

var templates = template.Must(template.ParseFiles("templates/index.html"))

var configFilePath = flag.String("config_file", "", "the path to the file that stores the configuration for the sign")
var port = flag.Int("port", 8080, "the port to serve this webserver")

type server struct {
	cfg  config.SignConfig
	port int
}

func main() {
	flag.Parse()

	if *configFilePath == "" {
		fmt.Fprintln(os.Stderr, "A config file path is required.")
		os.Exit(1)
	}

	srv := newServer(*port, config.NewFileSignConfig(*configFilePath)).serve()

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt)
	<-sigs
	srv.Shutdown(context.Background())
	os.Exit(0)
}

func newServer(port int, cfg config.SignConfig) *server {
	return &server{
		port: port,
		cfg:  cfg,
	}
}

func (s *server) serve() *http.Server {
	srv := &http.Server{
		Addr: fmt.Sprintf(":%d", s.port),
	}

	http.HandleFunc("/", s.rootHandler)

	go func() {
		if err := srv.ListenAndServe(); err != nil {
			log.Printf("Error serving: %v", err)
		}
	}()

	return srv
}

func (s *server) rootHandler(w http.ResponseWriter, r *http.Request) {
	c, err := s.cfg.Get()
	if err != nil {
		http.Error(w, fmt.Sprintf("Internal error: %v", err), http.StatusInternalServerError)
		return
	}
	if err := templates.ExecuteTemplate(w, "index.html", c); err != nil {
		http.Error(w, fmt.Sprintf("Internal error: %v", err), http.StatusInternalServerError)
		return
	}
}
<commit_msg>Check for nil config in the admin server and return an error.<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"os"
	"os/signal"

	"github.com/wallaceicy06/muni-sign/admin/config"
)

var templates = template.Must(template.ParseFiles("templates/index.html"))

var configFilePath = flag.String("config_file", "", "the path to the file that stores the configuration for the sign")
var port = flag.Int("port", 8080, "the port to serve this webserver")

type server struct {
	cfg  config.SignConfig
	port int
}

func main() {
	flag.Parse()

	if *configFilePath == "" {
		fmt.Fprintln(os.Stderr, "A config file path is required.")
		os.Exit(1)
	}

	srv := newServer(*port, config.NewFileSignConfig(*configFilePath)).serve()

	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, os.Interrupt)
	<-sigs
	srv.Shutdown(context.Background())
	os.Exit(0)
}

func newServer(port int, cfg config.SignConfig) *server {
	return &server{
		port: port,
		cfg:  cfg,
	}
}

func (s *server) serve() *http.Server {
	srv := &http.Server{
		Addr: fmt.Sprintf(":%d", s.port),
	}

	http.HandleFunc("/", s.rootHandler)

	go func() {
		if err := srv.ListenAndServe(); err != nil {
			log.Printf("Error serving: %v", err)
		}
	}()

	return srv
}

func (s *server) rootHandler(w http.ResponseWriter, r *http.Request) {
	c, err := s.cfg.Get()
	if err != nil {
		http.Error(w, fmt.Sprintf("Internal error: %v", err), http.StatusInternalServerError)
		return
	}
	// Make sure that the configuration is not nil so that the server can return
	// an error before rendering the template.
	if c == nil {
		http.Error(w, fmt.Sprintf("Internal error: %v", err), http.StatusInternalServerError)
		return
	}
	if err := templates.ExecuteTemplate(w, "index.html", c); err != nil {
		log.Printf("Problem rendering HTML template: %v", err)
		return
	}
}
<|endoftext|>


<commit_before>package stringz

type acNode struct {
  // Index into the acNodeArray for a given character
  next [256]int

  // failure link index
  failure int

  // This node indicates that the following elements matched
  matches []int
}

type ahBfs struct {
  node, data, index int
}

func AhoCorasickPreprocessSet(datas [][]byte) []acNode {
  nodes := []acNode{ acNode{} }
  for i,data := range datas {
    cur := 0
    for _,b := range data {
      if nodes[cur].next[b] != 0 {
        cur = nodes[cur].next[b]
        continue
      }
      nodes[cur].next[b] = len(nodes)
      cur = len(nodes)
      nodes = append(nodes, acNode{})
    }
    nodes[cur].matches = append(nodes[cur].matches, i)
  }

  // The skeleton of the graph is done, now we do a BFS on the nodes and form
  // failure links as we go.
  var q []ahBfs
  for i := range datas {
    // TODO: Figure out if this makes sense, maybe we should fix how the BFS
    // works instead?
    if len(datas[i]) > 1 {
      bfs := ahBfs{
        node:  nodes[0].next[datas[i][0]],
        data:  i,
        index: 1,
      }
      q = append(q, bfs)
    }
  }
  for len(q) > 0 {
    bfs := q[0]
    q = q[1:]
    mod := nodes[bfs.node].failure
    edge := datas[bfs.data][bfs.index]
    for mod != 0 && nodes[mod].next[edge] == 0 {
      mod = nodes[mod].failure
    }
    source := nodes[bfs.node].next[edge]
    if nodes[source].failure == 0 {
      target := nodes[mod].next[edge]
      nodes[source].failure = target
      for _, m := range nodes[target].matches {
        nodes[source].matches = append(nodes[source].matches, m)
      }
    }
    bfs.node = nodes[bfs.node].next[edge]
    bfs.index++
    if bfs.index < len(datas[bfs.data]) {
      q = append(q, bfs)
    }
  }

  return nodes
}

func AhoCorasick(datas [][]byte, t []byte) [][]int {
  nodes := AhoCorasickPreprocessSet(datas)
  cur := 0
  matches := make([][]int, len(datas))
  for i, c := range t {
    for _, m := range nodes[cur].matches {
      matches[m] = append(matches[m], i - len(datas[m]))
    }
    for nodes[cur].next[c] == 0 {
      if nodes[cur].failure != 0 {
        cur = nodes[cur].failure
      } else {
        cur = 0
        break
      }
    }
    cur = nodes[cur].next[c]
  }
  for _, m := range nodes[cur].matches {
    matches[m] = append(matches[m], len(t) - len(datas[m]))
  }
  return matches
}
<commit_msg>sped up preprocessing a bit in aho corasick<commit_after>package stringz

type acNode struct {
  // Index into the acNodeArray for a given character
  next [256]int

  // failure link index
  failure int

  // This node indicates that the following elements matched
  matches []int
}

type ahBfs struct {
  node, data, index int
}

func AhoCorasickPreprocessSet(datas [][]byte) []acNode {
  total_len := 0
  for i := range datas {
    total_len += len(datas[i])
  }
  nodes := make([]acNode, total_len + 1)[0:1]
  for i,data := range datas {
    cur := 0
    for _,b := range data {
      if nodes[cur].next[b] != 0 {
        cur = nodes[cur].next[b]
        continue
      }
      nodes[cur].next[b] = len(nodes)
      cur = len(nodes)
      nodes = append(nodes, acNode{})
    }
    nodes[cur].matches = append(nodes[cur].matches, i)
  }

  // The skeleton of the graph is done, now we do a BFS on the nodes and form
  // failure links as we go.
  var q []ahBfs
  for i := range datas {
    // TODO: Figure out if this makes sense, maybe we should fix how the BFS
    // works instead?
    if len(datas[i]) > 1 {
      bfs := ahBfs{
        node:  nodes[0].next[datas[i][0]],
        data:  i,
        index: 1,
      }
      q = append(q, bfs)
    }
  }
  for len(q) > 0 {
    bfs := q[0]
    q = q[1:]
    mod := nodes[bfs.node].failure
    edge := datas[bfs.data][bfs.index]
    for mod != 0 && nodes[mod].next[edge] == 0 {
      mod = nodes[mod].failure
    }
    source := nodes[bfs.node].next[edge]
    if nodes[source].failure == 0 {
      target := nodes[mod].next[edge]
      nodes[source].failure = target
      for _, m := range nodes[target].matches {
        nodes[source].matches = append(nodes[source].matches, m)
      }
    }
    bfs.node = nodes[bfs.node].next[edge]
    bfs.index++
    if bfs.index < len(datas[bfs.data]) {
      q = append(q, bfs)
    }
  }

  return nodes
}

func AhoCorasick(datas [][]byte, t []byte) [][]int {
  nodes := AhoCorasickPreprocessSet(datas)
  cur := 0
  matches := make([][]int, len(datas))
  for i, c := range t {
    for _, m := range nodes[cur].matches {
      matches[m] = append(matches[m], i - len(datas[m]))
    }
    for nodes[cur].next[c] == 0 {
      if nodes[cur].failure != 0 {
        cur = nodes[cur].failure
      } else {
        cur = 0
        break
      }
    }
    cur = nodes[cur].next[c]
  }
  for _, m := range nodes[cur].matches {
    matches[m] = append(matches[m], len(t) - len(datas[m]))
  }
  return matches
}
<|endoftext|>


<commit_before>package clouddriveclient

import (
	"encoding/json"
	"github.com/koofr/go-httpclient"
)

type CloudDriveError struct {
	Code            string `json:"code"`
	Message         string `json:"message"`
	Logref          string `json:"logref"`
	HttpClientError *httpclient.InvalidStatusError
}

func (e *CloudDriveError) Error() string {
	return e.Message
}

func IsCloudDriveError(err error) (cloudDriveErr *CloudDriveError, ok bool) {
	if cde, ok := err.(*CloudDriveError); ok {
		return cde, true
	} else {
		return nil, false
	}
}

func HandleError(err error) error {
	if ise, ok := httpclient.IsInvalidStatusError(err); ok {
		cloudDriveErr := &CloudDriveError{}

		if ise.Headers.Get("Content-Type") == "application/vnd.error+json" {
			if jsonErr := json.Unmarshal([]byte(ise.Content), &cloudDriveErr); jsonErr != nil {
				cloudDriveErr.Code = "unknown"
				cloudDriveErr.Message = ise.Content
			}
		} else {
			cloudDriveErr.Code = "unknown"
			cloudDriveErr.Message = ise.Content
		}

		cloudDriveErr.HttpClientError = ise

		return cloudDriveErr
	} else {
		return err
	}
}
<commit_msg>Be tolerant of error content types.<commit_after>package clouddriveclient

import (
	"encoding/json"
	"github.com/koofr/go-httpclient"
)

type CloudDriveError struct {
	Code            string `json:"code"`
	Message         string `json:"message"`
	Logref          string `json:"logref"`
	HttpClientError *httpclient.InvalidStatusError
}

func (e *CloudDriveError) Error() string {
	return e.Message
}

func IsCloudDriveError(err error) (cloudDriveErr *CloudDriveError, ok bool) {
	if cde, ok := err.(*CloudDriveError); ok {
		return cde, true
	} else {
		return nil, false
	}
}

func HandleError(err error) error {
	if ise, ok := httpclient.IsInvalidStatusError(err); ok {
		cloudDriveErr := &CloudDriveError{}

		ct := ise.Headers.Get("Content-Type")
		if ct == "application/vnd.error+json" || ct == "application/json" {
			if jsonErr := json.Unmarshal([]byte(ise.Content), &cloudDriveErr); jsonErr != nil {
				cloudDriveErr.Code = "unknown"
				cloudDriveErr.Message = ise.Content
			}
		} else {
			cloudDriveErr.Code = "unknown"
			cloudDriveErr.Message = ise.Content
		}

		cloudDriveErr.HttpClientError = ise

		return cloudDriveErr
	} else {
		return err
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"time"

	"github.com/gorilla/websocket"
)

const (
	// Time allowed to write a message to the peer.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the peer.
	pongWait = 60 * time.Second

	// Send pings to peer with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10

	// Maximum message size allowed from peer.
	maxMessageSize = 512
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

// connection is an middleman between the websocket connection and the hub.
type connection struct {
	// The websocket connection.
	ws *websocket.Conn

	// Buffered channel of outbound messages.
	send chan []byte
}

// write writes a message with the given message type and payload.
func (c *connection) write(mt int, payload []byte) error {
	c.ws.SetWriteDeadline(time.Now().Add(writeWait))
	return c.ws.WriteMessage(mt, payload)
}

// writePump pumps messages from the hub to the websocket connection.
func (c *connection) writePump() {
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		c.ws.Close()
	}()

	for {
		select {
		case message, ok := <-c.send:
			if !ok {
				c.write(websocket.CloseMessage, []byte{})
				return
			}
			if err := c.write(websocket.TextMessage, message); err != nil {
				return
			}
		case <-ticker.C:
			if err := c.write(websocket.PingMessage, []byte{}); err != nil {
				return
			}
		}
	}
}
<commit_msg>make go websocket not check the origin<commit_after>// Copyright 2013 The Gorilla WebSocket Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"net/http"
	"time"

	"github.com/gorilla/websocket"
)

const (
	// Time allowed to write a message to the peer.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the peer.
	pongWait = 60 * time.Second

	// Send pings to peer with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10

	// Maximum message size allowed from peer.
	maxMessageSize = 512
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

// connection is an middleman between the websocket connection and the hub.
type connection struct {
	// The websocket connection.
	ws *websocket.Conn

	// Buffered channel of outbound messages.
	send chan []byte
}

// write writes a message with the given message type and payload.
func (c *connection) write(mt int, payload []byte) error {
	c.ws.SetWriteDeadline(time.Now().Add(writeWait))
	return c.ws.WriteMessage(mt, payload)
}

// writePump pumps messages from the hub to the websocket connection.
func (c *connection) writePump() {
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		c.ws.Close()
	}()

	for {
		select {
		case message, ok := <-c.send:
			if !ok {
				c.write(websocket.CloseMessage, []byte{})
				return
			}
			if err := c.write(websocket.TextMessage, message); err != nil {
				return
			}
		case <-ticker.C:
			if err := c.write(websocket.PingMessage, []byte{}); err != nil {
				return
			}
		}
	}
}
<|endoftext|>