<commit_before>/********************************
*** Web server API for Go     ***
*** Code is under MIT license ***
*** Code by CodingFerret      ***
*** github.com/squiidz        ***
*********************************/

package fur

import (
	"fmt"
	"log"
	"net/http"
)

var (
	origin Origin
)

// Simple Server structure for a web server.
type Server struct {
	Host string
	Port string
	Log  bool
	Mux  *http.ServeMux
}

type MiddleWare func(http.Handler) http.Handler

type Origin []MiddleWare

// Create a NewServer instance with the given value.
// Host: "localhost"
// Port: ":8080"
// Log: true/false
// Options: functions to run on the server instance who's gonna be return.
func NewServer(host string, port string, log bool, options ...func(s *Server)) *Server {
	svr := Server{host, port, log, http.NewServeMux()}
	for _, option := range options {
		option(&svr)
	}
	return &svr
}

func (s *Server) Stack(middles ...MiddleWare) {
	for _, middle := range middles {
		origin = append(origin, middle)
	}
}

// Start Listening on host and port of the Server.
// Log the request if the log was initiated as true in NewServer.
func (s *Server) Start() {
	fmt.Printf("[+] Server Running on %s ... \n", s.Port)
	if s.Log {
		http.ListenAndServe(s.Host+s.Port, s.logger(s.Mux))
	}
	http.ListenAndServe(s.Host+s.Port, s.Mux)
}

// Add function with the right sigature to the Server Mux
// and chain the provided middlewares on it.
func (s *Server) AddRoute(pat string, f func(rw http.ResponseWriter, req *http.Request), middles ...MiddleWare) {
	var stack http.Handler
	var midStack []MiddleWare

	if origin != nil {
		for _, or := range origin {
			midStack = append(midStack, or)
		}
	}

	if middles != nil {
		for _, mid := range middles {
			midStack = append(midStack, mid)
		}
	}

	if midStack != nil {

		stack = midStack[0](http.HandlerFunc(f))
		stack = wrap(stack, midStack[1:])

		s.Mux.Handle(pat, stack)
	} else {
		s.Mux.Handle(pat, http.HandlerFunc(f))
	}

}

// Log request to the Server.
func (s *Server) logger(mux http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		log.Printf("%s %s %s", req.RemoteAddr, req.Method, req.URL)
		mux.ServeHTTP(rw, req)
	})
}

func wrap(stack http.Handler, middles []MiddleWare) http.Handler {
	for i := len(middles) - 1; i >= 0; i-- {
		stack = middles[i](stack)
	}

	return stack
}
<commit_msg>Fix AddRoute loop<commit_after>/********************************
*** Web server API for Go     ***
*** Code is under MIT license ***
*** Code by CodingFerret      ***
*** github.com/squiidz        ***
*********************************/

package fur

import (
	"fmt"
	"log"
	"net/http"
)

var (
	origin Origin
)

// Simple Server structure for a web server.
type Server struct {
	Host string
	Port string
	Log  bool
	Mux  *http.ServeMux
}

type MiddleWare func(http.Handler) http.Handler

type Origin []MiddleWare

// Create a NewServer instance with the given value.
// Host: "localhost"
// Port: ":8080"
// Log: true/false
// Options: functions to run on the server instance who's gonna be return.
func NewServer(host string, port string, log bool, options ...func(s *Server)) *Server {
	svr := Server{host, port, log, http.NewServeMux()}
	for _, option := range options {
		option(&svr)
	}
	return &svr
}

func (s *Server) Stack(middles ...MiddleWare) {
	for _, middle := range middles {
		origin = append(origin, middle)
	}
}

// Start Listening on host and port of the Server.
// Log the request if the log was initiated as true in NewServer.
func (s *Server) Start() {
	fmt.Printf("[+] Server Running on %s ... \n", s.Port)
	if s.Log {
		http.ListenAndServe(s.Host+s.Port, s.logger(s.Mux))
	}
	http.ListenAndServe(s.Host+s.Port, s.Mux)
}

// Add function with the right sigature to the Server Mux
// and chain the provided middlewares on it.
func (s *Server) AddRoute(pat string, f func(rw http.ResponseWriter, req *http.Request), middles ...MiddleWare) {
	var stack http.Handler
	var midStack = origin

	if middles != nil {
		for _, mid := range middles {
			midStack = append(midStack, mid)
		}
	}

	if midStack != nil {

		stack = midStack[0](http.HandlerFunc(f))
		stack = wrap(stack, midStack[1:])

		s.Mux.Handle(pat, stack)
	} else {
		s.Mux.Handle(pat, http.HandlerFunc(f))
	}

}

// Log request to the Server.
func (s *Server) logger(mux http.Handler) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		log.Printf("%s %s %s", req.RemoteAddr, req.Method, req.URL)
		mux.ServeHTTP(rw, req)
	})
}

func wrap(stack http.Handler, middles []MiddleWare) http.Handler {
	for i := len(middles) - 1; i >= 0; i-- {
		stack = middles[i](stack)
	}

	return stack
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"syscall"
	"unsafe"
)

const (
	ansiEraseDisplay = "\033[2J"
	ansiResetCursor  = "\033[H"
	carriageReturn   = "\015"
)

var originalSttyState bytes.Buffer
var winRows uint16
var winCols uint16

type winsize struct {
	rows, cols, xpixel, ypixel uint16
}

func getWinsize() winsize {
	ws := winsize{}
	syscall.Syscall(syscall.SYS_IOCTL,
		uintptr(0), uintptr(syscall.TIOCGWINSZ),
		uintptr(unsafe.Pointer(&ws)))
	return ws
}

func getSttyState(state *bytes.Buffer) (err error) {
	cmd := exec.Command("stty", "-g")
	cmd.Stdin = os.Stdin
	cmd.Stdout = state
	return cmd.Run()
}

func setSttyState(state *bytes.Buffer) (err error) {
	cmd := exec.Command("stty", state.String())
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func NewTTY() (t *TTY, err error) {
	fh, err := os.OpenFile("/dev/tty", os.O_RDWR, 0666)
	if err != nil {
		return
	}
	t = &TTY{fh, ">> "}
	return
}

type TTY struct {
	*os.File
	prompt string
}

// Clears the screen and sets the cursor to first row, first column
func (t *TTY) resetScreen() {
	fmt.Fprint(t.File, ansiEraseDisplay+ansiResetCursor)
}

// Print prompt with `in`
func (t *TTY) printPrompt(in []byte) {
	fmt.Fprintf(t.File, t.prompt+"%s", in)
}

// Positions the cursor after the prompt and `inlen` colums to the right
func (t *TTY) cursorAfterPrompt(inlen int) {
	t.setCursorPos(0, len(t.prompt)+inlen)
}

// Sets the cursor to `line` and `col`
func (t *TTY) setCursorPos(line int, col int) {
	fmt.Fprintf(t.File, "\033[%d;%dH", line+1, col+1)
}

func init() {
	ws := getWinsize()
	winRows = ws.rows
	winCols = ws.cols
}

func main() {
	err := getSttyState(&originalSttyState)
	if err != nil {
		log.Fatal(err)
	}
	defer setSttyState(&originalSttyState)

	setSttyState(bytes.NewBufferString("cbreak"))
	setSttyState(bytes.NewBufferString("-echo"))

	tty, err := NewTTY()
	if err != nil {
		log.Fatal(err)
	}

	var input []byte = make([]byte, 0)
	var b []byte = make([]byte, 1)

	// TODO: Do not print results and input to STDOUT, only to TTY, only print
	// selected result to STDOUT

	// TODO: Run command in the background, draw as long as no new input given
	// if input is given, cancel command in background

	// TODO: Only print as many result lines as we have lines on the screen

	for {
		tty.resetScreen()
		tty.printPrompt(input[:len(input)])

		if len(input) > 0 {
			// TODO: move this in a goroutine, give it a quit-channel, stream the output
			// to the current position (line after prompt)

			fmt.Fprintf(tty, "\n")
			arg := fmt.Sprintf("%s", input[:len(input)])
			out, err := exec.Command("ag", arg).Output()
			if err != nil {
				log.Fatal(err)
			}

			outb := bytes.NewBuffer(out)
			outr := bufio.NewReader(outb)
			for i := 1; i < int(winRows)-1; i++ {
				line, err := outr.ReadBytes('\n')
				if err != nil && err != io.EOF {
					log.Fatal(err)
				}
				fmt.Fprintf(tty, "%s", line)
			}

			// Jump back to last typing position
			tty.cursorAfterPrompt(len(input))
		}

		tty.Read(b)
		switch b[0] {
		case 127:
			// Backspace
			if len(input) > 0 {
				input = input[:len(input)-1]
			}
		case 4, 13:
			// Return or Ctrl-D
			// fmt.Println("Result:")
			// fmt.Printf("%s%s\n", carriageReturn, string(input[:len(input)]))
			return
		default:
			// TODO: Default is wrong here. Only append printable characters to
			// input

			// TODO: Send a signal through the quit channel to the command in the background,
			// to cancel it
			// Everything else
			input = append(input, b...)
		}
	}
}
<commit_msg>Add TODO<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"syscall"
	"unsafe"
)

const (
	ansiEraseDisplay = "\033[2J"
	ansiResetCursor  = "\033[H"
	carriageReturn   = "\015"
)

var originalSttyState bytes.Buffer
var winRows uint16
var winCols uint16

type winsize struct {
	rows, cols, xpixel, ypixel uint16
}

func getWinsize() winsize {
	ws := winsize{}
	syscall.Syscall(syscall.SYS_IOCTL,
		uintptr(0), uintptr(syscall.TIOCGWINSZ),
		uintptr(unsafe.Pointer(&ws)))
	return ws
}

func getSttyState(state *bytes.Buffer) (err error) {
	cmd := exec.Command("stty", "-g")
	cmd.Stdin = os.Stdin
	cmd.Stdout = state
	return cmd.Run()
}

func setSttyState(state *bytes.Buffer) (err error) {
	cmd := exec.Command("stty", state.String())
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func NewTTY() (t *TTY, err error) {
	fh, err := os.OpenFile("/dev/tty", os.O_RDWR, 0666)
	if err != nil {
		return
	}
	t = &TTY{fh, ">> "}
	return
}

type TTY struct {
	*os.File
	prompt string
}

// Clears the screen and sets the cursor to first row, first column
func (t *TTY) resetScreen() {
	fmt.Fprint(t.File, ansiEraseDisplay+ansiResetCursor)
}

// Print prompt with `in`
func (t *TTY) printPrompt(in []byte) {
	fmt.Fprintf(t.File, t.prompt+"%s", in)
}

// Positions the cursor after the prompt and `inlen` colums to the right
func (t *TTY) cursorAfterPrompt(inlen int) {
	t.setCursorPos(0, len(t.prompt)+inlen)
}

// Sets the cursor to `line` and `col`
func (t *TTY) setCursorPos(line int, col int) {
	fmt.Fprintf(t.File, "\033[%d;%dH", line+1, col+1)
}

func init() {
	ws := getWinsize()
	winRows = ws.rows
	winCols = ws.cols
}

func main() {
	err := getSttyState(&originalSttyState)
	if err != nil {
		log.Fatal(err)
	}
	// TODO: this needs to be run when the process is interrupted
	defer setSttyState(&originalSttyState)

	setSttyState(bytes.NewBufferString("cbreak"))
	setSttyState(bytes.NewBufferString("-echo"))

	tty, err := NewTTY()
	if err != nil {
		log.Fatal(err)
	}

	var input []byte = make([]byte, 0)
	var b []byte = make([]byte, 1)

	// TODO: Do not print results and input to STDOUT, only to TTY, only print
	// selected result to STDOUT

	// TODO: Run command in the background, draw as long as no new input given
	// if input is given, cancel command in background

	// TODO: Only print as many result lines as we have lines on the screen

	for {
		tty.resetScreen()
		tty.printPrompt(input[:len(input)])

		if len(input) > 0 {
			// TODO: move this in a goroutine, give it a quit-channel, stream the output
			// to the current position (line after prompt)

			fmt.Fprintf(tty, "\n")
			arg := fmt.Sprintf("%s", input[:len(input)])
			out, err := exec.Command("ag", arg).Output()
			if err != nil {
				log.Fatal(err)
			}

			outb := bytes.NewBuffer(out)
			outr := bufio.NewReader(outb)
			for i := 1; i < int(winRows)-1; i++ {
				line, err := outr.ReadBytes('\n')
				if err != nil && err != io.EOF {
					log.Fatal(err)
				}
				fmt.Fprintf(tty, "%s", line)
			}

			// Jump back to last typing position
			tty.cursorAfterPrompt(len(input))
		}

		tty.Read(b)
		switch b[0] {
		case 127:
			// Backspace
			if len(input) > 0 {
				input = input[:len(input)-1]
			}
		case 4, 13:
			// Return or Ctrl-D
			// fmt.Println("Result:")
			// fmt.Printf("%s%s\n", carriageReturn, string(input[:len(input)]))
			return
		default:
			// TODO: Default is wrong here. Only append printable characters to
			// input

			// TODO: Send a signal through the quit channel to the command in the background,
			// to cancel it
			// Everything else
			input = append(input, b...)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type mpath struct {
	name string
	path string
}
type vdir string

var mpathre = regexp.MustCompile(`^\.(.*)###(.*)$`)
var vdirre = regexp.MustCompile(`^[a-f0-9]{64}$`)

func newmpath(mp string) *mpath {
	res := mpathre.FindAllStringSubmatch(mp, -1)
	var mres *mpath
	if res != nil && len(res) == 1 && len(res[0]) == 3 {
		name := res[0][1]
		path := res[0][2]
		path = strings.Replace(path, ",#,", "/", -1)
		mres = &mpath{name, path}
	}
	return mres
}

func (mp *mpath) String() string {
	return "." + mp.name + "###" + strings.Replace(mp.path, "/", ",#,", -1)
}

func newvdir(vd string) vdir {
	res := vdirre.FindString(vd)
	vres := vdir("")
	if res != "" && res == vd {
		vres = vdir(vd)
	}
	return vres
}

type marker struct {
	mp  *mpath
	dir vdir
}

type markers []*marker

var allmarkers = markers{}

func (marks markers) getMarker(name string, path string, dir vdir) *marker {
	var mark *marker
	for _, marker := range marks {
		if marker.mp.name == name && marker.mp.path == path {
			mark = marker
			break
		}
	}
	if mark == nil {
		mark = &marker{mp: &mpath{name: name, path: path}, dir: dir}
	}
	if mark.dir != dir {
		// TODO: move dir and ln -s mark.dir dir
		fmt.Printf("Mark container named '%s' for path '%s' as link to '%s' (from '%s')\n", name, path, mark.dir, dir)
	}
	// TODO: check that link exists
	return mark
}

type volume struct {
	path string
	dir  vdir
	mark *marker
}

type volumes []*volume

var allvolumes = volumes{}

func (v *volume) String() string {
	return "vol '" + string(v.dir) + "'"
}

func (vols volumes) getVolume(vd vdir, path string, name string) *volume {
	var vol *volume
	for _, volume := range vols {
		if string(volume.dir) == string(vd) {
			vol = volume
			if vol.path == "" {
				vol.path = path
			}
			if vol.path != path {
				fmt.Printf("Invalid volume path detected: '%s' (vs. container volume path '%s')\n", vol.path, path)
			}
			if vol.mark == nil {
				vol.mark = allmarkers.getMarker(name, vol.path, vol.dir)
			}
			break
		}
	}
	if vol == nil {
		// TODO make marker
		vol = &volume{path: path, dir: vd}
	}
	return vol
}

type container struct {
	name    string
	id      string
	stopped bool
	volumes []*volume
}

func (c *container) trunc() string {
	if len(c.id) > 0 {
		return c.id[:7]
	}
	return ""
}

func (c *container) String() string {
	return "cnt '" + c.name + "' (" + c.trunc() + ")" + fmt.Sprintf("[%v] - %d vol", c.stopped, len(c.volumes))
}

var containers = []*container{}

func mustcmd(acmd string) string {
	out, err := cmd(acmd)
	if err != nil {
		log.Fatal(fmt.Sprintf("out='%s', err='%s'", out, err))
	}
	return string(out)
}

func cmd(cmd string) (string, error) {
	fmt.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	return strings.TrimSpace(string(out)), err
}

func readVolumes() {
	out := mustcmd("sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir")
	vollines := strings.Split(out, "\n")
	// fmt.Println(vollines)
	for _, volline := range vollines {
		dir := volline
		if dir == "./" || dir == "../" {
			continue
		}
		if strings.HasSuffix(dir, "@") {
			dir = dir[:len(dir)-1]
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			mp := newmpath(dir)
			if mp == nil {
				fmt.Printf("Invalid marker detected: '%s'\n", dir)
				mustcmd("sudo rm " + fdir)
			} else {
				dirlink, err := cmd("sudo readlink " + fdir)
				fmt.Printf("---\ndir: '%s'\ndlk: '%s'\nerr='%v'\n", dir, dirlink, err)
				if err != nil {
					fmt.Printf("Invalid marker (no readlink) detected: '%s'\n", dir)
					mustcmd("sudo rm " + fdir)
				} else {
					_, err := cmd("sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" + dirlink)
					if err != nil {
						fmt.Printf("Invalid marker (readlink no ls) detected: '%s'\n", dir)
						mustcmd("sudo rm " + fdir)
					} else {
						vd := newvdir(dirlink)
						if vd == "" {
							fmt.Printf("Invalid marker (readlink no vdir) detected: '%s'\n", dir)
							mustcmd("sudo rm " + fdir)
						} else {
							allmarkers = append(allmarkers, &marker{mp, vd})
						}
					}
				}
			}
		} else if strings.HasSuffix(dir, "/") {

			dir = dir[:len(dir)-1]
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			vd := newvdir(dir)
			if vd == "" {
				fmt.Printf("Invalid volume folder detected: '%s'\n", dir)
				mustcmd("sudo rm " + fdir)
			} else {
				allvolumes = append(allvolumes, &volume{dir: vd})
			}
		} else {
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			fmt.Printf("Invalid file detected: '%s'\n", dir)
			mustcmd("sudo rm " + fdir)
		}
	}
	fmt.Printf("volumes: %v\nmarkers: %v\n", allvolumes, allmarkers)
}

func readContainer() {

	out := mustcmd("docker ps -aq --no-trunc")
	contlines := strings.Split(out, "\n")
	// fmt.Println(contlines)
	for _, contline := range contlines {
		id := contline
		res := mustcmd("docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' " + id)
		// fmt.Println("res1: '" + res + "'")
		name := res[1:strings.Index(res, ",")]
		cont := &container{name: name, id: id}
		res = res[strings.Index(res, ",")+1:]
		// fmt.Println("res2: '" + res + "'")
		vols := strings.Split(res, "##~#")
		// fmt.Println(vols)
		for _, vol := range vols {
			elts := strings.Split(vol, ",")
			if len(elts) == 2 {
				// fmt.Printf("elts: '%v'\n", elts)
				path := elts[0]
				vfs := elts[1]
				if strings.Contains(vfs, "/var/lib/docker/vfs/dir/") {
					vd := newvdir(filepath.Base(vfs))
					if vd == "" {
						fmt.Printf("Invalid volume folder detected: '%s'\n", vfs)
						break
					}
					newvol := allvolumes.getVolume(vd, path, name)
					cont.volumes = append(cont.volumes, newvol)
				}
			}
		}
		containers = append(containers, cont)
	}
	fmt.Printf("containers: %v\n", containers)
}

func (v *volume) accept(m *marker) bool {
	// TODO
	return false
}

func checkContainers() {
	for _, container := range containers {
		for _, volume := range container.volumes {
			if volume.mark == nil {
				for _, mark := range allmarkers {
					if volume.accept(mark) {
						fmt.Printf("Set mark '%v' to volume '%v' of container '%v'\n", mark, volume, container)
						volume.mark = mark
						// TODO check if ln is needed
					}
				}
			}
			if volume.mark == nil {
				// TODO check if vfs folder exist.
				// If it does, make the marker
			}
		}
	}
}

func (c *container) accept(v *volume) bool {
	// TODO
	return false
}

func checkVolumes() {
	for _, volume := range allvolumes {
		orphan := true
		for _, container := range containers {
			if container.accept(volume) {
				orphan = false
				break
			}
		}
		if orphan {
			fmt.Printf("Orphan detected, volume '%v'\n", volume)
			// TODO rm if necessary or at least mv _xxx
		}
	}
}

// docker run --rm -i -t -v `pwd`:`pwd` -w `pwd` --entrypoint="/bin/bash" go -c 'go build gcl.go'
func main() {
	fmt.Println("### 1/4 readVolumes     ###")
	readVolumes()
	fmt.Println("### 2/4 readContainer   ###")
	readContainer()
	fmt.Println("### 3/4 checkContainers ###")
	checkContainers()
	fmt.Println("### 4/4 checkVolumes    ###")
	checkVolumes()
	fmt.Println("---      All done       ---")
	os.Exit(0)
}
<commit_msg>gcl.go: add Strings() for vdir and marker<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
)

type mpath struct {
	name string
	path string
}

type vdir string

func (vd vdir) trunc() string {
	s := string(vd)
	if len(s) > 0 {
		return s[:7]
	}
	return ""
}

var mpathre = regexp.MustCompile(`^\.(.*)###(.*)$`)
var vdirre = regexp.MustCompile(`^[a-f0-9]{64}$`)

func newmpath(mp string) *mpath {
	res := mpathre.FindAllStringSubmatch(mp, -1)
	var mres *mpath
	if res != nil && len(res) == 1 && len(res[0]) == 3 {
		name := res[0][1]
		path := res[0][2]
		path = strings.Replace(path, ",#,", "/", -1)
		mres = &mpath{name, path}
	}
	return mres
}

func (mp *mpath) String() string {
	return "." + mp.name + "###" + strings.Replace(mp.path, "/", ",#,", -1)
}

func newvdir(vd string) vdir {
	res := vdirre.FindString(vd)
	vres := vdir("")
	if res != "" && res == vd {
		vres = vdir(vd)
	}
	return vres
}

type marker struct {
	mp  *mpath
	dir vdir
}

func (mk *marker) String() string {
	return fmt.Sprintf("marker '%s'<%s->%s>", mk.dir.trunc(), mk.mp.name, mk.mp.path)
}

type markers []*marker

var allmarkers = markers{}

func (marks markers) getMarker(name string, path string, dir vdir) *marker {
	var mark *marker
	for _, marker := range marks {
		if marker.mp.name == name && marker.mp.path == path {
			mark = marker
			break
		}
	}
	if mark == nil {
		mark = &marker{mp: &mpath{name: name, path: path}, dir: dir}
	}
	if mark.dir != dir {
		// TODO: move dir and ln -s mark.dir dir
		fmt.Printf("Mark container named '%s' for path '%s' as link to '%s' (from '%s')\n", name, path, mark.dir, dir)
	}
	// TODO: check that link exists
	return mark
}

type volume struct {
	path string
	dir  vdir
	mark *marker
}

type volumes []*volume

var allvolumes = volumes{}

func (v *volume) String() string {
	return "vol '" + string(v.dir) + "'"
}

func (vols volumes) getVolume(vd vdir, path string, name string) *volume {
	var vol *volume
	for _, volume := range vols {
		if string(volume.dir) == string(vd) {
			vol = volume
			if vol.path == "" {
				vol.path = path
			}
			if vol.path != path {
				fmt.Printf("Invalid volume path detected: '%s' (vs. container volume path '%s')\n", vol.path, path)
			}
			if vol.mark == nil {
				vol.mark = allmarkers.getMarker(name, vol.path, vol.dir)
			}
			break
		}
	}
	if vol == nil {
		// TODO make marker
		vol = &volume{path: path, dir: vd}
	}
	return vol
}

type container struct {
	name    string
	id      string
	stopped bool
	volumes []*volume
}

func (c *container) trunc() string {
	if len(c.id) > 0 {
		return c.id[:7]
	}
	return ""
}

func (c *container) String() string {
	return "cnt '" + c.name + "' (" + c.trunc() + ")" + fmt.Sprintf("[%v] - %d vol", c.stopped, len(c.volumes))
}

var containers = []*container{}

func mustcmd(acmd string) string {
	out, err := cmd(acmd)
	if err != nil {
		log.Fatal(fmt.Sprintf("out='%s', err='%s'", out, err))
	}
	return string(out)
}

func cmd(cmd string) (string, error) {
	fmt.Println(cmd)
	out, err := exec.Command("sh", "-c", cmd).Output()
	return strings.TrimSpace(string(out)), err
}

func readVolumes() {
	out := mustcmd("sudo ls -a1F /mnt/sda1/var/lib/docker/vfs/dir")
	vollines := strings.Split(out, "\n")
	// fmt.Println(vollines)
	for _, volline := range vollines {
		dir := volline
		if dir == "./" || dir == "../" {
			continue
		}
		if strings.HasSuffix(dir, "@") {
			dir = dir[:len(dir)-1]
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			mp := newmpath(dir)
			if mp == nil {
				fmt.Printf("Invalid marker detected: '%s'\n", dir)
				mustcmd("sudo rm " + fdir)
			} else {
				dirlink, err := cmd("sudo readlink " + fdir)
				fmt.Printf("---\ndir: '%s'\ndlk: '%s'\nerr='%v'\n", dir, dirlink, err)
				if err != nil {
					fmt.Printf("Invalid marker (no readlink) detected: '%s'\n", dir)
					mustcmd("sudo rm " + fdir)
				} else {
					_, err := cmd("sudo ls /mnt/sda1/var/lib/docker/vfs/dir/" + dirlink)
					if err != nil {
						fmt.Printf("Invalid marker (readlink no ls) detected: '%s'\n", dir)
						mustcmd("sudo rm " + fdir)
					} else {
						vd := newvdir(dirlink)
						if vd == "" {
							fmt.Printf("Invalid marker (readlink no vdir) detected: '%s'\n", dir)
							mustcmd("sudo rm " + fdir)
						} else {
							allmarkers = append(allmarkers, &marker{mp, vd})
						}
					}
				}
			}
		} else if strings.HasSuffix(dir, "/") {

			dir = dir[:len(dir)-1]
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			vd := newvdir(dir)
			if vd == "" {
				fmt.Printf("Invalid volume folder detected: '%s'\n", dir)
				mustcmd("sudo rm " + fdir)
			} else {
				allvolumes = append(allvolumes, &volume{dir: vd})
			}
		} else {
			fdir := fmt.Sprintf("/mnt/sda1/var/lib/docker/vfs/dir/%s", dir)
			fmt.Printf("Invalid file detected: '%s'\n", dir)
			mustcmd("sudo rm " + fdir)
		}
	}
	fmt.Printf("volumes: %v\nmarkers: %v\n", allvolumes, allmarkers)
}

func readContainer() {

	out := mustcmd("docker ps -aq --no-trunc")
	contlines := strings.Split(out, "\n")
	// fmt.Println(contlines)
	for _, contline := range contlines {
		id := contline
		res := mustcmd("docker inspect -f '{{ .Name }},{{ range $key, $value := .Volumes }}{{ $key }},{{ $value }}##~#{{ end }}' " + id)
		// fmt.Println("res1: '" + res + "'")
		name := res[1:strings.Index(res, ",")]
		cont := &container{name: name, id: id}
		res = res[strings.Index(res, ",")+1:]
		// fmt.Println("res2: '" + res + "'")
		vols := strings.Split(res, "##~#")
		// fmt.Println(vols)
		for _, vol := range vols {
			elts := strings.Split(vol, ",")
			if len(elts) == 2 {
				// fmt.Printf("elts: '%v'\n", elts)
				path := elts[0]
				vfs := elts[1]
				if strings.Contains(vfs, "/var/lib/docker/vfs/dir/") {
					vd := newvdir(filepath.Base(vfs))
					if vd == "" {
						fmt.Printf("Invalid volume folder detected: '%s'\n", vfs)
						break
					}
					newvol := allvolumes.getVolume(vd, path, name)
					cont.volumes = append(cont.volumes, newvol)
				}
			}
		}
		containers = append(containers, cont)
	}
	fmt.Printf("containers: %v\n", containers)
}

func (v *volume) accept(m *marker) bool {
	// TODO
	return false
}

func checkContainers() {
	for _, container := range containers {
		for _, volume := range container.volumes {
			if volume.mark == nil {
				for _, mark := range allmarkers {
					if volume.accept(mark) {
						fmt.Printf("Set mark '%v' to volume '%v' of container '%v'\n", mark, volume, container)
						volume.mark = mark
						// TODO check if ln is needed
					}
				}
			}
			if volume.mark == nil {
				// TODO check if vfs folder exist.
				// If it does, make the marker
			}
		}
	}
}

func (c *container) accept(v *volume) bool {
	// TODO
	return false
}

func checkVolumes() {
	for _, volume := range allvolumes {
		orphan := true
		for _, container := range containers {
			if container.accept(volume) {
				orphan = false
				break
			}
		}
		if orphan {
			fmt.Printf("Orphan detected, volume '%v'\n", volume)
			// TODO rm if necessary or at least mv _xxx
		}
	}
}

// docker run --rm -i -t -v `pwd`:`pwd` -w `pwd` --entrypoint="/bin/bash" go -c 'go build gcl.go'
func main() {
	fmt.Println("### 1/4 readVolumes     ###")
	readVolumes()
	fmt.Println("### 2/4 readContainer   ###")
	readContainer()
	fmt.Println("### 3/4 checkContainers ###")
	checkContainers()
	fmt.Println("### 4/4 checkVolumes    ###")
	checkVolumes()
	fmt.Println("---      All done       ---")
	os.Exit(0)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"os/exec"
	"regexp"
)

type Matcher struct {
	re *regexp.Regexp
}

func NewMatcher(expr string) (m *Matcher, err error) {
	m = &Matcher{}
	m.re, err = regexp.Compile(expr)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (m *Matcher) MatchString(s string) bool {
	return m.re.MatchString(s)
}

type Processor struct {
	cmd *exec.Cmd
}

func NewProcessor(name string, arg ...string) (p *Processor, err error) {
	if _, err = exec.LookPath(name); err != nil {
		return nil, err
	}
	p = &Processor{}
	p.cmd = exec.Command(name, arg...)
	return p, nil
}

func (p *Processor) Process(a []string) error {
	in, err := p.cmd.StdinPipe()
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := p.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	defer out.Close()

	if err = p.cmd.Start(); err != nil {
		return err
	}
	for _, s := range a {
		fmt.Fprintln(in, s)
	}
	if err = in.Close(); err != nil {
		return err
	}

	b := bufio.NewScanner(out)
	for i := 0; i < len(a) && b.Scan(); i++ {
		a[i] = b.Text()
	}
	return b.Err()
}

type Lines struct {
	matcher        *Matcher
	processor      *Processor
	lines          []string
	matchedLines   []string
	matchedIndexes map[int]bool
}

func NewLines(m *Matcher, p *Processor) *Lines {
	return &Lines{
		matcher:        m,
		processor:      p,
		lines:          []string{},
		matchedLines:   []string{},
		matchedIndexes: make(map[int]bool),
	}
}

func (l *Lines) LoadLines(r io.Reader) error {
	b := bufio.NewScanner(r)
	for i := 0; b.Scan(); i++ {
		line := b.Text()
		if l.matcher.MatchString(line) {
			l.matchedLines = append(l.matchedLines, line)
			l.matchedIndexes[i] = true
		}
		l.lines = append(l.lines, line)
	}
	return b.Err()
}
<commit_msg>Remove Matcher and Processor from Lines's fields<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"os/exec"
	"regexp"
)

type Matcher struct {
	re *regexp.Regexp
}

func NewMatcher(expr string) (m *Matcher, err error) {
	m = &Matcher{}
	m.re, err = regexp.Compile(expr)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func (m *Matcher) MatchString(s string) bool {
	return m.re.MatchString(s)
}

type Processor struct {
	cmd *exec.Cmd
}

func NewProcessor(name string, arg ...string) (p *Processor, err error) {
	if _, err = exec.LookPath(name); err != nil {
		return nil, err
	}
	p = &Processor{}
	p.cmd = exec.Command(name, arg...)
	return p, nil
}

func (p *Processor) Process(a []string) error {
	in, err := p.cmd.StdinPipe()
	if err != nil {
		return err
	}
	defer in.Close()

	out, err := p.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	defer out.Close()

	if err = p.cmd.Start(); err != nil {
		return err
	}
	for _, s := range a {
		fmt.Fprintln(in, s)
	}
	if err = in.Close(); err != nil {
		return err
	}

	b := bufio.NewScanner(out)
	for i := 0; i < len(a) && b.Scan(); i++ {
		a[i] = b.Text()
	}
	return b.Err()
}

type Lines struct {
	lines          []string
	matchedLines   []string
	matchedIndexes map[int]bool
}

func NewLines() *Lines {
	return &Lines{
		lines:          []string{},
		matchedLines:   []string{},
		matchedIndexes: make(map[int]bool),
	}
}

func (l *Lines) LoadLines(r io.Reader, m *Matcher) error {
	b := bufio.NewScanner(r)
	for i := 0; b.Scan(); i++ {
		line := b.Text()
		if m.MatchString(line) {
			l.matchedLines = append(l.matchedLines, line)
			l.matchedIndexes[i] = true
		}
		l.lines = append(l.lines, line)
	}
	return b.Err()
}
<|endoftext|>


<commit_before>package creeperkeeper

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"sync"
	"time"
)

const vineDateFormat = "2006-01-02T15:04:05.999999"

var fallbackUUIDCount = 0
var uuidMutex = sync.Mutex{}

type vineExtractor func(url string) (vines []Vine, err error)

// DownloadVines downloads vines to files named after their shortIDs, eg
// bnmHnwVILKD.mp4.
func DownloadVines(vines []Vine) error {
	f := func(i interface{}) (err error) {
		vine := i.(Vine)
		file, err := os.Create(vine.VideoFilename())
		if err != nil {
			return err
		}
		defer func() {
			if cerr := file.Close(); err == nil && cerr != nil {
				err = cerr
			}
		}()
		err = vine.Download(file)
		if err != nil {
			log.Printf("get %.20q: %s", vine.Title, err)
		} else if Verbose {
			log.Printf("got %q", vine.Title)
		}
		return err
	}

	// Convert []Vine to []interface{}
	jobs := make([]interface{}, len(vines))
	for i, v := range vines {
		jobs[i] = v
	}

	nerr := parallel(jobs, f, 4)
	if nerr > 0 {
		return fmt.Errorf("%d/%d failed", nerr, len(vines))
	}
	return nil
}

// ExtractVines gets vine metadata related to a url for a single vine, a user
// profile, or a user's likes. API requests are made as necessary to get all of
// a user's posts or likes.
func ExtractVines(url string) (vines []Vine, err error) {
	extractors := map[string]vineExtractor{
		"individual": vineExtractor(vineURLToVines),
		"user":       vineExtractor(userURLToVines),
	}
	errs := map[string]error{}
	for name, extractor := range extractors {
		vines, err = extractor(url)
		if err != nil {
			err = fmt.Errorf("%s: %s", name, err.Error())
			errs[name] = err
		}
		if len(vines) > 0 {
			return vines, err
		}
	}
	s := "vine extraction: "
	for name, err := range errs {
		s += fmt.Sprintf("%s: %s", name, err.Error())
	}
	return nil, errors.New(s)
}

// vineURLToVines gets vine metadata for the vine referred to by the given URL.
func vineURLToVines(url string) (vines []Vine, err error) {
	vineURLRE := regexp.MustCompile(`https?://(?:www\.)?vine\.co/(?:v|oembed)/([^?/]+)`)
	m := vineURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		return nil, fmt.Errorf("vineURLToVines: unrecognized url: %s", url)
	}
	id := m[1]
	vine, err := getVine(id)
	if err != nil {
		return nil, err
	}
	return []Vine{vine}, nil
}

func getVine(id string) (Vine, error) {
	var jv jsonVine
	url := fmt.Sprintf("https://archive.vine.co/posts/%s.json", id)
	err := deserialize(url, &jv)
	if err != nil {
		return Vine{}, fmt.Errorf("getVine %s: %s", id, err)
	}
	created, err := time.Parse(vineDateFormat, jv.Created)
	if err != nil {
		return Vine{}, fmt.Errorf("getVine %s: %s", id, err)
	}
	return Vine{
		Title:      jv.Description,
		Uploader:   jv.Username,
		UploaderID: jv.UserIdStr,
		URL:        jv.VideoURL,
		UUID:       id,
		Created:    created,
	}, nil
}

func userURLToVines(url string) ([]Vine, error) {
	userID, err := userURLToUserID(url)
	if err != nil {
		return nil, fmt.Errorf("userURLToVines: %s", err)
	}
	var ju jsonUser
	postsURL := fmt.Sprintf("https://archive.vine.co/profiles/%s.json", userID)
	err = deserialize(postsURL, &ju)
	if err != nil {
		return nil, fmt.Errorf("userURLToVines: %s", err)
	}

	var vines []Vine
	vineq := make(chan Vine)
	wg := sync.WaitGroup{}
	go func() {
		wg.Add(1)
		for vine := range vineq {
			vines = append(vines, vine)
		}
		wg.Done()
	}()
	jobs := make([]interface{}, len(ju.Posts))
	for i, v := range ju.Posts {
		jobs[i] = v
	}
	f := func(i interface{}) error {
		id := i.(string)
		vine, err := getVine(id)
		if err != nil {
			return err
		}
		vineq <- vine
		return nil
	}
	nerr := parallel(jobs, f, 8)
	close(vineq)
	wg.Wait()
	if nerr > 0 {
		return vines, fmt.Errorf("get vine metadata: %d/%d failed", nerr, len(ju.Posts))
	}
	return vines, nil
}

func userURLToUserID(url string) (string, error) {
	userURLRE := regexp.MustCompile(`(?:https?://)?vine\.co/(u/)?([^/]+)/?(?:\?.*)?$`)
	m := userURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		return "", fmt.Errorf("unrecognized vine user url: %q", url)
	}
	isVanity := len(m[1]) == 0
	if isVanity {
		profileURL := fmt.Sprintf("https://vine.co/api/users/profiles/vanity/%s", m[2])
		var jve jsonVanityEnvelope
		err := deserialize(profileURL, &jve)
		if err != nil {
			return "", err
		}
		return fmt.Sprint(jve.Data.UserID), nil
	} else {
		return m[2], nil
	}
}

// deserialize GETs a JSON API endpoint, unwraps the enveloping object and
// unmarshals the response.
func deserialize(url string, d interface{}) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("HTTP %d: %s", resp.StatusCode, body)
	}
	err = json.Unmarshal(body, &d)
	if err != nil {
		return fmt.Errorf("unrecognized json %s", err)
	}
	return nil
}

func vineURLToUUID(url string) string {
	vineURLRE := regexp.MustCompile(`https://vine\.co/v/([a-zA-Z0-9]+)$`)
	m := vineURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		uuidMutex.Lock()
		defer uuidMutex.Unlock()
		return fmt.Sprintf("fallbackID%d", fallbackUUIDCount)
	}
	return string(m[1])
}

type jsonUser struct {
	Posts []string
}

type jsonVine struct {
	Description string
	Username    string
	UserIdStr   string
	VideoURL    string
	Created     string
}

// User API JSON structures
type jsonVanityEnvelope struct {
	Data jsonVanity
}
type jsonVanity struct {
	UserID int64
}
<commit_msg>Log number of vines in user metadata<commit_after>package creeperkeeper

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"regexp"
	"sync"
	"time"
)

const vineDateFormat = "2006-01-02T15:04:05.999999"

var fallbackUUIDCount = 0
var uuidMutex = sync.Mutex{}

type vineExtractor func(url string) (vines []Vine, err error)

// DownloadVines downloads vines to files named after their shortIDs, eg
// bnmHnwVILKD.mp4.
func DownloadVines(vines []Vine) error {
	f := func(i interface{}) (err error) {
		vine := i.(Vine)
		file, err := os.Create(vine.VideoFilename())
		if err != nil {
			return err
		}
		defer func() {
			if cerr := file.Close(); err == nil && cerr != nil {
				err = cerr
			}
		}()
		err = vine.Download(file)
		if err != nil {
			log.Printf("get %.20q: %s", vine.Title, err)
		} else if Verbose {
			log.Printf("got %q", vine.Title)
		}
		return err
	}

	// Convert []Vine to []interface{}
	jobs := make([]interface{}, len(vines))
	for i, v := range vines {
		jobs[i] = v
	}

	nerr := parallel(jobs, f, 4)
	if nerr > 0 {
		return fmt.Errorf("%d/%d failed", nerr, len(vines))
	}
	return nil
}

// ExtractVines gets vine metadata related to a url for a single vine, a user
// profile, or a user's likes. API requests are made as necessary to get all of
// a user's posts or likes.
func ExtractVines(url string) (vines []Vine, err error) {
	extractors := map[string]vineExtractor{
		"individual": vineExtractor(vineURLToVines),
		"user":       vineExtractor(userURLToVines),
	}
	errs := map[string]error{}
	for name, extractor := range extractors {
		vines, err = extractor(url)
		if err != nil {
			err = fmt.Errorf("%s: %s", name, err.Error())
			errs[name] = err
		}
		if len(vines) > 0 {
			return vines, err
		}
	}
	s := "vine extraction: "
	for name, err := range errs {
		s += fmt.Sprintf("%s: %s", name, err.Error())
	}
	return nil, errors.New(s)
}

// vineURLToVines gets vine metadata for the vine referred to by the given URL.
func vineURLToVines(url string) (vines []Vine, err error) {
	vineURLRE := regexp.MustCompile(`https?://(?:www\.)?vine\.co/(?:v|oembed)/([^?/]+)`)
	m := vineURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		return nil, fmt.Errorf("vineURLToVines: unrecognized url: %s", url)
	}
	id := m[1]
	vine, err := getVine(id)
	if err != nil {
		return nil, err
	}
	return []Vine{vine}, nil
}

func getVine(id string) (Vine, error) {
	var jv jsonVine
	url := fmt.Sprintf("https://archive.vine.co/posts/%s.json", id)
	err := deserialize(url, &jv)
	if err != nil {
		return Vine{}, fmt.Errorf("getVine %s: %s", id, err)
	}
	created, err := time.Parse(vineDateFormat, jv.Created)
	if err != nil {
		return Vine{}, fmt.Errorf("getVine %s: %s", id, err)
	}
	return Vine{
		Title:      jv.Description,
		Uploader:   jv.Username,
		UploaderID: jv.UserIdStr,
		URL:        jv.VideoURL,
		UUID:       id,
		Created:    created,
	}, nil
}

func userURLToVines(url string) ([]Vine, error) {
	userID, err := userURLToUserID(url)
	if err != nil {
		return nil, fmt.Errorf("userURLToVines: %s", err)
	}
	var ju jsonUser
	postsURL := fmt.Sprintf("https://archive.vine.co/profiles/%s.json", userID)
	err = deserialize(postsURL, &ju)
	if err != nil {
		return nil, fmt.Errorf("userURLToVines: %s", err)
	}

	if Verbose {
		log.Printf("getting metadata for %d vines", len(ju.Posts))
	}

	var vines []Vine
	vineq := make(chan Vine)
	wg := sync.WaitGroup{}
	go func() {
		wg.Add(1)
		for vine := range vineq {
			vines = append(vines, vine)
		}
		wg.Done()
	}()
	jobs := make([]interface{}, len(ju.Posts))
	for i, v := range ju.Posts {
		jobs[i] = v
	}
	f := func(i interface{}) error {
		id := i.(string)
		vine, err := getVine(id)
		if err != nil {
			return err
		}
		vineq <- vine
		return nil
	}
	nerr := parallel(jobs, f, 8)
	close(vineq)
	wg.Wait()
	if nerr > 0 {
		return vines, fmt.Errorf("get vine metadata: %d/%d failed", nerr, len(ju.Posts))
	}
	return vines, nil
}

func userURLToUserID(url string) (string, error) {
	userURLRE := regexp.MustCompile(`(?:https?://)?vine\.co/(u/)?([^/]+)/?(?:\?.*)?$`)
	m := userURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		return "", fmt.Errorf("unrecognized vine user url: %q", url)
	}
	isVanity := len(m[1]) == 0
	if isVanity {
		profileURL := fmt.Sprintf("https://vine.co/api/users/profiles/vanity/%s", m[2])
		var jve jsonVanityEnvelope
		err := deserialize(profileURL, &jve)
		if err != nil {
			return "", err
		}
		return fmt.Sprint(jve.Data.UserID), nil
	} else {
		return m[2], nil
	}
}

// deserialize GETs a JSON API endpoint, unwraps the enveloping object and
// unmarshals the response.
func deserialize(url string, d interface{}) error {
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	if resp.StatusCode != 200 {
		return fmt.Errorf("HTTP %d: %s", resp.StatusCode, body)
	}
	err = json.Unmarshal(body, &d)
	if err != nil {
		return fmt.Errorf("unrecognized json %s", err)
	}
	return nil
}

func vineURLToUUID(url string) string {
	vineURLRE := regexp.MustCompile(`https://vine\.co/v/([a-zA-Z0-9]+)$`)
	m := vineURLRE.FindStringSubmatch(url)
	if len(m) == 0 {
		uuidMutex.Lock()
		defer uuidMutex.Unlock()
		return fmt.Sprintf("fallbackID%d", fallbackUUIDCount)
	}
	return string(m[1])
}

type jsonUser struct {
	Posts []string
}

type jsonVine struct {
	Description string
	Username    string
	UserIdStr   string
	VideoURL    string
	Created     string
}

// User API JSON structures
type jsonVanityEnvelope struct {
	Data jsonVanity
}
type jsonVanity struct {
	UserID int64
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/garyburd/redigo/redis"
	"log"
	"os"
	"strings"
)

func remove(c redis.Conn, domain string, ip string) {
	// get all keys
	var domains, _ = redis.Strings(c.Do("KEYS", "*"))

	// for each keys(frontend:*)
	for _, tmp_domain := range domains {

		protocol := "http://"
		new_domain := domain

		if strings.Contains(domain, "http://") {
			new_domain = strings.Replace(domain, "http://", "", -1)
		}

		if strings.Contains(domain, "https://") {
			new_domain = strings.Replace(domain, "https://", "", -1)
			protocol = "https://"
		}

		key := fmt.Sprintf("frontend:%s", new_domain)

		// check requested domain same with current domain or not
		if key == tmp_domain {

			if ip != "" { //if specific ip was specify
				// for each value in current domain
				ips, _ := redis.Strings(c.Do("LRANGE", key, 0, -1))
				for _, local_ip := range ips {
					value := fmt.Sprintf("%s%s", protocol, ip) // 'http://' + ip

					// if value match with requested ip, then delete
					if value == local_ip {
						c.Do("LREM", key, 0, value)
					}
				}
				//if no specific ip specify
			} else {
				// for each value in current domain
				ips, _ := redis.Strings(c.Do("LRANGE", key, 0, -1))
				for _, local_ip := range ips {
					c.Do("LREM", key, 0, local_ip)
				}
			}

			break // break early if domain found
		}

	}
}

func add(c redis.Conn, domain string, ip string) {
	protocol := "http://"
	new_domain := domain

	if strings.Contains(domain, "http://") {
		new_domain = strings.Replace(domain, "http://", "", -1)
	}

	if strings.Contains(domain, "https://") {
		new_domain = strings.Replace(domain, "https://", "", -1)
		protocol = "https://"
	}

	remove(c, domain, ip)

	key := fmt.Sprintf("frontend:%s", new_domain)

	title, _ := redis.Strings(c.Do("LRANGE", key, 0, 0))
	if len(title) != 0 {
		fmt.Println("adsfasfsdaf")
		if title[0] != "_"+new_domain {
			c.Do("RPUSH", key, "_"+new_domain)
		}
	} else {
		c.Do("RPUSH", key, "_"+new_domain)
	}

	new_ip := protocol + ip
	c.Do("RPUSH", key, new_ip)

	fmt.Println(" => Added!")
}

func show(c redis.Conn) {
	var domains, _ = redis.Strings(c.Do("KEYS", "*"))
	fmt.Println("|--------------------------------------------")
	fmt.Printf("| %-20s        %-20s\n", "Domain", "IP")
	fmt.Println("|--------------------------------------------")
	for _, domain := range domains {
		clean_domain := strings.Replace(domain, "frontend:", "", -1)

		ips, _ := redis.Strings(c.Do("LRANGE", domain, 0, -1))
		for _, ip := range ips {
			fmt.Printf("| %-20s   --   %-20s\n", clean_domain, ip)
		}

	}
	fmt.Println("|--------------------------------------------")
}

func main() {

	var (
		c, err = redis.Dial("tcp", "127.0.0.1:6379")
	)

	if err != nil {
		log.Fatal(err)
	}

	argc := len(os.Args)
	argv := os.Args

	if argc == 2 {
		if argv[1] == "show" {
			show(c)
		}
	} else if argc == 3 {
		if argv[1] == "remove" {
			remove(c, argv[2], "")
			show(c)
		}
	} else if argc == 4 {
		if argv[1] == "add" {
			add(c, argv[2], argv[3])
			show(c)
		} else if argv[1] == "remove" {
			remove(c, argv[2], argv[3])
			show(c)
		}
	} else {
		fmt.Println("Hipache cli ver. 1.0")
		fmt.Println("   Usage:\n")

		fmt.Println("   List all mappings")
		fmt.Println("   hic show\n")

		fmt.Println("   Add a url mapping")
		fmt.Println("   hic add http://mywebsite.com 192.168.1.1:3475\n")

		fmt.Println("   Remove 192.168.1.1:3475 from mywebsite.com")
		fmt.Println("   hic remove http://mywebsite.com 192.168.1.1:3475\n")

		fmt.Println("   Remove mywebsite.com")
		fmt.Println("   hic remove http://mywebsite.com\n")
	}

}
<commit_msg>hic can get ip from container<commit_after>package main

import (
	"os"
	"log"
	"fmt"
	"errors"
	"strconv"
	"strings"
	"encoding/json"
	"github.com/samalba/dockerclient"
	"github.com/garyburd/redigo/redis"
	"github.com/olekukonko/tablewriter"
)

var (
	CONFIG_FILENAME = ".hic"
)

var (
	ContainerNotFound = errors.New("Container not found.")
	PortNotFound      = errors.New("Cannot find port.")
	IpNotFound        = errors.New("Cannot find Ip.")
)

type MappingType struct {
	url  string
	name string
	port int
}

type RedisType struct {
	url   string
	ip    string
	port  int
	key   string
	value string
}

func getContainerIpByName(docker *dockerclient.DockerClient, container_name string) (string, error) {
	containers, err := docker.ListContainers(false)
	if err != nil {
		log.Fatal(err)
	}

	container_id := "0"
	for _, c := range containers {
		names := c.Names
		for _, name := range names {
			if name == "/"+container_name {
				container_id = c.Id
				break
			}
		}
		if container_id != "0" {
			break
		}
	}

	if container_id == "0" {
		return "", ContainerNotFound
	}

	info, err := docker.InspectContainer(container_id)
	if err != nil {
		return "", err
	}

	container_ip := info.NetworkSettings.IpAddress

	return container_ip, nil
}

func getContainerIpByPort(docker *dockerclient.DockerClient, container_port int) (string, error) {
	containers, err := docker.ListContainers(false)
	if err != nil {
		log.Fatal(err)
	}

	container_id := "0"
	for _, c := range containers {
		ports := c.Ports
		for _, p := range ports {
			if p.PrivatePort == container_port {
				container_id = c.Id
				break
			}
		}
		if container_id != "0" {
			break
		}
	}

	if container_id == "0" {
		return "", ContainerNotFound
	}

	info, err := docker.InspectContainer(container_id)
	if err != nil {
		return "", err
	}

	container_ip := info.NetworkSettings.IpAddress

	return container_ip, nil
}

func getPortFromMixIp(mix_ip string) (int, error) {
	ip_components := strings.Split(mix_ip, ":")
	port := 0
	if len(ip_components) == 3 {
		port_s := ip_components[2]
		port_tmp, err := strconv.Atoi(port_s)
		if err != nil {
			log.Fatal(err)
		}
		port = port_tmp
	} else {
		return 0, PortNotFound
	}
	return port, nil
}

func getUrlFromMixKey(mix_ip string) (string, error) {
	url_components := strings.Split(mix_ip, "frontend:")
	url := ""
	if len(url_components) == 2 {
		url = url_components[1]
	} else {
		return "", IpNotFound
	}
	return url, nil
}

func getIpFromMixIp(mix_ip string) (string, error) {
	ip_components := strings.Split(mix_ip, ":")
	ip := ""
	if len(ip_components) == 3 {
		ip = ip_components[1]
		ip = strings.Replace(ip, "//", "", -1)
	} else {
		return "", IpNotFound
	}
	return ip, nil
}

func getRedisDataAsList(c redis.Conn) ([]RedisType, error) {
	var list []RedisType

	data, err := redis.Strings(c.Do("KEYS", "*"))
	if err != nil {
		log.Fatal(err)
	}

	for _, item := range data {
		sub_data, _ := redis.Strings(c.Do("LRANGE", item, 0, -1))
		url, _ := getUrlFromMixKey(item)
		for _, sub_item := range sub_data {
			var i RedisType

			i.url = url

			ip, _ := getIpFromMixIp(sub_item)
			i.ip = ip

			port, _ := getPortFromMixIp(sub_item)
			i.port = port

			i.key = item
			i.value = sub_item

			list = append(list, i)
		}
	}

	return list, nil
}

func _Remove(c redis.Conn, query RedisType, remove_type int) {
	list, err := getRedisDataAsList(c)
	if err != nil {
		log.Fatal(err)
	}

	for _, item := range list {
		if remove_type == 1 {
			if item.ip == query.ip && item.url == query.url && item.port == query.port {
				key := fmt.Sprintf("frontend:%s", item.url)

				protocol := "http://"
				if item.port == 443 {
					protocol = "https://"
				}

				value := protocol + item.ip + ":" + strconv.Itoa(item.port)

				c.Do("LREM", key, 0, value)
			}
		} else if remove_type == 2 {
			if item.ip == query.ip && item.url == query.url {
				key := fmt.Sprintf("frontend:%s", item.url)

				protocol := "http://"
				if item.port == 443 {
					protocol = "https://"
				}

				value := protocol + item.ip + ":" + strconv.Itoa(item.port)

				c.Do("LREM", key, 0, value)
			}
		} else if remove_type == 3 {
			if item.url == query.url {
				key := fmt.Sprintf("frontend:%s", item.url)

				protocol := "http://"
				if item.port == 443 {
					protocol = "https://"
				}

				value := protocol + item.ip + ":" + strconv.Itoa(item.port)
				c.Do("LREM", key, 0, value)
			}
		}
	}

	list, err = getRedisDataAsList(c)
	if err != nil {
		log.Fatal(err)
	}

	count := 0
	for _, item := range list {
		if item.url == query.url {
			count = count + 1
		}
	}

	if count == 1 {
		key := fmt.Sprintf("frontend:%s", query.url)
		items, _ := redis.Strings(c.Do("LRANGE", key, 0, -1))
		for _, item := range items {
			c.Do("LREM", key, 0, item)
		}
	}
}

func _Add(c redis.Conn, query RedisType) {
	_Remove(c, query, 1)

	key := fmt.Sprintf("frontend:%s", query.url)
	title, _ := redis.Strings(c.Do("LRANGE", key, 0, 0))
	if len(title) != 0 {
		if title[0] != "_"+query.url {
			c.Do("RPUSH", key, "_"+query.url)
		}
	} else {
		c.Do("RPUSH", key, "_"+query.url)
	}

	protocol := "http://"
	if query.port == 443 {
		protocol = "https://"
	}

	new_ip := protocol + query.ip + ":" + strconv.Itoa(query.port)
	c.Do("RPUSH", key, new_ip)
}

func Clear(c redis.Conn) {
	data, err := getRedisDataAsList(c)
	if err != nil {
		log.Fatal(err)
	}

	for _, item := range data {
		c.Do("LREM", item.key, 0, item.value)
	}
}

// c, <url>, <container name>, <private port>
func Remove(docker *dockerclient.DockerClient, c redis.Conn, url string, ip string, port int) {
	new_url := url
	if strings.HasPrefix(url, "http://") {
		new_url = strings.Replace(url, "http://", "", -1)
	}

	if strings.HasPrefix(url, "https://") {
		new_url = strings.Replace(url, "https://", "", -1)
	}

	var query RedisType
	query.port = port
	query.ip = ip
	query.url = new_url

	if ip != "" && port != 0 {
		_Remove(c, query, 1)
	} else if ip != "" && port == 0 {
		_Remove(c, query, 2)
	} else if ip == "" && port == 0 {
		_Remove(c, query, 3)
	}

	fmt.Println(" => Removed!")
}

// c, <url>, <container name>, <private port>
func Add(docker *dockerclient.DockerClient, c redis.Conn, container_name string, url string, port int) {
	new_url := url
	if strings.HasPrefix(url, "http://") {
		new_url = strings.Replace(url, "http://", "", -1)
	}

	if strings.HasPrefix(url, "https://") {
		new_url = strings.Replace(url, "https://", "", -1)
	}

	ip, err := getContainerIpByName(docker, container_name)
	if err != nil {
		ip = container_name // user pass ip value in container_name parameter
	}

	var query RedisType
	query.port = port
	query.ip = ip
	query.url = new_url

	_Add(c, query)

	fmt.Println(" => Added!")
}

func Show(c redis.Conn) {
	data, err := getRedisDataAsList(c)
	if err != nil {
		log.Fatal(err)
	}

	rows := [][]string{}
	for _, item := range data {
		i := []string{}
		i = append(i,item.url)

		if item.ip != "" {
			i = append(i,item.ip)
		}else{
			i = append(i,"IDENTIFIER(" + item.value + ")")
		}

		if item.port != 0 {
			i = append(i,strconv.Itoa(item.port))
		}else{
			i = append(i, "-")
		}

		rows = append(rows, i)
	}

	table := tablewriter.NewWriter(os.Stdout)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeader([]string{"Url", "Ip", "Port"})

	for _, v := range rows {
		table.Append(v)
	}
	table.SetColumnSeparator(" ")
	table.Render()
	// var domains, _ = redis.Strings(c.Do("KEYS", "*"))
	// fmt.Println("|--------------------------------------------")
	// fmt.Printf("| %-20s        %-20s        %-20s\n", "Domain", "IP", "PORT")
	// fmt.Println("|--------------------------------------------")
	// for _, domain := range domains {

		// ips, _ := redis.Strings(c.Do("LRANGE", domain, 0, -1))
		// for _, ip := range ips {

			// fmt.Printf("| %-20s   --   %-20s   --   %-20s\n", domain, ip, "")
		// }

	// }
	// fmt.Println("|--------------------------------------------")
}

func Help() {
	fmt.Println("Hic (Hipache cli) ver. 2.0\n")
	fmt.Println("   Usage:\n")

	fmt.Println("   = List")
	fmt.Println("   ==============================")
	fmt.Println("   List all mappings.")
	fmt.Println("         hic\n")

	fmt.Println("   + Add")
	fmt.Println("   ==============================")
	fmt.Println("   No need to insert http or https for url...")
	fmt.Println("   ..., the protocol is detected by port number. If port...")
	fmt.Println("   ... number not given, default is 80.\n")

	fmt.Println("   Add a mapping by container name")
	fmt.Println("         hic add <container name> <url> <private port>")
	fmt.Println("    e.g. hic add blog_web_1 mywebsite.com 80\n")

	fmt.Println("   Add a mapping by ip")
	fmt.Println("         hic add <ip> <url> <private port>")
	fmt.Println("    e.g. hic add 192.168.1.6 mywebsite.com 80\n")

	fmt.Println("   - Remove")
	fmt.Println("   ==============================")
	fmt.Println("   Remove url(s).")
	fmt.Println("         hic rm <url>")
	fmt.Println("    e.g. hic rm mywebsite.com\n")

	fmt.Println("   Remove url(s) by ip. This will result...")
	fmt.Println("   ...removing all ports numbers (80, 443, etc.).")
	fmt.Println("         hic rm <url> <ip>")
	fmt.Println("    e.g. hic rm mywebsite.com 192.168.1.6\n")

	fmt.Println("   Remove an url mapping by container name and port.")
	fmt.Println("         hic rm <url> <ip> <private port>")
	fmt.Println("    e.g. hic rm mywebsite.com 192.168.1.6 80\n")

	fmt.Println("   <> Synchronization")
	fmt.Println("   ==============================")
	fmt.Println("   Sync all Ips between containers and Redis in Haraka.")
	fmt.Println("         hic sync\n")
}
func Sync(c redis.Conn) {

}
func main() {
	mapD := map[string]int{"apple": 5, "lettuce": 7}
	mapB, _ := json.Marshal(mapD)
	fmt.Println(string(mapB))

	docker, err := dockerclient.NewDockerClient("unix:///var/run/docker.sock", nil)
	if err != nil {
		log.Fatal("Cannot connect to Docker API.")
	}

	hipache_ip, err := getContainerIpByPort(docker, 6379)
	if err != nil {
		log.Fatal(err)
	}

	// Yeah, let's connect to Hipache server!
	c, err := redis.Dial("tcp", hipache_ip+":6379")
	if err != nil {
		log.Fatal(err)
	}

	argc := len(os.Args)
	argv := os.Args

	if argc == 1 {
		Show(c)
	} else if argc == 2 {
		if argv[1] == "sync" {
			Sync(c)
			Show(c)
		} else if argv[1] == "clear" {
			Clear(c)
			Show(c)
		} else {
			Help()
		}
	} else if argc == 3 {
		if argv[1] == "rm" {
			// remove(docker, c, <url>, "", 0)
			Remove(docker, c, argv[2], "", 0)
			Show(c)
		} else {
			Help()
		}
	} else if argc == 4 {
		if argv[1] == "add" {
			// add(docker, c, <container name>, <url>, "")
			Add(docker, c, argv[2], argv[3], 80)
			Show(c)
		} else if argv[1] == "rm" {
			// remove(docker, c, <url>, <ip>, 0)
			Remove(docker, c, argv[2], argv[3], 0)
			Show(c)
		} else {
			Help()
		}
	} else if argc == 5 {
		if argv[1] == "add" {
			port, err := strconv.Atoi(argv[4])
			if err != nil {
				log.Fatal(err)
			}
			// add(docker, c, <container name>, <url>, <private port>)
			Add(docker, c, argv[2], argv[3], port)
			Show(c)
		} else if argv[1] == "rm" {
			port, err := strconv.Atoi(argv[4])
			if err != nil {
				log.Fatal(err)
			}
			// remove(docker, c, <url>, <ip>, <private port>)
			Remove(docker, c, argv[2], argv[3], port)
			Show(c)
		} else {
			Help()
		}
	} else {
		Help()
	}

}
<|endoftext|>


<commit_before>package hyb

import (
	"container/heap"
	"encoding/gob"
	"fmt"
	"io"
	"math"
	"sort"
	"strings"
	"unicode/utf8"
	"unsafe"

	"github.com/robskie/bp128"
)

type mergeElem struct {
	ptr *[]iposting

	idx   int
	value uint32
}

// mergeHeap is a minimum heap of mergeElem.
// This is used to merge k arrays of ipostings.
type mergeHeap []mergeElem

func (h mergeHeap) Len() int           { return len(h) }
func (h mergeHeap) Less(i, j int) bool { return h[i].value < h[j].value }
func (h mergeHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

func (h *mergeHeap) Push(x interface{}) {
	*h = append(*h, x.(mergeElem))
}

func (h *mergeHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

type iposting struct {
	id   uint32
	word uint32
	rank uint32
}

// Index represents a group of searchable documents.
type Index struct {
	blocks []*pblock
	words  []string

	// freqword maps the word
	// frequency (index) to word
	// id (value).
	freqword []uint32

	// charfreq[x][y] returns the
	// max frequency of a character (x)
	// given its word position (y).
	charfreq [][]uint32

	size int
}

// NewIndex returns an empty index.
// Call Index.Read to populate it.
func NewIndex() *Index {
	return &Index{}
}

// Write serializes the index.
func (idx *Index) Write(w io.Writer) error {
	enc := gob.NewEncoder(w)

	err := checkErr(
		enc.Encode(idx.blocks),
		enc.Encode(idx.words),
		enc.Encode(idx.freqword),
		enc.Encode(idx.charfreq),
		enc.Encode(idx.size),
	)

	if err != nil {
		err = fmt.Errorf("hyb: write failed (%v)", err)
	}

	return err
}

// Read deserializes the index.
func (idx *Index) Read(r io.Reader) error {
	dec := gob.NewDecoder(r)

	err := checkErr(
		dec.Decode(&idx.blocks),
		dec.Decode(&idx.words),
		dec.Decode(&idx.freqword),
		dec.Decode(&idx.charfreq),
		dec.Decode(&idx.size),
	)

	if err != nil {
		err = fmt.Errorf("hyb: read failed (%v)", err)
	}

	return err
}

// Size returns the size of the index in bytes.
func (idx *Index) Size() int {
	return idx.size
}

// Search performs a search on the index given a query.
// If this search is a continuation of a previous search,
// prev should point to the previous result. This speeds
// up the search because it only needs to consider the
// documents included in the previous result.
func (idx *Index) Search(query []string, prev *Result) {
	// Check if the current query is a
	// continuation of the previous query
	cont, cquery := continuation(prev.query, query)

	// If the previous query returns no
	// results, no need to search again
	if cont && len(prev.results) == 0 {
		return
	} else if !cont {
		prev.clear()
		prev.words = idx.words
	}

	pquery := ""
	if cont {
		pquery = prev.query[len(prev.query)-1]
	}

	for _, q := range cquery {
		if len(pquery) > 0 && strings.HasPrefix(q, pquery) {
			wrange := getWordRange(q, idx.words, int(prev.wrange[0]))

			var start, end uint32
			if wrange != nil {
				start = wrange[0] - prev.wrange[0]
				end = start + (wrange[1] - wrange[0] + 1)
			}
			prev.wrange = wrange
			prev.completions = prev.completions[start:end]

			// If previous query is a prefix of the current
			// query, just filter IDs not in word range.
			prev.results = filter(prev.results, wrange)
		} else {
			idx.search(q, prev)
		}

		if len(prev.results) == 0 {
			break
		}

		pquery = q
	}

	prev.query = query
}

func (idx *Index) search(query string, prev *Result) {
	// Get blocks that contain the query
	blocks := []*pblock{}
	for _, b := range idx.blocks {
		// The first condition in the parenthesis finds the first
		// block that contains the query. The or'd condition finds
		// the succeeding blocks.
		if (query >= b.wboundary[0] && query <= b.wboundary[1]) ||
			strings.HasPrefix(b.wboundary[0], query) {

			blocks = append(blocks, b)
		}
	}

	if len(blocks) == 0 {
		prev.clear()
		return
	}

	wrange := getWordRange(query, idx.words, blocks[0].boundary[0])
	prev.wrange = wrange
	if wrange == nil {
		prev.clear()
		return
	}

	// Extend completions buffer if necessary
	rlen := int(wrange[1] - wrange[0] + 1)
	extension := rlen - len(prev.compbuf)
	if extension > 0 {
		prev.compbuf = append(prev.compbuf, make([]completion, extension)...)
	}

	// Create completions
	wid := wrange[0]
	comps := prev.compbuf[:rlen]
	for i := range comps {
		comps[i] = completion{wid, 0}
		wid++
	}

	// Estimate number of results
	cout := 0
	if len(prev.results) == 0 {
		cout = calcLen(query, idx.charfreq)
	}

	// Intersect postings to blocks
	var posts []iposting
	postings := make([][]iposting, 0, len(blocks))
	for _, b := range blocks {
		posts, comps = intersect(
			prev.results,
			comps,
			cout,
			b,
			wrange,
			idx.freqword,
		)

		if len(posts) > 0 {
			postings = append(postings, posts)
		}
	}
	prev.completions = comps

	// Merge postings
	merge(&prev.results, postings)
}

func merge(results *[]iposting, posts [][]iposting) {
	// If there is only one array,
	// just copy to results.
	if len(posts) == 1 {
		*results = posts[0]
		return
	}

	// Initialize heap
	h := &mergeHeap{}
	for i, p := range posts {
		h.Push(mergeElem{&posts[i], 0, p[0].id})
	}

	// Perform k-way merge
	heap.Init(h)
	out := (*results)[:0]
	for h.Len() > 0 {
		e := heap.Pop(h).(mergeElem)

		slice := *e.ptr
		p := slice[e.idx]
		out = append(out, p)

		e.idx++
		if e.idx < len(slice) {
			e.value = slice[e.idx].id
			heap.Push(h, e)
		}
	}

	*results = out
}

// filter removes IDs with
// words not in the word range.
func filter(posts []iposting, wrange *[2]uint32) []iposting {
	if wrange == nil {
		return nil
	}

	out := posts[:0]
	for i := range posts {
		wid := posts[i].word
		if wid >= wrange[0] && wid <= wrange[1] {
			out = append(out, posts[i])
		}
	}

	return out
}

func intersect(
	results []iposting,
	comps []completion,
	cout int,
	block *pblock,
	wrange *[2]uint32,
	freqword []uint32) ([]iposting, []completion) {

	const offset = 4
	const chunkSize = 2048
	buffer := make([]uint32, (chunkSize+offset)*3)

	ids := align(buffer[0:])
	words := align(ids[chunkSize:])
	ranks := align(words[chunkSize:])

	i, j := 0, 0
	out := make([]iposting, 0, cout)
	for _, p := range block.posts {
		if len(results) > 0 {
			if i >= len(results) {
				break
			} else if results[i].id > p.iboundary {
				continue
			}
		}

		bp128.Unpack(p.ids, &ids)
		bp128.Unpack(p.words, &words)
		bp128.Unpack(p.ranks, &ranks)

		var pid, pwid uint32 = math.MaxUint32, math.MaxUint32
		if len(results) > 0 {
			for j = 0; i < len(results) && j < len(ids); {
				jid := ids[j]
				rid := results[i].id

				if rid < jid {
					i++
				} else if rid > jid {
					j++
				} else {
					wid := freqword[words[j]]
					if wid >= wrange[0] && wid <= wrange[1] {
						ip := iposting{rid, wid, ranks[j]}
						out = append(out, ip)

						if pid != rid || pwid != wid {
							comps[wid-wrange[0]].hits++
						}

						pid = rid
						pwid = wid
					}

					j++
				}
			}
		} else {
			for j, wf := range words {
				wid := freqword[wf]
				if wid >= wrange[0] && wid <= wrange[1] {
					id := ids[j]
					ip := iposting{id, wid, ranks[j]}
					out = append(out, ip)

					if pid != id || pwid != wid {
						comps[wid-wrange[0]].hits++
					}

					pid = id
					pwid = wid
				}
			}
		}
	}

	return out, comps
}

// continuation returns true if the current
// query is a continuation of the previous query.
func continuation(prev []string, curr []string) (bool, []string) {
	if len(prev) > len(curr) {
		return false, curr
	} else if len(prev) == 0 {
		return false, curr
	}

	count := 0
	query := curr[len(prev):]
	for i := range prev {
		if strings.HasPrefix(curr[i], prev[i]) {
			if curr[i] != prev[i] {
				query = curr[i:]
			}
			count++
		} else {
			break
		}
	}

	// The current query is only a continuation
	// of the previous query if all the words in
	// the previous query is also in the current
	// query or prefix of the current query.
	if count == len(prev) {
		return true, query
	}

	return false, curr
}

func getWordRange(query string, words []string, offset int) *[2]uint32 {
	// Get the first word that is a prefix of the query
	words = words[offset:]
	rstart := sort.SearchStrings(words, query)

	// Return if no prefix found
	if rstart == len(words) || !strings.HasPrefix(words[rstart], query) {
		return nil
	}

	// Get the last prefix word
	query += string(utf8.MaxRune)
	rend := rstart + sort.SearchStrings(words[rstart:], query) - 1

	return &[2]uint32{uint32(rstart + offset), uint32(rend + offset)}
}

// calcLen estimates the number of words that matches the query.
func calcLen(query string, charfreq [][]uint32) int {
	cout := math.MaxUint32
	cmax := len(charfreq[0])
	for i := 0; i < len(query) && i < cmax; i++ {
		cf := int(charfreq[query[i]][i])
		if cout > cf {
			cout = cf
		}
	}

	return cout
}

// align returns a 16-byte aligned subarray
// of the given array. Returns nil if it cannot
// find an aligned subarray.
func align(a []uint32) []uint32 {
	for i := range a {
		addr := unsafe.Pointer(&a[i])
		if uintptr(addr)&15 == 0 {
			return a[i:]
		}
	}

	return nil
}

func checkErr(err ...error) error {
	for _, e := range err {
		if e != nil {
			return e
		}
	}

	return nil
}
<commit_msg>Added gob encoding and decoding methods<commit_after>package hyb

import (
	"bytes"
	"container/heap"
	"encoding/gob"
	"fmt"
	"io"
	"math"
	"sort"
	"strings"
	"unicode/utf8"
	"unsafe"

	"github.com/robskie/bp128"
)

type mergeElem struct {
	ptr *[]iposting

	idx   int
	value uint32
}

// mergeHeap is a minimum heap of mergeElem.
// This is used to merge k arrays of ipostings.
type mergeHeap []mergeElem

func (h mergeHeap) Len() int           { return len(h) }
func (h mergeHeap) Less(i, j int) bool { return h[i].value < h[j].value }
func (h mergeHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

func (h *mergeHeap) Push(x interface{}) {
	*h = append(*h, x.(mergeElem))
}

func (h *mergeHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

type iposting struct {
	id   uint32
	word uint32
	rank uint32
}

// Index represents a group of searchable documents.
type Index struct {
	blocks []*pblock
	words  []string

	// freqword maps the word
	// frequency (index) to word
	// id (value).
	freqword []uint32

	// charfreq[x][y] returns the
	// max frequency of a character (x)
	// given its word position (y).
	charfreq [][]uint32

	size int
}

// NewIndex returns an empty index.
// Call Index.Read to populate it.
func NewIndex() *Index {
	return &Index{}
}

// Write serializes the index.
func (idx *Index) Write(w io.Writer) error {
	return gob.NewEncoder(w).Encode(idx)
}

// Read deserializes the index.
func (idx *Index) Read(r io.Reader) error {
	return gob.NewDecoder(r).Decode(idx)
}

// GobEncode transforms an index into gob streams.
func (idx *Index) GobEncode() ([]byte, error) {
	buf := &bytes.Buffer{}
	enc := gob.NewEncoder(buf)

	err := checkErr(
		enc.Encode(idx.blocks),
		enc.Encode(idx.words),
		enc.Encode(idx.freqword),
		enc.Encode(idx.charfreq),
		enc.Encode(idx.size),
	)

	if err != nil {
		err = fmt.Errorf("hyb: encode failed (%v)", err)
	}

	return buf.Bytes(), err
}

// GobDecode decodes an index from gob streams.
func (idx *Index) GobDecode(data []byte) error {
	buf := bytes.NewReader(data)
	dec := gob.NewDecoder(buf)

	err := checkErr(
		dec.Decode(&idx.blocks),
		dec.Decode(&idx.words),
		dec.Decode(&idx.freqword),
		dec.Decode(&idx.charfreq),
		dec.Decode(&idx.size),
	)

	if err != nil {
		err = fmt.Errorf("hyb: decode failed (%v)", err)
	}

	return err
}

// Size returns the size of the index in bytes.
func (idx *Index) Size() int {
	return idx.size
}

// Search performs a search on the index given a query.
// If this search is a continuation of a previous search,
// prev should point to the previous result. This speeds
// up the search because it only needs to consider the
// documents included in the previous result.
func (idx *Index) Search(query []string, prev *Result) {
	// Check if the current query is a
	// continuation of the previous query
	cont, cquery := continuation(prev.query, query)

	// If the previous query returns no
	// results, no need to search again
	if cont && len(prev.results) == 0 {
		return
	} else if !cont {
		prev.clear()
		prev.words = idx.words
	}

	pquery := ""
	if cont {
		pquery = prev.query[len(prev.query)-1]
	}

	for _, q := range cquery {
		if len(pquery) > 0 && strings.HasPrefix(q, pquery) {
			wrange := getWordRange(q, idx.words, int(prev.wrange[0]))

			var start, end uint32
			if wrange != nil {
				start = wrange[0] - prev.wrange[0]
				end = start + (wrange[1] - wrange[0] + 1)
			}
			prev.wrange = wrange
			prev.completions = prev.completions[start:end]

			// If previous query is a prefix of the current
			// query, just filter IDs not in word range.
			prev.results = filter(prev.results, wrange)
		} else {
			idx.search(q, prev)
		}

		if len(prev.results) == 0 {
			break
		}

		pquery = q
	}

	prev.query = query
}

func (idx *Index) search(query string, prev *Result) {
	// Get blocks that contain the query
	blocks := []*pblock{}
	for _, b := range idx.blocks {
		// The first condition in the parenthesis finds the first
		// block that contains the query. The or'd condition finds
		// the succeeding blocks.
		if (query >= b.wboundary[0] && query <= b.wboundary[1]) ||
			strings.HasPrefix(b.wboundary[0], query) {

			blocks = append(blocks, b)
		}
	}

	if len(blocks) == 0 {
		prev.clear()
		return
	}

	wrange := getWordRange(query, idx.words, blocks[0].boundary[0])
	prev.wrange = wrange
	if wrange == nil {
		prev.clear()
		return
	}

	// Extend completions buffer if necessary
	rlen := int(wrange[1] - wrange[0] + 1)
	extension := rlen - len(prev.compbuf)
	if extension > 0 {
		prev.compbuf = append(prev.compbuf, make([]completion, extension)...)
	}

	// Create completions
	wid := wrange[0]
	comps := prev.compbuf[:rlen]
	for i := range comps {
		comps[i] = completion{wid, 0}
		wid++
	}

	// Estimate number of results
	cout := 0
	if len(prev.results) == 0 {
		cout = calcLen(query, idx.charfreq)
	}

	// Intersect postings to blocks
	var posts []iposting
	postings := make([][]iposting, 0, len(blocks))
	for _, b := range blocks {
		posts, comps = intersect(
			prev.results,
			comps,
			cout,
			b,
			wrange,
			idx.freqword,
		)

		if len(posts) > 0 {
			postings = append(postings, posts)
		}
	}
	prev.completions = comps

	// Merge postings
	merge(&prev.results, postings)
}

func merge(results *[]iposting, posts [][]iposting) {
	// If there is only one array,
	// just copy to results.
	if len(posts) == 1 {
		*results = posts[0]
		return
	}

	// Initialize heap
	h := &mergeHeap{}
	for i, p := range posts {
		h.Push(mergeElem{&posts[i], 0, p[0].id})
	}

	// Perform k-way merge
	heap.Init(h)
	out := (*results)[:0]
	for h.Len() > 0 {
		e := heap.Pop(h).(mergeElem)

		slice := *e.ptr
		p := slice[e.idx]
		out = append(out, p)

		e.idx++
		if e.idx < len(slice) {
			e.value = slice[e.idx].id
			heap.Push(h, e)
		}
	}

	*results = out
}

// filter removes IDs with
// words not in the word range.
func filter(posts []iposting, wrange *[2]uint32) []iposting {
	if wrange == nil {
		return nil
	}

	out := posts[:0]
	for i := range posts {
		wid := posts[i].word
		if wid >= wrange[0] && wid <= wrange[1] {
			out = append(out, posts[i])
		}
	}

	return out
}

func intersect(
	results []iposting,
	comps []completion,
	cout int,
	block *pblock,
	wrange *[2]uint32,
	freqword []uint32) ([]iposting, []completion) {

	const offset = 4
	const chunkSize = 2048
	buffer := make([]uint32, (chunkSize+offset)*3)

	ids := align(buffer[0:])
	words := align(ids[chunkSize:])
	ranks := align(words[chunkSize:])

	i, j := 0, 0
	out := make([]iposting, 0, cout)
	for _, p := range block.posts {
		if len(results) > 0 {
			if i >= len(results) {
				break
			} else if results[i].id > p.iboundary {
				continue
			}
		}

		bp128.Unpack(p.ids, &ids)
		bp128.Unpack(p.words, &words)
		bp128.Unpack(p.ranks, &ranks)

		var pid, pwid uint32 = math.MaxUint32, math.MaxUint32
		if len(results) > 0 {
			for j = 0; i < len(results) && j < len(ids); {
				jid := ids[j]
				rid := results[i].id

				if rid < jid {
					i++
				} else if rid > jid {
					j++
				} else {
					wid := freqword[words[j]]
					if wid >= wrange[0] && wid <= wrange[1] {
						ip := iposting{rid, wid, ranks[j]}
						out = append(out, ip)

						if pid != rid || pwid != wid {
							comps[wid-wrange[0]].hits++
						}

						pid = rid
						pwid = wid
					}

					j++
				}
			}
		} else {
			for j, wf := range words {
				wid := freqword[wf]
				if wid >= wrange[0] && wid <= wrange[1] {
					id := ids[j]
					ip := iposting{id, wid, ranks[j]}
					out = append(out, ip)

					if pid != id || pwid != wid {
						comps[wid-wrange[0]].hits++
					}

					pid = id
					pwid = wid
				}
			}
		}
	}

	return out, comps
}

// continuation returns true if the current
// query is a continuation of the previous query.
func continuation(prev []string, curr []string) (bool, []string) {
	if len(prev) > len(curr) {
		return false, curr
	} else if len(prev) == 0 {
		return false, curr
	}

	count := 0
	query := curr[len(prev):]
	for i := range prev {
		if strings.HasPrefix(curr[i], prev[i]) {
			if curr[i] != prev[i] {
				query = curr[i:]
			}
			count++
		} else {
			break
		}
	}

	// The current query is only a continuation
	// of the previous query if all the words in
	// the previous query is also in the current
	// query or prefix of the current query.
	if count == len(prev) {
		return true, query
	}

	return false, curr
}

func getWordRange(query string, words []string, offset int) *[2]uint32 {
	// Get the first word that is a prefix of the query
	words = words[offset:]
	rstart := sort.SearchStrings(words, query)

	// Return if no prefix found
	if rstart == len(words) || !strings.HasPrefix(words[rstart], query) {
		return nil
	}

	// Get the last prefix word
	query += string(utf8.MaxRune)
	rend := rstart + sort.SearchStrings(words[rstart:], query) - 1

	return &[2]uint32{uint32(rstart + offset), uint32(rend + offset)}
}

// calcLen estimates the number of words that matches the query.
func calcLen(query string, charfreq [][]uint32) int {
	cout := math.MaxUint32
	cmax := len(charfreq[0])
	for i := 0; i < len(query) && i < cmax; i++ {
		cf := int(charfreq[query[i]][i])
		if cout > cf {
			cout = cf
		}
	}

	return cout
}

// align returns a 16-byte aligned subarray
// of the given array. Returns nil if it cannot
// find an aligned subarray.
func align(a []uint32) []uint32 {
	for i := range a {
		addr := unsafe.Pointer(&a[i])
		if uintptr(addr)&15 == 0 {
			return a[i:]
		}
	}

	return nil
}

func checkErr(err ...error) error {
	for _, e := range err {
		if e != nil {
			return e
		}
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Ion is a small web framework for people in a hurry.
Ion provides a fast trie based router (julienschmidt/httprouter),
a integrated middleware support (justinas/alice), automatic
context support (via gorilla/context) and some handful helpers.

A short example:

	package main

	import (
		"fmt"
		"github.com/estebarb/ion"
		"net/http"
	)

	func hello(w http.ResponseWriter, r *http.Request) {
		value := ion.URLArgs(r, "name")
		if value != "" {
			fmt.Fprintf(w, "Hello, %v!", value)
		} else {
			fmt.Fprint(w, "Hello world!")
		}
	}

	func main() {
		r := ion.NewRouter()
		r.GetFunc("/", hello)
		r.GetFunc("/:name", hello)
		http.ListenAndServe(":8080", r)
	}

At this point the framework is highly experimental, so please don't
use it in production for now...
*/
package ion

import (
	"github.com/estebarb/ion/context"
	"github.com/julienschmidt/httprouter"
	"github.com/justinas/alice"
	"html/template"
	"net/http"
	"encoding/json"
	"io/ioutil"
	"time"
	"log"
)

// This is the router used by Ion. It contains the high performance
// trie based httprouter, the middleware manager alice and adapter
// functions that make trivial to use the Go http.Handler
type Router struct {
	*httprouter.Router
	Middleware alice.Chain
}

/*
Ion adds the path arguments by httprouter to
context, so they can be retrieved using:

	// asuming a path like /:name
	// r is a *http.Request
	params := context.Get(r, ion.Urlargs)
	name := params.ByName("name")

But the preferred way is the following:

	name := ion.URLArgs(r, "name")

*/
const urlargs = "urlargs"

/*
Returns a new router, with no middleware.
*/
func NewRouter() *Router {
	return &Router{httprouter.New(), alice.New()}
}

/*
Returns a new router, configured with the middlewares
provided.
*/
func NewRouterDefaults(middleware ...alice.Constructor) *Router {
	r := &Router{
		httprouter.New(),
		alice.New(middleware...),
	}
	return r
}

/*
wrapHandler transforms a http.Handler handler to a httprouter.Handle
*/
func wrapHandler(h http.Handler) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		context.Set(r, urlargs, ps)
		h.ServeHTTP(w, r)
	}
}

// Registers a new request handler (http.Handler) for the given path and method.
// It also executes the current Middleware in the settings.
func (r *Router) MethodHandle(method, path string, handle http.Handler) {
	r.Handle(method, path, wrapHandler(r.Middleware.Then(handle)))
}

// Registers a new request handler (http.HandlerFunc) for the given path and method.
// It also executes the current Middleware in the settings.
func (r *Router) MethodHandleFunc(method, path string, handle http.HandlerFunc) {
	r.Handle(method, path, wrapHandler(r.Middleware.ThenFunc(handle)))
}

// Shortcut for router.MethodHandle("DELETE", path, handler)
func (r *Router) DELETE(path string, handler http.Handler) {
	r.MethodHandle("DELETE", path, handler)
}

// Shortcut for router.MethodHandle("GET", path, handler)
func (r *Router) GET(path string, handler http.Handler) {
	r.MethodHandle("GET", path, handler)
}

// Shortcut for router.MethodHandle("POST", path, handler)
func (r *Router) POST(path string, handler http.Handler) {
	r.MethodHandle("POST", path, handler)
}

// Shortcut for router.MethodHandle("PATCH", path, handler)
func (r *Router) PATCH(path string, handler http.Handler) {
	r.MethodHandle("PATCH", path, handler)
}

// Shortcut for router.MethodHandle("PUT", path, handler)
func (r *Router) PUT(path string, handler http.Handler) {
	r.MethodHandle("PUT", path, handler)
}

// Shortcut for router.MethodHandleFunc("DELETE", path, handler)
func (r *Router) DeleteFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("DELETE", path, handler)
}

// Shortcut for router.MethodHandleFunc("GET", path, handler)
func (r *Router) GetFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("GET", path, handler)
}

// Shortcut for router.MethodHandleFunc("POST", path, handler)
func (r *Router) PostFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("POST", path, handler)
}

// Shortcut for router.MethodHandleFunc("PATCH", path, handler)
func (r *Router) PatchFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("PATCH", path, handler)
}

// Shortcut for router.MethodHandleFunc("PUT", path, handler)
func (r *Router) PutFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("PUT", path, handler)
}

// Returns a handler that can render the given templates. The templates
// receives as parameters the context associated to the current
// request.
func RenderTemplate(t *template.Template) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		// Adds URL params to context
		args, _ := context.Get(r, urlargs)
		context.Set(r, urlargs, paramsToMap(args.(httprouter.Params)))
		// Adds data in context to template context
		ctx, _ := context.GetAll(r)
		t.Execute(w, ctx)
	}
}

// Converts the httprouter.Params array to a map, that can
// be consumed easily from templates.
func paramsToMap(p httprouter.Params) map[string]string {
	ret := make(map[string]string)
	for _, v := range p {
		ret[v.Key] = v.Value
	}
	return ret
}

// This interface works with RegisterREST to provide a shortcut
// to register an RESTful endpoint.

type RESTendpoint interface {
	LIST(w http.ResponseWriter, r *http.Request)
	POST(w http.ResponseWriter, r *http.Request)
	PUT(w http.ResponseWriter, r *http.Request)
	GET(w http.ResponseWriter, r *http.Request)
	DELETE(w http.ResponseWriter, r *http.Request)
}

// Register a RESTendpoint in a router.
// It will register the following routes:
// - GET  path		(list function)
// - POST path		(post function)
// - GET  path/:id	(get function)
// - PUT  path/:id	(put function)
// - DELETE  path/:id	(delete function)
// The path MUST include the trailing slash.
func (r *Router) RegisterREST(path string, handler RESTendpoint) {
	r.GetFunc(path+":id", handler.GET)
	r.PutFunc(path+":id", handler.PUT)
	r.DeleteFunc(path+":id", handler.DELETE)
	r.GetFunc(path, handler.LIST)
	r.PostFunc(path, handler.POST)
}

// Returns a named argument from the request URL
func URLArgs(r *http.Request, name string) string {
	val, _ := context.Get(r, urlargs)
	v2 := val.(httprouter.Params)
	return v2.ByName(name)
}

// Writes a JSON value
func MarshalJSON(w http.ResponseWriter, value interface{}) error{
	out, err := json.Marshal(value)
	if err != nil{
		return err
	}
	w.Write(out)
	return nil
}

// Returns the unmarshaled value from the request body
func UnmarshalJSON(r *http.Request, value interface{}) error{
	body, err := ioutil.ReadAll(r.Body)
	if err != nil{
		return err
	}
	return json.Unmarshal(body, value)
}

// Provides a logging middleware
func LoggingMiddleware(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		t1 := time.Now()
		next.ServeHTTP(w, r)
		t2 := time.Now()
		log.Printf("[%s] %q %v\n", r.Method, r.URL.String(), t2.Sub(t1))
	}

	return http.HandlerFunc(fn)
}

// Provides a recovery from panics in other handlers
func PanicMiddleware(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				log.Printf("panic: %+v", err)
				http.Error(w, http.StatusText(500), 500)
			}
		}()

		next.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}<commit_msg>Added a DoNothing handler<commit_after>/*
Ion is a small web framework for people in a hurry.
Ion provides a fast trie based router (julienschmidt/httprouter),
a integrated middleware support (justinas/alice), automatic
context support (via gorilla/context) and some handful helpers.

A short example:

	package main

	import (
		"fmt"
		"github.com/estebarb/ion"
		"net/http"
	)

	func hello(w http.ResponseWriter, r *http.Request) {
		value := ion.URLArgs(r, "name")
		if value != "" {
			fmt.Fprintf(w, "Hello, %v!", value)
		} else {
			fmt.Fprint(w, "Hello world!")
		}
	}

	func main() {
		r := ion.NewRouter()
		r.GetFunc("/", hello)
		r.GetFunc("/:name", hello)
		http.ListenAndServe(":8080", r)
	}

At this point the framework is highly experimental, so please don't
use it in production for now...
*/
package ion

import (
	"github.com/estebarb/ion/context"
	"github.com/julienschmidt/httprouter"
	"github.com/justinas/alice"
	"html/template"
	"net/http"
	"encoding/json"
	"io/ioutil"
	"time"
	"log"
)

// This is the router used by Ion. It contains the high performance
// trie based httprouter, the middleware manager alice and adapter
// functions that make trivial to use the Go http.Handler
type Router struct {
	*httprouter.Router
	Middleware alice.Chain
}

/*
Ion adds the path arguments by httprouter to
context, so they can be retrieved using:

	// asuming a path like /:name
	// r is a *http.Request
	params := context.Get(r, ion.Urlargs)
	name := params.ByName("name")

But the preferred way is the following:

	name := ion.URLArgs(r, "name")

*/
const urlargs = "urlargs"

/*
Returns a new router, with no middleware.
*/
func NewRouter() *Router {
	return &Router{httprouter.New(), alice.New()}
}

/*
Returns a new router, configured with the middlewares
provided.
*/
func NewRouterDefaults(middleware ...alice.Constructor) *Router {
	r := &Router{
		httprouter.New(),
		alice.New(middleware...),
	}
	return r
}

/*
wrapHandler transforms a http.Handler handler to a httprouter.Handle
*/
func wrapHandler(h http.Handler) httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
		context.Set(r, urlargs, ps)
		h.ServeHTTP(w, r)
	}
}

// Registers a new request handler (http.Handler) for the given path and method.
// It also executes the current Middleware in the settings.
func (r *Router) MethodHandle(method, path string, handle http.Handler) {
	r.Handle(method, path, wrapHandler(r.Middleware.Then(handle)))
}

// Registers a new request handler (http.HandlerFunc) for the given path and method.
// It also executes the current Middleware in the settings.
func (r *Router) MethodHandleFunc(method, path string, handle http.HandlerFunc) {
	r.Handle(method, path, wrapHandler(r.Middleware.ThenFunc(handle)))
}

// Shortcut for router.MethodHandle("DELETE", path, handler)
func (r *Router) DELETE(path string, handler http.Handler) {
	r.MethodHandle("DELETE", path, handler)
}

// Shortcut for router.MethodHandle("GET", path, handler)
func (r *Router) GET(path string, handler http.Handler) {
	r.MethodHandle("GET", path, handler)
}

// Shortcut for router.MethodHandle("POST", path, handler)
func (r *Router) POST(path string, handler http.Handler) {
	r.MethodHandle("POST", path, handler)
}

// Shortcut for router.MethodHandle("PATCH", path, handler)
func (r *Router) PATCH(path string, handler http.Handler) {
	r.MethodHandle("PATCH", path, handler)
}

// Shortcut for router.MethodHandle("PUT", path, handler)
func (r *Router) PUT(path string, handler http.Handler) {
	r.MethodHandle("PUT", path, handler)
}

// Shortcut for router.MethodHandleFunc("DELETE", path, handler)
func (r *Router) DeleteFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("DELETE", path, handler)
}

// Shortcut for router.MethodHandleFunc("GET", path, handler)
func (r *Router) GetFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("GET", path, handler)
}

// Shortcut for router.MethodHandleFunc("POST", path, handler)
func (r *Router) PostFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("POST", path, handler)
}

// Shortcut for router.MethodHandleFunc("PATCH", path, handler)
func (r *Router) PatchFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("PATCH", path, handler)
}

// Shortcut for router.MethodHandleFunc("PUT", path, handler)
func (r *Router) PutFunc(path string, handler http.HandlerFunc) {
	r.MethodHandleFunc("PUT", path, handler)
}

// Returns a handler that can render the given templates. The templates
// receives as parameters the context associated to the current
// request.
func RenderTemplate(t *template.Template) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		// Adds URL params to context
		args, _ := context.Get(r, urlargs)
		context.Set(r, urlargs, paramsToMap(args.(httprouter.Params)))
		// Adds data in context to template context
		ctx, _ := context.GetAll(r)
		t.Execute(w, ctx)
	}
}

// Converts the httprouter.Params array to a map, that can
// be consumed easily from templates.
func paramsToMap(p httprouter.Params) map[string]string {
	ret := make(map[string]string)
	for _, v := range p {
		ret[v.Key] = v.Value
	}
	return ret
}

// This interface works with RegisterREST to provide a shortcut
// to register an RESTful endpoint.

type RESTendpoint interface {
	LIST(w http.ResponseWriter, r *http.Request)
	POST(w http.ResponseWriter, r *http.Request)
	PUT(w http.ResponseWriter, r *http.Request)
	GET(w http.ResponseWriter, r *http.Request)
	DELETE(w http.ResponseWriter, r *http.Request)
}

// Register a RESTendpoint in a router.
// It will register the following routes:
// - GET  path		(list function)
// - POST path		(post function)
// - GET  path/:id	(get function)
// - PUT  path/:id	(put function)
// - DELETE  path/:id	(delete function)
// The path MUST include the trailing slash.
func (r *Router) RegisterREST(path string, handler RESTendpoint) {
	r.GetFunc(path+":id", handler.GET)
	r.PutFunc(path+":id", handler.PUT)
	r.DeleteFunc(path+":id", handler.DELETE)
	r.GetFunc(path, handler.LIST)
	r.PostFunc(path, handler.POST)
}

// Returns a named argument from the request URL
func URLArgs(r *http.Request, name string) string {
	val, _ := context.Get(r, urlargs)
	v2 := val.(httprouter.Params)
	return v2.ByName(name)
}

// Writes a JSON value
func MarshalJSON(w http.ResponseWriter, value interface{}) error{
	out, err := json.Marshal(value)
	if err != nil{
		return err
	}
	w.Write(out)
	return nil
}

// Returns the unmarshaled value from the request body
func UnmarshalJSON(r *http.Request, value interface{}) error{
	body, err := ioutil.ReadAll(r.Body)
	if err != nil{
		return err
	}
	return json.Unmarshal(body, value)
}

// Provides a logging middleware
func LoggingMiddleware(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		t1 := time.Now()
		next.ServeHTTP(w, r)
		t2 := time.Now()
		log.Printf("[%s] %q %v\n", r.Method, r.URL.String(), t2.Sub(t1))
	}

	return http.HandlerFunc(fn)
}

// Provides a recovery from panics in other handlers
func PanicMiddleware(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			if err := recover(); err != nil {
				log.Printf("panic: %+v", err)
				http.Error(w, http.StatusText(500), 500)
			}
		}()

		next.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

// This is the simpler possible handler: DoNothing do nothing.
// It can be used as a placeholder, or when we want to run
// the middleware for the collateral effects, but we don't want
// to do something specific.
// Also can be used with IonMVC.
func DoNothing(w http.ResponseWriter, r *http.Request) {
}<|endoftext|>