<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"syscall"
	"unsafe"
)

/*** defines ***/

const KILO_VERSION = "0.0.1"
const KILO_TAB_STOP = 8
const (
	ARROW_LEFT  = 1000 + iota
	ARROW_RIGHT = 1000 + iota
	ARROW_UP    = 1000 + iota
	ARROW_DOWN  = 1000 + iota
	DEL_KEY     = 1000 + iota
	HOME_KEY    = 1000 + iota
	END_KEY     = 1000 + iota
	PAGE_UP     = 1000 + iota
	PAGE_DOWN   = 1000 + iota
)

/*** data ***/

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type erow struct {
	size   int
	rsize  int
	chars  []byte
	render []byte
}

type editorConfig struct {
	cx          int
	cy          int
	rowoff      int
	coloff      int
	screenRows  int
	screenCols  int
	numRows     int
	rows        []erow
	origTermios *Termios
}

type WinSize struct {
	Row    uint16
	Col    uint16
	Xpixel uint16
	Ypixel uint16
}

var E editorConfig

/*** terminal ***/

func die(err error) {
	disableRawMode()
	io.WriteString(os.Stdout, "\x1b[2J")
	io.WriteString(os.Stdout, "\x1b[H")
	log.Fatal(err)
}

func TcSetAttr(fd uintptr, termios *Termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func enableRawMode() {
	E.origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *E.origTermios
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), E.origTermios); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}

func editorReadKey() int {
	var buffer [1]byte
	var cc int
	var err error
	for cc, err = os.Stdin.Read(buffer[:]); cc != 1; cc, err = os.Stdin.Read(buffer[:]) {
	}
	if err != nil {
		die(err)
	}
	if buffer[0] == '\x1b' {
		var seq [2]byte
		if cc, _ = os.Stdin.Read(seq[:]); cc != 2 {
			return '\x1b'
		}

		if seq[0] == '[' {
			if seq[1] >= '0' && seq[1] <= '9' {
				if cc, err = os.Stdin.Read(buffer[:]); cc != 1 {
					return '\x1b'
				}
				if buffer[0] == '~' {
					switch seq[1] {
					case '1':
						return HOME_KEY
					case '3':
						return DEL_KEY
					case '4':
						return END_KEY
					case '5':
						return PAGE_UP
					case '6':
						return PAGE_DOWN
					case '7':
						return HOME_KEY
					case '8':
						return END_KEY
					}
				}
				// XXX - what happens here?
			} else {
				switch seq[1] {
				case 'A':
					return ARROW_UP
				case 'B':
					return ARROW_DOWN
				case 'C':
					return ARROW_RIGHT
				case 'D':
					return ARROW_LEFT
				case 'H':
					return HOME_KEY
				case 'F':
					return END_KEY
				}
			}
		} else if seq[0] == '0' {
			switch seq[1] {
			case 'H':
				return HOME_KEY
			case 'F':
				return END_KEY
			}
		}

		return '\x1b'
	}
	return int(buffer[0])
}

func getCursorPosition(rows *int, cols *int) int {
	io.WriteString(os.Stdout, "\x1b[6n")
	var buffer [1]byte
	var buf []byte
	var cc int
	for cc, _ = os.Stdin.Read(buffer[:]); cc == 1; cc, _ = os.Stdin.Read(buffer[:]) {
		if buffer[0] == 'R' {
			break
		}
		buf = append(buf, buffer[0])
	}
	if string(buf[0:2]) != "\x1b[" {
		log.Printf("Failed to read rows;cols from tty\n")
		return -1
	}
	if n, e := fmt.Sscanf(string(buf[2:]), "%d;%d", rows, cols); n != 2 || e != nil {
		if e != nil {
			log.Printf("getCursorPosition: fmt.Sscanf() failed: %s\n", e)
		}
		if n != 2 {
			log.Printf("getCursorPosition: got %d items, wanted 2\n", n)
		}
		return -1
	}
	return 0
}

func getWindowSize(rows *int, cols *int) int {
	var w WinSize
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		os.Stdout.Fd(),
		syscall.TIOCGWINSZ,
		uintptr(unsafe.Pointer(&w)),
	)
	if err != 0 { // type syscall.Errno
		io.WriteString(os.Stdout, "\x1b[999C\x1b[999B")
		return getCursorPosition(rows, cols)
	} else {
		*rows = int(w.Row)
		*cols = int(w.Col)
		return 0
	}
	return -1
}

/*** row operations ***/

func editorUpdateRow(row *erow) {
	tabs := 0
	for _, c := range row.chars {
		if c == '\t' {
			tabs++
		}
	}
	row.render = make([]byte, row.size + tabs*(KILO_TAB_STOP - 1))

	idx := 0
	for _, c := range row.chars {
		if c == '\t' {
			row.render[idx] = ' '
			idx++
			for (idx%KILO_TAB_STOP) != 0 {
				row.render[idx] = ' '
				idx++
			}
		} else {
			row.render[idx] = c
			idx++
		}
	}
	row.rsize = idx
}

func editorAppendRow(s []byte) {
	var r erow
	r.chars = s
	r.size = len(s)
	E.rows = append(E.rows, r)
	editorUpdateRow(&E.rows[E.numRows])
	E.numRows++
}

/*** file I/O ***/

func editorOpen(filename string) {
	fd, err := os.Open(filename)
	if err != nil {
		die(err)
	}
	defer fd.Close()
	fp := bufio.NewReader(fd)

	for line, err := fp.ReadBytes('\n'); err == nil; line, err = fp.ReadBytes('\n') {
		// Trim trailing newlines and carriage returns
		for c := line[len(line) - 1]; len(line) > 0 && (c == '\n' || c == '\r'); {
			line = line[:len(line)-1]
			if len(line) > 0 {
				c = line[len(line) - 1]
			}
		}
		editorAppendRow(line)
	}

	if err != nil && err != io.EOF {
		die(err)
	}
}

/*** input ***/

func editorMoveCursor(key int) {
	switch key {
	case ARROW_LEFT:
		if E.cx != 0 {
			E.cx--
		} else if E.cy > 0 {
			E.cy--
			E.cx = E.rows[E.cy].rsize
		}
	case ARROW_RIGHT:
		if E.cy < E.numRows {
			if E.cx < E.rows[E.cy].rsize {
				E.cx++
			} else if E.cx == E.rows[E.cy].rsize {
				E.cy++
				E.cx = 0
			}
		}
	case ARROW_UP:
		if E.cy != 0 {
			E.cy--
		}
	case ARROW_DOWN:
		if E.cy < E.numRows {
			E.cy++
		}
	}

	rowlen := 0
	if E.cy < E.numRows {
		rowlen = E.rows[E.cy].rsize
	}
	if E.cx > rowlen {
		E.cx = rowlen
	}
}

func editorProcessKeypress() {
	c := editorReadKey()
	switch c {
	case ('q' & 0x1f):
		io.WriteString(os.Stdout, "\x1b[2J")
		io.WriteString(os.Stdout, "\x1b[H")
		disableRawMode()
		os.Exit(0)
	case HOME_KEY:
		E.cx = 0
	case END_KEY:
		E.cx = E.screenCols - 1
	case PAGE_UP, PAGE_DOWN:
		dir := ARROW_DOWN
		if c == PAGE_UP {
			dir = ARROW_UP
		}
		for times := E.screenRows; times > 0; times-- {
			editorMoveCursor(dir)
		}
	case ARROW_UP, ARROW_DOWN, ARROW_LEFT, ARROW_RIGHT:
		editorMoveCursor(c)
	}
}

/*** append buffer ***/

type abuf struct {
	buf []byte
}

func (p abuf) String() string {
	return string(p.buf)
}

func (p *abuf) abAppend(s string) {
	p.buf = append(p.buf, []byte(s)...)
}

func (p *abuf) abAppendBytes(b []byte) {
	p.buf = append(p.buf, b...)
}

/*** output ***/

func editorScroll() {
	if E.cy < E.rowoff {
		E.rowoff = E.cy
	}
	if E.cy >= E.rowoff + E.screenRows {
		E.rowoff = E.cy - E.screenRows + 1
	}
	if E.cx < E.coloff {
		E.coloff = E.cx
	}
	if E.cx >= E.coloff + E.screenCols {
		E.coloff = E.cx - E.screenCols + 1
	}
}

func editorRefreshScreen() {
	editorScroll()
	var ab abuf
	ab.abAppend("\x1b[25l")
	ab.abAppend("\x1b[H")
	editorDrawRows(&ab)
	ab.abAppend(fmt.Sprintf("\x1b[%d;%dH", (E.cy - E.rowoff) + 1, (E.cx - E.coloff) + 1))
	ab.abAppend("\x1b[25h")
	_, e := io.WriteString(os.Stdout, ab.String())
	if e != nil {
		log.Fatal(e)
	}
}

func editorDrawRows(ab *abuf) {
	for y := 0; y < E.screenRows; y++ {
		filerow := y + E.rowoff
		if filerow >= E.numRows {
			if E.numRows == 0 && y == E.screenRows/3 {
				w := fmt.Sprintf("Kilo editor -- version %s", KILO_VERSION)
				if len(w) > E.screenCols {
					w = w[0:E.screenCols]
				}
				pad := "~ "
				for padding := (E.screenCols - len(w)) / 2; padding > 0; padding-- {
					ab.abAppend(pad)
					pad = " "
				}
				ab.abAppend(w)
			} else {
				ab.abAppend("~")
			}
		} else {
			len := E.rows[filerow].rsize - E.coloff
			if len < 0 { len = 0 }
			if len > E.screenCols { len = E.screenCols }
			ab.abAppendBytes(E.rows[filerow].render[E.coloff:E.coloff+len])
		}
		ab.abAppend("\x1b[K")
		if y < E.screenRows-1 {
			ab.abAppend("\r\n")
		}
	}
}

/*** init ***/

func initEditor() {
	// Initialization a la C not necessary.
	if getWindowSize(&E.screenRows, &E.screenCols) == -1 {
		die(fmt.Errorf("couldn't get screen size"))
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	initEditor()
	if len(os.Args) > 1 {
		editorOpen(os.Args[1])
	}

	for {
		editorRefreshScreen()
		editorProcessKeypress()
	}
}
<commit_msg>Step 85<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"os"
	"syscall"
	"unsafe"
)

/*** defines ***/

const KILO_VERSION = "0.0.1"
const KILO_TAB_STOP = 8
const (
	ARROW_LEFT  = 1000 + iota
	ARROW_RIGHT = 1000 + iota
	ARROW_UP    = 1000 + iota
	ARROW_DOWN  = 1000 + iota
	DEL_KEY     = 1000 + iota
	HOME_KEY    = 1000 + iota
	END_KEY     = 1000 + iota
	PAGE_UP     = 1000 + iota
	PAGE_DOWN   = 1000 + iota
)

/*** data ***/

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type erow struct {
	size   int
	rsize  int
	chars  []byte
	render []byte
}

type editorConfig struct {
	cx          int
	cy          int
	rx          int
	rowoff      int
	coloff      int
	screenRows  int
	screenCols  int
	numRows     int
	rows        []erow
	origTermios *Termios
}

type WinSize struct {
	Row    uint16
	Col    uint16
	Xpixel uint16
	Ypixel uint16
}

var E editorConfig

/*** terminal ***/

func die(err error) {
	disableRawMode()
	io.WriteString(os.Stdout, "\x1b[2J")
	io.WriteString(os.Stdout, "\x1b[H")
	log.Fatal(err)
}

func TcSetAttr(fd uintptr, termios *Termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func enableRawMode() {
	E.origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *E.origTermios
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), E.origTermios); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}

func editorReadKey() int {
	var buffer [1]byte
	var cc int
	var err error
	for cc, err = os.Stdin.Read(buffer[:]); cc != 1; cc, err = os.Stdin.Read(buffer[:]) {
	}
	if err != nil {
		die(err)
	}
	if buffer[0] == '\x1b' {
		var seq [2]byte
		if cc, _ = os.Stdin.Read(seq[:]); cc != 2 {
			return '\x1b'
		}

		if seq[0] == '[' {
			if seq[1] >= '0' && seq[1] <= '9' {
				if cc, err = os.Stdin.Read(buffer[:]); cc != 1 {
					return '\x1b'
				}
				if buffer[0] == '~' {
					switch seq[1] {
					case '1':
						return HOME_KEY
					case '3':
						return DEL_KEY
					case '4':
						return END_KEY
					case '5':
						return PAGE_UP
					case '6':
						return PAGE_DOWN
					case '7':
						return HOME_KEY
					case '8':
						return END_KEY
					}
				}
				// XXX - what happens here?
			} else {
				switch seq[1] {
				case 'A':
					return ARROW_UP
				case 'B':
					return ARROW_DOWN
				case 'C':
					return ARROW_RIGHT
				case 'D':
					return ARROW_LEFT
				case 'H':
					return HOME_KEY
				case 'F':
					return END_KEY
				}
			}
		} else if seq[0] == '0' {
			switch seq[1] {
			case 'H':
				return HOME_KEY
			case 'F':
				return END_KEY
			}
		}

		return '\x1b'
	}
	return int(buffer[0])
}

func getCursorPosition(rows *int, cols *int) int {
	io.WriteString(os.Stdout, "\x1b[6n")
	var buffer [1]byte
	var buf []byte
	var cc int
	for cc, _ = os.Stdin.Read(buffer[:]); cc == 1; cc, _ = os.Stdin.Read(buffer[:]) {
		if buffer[0] == 'R' {
			break
		}
		buf = append(buf, buffer[0])
	}
	if string(buf[0:2]) != "\x1b[" {
		log.Printf("Failed to read rows;cols from tty\n")
		return -1
	}
	if n, e := fmt.Sscanf(string(buf[2:]), "%d;%d", rows, cols); n != 2 || e != nil {
		if e != nil {
			log.Printf("getCursorPosition: fmt.Sscanf() failed: %s\n", e)
		}
		if n != 2 {
			log.Printf("getCursorPosition: got %d items, wanted 2\n", n)
		}
		return -1
	}
	return 0
}

func getWindowSize(rows *int, cols *int) int {
	var w WinSize
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		os.Stdout.Fd(),
		syscall.TIOCGWINSZ,
		uintptr(unsafe.Pointer(&w)),
	)
	if err != 0 { // type syscall.Errno
		io.WriteString(os.Stdout, "\x1b[999C\x1b[999B")
		return getCursorPosition(rows, cols)
	} else {
		*rows = int(w.Row)
		*cols = int(w.Col)
		return 0
	}
	return -1
}

/*** row operations ***/

func editorUpdateRow(row *erow) {
	tabs := 0
	for _, c := range row.chars {
		if c == '\t' {
			tabs++
		}
	}
	row.render = make([]byte, row.size + tabs*(KILO_TAB_STOP - 1))

	idx := 0
	for _, c := range row.chars {
		if c == '\t' {
			row.render[idx] = ' '
			idx++
			for (idx%KILO_TAB_STOP) != 0 {
				row.render[idx] = ' '
				idx++
			}
		} else {
			row.render[idx] = c
			idx++
		}
	}
	row.rsize = idx
}

func editorAppendRow(s []byte) {
	var r erow
	r.chars = s
	r.size = len(s)
	E.rows = append(E.rows, r)
	editorUpdateRow(&E.rows[E.numRows])
	E.numRows++
}

/*** file I/O ***/

func editorOpen(filename string) {
	fd, err := os.Open(filename)
	if err != nil {
		die(err)
	}
	defer fd.Close()
	fp := bufio.NewReader(fd)

	for line, err := fp.ReadBytes('\n'); err == nil; line, err = fp.ReadBytes('\n') {
		// Trim trailing newlines and carriage returns
		for c := line[len(line) - 1]; len(line) > 0 && (c == '\n' || c == '\r'); {
			line = line[:len(line)-1]
			if len(line) > 0 {
				c = line[len(line) - 1]
			}
		}
		editorAppendRow(line)
	}

	if err != nil && err != io.EOF {
		die(err)
	}
}

/*** input ***/

func editorMoveCursor(key int) {
	switch key {
	case ARROW_LEFT:
		if E.cx != 0 {
			E.cx--
		} else if E.cy > 0 {
			E.cy--
			E.cx = E.rows[E.cy].rsize
		}
	case ARROW_RIGHT:
		if E.cy < E.numRows {
			if E.cx < E.rows[E.cy].rsize {
				E.cx++
			} else if E.cx == E.rows[E.cy].rsize {
				E.cy++
				E.cx = 0
			}
		}
	case ARROW_UP:
		if E.cy != 0 {
			E.cy--
		}
	case ARROW_DOWN:
		if E.cy < E.numRows {
			E.cy++
		}
	}

	rowlen := 0
	if E.cy < E.numRows {
		rowlen = E.rows[E.cy].rsize
	}
	if E.cx > rowlen {
		E.cx = rowlen
	}
}

func editorProcessKeypress() {
	c := editorReadKey()
	switch c {
	case ('q' & 0x1f):
		io.WriteString(os.Stdout, "\x1b[2J")
		io.WriteString(os.Stdout, "\x1b[H")
		disableRawMode()
		os.Exit(0)
	case HOME_KEY:
		E.cx = 0
	case END_KEY:
		E.cx = E.screenCols - 1
	case PAGE_UP, PAGE_DOWN:
		dir := ARROW_DOWN
		if c == PAGE_UP {
			dir = ARROW_UP
		}
		for times := E.screenRows; times > 0; times-- {
			editorMoveCursor(dir)
		}
	case ARROW_UP, ARROW_DOWN, ARROW_LEFT, ARROW_RIGHT:
		editorMoveCursor(c)
	}
}

/*** append buffer ***/

type abuf struct {
	buf []byte
}

func (p abuf) String() string {
	return string(p.buf)
}

func (p *abuf) abAppend(s string) {
	p.buf = append(p.buf, []byte(s)...)
}

func (p *abuf) abAppendBytes(b []byte) {
	p.buf = append(p.buf, b...)
}

/*** output ***/

func editorScroll() {
	if E.cy < E.rowoff {
		E.rowoff = E.cy
	}
	if E.cy >= E.rowoff + E.screenRows {
		E.rowoff = E.cy - E.screenRows + 1
	}
	if E.cx < E.coloff {
		E.coloff = E.cx
	}
	if E.cx >= E.coloff + E.screenCols {
		E.coloff = E.cx - E.screenCols + 1
	}
}

func editorRefreshScreen() {
	editorScroll()
	var ab abuf
	ab.abAppend("\x1b[25l")
	ab.abAppend("\x1b[H")
	editorDrawRows(&ab)
	ab.abAppend(fmt.Sprintf("\x1b[%d;%dH", (E.cy - E.rowoff) + 1, (E.cx - E.coloff) + 1))
	ab.abAppend("\x1b[25h")
	_, e := io.WriteString(os.Stdout, ab.String())
	if e != nil {
		log.Fatal(e)
	}
}

func editorDrawRows(ab *abuf) {
	for y := 0; y < E.screenRows; y++ {
		filerow := y + E.rowoff
		if filerow >= E.numRows {
			if E.numRows == 0 && y == E.screenRows/3 {
				w := fmt.Sprintf("Kilo editor -- version %s", KILO_VERSION)
				if len(w) > E.screenCols {
					w = w[0:E.screenCols]
				}
				pad := "~ "
				for padding := (E.screenCols - len(w)) / 2; padding > 0; padding-- {
					ab.abAppend(pad)
					pad = " "
				}
				ab.abAppend(w)
			} else {
				ab.abAppend("~")
			}
		} else {
			len := E.rows[filerow].rsize - E.coloff
			if len < 0 { len = 0 }
			if len > E.screenCols { len = E.screenCols }
			ab.abAppendBytes(E.rows[filerow].render[E.coloff:E.coloff+len])
		}
		ab.abAppend("\x1b[K")
		if y < E.screenRows-1 {
			ab.abAppend("\r\n")
		}
	}
}

/*** init ***/

func initEditor() {
	// Initialization a la C not necessary.
	if getWindowSize(&E.screenRows, &E.screenCols) == -1 {
		die(fmt.Errorf("couldn't get screen size"))
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	initEditor()
	if len(os.Args) > 1 {
		editorOpen(os.Args[1])
	}

	for {
		editorRefreshScreen()
		editorProcessKeypress()
	}
}
<|endoftext|>


<commit_before>/*  lMDQ is a MDQ server that caches metadata locally so it's local clients can lookup
    pre-checked metadata and not depend on a working connection to a remote MDQ server.

    It uses SQLite as it's datastore and allows lookup by either entityID or Location.
    The latter is used by WAYF for it's mass hosting services BIRK and KRIB.

    It can also be used as a library for just looking up metadata inside a go program.

    Or a client can use the SQLite database directly using the following query:

		"select e.md, e.hash from entity e, lookup l where l.hash = $1 and l.entity_id_fk = e.id and e.validuntil >= $2"

    where $1 is the lowercase hex sha1 of the entityID or location without the {sha1} prefix
    $2 is the current epoch.

    to-do:
        √ caching interface
          invalidate cache ???
*/

package lMDQ

import (
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	// 	_ "github.com/mattn/go-sqlite3" for handling sqlite3
	_ "github.com/mattn/go-sqlite3"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"sort"
	"strings"
	"sync"
	"time"
)

type (
	// EntityRec refers entity info
	EntityRec struct {
		id       int64
		entityid string
		hash     string
	}
	// MDQ refers to metadata query
	MDQ struct {
		db    *sql.DB
		stmt  *sql.Stmt
		Path  string
		Cache map[string]*MdXp
		Lock  sync.Mutex
		Table string
	}
	// MdXp refers to check validity
	MdXp struct {
		*goxml.Xp
		created time.Time
	}
)

var (
	cacheduration = time.Minute * 60
	// MetaDataNotFoundError refers to error
	MetaDataNotFoundError = errors.New("Metadata not found")
)

// Valid refers to check the validity of metadata
func (xp *MdXp) Valid(duration time.Duration) bool {
	since := time.Since(xp.created)
	//log.Println(since, duration, since  < duration)
	return since < duration
}

// Open refers to open metadata file
func (mdq *MDQ) Open() (err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()
	mdq.Cache = make(map[string]*MdXp)
	mdq.db, err = sql.Open("sqlite3", mdq.Path)
	if err != nil {
		return
	}
	// This is supposed to be a very smart wayf do to prefix search - keep an eye on whether a 10 char prefix ie. using 40 bits is enough
	mdq.stmt, err = mdq.db.Prepare("select e.md md from entity_" + mdq.Table + " e, lookup_" + mdq.Table + " l where ? < l.hash||'z' and l.hash||'z' <= ? and l.entity_id_fk = e.id")
	if err != nil {
		return
	}
	return
}

// MDQ looks up an entity using the supplied feed and key.
// The key can be an entityID or a location, optionally in {sha1} format
// It returns a non nil err if the entity is not found
// and the metadata and a hash/etag over the content if it is.
// The hash can be used to decide if a cached dom object is still valid,
// This might be an optimization as the database lookup is much faster that the parsing.
func (mdq *MDQ) MDQ(key string) (xp *goxml.Xp, err error) {
	return mdq.dbget(key, true)
}

func (mdq *MDQ) dbget(key string, cache bool) (xp *goxml.Xp, err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()

	k := key
	if strings.HasPrefix(key, "{sha1}") {
		key = key[6:]
	} else {
		hash := sha1.Sum([]byte(key))
		key = hex.EncodeToString(append(hash[:]))
	}
	key = key[:10] // only use the first 10 chars for key
	cachedxp := mdq.Cache[key]
	if cachedxp != nil && cachedxp.Valid(cacheduration) {
		xp = cachedxp.Xp.CpXp()
		return
	}

	var xml []byte
	err = mdq.stmt.QueryRow(key, key+"z").Scan(&xml)
	switch {
	case err == sql.ErrNoRows:
		err = goxml.Wrap(MetaDataNotFoundError, "err:Metadata not found", "key:"+k, "table:"+mdq.Table)
		return
	case err != nil:
		return
	default:
		md := gosaml.Inflate(xml)
		xp = goxml.NewXp(md)
	}
	if cache {
		mdxp := new(MdXp)
		mdxp.Xp = xp
		mdxp.created = time.Now()
		mdq.Cache[key] = mdxp
	}
	return
}

// MDQFilter refers Filtering by xpath for testing purposes
func (mdq *MDQ) MDQFilter(xpathfilter string) (xp *goxml.Xp, numberOfEntities int, err error) {
	recs, err := mdq.getEntityList()
	if err != nil {
		return
	}

	// get the entities into an ordered slice
	index := make([]string, len(recs))
	i := 0
	for k := range recs {
		index[i] = k
		i++
	}
	sort.Strings(index)

	//log.Println(xpathfilter)
	xp = goxml.NewXpFromString(`<md:EntitiesDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" />`)

	root, _ := xp.Doc.DocumentElement()
	for _, entityID := range index {
		ent, _ := mdq.dbget(entityID, false)

		if xpathfilter == "" || len(ent.Query(nil, xpathfilter)) > 0 {
			entity, _ := ent.Doc.DocumentElement()
			root.AddChild(xp.CopyNode(entity, 1))
			numberOfEntities++
		}
	}
	return
}

// getEntityList returns a map keyed by entityIDs for the
// current entities in the database
func (mdq *MDQ) getEntityList() (entities map[string]EntityRec, err error) {

	entities = make(map[string]EntityRec)
	var rows *sql.Rows
	rows, err = mdq.db.Query("select id, entityid, hash from entity_" + mdq.Table)
	if err != nil {
		return
	}
	defer rows.Close()
	for rows.Next() {
		var rec EntityRec
		if err = rows.Scan(&rec.id, &rec.entityid, &rec.hash); err != nil {
			return
		}
		entities[rec.entityid] = rec
	}
	if err = rows.Err(); err != nil { // no reason to actually check err, but if we later forget ...
		return
	}
	return
}
<commit_msg>Explicitly close the db, when renewing md.<commit_after>/*  lMDQ is a MDQ server that caches metadata locally so it's local clients can lookup
    pre-checked metadata and not depend on a working connection to a remote MDQ server.

    It uses SQLite as it's datastore and allows lookup by either entityID or Location.
    The latter is used by WAYF for it's mass hosting services BIRK and KRIB.

    It can also be used as a library for just looking up metadata inside a go program.

    Or a client can use the SQLite database directly using the following query:

		"select e.md, e.hash from entity e, lookup l where l.hash = $1 and l.entity_id_fk = e.id and e.validuntil >= $2"

    where $1 is the lowercase hex sha1 of the entityID or location without the {sha1} prefix
    $2 is the current epoch.

    to-do:
        √ caching interface
          invalidate cache ???
*/

package lMDQ

import (
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	// 	_ "github.com/mattn/go-sqlite3" for handling sqlite3
	_ "github.com/mattn/go-sqlite3"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"sort"
	"strings"
	"sync"
	"time"
)

type (
	// EntityRec refers entity info
	EntityRec struct {
		id       int64
		entityid string
		hash     string
	}
	// MDQ refers to metadata query
	MDQ struct {
		db    *sql.DB
		stmt  *sql.Stmt
		Path  string
		Cache map[string]*MdXp
		Lock  sync.Mutex
		Table string
	}
	// MdXp refers to check validity
	MdXp struct {
		*goxml.Xp
		created time.Time
	}
)

var (
	cacheduration = time.Minute * 60
	// MetaDataNotFoundError refers to error
	MetaDataNotFoundError = errors.New("Metadata not found")
)

// Valid refers to check the validity of metadata
func (xp *MdXp) Valid(duration time.Duration) bool {
	since := time.Since(xp.created)
	//log.Println(since, duration, since  < duration)
	return since < duration
}

// Open refers to open metadata file
func (mdq *MDQ) Open() (err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()
	mdq.Cache = make(map[string]*MdXp)
	if mdq.db != nil {
	    mdq.db.Close()
	}
	mdq.db, err = sql.Open("sqlite3", mdq.Path)
	if err != nil {
		return
	}
	// This is supposed to be a very smart wayf do to prefix search - keep an eye on whether a 10 char prefix ie. using 40 bits is enough
	mdq.stmt, err = mdq.db.Prepare("select e.md md from entity_" + mdq.Table + " e, lookup_" + mdq.Table + " l where ? < l.hash||'z' and l.hash||'z' <= ? and l.entity_id_fk = e.id")
	if err != nil {
		return
	}
	return
}

// MDQ looks up an entity using the supplied feed and key.
// The key can be an entityID or a location, optionally in {sha1} format
// It returns a non nil err if the entity is not found
// and the metadata and a hash/etag over the content if it is.
// The hash can be used to decide if a cached dom object is still valid,
// This might be an optimization as the database lookup is much faster that the parsing.
func (mdq *MDQ) MDQ(key string) (xp *goxml.Xp, err error) {
	return mdq.dbget(key, true)
}

func (mdq *MDQ) dbget(key string, cache bool) (xp *goxml.Xp, err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()

	k := key
	if strings.HasPrefix(key, "{sha1}") {
		key = key[6:]
	} else {
		hash := sha1.Sum([]byte(key))
		key = hex.EncodeToString(append(hash[:]))
	}
	key = key[:10] // only use the first 10 chars for key
	cachedxp := mdq.Cache[key]
	if cachedxp != nil && cachedxp.Valid(cacheduration) {
		xp = cachedxp.Xp.CpXp()
		return
	}

	var xml []byte
	err = mdq.stmt.QueryRow(key, key+"z").Scan(&xml)
	switch {
	case err == sql.ErrNoRows:
		err = goxml.Wrap(MetaDataNotFoundError, "err:Metadata not found", "key:"+k, "table:"+mdq.Table)
		return
	case err != nil:
		return
	default:
		md := gosaml.Inflate(xml)
		xp = goxml.NewXp(md)
	}
	if cache {
		mdxp := new(MdXp)
		mdxp.Xp = xp
		mdxp.created = time.Now()
		mdq.Cache[key] = mdxp
	}
	return
}

// MDQFilter refers Filtering by xpath for testing purposes
func (mdq *MDQ) MDQFilter(xpathfilter string) (xp *goxml.Xp, numberOfEntities int, err error) {
	recs, err := mdq.getEntityList()
	if err != nil {
		return
	}

	// get the entities into an ordered slice
	index := make([]string, len(recs))
	i := 0
	for k := range recs {
		index[i] = k
		i++
	}
	sort.Strings(index)

	//log.Println(xpathfilter)
	xp = goxml.NewXpFromString(`<md:EntitiesDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" />`)

	root, _ := xp.Doc.DocumentElement()
	for _, entityID := range index {
		ent, _ := mdq.dbget(entityID, false)

		if xpathfilter == "" || len(ent.Query(nil, xpathfilter)) > 0 {
			entity, _ := ent.Doc.DocumentElement()
			root.AddChild(xp.CopyNode(entity, 1))
			numberOfEntities++
		}
	}
	return
}

// getEntityList returns a map keyed by entityIDs for the
// current entities in the database
func (mdq *MDQ) getEntityList() (entities map[string]EntityRec, err error) {

	entities = make(map[string]EntityRec)
	var rows *sql.Rows
	rows, err = mdq.db.Query("select id, entityid, hash from entity_" + mdq.Table)
	if err != nil {
		return
	}
	defer rows.Close()
	for rows.Next() {
		var rec EntityRec
		if err = rows.Scan(&rec.id, &rec.entityid, &rec.hash); err != nil {
			return
		}
		entities[rec.entityid] = rec
	}
	if err = rows.Err(); err != nil { // no reason to actually check err, but if we later forget ...
		return
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides LDAP client functions.
package ldap

import (
	"errors"
	"fmt"
	"github.com/mavricknz/asn1-ber"
	"io/ioutil"
)

// LDAP Application Codes
const (
	ApplicationBindRequest           = 0
	ApplicationBindResponse          = 1
	ApplicationUnbindRequest         = 2
	ApplicationSearchRequest         = 3
	ApplicationSearchResultEntry     = 4
	ApplicationSearchResultDone      = 5
	ApplicationModifyRequest         = 6
	ApplicationModifyResponse        = 7
	ApplicationAddRequest            = 8
	ApplicationAddResponse           = 9
	ApplicationDelRequest            = 10
	ApplicationDelResponse           = 11
	ApplicationModifyDNRequest       = 12
	ApplicationModifyDNResponse      = 13
	ApplicationCompareRequest        = 14
	ApplicationCompareResponse       = 15
	ApplicationAbandonRequest        = 16
	ApplicationSearchResultReference = 19
	ApplicationExtendedRequest       = 23
	ApplicationExtendedResponse      = 24
)

var ApplicationMap = map[uint8]string{
	ApplicationBindRequest:           "Bind Request",
	ApplicationBindResponse:          "Bind Response",
	ApplicationUnbindRequest:         "Unbind Request",
	ApplicationSearchRequest:         "Search Request",
	ApplicationSearchResultEntry:     "Search Result Entry",
	ApplicationSearchResultDone:      "Search Result Done",
	ApplicationModifyRequest:         "Modify Request",
	ApplicationModifyResponse:        "Modify Response",
	ApplicationAddRequest:            "Add Request",
	ApplicationAddResponse:           "Add Response",
	ApplicationDelRequest:            "Del Request",
	ApplicationDelResponse:           "Del Response",
	ApplicationModifyDNRequest:       "Modify DN Request",
	ApplicationModifyDNResponse:      "Modify DN Response",
	ApplicationCompareRequest:        "Compare Request",
	ApplicationCompareResponse:       "Compare Response",
	ApplicationAbandonRequest:        "Abandon Request",
	ApplicationSearchResultReference: "Search Result Reference",
	ApplicationExtendedRequest:       "Extended Request",
	ApplicationExtendedResponse:      "Extended Response",
}

// LDAP Result Codes
const (
	LDAPResultSuccess                      = 0
	LDAPResultOperationsError              = 1
	LDAPResultProtocolError                = 2
	LDAPResultTimeLimitExceeded            = 3
	LDAPResultSizeLimitExceeded            = 4
	LDAPResultCompareFalse                 = 5
	LDAPResultCompareTrue                  = 6
	LDAPResultAuthMethodNotSupported       = 7
	LDAPResultStrongAuthRequired           = 8
	LDAPResultReferral                     = 10
	LDAPResultAdminLimitExceeded           = 11
	LDAPResultUnavailableCriticalExtension = 12
	LDAPResultConfidentialityRequired      = 13
	LDAPResultSaslBindInProgress           = 14
	LDAPResultNoSuchAttribute              = 16
	LDAPResultUndefinedAttributeType       = 17
	LDAPResultInappropriateMatching        = 18
	LDAPResultConstraintViolation          = 19
	LDAPResultAttributeOrValueExists       = 20
	LDAPResultInvalidAttributeSyntax       = 21
	LDAPResultNoSuchObject                 = 32
	LDAPResultAliasProblem                 = 33
	LDAPResultInvalidDNSyntax              = 34
	LDAPResultAliasDereferencingProblem    = 36
	LDAPResultInappropriateAuthentication  = 48
	LDAPResultInvalidCredentials           = 49
	LDAPResultInsufficientAccessRights     = 50
	LDAPResultBusy                         = 51
	LDAPResultUnavailable                  = 52
	LDAPResultUnwillingToPerform           = 53
	LDAPResultLoopDetect                   = 54
	LDAPResultNamingViolation              = 64
	LDAPResultObjectClassViolation         = 65
	LDAPResultNotAllowedOnNonLeaf          = 66
	LDAPResultNotAllowedOnRDN              = 67
	LDAPResultEntryAlreadyExists           = 68
	LDAPResultObjectClassModsProhibited    = 69
	LDAPResultAffectsMultipleDSAs          = 71
	LDAPResultOther                        = 80

	ErrorNetwork         = 200
	ErrorFilterCompile   = 201
	ErrorFilterDecompile = 202
	ErrorDebugging       = 203
	ErrorEncoding        = 204
)

var LDAPResultCodeMap = map[uint8]string{
	LDAPResultSuccess:                      "Success",
	LDAPResultOperationsError:              "Operations Error",
	LDAPResultProtocolError:                "Protocol Error",
	LDAPResultTimeLimitExceeded:            "Time Limit Exceeded",
	LDAPResultSizeLimitExceeded:            "Size Limit Exceeded",
	LDAPResultCompareFalse:                 "Compare False",
	LDAPResultCompareTrue:                  "Compare True",
	LDAPResultAuthMethodNotSupported:       "Auth Method Not Supported",
	LDAPResultStrongAuthRequired:           "Strong Auth Required",
	LDAPResultReferral:                     "Referral",
	LDAPResultAdminLimitExceeded:           "Admin Limit Exceeded",
	LDAPResultUnavailableCriticalExtension: "Unavailable Critical Extension",
	LDAPResultConfidentialityRequired:      "Confidentiality Required",
	LDAPResultSaslBindInProgress:           "Sasl Bind In Progress",
	LDAPResultNoSuchAttribute:              "No Such Attribute",
	LDAPResultUndefinedAttributeType:       "Undefined Attribute Type",
	LDAPResultInappropriateMatching:        "Inappropriate Matching",
	LDAPResultConstraintViolation:          "Constraint Violation",
	LDAPResultAttributeOrValueExists:       "Attribute Or Value Exists",
	LDAPResultInvalidAttributeSyntax:       "Invalid Attribute Syntax",
	LDAPResultNoSuchObject:                 "No Such Object",
	LDAPResultAliasProblem:                 "Alias Problem",
	LDAPResultInvalidDNSyntax:              "Invalid DN Syntax",
	LDAPResultAliasDereferencingProblem:    "Alias Dereferencing Problem",
	LDAPResultInappropriateAuthentication:  "Inappropriate Authentication",
	LDAPResultInvalidCredentials:           "Invalid Credentials",
	LDAPResultInsufficientAccessRights:     "Insufficient Access Rights",
	LDAPResultBusy:                         "Busy",
	LDAPResultUnavailable:                  "Unavailable",
	LDAPResultUnwillingToPerform:           "Unwilling To Perform",
	LDAPResultLoopDetect:                   "Loop Detect",
	LDAPResultNamingViolation:              "Naming Violation",
	LDAPResultObjectClassViolation:         "Object Class Violation",
	LDAPResultNotAllowedOnNonLeaf:          "Not Allowed On Non Leaf",
	LDAPResultNotAllowedOnRDN:              "Not Allowed On RDN",
	LDAPResultEntryAlreadyExists:           "Entry Already Exists",
	LDAPResultObjectClassModsProhibited:    "Object Class Mods Prohibited",
	LDAPResultAffectsMultipleDSAs:          "Affects Multiple DSAs",
	LDAPResultOther:                        "Other",
}

// Adds descriptions to an LDAP Response packet for debugging
func addLDAPDescriptions(packet *ber.Packet) (err *Error) {
	defer func() {
		if r := recover(); r != nil {
			err = NewError(ErrorDebugging, errors.New("Cannot process packet to add descriptions"))
		}
	}()
	packet.Description = "LDAP Response"
	packet.Children[0].Description = "Message ID"

	application := packet.Children[1].Tag
	packet.Children[1].Description = ApplicationMap[application]

	switch application {
	case ApplicationBindRequest:
		addRequestDescriptions(packet)
	case ApplicationBindResponse:
		addDefaultLDAPResponseDescriptions(packet)
	case ApplicationUnbindRequest:
		addRequestDescriptions(packet)
	case ApplicationSearchRequest:
		addRequestDescriptions(packet)
	case ApplicationSearchResultEntry:
		packet.Children[1].Children[0].Description = "Object Name"
		packet.Children[1].Children[1].Description = "Attributes"
		for _, child := range packet.Children[1].Children[1].Children {
			child.Description = "Attribute"
			child.Children[0].Description = "Attribute Name"
			child.Children[1].Description = "Attribute Values"
			for _, grandchild := range child.Children[1].Children {
				grandchild.Description = "Attribute Value"
			}
		}
		if len(packet.Children) == 3 {
			addControlDescriptions(packet.Children[2])
		}
	case ApplicationSearchResultDone:
		addDefaultLDAPResponseDescriptions(packet)
	case ApplicationModifyRequest:
		addRequestDescriptions(packet)
	case ApplicationModifyResponse:
	case ApplicationAddRequest:
		addRequestDescriptions(packet)
	case ApplicationAddResponse:
	case ApplicationDelRequest:
		addRequestDescriptions(packet)
	case ApplicationDelResponse:
	case ApplicationModifyDNRequest:
		addRequestDescriptions(packet)
	case ApplicationModifyDNResponse:
	case ApplicationCompareRequest:
		addRequestDescriptions(packet)
	case ApplicationCompareResponse:
	case ApplicationAbandonRequest:
		addRequestDescriptions(packet)
	case ApplicationSearchResultReference:
	case ApplicationExtendedRequest:
		addRequestDescriptions(packet)
	case ApplicationExtendedResponse:
	}

	return nil
}

func addControlDescriptions(packet *ber.Packet) {
	packet.Description = "Controls"
	for _, child := range packet.Children {
		child.Description = "Control"
		child.Children[0].Description = "Control Type (" + ControlTypeMap[child.Children[0].Value.(string)] + ")"
		value := child.Children[1]
		if len(child.Children) == 3 {
			child.Children[1].Description = "Criticality"
			value = child.Children[2]
		}
		value.Description = "Control Value"

		switch child.Children[0].Value.(string) {
		case ControlTypePaging:
			value.Description += " (Paging)"
			if value.Value != nil {
				value_children := ber.DecodePacket(value.Data.Bytes())
				value.Data.Truncate(0)
				value.Value = nil
				value_children.Children[1].Value = value_children.Children[1].Data.Bytes()
				value.AppendChild(value_children)
			}
			value.Children[0].Description = "Real Search Control Value"
			value.Children[0].Children[0].Description = "Paging Size"
			value.Children[0].Children[1].Description = "Cookie"
		}
	}
}

func addRequestDescriptions(packet *ber.Packet) {
	packet.Description = "LDAP Request"
	packet.Children[0].Description = "Message ID"
	packet.Children[1].Description = ApplicationMap[packet.Children[1].Tag]
	if len(packet.Children) == 3 {
		addControlDescriptions(packet.Children[2])
	}
}

func addDefaultLDAPResponseDescriptions(packet *ber.Packet) {
	resultCode := packet.Children[1].Children[0].Value.(uint64)
	packet.Children[1].Children[0].Description = "Result Code (" + LDAPResultCodeMap[uint8(resultCode)] + ")"
	packet.Children[1].Children[1].Description = "Matched DN"
	packet.Children[1].Children[2].Description = "Error Message"
	if len(packet.Children[1].Children) > 3 {
		packet.Children[1].Children[3].Description = "Referral"
	}
	if len(packet.Children) == 3 {
		addControlDescriptions(packet.Children[2])
	}
}

func DebugBinaryFile(FileName string) *Error {
	file, err := ioutil.ReadFile(FileName)
	if err != nil {
		return NewError(ErrorDebugging, err)
	}
	ber.PrintBytes(file, "")
	packet := ber.DecodePacket(file)
	addLDAPDescriptions(packet)
	ber.PrintPacket(packet)

	return nil
}

type Error struct {
	Err        error
	ResultCode uint8
}

func (e *Error) Error() string {
	return fmt.Sprintf("LDAP Result Code %d %q: %s", e.ResultCode, LDAPResultCodeMap[e.ResultCode], e.Err.Error())
}

func NewError(ResultCode uint8, Err error) *Error {
	return &Error{ResultCode: ResultCode, Err: Err}
}

func getLDAPResultCode(p *ber.Packet) (code uint8, description string) {
	if len(p.Children) >= 2 {
		response := p.Children[1]
		if response.ClassType == ber.ClassApplication && response.TagType == ber.TypeConstructed && len(response.Children) == 3 {
			code = uint8(response.Children[0].Value.(uint64))
			description = response.Children[2].Value.(string)
			return
		}
	}

	code = ErrorNetwork
	description = "Invalid packet format"
	return
}
<commit_msg>ErrorDecoding Const added<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package provides LDAP client functions.
package ldap

import (
	"errors"
	"fmt"
	"github.com/mavricknz/asn1-ber"
	"io/ioutil"
)

// LDAP Application Codes
const (
	ApplicationBindRequest           = 0
	ApplicationBindResponse          = 1
	ApplicationUnbindRequest         = 2
	ApplicationSearchRequest         = 3
	ApplicationSearchResultEntry     = 4
	ApplicationSearchResultDone      = 5
	ApplicationModifyRequest         = 6
	ApplicationModifyResponse        = 7
	ApplicationAddRequest            = 8
	ApplicationAddResponse           = 9
	ApplicationDelRequest            = 10
	ApplicationDelResponse           = 11
	ApplicationModifyDNRequest       = 12
	ApplicationModifyDNResponse      = 13
	ApplicationCompareRequest        = 14
	ApplicationCompareResponse       = 15
	ApplicationAbandonRequest        = 16
	ApplicationSearchResultReference = 19
	ApplicationExtendedRequest       = 23
	ApplicationExtendedResponse      = 24
)

var ApplicationMap = map[uint8]string{
	ApplicationBindRequest:           "Bind Request",
	ApplicationBindResponse:          "Bind Response",
	ApplicationUnbindRequest:         "Unbind Request",
	ApplicationSearchRequest:         "Search Request",
	ApplicationSearchResultEntry:     "Search Result Entry",
	ApplicationSearchResultDone:      "Search Result Done",
	ApplicationModifyRequest:         "Modify Request",
	ApplicationModifyResponse:        "Modify Response",
	ApplicationAddRequest:            "Add Request",
	ApplicationAddResponse:           "Add Response",
	ApplicationDelRequest:            "Del Request",
	ApplicationDelResponse:           "Del Response",
	ApplicationModifyDNRequest:       "Modify DN Request",
	ApplicationModifyDNResponse:      "Modify DN Response",
	ApplicationCompareRequest:        "Compare Request",
	ApplicationCompareResponse:       "Compare Response",
	ApplicationAbandonRequest:        "Abandon Request",
	ApplicationSearchResultReference: "Search Result Reference",
	ApplicationExtendedRequest:       "Extended Request",
	ApplicationExtendedResponse:      "Extended Response",
}

// LDAP Result Codes
const (
	LDAPResultSuccess                      = 0
	LDAPResultOperationsError              = 1
	LDAPResultProtocolError                = 2
	LDAPResultTimeLimitExceeded            = 3
	LDAPResultSizeLimitExceeded            = 4
	LDAPResultCompareFalse                 = 5
	LDAPResultCompareTrue                  = 6
	LDAPResultAuthMethodNotSupported       = 7
	LDAPResultStrongAuthRequired           = 8
	LDAPResultReferral                     = 10
	LDAPResultAdminLimitExceeded           = 11
	LDAPResultUnavailableCriticalExtension = 12
	LDAPResultConfidentialityRequired      = 13
	LDAPResultSaslBindInProgress           = 14
	LDAPResultNoSuchAttribute              = 16
	LDAPResultUndefinedAttributeType       = 17
	LDAPResultInappropriateMatching        = 18
	LDAPResultConstraintViolation          = 19
	LDAPResultAttributeOrValueExists       = 20
	LDAPResultInvalidAttributeSyntax       = 21
	LDAPResultNoSuchObject                 = 32
	LDAPResultAliasProblem                 = 33
	LDAPResultInvalidDNSyntax              = 34
	LDAPResultAliasDereferencingProblem    = 36
	LDAPResultInappropriateAuthentication  = 48
	LDAPResultInvalidCredentials           = 49
	LDAPResultInsufficientAccessRights     = 50
	LDAPResultBusy                         = 51
	LDAPResultUnavailable                  = 52
	LDAPResultUnwillingToPerform           = 53
	LDAPResultLoopDetect                   = 54
	LDAPResultNamingViolation              = 64
	LDAPResultObjectClassViolation         = 65
	LDAPResultNotAllowedOnNonLeaf          = 66
	LDAPResultNotAllowedOnRDN              = 67
	LDAPResultEntryAlreadyExists           = 68
	LDAPResultObjectClassModsProhibited    = 69
	LDAPResultAffectsMultipleDSAs          = 71
	LDAPResultOther                        = 80

	ErrorNetwork         = 200
	ErrorFilterCompile   = 201
	ErrorFilterDecompile = 202
	ErrorDebugging       = 203
	ErrorEncoding        = 204
	ErrorDecoding        = 205
)

var LDAPResultCodeMap = map[uint8]string{
	LDAPResultSuccess:                      "Success",
	LDAPResultOperationsError:              "Operations Error",
	LDAPResultProtocolError:                "Protocol Error",
	LDAPResultTimeLimitExceeded:            "Time Limit Exceeded",
	LDAPResultSizeLimitExceeded:            "Size Limit Exceeded",
	LDAPResultCompareFalse:                 "Compare False",
	LDAPResultCompareTrue:                  "Compare True",
	LDAPResultAuthMethodNotSupported:       "Auth Method Not Supported",
	LDAPResultStrongAuthRequired:           "Strong Auth Required",
	LDAPResultReferral:                     "Referral",
	LDAPResultAdminLimitExceeded:           "Admin Limit Exceeded",
	LDAPResultUnavailableCriticalExtension: "Unavailable Critical Extension",
	LDAPResultConfidentialityRequired:      "Confidentiality Required",
	LDAPResultSaslBindInProgress:           "Sasl Bind In Progress",
	LDAPResultNoSuchAttribute:              "No Such Attribute",
	LDAPResultUndefinedAttributeType:       "Undefined Attribute Type",
	LDAPResultInappropriateMatching:        "Inappropriate Matching",
	LDAPResultConstraintViolation:          "Constraint Violation",
	LDAPResultAttributeOrValueExists:       "Attribute Or Value Exists",
	LDAPResultInvalidAttributeSyntax:       "Invalid Attribute Syntax",
	LDAPResultNoSuchObject:                 "No Such Object",
	LDAPResultAliasProblem:                 "Alias Problem",
	LDAPResultInvalidDNSyntax:              "Invalid DN Syntax",
	LDAPResultAliasDereferencingProblem:    "Alias Dereferencing Problem",
	LDAPResultInappropriateAuthentication:  "Inappropriate Authentication",
	LDAPResultInvalidCredentials:           "Invalid Credentials",
	LDAPResultInsufficientAccessRights:     "Insufficient Access Rights",
	LDAPResultBusy:                         "Busy",
	LDAPResultUnavailable:                  "Unavailable",
	LDAPResultUnwillingToPerform:           "Unwilling To Perform",
	LDAPResultLoopDetect:                   "Loop Detect",
	LDAPResultNamingViolation:              "Naming Violation",
	LDAPResultObjectClassViolation:         "Object Class Violation",
	LDAPResultNotAllowedOnNonLeaf:          "Not Allowed On Non Leaf",
	LDAPResultNotAllowedOnRDN:              "Not Allowed On RDN",
	LDAPResultEntryAlreadyExists:           "Entry Already Exists",
	LDAPResultObjectClassModsProhibited:    "Object Class Mods Prohibited",
	LDAPResultAffectsMultipleDSAs:          "Affects Multiple DSAs",
	LDAPResultOther:                        "Other",
}

// Adds descriptions to an LDAP Response packet for debugging
func addLDAPDescriptions(packet *ber.Packet) (err *Error) {
	defer func() {
		if r := recover(); r != nil {
			err = NewError(ErrorDebugging, errors.New("Cannot process packet to add descriptions"))
		}
	}()
	packet.Description = "LDAP Response"
	packet.Children[0].Description = "Message ID"

	application := packet.Children[1].Tag
	packet.Children[1].Description = ApplicationMap[application]

	switch application {
	case ApplicationBindRequest:
		addRequestDescriptions(packet)
	case ApplicationBindResponse:
		addDefaultLDAPResponseDescriptions(packet)
	case ApplicationUnbindRequest:
		addRequestDescriptions(packet)
	case ApplicationSearchRequest:
		addRequestDescriptions(packet)
	case ApplicationSearchResultEntry:
		packet.Children[1].Children[0].Description = "Object Name"
		packet.Children[1].Children[1].Description = "Attributes"
		for _, child := range packet.Children[1].Children[1].Children {
			child.Description = "Attribute"
			child.Children[0].Description = "Attribute Name"
			child.Children[1].Description = "Attribute Values"
			for _, grandchild := range child.Children[1].Children {
				grandchild.Description = "Attribute Value"
			}
		}
		if len(packet.Children) == 3 {
			addControlDescriptions(packet.Children[2])
		}
	case ApplicationSearchResultDone:
		addDefaultLDAPResponseDescriptions(packet)
	case ApplicationModifyRequest:
		addRequestDescriptions(packet)
	case ApplicationModifyResponse:
	case ApplicationAddRequest:
		addRequestDescriptions(packet)
	case ApplicationAddResponse:
	case ApplicationDelRequest:
		addRequestDescriptions(packet)
	case ApplicationDelResponse:
	case ApplicationModifyDNRequest:
		addRequestDescriptions(packet)
	case ApplicationModifyDNResponse:
	case ApplicationCompareRequest:
		addRequestDescriptions(packet)
	case ApplicationCompareResponse:
	case ApplicationAbandonRequest:
		addRequestDescriptions(packet)
	case ApplicationSearchResultReference:
	case ApplicationExtendedRequest:
		addRequestDescriptions(packet)
	case ApplicationExtendedResponse:
	}

	return nil
}

func addControlDescriptions(packet *ber.Packet) {
	packet.Description = "Controls"
	for _, child := range packet.Children {
		child.Description = "Control"
		child.Children[0].Description = "Control Type (" + ControlTypeMap[child.Children[0].Value.(string)] + ")"
		value := child.Children[1]
		if len(child.Children) == 3 {
			child.Children[1].Description = "Criticality"
			value = child.Children[2]
		}
		value.Description = "Control Value"

		switch child.Children[0].Value.(string) {
		case ControlTypePaging:
			value.Description += " (Paging)"
			if value.Value != nil {
				value_children := ber.DecodePacket(value.Data.Bytes())
				value.Data.Truncate(0)
				value.Value = nil
				value_children.Children[1].Value = value_children.Children[1].Data.Bytes()
				value.AppendChild(value_children)
			}
			value.Children[0].Description = "Real Search Control Value"
			value.Children[0].Children[0].Description = "Paging Size"
			value.Children[0].Children[1].Description = "Cookie"
		}
	}
}

func addRequestDescriptions(packet *ber.Packet) {
	packet.Description = "LDAP Request"
	packet.Children[0].Description = "Message ID"
	packet.Children[1].Description = ApplicationMap[packet.Children[1].Tag]
	if len(packet.Children) == 3 {
		addControlDescriptions(packet.Children[2])
	}
}

func addDefaultLDAPResponseDescriptions(packet *ber.Packet) {
	resultCode := packet.Children[1].Children[0].Value.(uint64)
	packet.Children[1].Children[0].Description = "Result Code (" + LDAPResultCodeMap[uint8(resultCode)] + ")"
	packet.Children[1].Children[1].Description = "Matched DN"
	packet.Children[1].Children[2].Description = "Error Message"
	if len(packet.Children[1].Children) > 3 {
		packet.Children[1].Children[3].Description = "Referral"
	}
	if len(packet.Children) == 3 {
		addControlDescriptions(packet.Children[2])
	}
}

func DebugBinaryFile(FileName string) *Error {
	file, err := ioutil.ReadFile(FileName)
	if err != nil {
		return NewError(ErrorDebugging, err)
	}
	ber.PrintBytes(file, "")
	packet := ber.DecodePacket(file)
	addLDAPDescriptions(packet)
	ber.PrintPacket(packet)

	return nil
}

type Error struct {
	Err        error
	ResultCode uint8
}

func (e *Error) Error() string {
	return fmt.Sprintf("LDAP Result Code %d %q: %s", e.ResultCode, LDAPResultCodeMap[e.ResultCode], e.Err.Error())
}

func NewError(ResultCode uint8, Err error) *Error {
	return &Error{ResultCode: ResultCode, Err: Err}
}

func getLDAPResultCode(p *ber.Packet) (code uint8, description string) {
	if len(p.Children) >= 2 {
		response := p.Children[1]
		if response.ClassType == ber.ClassApplication && response.TagType == ber.TypeConstructed && len(response.Children) == 3 {
			code = uint8(response.Children[0].Value.(uint64))
			description = response.Children[2].Value.(string)
			return
		}
	}

	code = ErrorNetwork
	description = "Invalid packet format"
	return
}
<|endoftext|>


<commit_before>package main

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type lead struct {
	Id                 bson.ObjectId `bson:"_id"`
	Contact            *mgo.DBRef    `bson:"contact,omitempty"`
	Source             string        `bson:"source,omitempty"`
	Owner              string        `bson:"owner,omitempty"`
	Status             string        `bson:"status,omitempty"`
	TeamSize           float64       `bson:"teamsize,omitempty"`
	RatePerHour        float64       `bson:"rateperhour,omitempty"`
	DurationInMonths   float64       `bson:"durationinmonths,omitempty"`
	EstimatedStartDate string        `bson:"estimatedstartdate,omitempty"`
	//Here we choose not to use time.Time because omitempty isn't supported for time.Time
	Comments []string `bson:"comments,omitempty"`
}

// NewLead takes the fields of a lead, initializes a struct of lead type and returns
// the pointer to that struct.
// Also, It inserts the lead data into a mongoDB collection, which is passed as the first parameter.
func NewLead(c *mgo.Collection, r *mgo.DBRef, source, owner, status string,
	teamsize, rate, duration float64, start string, comments []string) (*lead, error) {

	doc := lead{
		Id:                 bson.NewObjectId(),
		Contact:            r,
		Source:             source,
		Owner:              owner,
		Status:             status,
		TeamSize:           teamsize,
		RatePerHour:        rate,
		DurationInMonths:   duration,
		EstimatedStartDate: start,
		Comments:           comments,
	}
	err := c.Insert(doc)
	if err != nil {
		return &lead{}, err
	}
	return &doc, nil
}
<commit_msg>Add json tags for lead type<commit_after>package main

import (
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type lead struct {
	Id                 bson.ObjectId `bson:"_id"                          json:"id"`
	Contact            *mgo.DBRef    `bson:"contact,omitempty"            json:"contact,omitempty"`
	Source             string        `bson:"source,omitempty"             json:"source,omitempty"`
	Owner              string        `bson:"owner,omitempty"              json:"owner,omitempty"`
	Status             string        `bson:"status,omitempty"             json:"status,omitempty"`
	TeamSize           float64       `bson:"teamsize,omitempty"           json:"teamsize,omitempty"`
	RatePerHour        float64       `bson:"rateperhour,omitempty"        json:"rateperhour,omitempty"`
	DurationInMonths   float64       `bson:"durationinmonths,omitempty"   json:"durationinmonths,omitempty"`
	EstimatedStartDate string        `bson:"estimatedstartdate,omitempty" json:"estimatedstartdate,omitempty"`
	//Here we choose not to use time.Time because omitempty isn't supported for time.Time
	Comments []string `bson:"comments,omitempty"`
}

// NewLead takes the fields of a lead, initializes a struct of lead type and returns
// the pointer to that struct.
// Also, It inserts the lead data into a mongoDB collection, which is passed as the first parameter.
func NewLead(c *mgo.Collection, r *mgo.DBRef, source, owner, status string,
	teamsize, rate, duration float64, start string, comments []string) (*lead, error) {

	doc := lead{
		Id:                 bson.NewObjectId(),
		Contact:            r,
		Source:             source,
		Owner:              owner,
		Status:             status,
		TeamSize:           teamsize,
		RatePerHour:        rate,
		DurationInMonths:   duration,
		EstimatedStartDate: start,
		Comments:           comments,
	}
	err := c.Insert(doc)
	if err != nil {
		return &lead{}, err
	}
	return &doc, nil
}
<|endoftext|>


<commit_before>package async

import (
  "container/list"
  "sync"
)

/*

  Used to contain the Routine functions to be processed

  This list inherits http://golang.org/pkg/container/list/ and contains all
  of the functionality that it contains, with a minor tweak to Remove. Instead
  of Remove returning the element, it returns our routine. This is used to
  ensure that our Routine is removed from the list before it's ran, and
  therefore isn't able to be called again.

*/
type List struct {
  *list.List

  Wait sync.WaitGroup
}

/*
  Create a new list
*/
func New() *List {
  return &List{
    List: list.New(),
  }
}

/*
  Add a Routine function to the current list
*/
func (l *List) Add(routine Routine) (*List, *list.Element) {
  element := l.PushBack(routine)
  return l, element
}

/*
  Add multiple Routine functions to the current list
*/
func (l *List) Multiple(routines ...Routine) (*List, []*list.Element) {
  var (
    elements = make([]*list.Element, 0)
  )

  for i := 0; i < len(routines); i++ {
    _, e := l.Add(routines[i])
    elements = append(elements, e)
  }

  return l, elements
}

/*
  Remove an element from the current list
*/
func (l *List) Remove(element *list.Element) (*List, Routine) {
  routine := l.List.Remove(element).(Routine)
  return l, routine
}
<commit_msg>Replace golang.org link with godoc.org link<commit_after>package async

import (
  "container/list"
  "sync"
)

/*

  Used to contain the Routine functions to be processed

  This list inherits https://godoc.org/container/list and contains all
  of the functionality that it contains, with a minor tweak to Remove. Instead
  of Remove returning the element, it returns our routine. This is used to
  ensure that our Routine is removed from the list before it's ran, and
  therefore isn't able to be called again.

*/
type List struct {
  *list.List

  Wait sync.WaitGroup
}

/*
  Create a new list
*/
func New() *List {
  return &List{
    List: list.New(),
  }
}

/*
  Add a Routine function to the current list
*/
func (l *List) Add(routine Routine) (*List, *list.Element) {
  element := l.PushBack(routine)
  return l, element
}

/*
  Add multiple Routine functions to the current list
*/
func (l *List) Multiple(routines ...Routine) (*List, []*list.Element) {
  var (
    elements = make([]*list.Element, 0)
  )

  for i := 0; i < len(routines); i++ {
    _, e := l.Add(routines[i])
    elements = append(elements, e)
  }

  return l, elements
}

/*
  Remove an element from the current list
*/
func (l *List) Remove(element *list.Element) (*List, Routine) {
  routine := l.List.Remove(element).(Routine)
  return l, routine
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"time"

	"github.com/cloudfoundry/cli/cf/configuration/config_helpers"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/plugin"
)

type Application struct {
	Name string
}

type BlueGreenDeployPlugin struct {
	Connection plugin.CliConnection
}

func (p *BlueGreenDeployPlugin) Run(cliConnection plugin.CliConnection, args []string) {
	p.Connection = cliConnection

	if len(args) < 2 {
		fmt.Printf("appname must be specified")
		os.Exit(1)
	}

	appName := args[1]
	err := p.DeleteOldAppVersions(appName)
	if err != nil {
		fmt.Printf("Could not delete old app version - %s", err.Error())
		os.Exit(1)
	}

	err = p.PushNewAppVersion(appName)
	if err != nil {
		fmt.Printf("Could not push new version - %s", err.Error())
		os.Exit(1)
	}
}

func (p *BlueGreenDeployPlugin) GetMetadata() plugin.PluginMetadata {
	return plugin.PluginMetadata{
		Name: "blue-green-deploy",
		Version: plugin.VersionType{
			Major: 0,
			Minor: 1,
			Build: 0,
		},
		Commands: []plugin.Command{
			{
				Name:     "blue-green-deploy",
				Alias:    "bgd",
				HelpText: "Do zero-time deploys in a non-sucky way",
			},
		},
	}
}

func (p *BlueGreenDeployPlugin) deleteApps(apps []Application) error {
	for _, app := range apps {
		if _, err := p.Connection.CliCommand("delete", app.Name, "-f", "-r"); err != nil {
			return err
		}
	}

	return nil
}

func (p *BlueGreenDeployPlugin) DeleteOldAppVersions(appName string) error {
	apps, err := p.appsInCurrentSpace()
	if err != nil {
		return err
	}
	return p.deleteApps(FilterOldApps(appName, apps))
}

func (p *BlueGreenDeployPlugin) PushNewAppVersion(appName string) error {
	_, err := p.Connection.CliCommand("push", GenerateAppName(appName))
	return err
}

func (p *BlueGreenDeployPlugin) appsInCurrentSpace() ([]Application, error) {
	path := fmt.Sprintf("/v2/spaces/%s/summary", getSpaceGuid())

	output, err := p.Connection.CliCommandWithoutTerminalOutput("curl", path)
	if err != nil {
		return nil, err
	}

	apps := struct {
		Apps []Application
	}{}

	json.Unmarshal([]byte(output[0]), &apps)
	return apps.Apps, nil
}

func getSpaceGuid() string {
	configRepo := core_config.NewRepositoryFromFilepath(config_helpers.DefaultFilePath(), func(err error) {
		if err != nil {
			fmt.Printf("Config error: %s", err)
		}
	})

	return configRepo.SpaceFields().Guid
}

func FilterOldApps(appName string, apps []Application) (oldApps []Application) {
	r := regexp.MustCompile(fmt.Sprintf("^%s-[0-9]{14}-old$", appName))
	oldApps = []Application{}
	for _, app := range apps {
		if r.MatchString(app.Name) {
			oldApps = append(oldApps, app)
		}
	}
	return
}

func GenerateAppName(base string) string {
	return fmt.Sprintf("%s-%s", base, time.Now().Format("20060102150405"))
}

func main() {
	plugin.Start(&BlueGreenDeployPlugin{})
}
<commit_msg>because consistent versioning is very important<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"regexp"
	"time"

	"github.com/cloudfoundry/cli/cf/configuration/config_helpers"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/plugin"
)

type Application struct {
	Name string
}

type BlueGreenDeployPlugin struct {
	Connection plugin.CliConnection
}

func (p *BlueGreenDeployPlugin) Run(cliConnection plugin.CliConnection, args []string) {
	p.Connection = cliConnection

	if len(args) < 2 {
		fmt.Printf("appname must be specified")
		os.Exit(1)
	}

	appName := args[1]
	err := p.DeleteOldAppVersions(appName)
	if err != nil {
		fmt.Printf("Could not delete old app version - %s", err.Error())
		os.Exit(1)
	}

	err = p.PushNewAppVersion(appName)
	if err != nil {
		fmt.Printf("Could not push new version - %s", err.Error())
		os.Exit(1)
	}
}

func (p *BlueGreenDeployPlugin) GetMetadata() plugin.PluginMetadata {
	return plugin.PluginMetadata{
		Name: "blue-green-deploy",
		Version: plugin.VersionType{
			Major: 0,
			Minor: 3,
			Build: 0,
		},
		Commands: []plugin.Command{
			{
				Name:     "blue-green-deploy",
				Alias:    "bgd",
				HelpText: "Do zero-time deploys in a non-sucky way",
			},
		},
	}
}

func (p *BlueGreenDeployPlugin) deleteApps(apps []Application) error {
	for _, app := range apps {
		if _, err := p.Connection.CliCommand("delete", app.Name, "-f", "-r"); err != nil {
			return err
		}
	}

	return nil
}

func (p *BlueGreenDeployPlugin) DeleteOldAppVersions(appName string) error {
	apps, err := p.appsInCurrentSpace()
	if err != nil {
		return err
	}
	return p.deleteApps(FilterOldApps(appName, apps))
}

func (p *BlueGreenDeployPlugin) PushNewAppVersion(appName string) error {
	_, err := p.Connection.CliCommand("push", GenerateAppName(appName))
	return err
}

func (p *BlueGreenDeployPlugin) appsInCurrentSpace() ([]Application, error) {
	path := fmt.Sprintf("/v2/spaces/%s/summary", getSpaceGuid())

	output, err := p.Connection.CliCommandWithoutTerminalOutput("curl", path)
	if err != nil {
		return nil, err
	}

	apps := struct {
		Apps []Application
	}{}

	json.Unmarshal([]byte(output[0]), &apps)
	return apps.Apps, nil
}

func getSpaceGuid() string {
	configRepo := core_config.NewRepositoryFromFilepath(config_helpers.DefaultFilePath(), func(err error) {
		if err != nil {
			fmt.Printf("Config error: %s", err)
		}
	})

	return configRepo.SpaceFields().Guid
}

func FilterOldApps(appName string, apps []Application) (oldApps []Application) {
	r := regexp.MustCompile(fmt.Sprintf("^%s-[0-9]{14}-old$", appName))
	oldApps = []Application{}
	for _, app := range apps {
		if r.MatchString(app.Name) {
			oldApps = append(oldApps, app)
		}
	}
	return
}

func GenerateAppName(base string) string {
	return fmt.Sprintf("%s-%s", base, time.Now().Format("20060102150405"))
}

func main() {
	plugin.Start(&BlueGreenDeployPlugin{})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/ninjasphere/app-scheduler/controller"
	"github.com/ninjasphere/app-scheduler/model"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/support"
)

var (
	info = ninja.LoadModuleInfo("./package.json")
)

type service struct {
}

type SchedulerApp struct {
	support.AppSupport
	scheduler *controller.Scheduler
}

type Config struct {
}

func (a *SchedulerApp) Start(model *model.Schedule) error {
	if a.scheduler != nil {
		return fmt.Errorf("illegal state: scheduler is already running")
	}
	a.scheduler = &controller.Scheduler{}
	err := a.scheduler.Start(model)
	if err == nil {
		a.SendEvent("config", model)
	}
	return err
}

func (a *SchedulerApp) Stop() error {
	var err error
	if a.scheduler != nil {
		tmp := a.scheduler
		a.scheduler = nil
		err = tmp.Stop()
	}
	return err
}

func main() {
	app := &SchedulerApp{}
	err := app.Init(info)
	if err != nil {
		app.Log.Fatalf("failed to initialize app: %v", err)
	}

	err = app.Export(app)
	if err != nil {
		app.Log.Fatalf("failed to export app: %v", err)
	}

	support.WaitUntilSignal()
}
<commit_msg>Remove unused type definitions.<commit_after>package main

import (
	"fmt"

	"github.com/ninjasphere/app-scheduler/controller"
	"github.com/ninjasphere/app-scheduler/model"
	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/support"
)

var (
	info = ninja.LoadModuleInfo("./package.json")
)

type SchedulerApp struct {
	support.AppSupport
	scheduler *controller.Scheduler
	model     *model.Schedule
}

func (a *SchedulerApp) Start(model *model.Schedule) error {
	if a.scheduler != nil {
		return fmt.Errorf("illegal state: scheduler is already running")
	}
	a.scheduler = &controller.Scheduler{}
	err := a.scheduler.Start(model)
	if err == nil {
		a.SendEvent("config", model)
	}
	return err
}

func (a *SchedulerApp) Stop() error {
	var err error
	if a.scheduler != nil {
		tmp := a.scheduler
		a.scheduler = nil
		err = tmp.Stop()
	}
	return err
}

func main() {
	app := &SchedulerApp{}
	err := app.Init(info)
	if err != nil {
		app.Log.Fatalf("failed to initialize app: %v", err)
	}

	err = app.Export(app)
	if err != nil {
		app.Log.Fatalf("failed to export app: %v", err)
	}

	support.WaitUntilSignal()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"fmt"
	"math/rand"
	"net/http"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/codegangsta/martini"
	"github.com/russross/blackfriday"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
)

var PostTemplate = template.Must(template.ParseFiles("public2/pagetempl.html"))
var HomeTemplate = template.Must(template.ParseFiles("public2/hometempl.html"))

type Post struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    time.Time
	Slug    string
	Title   string
	Type    string // Possible types [Comedy,Hardware,Mystery,Networking,Problem,Quirk]
	R1      string // Override of the Rec 1
	R2      string // Override of the Rec 2
}

type PostFormatted struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    string
	Slug    string
	Title   string
}

func main() {
	m := martini.Classic()
	m.Get("/post/:name", ReadPost)
	m.Get("/raw/:name", ReadRawPost)
	m.Post("/admin/new", PublishPost)
	m.Get("/admin/", Admin)
	m.Get("/", ListPosts)
	m.Get("/all", ListPosts)
	m.Get("/rss.xml", GetRSS)
	m.Get("/sitemap.xml", GetSitemap)
	m.Get("/admin/run_gc", Run_GC)
	m.Get("/admin/backup.tar", Producebackup)
	m.Get("/admin/remove/:name", RemovePost)
	m.Post("/admin/uploadfile", UploadFile)

	m.Get("/lessons/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/lessons/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/asset/:tag", ReadFile)

	m.Use(func(res http.ResponseWriter, req *http.Request) {
		res.Header().Add("Cache-Control", "public")
		res.Header().Add("X-Served-By", GimmeDC(res, req))
		res.Header().Add("X-Served-For", req.Header.Get("CF-RAY"))
	})

	m.Use(martini.Static("public2"))

	http.Handle("/", m)

	PostTitleCache = make(map[string]Post)
	appengine.Main()
}

func MigrateOldURLS(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	http.Redirect(rw, req, fmt.Sprintf("https://blog.benjojo.co.uk/post/%s-%s-%s-%s.md", params["year"], params["month"], params["day"], params["title"]), http.StatusMovedPermanently)
}

func ReadPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	// c := appengine.NewContext(r)
	// key := datastore.NewIncompleteKey(c, "Greeting", PostKey(c))
	// _, err := datastore.Put(c, key, &g)
	if len(PostTitleCache) == 0 {
		forceUpdatePostTitleCache(rw, req)
	}

	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)

	if err != nil {
		if fmt.Sprint(err) == "datastore: no such entity" {
			http.Error(rw, "This blog post cannot be found, Please check your URL", http.StatusNotFound)
			return
		}

		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	if post.Type != "" {
		findReccomendations(&post)
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	// post.Content = strings.Replace(string(postd), "\n", "\r\n\r\n", -1)
	post.Content = string(postd)
	output := blackfriday.MarkdownCommon([]byte(post.Content))
	lines := strings.Split(string(postd), "\n")
	layoutData := struct {
		Title   string
		Content string
		Date    string
		// Rec links at the bottom
		HasReccomendations bool
		FirstRecLink       string
		FirstTitle         string
		FirstYear          int
		SecondRecLink      string
		SecondTitle        string
		SecondYear         int
		RandomLink         string
		RandomTitle        string
		RandomYear         int
	}{
		Title:   lines[0],
		Content: string(output),
		Date:    post.Date.Format("Jan 2 2006"),
	}
	if post.R1 != "" && post.R2 != "" {
		layoutData.HasReccomendations = true
		layoutData.FirstRecLink = "/post/" + post.R1
		layoutData.FirstTitle = PostTitleCache[post.R1].Title
		layoutData.FirstYear = PostTitleCache[post.R1].Date.Year()
		layoutData.SecondRecLink = "/post/" + post.R2
		layoutData.SecondTitle = PostTitleCache[post.R2].Title
		layoutData.SecondYear = PostTitleCache[post.R2].Date.Year()

		for _, v := range PostTitleCache {
			if !strings.HasPrefix(v.Slug, "DRAFT") {
				layoutData.RandomLink = "/post/" + v.Slug
				layoutData.RandomTitle = PostTitleCache[v.Slug].Title
				layoutData.RandomYear = PostTitleCache[v.Slug].Date.Year()
				break
			}
		}

	}

	err = PostTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}
}

func findReccomendations(Incoming *Post) {
	rand.Seed(Incoming.Date.Unix())
	// Make an array of candidates
	Candidates := make([]string, 0)
	for _, v := range PostTitleCache {
		if v.Type == Incoming.Type {
			if v.Date.Unix() < Incoming.Date.Unix() {
				// If the post is older
				if !strings.HasPrefix("DRAFT-", v.Title) {
					Candidates = append(Candidates, v.Slug)
				}
			}
		}
	}
	if len(Candidates) < 2 {
		return
	}

	sort.Strings(Candidates)

	if Incoming.R1 == "" {
		Incoming.R1 = Candidates[rand.Intn(len(Candidates)-1)]
	}

	if Incoming.R2 == "" {
		Incoming.R2 = Candidates[rand.Intn(len(Candidates)-1)]
	}
}

func ReadRawPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	rw.Write(postd)
}

var PostTitleCache map[string]Post

func updatePostCache(Posts []Post) {
	for _, v := range Posts {
		PostTitleCache[v.Slug] = v
	}
}

func forceUpdatePostTitleCache(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	updatePostCache(posts)
}

func ListPosts(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	updatePostCache(posts)

	FormattedPosts := make([]PostFormatted, 0)

	for _, v := range posts {
		if !strings.HasPrefix(v.Slug, "DRAFT-") {
			newpost := PostFormatted{
				Author:  v.Author,
				Content: v.Content,
				Date:    v.Date.Format("2006-01-02 15:04:05"),
				Slug:    v.Slug,
				Title:   v.Title,
			}
			FormattedPosts = append(FormattedPosts, newpost)
		}
	}

	layoutData := struct {
		Posts []PostFormatted
	}{
		Posts: FormattedPosts,
	}

	err := HomeTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}

}
<commit_msg>Prevent two of the same reccomendations from showing up at once<commit_after>package main

import (
	"encoding/base64"
	"fmt"
	"math/rand"
	"net/http"
	"sort"
	"strings"
	"text/template"
	"time"

	"github.com/codegangsta/martini"
	"github.com/russross/blackfriday"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
)

var PostTemplate = template.Must(template.ParseFiles("public2/pagetempl.html"))
var HomeTemplate = template.Must(template.ParseFiles("public2/hometempl.html"))

type Post struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    time.Time
	Slug    string
	Title   string
	Type    string // Possible types [Comedy,Hardware,Mystery,Networking,Problem,Quirk]
	R1      string // Override of the Rec 1
	R2      string // Override of the Rec 2
}

type PostFormatted struct {
	Author  string
	Content string `datastore:",noindex"`
	Date    string
	Slug    string
	Title   string
}

func main() {
	m := martini.Classic()
	m.Get("/post/:name", ReadPost)
	m.Get("/raw/:name", ReadRawPost)
	m.Post("/admin/new", PublishPost)
	m.Get("/admin/", Admin)
	m.Get("/", ListPosts)
	m.Get("/all", ListPosts)
	m.Get("/rss.xml", GetRSS)
	m.Get("/sitemap.xml", GetSitemap)
	m.Get("/admin/run_gc", Run_GC)
	m.Get("/admin/backup.tar", Producebackup)
	m.Get("/admin/remove/:name", RemovePost)
	m.Post("/admin/uploadfile", UploadFile)

	m.Get("/lessons/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/lessons/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title", MigrateOldURLS)
	m.Get("/posts/errors/:year/:month/:day/:title/", MigrateOldURLS)
	m.Get("/asset/:tag", ReadFile)

	m.Use(func(res http.ResponseWriter, req *http.Request) {
		res.Header().Add("Cache-Control", "public")
		res.Header().Add("X-Served-By", GimmeDC(res, req))
		res.Header().Add("X-Served-For", req.Header.Get("CF-RAY"))
	})

	m.Use(martini.Static("public2"))

	http.Handle("/", m)

	PostTitleCache = make(map[string]Post)
	appengine.Main()
}

func MigrateOldURLS(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	http.Redirect(rw, req, fmt.Sprintf("https://blog.benjojo.co.uk/post/%s-%s-%s-%s.md", params["year"], params["month"], params["day"], params["title"]), http.StatusMovedPermanently)
}

func ReadPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	// c := appengine.NewContext(r)
	// key := datastore.NewIncompleteKey(c, "Greeting", PostKey(c))
	// _, err := datastore.Put(c, key, &g)
	if len(PostTitleCache) == 0 {
		forceUpdatePostTitleCache(rw, req)
	}

	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)

	if err != nil {
		if fmt.Sprint(err) == "datastore: no such entity" {
			http.Error(rw, "This blog post cannot be found, Please check your URL", http.StatusNotFound)
			return
		}

		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	if post.Type != "" {
		findReccomendations(&post)
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	// post.Content = strings.Replace(string(postd), "\n", "\r\n\r\n", -1)
	post.Content = string(postd)
	output := blackfriday.MarkdownCommon([]byte(post.Content))
	lines := strings.Split(string(postd), "\n")
	layoutData := struct {
		Title   string
		Content string
		Date    string
		// Rec links at the bottom
		HasReccomendations bool
		FirstRecLink       string
		FirstTitle         string
		FirstYear          int
		SecondRecLink      string
		SecondTitle        string
		SecondYear         int
		RandomLink         string
		RandomTitle        string
		RandomYear         int
	}{
		Title:   lines[0],
		Content: string(output),
		Date:    post.Date.Format("Jan 2 2006"),
	}
	if post.R1 != "" && post.R2 != "" {
		layoutData.HasReccomendations = true
		layoutData.FirstRecLink = "/post/" + post.R1
		layoutData.FirstTitle = PostTitleCache[post.R1].Title
		layoutData.FirstYear = PostTitleCache[post.R1].Date.Year()
		layoutData.SecondRecLink = "/post/" + post.R2
		layoutData.SecondTitle = PostTitleCache[post.R2].Title
		layoutData.SecondYear = PostTitleCache[post.R2].Date.Year()

		for _, v := range PostTitleCache {
			if !strings.HasPrefix(v.Slug, "DRAFT") {
				layoutData.RandomLink = "/post/" + v.Slug
				layoutData.RandomTitle = PostTitleCache[v.Slug].Title
				layoutData.RandomYear = PostTitleCache[v.Slug].Date.Year()
				break
			}
		}

	}

	err = PostTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}
}

func findReccomendations(Incoming *Post) {
	rand.Seed(Incoming.Date.Unix())
	// Make an array of candidates
	Candidates := make([]string, 0)
	for _, v := range PostTitleCache {
		if v.Type == Incoming.Type {
			if v.Date.Unix() < Incoming.Date.Unix() {
				// If the post is older
				if !strings.HasPrefix("DRAFT-", v.Title) {
					Candidates = append(Candidates, v.Slug)
				}
			}
		}
	}
	if len(Candidates) < 2 {
		return
	}

	sort.Strings(Candidates)

	if Incoming.R1 == "" {
		Incoming.R1 = Candidates[rand.Intn(len(Candidates)-1)]
	}
	if Incoming.R2 == "" {
		for i := 0; i < 100; i++ {
			Incoming.R2 = Candidates[rand.Intn(len(Candidates)-1)]
			if Incoming.R2 == Incoming.R1 {
				continue
			}
			break
		}
	}
}

func ReadRawPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)
	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	post := Post{}
	err := datastore.Get(c, k, &post)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}

	postd, _ := base64.StdEncoding.DecodeString(post.Content)
	rw.Write(postd)
}

var PostTitleCache map[string]Post

func updatePostCache(Posts []Post) {
	for _, v := range Posts {
		PostTitleCache[v.Slug] = v
	}
}

func forceUpdatePostTitleCache(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	updatePostCache(posts)
}

func ListPosts(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	q := datastore.NewQuery("Post").Order("-Date").Limit(100)
	posts := make([]Post, 0, 100)

	if _, err := q.GetAll(c, &posts); err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
		return
	}
	updatePostCache(posts)

	FormattedPosts := make([]PostFormatted, 0)

	for _, v := range posts {
		if !strings.HasPrefix(v.Slug, "DRAFT-") {
			newpost := PostFormatted{
				Author:  v.Author,
				Content: v.Content,
				Date:    v.Date.Format("2006-01-02 15:04:05"),
				Slug:    v.Slug,
				Title:   v.Title,
			}
			FormattedPosts = append(FormattedPosts, newpost)
		}
	}

	layoutData := struct {
		Posts []PostFormatted
	}{
		Posts: FormattedPosts,
	}

	err := HomeTemplate.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}

}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
	"github.com/syncthing/syncthing/discover"
	"github.com/syncthing/syncthing/protocol"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
)

const cacheLimitSeconds = 3600

var (
	lock       sync.Mutex
	queries    = 0
	announces  = 0
	answered   = 0
	limited    = 0
	unknowns   = 0
	debug      = false
	lruSize    = 1024
	limitAvg   = 1
	limitBurst = 10
	limiter    *lru.Cache
)

func main() {
	var listen string
	var timestamp bool
	var statsIntv int
	var statsFile string
	var dbDir string

	flag.StringVar(&listen, "listen", ":22026", "Listen address")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
	flag.IntVar(&statsIntv, "stats-intv", 0, "Statistics output interval (s)")
	flag.StringVar(&statsFile, "stats-file", "/var/discosrv/stats", "Statistics file name")
	flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
	flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
	flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
	flag.StringVar(&dbDir, "db-dir", "/var/discosrv/db", "Database directory")
	flag.Parse()

	limiter = lru.New(lruSize)

	log.SetOutput(os.Stdout)
	if !timestamp {
		log.SetFlags(0)
	}

	addr, _ := net.ResolveUDPAddr("udp", listen)
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatal(err)
	}

	parentDir := filepath.Dir(dbDir)
	if _, err := os.Stat(parentDir); err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(parentDir, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	db, err := leveldb.OpenFile(dbDir, &opt.Options{CachedOpenFiles: 32})
	if err != nil {
		log.Fatal(err)
	}

	statsLog, err := os.OpenFile(statsFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}

	if statsIntv > 0 {
		go logStats(statsLog, statsIntv)
	}

	go clean(statsLog, db)

	var buf = make([]byte, 1024)
	for {
		buf = buf[:cap(buf)]
		n, addr, err := conn.ReadFromUDP(buf)

		if limit(addr) {
			// Rate limit in effect for source
			continue
		}

		if err != nil {
			log.Fatal(err)
		}

		if n < 4 {
			log.Printf("Received short packet (%d bytes)", n)
			continue
		}

		buf = buf[:n]
		magic := binary.BigEndian.Uint32(buf)

		switch magic {
		case discover.AnnouncementMagic:
			handleAnnounceV2(db, addr, buf)

		case discover.QueryMagic:
			handleQueryV2(db, conn, addr, buf)

		default:
			lock.Lock()
			unknowns++
			lock.Unlock()
		}
	}
}

func limit(addr *net.UDPAddr) bool {
	key := addr.IP.String()

	lock.Lock()
	defer lock.Unlock()

	bkt, ok := limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			if debug {
				log.Println("Rate limit exceeded for", key)
			}
			limited++
			return true
		}
	} else {
		if debug {
			log.Println("New limiter for", key)
		}
		// One packet per ten seconds average rate, burst ten packets
		limiter.Add(key, ratelimit.NewBucket(10*time.Second/time.Duration(limitAvg), int64(limitBurst)))
	}

	return false
}

func handleAnnounceV2(db *leveldb.DB, addr *net.UDPAddr, buf []byte) {
	var pkt discover.Announce
	err := pkt.UnmarshalXDR(buf)
	if err != nil && err != io.EOF {
		log.Println("AnnounceV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	announces++
	lock.Unlock()

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}

	var addrs []address
	now := time.Now().Unix()
	for _, addr := range pkt.This.Addresses {
		tip := addr.IP
		if len(tip) == 0 {
			tip = ip
		}
		addrs = append(addrs, address{
			ip:   tip,
			port: addr.Port,
			seen: now,
		})
	}

	var id protocol.NodeID
	if len(pkt.This.ID) == 32 {
		// Raw node ID
		copy(id[:], pkt.This.ID)
	} else {
		id.UnmarshalText(pkt.This.ID)
	}

	update(db, id, addrs)
}

func handleQueryV2(db *leveldb.DB, conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
	var pkt discover.Query
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("QueryV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	var id protocol.NodeID
	if len(pkt.NodeID) == 32 {
		// Raw node ID
		copy(id[:], pkt.NodeID)
	} else {
		id.UnmarshalText(pkt.NodeID)
	}

	lock.Lock()
	queries++
	lock.Unlock()

	addrs := get(db, id)

	now := time.Now().Unix()
	if len(addrs) > 0 {
		ann := discover.Announce{
			Magic: discover.AnnouncementMagic,
			This: discover.Node{
				ID: pkt.NodeID,
			},
		}
		for _, addr := range addrs {
			if now-addr.seen > cacheLimitSeconds {
				continue
			}
			ann.This.Addresses = append(ann.This.Addresses, discover.Address{IP: addr.ip, Port: addr.port})
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		if len(ann.This.Addresses) == 0 {
			return
		}

		tb := ann.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV2 response write:", err)
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
}

func next(intv int) time.Time {
	d := time.Duration(intv) * time.Second
	t0 := time.Now()
	t1 := t0.Add(d).Truncate(d)
	time.Sleep(t1.Sub(t0))
	return t1
}

func logStats(statsLog io.Writer, intv int) {
	for {
		t := next(intv)

		lock.Lock()

		fmt.Fprintf(statsLog, "%d Queries:%d Answered:%d Announces:%d Unknown:%d Limited:%d\n",
			t.Unix(), queries, answered, announces, unknowns, limited)

		queries = 0
		announces = 0
		answered = 0
		limited = 0
		unknowns = 0

		lock.Unlock()
	}
}

func get(db *leveldb.DB, id protocol.NodeID) []address {
	var addrs addressList
	val, err := db.Get(id[:], nil)
	if err == nil {
		addrs.UnmarshalXDR(val)
	}
	return addrs.addresses
}

func update(db *leveldb.DB, id protocol.NodeID, addrs []address) {
	var newAddrs addressList

	val, err := db.Get(id[:], nil)
	if err == nil {
		newAddrs.UnmarshalXDR(val)
	}

nextAddr:
	for _, newAddr := range addrs {
		for i, exAddr := range newAddrs.addresses {
			if bytes.Compare(newAddr.ip, exAddr.ip) == 0 {
				newAddrs.addresses[i] = newAddr
				continue nextAddr
			}
		}
		newAddrs.addresses = append(newAddrs.addresses, newAddr)
	}

	db.Put(id[:], newAddrs.MarshalXDR(), nil)
}

func clean(statsLog io.Writer, db *leveldb.DB) {
	for {
		now := time.Now()
		nowSecs := now.Unix()

		var kept, deleted int64
		iter := db.NewIterator(nil, nil)
		for iter.Next() {
			var addrs addressList
			addrs.UnmarshalXDR(iter.Value())

			// Remove expired addresses
			newAddrs := addrs.addresses
			for i := 0; i < len(newAddrs); i++ {
				if nowSecs-newAddrs[i].seen > cacheLimitSeconds {
					newAddrs[i] = newAddrs[len(newAddrs)-1]
					newAddrs = newAddrs[:len(newAddrs)-1]
				}
			}

			// Delete empty records
			if len(newAddrs) == 0 {
				db.Delete(iter.Key(), nil)
				deleted++
				continue
			}

			// Update changed records
			if len(newAddrs) != len(addrs.addresses) {
				addrs.addresses = newAddrs
				db.Put(iter.Key(), addrs.MarshalXDR(), nil)
			}
			kept++
		}
		iter.Release()

		fmt.Fprintf(statsLog, "%d Kept:%d Deleted:%d Took:%0.04fs\n", nowSecs, kept, deleted, time.Since(now).Seconds())

		time.Sleep(cacheLimitSeconds * time.Second / 2)
	}
}
<commit_msg>Align cleaning routine in time<commit_after>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"flag"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/golang/groupcache/lru"
	"github.com/juju/ratelimit"
	"github.com/syncthing/syncthing/discover"
	"github.com/syncthing/syncthing/protocol"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
)

const cacheLimitSeconds = 3600

var (
	lock       sync.Mutex
	queries    = 0
	announces  = 0
	answered   = 0
	limited    = 0
	unknowns   = 0
	debug      = false
	lruSize    = 1024
	limitAvg   = 1
	limitBurst = 10
	limiter    *lru.Cache
)

func main() {
	var listen string
	var timestamp bool
	var statsIntv int
	var statsFile string
	var dbDir string

	flag.StringVar(&listen, "listen", ":22026", "Listen address")
	flag.BoolVar(&debug, "debug", false, "Enable debug output")
	flag.BoolVar(&timestamp, "timestamp", true, "Timestamp the log output")
	flag.IntVar(&statsIntv, "stats-intv", 0, "Statistics output interval (s)")
	flag.StringVar(&statsFile, "stats-file", "/var/discosrv/stats", "Statistics file name")
	flag.IntVar(&lruSize, "limit-cache", lruSize, "Limiter cache entries")
	flag.IntVar(&limitAvg, "limit-avg", limitAvg, "Allowed average package rate, per 10 s")
	flag.IntVar(&limitBurst, "limit-burst", limitBurst, "Allowed burst size, packets")
	flag.StringVar(&dbDir, "db-dir", "/var/discosrv/db", "Database directory")
	flag.Parse()

	limiter = lru.New(lruSize)

	log.SetOutput(os.Stdout)
	if !timestamp {
		log.SetFlags(0)
	}

	addr, _ := net.ResolveUDPAddr("udp", listen)
	conn, err := net.ListenUDP("udp", addr)
	if err != nil {
		log.Fatal(err)
	}

	parentDir := filepath.Dir(dbDir)
	if _, err := os.Stat(parentDir); err != nil && os.IsNotExist(err) {
		err = os.MkdirAll(parentDir, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	db, err := leveldb.OpenFile(dbDir, &opt.Options{CachedOpenFiles: 32})
	if err != nil {
		log.Fatal(err)
	}

	statsLog, err := os.OpenFile(statsFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		log.Fatal(err)
	}

	if statsIntv > 0 {
		go logStats(statsLog, statsIntv)
	}

	go clean(statsLog, db)

	var buf = make([]byte, 1024)
	for {
		buf = buf[:cap(buf)]
		n, addr, err := conn.ReadFromUDP(buf)

		if limit(addr) {
			// Rate limit in effect for source
			continue
		}

		if err != nil {
			log.Fatal(err)
		}

		if n < 4 {
			log.Printf("Received short packet (%d bytes)", n)
			continue
		}

		buf = buf[:n]
		magic := binary.BigEndian.Uint32(buf)

		switch magic {
		case discover.AnnouncementMagic:
			handleAnnounceV2(db, addr, buf)

		case discover.QueryMagic:
			handleQueryV2(db, conn, addr, buf)

		default:
			lock.Lock()
			unknowns++
			lock.Unlock()
		}
	}
}

func limit(addr *net.UDPAddr) bool {
	key := addr.IP.String()

	lock.Lock()
	defer lock.Unlock()

	bkt, ok := limiter.Get(key)
	if ok {
		bkt := bkt.(*ratelimit.Bucket)
		if bkt.TakeAvailable(1) != 1 {
			// Rate limit exceeded; ignore packet
			if debug {
				log.Println("Rate limit exceeded for", key)
			}
			limited++
			return true
		}
	} else {
		if debug {
			log.Println("New limiter for", key)
		}
		// One packet per ten seconds average rate, burst ten packets
		limiter.Add(key, ratelimit.NewBucket(10*time.Second/time.Duration(limitAvg), int64(limitBurst)))
	}

	return false
}

func handleAnnounceV2(db *leveldb.DB, addr *net.UDPAddr, buf []byte) {
	var pkt discover.Announce
	err := pkt.UnmarshalXDR(buf)
	if err != nil && err != io.EOF {
		log.Println("AnnounceV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	lock.Lock()
	announces++
	lock.Unlock()

	ip := addr.IP.To4()
	if ip == nil {
		ip = addr.IP.To16()
	}

	var addrs []address
	now := time.Now().Unix()
	for _, addr := range pkt.This.Addresses {
		tip := addr.IP
		if len(tip) == 0 {
			tip = ip
		}
		addrs = append(addrs, address{
			ip:   tip,
			port: addr.Port,
			seen: now,
		})
	}

	var id protocol.NodeID
	if len(pkt.This.ID) == 32 {
		// Raw node ID
		copy(id[:], pkt.This.ID)
	} else {
		id.UnmarshalText(pkt.This.ID)
	}

	update(db, id, addrs)
}

func handleQueryV2(db *leveldb.DB, conn *net.UDPConn, addr *net.UDPAddr, buf []byte) {
	var pkt discover.Query
	err := pkt.UnmarshalXDR(buf)
	if err != nil {
		log.Println("QueryV2 Unmarshal:", err)
		log.Println(hex.Dump(buf))
		return
	}
	if debug {
		log.Printf("<- %v %#v", addr, pkt)
	}

	var id protocol.NodeID
	if len(pkt.NodeID) == 32 {
		// Raw node ID
		copy(id[:], pkt.NodeID)
	} else {
		id.UnmarshalText(pkt.NodeID)
	}

	lock.Lock()
	queries++
	lock.Unlock()

	addrs := get(db, id)

	now := time.Now().Unix()
	if len(addrs) > 0 {
		ann := discover.Announce{
			Magic: discover.AnnouncementMagic,
			This: discover.Node{
				ID: pkt.NodeID,
			},
		}
		for _, addr := range addrs {
			if now-addr.seen > cacheLimitSeconds {
				continue
			}
			ann.This.Addresses = append(ann.This.Addresses, discover.Address{IP: addr.ip, Port: addr.port})
		}
		if debug {
			log.Printf("-> %v %#v", addr, pkt)
		}

		if len(ann.This.Addresses) == 0 {
			return
		}

		tb := ann.MarshalXDR()
		_, _, err = conn.WriteMsgUDP(tb, nil, addr)
		if err != nil {
			log.Println("QueryV2 response write:", err)
		}

		lock.Lock()
		answered++
		lock.Unlock()
	}
}

func next(intv int) time.Time {
	d := time.Duration(intv) * time.Second
	t0 := time.Now()
	t1 := t0.Add(d).Truncate(d)
	time.Sleep(t1.Sub(t0))
	return t1
}

func logStats(statsLog io.Writer, intv int) {
	for {
		t := next(intv)

		lock.Lock()

		fmt.Fprintf(statsLog, "%d Queries:%d Answered:%d Announces:%d Unknown:%d Limited:%d\n",
			t.Unix(), queries, answered, announces, unknowns, limited)

		queries = 0
		announces = 0
		answered = 0
		limited = 0
		unknowns = 0

		lock.Unlock()
	}
}

func get(db *leveldb.DB, id protocol.NodeID) []address {
	var addrs addressList
	val, err := db.Get(id[:], nil)
	if err == nil {
		addrs.UnmarshalXDR(val)
	}
	return addrs.addresses
}

func update(db *leveldb.DB, id protocol.NodeID, addrs []address) {
	var newAddrs addressList

	val, err := db.Get(id[:], nil)
	if err == nil {
		newAddrs.UnmarshalXDR(val)
	}

nextAddr:
	for _, newAddr := range addrs {
		for i, exAddr := range newAddrs.addresses {
			if bytes.Compare(newAddr.ip, exAddr.ip) == 0 {
				newAddrs.addresses[i] = newAddr
				continue nextAddr
			}
		}
		newAddrs.addresses = append(newAddrs.addresses, newAddr)
	}

	db.Put(id[:], newAddrs.MarshalXDR(), nil)
}

func clean(statsLog io.Writer, db *leveldb.DB) {
	for {
		now := next(cacheLimitSeconds)
		nowSecs := now.Unix()

		var kept, deleted int64
		iter := db.NewIterator(nil, nil)
		for iter.Next() {
			var addrs addressList
			addrs.UnmarshalXDR(iter.Value())

			// Remove expired addresses
			newAddrs := addrs.addresses
			for i := 0; i < len(newAddrs); i++ {
				if nowSecs-newAddrs[i].seen > cacheLimitSeconds {
					newAddrs[i] = newAddrs[len(newAddrs)-1]
					newAddrs = newAddrs[:len(newAddrs)-1]
				}
			}

			// Delete empty records
			if len(newAddrs) == 0 {
				db.Delete(iter.Key(), nil)
				deleted++
				continue
			}

			// Update changed records
			if len(newAddrs) != len(addrs.addresses) {
				addrs.addresses = newAddrs
				db.Put(iter.Key(), addrs.MarshalXDR(), nil)
			}
			kept++
		}
		iter.Release()

		fmt.Fprintf(statsLog, "%d Kept:%d Deleted:%d Took:%0.04fs\n", nowSecs, kept, deleted, time.Since(now).Seconds())
	}
}
<|endoftext|>