<commit_before>// Copyright (c) 2016 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package aa2d

import (
	"bytes"
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
)

const (
	// XScale is the default scale for the x-axis of a grid.
	XScale = 9
	// YScale is the default scale for the y-axis of a grid.
	YScale = 16
)

// A Parser for two-dimensional hierarchical ASCII art.
type Parser struct {
	xScale float64
	yScale float64
	refs   map[string]map[string]interface{}
}

// A Grid is an abstract representation of two-dimensional hierarchical ASCII
// art which various elements.
type Grid struct {
	W     float64                           // size of grid in x-dimension (width)
	H     float64                           // size of grid in y-dimension (height)
	Refs  map[string]map[string]interface{} // JSON references
	Elems []interface{}                     // list of elements on the grid
}

// The Rectangle element.
type Rectangle struct {
	X               float64       // x-axis coordinate
	Y               float64       // y-axis coordinate
	W               float64       // width of rectangle
	H               float64       // height of rectangle
	RoundUpperLeft  bool          // rounded upper-left corner
	RoundUpperRight bool          // rounded upper-right corner
	RoundLowerLeft  bool          // rounded lower-left corner
	RoundLowerRight bool          // rounded lower-right corner
	Ref             interface{}   // JSON reference of the rectangle, if def
	Elems           []interface{} // contained elements
}

// The Line element.
type Line struct {
	X1         float64                // x-axis coordinate of the start of the line
	Y1         float64                // y-axis coordinate of the start of the line
	X2         float64                // x-axis coordinate of the end of the line
	Y2         float64                // y-axis coordinate of the end of the line
	ArrowStart bool                   // arrow at the start of the line
	ArrowEnd   bool                   // arrow at the end of the line
	Dotted     bool                   // line is dotted
	Ref        map[string]interface{} // JSON reference of the line, if defined
}

// The Polyline element.
type Polyline struct {
	X      []float64              // x-axis coordinates of points on polyline
	Y      []float64              // y-axis coordinates of points on polyline
	Dotted []bool                 // polyline segment is dotted (len(Dotted) == len(X)-1)
	Ref    map[string]interface{} // JSON reference of the polyline, if defined
}

// The Polygon element.
type Polygon struct {
	X      []float64              // x-axis coordinates of points on polygon
	Y      []float64              // y-axis coordinates of points on polygon
	Dotted []bool                 // polygon segment is dotted (len(Dotted) == len(X))
	Ref    map[string]interface{} // JSON reference of the polygon, if defined
	Elems  []interface{}          // contained elements
}

// The Textline element.
type Textline struct {
	X    float64 // x-axis coordinate of the start of the text
	Y    float64 // y-axis coordinate of the start of the text
	Text string  // the actual text string
}

// NewParser returns a new parser for two-dimensional hierarchical ASCII art.
func NewParser() *Parser {
	return &Parser{
		xScale: XScale,
		yScale: YScale,
		refs:   make(map[string]map[string]interface{}),
	}
}

// SetScale sets the grid scale for parser p.
// xScale denotes the number of pixels to scale each unit on the x-axis to.
// yScale denotes the number of pixels to scale each unit on the y-axis to.
func (p *Parser) SetScale(xScale, yScale float64) error {
	if xScale <= 0 {
		return ErrWrongXScale
	}
	if yScale <= 0 {
		return ErrWrongYScale
	}
	p.xScale = xScale
	p.yScale = yScale
	return nil
}

// Parse parses asciiArt with parser p and returns a grid.
// If there is an error, it will be of type *ParseError.
func (p *Parser) Parse(asciiArt string) (*Grid, error) {
	var g Grid
	lines := bytes.Split([]byte(asciiArt), []byte("\n"))
	lines = removeEmptyTrailingLines(lines)
	for y, line := range lines {
		if len(line) > 0 && line[0] == '[' {
			if err := p.parseReference(&g, lines, y); err != nil {
				return nil, err
			}
			lines[y] = nil // remove reference line
		}
	}
	lines = removeEmptyTrailingLines(lines)
	var maxLen float64
	for _, line := range lines {
		if float64(len(line)) > maxLen {
			maxLen = float64(len(line))
		}
	}
	g.W = p.xScale * maxLen
	g.H = p.yScale * float64(len(lines))
	// add some extra space at the side for effects
	g.W += p.xScale
	g.H += p.yScale
	g.Elems = make([]interface{}, 0)
	if err := p.parseGrid(&g, lines); err != nil {
		return nil, err
	}
	return &g, nil
}

func removeEmptyTrailingLines(lines [][]byte) [][]byte {
	// remove empty lines at the end
	var rem int
	for i := len(lines) - 1; i >= 0; i-- {
		if len(lines[i]) == 0 {
			rem++
		} else {
			break
		}
	}
	return lines[:len(lines)-rem]
}

func (p *Parser) parseGrid(g *Grid, lines [][]byte) error {
	for y, line := range lines {
		if len(line) > 0 {
			switch line[0] {
			case '[':
				return fmt.Errorf("aa2d: reference on line %d defined in the "+
					"middle of document: %s", y, line)
			}
		}
	innerLoop:
		for x, cell := range line {
			switch cell {
			case '#':
				if err := p.parseRectangle(g, lines, x, y, false); err != nil {
					return err
				}
				goto innerLoop
			case '.':
				if err := p.parseRectangle(g, lines, x, y, true); err != nil {
					return err
				}
				goto innerLoop
			case ' ':
				continue
			default:
				return &ParseError{X: x, Y: y, Err: ErrUnknownCharacter}
			}
		}
	}
	return nil
}

func (p *Parser) parseRectangle(
	g *Grid,
	lines [][]byte,
	startX, startY int,
	roundUpperLeft bool,
) error {
	if startX+1 == len(lines[startY]) || lines[startY][startX+1] != '-' {
		return &ParseError{X: startX + 1, Y: startY, Err: ErrExpRecLine}
	}
	var r Rectangle
	r.X = float64(startX)
	r.Y = float64(startY)
	r.RoundUpperLeft = roundUpperLeft
	// go right
	x := startX + 2
	found := false
loopRight:
	for ; x < len(lines[startY]); x++ {
		switch lines[startY][x] {
		case '-':
			continue
		case '.':
			r.RoundUpperRight = true
			fallthrough
		case '#':
			r.W = float64(x) - r.X + 1
			found = true
			break loopRight
		default:
			return &ParseError{X: x, Y: startY, Err: ErrExpRecLineOrUpCorn}
		}
	}
	if !found {
		return &ParseError{X: x, Y: startY, Err: ErrNoRecUpRightCorn}
	}
	// go down
	y := startY + 1
	found = false
loopDown:
	for ; y < len(lines); y++ {
		if len(lines[y]) < x {
			return &ParseError{X: x, Y: y, Err: ErrExpRecLineOrLowCorn}
		}
		switch lines[y][x] {
		case '|':
			continue
		case '\'':
			r.RoundLowerRight = true
			fallthrough
		case '#':
			r.H = float64(y) - r.Y + 1
			found = true
			break loopDown
		default:
			return &ParseError{X: x, Y: y, Err: ErrExpRecLineOrLowCorn}
		}
	}
	if !found {
		return &ParseError{X: x, Y: y, Err: ErrNoRecLowRightCorn}
	}
	// go left
	x--
	for ; x > startX; x-- {
		switch lines[y][x] {
		case '-':
			continue
		default:
			return &ParseError{X: x, Y: y, Err: ErrExpRecHorizontalLine}
		}
	}
	switch lines[y][x] {
	case '\'':
		r.RoundLowerLeft = true
		fallthrough
	case '#':
		break
	default:
		return &ParseError{X: 0, Y: y, Err: ErrExpRecLowCorn}
	}
	// go up
	y--
	for ; y > startY; y-- {
		switch lines[y][startX] {
		case '|':
			continue
		default:
			return &ParseError{X: x, Y: y, Err: ErrExpRecVerticalLine}
		}
	}
	// remove rectangle
	r.remove(lines)
	// scale
	r.X = r.X*p.xScale + p.xScale/2
	r.Y = r.Y*p.yScale + p.yScale/2
	r.W = r.W*p.xScale - p.xScale
	r.H = r.H*p.yScale - p.yScale
	// add rectangle to grid
	g.Elems = append(g.Elems, r)
	return nil
}

func (r *Rectangle) remove(lines [][]byte) {
	// remove upper and lower line
	for x := int(r.X); x < int(r.X)+int(r.W); x++ {
		lines[int(r.Y)][x] = ' '
		lines[int(r.Y)+int(r.H)-1][x] = ' '
	}
	// remove side lines
	for y := int(r.Y) + 1; y < int(r.Y)+int(r.H)-1; y++ {
		lines[y][int(r.X)] = ' '
		lines[y][int(r.X)+int(r.W)-1] = ' '
	}
}

var matchReference = regexp.MustCompile(`^\[(.+)\]: (.*)$`)

func (p *Parser) parseReference(g *Grid, lines [][]byte, startY int) error {
	m := matchReference.FindStringSubmatch(string(lines[startY]))
	if m == nil {
		return fmt.Errorf("aa2d: cannot parse reference on line %d: %s",
			startY, string(lines[startY]))
	}
	key := m[1]
	jsn := m[2]
	var ref interface{}
	if err := json.Unmarshal([]byte(jsn), &ref); err != nil {
		return err
	}
	refMap, ok := ref.(map[string]interface{})
	if !ok {
		return fmt.Errorf("aa2d: reference on line %d is not a JSON object: %s",
			startY, jsn)
	}
	if strings.HasPrefix(key, "_") {
		if g.Refs == nil {
			g.Refs = make(map[string]map[string]interface{})
		}
		if g.Refs[key] != nil {
			return fmt.Errorf("aa2d: reference on line %d defined twice: %s",
				startY, key)
		}
		g.Refs[key] = refMap
	} else {
		if p.refs[key] != nil {
			return fmt.Errorf("aa2d: reference on line %d defined twice: %s",
				startY, key)
		}
		p.refs[key] = refMap
	}
	return nil
}
<commit_msg>define elem interface for composition<commit_after>// Copyright (c) 2016 Frank Braun <frank@cryptogroup.net>
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package aa2d

import (
	"bytes"
	"encoding/json"
	"fmt"
	"regexp"
	"strings"
)

const (
	// XScale is the default scale for the x-axis of a grid.
	XScale = 9
	// YScale is the default scale for the y-axis of a grid.
	YScale = 16
)

// A Parser for two-dimensional hierarchical ASCII art.
type Parser struct {
	xScale float64
	yScale float64
	refs   map[string]map[string]interface{}
}

// A Grid is an abstract representation of two-dimensional hierarchical ASCII
// art which various elements.
type Grid struct {
	W     float64                           // size of grid in x-dimension (width)
	H     float64                           // size of grid in y-dimension (height)
	Refs  map[string]map[string]interface{} // JSON references
	Elems []interface{}                     // list of elements on the grid
}

// The Rectangle element.
type Rectangle struct {
	X               float64       // x-axis coordinate
	Y               float64       // y-axis coordinate
	W               float64       // width of rectangle
	H               float64       // height of rectangle
	RoundUpperLeft  bool          // rounded upper-left corner
	RoundUpperRight bool          // rounded upper-right corner
	RoundLowerLeft  bool          // rounded lower-left corner
	RoundLowerRight bool          // rounded lower-right corner
	Ref             interface{}   // JSON reference of the rectangle, if def
	Elems           []interface{} // contained elements
}

// The Line element.
type Line struct {
	X1         float64                // x-axis coordinate of the start of the line
	Y1         float64                // y-axis coordinate of the start of the line
	X2         float64                // x-axis coordinate of the end of the line
	Y2         float64                // y-axis coordinate of the end of the line
	ArrowStart bool                   // arrow at the start of the line
	ArrowEnd   bool                   // arrow at the end of the line
	Dotted     bool                   // line is dotted
	Ref        map[string]interface{} // JSON reference of the line, if defined
}

// The Polyline element.
type Polyline struct {
	X      []float64              // x-axis coordinates of points on polyline
	Y      []float64              // y-axis coordinates of points on polyline
	Dotted []bool                 // polyline segment is dotted (len(Dotted) == len(X)-1)
	Ref    map[string]interface{} // JSON reference of the polyline, if defined
}

// The Polygon element.
type Polygon struct {
	X      []float64              // x-axis coordinates of points on polygon
	Y      []float64              // y-axis coordinates of points on polygon
	Dotted []bool                 // polygon segment is dotted (len(Dotted) == len(X))
	Ref    map[string]interface{} // JSON reference of the polygon, if defined
	Elems  []interface{}          // contained elements
}

// The Textline element.
type Textline struct {
	X    float64 // x-axis coordinate of the start of the text
	Y    float64 // y-axis coordinate of the start of the text
	Text string  // the actual text string
}

type elem interface {
	addElem(interface{})
}

func (g *Grid) addElem(e interface{}) {
	g.Elems = append(g.Elems, e)
}

// NewParser returns a new parser for two-dimensional hierarchical ASCII art.
func NewParser() *Parser {
	return &Parser{
		xScale: XScale,
		yScale: YScale,
		refs:   make(map[string]map[string]interface{}),
	}
}

// SetScale sets the grid scale for parser p.
// xScale denotes the number of pixels to scale each unit on the x-axis to.
// yScale denotes the number of pixels to scale each unit on the y-axis to.
func (p *Parser) SetScale(xScale, yScale float64) error {
	if xScale <= 0 {
		return ErrWrongXScale
	}
	if yScale <= 0 {
		return ErrWrongYScale
	}
	p.xScale = xScale
	p.yScale = yScale
	return nil
}

// Parse parses asciiArt with parser p and returns a grid.
// If there is an error, it will be of type *ParseError.
func (p *Parser) Parse(asciiArt string) (*Grid, error) {
	var g Grid
	lines := bytes.Split([]byte(asciiArt), []byte("\n"))
	lines = removeEmptyTrailingLines(lines)
	for y, line := range lines {
		if len(line) > 0 && line[0] == '[' {
			if err := p.parseReference(&g, lines, y); err != nil {
				return nil, err
			}
			lines[y] = nil // remove reference line
		}
	}
	lines = removeEmptyTrailingLines(lines)
	var maxLen float64
	for _, line := range lines {
		if float64(len(line)) > maxLen {
			maxLen = float64(len(line))
		}
	}
	g.W = p.xScale * maxLen
	g.H = p.yScale * float64(len(lines))
	// add some extra space at the side for effects
	g.W += p.xScale
	g.H += p.yScale
	g.Elems = make([]interface{}, 0)
	if err := p.parseGrid(&g, lines); err != nil {
		return nil, err
	}
	return &g, nil
}

func removeEmptyTrailingLines(lines [][]byte) [][]byte {
	// remove empty lines at the end
	var rem int
	for i := len(lines) - 1; i >= 0; i-- {
		if len(lines[i]) == 0 {
			rem++
		} else {
			break
		}
	}
	return lines[:len(lines)-rem]
}

func (p *Parser) parseGrid(g *Grid, lines [][]byte) error {
	for y, line := range lines {
		if len(line) > 0 {
			switch line[0] {
			case '[':
				return fmt.Errorf("aa2d: reference on line %d defined in the "+
					"middle of document: %s", y, line)
			}
		}
	innerLoop:
		for x, cell := range line {
			switch cell {
			case '#':
				if err := p.parseRectangle(g, lines, x, y, false); err != nil {
					return err
				}
				goto innerLoop
			case '.':
				if err := p.parseRectangle(g, lines, x, y, true); err != nil {
					return err
				}
				goto innerLoop
			case ' ':
				continue
			default:
				return &ParseError{X: x, Y: y, Err: ErrUnknownCharacter}
			}
		}
	}
	return nil
}

func (p *Parser) parseRectangle(
	parent elem,
	lines [][]byte,
	startX, startY int,
	roundUpperLeft bool,
) error {
	if startX+1 == len(lines[startY]) || lines[startY][startX+1] != '-' {
		return &ParseError{X: startX + 1, Y: startY, Err: ErrExpRecLine}
	}
	var r Rectangle
	r.X = float64(startX)
	r.Y = float64(startY)
	r.RoundUpperLeft = roundUpperLeft
	// go right
	x := startX + 2
	found := false
loopRight:
	for ; x < len(lines[startY]); x++ {
		switch lines[startY][x] {
		case '-':
			continue
		case '.':
			r.RoundUpperRight = true
			fallthrough
		case '#':
			r.W = float64(x) - r.X + 1
			found = true
			break loopRight
		default:
			return &ParseError{X: x, Y: startY, Err: ErrExpRecLineOrUpCorn}
		}
	}
	if !found {
		return &ParseError{X: x, Y: startY, Err: ErrNoRecUpRightCorn}
	}
	// go down
	y := startY + 1
	found = false
loopDown:
	for ; y < len(lines); y++ {
		if len(lines[y]) < x {
			return &ParseError{X: x, Y: y, Err: ErrExpRecLineOrLowCorn}
		}
		switch lines[y][x] {
		case '|':
			continue
		case '\'':
			r.RoundLowerRight = true
			fallthrough
		case '#':
			r.H = float64(y) - r.Y + 1
			found = true
			break loopDown
		default:
			return &ParseError{X: x, Y: y, Err: ErrExpRecLineOrLowCorn}
		}
	}
	if !found {
		return &ParseError{X: x, Y: y, Err: ErrNoRecLowRightCorn}
	}
	// go left
	x--
	for ; x > startX; x-- {
		switch lines[y][x] {
		case '-':
			continue
		default:
			return &ParseError{X: x, Y: y, Err: ErrExpRecHorizontalLine}
		}
	}
	switch lines[y][x] {
	case '\'':
		r.RoundLowerLeft = true
		fallthrough
	case '#':
		break
	default:
		return &ParseError{X: 0, Y: y, Err: ErrExpRecLowCorn}
	}
	// go up
	y--
	for ; y > startY; y-- {
		switch lines[y][startX] {
		case '|':
			continue
		default:
			return &ParseError{X: x, Y: y, Err: ErrExpRecVerticalLine}
		}
	}
	// remove rectangle
	r.remove(lines)
	// scale
	r.X = r.X*p.xScale + p.xScale/2
	r.Y = r.Y*p.yScale + p.yScale/2
	r.W = r.W*p.xScale - p.xScale
	r.H = r.H*p.yScale - p.yScale
	// add rectangle to parent
	parent.addElem(r)
	return nil
}

func (r *Rectangle) remove(lines [][]byte) {
	// remove upper and lower line
	for x := int(r.X); x < int(r.X)+int(r.W); x++ {
		lines[int(r.Y)][x] = ' '
		lines[int(r.Y)+int(r.H)-1][x] = ' '
	}
	// remove side lines
	for y := int(r.Y) + 1; y < int(r.Y)+int(r.H)-1; y++ {
		lines[y][int(r.X)] = ' '
		lines[y][int(r.X)+int(r.W)-1] = ' '
	}
}

var matchReference = regexp.MustCompile(`^\[(.+)\]: (.*)$`)

func (p *Parser) parseReference(g *Grid, lines [][]byte, startY int) error {
	m := matchReference.FindStringSubmatch(string(lines[startY]))
	if m == nil {
		return fmt.Errorf("aa2d: cannot parse reference on line %d: %s",
			startY, string(lines[startY]))
	}
	key := m[1]
	jsn := m[2]
	var ref interface{}
	if err := json.Unmarshal([]byte(jsn), &ref); err != nil {
		return err
	}
	refMap, ok := ref.(map[string]interface{})
	if !ok {
		return fmt.Errorf("aa2d: reference on line %d is not a JSON object: %s",
			startY, jsn)
	}
	if strings.HasPrefix(key, "_") {
		if g.Refs == nil {
			g.Refs = make(map[string]map[string]interface{})
		}
		if g.Refs[key] != nil {
			return fmt.Errorf("aa2d: reference on line %d defined twice: %s",
				startY, key)
		}
		g.Refs[key] = refMap
	} else {
		if p.refs[key] != nil {
			return fmt.Errorf("aa2d: reference on line %d defined twice: %s",
				startY, key)
		}
		p.refs[key] = refMap
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"time"
)

// recording pages filtered to access log

var accessLogName = flag.String("access-log", "", "path to access-log file")

// logChan is a channel for sending context objects, once processing is
// completed, to be logged in the access log.
var logChan = make(chan *context, 10)

// When a value is sent on logResetChan, the log file will be closed and
// reopened (for compatibility with logrotate).
var logResetChan = make(chan bool, 1)

// accessLog opens the log file and writes entries to it from logChan.
// It should be run in its own goroutine.
func accessLog() {
	var logfile = os.Stdout
	actualFile := false
	var err error
	if *accessLogName != "" {
		logfile, err = os.OpenFile(*accessLogName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
		if err != nil {
			log.Printf("Could not open access log file (%s): %s\n Sending access log messages to standard output instead.", *accessLogName, err)
			logfile = os.Stdout
		} else {
			actualFile = true
		}
	}

	for {
		select {
		case c := <-logChan:
			c.log(logfile)
		case <-logResetChan:
			if actualFile {
				logfile.Close()
				logfile, err = os.OpenFile(*accessLogName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
				if err != nil {
					log.Printf("Could not open access log file (%s): %s\n Sending access log messages to standard output instead.", *accessLogName, err)
					logfile = os.Stdout
					actualFile = false
				}
			}
		}
	}
}

// log writes a log entry (in CSV format) to w.
func (c *context) log(w io.Writer) {
	mode := "REQMOD"
	if c.resp != nil {
		mode = "RESPMOD"
	}

	modified := ""
	if c.modified {
		modified = "pruned"
	}

	fmt.Fprintf(w, "%q,%q,%q,%q,%q,%q,%d,%q,%q,%q,%q\n", time.Now().Format(time.Stamp), c.user, c.action, c.URL, mode, c.contentType, len(c.content), modified, listTally(c.stringTally()), listTally(c.scores), strings.Join(c.blocked, ", "))
}

// stringTally returns a copy of c.tally with strings instead of rules as keys.
func (c *context) stringTally() map[string]int {
	st := make(map[string]int)
	for r, n := range c.tally {
		st[r.String()] = n
	}
	return st
}

// listTally sorts the tally and formats it as a comma-separated string.
func listTally(tally map[string]int) string {
	b := new(bytes.Buffer)
	for i, rule := range sortedKeys(tally) {
		if i > 0 {
			b.WriteString(", ")
		}
		fmt.Fprint(b, rule, " ", tally[rule])
	}
	return b.String()
}

<commit_msg>Change date format in access log.<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"
	"time"
)

// recording pages filtered to access log

var accessLogName = flag.String("access-log", "", "path to access-log file")

// logChan is a channel for sending context objects, once processing is
// completed, to be logged in the access log.
var logChan = make(chan *context, 10)

// When a value is sent on logResetChan, the log file will be closed and
// reopened (for compatibility with logrotate).
var logResetChan = make(chan bool, 1)

// accessLog opens the log file and writes entries to it from logChan.
// It should be run in its own goroutine.
func accessLog() {
	var logfile = os.Stdout
	actualFile := false
	var err error
	if *accessLogName != "" {
		logfile, err = os.OpenFile(*accessLogName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
		if err != nil {
			log.Printf("Could not open access log file (%s): %s\n Sending access log messages to standard output instead.", *accessLogName, err)
			logfile = os.Stdout
		} else {
			actualFile = true
		}
	}

	for {
		select {
		case c := <-logChan:
			c.log(logfile)
		case <-logResetChan:
			if actualFile {
				logfile.Close()
				logfile, err = os.OpenFile(*accessLogName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
				if err != nil {
					log.Printf("Could not open access log file (%s): %s\n Sending access log messages to standard output instead.", *accessLogName, err)
					logfile = os.Stdout
					actualFile = false
				}
			}
		}
	}
}

// log writes a log entry (in CSV format) to w.
func (c *context) log(w io.Writer) {
	mode := "REQMOD"
	if c.resp != nil {
		mode = "RESPMOD"
	}

	modified := ""
	if c.modified {
		modified = "pruned"
	}

	fmt.Fprintf(w, "%q,%q,%q,%q,%q,%q,%d,%q,%q,%q,%q\n", time.Now().Format("2006-01-02 15:04:05"), c.user, c.action, c.URL, mode, c.contentType, len(c.content), modified, listTally(c.stringTally()), listTally(c.scores), strings.Join(c.blocked, ", "))
}

// stringTally returns a copy of c.tally with strings instead of rules as keys.
func (c *context) stringTally() map[string]int {
	st := make(map[string]int)
	for r, n := range c.tally {
		st[r.String()] = n
	}
	return st
}

// listTally sorts the tally and formats it as a comma-separated string.
func listTally(tally map[string]int) string {
	b := new(bytes.Buffer)
	for i, rule := range sortedKeys(tally) {
		if i > 0 {
			b.WriteString(", ")
		}
		fmt.Fprint(b, rule, " ", tally[rule])
	}
	return b.String()
}

<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

type Configuration struct {
	Token             string `json: "token"`
	Host              string `json: "host"`
	Port              int32  `json: "port"`
	WikiGitUrl        string `json: "wiki_git_url"`
	GitUsername       string `json: "git_username"`
	WikiChangelogPath string `json: "wiki_changelog_path"`
	WikiPath          string `json: "wiki_path"`
	VersionUrl        string `json: "version_url"`
}

type PullRequest struct {
	State          string `json: "state"`
	Body           string `json: "body"`
	MergeCommitSha string `json: "merge_commit_sha"`
	Title          string `json: "title"`
}

var configuration = Configuration{}

func parseGhPost(rw http.ResponseWriter, request *http.Request) {
	// When a POST is made, read the content and unmarshal it to a struc
	decoder := json.NewDecoder(request.Body)

	var pr PullRequest
	err := decoder.Decode(&pr)

	if err != nil {
		panic(err)
	}

	// if state is "closed" check "merge_commit_sha" if it's empty or not
	if pr.State == "closed" {
		if pr.MergeCommitSha != "" {
			// Read the "body" and split("\r\n") it into a list of string
			body_lines := strings.Split(pr.Body, "\r\n")

			// check if the project.wiki folder exist: If it doesn't exists git clone, if it exists
			// git pull
			_, err := os.Stat(configuration.WikiPath)

			if err != nil {
				// Wiki folder doesn't exists, we will have to clone from git
				clone_command := fmt.Sprintf("git clone https://%s:%s@%s",
					configuration.GitUsername, configuration.Token, configuration.WikiGitUrl)
				exec.Command("sh", "-c", clone_command)
			} else {
				// Wiki folder exists, we will have to pull to get updates
				pull_command := fmt.Sprintf("git -C %s pull", configuration.WikiPath)
				exec.Command("sh", "-c", pull_command)
			}

			// Check the VersionUrl and get the current released version of the software.
			// This number will be used to determinate if the current version is already in the
			// wiki changelog or if we have to add a new version section.
			version_resp, err := http.Get(configuration.VersionUrl)

			if err != nil {
				panic(err)
			}

			defer version_resp.Body.Close()
			version_content, err := ioutil.ReadAll(version_resp.Body)
			version := strings.Split(string(version_content), "\n")[0]

			// open fileToUpdate, read all the text and split by lines
			content, err := ioutil.ReadFile(configuration.WikiChangelogPath)

			if err != nil {
				panic(err)
			}

			wiki_lines := strings.Split(string(content), "\n")

			// Check if the released version is the same we have at the beginning of the wiki
			if wiki_lines[0] == fmt.Sprintf("## %s", version) {
				// Find the first empty line and insert the body_lines there
				for ln, line := range wiki_lines {
					if line == "" {
						first_part := wiki_lines[0 : ln-1]
						first_part = first_part[0 : len(first_part)+len(body_lines)]
					}
				}

			} else {
				// Add a new section with the new version at the beginning of the file
				fmt.Println(body_lines)
			}
		}
	}
}

func main() {
	// Read configuration from config.json
	file, _ := os.Open("config.json")
	decoder := json.NewDecoder(file)
	err := decoder.Decode(&configuration)

	if err != nil {
		fmt.Println("error:", err)
	}

	// Listen to the given host:port
	http.HandleFunc("/", parseGhPost)
	http.ListenAndServe(fmt.Sprintf("%s:%d", configuration.Host, configuration.Port), nil)
}
<commit_msg>Fix payload structure<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strings"
)

type Configuration struct {
	Token             string `json:"token"`
	Host              string `json:"host"`
	Port              int32  `json:"port"`
	WikiGitUrl        string `json:"wiki_git_url"`
	GitUsername       string `json:"git_username"`
	WikiChangelogPath string `json:"wiki_changelog_path"`
	WikiPath          string `json:"wiki_path"`
	VersionUrl        string `json:"version_url"`
}

type PullRequest struct {
	State          string `json:"state"`
	Body           string `json:"body"`
	MergeCommitSha string `json:"merge_commit_sha"`
	Title          string `json:"title"`
}

type PayLoad struct {
	Request PullRequest `json:"pull_request"`
}

var configuration = Configuration{}

func parseGhPost(rw http.ResponseWriter, request *http.Request) {
	// When a POST is made, read the content and unmarshal it to a struc
	payload_decoder := json.NewDecoder(request.Body)

	var payload PayLoad
	err_payload := payload_decoder.Decode(&payload)

	if err_payload != nil {
		panic(err_payload)
	}

	fmt.Println(payload)

	// if state is "closed" check "merge_commit_sha" if it's empty or not
	if payload.Request.State == "closed" {
		if payload.Request.MergeCommitSha != "" {
			// Read the "body" and split("\r\n") it into a list of string
			body_lines := strings.Split(payload.Request.Body, "\r\n")

			// check if the project.wiki folder exist: If it doesn't exists git clone, if it exists
			// git pull
			_, err := os.Stat(configuration.WikiPath)

			if err != nil {
				// Wiki folder doesn't exists, we will have to clone from git
				clone_command := fmt.Sprintf("git clone https://%s:%s@%s",
					configuration.GitUsername, configuration.Token, configuration.WikiGitUrl)
				exec.Command("sh", "-c", clone_command)
			} else {
				// Wiki folder exists, we will have to pull to get updates
				pull_command := fmt.Sprintf("git -C %s pull", configuration.WikiPath)
				exec.Command("sh", "-c", pull_command)
			}

			// Check the VersionUrl and get the current released version of the software.
			// This number will be used to determinate if the current version is already in the
			// wiki changelog or if we have to add a new version section.
			version_resp, err := http.Get(configuration.VersionUrl)

			if err != nil {
				panic(err)
			}

			defer version_resp.Body.Close()
			version_content, err := ioutil.ReadAll(version_resp.Body)
			version := strings.Split(string(version_content), "\n")[0]

			// open fileToUpdate, read all the text and split by lines
			content, err := ioutil.ReadFile(configuration.WikiChangelogPath)

			if err != nil {
				panic(err)
			}

			wiki_lines := strings.Split(string(content), "\n")

			// Check if the released version is the same we have at the beginning of the wiki
			if wiki_lines[0] == fmt.Sprintf("## %s", version) {
				// Find the first empty line and insert the body_lines there
				for ln, line := range wiki_lines {
					if line == "" {
						wiki_lines = append(wiki_lines[:ln],
							append(body_lines, wiki_lines[ln:]...)...)
						break
					}
				}

			} else {
				// Add a new section with the new version at the beginning of the file
				fmt.Println(body_lines)
			}

			fmt.Println(wiki_lines)
		}
	}
}

func main() {
	// Read configuration from config.json
	file, _ := os.Open("config.json")
	decoder := json.NewDecoder(file)
	err := decoder.Decode(&configuration)

	if err != nil {
		fmt.Println("error:", err)
	}

	// Listen to the given host:port
	http.HandleFunc("/", parseGhPost)
	http.ListenAndServe(fmt.Sprintf("%s:%d", configuration.Host, configuration.Port), nil)
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 - Michael Woolnough <michael.woolnough@gmail.com>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Package Minecraft will be a full featured minecraft level editor/viewer.
package minecraft

import "fmt"

type MissingTagError struct {
	tagName string
}

func (m MissingTagError) Error() string {
	return fmt.Sprintf("minecraft: missing %q tag", m.tagName)
}

type WrongTypeError struct {
	tagName        string
	expecting, got nbt.TagId
}

func (w WrongTypeError) Error() string {
	return fmt.Sprintf("minecraft: tag %q is of incorrect type, expecting %q, got %q", w.tagName, w.expecting, w.got)
}

type OOB struct{}

func (o OOB) Error() string {
	return "Received Out-of-bounds error"
}

type UnexpectedValue struct {
	tagName, expecting, got string
}

func (u UnexpectedValue) Error() string {
	return fmt.Sprintf("minecraft: tag %q was expecting %s, but got %q", u.tagName, u.expecting, u.got)
}

var (
	transparentBlocks []uint16
	lightBlocks       map[uint8]uint8
)

func init() {
	transparentBlocks = []uint16{0, 6, 18, 20, 26, 27, 28, 29, 30, 31, 33, 34, 37, 38, 39, 40, 50, 51, 52, 54, 55, 59, 63, 64, 65, 66, 69, 70, 71, 75, 76, 77, 78, 79, 81, 83, 85, 90, 92, 93, 94, 96, 102, 106, 107, 117, 118, 119, 120, 750}
	lightBlocks = map[uint8]uint8{
		10:  15,
		11:  15,
		39:  1,
		50:  14,
		51:  15,
		62:  13,
		74:  13,
		76:  7,
		89:  15,
		90:  11,
		91:  15,
		94:  9,
		95:  15,
		117: 1,
		119: 15,
		120: 1,
		122: 1,
		124: 16,
		130: 15,
		138: 15,
	}
}
<commit_msg>Added missing import<commit_after>// Copyright (c) 2013 - Michael Woolnough <michael.woolnough@gmail.com>
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Package Minecraft will be a full featured minecraft level editor/viewer.
package minecraft

import (
	"fmt"
	"github.com/MJKWoolnough/minecraft/nbt"
)

type MissingTagError struct {
	tagName string
}

func (m MissingTagError) Error() string {
	return fmt.Sprintf("minecraft: missing %q tag", m.tagName)
}

type WrongTypeError struct {
	tagName        string
	expecting, got nbt.TagId
}

func (w WrongTypeError) Error() string {
	return fmt.Sprintf("minecraft: tag %q is of incorrect type, expecting %q, got %q", w.tagName, w.expecting, w.got)
}

type OOB struct{}

func (o OOB) Error() string {
	return "Received Out-of-bounds error"
}

type UnexpectedValue struct {
	tagName, expecting, got string
}

func (u UnexpectedValue) Error() string {
	return fmt.Sprintf("minecraft: tag %q was expecting %s, but got %q", u.tagName, u.expecting, u.got)
}

var (
	transparentBlocks []uint16
	lightBlocks       map[uint8]uint8
)

func init() {
	transparentBlocks = []uint16{0, 6, 18, 20, 26, 27, 28, 29, 30, 31, 33, 34, 37, 38, 39, 40, 50, 51, 52, 54, 55, 59, 63, 64, 65, 66, 69, 70, 71, 75, 76, 77, 78, 79, 81, 83, 85, 90, 92, 93, 94, 96, 102, 106, 107, 117, 118, 119, 120, 750}
	lightBlocks = map[uint8]uint8{
		10:  15,
		11:  15,
		39:  1,
		50:  14,
		51:  15,
		62:  13,
		74:  13,
		76:  7,
		89:  15,
		90:  11,
		91:  15,
		94:  9,
		95:  15,
		117: 1,
		119: 15,
		120: 1,
		122: 1,
		124: 16,
		130: 15,
		138: 15,
	}
}
<|endoftext|>


<commit_before>package lnwire

import (
	"fmt"
	"net"

	"github.com/roasbeef/btcd/btcec"
	"github.com/roasbeef/btcd/wire"
)

// ServiceFlag identifies the services supported by a particular Lightning
// Network peer.
type ServiceFlag uint64

// NetAddress represents information pertaining to the identity and network
// reachability of a peer. Information stored includes the node's identity
// public key for establishing a confidential+authenticated connection, the
// service bits it supports, and a TCP address the node is reachable at.
//
// TODO(roasbeef): merge with LinkNode in some fashion
type NetAddress struct {
	// IdentityKey is the long-term static public key for a node. This node is
	// used throughout the network as a node's identity key. It is used to
	// authenticate any data sent to the network on behalf of the node, and
	// additionally to establish a confidential+authenticated connection with
	// the node.
	IdentityKey *btcec.PublicKey

	// Services defines the set of services supported by the node reachable at
	// the address and identity key defined in the struct.
	Services ServiceFlag

	// Address is is the IP address and port of the node.
	Address *net.TCPAddr

	// ChainNet is the Bitcoin network this node is associated with.
	// TODO(roasbeef): make a slice in the future for multi-chain
	ChainNet wire.BitcoinNet
}

// String returns a human readable string describing the target NetAddress. The
// current string format is: <pubkey>@host.
func (n *NetAddress) String() string {
	// TODO(roasbeef): use base58?
	pubkey := n.IdentityKey.SerializeCompressed()

	return fmt.Sprintf("%x@%v", pubkey, n.Address)
}
<commit_msg>lnwire: modify NetAddress to implement the net.Addr interface<commit_after>package lnwire

import (
	"fmt"
	"net"

	"github.com/roasbeef/btcd/btcec"
	"github.com/roasbeef/btcd/wire"
)

// ServiceFlag identifies the services supported by a particular Lightning
// Network peer.
type ServiceFlag uint64

// NetAddress represents information pertaining to the identity and network
// reachability of a peer. Information stored includes the node's identity
// public key for establishing a confidential+authenticated connection, the
// service bits it supports, and a TCP address the node is reachable at.
//
// TODO(roasbeef): merge with LinkNode in some fashion
type NetAddress struct {
	// IdentityKey is the long-term static public key for a node. This node is
	// used throughout the network as a node's identity key. It is used to
	// authenticate any data sent to the network on behalf of the node, and
	// additionally to establish a confidential+authenticated connection with
	// the node.
	IdentityKey *btcec.PublicKey

	// Services defines the set of services supported by the node reachable at
	// the address and identity key defined in the struct.
	Services ServiceFlag

	// Address is is the IP address and port of the node.
	Address *net.TCPAddr

	// ChainNet is the Bitcoin network this node is associated with.
	// TODO(roasbeef): make a slice in the future for multi-chain
	ChainNet wire.BitcoinNet
}

// A compile time assertion to ensure that NetAddress meets the net.Addr
// interface.
var _ net.Addr = (*NetAddress)(nil)

// String returns a human readable string describing the target NetAddress. The
// current string format is: <pubkey>@host.
//
// This part of the net.Addr interface.
func (n *NetAddress) String() string {
	// TODO(roasbeef): use base58?
	pubkey := n.IdentityKey.SerializeCompressed()

	return fmt.Sprintf("%x@%v", pubkey, n.Address)
}

// Network returns the name of the network this address is binded to.
//
// This part of the net.Addr interface.
func (n *NetAddress) Network() string {
	return n.Address.Network()
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gitindex

import (
	"bytes"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

func CloneRepos(destDir string, repos map[string]string) error {
	for name, cloneURL := range repos {
		parent := filepath.Join(destDir, filepath.Dir(name))
		if err := os.MkdirAll(parent, 0755); err != nil {
			return err
		}

		repoDest := filepath.Join(parent, filepath.Base(name)+".git")
		if _, err := os.Lstat(repoDest); err == nil {
			continue
		}

		cmd := exec.Command("git", "clone", "--bare", "--verbose", "--progress", "--recursive", cloneURL, repoDest)
		// Prevent prompting
		cmd.Stdin = &bytes.Buffer{}
		log.Println("running:", cmd.Args)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Set a fetch spec for cloned repositories.<commit_after>// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gitindex

import (
	"bytes"
	"log"
	"os"
	"os/exec"
	"path/filepath"
)

func CloneRepos(destDir string, repos map[string]string) error {
	for name, cloneURL := range repos {
		parent := filepath.Join(destDir, filepath.Dir(name))
		if err := os.MkdirAll(parent, 0755); err != nil {
			return err
		}

		repoDest := filepath.Join(parent, filepath.Base(name)+".git")
		if _, err := os.Lstat(repoDest); err == nil {
			continue
		}

		cmd := exec.Command(
			"git", "clone", "--bare", "--verbose", "--progress",
			// Only fetch branch heads, and ignore note branches.
			"--config", "remote.origin.fetch=+refs/heads/*:refs/heads/*",
			cloneURL, repoDest)
		// Prevent prompting
		cmd.Stdin = &bytes.Buffer{}
		log.Println("running:", cmd.Args)
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Rion Carter
// Licensed under the terms of the GPLv3
package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"github.com/miekg/dns"
	"io/ioutil"
	"scratches/dnsresolver/nmaps"
	"strings"
)

func main() {
	nmapFilePath := flag.String("NmapFile", "", "Pass in the full path to the NMAP XML file here")
	testDomainsPath := flag.String("DomainsFile", "", "Pass in the list of newline-delimited domains that are to be tested")
	flag.Parse()

	// Read in the specified nmap file
	nmapXml, _ := ioutil.ReadFile(*nmapFilePath)

	// Unmarshal the XML
	var nmapRun nmaps.Nmaprun
	xml.Unmarshal(nmapXml, &nmapRun)

	// Find DNS servers
	var dnsServers []string
	for _, host := range nmapRun.Host{
		for _, port := range host.Ports.Port{
			if port.Portid == "53" && port.State.State == "open" {
				dnsServers = append(dnsServers, host.Address.Addr)
			}
		}
	}

	// Get the list of domains to test
	//var domains []string
	domainsRaw, _ := ioutil.ReadFile(*testDomainsPath)
	domains := strings.Split(strings.Replace(string(domainsRaw), "\r", "", -1), "\n")

	fmt.Println("These DNS servers were identified:")
	fmt.Println(dnsServers)

	fmt.Println("Testing specified domain names against the dns resolvers")
	var responses []DnsResponse
	for _, dns := range dnsServers{
		for _, domain := range domains{
			response := testDns(dns, domain)
			responses = append(responses, response)
		}
	}

	// Check to see if every response has the same number of response and if the results are consistent:
	dnsDifferences := make(map[string]DnsDifferenceReport)
	for _, response := range responses{
		// Check against all others _EXCEPT_ this same DnsResolver/Domain pair
		for _, otherResponse := range responses{
			if !(response.Dns == otherResponse.Dns && response.Domain == otherResponse.Domain) &&
				(response.Domain == otherResponse.Domain){
				// Not checking against itself (checking domain as represented by another dns resolver) so continue

				if len(response.DnsResponses) == len(otherResponse.DnsResponses){
					// Check each response string to see if there are any differences
					for _,dnsResponse := range response.DnsResponses {

						if response.Domain == otherResponse.Domain {
							match := false
							for _, otherDnsResponse := range otherResponse.DnsResponses {
								if dnsResponse == otherDnsResponse {
									match = true
								}
							}

							if !match {
								// Once we get a proposed 'insertion index', we need to check the string at that index
								// to see if it already exists or not. If the strings don't match, we have a difference
								dnsDifferences[response.Domain] = DnsDifferenceReport{
									Domain:      response.Domain,
									Differences: []DnsResponse{response, otherResponse},
								}
							}
						}
					}

				} else {
					// No need to check further: There's a difference!
					dnsDifferences[response.Domain] = DnsDifferenceReport{
						Domain:response.Domain,
						Differences:[]DnsResponse{response, otherResponse},
					}
				}
			}
		}
	}

	// JSON Format the differences
	marshaledDifferences, _ := json.MarshalIndent(dnsDifferences, "", " ")
	fmt.Print(string(marshaledDifferences))
	fmt.Println("\nDone")
}

type DnsResponse struct{
	Dns          string
	Domain       string
	DnsResponses []string
}

type DnsDifferenceReport struct{
	Domain string
	Differences []DnsResponse
}

func testDns(dnsResolver string, domain string) DnsResponse{
	dnsClient := dns.Client{}

	// Prepare the DNS message (append a period at the end if one is not already present
	if !strings.HasSuffix(domain, "."){
		domain = domain + "."
	}
	dnsRequest := &dns.Msg{}
	dnsRequest.SetQuestion(domain, dns.TypeA)

	response, _, err := dnsClient.Exchange(dnsRequest, dnsResolver + ":53")
	if err!= nil{
		fmt.Println(err)
	}
	//fmt.Print("Timing: ", t, "\n", "Response:", response)
	var responses []string
	for _, answer := range response.Answer{
		//fmt.Println(answer.String())
		// We need to grab the first, 3rd, 4th and 5th entries
		pieces := strings.Split(answer.String(), "\t")
		responses = append(responses, pieces[0]+"\t"+pieces[2]+"\t"+pieces[3]+"\t"+pieces[4])
	}

	return DnsResponse{
		Dns:          dnsResolver,
		Domain:       domain,
		DnsResponses: responses,
	}
}
<commit_msg>Adding verbosity flag<commit_after>// Copyright 2018 Rion Carter
// Licensed under the terms of the GPLv3
package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"github.com/miekg/dns"
	"io/ioutil"
	"scratches/dnsresolver/nmaps"
	"strings"
)

func main() {
	nmapFilePath := flag.String("NmapFile", "", "Pass in the full path to the NMAP XML file here")
	testDomainsPath := flag.String("DomainsFile", "", "Pass in the list of newline-delimited domains that are to be tested")
	verbose := flag.Bool("Verbose", false, "Pass in 'true' to get more verbose output")
	flag.Parse()

	// Read in the specified nmap file
	nmapXml, _ := ioutil.ReadFile(*nmapFilePath)

	// Unmarshal the XML
	var nmapRun nmaps.Nmaprun
	xml.Unmarshal(nmapXml, &nmapRun)

	// Find DNS servers
	var dnsServers []string
	for _, host := range nmapRun.Host{
		for _, port := range host.Ports.Port{
			if port.Portid == "53" && port.State.State == "open"{
				dnsServers = append(dnsServers, host.Address.Addr)
			}
		}
	}

	// Get the list of domains to test
	//var domains []string
	domainsRaw, _ := ioutil.ReadFile(*testDomainsPath)
	domains := strings.Split(strings.Replace(string(domainsRaw), "\r", "", -1), "\n")

	if *verbose {
		fmt.Println("These DNS servers were identified:")
		fmt.Println(dnsServers)

		fmt.Println("Testing specified domain names against the dns resolvers")
	}
	var responses []DnsResponse
	for _, dns := range dnsServers{
		for _, domain := range domains{
			response := testDns(dns, domain)
			responses = append(responses, response)
		}
	}

	// Check to see if every response has the same number of response and if the results are consistent:
	dnsDifferences := make(map[string]DnsDifferenceReport)
	for _, response := range responses{
		// Check against all others _EXCEPT_ this same DnsResolver/Domain pair
		for _, otherResponse := range responses{
			if !(response.Dns == otherResponse.Dns && response.Domain == otherResponse.Domain) &&
				(response.Domain == otherResponse.Domain){
				// Not checking against itself (checking domain as represented by another dns resolver) so continue

				if len(response.DnsResponses) == len(otherResponse.DnsResponses){
					// Check each response string to see if there are any differences
					for _,dnsResponse := range response.DnsResponses {

						if response.Domain == otherResponse.Domain {
							match := false
							for _, otherDnsResponse := range otherResponse.DnsResponses {
								if dnsResponse == otherDnsResponse {
									match = true
								}
							}

							if !match {
								// Once we get a proposed 'insertion index', we need to check the string at that index
								// to see if it already exists or not. If the strings don't match, we have a difference
								dnsDifferences[response.Domain] = DnsDifferenceReport{
									Domain:      response.Domain,
									Differences: []DnsResponse{response, otherResponse},
								}
							}
						}
					}

				} else {
					// No need to check further: There's a difference!
					dnsDifferences[response.Domain] = DnsDifferenceReport{
						Domain:response.Domain,
						Differences:[]DnsResponse{response, otherResponse},
					}
				}
			}
		}
	}

	// JSON Format the differences
	marshaledDifferences, _ := json.MarshalIndent(dnsDifferences, "", " ")
	fmt.Print(string(marshaledDifferences))

	if *verbose{
		fmt.Println("\nDone")
	}
}

type DnsResponse struct{
	Dns          string
	Domain       string
	DnsResponses []string
}

type DnsDifferenceReport struct{
	Domain string
	Differences []DnsResponse
}

func testDns(dnsResolver string, domain string) DnsResponse{
	dnsClient := dns.Client{}

	// Prepare the DNS message (append a period at the end if one is not already present
	if !strings.HasSuffix(domain, "."){
		domain = domain + "."
	}
	dnsRequest := &dns.Msg{}
	dnsRequest.SetQuestion(domain, dns.TypeA)

	response, _, err := dnsClient.Exchange(dnsRequest, dnsResolver + ":53")
	if err!= nil{
		fmt.Println(err)
	}
	//fmt.Print("Timing: ", t, "\n", "Response:", response)
	var responses []string
	for _, answer := range response.Answer{
		//fmt.Println(answer.String())
		// We need to grab the first, 3rd, 4th and 5th entries
		pieces := strings.Split(answer.String(), "\t")
		responses = append(responses, pieces[0]+"\t"+pieces[2]+"\t"+pieces[3]+"\t"+pieces[4])
	}

	return DnsResponse{
		Dns:          dnsResolver,
		Domain:       domain,
		DnsResponses: responses,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/spf13/cobra"

	"github.com/lxc/lxd/client"
)

type cmdCallhook struct {
	global *cmdGlobal
}

func (c *cmdCallhook) Command() *cobra.Command {
	cmd := &cobra.Command{}
	cmd.Use = "callhook <path> <id> <state>"
	cmd.Short = "Call container lifecycle hook in LXD"
	cmd.Long = `Description:
  Call container lifecycle hook in LXD

  This internal command notifies LXD about a container lifecycle event
  (start, stop, restart) and blocks until LXD has processed it.

`
	cmd.RunE = c.Run
	cmd.Hidden = true

	return cmd
}

func (c *cmdCallhook) Run(cmd *cobra.Command, args []string) error {
	// Sanity checks
	if len(args) < 2 {
		cmd.Help()

		if len(args) == 0 {
			return nil
		}

		return fmt.Errorf("Missing required arguments")
	}

	path := args[0]
	id := args[1]
	state := args[2]
	target := ""

	// Only root should run this
	if os.Geteuid() != 0 {
		return fmt.Errorf("This must be run as root")
	}

	// Connect to LXD
	d, err := lxd.ConnectLXDUnix(fmt.Sprintf("%s/unix.socket", path), nil)
	if err != nil {
		return err
	}

	// Prepare the request URL
	url := fmt.Sprintf("/internal/containers/%s/on%s", id, state)
	if state == "stop" {
		target = os.Getenv("LXC_TARGET")
		if target == "" {
			target = "unknown"
		}
		url = fmt.Sprintf("%s?target=%s", url, target)
	}

	// Setup the request
	hook := make(chan error, 1)
	go func() {
		_, _, err := d.RawQuery("GET", url, nil, "")
		if err != nil {
			hook <- err
			return
		}

		hook <- nil
	}()

	// Handle the timeout
	select {
	case err := <-hook:
		if err != nil {
			return err
		}
		break
	case <-time.After(30 * time.Second):
		return fmt.Errorf("Hook didn't finish within 30s")
	}

	if target == "reboot" {
		return fmt.Errorf("Reboot must be handled by LXD.")
	}

	return nil
}
<commit_msg>lxd/callhook: Respect LXD_SOCKET environment variable<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/spf13/cobra"

	"github.com/lxc/lxd/client"
)

type cmdCallhook struct {
	global *cmdGlobal
}

func (c *cmdCallhook) Command() *cobra.Command {
	cmd := &cobra.Command{}
	cmd.Use = "callhook <path> <id> <state>"
	cmd.Short = "Call container lifecycle hook in LXD"
	cmd.Long = `Description:
  Call container lifecycle hook in LXD

  This internal command notifies LXD about a container lifecycle event
  (start, stop, restart) and blocks until LXD has processed it.

`
	cmd.RunE = c.Run
	cmd.Hidden = true

	return cmd
}

func (c *cmdCallhook) Run(cmd *cobra.Command, args []string) error {
	// Sanity checks
	if len(args) < 2 {
		cmd.Help()

		if len(args) == 0 {
			return nil
		}

		return fmt.Errorf("Missing required arguments")
	}

	path := args[0]
	id := args[1]
	state := args[2]
	target := ""

	// Only root should run this
	if os.Geteuid() != 0 {
		return fmt.Errorf("This must be run as root")
	}

	// Connect to LXD
	socket := os.Getenv("LXD_SOCKET")
	if socket == "" {
		socket = filepath.Join(path, "unix.socket")
	}
	d, err := lxd.ConnectLXDUnix(socket, nil)
	if err != nil {
		return err
	}

	// Prepare the request URL
	url := fmt.Sprintf("/internal/containers/%s/on%s", id, state)
	if state == "stop" {
		target = os.Getenv("LXC_TARGET")
		if target == "" {
			target = "unknown"
		}
		url = fmt.Sprintf("%s?target=%s", url, target)
	}

	// Setup the request
	hook := make(chan error, 1)
	go func() {
		_, _, err := d.RawQuery("GET", url, nil, "")
		if err != nil {
			hook <- err
			return
		}

		hook <- nil
	}()

	// Handle the timeout
	select {
	case err := <-hook:
		if err != nil {
			return err
		}
		break
	case <-time.After(30 * time.Second):
		return fmt.Errorf("Hook didn't finish within 30s")
	}

	if target == "reboot" {
		return fmt.Errorf("Reboot must be handled by LXD.")
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	// zmq "github.com/alecthomas/gozmq"
	trisclient "github.com/fvbock/tris/client"
	trisserver "github.com/fvbock/tris/server"
	"github.com/sbinet/liner"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
)

// TODO: detect connection loss and handle reconnect + currentDB

var (
	term      *liner.State = nil
	dsnString              = flag.String("d", "tcp:localhost:6000", "dsn to connect to")
)

func init() {
	fmt.Printf("TriS cli %s\n", trisclient.VERSION)
	term = liner.NewLiner()

	fname := path.Join(os.Getenv("HOME"), ".tris.history")
	f, err := os.Open(fname)
	if err != nil {
		fmt.Printf("**warning: could not access history file [%s]\n", fname)
		return
	}
	defer f.Close()
	_, err = term.ReadHistory(f)
	if err != nil {
		fmt.Printf("**warning: could not read history file [%s]\n", fname)
		return
	}
}

func atexit() {
	fmt.Println("CLI atexit")
	fname := path.Join(os.Getenv("HOME"), ".tris.history")
	f, err := os.OpenFile(fname, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		fmt.Printf("**warning: could not access history file [%s]\n", fname)
		return
	}
	defer f.Close()
	_, err = term.WriteHistory(f)
	if err != nil {
		fmt.Printf("**warning: could not write history file [%s]\n", fname)
		return
	}

	err = term.Close()
	if err != nil {
		fmt.Printf("**warning: problem closing term: %v\n", err)
		return
	}
	fmt.Println("\n")
}

func main() {
	defer atexit()
	var ierr error = nil // previous interpreter error
	ps1 := "[not connected]> "
	// ps2 := "...  "
	prompt := &ps1
	command := ""

	// TRIS conn
	flag.Parse()
	dsnParts := strings.Split(*dsnString, ":")
	port, _ := strconv.ParseInt(dsnParts[2], 10, 32)
	dsn := &trisclient.DSN{
		Protocol: dsnParts[0],
		Host:     dsnParts[1],
		Port:     int(port),
	}
	fmt.Printf("Connecting to %s://%s:%v\n", dsn.Protocol, dsn.Host, dsn.Port)

	client, err := trisclient.NewClient(dsn)
	err = client.Dial()
	if err != nil {
		fmt.Println(command, ierr)
	} else {
		ps1 = fmt.Sprintf("%s:%v/[%s]> ", dsnParts[1], dsnParts[2], client.ActiveDb)
	}
	defer client.Close()

	// check conn
	// r, err := client.Send("PING")
	// if err != nil {
	// 	fmt.Println("Error:", err)
	// }
	response, err := client.Ping()
	// response := trisserver.Unserialize(r)
	if response.ReturnCode != trisserver.COMMAND_OK {
		fmt.Printf("Initial PING failed:\n%v\n", response)
	} else {
		for {
			ps1 = fmt.Sprintf("%s:%v/[%s]> ", dsnParts[1], dsnParts[2], client.ActiveDb)
			// fmt.Println(">>>", client.ActiveDb)
			line, err := term.Prompt(*prompt)
			if err != nil {
				if err != io.EOF {
					ierr = err
				} else {
					ierr = nil
				}
				break //os.Exit(0)
			}
			if line == "" || line == ";" {
				// no more input
				prompt = &ps1
			}

			command += line
			if command != "" {
				for _, ll := range strings.Split(command, "\n") {
					term.AppendHistory(ll)
				}
			} else {
				continue
			}

			// SOURCE cmd. read script file
			if strings.ToUpper(command[:6]) == "SOURCE" {
				fname := strings.Trim(command[6:], " \n")
				fmt.Println("OPEN source file:", fname)
				sourcedFile, err := ioutil.ReadFile(fname)
				if err != nil {
					fmt.Println("ERR opening source file:", err)
					continue
				}
				command = string(sourcedFile)
			}
			// // nama
			// r, err := client.Send(command)
			// if err != nil {
			// 	fmt.Println("Error:", err)
			// }

			// response := trisserver.Unserialize(r)
			// // fmt.Println(response)
			// response.Print()

			// use lib
			var cmds []string
			var args [][]string
			msgs := strings.Split(strings.Trim(command, " "), "\n")
			fmt.Println("msgs", msgs)
			for n, msg := range msgs {
				parts := strings.Split(strings.Trim(msg, " "), " ")
				for i, p := range parts {
					if len(p) == 0 {
						continue
					}
					if i == 0 {
						cmds = append(cmds, strings.ToUpper(p))
						args = append(args, make([]string, 0))
					} else {
						args[n] = append(args[n], p)
					}
				}
			}
			fmt.Println("cmds, args", cmds, args)

		cmdexec:
			for i, cmdname := range cmds {
				var response *trisserver.Reply
				var err error
				switch cmdname {
				case "PING":
					response, err = client.Ping()
				case "SELECT":
					response, err = client.Select(args[i][0])
				case "DBINFO":
					response, err = client.DbInfo()
				case "INFO":
					response, err = client.Info()
				case "SAVE":
					response, err = client.Save()
				case "IMPORT":
					response, err = client.ImportDb(args[i][0], args[i][1])
				case "MERGE":
					response, err = client.MergeDb(args[i][0])
				case "CREATE":
					response, err = client.Create(args[i][0])
				case "ADD":
					response, err = client.Add(args[i][0])
				case "DEL":
					response, err = client.Del(args[i][0])
				case "HAS":
					response, err = client.Has(args[i][0])
				case "HASCOUNT":
					response, err = client.HasCount(args[i][0])
				case "HASPREFIX":
					response, err = client.HasPrefix(args[i][0])
				case "MEMBERS":
					response, err = client.Members()
				case "PREFIXMEMBERS":
					response, err = client.PrefixMembers(args[i][0])
				default:
					fmt.Println("Unknown command.")
					break cmdexec
				}
				if response.ReturnCode != trisserver.COMMAND_OK || err != nil {
					// TODO
				}
				response.Print()
			}

			// reset state
			command = ""
			ierr = nil
		}
	}
}
<commit_msg>fix SOURCE cmd check and disallow nested SOURCE cmds<commit_after>package main

import (
	"flag"
	"fmt"
	// zmq "github.com/alecthomas/gozmq"
	trisclient "github.com/fvbock/tris/client"
	trisserver "github.com/fvbock/tris/server"
	"github.com/sbinet/liner"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"
)

// TODO: detect connection loss and handle reconnect + currentDB

var (
	term      *liner.State = nil
	dsnString              = flag.String("d", "tcp:localhost:6000", "dsn to connect to")
)

func init() {
	fmt.Printf("TriS cli %s\n", trisclient.VERSION)
	term = liner.NewLiner()

	fname := path.Join(os.Getenv("HOME"), ".tris.history")
	f, err := os.Open(fname)
	if err != nil {
		fmt.Printf("**warning: could not access history file [%s]\n", fname)
		return
	}
	defer f.Close()
	_, err = term.ReadHistory(f)
	if err != nil {
		fmt.Printf("**warning: could not read history file [%s]\n", fname)
		return
	}
}

func atexit() {
	fmt.Println("CLI atexit")
	fname := path.Join(os.Getenv("HOME"), ".tris.history")
	f, err := os.OpenFile(fname, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		fmt.Printf("**warning: could not access history file [%s]\n", fname)
		return
	}
	defer f.Close()
	_, err = term.WriteHistory(f)
	if err != nil {
		fmt.Printf("**warning: could not write history file [%s]\n", fname)
		return
	}

	err = term.Close()
	if err != nil {
		fmt.Printf("**warning: problem closing term: %v\n", err)
		return
	}
	fmt.Println("\n")
}

func main() {
	defer atexit()
	var ierr error = nil // previous interpreter error
	ps1 := "[not connected]> "
	// ps2 := "...  "
	prompt := &ps1
	command := ""

	// TRIS conn
	flag.Parse()
	dsnParts := strings.Split(*dsnString, ":")
	port, _ := strconv.ParseInt(dsnParts[2], 10, 32)
	dsn := &trisclient.DSN{
		Protocol: dsnParts[0],
		Host:     dsnParts[1],
		Port:     int(port),
	}
	fmt.Printf("Connecting to %s://%s:%v\n", dsn.Protocol, dsn.Host, dsn.Port)

	client, err := trisclient.NewClient(dsn)
	err = client.Dial()
	if err != nil {
		fmt.Println(command, ierr)
	} else {
		ps1 = fmt.Sprintf("%s:%v/[%s]> ", dsnParts[1], dsnParts[2], client.ActiveDb)
	}
	defer client.Close()

	// check conn
	// r, err := client.Send("PING")
	// if err != nil {
	// 	fmt.Println("Error:", err)
	// }
	response, err := client.Ping()
	// response := trisserver.Unserialize(r)
	if response.ReturnCode != trisserver.COMMAND_OK {
		fmt.Printf("Initial PING failed:\n%v\n", response)
	} else {
		for {
			ps1 = fmt.Sprintf("%s:%v/[%s]> ", dsnParts[1], dsnParts[2], client.ActiveDb)
			// fmt.Println(">>>", client.ActiveDb)
			line, err := term.Prompt(*prompt)
			if err != nil {
				if err != io.EOF {
					ierr = err
				} else {
					ierr = nil
				}
				break //os.Exit(0)
			}
			if line == "" || line == ";" {
				// no more input
				prompt = &ps1
			}

			command += line
			if command != "" {
				for _, ll := range strings.Split(command, "\n") {
					term.AppendHistory(ll)
				}
			} else {
				continue
			}

			// SOURCE cmd. read script file
			if len(command) > 8 && strings.ToUpper(command[:6]) == "SOURCE" {
				fname := strings.Trim(command[6:], " \n")
				fmt.Println("OPEN source file:", fname)
				sourcedFile, err := ioutil.ReadFile(fname)
				if err != nil {
					fmt.Println("ERR opening source file:", err)
					continue
				}
				command = string(sourcedFile)
			}
			// // nama
			// r, err := client.Send(command)
			// if err != nil {
			// 	fmt.Println("Error:", err)
			// }

			// response := trisserver.Unserialize(r)
			// // fmt.Println(response)
			// response.Print()

			// use lib
			var cmds []string
			var args [][]string
			msgs := strings.Split(strings.Trim(command, " "), "\n")
			fmt.Println("msgs", msgs)
			for n, msg := range msgs {
				parts := strings.Split(strings.Trim(msg, " "), " ")
				for i, p := range parts {
					if len(p) == 0 {
						continue
					}
					if i == 0 {
						cmds = append(cmds, strings.ToUpper(p))
						args = append(args, make([]string, 0))
					} else {
						args[n] = append(args[n], p)
					}
				}
			}
			fmt.Println("cmds, args", cmds, args)

		cmdexec:
			for i, cmdname := range cmds {
				var response *trisserver.Reply
				var err error
				switch cmdname {
				case "SOURCE":
					fmt.Println("Nested sourcing is currently not supported.")
					break cmdexec
				case "PING":
					response, err = client.Ping()
				case "SELECT":
					response, err = client.Select(args[i][0])
				case "DBINFO":
					response, err = client.DbInfo()
				case "INFO":
					response, err = client.Info()
				case "SAVE":
					response, err = client.Save()
				case "IMPORT":
					response, err = client.ImportDb(args[i][0], args[i][1])
				case "MERGE":
					response, err = client.MergeDb(args[i][0])
				case "CREATE":
					response, err = client.Create(args[i][0])
				case "ADD":
					response, err = client.Add(args[i][0])
				case "DEL":
					response, err = client.Del(args[i][0])
				case "HAS":
					response, err = client.Has(args[i][0])
				case "HASCOUNT":
					response, err = client.HasCount(args[i][0])
				case "HASPREFIX":
					response, err = client.HasPrefix(args[i][0])
				case "MEMBERS":
					response, err = client.Members()
				case "PREFIXMEMBERS":
					response, err = client.PrefixMembers(args[i][0])
				default:
					fmt.Println("Unknown command.")
					break cmdexec
				}
				if response.ReturnCode != trisserver.COMMAND_OK || err != nil {
					// TODO
				}
				response.Print()
			}

			// reset state
			command = ""
			ierr = nil
		}
	}
}
<|endoftext|>


<commit_before>package main

func ExampleRunner() {
	Runner()

	// Output: sleeping for 1 seconds
	// sleeping for 2 seconds
	// goroutine exiting
}
<commit_msg>add license information to example test file<commit_after>// Copyright (c) 2015 Tim Heckman
// Released under the MIT license

package main

func ExampleRunner() {
	Runner()

	// Output: sleeping for 1 seconds
	// sleeping for 2 seconds
	// goroutine exiting
}
<|endoftext|>


<commit_before>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	return Mandrake{make(chan string), c.MonitoredDirectory, analyzers, filter}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		fmeta, err := filemeta.NewFileMeta(fpath)
		if err != nil {
			log.Println(err)
		}

		fs, err := json.Marshal(fmeta)

		for _, analyzer := range m.Analyzers {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			log.Print(result)
		}

		log.Println(string(fs))
		log.Printf("%s", fpath)
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}<commit_msg>updated mandrake to use filter for analysis<commit_after>/*

*/
package mandrake

import (
	"log"
	"encoding/json"

	"golang.org/x/exp/inotify"
	"github.com/hosom/gomandrake/config"
	"github.com/hosom/gomandrake/filemeta"
	"github.com/hosom/gomandrake/plugin"
)

// Mandrake is a wrapper struct for the bulk of the application logic
type Mandrake struct {
	AnalysisPipeline	chan string
	MonitoredDirectory	string
	Analyzers			[]plugin.AnalyzerCaller
	AnalyzerFilter		map[string][]plugin.AnalyzerCaller
}

// NewMandrake creates and returns a Mandrake struct utilizing a passed
// parsed configuration file to create the correct fields.
func NewMandrake(c config.Config) Mandrake {
	analyzers := []plugin.AnalyzerCaller{}
	filter := make(map[string][]plugin.AnalyzerCaller)
	for _, plug := range c.Analyzers {
		analyzer := plugin.NewAnalyzerCaller(plug)
		// Build a slice of all AnalyzerCaller structs
		analyzers = append(analyzers, analyzer)

		// Create a map to function as a mime_type filter for analyzers
		for _, mime := range analyzer.MimeFilter {
			filter[mime] = append(filter[mime], analyzer)
		}
	}

	return Mandrake{make(chan string), c.MonitoredDirectory, analyzers, filter}
}

// ListenAndServe starts the goroutines that perform all of the heavy lifting
// including Monitor() and DispatchAnalysis().
func (m Mandrake) ListenAndServe() {
	log.SetPrefix("[mandrake] ")
	log.Println(m.Analyzers[0])
	go m.DispatchAnalysis()
	m.Monitor()
}

// DispatchAnalysis intelligently sends a new file to registered plugins so
// that it can be analyzed.
func (m Mandrake) DispatchAnalysis() {
	for fpath := range m.AnalysisPipeline {
		fmeta, err := filemeta.NewFileMeta(fpath)
		if err != nil {
			log.Println(err)
		}

		// Create JSON filemeta object to pass to plugins so that plugins
		// receive basic contextual information about the file.
		fs, err := json.Marshal(fmeta)

		for _, analyzer := range m.AnalyzerFilter["all"] {
			result, err := analyzer.Analyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			log.Print(result)
		}

		for _, analyzer := range m.AnalyzerFilter[fmeta.Mime] {
			result, err := analyzer.ANalyze(string(fs))
			if err != nil {
				log.Print(err)
			}
			log.Print(result)
		}

		log.Println(string(fs))
		log.Printf("%s", fpath)
	}
}

// Monitor uses inotify to monitor the MonitoredDirectory for IN_CLOSE_WRITE
// events. Files written to the MonitoredDirectory will be sent to the
// analysis pipeline to be analyzed.
func (m Mandrake) Monitor() {
	log.Println("starting inotify watcher")
	watcher, err := inotify.NewWatcher()
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("adding watcher to %s directory", m.MonitoredDirectory)
	err = watcher.AddWatch(m.MonitoredDirectory, inotify.IN_CLOSE_WRITE)
	if err != nil {
		log.Fatal(err)
	}

	for {
		select {
		case ev := <- watcher.Event:
			m.AnalysisPipeline <- ev.Name
		case err := <- watcher.Error:
			log.Printf("inotify error: %s", err)
		}
	}
}<|endoftext|>


<commit_before>package maps

import "testing"

func TestCounter(t *testing.T) {
	c := NewCounter()
	if c == nil {
		t.Error("cannot instantiate a Counter")
	}

	if c.Size() != 0 {
		t.Error("must be empty")
	}

	c.Put("key", 2)

	if c.Size() != 1 {
		t.Error("must be empty")
	}

	v := c.Get("key")

	if v != 2 {
		t.Errorf("must be 2, but was %v", v)
	}
}
<commit_msg>map: Counter tests (#67)<commit_after>package maps

import "testing"

func TestCounter(t *testing.T) {
	c := NewCounter()
	if c == nil {
		t.Error("cannot instantiate a Counter")
	}

	if c.Size() != 0 {
		t.Error("must be empty")
	}

	c.Inc("key")
	c.Put("key", 2)

	if c.Size() != 1 || !c.Has("key") {
		t.Error("must be non-empty")
	}

	if value := c.Get("key"); value != 2 {
		t.Errorf("must be 2, but was %v", value)
	}

	c.Inc("key")
	if value := c.Get("key"); value != 3 {
		t.Errorf("must be 3, but was %v", value)
	}

	c.Dec("key")
	if value := c.Get("key"); value != 2 {
		t.Errorf("must be 2, but was %v", value)
	}

	c.Del("key")
	if value := c.Get("key"); value != -1 {
		t.Errorf("must be -1, but was %v", value)
	}

	c.Dec("key")
	if value := c.Get("key"); value != 0 {
		t.Errorf("must be 0, but was %v", value)
	}

	if keys := c.Keys(); len(keys) != 1 {
		t.Errorf("should have %v keys, got %v", 1, len(keys))
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package winfsnotify

import (
	"os"
	"time"
	"testing"
)

func expect(t *testing.T, eventstream <-chan *Event, name string, mask uint32) {
	t.Logf(`expected: "%s": 0x%x`, name, mask)
	select {
	case event := <-eventstream:
		if event == nil {
			t.Fatal("nil event received")
		}
		t.Logf("received: %s", event)
		if event.Name != name || event.Mask != mask {
			t.Fatal("did not receive expected event")
		}
	case <-time.After(1e9):
		t.Fatal("timed out waiting for event")
	}
}

func TestNotifyEvents(t *testing.T) {
	watcher, err := NewWatcher()
	if err != nil {
		t.Fatalf("NewWatcher() failed: %s", err)
	}

	testDir := "TestNotifyEvents.testdirectory"
	testFile := testDir + "/TestNotifyEvents.testfile"
	testFile2 := testFile + ".new"
	const mask = FS_ALL_EVENTS & ^(FS_ATTRIB|FS_CLOSE) | FS_IGNORED

	// Add a watch for testDir
	os.RemoveAll(testDir)
	if err = os.Mkdir(testDir, 0777); err != nil {
		t.Fatalf("Failed to create test directory", err)
	}
	defer os.RemoveAll(testDir)
	err = watcher.AddWatch(testDir, mask)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	// Receive errors on the error channel on a separate goroutine
	go func() {
		for err := range watcher.Error {
			t.Fatalf("error received: %s", err)
		}
	}()

	// Create a file
	file, err := os.Create(testFile)
	if err != nil {
		t.Fatalf("creating test file failed: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_CREATE)

	err = watcher.AddWatch(testFile, mask)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	if _, err = file.WriteString("hello, world"); err != nil {
		t.Fatalf("failed to write to test file: %s", err)
	}
	if err = file.Close(); err != nil {
		t.Fatalf("failed to close test file: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_MODIFY)
	expect(t, watcher.Event, testFile, FS_MODIFY)

	if err = os.Rename(testFile, testFile2); err != nil {
		t.Fatalf("failed to rename test file: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_MOVED_FROM)
	expect(t, watcher.Event, testFile2, FS_MOVED_TO)
	expect(t, watcher.Event, testFile, FS_MOVE_SELF)

	if err = os.RemoveAll(testDir); err != nil {
		t.Fatalf("failed to remove test directory: %s", err)
	}
	expect(t, watcher.Event, testFile2, FS_DELETE_SELF)
	expect(t, watcher.Event, testFile2, FS_IGNORED)
	expect(t, watcher.Event, testFile2, FS_DELETE)
	expect(t, watcher.Event, testDir, FS_DELETE_SELF)
	expect(t, watcher.Event, testDir, FS_IGNORED)

	t.Log("calling Close()")
	if err = watcher.Close(); err != nil {
		t.Fatalf("failed to close watcher: %s", err)
	}
}

func TestNotifyClose(t *testing.T) {
	watcher, _ := NewWatcher()
	watcher.Close()

	done := false
	go func() {
		watcher.Close()
		done = true
	}()

	time.Sleep(50e6) // 50 ms
	if !done {
		t.Fatal("double Close() test failed: second Close() call didn't return")
	}

	err := watcher.Watch("_test")
	if err == nil {
		t.Fatal("expected error on Watch() after Close(), got nil")
	}
}
<commit_msg>exp/winfsnotify: fix govet-found bug<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package winfsnotify

import (
	"os"
	"time"
	"testing"
)

func expect(t *testing.T, eventstream <-chan *Event, name string, mask uint32) {
	t.Logf(`expected: "%s": 0x%x`, name, mask)
	select {
	case event := <-eventstream:
		if event == nil {
			t.Fatal("nil event received")
		}
		t.Logf("received: %s", event)
		if event.Name != name || event.Mask != mask {
			t.Fatal("did not receive expected event")
		}
	case <-time.After(1e9):
		t.Fatal("timed out waiting for event")
	}
}

func TestNotifyEvents(t *testing.T) {
	watcher, err := NewWatcher()
	if err != nil {
		t.Fatalf("NewWatcher() failed: %s", err)
	}

	testDir := "TestNotifyEvents.testdirectory"
	testFile := testDir + "/TestNotifyEvents.testfile"
	testFile2 := testFile + ".new"
	const mask = FS_ALL_EVENTS & ^(FS_ATTRIB|FS_CLOSE) | FS_IGNORED

	// Add a watch for testDir
	os.RemoveAll(testDir)
	if err = os.Mkdir(testDir, 0777); err != nil {
		t.Fatalf("Failed to create test directory: %s", err)
	}
	defer os.RemoveAll(testDir)
	err = watcher.AddWatch(testDir, mask)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	// Receive errors on the error channel on a separate goroutine
	go func() {
		for err := range watcher.Error {
			t.Fatalf("error received: %s", err)
		}
	}()

	// Create a file
	file, err := os.Create(testFile)
	if err != nil {
		t.Fatalf("creating test file failed: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_CREATE)

	err = watcher.AddWatch(testFile, mask)
	if err != nil {
		t.Fatalf("Watcher.Watch() failed: %s", err)
	}

	if _, err = file.WriteString("hello, world"); err != nil {
		t.Fatalf("failed to write to test file: %s", err)
	}
	if err = file.Close(); err != nil {
		t.Fatalf("failed to close test file: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_MODIFY)
	expect(t, watcher.Event, testFile, FS_MODIFY)

	if err = os.Rename(testFile, testFile2); err != nil {
		t.Fatalf("failed to rename test file: %s", err)
	}
	expect(t, watcher.Event, testFile, FS_MOVED_FROM)
	expect(t, watcher.Event, testFile2, FS_MOVED_TO)
	expect(t, watcher.Event, testFile, FS_MOVE_SELF)

	if err = os.RemoveAll(testDir); err != nil {
		t.Fatalf("failed to remove test directory: %s", err)
	}
	expect(t, watcher.Event, testFile2, FS_DELETE_SELF)
	expect(t, watcher.Event, testFile2, FS_IGNORED)
	expect(t, watcher.Event, testFile2, FS_DELETE)
	expect(t, watcher.Event, testDir, FS_DELETE_SELF)
	expect(t, watcher.Event, testDir, FS_IGNORED)

	t.Log("calling Close()")
	if err = watcher.Close(); err != nil {
		t.Fatalf("failed to close watcher: %s", err)
	}
}

func TestNotifyClose(t *testing.T) {
	watcher, _ := NewWatcher()
	watcher.Close()

	done := false
	go func() {
		watcher.Close()
		done = true
	}()

	time.Sleep(50e6) // 50 ms
	if !done {
		t.Fatal("double Close() test failed: second Close() call didn't return")
	}

	err := watcher.Watch("_test")
	if err == nil {
		t.Fatal("expected error on Watch() after Close(), got nil")
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015, Shinya Yagyu
 * All rights reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 * 3. Neither the name of the copyright holder nor the names of its
 *    contributors may be used to endorse or promote products derived from this
 *    software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package keylib

import (
	"fmt"
	"log"
	"regexp"
	"time"

	"github.com/boltdb/bolt"
	"github.com/shingetsu-gou/shingetsu-gou/db"
	"github.com/shingetsu-gou/shingetsu-gou/mch"
	"github.com/shingetsu-gou/shingetsu-gou/recentlist"
	"github.com/shingetsu-gou/shingetsu-gou/record"
	"github.com/shingetsu-gou/shingetsu-gou/thread"
	"github.com/shingetsu-gou/shingetsu-gou/util"
)

func getThread(stamp int64) (string, error) {
	var thread string
	k := db.MustTob(stamp)
	err := db.DB.View(func(tx *bolt.Tx) error {
		_, err := db.Get(tx, "keylibST", k, &thread)
		return err
	})
	return thread, err
}

func getTime(thread string) (int64, error) {
	var stamp int64
	k := db.MustTob(thread)
	err := db.DB.View(func(tx *bolt.Tx) error {
		_, err := db.Get(tx, "keylibTS", k, &stamp)
		return err
	})
	return stamp, err
}

//Load loads from the file, adds stamps/datfile pairs from cachelist and recentlist.
//and saves to file.
func Load() {
	allCaches := thread.AllCaches()
	allRecs := recentlist.GetRecords()
	err := db.DB.Update(func(tx *bolt.Tx) error {
		for _, c := range allCaches {
			setFromCache(tx, c)
		}
		for _, rec := range allRecs {
			c := thread.NewCache(rec.Datfile)
			setFromCache(tx, c)
		}
		return nil
	})
	if err != nil {
		log.Println(err)
	}
}

//setEntry stores stamp/value.
func setEntry(tx *bolt.Tx, stamp int64, filekey string) {
	sb := db.MustTob(stamp)
	fb := db.MustTob(filekey)
	err := db.Put(tx, "keylibST", sb, fb)
	if err != nil {
		log.Print(err)
	}
	err = db.Put(tx, "keylibTS", fb, sb)
	if err != nil {
		log.Print(err)
	}
}

//setFromCache adds cache.datfile/timestamp pair if not exists.
func setFromCache(tx *bolt.Tx, ca *thread.Cache) {
	_, err := getTime(ca.Datfile)
	if err == nil {
		return
	}
	var firstStamp int64
	if !ca.HasRecord() {
		firstStamp = ca.RecentStamp()
	} else {
		if rec := ca.LoadRecords(record.Alive); len(rec) > 0 {
			firstStamp = rec[rec.Keys()[0]].Stamp
		}
	}
	if firstStamp == 0 {
		firstStamp = time.Now().Add(-24 * time.Hour).Unix()
	}
	for err = nil; err != nil; firstStamp++ {
		_, err = getThread(firstStamp)
	}
	setEntry(tx, firstStamp, ca.Datfile)
}

//GetDatkey returns stamp from filekey.
//if not found, tries to read from cache.
func GetDatkey(filekey string) (int64, error) {
	v, err := getTime(filekey)
	if err == nil {
		return v, nil
	}
	c := thread.NewCache(filekey)
	err = db.DB.Update(func(tx *bolt.Tx) error {
		setFromCache(tx, c)
		return nil
	})
	if err == nil {
		log.Println(err)
	}
	return getTime(filekey)
}

//GetFilekey returns value from datkey(stamp).
func GetFilekey(nDatkey int64) string {
	v, err := getThread(nDatkey)
	if err == nil {
		return v
	}
	return ""
}

//MakeBracketLink changes str in brackets to the html links format.
func MakeBracketLink(body, datHost, board string, table *mch.ResTable) string {
	regs := []*regexp.Regexp{
		regexp.MustCompile("^(?P<title>[^/]+)$"),
		regexp.MustCompile("^/(?P<type>[a-z]+)/(?P<title>[^/]+)$"),
		regexp.MustCompile("^(?P<title>[^/]+)/(?P<id>[0-9a-f]{8})$"),
		regexp.MustCompile("^/(?P<type>[a-z]+)/(?P<title>[^/]+)/(?P<id>[0-9a-f]{8})$"),
	}
	reg := regexp.MustCompile(`\[\[([^<>]+?)\]\]`)
	return reg.ReplaceAllStringFunc(body, func(str string) string {
		link := reg.FindStringSubmatch(str)[1]
		result := make(map[string]string)
		for _, r := range regs {
			if match := r.FindStringSubmatch(link); match != nil {
				for i, name := range r.SubexpNames() {
					result[name] = match[i]
				}
				break
			}
		}
		if result["title"] == "" {
			return result["body"]
		}
		if result["type"] == "" {
			result["type"] = "thread"
		}
		file := util.FileEncode(result["type"], result["title"])
		datkey, err := GetDatkey(file)
		if err != nil {
			log.Println(err)
			return body
		}
		if result["id"] == "" {
			url := fmt.Sprintf("http://%s/test/read.cgi/%s/%d/", datHost, board, datkey)
			return fmt.Sprintf("[[%s(%s)]]", result["title"], url)
		}
		ca := thread.NewCache(file)
		table = mch.NewResTable(ca)
		no := table.ID2num[result["id"]]
		url := fmt.Sprintf("http://%s/test/read.cgi/%s/%d/%d", datHost, board, datkey, no)
		return fmt.Sprintf("[[%s(>>%d %s)]]", result["title"], no, url)
	})
}

//MakeBody makes a dat body(message) line after stamp.
func MakeBody(rec *record.Record, host, board string, table *mch.ResTable) string {
	body := rec.GetBodyValue("body", "")
	body += rec.MakeAttachLink(host)
	body = table.MakeRSSAnchor(body)
	body = MakeBracketLink(body, host, board, table)
	return body
}

//MakeDat makes dat lines of 2ch from cache.
func MakeDat(ca *thread.Cache, board, host string) []string {
	recs := ca.LoadRecords(record.Alive)
	dat := make([]string, len(recs))
	table := mch.NewResTable(ca)

	i := 0
	for _, datfile := range recs.Keys() {
		rec := recs[datfile]
		err := rec.Load()
		if err != nil {
			log.Println(err)
			continue
		}
		name := rec.GetBodyValue("name", "")
		if name == "" {
			name = "名無しさん"
		}
		if rec.GetBodyValue("pubkey", "") != "" {
			name += "◆" + rec.GetBodyValue("pubkey", "")[:10]
		}
		comment := fmt.Sprintf("%s<>%s<>%s<>%s<>",
			name, rec.GetBodyValue("main", ""), util.Datestr2ch(rec.Stamp), MakeBody(rec, host, board, table))
		if i == 0 {
			comment += util.FileDecode(ca.Datfile)
		}
		dat[i] = comment
		i++
	}

	return dat
}
<commit_msg>fixed same stamp in subject.txt<commit_after>/*
 * Copyright (c) 2015, Shinya Yagyu
 * All rights reserved.
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 * 3. Neither the name of the copyright holder nor the names of its
 *    contributors may be used to endorse or promote products derived from this
 *    software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package keylib

import (
	"fmt"
	"log"
	"regexp"
	"time"

	"github.com/boltdb/bolt"
	"github.com/shingetsu-gou/shingetsu-gou/db"
	"github.com/shingetsu-gou/shingetsu-gou/mch"
	"github.com/shingetsu-gou/shingetsu-gou/recentlist"
	"github.com/shingetsu-gou/shingetsu-gou/record"
	"github.com/shingetsu-gou/shingetsu-gou/thread"
	"github.com/shingetsu-gou/shingetsu-gou/util"
)

func getThread(tx *bolt.Tx, stamp int64) (string, error) {
	var thread string
	k := db.MustTob(stamp)
	_, err := db.Get(tx, "keylibST", k, &thread)
	return thread, err
}

func getTime(tx *bolt.Tx, thread string) (int64, error) {
	var stamp int64
	k := db.MustTob(thread)
	_, err := db.Get(tx, "keylibTS", k, &stamp)
	return stamp, err
}

//Load loads from the file, adds stamps/datfile pairs from cachelist and recentlist.
//and saves to file.
func Load() {
	allCaches := thread.AllCaches()
	allRecs := recentlist.GetRecords()
	err := db.DB.Update(func(tx *bolt.Tx) error {
		for _, c := range allCaches {
			setFromCache(tx, c)
		}
		for _, rec := range allRecs {
			c := thread.NewCache(rec.Datfile)
			setFromCache(tx, c)
		}
		return nil
	})
	if err != nil {
		log.Println(err)
	}
}

//setEntry stores stamp/value.
func setEntry(tx *bolt.Tx, stamp int64, filekey string) {
	sb := db.MustTob(stamp)
	fb := db.MustTob(filekey)
	err := db.Put(tx, "keylibST", sb, fb)
	if err != nil {
		panic(err)
	}
	err = db.Put(tx, "keylibTS", fb, sb)
	if err != nil {
		panic(err)
	}
}

//setFromCache adds cache.datfile/timestamp pair if not exists.
func setFromCache(tx *bolt.Tx, ca *thread.Cache) {
	_, err := getTime(tx, ca.Datfile)
	if err == nil {
		return
	}
	var firstStamp int64
	rec := ca.LoadRecords(record.Alive)
	switch {
	case !ca.HasRecord():
		firstStamp = ca.RecentStamp()
	case len(rec) > 0:
		firstStamp = rec[rec.Keys()[0]].Stamp
	default:
		firstStamp = time.Now().Add(-24 * time.Hour).Unix()
	}
	for {
		_, err = getThread(tx, firstStamp)
		if err != nil {
			break
		}
		firstStamp++
	}
	setEntry(tx, firstStamp, ca.Datfile)
}

//GetDatkey returns stamp from filekey.
//if not found, tries to read from cache.
func GetDatkey(filekey string) (int64, error) {
	var v int64
	err := db.DB.Update(func(tx *bolt.Tx) error {
		var errr error
		v, errr = getTime(tx, filekey)
		if errr == nil {
			return nil
		}
		c := thread.NewCache(filekey)
		setFromCache(tx, c)
		v, errr = getTime(tx, filekey)
		return errr
	})
	return v, err
}

//GetFilekey returns value from datkey(stamp).
func GetFilekey(nDatkey int64) string {
	var v string
	err := db.DB.View(func(tx *bolt.Tx) error {
		var errr error
		v, errr = getThread(tx, nDatkey)
		return errr
	})
	if err == nil {
		return v
	}
	return ""
}

//MakeBracketLink changes str in brackets to the html links format.
func MakeBracketLink(body, datHost, board string, table *mch.ResTable) string {
	regs := []*regexp.Regexp{
		regexp.MustCompile("^(?P<title>[^/]+)$"),
		regexp.MustCompile("^/(?P<type>[a-z]+)/(?P<title>[^/]+)$"),
		regexp.MustCompile("^(?P<title>[^/]+)/(?P<id>[0-9a-f]{8})$"),
		regexp.MustCompile("^/(?P<type>[a-z]+)/(?P<title>[^/]+)/(?P<id>[0-9a-f]{8})$"),
	}
	reg := regexp.MustCompile(`\[\[([^<>]+?)\]\]`)
	return reg.ReplaceAllStringFunc(body, func(str string) string {
		link := reg.FindStringSubmatch(str)[1]
		result := make(map[string]string)
		for _, r := range regs {
			if match := r.FindStringSubmatch(link); match != nil {
				for i, name := range r.SubexpNames() {
					result[name] = match[i]
				}
				break
			}
		}
		if result["title"] == "" {
			return result["body"]
		}
		if result["type"] == "" {
			result["type"] = "thread"
		}
		file := util.FileEncode(result["type"], result["title"])
		datkey, err := GetDatkey(file)
		if err != nil {
			log.Println(err)
			return body
		}
		if result["id"] == "" {
			url := fmt.Sprintf("http://%s/test/read.cgi/%s/%d/", datHost, board, datkey)
			return fmt.Sprintf("[[%s(%s)]]", result["title"], url)
		}
		ca := thread.NewCache(file)
		table = mch.NewResTable(ca)
		no := table.ID2num[result["id"]]
		url := fmt.Sprintf("http://%s/test/read.cgi/%s/%d/%d", datHost, board, datkey, no)
		return fmt.Sprintf("[[%s(>>%d %s)]]", result["title"], no, url)
	})
}

//MakeBody makes a dat body(message) line after stamp.
func MakeBody(rec *record.Record, host, board string, table *mch.ResTable) string {
	body := rec.GetBodyValue("body", "")
	body += rec.MakeAttachLink(host)
	body = table.MakeRSSAnchor(body)
	body = MakeBracketLink(body, host, board, table)
	return body
}

//MakeDat makes dat lines of 2ch from cache.
func MakeDat(ca *thread.Cache, board, host string) []string {
	recs := ca.LoadRecords(record.Alive)
	dat := make([]string, len(recs))
	table := mch.NewResTable(ca)

	i := 0
	for _, datfile := range recs.Keys() {
		rec := recs[datfile]
		err := rec.Load()
		if err != nil {
			log.Println(err)
			continue
		}
		name := rec.GetBodyValue("name", "")
		if name == "" {
			name = "名無しさん"
		}
		if rec.GetBodyValue("pubkey", "") != "" {
			name += "◆" + rec.GetBodyValue("pubkey", "")[:10]
		}
		comment := fmt.Sprintf("%s<>%s<>%s<>%s<>",
			name, rec.GetBodyValue("main", ""), util.Datestr2ch(rec.Stamp), MakeBody(rec, host, board, table))
		if i == 0 {
			comment += util.FileDecode(ca.Datfile)
		}
		dat[i] = comment
		i++
	}

	return dat
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package mode

import (
	"sync"

	"github.com/bitmark-inc/bitmarkd/chain"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/logger"
)

// Mode - type to hold the mode
type Mode int

// all possible modes
const (
	Stopped Mode = iota
	Resynchronise
	Normal
	maximum
)

var globalData struct {
	sync.RWMutex
	log     *logger.L
	mode    Mode
	testing bool
	chain   string

	// set once during initialise
	initialised bool
}

// Initialise - set up the mode system
func Initialise(chainName string) error {

	// ensure start up in resynchronise mode
	globalData.Lock()
	defer globalData.Unlock()

	// no need to start if already started
	if globalData.initialised {
		return fault.AlreadyInitialised
	}

	globalData.log = logger.New("mode")
	globalData.log.Info("starting…")

	// default settings
	globalData.chain = chainName
	globalData.testing = false
	globalData.mode = Resynchronise

	// override for specific chain
	switch chainName {
	case chain.Bitmark:
		// no change
	case chain.Testing, chain.Local:
		globalData.testing = true
	default:
		globalData.log.Criticalf("mode cannot handle chain: '%s'", chainName)
		return fault.InvalidChain
	}

	// all data initialised
	globalData.initialised = true

	return nil
}

// Finalise - shutdown mode handling
func Finalise() error {

	if !globalData.initialised {
		return fault.NotInitialised
	}

	globalData.log.Info("shutting down…")
	globalData.log.Flush()

	Set(Stopped)

	// finally...
	globalData.initialised = false

	globalData.log.Info("finished")
	globalData.log.Flush()

	return nil
}

// Set - change mode
func Set(mode Mode) {

	if mode >= Stopped && mode < maximum {
		globalData.Lock()
		globalData.mode = mode
		globalData.Unlock()

		globalData.log.Infof("set: %s", mode)
	} else {
		globalData.log.Errorf("ignore invalid set: %d", mode)
	}
}

// Is - detect mode
func Is(mode Mode) bool {
	globalData.RLock()
	defer globalData.RUnlock()
	return mode == globalData.mode
}

// IsNot - detect mode
func IsNot(mode Mode) bool {
	globalData.RLock()
	defer globalData.RUnlock()
	return mode != globalData.mode
}

// IsTesting - special for testing
func IsTesting() bool {
	globalData.RLock()
	defer globalData.RUnlock()
	return globalData.testing
}

// ChainName - name of the current chain
func ChainName() string {
	globalData.RLock()
	defer globalData.RUnlock()
	return globalData.chain
}

// String - current mode represented as a string
func String() string {
	globalData.RLock()
	defer globalData.RUnlock()
	return globalData.mode.String()
}

// String - current mode rep[resented as a string
func (m Mode) String() string {
	switch m {
	case Stopped:
		return "Stopped"
	case Resynchronise:
		return "Resynchronise"
	case Normal:
		return "Normal"
	default:
		return "*Unknown*"
	}
}
<commit_msg>[mode] fix spelling mistake<commit_after>// SPDX-License-Identifier: ISC
// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package mode

import (
	"sync"

	"github.com/bitmark-inc/bitmarkd/chain"
	"github.com/bitmark-inc/bitmarkd/fault"
	"github.com/bitmark-inc/logger"
)

// Mode - type to hold the mode
type Mode int

// all possible modes
const (
	Stopped Mode = iota
	Resynchronise
	Normal
	maximum
)

var globalData struct {
	sync.RWMutex
	log     *logger.L
	mode    Mode
	testing bool
	chain   string

	// set once during initialise
	initialised bool
}

// Initialise - set up the mode system
func Initialise(chainName string) error {

	// ensure start up in resynchronise mode
	globalData.Lock()
	defer globalData.Unlock()

	// no need to start if already started
	if globalData.initialised {
		return fault.AlreadyInitialised
	}

	globalData.log = logger.New("mode")
	globalData.log.Info("starting…")

	// default settings
	globalData.chain = chainName
	globalData.testing = false
	globalData.mode = Resynchronise

	// override for specific chain
	switch chainName {
	case chain.Bitmark:
		// no change
	case chain.Testing, chain.Local:
		globalData.testing = true
	default:
		globalData.log.Criticalf("mode cannot handle chain: '%s'", chainName)
		return fault.InvalidChain
	}

	// all data initialised
	globalData.initialised = true

	return nil
}

// Finalise - shutdown mode handling
func Finalise() error {

	if !globalData.initialised {
		return fault.NotInitialised
	}

	globalData.log.Info("shutting down…")
	globalData.log.Flush()

	Set(Stopped)

	// finally...
	globalData.initialised = false

	globalData.log.Info("finished")
	globalData.log.Flush()

	return nil
}

// Set - change mode
func Set(mode Mode) {

	if mode >= Stopped && mode < maximum {
		globalData.Lock()
		globalData.mode = mode
		globalData.Unlock()

		globalData.log.Infof("set: %s", mode)
	} else {
		globalData.log.Errorf("ignore invalid set: %d", mode)
	}
}

// Is - detect mode
func Is(mode Mode) bool {
	globalData.RLock()
	defer globalData.RUnlock()
	return mode == globalData.mode
}

// IsNot - detect mode
func IsNot(mode Mode) bool {
	globalData.RLock()
	defer globalData.RUnlock()
	return mode != globalData.mode
}

// IsTesting - special for testing
func IsTesting() bool {
	globalData.RLock()
	defer globalData.RUnlock()
	return globalData.testing
}

// ChainName - name of the current chain
func ChainName() string {
	globalData.RLock()
	defer globalData.RUnlock()
	return globalData.chain
}

// String - current mode represented as a string
func String() string {
	globalData.RLock()
	defer globalData.RUnlock()
	return globalData.mode.String()
}

// String - current mode represented as a string
func (m Mode) String() string {
	switch m {
	case Stopped:
		return "Stopped"
	case Resynchronise:
		return "Resynchronise"
	case Normal:
		return "Normal"
	default:
		return "*Unknown*"
	}
}
<|endoftext|>