<commit_before>package gom

import (
	"errors"
	"fmt"
	"reflect"
	"strings"
	"time"
	"encoding/json"
)

func IsEmpty(v interface{}) bool {
	times := time.Time{}
	if times == v {
		return true
	}
	if v == "" {
		return true
	}
	if v == 0 {
		return true
	}
	if v == 0.0 {
		return true
	}
	if v == nil {
		return true
	}
	return false
}
func getType(v interface{}) (reflect.Type, bool, bool) {
	tt := reflect.TypeOf(v)
	isPtr := false
	islice := false
	if tt.Kind() == reflect.Ptr {
		tt = tt.Elem()
		isPtr = true
	}
	if tt.Kind() == reflect.Slice || tt.Kind() == reflect.Array {
		tt = tt.Elem()
		islice = true
	}
	if debug {
		fmt.Println("Test getType, result:", tt, isPtr, islice)
	}
	return tt, isPtr, islice
}

func GetTableModel(v interface{}, names ...string) (TableModel, error) {
	tms, err := getTableModel(v)
	if err != nil {
		return TableModel{}, nil
	}
	tm := tms[0]
	var cc []Column
	for _, ct := range tm.Columns {
		for _, name := range names {
			if ct.ColumnName == name {
				cc = append(cc, ct)
			}
		}
	}
	tm.Columns = cc
	return tm, nil

}
func CreateSingleValueTableModel(v interface{}, table string, field string) TableModel {
	tt, _, _ := getType(v)
	vals := reflect.New(tt).Elem()
	columns := []Column{{ColumnName: field, Type: tt, IsPrimary: false, Auto: false}}
	return TableModel{Columns: columns, TableName: table, ModelType: tt, ModelValue: vals}
}
func getTableModels(vs ...interface{}) ([]TableModel, error) {
	tablemodels := []TableModel{}
	for _, v := range vs {
		tt, _, _ := getType(v)
		if tt.Kind() == reflect.Interface {
			return tablemodels, errors.New("can't use interface as struct")
		}
		tbs, err := getTableModel(v)
		if err != nil {
			return tablemodels, err
		}
		tablemodels = append(tablemodels, tbs...)
	}
	return tablemodels, nil
}
func getTableModel(v interface{}) ([]TableModel, error) {
	var tableModels []TableModel
	var values []reflect.Value
	tt, isPtr, isSlice := getType(v)
	if tt.NumField() == 0 || tt.NumMethod() == 0 {
		return tableModels, errors.New(tt.Name() + " is not a valid struct")
	}
	value := reflect.ValueOf(v)
	if v != nil && tt.Kind() != reflect.Interface {

		if isPtr {
			value = value.Elem()
		}
		if debug {
			fmt.Println("model info:", tt, isPtr, isSlice, value)
		}
		if isSlice {
			if value.Len() > 0 {
				for i := 0; i < value.Len(); i++ {
					val := value.Index(i)
					values = append(values, val)
				}
			} else {
				values = append(values, reflect.New(tt).Elem())
			}
		} else {
			values = append(values, value)
		}
		if debug {
			fmt.Println(values)
		}
		for _, val := range values {
			nameMethod := val.MethodByName("TableName")
			if debug {
				fmt.Println(nameMethod)
			}
			tableName := nameMethod.Call(nil)[0].String()
			columns, primary := getColumns(val)
			ccs := []Column{primary}
			ccs = append(ccs, columns...)
			tableModels = append(tableModels, TableModel{ModelType: tt, ModelValue: val, Columns: ccs, TableName: tableName, Primary: primary})
		}
		return tableModels, nil

	} else {
		return tableModels, errors.New("can't use interface to build TableModel")
	}
}
func getColumns(v reflect.Value) ([]Column, Column) {
	var columns []Column
	var primary Column
	results := reflect.Indirect(reflect.ValueOf(&columns))
	oo := v.Type()

	for i := 0; i < oo.NumField(); i++ {
		field := oo.Field(i)
		col, tps := getColumnFromField(field)
		if tps != -1 {
			if tps == 1 || tps == 2 {
				primary = col
			} else {
				n := reflect.Indirect(reflect.ValueOf(&col))
				if results.Kind() == reflect.Ptr {
					results.Set(reflect.Append(results, n.Addr()))
				} else {
					results.Set(reflect.Append(results, n))
				}
			}

		}
	}
	if debug {
		fmt.Println("columns is:", columns)
	}
	return columns, primary
}
func getColumnFromField(filed reflect.StructField) (Column, int) {
	tag, tps := getTagFromField(filed)
	if debug {
		fmt.Println("Tag is:", tag, "type is:", tps)
	}
	v:=reflect.New(filed.Type)
	if v.Kind()==reflect.Ptr{
		v=v.Elem()
	}
	if tps != -1 {
		return Column{Type: v.Type(), ColumnName: tag, FieldName: filed.Name, Auto: tps == 2, IsPrimary: tps == 1 || tps == 2}, tps
	} else {
		return Column{}, -1
	}

}
func getTagFromField(field reflect.StructField) (string, int) {
	tag, hasTag := field.Tag.Lookup("gom")
	if hasTag {
		if strings.EqualFold(tag, "-") || len(tag) == 0 {
			return "", -1
		} else if len(tag) == 1 {
			tps := 0
			if strings.EqualFold(tag, "@") {
				tps = 2
			}
			if strings.EqualFold(tag, "!") {
				tps = 1
			}
			return strings.ToLower(field.Name), tps
		} else {
			if strings.Contains(tag, ",") {
				tags := strings.Split(tag, ",")
				if len(tags) == 2 {
					if strings.EqualFold(tags[0], "!") || strings.EqualFold(tags[0], "primary") {
						return tags[1], 1
					} else if strings.EqualFold(tags[0], "@") || strings.EqualFold(tags[0], "auto") {
						return tags[1], 2
					} else if strings.EqualFold(tags[0], "#") || strings.EqualFold(tags[0], "column") {
						return tags[1], 0
					} else {
						return "", -1
					}
				} else {
					return "", -1
				}
			} else {
				return tag, 0
			}
		}
	} else {
		return "", -1
	}
}
func getValueOfTableRow(model TableModel, row RowChooser) reflect.Value {
	maps := getDataMap(model, row)
	vv := reflect.New(model.ModelType).Elem()
	isStruct := model.ModelType.Kind() == reflect.Struct && model.ModelType != reflect.TypeOf(time.Time{})
	if debug {
		fmt.Println("vv kind is:",vv.Kind())
	}
	for _, c := range model.Columns {
		if isStruct {
			vv.FieldByName(c.FieldName).Set(reflect.ValueOf(maps[c.FieldName]).Elem())
		} else {
			vv.Set(reflect.ValueOf(maps[c.FieldName]).Elem())
		}
	}
	return vv
}
func getDataMap(model TableModel, row RowChooser) map[string]interface{} {
	dest := getArrayFromColumns(model.Columns)
	err := row.Scan(dest...)
	if err != nil {
		fmt.Println(err)
		return map[string]interface{}{}
	}
	result := make(map[string]interface{}, len(model.Columns))
	ccs := model.Columns
	for i, dd := range ccs {
		result[dd.FieldName] = dest[i]
	}
	if debug {
		fmt.Println(json.Marshal(result))
	}
	return result

}
func getArrayFromColumns(columns []Column) []interface{}{
	dest := make([]interface{}, len(columns)) // A temporary interface{} slice
	for i,v:=range columns{
		vv:=reflect.New(v.Type)
		if vv.Kind()==reflect.Ptr{
			vv=vv.Elem()
		}
		vt:=vv.Interface()
		dest[i]=&vt
	}
	return dest
}
<commit_msg>修复对bool类型的支持<commit_after>package gom

import (
	"errors"
	"fmt"
	"reflect"
	"strings"
	"time"
	"encoding/json"
)

func IsEmpty(v interface{}) bool {
	times := time.Time{}
	if times == v {
		return true
	}
	if v == "" {
		return true
	}
	if v == 0 {
		return true
	}
	if v == 0.0 {
		return true
	}
	if v == nil {
		return true
	}
	return false
}
func getType(v interface{}) (reflect.Type, bool, bool) {
	tt := reflect.TypeOf(v)
	isPtr := false
	islice := false
	if tt.Kind() == reflect.Ptr {
		tt = tt.Elem()
		isPtr = true
	}
	if tt.Kind() == reflect.Slice || tt.Kind() == reflect.Array {
		tt = tt.Elem()
		islice = true
	}
	if debug {
		fmt.Println("Test getType, result:", tt, isPtr, islice)
	}
	return tt, isPtr, islice
}

func GetTableModel(v interface{}, names ...string) (TableModel, error) {
	tms, err := getTableModel(v)
	if err != nil {
		return TableModel{}, nil
	}
	tm := tms[0]
	var cc []Column
	for _, ct := range tm.Columns {
		for _, name := range names {
			if ct.ColumnName == name {
				cc = append(cc, ct)
			}
		}
	}
	tm.Columns = cc
	return tm, nil

}
func CreateSingleValueTableModel(v interface{}, table string, field string) TableModel {
	tt, _, _ := getType(v)
	vals := reflect.New(tt).Elem()
	columns := []Column{{ColumnName: field, Type: tt, IsPrimary: false, Auto: false}}
	return TableModel{Columns: columns, TableName: table, ModelType: tt, ModelValue: vals}
}
func getTableModels(vs ...interface{}) ([]TableModel, error) {
	tablemodels := []TableModel{}
	for _, v := range vs {
		tt, _, _ := getType(v)
		if tt.Kind() == reflect.Interface {
			return tablemodels, errors.New("can't use interface as struct")
		}
		tbs, err := getTableModel(v)
		if err != nil {
			return tablemodels, err
		}
		tablemodels = append(tablemodels, tbs...)
	}
	return tablemodels, nil
}
func getTableModel(v interface{}) ([]TableModel, error) {
	var tableModels []TableModel
	var values []reflect.Value
	tt, isPtr, isSlice := getType(v)
	if tt.NumField() == 0 || tt.NumMethod() == 0 {
		return tableModels, errors.New(tt.Name() + " is not a valid struct")
	}
	value := reflect.ValueOf(v)
	if v != nil && tt.Kind() != reflect.Interface {

		if isPtr {
			value = value.Elem()
		}
		if debug {
			fmt.Println("model info:", tt, isPtr, isSlice, value)
		}
		if isSlice {
			if value.Len() > 0 {
				for i := 0; i < value.Len(); i++ {
					val := value.Index(i)
					values = append(values, val)
				}
			} else {
				values = append(values, reflect.New(tt).Elem())
			}
		} else {
			values = append(values, value)
		}
		if debug {
			fmt.Println(values)
		}
		for _, val := range values {
			nameMethod := val.MethodByName("TableName")
			if debug {
				fmt.Println(nameMethod)
			}
			tableName := nameMethod.Call(nil)[0].String()
			columns, primary := getColumns(val)
			ccs := []Column{primary}
			ccs = append(ccs, columns...)
			tableModels = append(tableModels, TableModel{ModelType: tt, ModelValue: val, Columns: ccs, TableName: tableName, Primary: primary})
		}
		return tableModels, nil

	} else {
		return tableModels, errors.New("can't use interface to build TableModel")
	}
}
func getColumns(v reflect.Value) ([]Column, Column) {
	var columns []Column
	var primary Column
	results := reflect.Indirect(reflect.ValueOf(&columns))
	oo := v.Type()

	for i := 0; i < oo.NumField(); i++ {
		field := oo.Field(i)
		col, tps := getColumnFromField(field)
		if tps != -1 {
			if tps == 1 || tps == 2 {
				primary = col
			} else {
				n := reflect.Indirect(reflect.ValueOf(&col))
				if results.Kind() == reflect.Ptr {
					results.Set(reflect.Append(results, n.Addr()))
				} else {
					results.Set(reflect.Append(results, n))
				}
			}

		}
	}
	if debug {
		fmt.Println("columns is:", columns)
	}
	return columns, primary
}
func getColumnFromField(filed reflect.StructField) (Column, int) {
	tag, tps := getTagFromField(filed)
	if debug {
		fmt.Println("Tag is:", tag, "type is:", tps)
	}
	v:=reflect.New(filed.Type)
	if v.Kind()==reflect.Ptr{
		v=v.Elem()
	}
	if tps != -1 {
		return Column{Type: v.Type(), ColumnName: tag, FieldName: filed.Name, Auto: tps == 2, IsPrimary: tps == 1 || tps == 2}, tps
	} else {
		return Column{}, -1
	}

}
func getTagFromField(field reflect.StructField) (string, int) {
	tag, hasTag := field.Tag.Lookup("gom")
	if hasTag {
		if strings.EqualFold(tag, "-") || len(tag) == 0 {
			return "", -1
		} else if len(tag) == 1 {
			tps := 0
			if strings.EqualFold(tag, "@") {
				tps = 2
			}
			if strings.EqualFold(tag, "!") {
				tps = 1
			}
			return strings.ToLower(field.Name), tps
		} else {
			if strings.Contains(tag, ",") {
				tags := strings.Split(tag, ",")
				if len(tags) == 2 {
					if strings.EqualFold(tags[0], "!") || strings.EqualFold(tags[0], "primary") {
						return tags[1], 1
					} else if strings.EqualFold(tags[0], "@") || strings.EqualFold(tags[0], "auto") {
						return tags[1], 2
					} else if strings.EqualFold(tags[0], "#") || strings.EqualFold(tags[0], "column") {
						return tags[1], 0
					} else {
						return "", -1
					}
				} else {
					return "", -1
				}
			} else {
				return tag, 0
			}
		}
	} else {
		return "", -1
	}
}
func getValueOfTableRow(model TableModel, row RowChooser) reflect.Value {
	maps := getDataMap(model, row)
	vv := reflect.New(model.ModelType).Elem()
	isStruct := model.ModelType.Kind() == reflect.Struct && model.ModelType != reflect.TypeOf(time.Time{})
	if debug {
		fmt.Println("vv kind is:",vv.Kind())
	}
	for _, c := range model.Columns {
		if isStruct {
			vv.FieldByName(c.FieldName).Set(reflect.ValueOf(maps[c.FieldName]).Elem())
		} else {
			vv.Set(reflect.ValueOf(maps[c.FieldName]).Elem())
		}
	}
	return vv
}
func getDataMap(model TableModel, row RowChooser) map[string]interface{} {
	dest := getArrayFromColumns(model.Columns)
	err := row.Scan(dest...)
	if err != nil {
		fmt.Println(err)
		return map[string]interface{}{}
	}
	result := make(map[string]interface{}, len(model.Columns))
	ccs := model.Columns
	for i, dd := range ccs {
		result[dd.FieldName] = dest[i]
	}
	if debug {
		fmt.Println(json.Marshal(result))
	}
	return result

}
func getArrayFromColumns(columns []Column) []interface{}{
	dest := make([]interface{}, len(columns)) // A temporary interface{} slice
	for i,v:=range columns{
		vv:=reflect.New(v.Type)
		if vv.Kind()==reflect.Ptr{
			vv=vv.Elem()
		}
		vt:=reflect.New
		dest[i]=&vt
	}
	return dest
}
<|endoftext|>


<commit_before>package peco

import (
	"sync"
	"time"
)

// View handles the drawing/updating the screen
type View struct {
	*Ctx
	mutex  sync.Locker
	layout Layout
}

// PagingRequest can be sent to move the selection cursor
type PagingRequest int

const (
	// ToLineAbove moves the selection to the line above
	ToLineAbove PagingRequest = iota
	// ToScrollPageDown moves the selection to the next page
	ToScrollPageDown
	// ToLineBelow moves the selection to the line below
	ToLineBelow
	// ToScrollPageUp moves the selection to the previous page
	ToScrollPageUp
	// ToScrollLeft scrolls screen to the left
	ToScrollLeft
	// ToScrollRight scrolls screen to the right
	ToScrollRight
)

// StatusMsgRequest specifies the string to be drawn
// on the status message bar and an optional delay that tells
// the view to clear that message
type StatusMsgRequest struct {
	message    string
	clearDelay time.Duration
}

// Loop receives requests to update the screen
func (v *View) Loop() {
	defer v.ReleaseWaitGroup()
	for {
		select {
		case <-v.LoopCh():
			return
		case m := <-v.StatusMsgCh():
			v.printStatus(m.DataInterface().(StatusMsgRequest))
			m.Done()
		case r := <-v.PagingCh():
			v.movePage(r.DataInterface().(PagingRequest))
			r.Done()
		case lines := <-v.DrawCh():
			tmp := lines.DataInterface()
			if name, ok := tmp.(string); ok {
				if name == "prompt" {
					v.drawPrompt()
				}
			} else {
				v.drawScreen()
			}
			lines.Done()
		}
	}
}

func (v *View) printStatus(r StatusMsgRequest) {
	v.layout.PrintStatus(r.message, r.clearDelay)
}

func (v *View) drawScreen() {
	v.mutex.Lock()
	defer v.mutex.Unlock()
	v.layout.DrawScreen()
}

func (v *View) drawPrompt() {
	v.mutex.Lock()
	defer v.mutex.Unlock()
	v.layout.DrawPrompt()
}

func (v *View) movePage(p PagingRequest) {
	v.mutex.Lock()
	defer v.mutex.Unlock()

	if moved := v.layout.MovePage(p); moved {
		v.layout.DrawScreen()
	}
}
<commit_msg>remove useless temp variable<commit_after>package peco

import (
	"sync"
	"time"
)

// View handles the drawing/updating the screen
type View struct {
	*Ctx
	mutex  sync.Locker
	layout Layout
}

// PagingRequest can be sent to move the selection cursor
type PagingRequest int

const (
	// ToLineAbove moves the selection to the line above
	ToLineAbove PagingRequest = iota
	// ToScrollPageDown moves the selection to the next page
	ToScrollPageDown
	// ToLineBelow moves the selection to the line below
	ToLineBelow
	// ToScrollPageUp moves the selection to the previous page
	ToScrollPageUp
	// ToScrollLeft scrolls screen to the left
	ToScrollLeft
	// ToScrollRight scrolls screen to the right
	ToScrollRight
)

// StatusMsgRequest specifies the string to be drawn
// on the status message bar and an optional delay that tells
// the view to clear that message
type StatusMsgRequest struct {
	message    string
	clearDelay time.Duration
}

// Loop receives requests to update the screen
func (v *View) Loop() {
	defer v.ReleaseWaitGroup()
	for {
		select {
		case <-v.LoopCh():
			return
		case m := <-v.StatusMsgCh():
			v.printStatus(m.DataInterface().(StatusMsgRequest))
			m.Done()
		case r := <-v.PagingCh():
			v.movePage(r.DataInterface().(PagingRequest))
			r.Done()
		case lines := <-v.DrawCh():
			tmp := lines.DataInterface()
			if name, ok := tmp.(string); ok {
				if name == "prompt" {
					v.drawPrompt()
				}
			} else {
				v.drawScreen()
			}
			lines.Done()
		}
	}
}

func (v *View) printStatus(r StatusMsgRequest) {
	v.layout.PrintStatus(r.message, r.clearDelay)
}

func (v *View) drawScreen() {
	v.mutex.Lock()
	defer v.mutex.Unlock()
	v.layout.DrawScreen()
}

func (v *View) drawPrompt() {
	v.mutex.Lock()
	defer v.mutex.Unlock()
	v.layout.DrawPrompt()
}

func (v *View) movePage(p PagingRequest) {
	v.mutex.Lock()
	defer v.mutex.Unlock()

	if v.layout.MovePage(p) {
		v.layout.DrawScreen()
	}
}
<|endoftext|>


<commit_before>package hdfs

import (
	"os"
	"path/filepath"
	"sort"
)

// Walk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root. All errors that arise visiting files
// and directories are filtered by walkFn. The files are walked in lexical
// order, which makes the output deterministic but means that for very large
// directories Walk can be inefficient. Walk does not follow symbolic links.
func (c *Client) Walk(root string, walkFn filepath.WalkFunc) error {
	return c.walk(root, walkFn)
}

func (c *Client) walk(path string, walkFn filepath.WalkFunc) error {
	file, err := c.Open(path)
	var info os.FileInfo
	if file != nil {
		info = file.Stat()
	}

	err = walkFn(path, info, err)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}

		return err
	}

	if info == nil || !info.IsDir() {
		return nil
	}

	names, err := file.Readdirnames(0)
	if err != nil {
		return walkFn(path, info, err)
	}

	sort.Strings(names)
	for _, name := range names {
		err = c.walk(filepath.Join(path, name), walkFn)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Join using regular separators on the client, not OS-specific ones<commit_after>package hdfs

import (
	"os"
	"path/filepath"
	"sort"
)

// Walk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root. All errors that arise visiting files
// and directories are filtered by walkFn. The files are walked in lexical
// order, which makes the output deterministic but means that for very large
// directories Walk can be inefficient. Walk does not follow symbolic links.
func (c *Client) Walk(root string, walkFn filepath.WalkFunc) error {
	return c.walk(root, walkFn)
}

func (c *Client) walk(path string, walkFn filepath.WalkFunc) error {
	file, err := c.Open(path)
	var info os.FileInfo
	if file != nil {
		info = file.Stat()
	}

	err = walkFn(path, info, err)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}

		return err
	}

	if info == nil || !info.IsDir() {
		return nil
	}

	names, err := file.Readdirnames(0)
	if err != nil {
		return walkFn(path, info, err)
	}

	sort.Strings(names)
	for _, name := range names {
		err = c.walk(filepath.ToSlash(filepath.Join(path, name)), walkFn)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package godirwalk

import (
	"os"
	"path/filepath"
	"sort"

	"github.com/pkg/errors"
)

// Options provide parameters for how the Walk function operates.
type Options struct {
	// FollowSymbolicLinks specifies whether Walk will follow symbolic links
	// that refer to directories. When set to false or left as its zero-value,
	// Walk will still invoke the callback function with symbolic link nodes,
	// but if the symbolic link refers to a directory, it will not recurse on
	// that directory. When set to true, Walk will recurse on symbolic links
	// that refer to a directory.
	FollowSymbolicLinks bool

	// Unsorted controls whether or not Walk will sort the immediate descendants
	// of a directory by their relative names prior to visiting each of those
	// entries.
	//
	// When set to false or left at its zero-value, Walk will get the list of
	// immediate descendants of a particular directory, sort that list by
	// lexical order of their names, and then visit each node in the list in
	// sorted order. This will cause Walk to always traverse the same directory
	// tree in the same order, however may be inefficient for directories with
	// many immediate descendants.
	//
	// When set to true, Walk skips sorting the list of immediate descendants
	// for a directory, and simply visits each node in the order the operating
	// system enumerated them. This will be more fast, but with the side effect
	// that the traversal order may be different from one invocation to the
	// next.
	Unsorted bool

	// Callback is a required function that Walk will invoke for every file
	// system node it encounters.
	Callback WalkFunc

	// PostChildrenCallback is an option function that Walk will invoke for
	// every file system directory it encounters after its children have been
	// processed.
	PostChildrenCallback WalkFunc

	// ScratchBuffer is an optional scratch buffer for Walk to use when reading
	// directory entries, to reduce amount of garbage generation. Not all
	// architectures take advantage of the scratch buffer.
	ScratchBuffer []byte
}

// WalkFunc is the type of the function called for each file system node visited
// by Walk. The pathname argument will contain the argument to Walk as a prefix;
// that is, if Walk is called with "dir", which is a directory containing the
// file "a", the provided WalkFunc will be invoked with the argument "dir/a",
// using the correct os.PathSeparator for the Go Operating System architecture,
// GOOS. The directory entry argument is a pointer to a Dirent for the node,
// providing access to both the basename and the mode type of the file system
// node.
//
// If an error is returned by the walk function, processing stops. The sole
// exception is when the function returns the special value filepath.SkipDir. If
// the function returns filepath.SkipDir when invoked on a directory, Walk skips
// the directory's contents entirely. If the function returns filepath.SkipDir
// when invoked on a non-directory file system node, Walk skips the remaining
// files in the containing directory.
type WalkFunc func(osPathname string, directoryEntry *Dirent) error

// Walk walks the file tree rooted at the specified directory, calling the
// specified callback function for each file system node in the tree, including
// root, symbolic links, and other node types. The nodes are walked in lexical
// order, which makes the output deterministic but means that for very large
// directories this function can be inefficient.
//
// This function is often much faster than filepath.Walk because it does not
// invoke os.Stat for every node it encounters, but rather obtains the file
// system node type when it reads the parent directory.
//
//    func main() {
//        dirname := "."
//        if len(os.Args) > 1 {
//            dirname = os.Args[1]
//        }
//        err := godirwalk.Walk(dirname, &godirwalk.Options{
//            Callback: func(osPathname string, de *godirwalk.Dirent) error {
//                fmt.Printf("%s %s\n", de.ModeType(), osPathname)
//                return nil
//            },
//        })
//        if err != nil {
//            fmt.Fprintf(os.Stderr, "%s\n", err)
//            os.Exit(1)
//        }
//    }
func Walk(pathname string, options *Options) error {
	pathname = filepath.Clean(pathname)

	var fi os.FileInfo
	var err error

	if options.FollowSymbolicLinks {
		fi, err = os.Stat(pathname)
		if err != nil {
			return errors.Wrap(err, "cannot Stat")
		}
	} else {
		fi, err = os.Lstat(pathname)
		if err != nil {
			return errors.Wrap(err, "cannot Lstat")
		}
	}

	mode := fi.Mode()
	if mode&os.ModeDir == 0 {
		return errors.Errorf("cannot Walk non-directory: %s", pathname)
	}

	dirent := &Dirent{
		name:     filepath.Base(pathname),
		modeType: mode & os.ModeType,
	}

	err = walk(pathname, dirent, options)
	if err == filepath.SkipDir {
		return nil // silence SkipDir for top level
	}
	return err
}

// walk recursively traverses the file system node specified by pathname and the
// Dirent.
func walk(osPathname string, dirent *Dirent, options *Options) error {
	err := options.Callback(osPathname, dirent)
	if err != nil {
		if err != filepath.SkipDir {
			return errors.Wrap(err, "Callback") // wrap potential errors returned by walkFn
		}
		return err
	}

	// On some platforms, an entry can have more than one mode type bit set.
	// For instance, it could have both the symlink bit and the directory bit
	// set indicating it's a symlink to a directory.
	if dirent.IsSymlink() {
		if !options.FollowSymbolicLinks {
			return nil
		}
		// Only need to Stat entry if platform did not already have os.ModeDir
		// set, such as would be the case for unix like operating systems. (This
		// guard eliminates extra os.Stat check on Windows.)
		if !dirent.IsDir() {
			referent, err := os.Readlink(osPathname)
			if err != nil {
				return errors.Wrap(err, "cannot Readlink")
			}
			fi, err := os.Stat(filepath.Join(filepath.Dir(osPathname), referent))
			if err != nil {
				return errors.Wrap(err, "cannot Stat")
			}
			dirent.modeType = fi.Mode() & os.ModeType
		}
	}

	if !dirent.IsDir() {
		return nil
	}

	// If get here, then specified pathname refers to a directory.
	deChildren, err := ReadDirents(osPathname, options.ScratchBuffer)
	if err != nil {
		return errors.Wrap(err, "cannot ReadDirents")
	}

	if !options.Unsorted {
		sort.Sort(deChildren) // sort children entries unless upstream says to leave unsorted
	}

	for _, deChild := range deChildren {
		osChildname := filepath.Join(osPathname, deChild.name)
		err = walk(osChildname, deChild, options)
		if err != nil {
			if err != filepath.SkipDir {
				return err
			}
			// If received skipdir on a directory, stop processing that
			// directory, but continue to its siblings. If received skipdir on a
			// non-directory, stop processing remaining siblings.
			if deChild.IsSymlink() {
				// Only need to Stat entry if platform did not already have
				// os.ModeDir set, such as would be the case for unix like
				// operating systems. (This guard eliminates extra os.Stat check
				// on Windows.)
				if !deChild.IsDir() {
					// Resolve symbolic link referent to determine whether node
					// is directory or not.
					referent, err := os.Readlink(osChildname)
					if err != nil {
						return errors.Wrap(err, "cannot Readlink")
					}
					fi, err := os.Stat(filepath.Join(osPathname, referent))
					if err != nil {
						return errors.Wrap(err, "cannot Stat")
					}
					deChild.modeType = fi.Mode() & os.ModeType
				}
			}
			if !deChild.IsDir() {
				// If not directory, return immediately, thus skipping remainder
				// of siblings.
				return nil
			}
		}
	}

	if options.PostChildrenCallback == nil {
		return nil
	}
	err = options.PostChildrenCallback(osPathname, dirent)
	if err != nil {
		if err != filepath.SkipDir {
			return errors.Wrap(err, "PostChildrenCallback") // wrap potential errors returned by walkFn
		}
		return err
	}
	return nil
}
<commit_msg>ErrorCallback option allows user to specify action to take for OS errors<commit_after>package godirwalk

import (
	"os"
	"path/filepath"
	"sort"

	"github.com/pkg/errors"
)

// Options provide parameters for how the Walk function operates.
type Options struct {
	// ErrorCallback specifies a function to be invoked in the case of an error
	// that could potentially be ignored while walking a file system
	// hierarchy. When set to nil or left as its zero-value, any error condition
	// causes Walk to immediately return the error describing what took
	// place. When non-nil, this user supplied function is invoked with the OS
	// pathname of the file system object that caused the error along with the
	// error that took place. The return value of the supplied ErrorCallback
	// function determines whether the error will cause Walk to halt immediately
	// as it would were no ErrorCallback value provided, or skip this file
	// system node yet continue on with the remaining nodes in the file system
	// hierarchy.
	//
	// NOTE: ErrorCallback is only invoked for errors that are returned by the
	// runtime, and not for errors returned by other user supplied callback
	// functions.
	ErrorCallback func(string, error) ErrorAction

	// FollowSymbolicLinks specifies whether Walk will follow symbolic links
	// that refer to directories. When set to false or left as its zero-value,
	// Walk will still invoke the callback function with symbolic link nodes,
	// but if the symbolic link refers to a directory, it will not recurse on
	// that directory. When set to true, Walk will recurse on symbolic links
	// that refer to a directory.
	FollowSymbolicLinks bool

	// Unsorted controls whether or not Walk will sort the immediate descendants
	// of a directory by their relative names prior to visiting each of those
	// entries.
	//
	// When set to false or left at its zero-value, Walk will get the list of
	// immediate descendants of a particular directory, sort that list by
	// lexical order of their names, and then visit each node in the list in
	// sorted order. This will cause Walk to always traverse the same directory
	// tree in the same order, however may be inefficient for directories with
	// many immediate descendants.
	//
	// When set to true, Walk skips sorting the list of immediate descendants
	// for a directory, and simply visits each node in the order the operating
	// system enumerated them. This will be more fast, but with the side effect
	// that the traversal order may be different from one invocation to the
	// next.
	Unsorted bool

	// Callback is a required function that Walk will invoke for every file
	// system node it encounters.
	Callback WalkFunc

	// PostChildrenCallback is an option function that Walk will invoke for
	// every file system directory it encounters after its children have been
	// processed.
	PostChildrenCallback WalkFunc

	// ScratchBuffer is an optional scratch buffer for Walk to use when reading
	// directory entries, to reduce amount of garbage generation. Not all
	// architectures take advantage of the scratch buffer.
	ScratchBuffer []byte
}

// ErrorAction defines a set of actions the Walk function could take based on
// the occurrence of an error returned by the operating system. See the
// documentation for the ErrorCallback field of the Options structure for more
// information.
type ErrorAction int

const (
	// Halt is the ErrorAction return value when the upstream code wants to halt
	// the walk process when a runtime error takes place. It matches the default
	// action the Walk function would take were no ErrorCallback provided.
	Halt ErrorAction = iota

	// SkipNode is the ErrorAction return value when the upstream code wants to
	// ignore the runtime error for the current file system node, skip
	// processing of the node that caused the error, and continue walking the
	// file system hierarchy with the remaining nodes.
	SkipNode
)

// WalkFunc is the type of the function called for each file system node visited
// by Walk. The pathname argument will contain the argument to Walk as a prefix;
// that is, if Walk is called with "dir", which is a directory containing the
// file "a", the provided WalkFunc will be invoked with the argument "dir/a",
// using the correct os.PathSeparator for the Go Operating System architecture,
// GOOS. The directory entry argument is a pointer to a Dirent for the node,
// providing access to both the basename and the mode type of the file system
// node.
//
// If an error is returned by the Callback or PostChildrenCallback functions,
// processing stops. The sole exception is when the function returns the special
// value filepath.SkipDir. If the function returns filepath.SkipDir when invoked
// on a directory, Walk skips the directory's contents entirely. If the function
// returns filepath.SkipDir when invoked on a non-directory file system node,
// Walk skips the remaining files in the containing directory.
type WalkFunc func(osPathname string, directoryEntry *Dirent) error

// Walk walks the file tree rooted at the specified directory, calling the
// specified callback function for each file system node in the tree, including
// root, symbolic links, and other node types. The nodes are walked in lexical
// order, which makes the output deterministic but means that for very large
// directories this function can be inefficient.
//
// This function is often much faster than filepath.Walk because it does not
// invoke os.Stat for every node it encounters, but rather obtains the file
// system node type when it reads the parent directory.
//
// If an error is returned by an operating system, processing typically
// stops. However, when an ErrorCallback function is provided in the provided
// Options structure, that function is invoked with the error along with the OS
// pathname of the file system node that caused the error. The ErrorCallback
// function's return value determines the action that Walk will then take.
//
//    func main() {
//        dirname := "."
//        if len(os.Args) > 1 {
//            dirname = os.Args[1]
//        }
//        err := godirwalk.Walk(dirname, &godirwalk.Options{
//            Callback: func(osPathname string, de *godirwalk.Dirent) error {
//                fmt.Printf("%s %s\n", de.ModeType(), osPathname)
//                return nil
//            },
//        })
//        if err != nil {
//            fmt.Fprintf(os.Stderr, "%s\n", err)
//            os.Exit(1)
//        }
//    }
func Walk(pathname string, options *Options) error {
	pathname = filepath.Clean(pathname)

	var fi os.FileInfo
	var err error

	if options.FollowSymbolicLinks {
		fi, err = os.Stat(pathname)
		if err != nil {
			return errors.Wrap(err, "cannot Stat")
		}
	} else {
		fi, err = os.Lstat(pathname)
		if err != nil {
			return errors.Wrap(err, "cannot Lstat")
		}
	}

	mode := fi.Mode()
	if mode&os.ModeDir == 0 {
		return errors.Errorf("cannot Walk non-directory: %s", pathname)
	}

	dirent := &Dirent{
		name:     filepath.Base(pathname),
		modeType: mode & os.ModeType,
	}

	// If ErrorCallback is nil, set to a default value that halts the walk
	// process on all operating system errors. This is done to allow error
	// handling to be more succinct in the walk code.
	if options.ErrorCallback == nil {
		options.ErrorCallback = defaultErrorCallback
	}

	err = walk(pathname, dirent, options)
	if err == filepath.SkipDir {
		return nil // silence SkipDir for top level
	}
	return err
}

// defaultErrorCallback always returns Halt because if the upstream code did not
// provide an ErrorCallback function, walking the file system hierarchy ought to
// halt upon any operating system error.
func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt }

// walk recursively traverses the file system node specified by pathname and the
// Dirent.
func walk(osPathname string, dirent *Dirent, options *Options) error {
	err := options.Callback(osPathname, dirent)
	if err != nil {
		if err != filepath.SkipDir {
			return errors.Wrap(err, "Callback") // wrap potential errors returned by walkFn
		}
		return err
	}

	// On some platforms, an entry can have more than one mode type bit set.
	// For instance, it could have both the symlink bit and the directory bit
	// set indicating it's a symlink to a directory.
	if dirent.IsSymlink() {
		if !options.FollowSymbolicLinks {
			return nil
		}
		// Only need to Stat entry if platform did not already have os.ModeDir
		// set, such as would be the case for unix like operating systems. (This
		// guard eliminates extra os.Stat check on Windows.)
		if !dirent.IsDir() {
			referent, err := os.Readlink(osPathname)
			if err != nil {
				err = errors.Wrap(err, "cannot Readlink")
				if action := options.ErrorCallback(osPathname, err); action == SkipNode {
					return nil
				}
				return err
			}

			osp := filepath.Join(filepath.Dir(osPathname), referent)
			fi, err := os.Stat(osp)
			if err != nil {
				err = errors.Wrap(err, "cannot Stat")
				if action := options.ErrorCallback(osp, err); action == SkipNode {
					return nil
				}
				return err
			}
			dirent.modeType = fi.Mode() & os.ModeType
		}
	}

	if !dirent.IsDir() {
		return nil
	}

	// If get here, then specified pathname refers to a directory.
	deChildren, err := ReadDirents(osPathname, options.ScratchBuffer)
	if err != nil {
		err = errors.Wrap(err, "cannot ReadDirents")
		if action := options.ErrorCallback(osPathname, err); action == SkipNode {
			return nil
		}
		return err
	}

	if !options.Unsorted {
		sort.Sort(deChildren) // sort children entries unless upstream says to leave unsorted
	}

	for _, deChild := range deChildren {
		osChildname := filepath.Join(osPathname, deChild.name)
		err = walk(osChildname, deChild, options)
		if err != nil {
			if err != filepath.SkipDir {
				return err
			}
			// If received skipdir on a directory, stop processing that
			// directory, but continue to its siblings. If received skipdir on a
			// non-directory, stop processing remaining siblings.
			if deChild.IsSymlink() {
				// Only need to Stat entry if platform did not already have
				// os.ModeDir set, such as would be the case for unix like
				// operating systems. (This guard eliminates extra os.Stat check
				// on Windows.)
				if !deChild.IsDir() {
					// Resolve symbolic link referent to determine whether node
					// is directory or not.
					referent, err := os.Readlink(osChildname)
					if err != nil {
						err = errors.Wrap(err, "cannot Readlink")
						if action := options.ErrorCallback(osChildname, err); action == SkipNode {
							continue // with next child
						}
						return err
					}
					osp := filepath.Join(osPathname, referent)
					fi, err := os.Stat(osp)
					if err != nil {
						err = errors.Wrap(err, "cannot Stat")
						if action := options.ErrorCallback(osp, err); action == SkipNode {
							continue // with next child
						}
						return err
					}
					deChild.modeType = fi.Mode() & os.ModeType
				}
			}
			if !deChild.IsDir() {
				// If not directory, return immediately, thus skipping remainder
				// of siblings.
				return nil
			}
		}
	}

	if options.PostChildrenCallback == nil {
		return nil
	}
	err = options.PostChildrenCallback(osPathname, dirent)
	if err != nil {
		if err != filepath.SkipDir {
			return errors.Wrap(err, "PostChildrenCallback") // wrap potential errors returned by walkFn
		}
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package mtree

import (
	"fmt"
	"io"
	"os"
	"os/user"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/vbatts/go-mtree/pkg/govis"
)

// ExcludeFunc is the type of function called on each path walked to determine
// whether to be excluded from the assembled DirectoryHierarchy. If the func
// returns true, then the path is not included in the spec.
type ExcludeFunc func(path string, info os.FileInfo) bool

// ExcludeNonDirectories is an ExcludeFunc for excluding all paths that are not directories
var ExcludeNonDirectories = func(path string, info os.FileInfo) bool {
	return !info.IsDir()
}

var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"}

// Walk from root directory and assemble the DirectoryHierarchy. excludes
// provided are used to skip paths. keywords are the set to collect from the
// walked paths. The recommended default list is DefaultKeywords.
func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval) (*DirectoryHierarchy, error) {
	if fsEval == nil {
		fsEval = DefaultFsEval{}
	}
	creator := dhCreator{DH: &DirectoryHierarchy{}, fs: fsEval}
	// insert signature and metadata comments first (user, machine, tree, date)
	for _, e := range signatureEntries(root) {
		e.Pos = len(creator.DH.Entries)
		creator.DH.Entries = append(creator.DH.Entries, e)
	}
	// insert keyword metadata next
	for _, e := range keywordEntries(keywords) {
		e.Pos = len(creator.DH.Entries)
		creator.DH.Entries = append(creator.DH.Entries, e)
	}
	// walk the directory and add entries
	err := startWalk(&creator, root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		for _, ex := range excludes {
			if ex(path, info) {
				return nil
			}
		}

		entryPathName := filepath.Base(path)
		if info.IsDir() {
			creator.DH.Entries = append(creator.DH.Entries, Entry{
				Type: BlankType,
				Pos:  len(creator.DH.Entries),
			})

			// Insert a comment of the full path of the directory's name
			if creator.curDir != nil {
				dirname, err := creator.curDir.Path()
				if err != nil {
					return err
				}
				creator.DH.Entries = append(creator.DH.Entries, Entry{
					Pos:  len(creator.DH.Entries),
					Raw:  "# " + filepath.Join(dirname, entryPathName),
					Type: CommentType,
				})
			} else {
				entryPathName = "."
				creator.DH.Entries = append(creator.DH.Entries, Entry{
					Pos:  len(creator.DH.Entries),
					Raw:  "# .",
					Type: CommentType,
				})
			}

			// set the initial /set keywords
			if creator.curSet == nil {
				e := Entry{
					Name:     "/set",
					Type:     SpecialType,
					Pos:      len(creator.DH.Entries),
					Keywords: keyvalSelector(defaultSetKeywords, keywords),
				}
				for _, keyword := range SetKeywords {
					err := func() error {
						var r io.Reader
						if info.Mode().IsRegular() {
							fh, err := creator.fs.Open(path)
							if err != nil {
								return err
							}
							defer fh.Close()
							r = fh
						}
						keywordFunc, ok := KeywordFuncs[keyword]
						if !ok {
							return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
						}
						if str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r); err == nil && str != "" {
							e.Keywords = append(e.Keywords, str)
						} else if err != nil {
							return err
						}
						return nil
					}()
					if err != nil {
						return err
					}
				}
				creator.curSet = &e
				creator.DH.Entries = append(creator.DH.Entries, e)
			} else if creator.curSet != nil {
				// check the attributes of the /set keywords and re-set if changed
				klist := []KeyVal{}
				for _, keyword := range SetKeywords {
					err := func() error {
						var r io.Reader
						if info.Mode().IsRegular() {
							fh, err := creator.fs.Open(path)
							if err != nil {
								return err
							}
							defer fh.Close()
							r = fh
						}
						keywordFunc, ok := KeywordFuncs[keyword]
						if !ok {
							return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
						}
						str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
						if err != nil {
							return err
						}
						if str != "" {
							klist = append(klist, str)
						}
						return nil
					}()
					if err != nil {
						return err
					}
				}

				needNewSet := false
				for _, k := range klist {
					if !inKeyValSlice(k, creator.curSet.Keywords) {
						needNewSet = true
					}
				}
				if needNewSet {
					e := Entry{
						Name:     "/set",
						Type:     SpecialType,
						Pos:      len(creator.DH.Entries),
						Keywords: keyvalSelector(append(defaultSetKeywords, klist...), keywords),
					}
					creator.curSet = &e
					creator.DH.Entries = append(creator.DH.Entries, e)
				}
			}
		}
		encodedEntryName, err := govis.Vis(entryPathName, DefaultVisFlags)
		if err != nil {
			return err
		}
		e := Entry{
			Name:   encodedEntryName,
			Pos:    len(creator.DH.Entries),
			Type:   RelativeType,
			Set:    creator.curSet,
			Parent: creator.curDir,
		}
		for _, keyword := range keywords {
			err := func() error {
				var r io.Reader
				if info.Mode().IsRegular() {
					fh, err := creator.fs.Open(path)
					if err != nil {
						return err
					}
					defer fh.Close()
					r = fh
				}
				keywordFunc, ok := KeywordFuncs[keyword]
				if !ok {
					return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
				}
				str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
				if err != nil {
					return err
				}
				if str != "" && !inKeyValSlice(str, creator.curSet.Keywords) {
					e.Keywords = append(e.Keywords, str)
				}
				return nil
			}()
			if err != nil {
				return err
			}
		}
		if info.IsDir() {
			if creator.curDir != nil {
				creator.curDir.Next = &e
			}
			e.Prev = creator.curDir
			creator.curDir = &e
		} else {
			if creator.curEnt != nil {
				creator.curEnt.Next = &e
			}
			e.Prev = creator.curEnt
			creator.curEnt = &e
		}
		creator.DH.Entries = append(creator.DH.Entries, e)
		return nil
	})
	return creator.DH, err
}

// startWalk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root. All errors that arise visiting files
// and directories are filtered by walkFn. The files are walked in lexical
// order, which makes the output deterministic but means that for very
// large directories Walk can be inefficient.
// Walk does not follow symbolic links.
func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error {
	info, err := c.fs.Lstat(root)
	if err != nil {
		return walkFn(root, nil, err)
	}
	return walk(c, root, info, walkFn)
}

// walk recursively descends path, calling w.
func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
	err := walkFn(path, info, nil)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}
		return err
	}

	if !info.IsDir() {
		return nil
	}

	names, err := readOrderedDirNames(c, path)
	if err != nil {
		return walkFn(path, info, err)
	}

	for _, name := range names {
		filename := filepath.Join(path, name)
		fileInfo, err := c.fs.Lstat(filename)
		if err != nil {
			if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
				return err
			}
		} else {
			err = walk(c, filename, fileInfo, walkFn)
			if err != nil {
				if !fileInfo.IsDir() || err != filepath.SkipDir {
					return err
				}
			}
		}
	}
	c.DH.Entries = append(c.DH.Entries, Entry{
		Name: "..",
		Type: DotDotType,
		Pos:  len(c.DH.Entries),
	})
	if c.curDir != nil {
		c.curDir = c.curDir.Parent
	}
	return nil
}

// readOrderedDirNames reads the directory and returns a sorted list of all
// entries with non-directories first, followed by directories.
func readOrderedDirNames(c *dhCreator, dirname string) ([]string, error) {
	infos, err := c.fs.Readdir(dirname)
	if err != nil {
		return nil, err
	}

	names := []string{}
	dirnames := []string{}
	for _, info := range infos {
		if info.IsDir() {
			dirnames = append(dirnames, info.Name())
			continue
		}
		names = append(names, info.Name())
	}
	sort.Strings(names)
	sort.Strings(dirnames)
	return append(names, dirnames...), nil
}

// signatureEntries is a simple helper function that returns a slice of Entry's
// that describe the metadata signature about the host. Items like date, user,
// machine, and tree (which is specified by argument `root`), are considered.
// These Entry's construct comments in the mtree specification, so if there is
// an error trying to obtain a particular metadata, we simply don't construct
// the Entry.
func signatureEntries(root string) []Entry {
	var sigEntries []Entry
	user, err := user.Current()
	if err == nil {
		userEntry := Entry{
			Type: CommentType,
			Raw:  fmt.Sprintf("#%16s%s", "user: ", user.Username),
		}
		sigEntries = append(sigEntries, userEntry)
	}

	hostname, err := os.Hostname()
	if err == nil {
		hostEntry := Entry{
			Type: CommentType,
			Raw:  fmt.Sprintf("#%16s%s", "machine: ", hostname),
		}
		sigEntries = append(sigEntries, hostEntry)
	}

	if tree := filepath.Clean(root); tree == "." || tree == ".." {
		root, err := os.Getwd()
		if err == nil {
			// use parent directory of current directory
			if tree == ".." {
				root = filepath.Dir(root)
			}
			treeEntry := Entry{
				Type: CommentType,
				Raw:  fmt.Sprintf("#%16s%s", "tree: ", filepath.Clean(root)),
			}
			sigEntries = append(sigEntries, treeEntry)
		}
	} else {
		treeEntry := Entry{
			Type: CommentType,
			Raw:  fmt.Sprintf("#%16s%s", "tree: ", filepath.Clean(root)),
		}
		sigEntries = append(sigEntries, treeEntry)
	}

	dateEntry := Entry{
		Type: CommentType,
		Raw:  fmt.Sprintf("#%16s%s", "date: ", time.Now().Format("Mon Jan 2 15:04:05 2006")),
	}
	sigEntries = append(sigEntries, dateEntry)

	return sigEntries
}

// keywordEntries returns a slice of entries including a comment of the
// keywords requested when generating this manifest.
func keywordEntries(keywords []Keyword) []Entry {
	// Convert all of the keywords to zero-value keyvals.
	return []Entry{
		{
			Type: CommentType,
			Raw:  fmt.Sprintf("#%16s%s", "keywords: ", strings.Join(FromKeywords(keywords), ",")),
		},
	}
}
<commit_msg>walk: update the comment<commit_after>package mtree

import (
	"fmt"
	"io"
	"os"
	"os/user"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/vbatts/go-mtree/pkg/govis"
)

// ExcludeFunc is the type of function called on each path walked to determine
// whether to be excluded from the assembled DirectoryHierarchy. If the func
// returns true, then the path is not included in the spec.
type ExcludeFunc func(path string, info os.FileInfo) bool

// ExcludeNonDirectories is an ExcludeFunc for excluding all paths that are not directories
var ExcludeNonDirectories = func(path string, info os.FileInfo) bool {
	return !info.IsDir()
}

var defaultSetKeywords = []KeyVal{"type=file", "nlink=1", "flags=none", "mode=0664"}

// Walk from root directory and assemble the DirectoryHierarchy
// * `excludes` provided are used to skip paths
// * `keywords` are the set to collect from the walked paths. The recommended default list is DefaultKeywords.
// * `fsEval` is the interface to use in evaluating files. If `nil`, then DefaultFsEval is used.
func Walk(root string, excludes []ExcludeFunc, keywords []Keyword, fsEval FsEval) (*DirectoryHierarchy, error) {
	if fsEval == nil {
		fsEval = DefaultFsEval{}
	}
	creator := dhCreator{DH: &DirectoryHierarchy{}, fs: fsEval}
	// insert signature and metadata comments first (user, machine, tree, date)
	for _, e := range signatureEntries(root) {
		e.Pos = len(creator.DH.Entries)
		creator.DH.Entries = append(creator.DH.Entries, e)
	}
	// insert keyword metadata next
	for _, e := range keywordEntries(keywords) {
		e.Pos = len(creator.DH.Entries)
		creator.DH.Entries = append(creator.DH.Entries, e)
	}
	// walk the directory and add entries
	err := startWalk(&creator, root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		for _, ex := range excludes {
			if ex(path, info) {
				return nil
			}
		}

		entryPathName := filepath.Base(path)
		if info.IsDir() {
			creator.DH.Entries = append(creator.DH.Entries, Entry{
				Type: BlankType,
				Pos:  len(creator.DH.Entries),
			})

			// Insert a comment of the full path of the directory's name
			if creator.curDir != nil {
				dirname, err := creator.curDir.Path()
				if err != nil {
					return err
				}
				creator.DH.Entries = append(creator.DH.Entries, Entry{
					Pos:  len(creator.DH.Entries),
					Raw:  "# " + filepath.Join(dirname, entryPathName),
					Type: CommentType,
				})
			} else {
				entryPathName = "."
				creator.DH.Entries = append(creator.DH.Entries, Entry{
					Pos:  len(creator.DH.Entries),
					Raw:  "# .",
					Type: CommentType,
				})
			}

			// set the initial /set keywords
			if creator.curSet == nil {
				e := Entry{
					Name:     "/set",
					Type:     SpecialType,
					Pos:      len(creator.DH.Entries),
					Keywords: keyvalSelector(defaultSetKeywords, keywords),
				}
				for _, keyword := range SetKeywords {
					err := func() error {
						var r io.Reader
						if info.Mode().IsRegular() {
							fh, err := creator.fs.Open(path)
							if err != nil {
								return err
							}
							defer fh.Close()
							r = fh
						}
						keywordFunc, ok := KeywordFuncs[keyword]
						if !ok {
							return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
						}
						if str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r); err == nil && str != "" {
							e.Keywords = append(e.Keywords, str)
						} else if err != nil {
							return err
						}
						return nil
					}()
					if err != nil {
						return err
					}
				}
				creator.curSet = &e
				creator.DH.Entries = append(creator.DH.Entries, e)
			} else if creator.curSet != nil {
				// check the attributes of the /set keywords and re-set if changed
				klist := []KeyVal{}
				for _, keyword := range SetKeywords {
					err := func() error {
						var r io.Reader
						if info.Mode().IsRegular() {
							fh, err := creator.fs.Open(path)
							if err != nil {
								return err
							}
							defer fh.Close()
							r = fh
						}
						keywordFunc, ok := KeywordFuncs[keyword]
						if !ok {
							return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
						}
						str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
						if err != nil {
							return err
						}
						if str != "" {
							klist = append(klist, str)
						}
						return nil
					}()
					if err != nil {
						return err
					}
				}

				needNewSet := false
				for _, k := range klist {
					if !inKeyValSlice(k, creator.curSet.Keywords) {
						needNewSet = true
					}
				}
				if needNewSet {
					e := Entry{
						Name:     "/set",
						Type:     SpecialType,
						Pos:      len(creator.DH.Entries),
						Keywords: keyvalSelector(append(defaultSetKeywords, klist...), keywords),
					}
					creator.curSet = &e
					creator.DH.Entries = append(creator.DH.Entries, e)
				}
			}
		}
		encodedEntryName, err := govis.Vis(entryPathName, DefaultVisFlags)
		if err != nil {
			return err
		}
		e := Entry{
			Name:   encodedEntryName,
			Pos:    len(creator.DH.Entries),
			Type:   RelativeType,
			Set:    creator.curSet,
			Parent: creator.curDir,
		}
		for _, keyword := range keywords {
			err := func() error {
				var r io.Reader
				if info.Mode().IsRegular() {
					fh, err := creator.fs.Open(path)
					if err != nil {
						return err
					}
					defer fh.Close()
					r = fh
				}
				keywordFunc, ok := KeywordFuncs[keyword]
				if !ok {
					return fmt.Errorf("Unknown keyword %q for file %q", keyword, path)
				}
				str, err := creator.fs.KeywordFunc(keywordFunc)(path, info, r)
				if err != nil {
					return err
				}
				if str != "" && !inKeyValSlice(str, creator.curSet.Keywords) {
					e.Keywords = append(e.Keywords, str)
				}
				return nil
			}()
			if err != nil {
				return err
			}
		}
		if info.IsDir() {
			if creator.curDir != nil {
				creator.curDir.Next = &e
			}
			e.Prev = creator.curDir
			creator.curDir = &e
		} else {
			if creator.curEnt != nil {
				creator.curEnt.Next = &e
			}
			e.Prev = creator.curEnt
			creator.curEnt = &e
		}
		creator.DH.Entries = append(creator.DH.Entries, e)
		return nil
	})
	return creator.DH, err
}

// startWalk walks the file tree rooted at root, calling walkFn for each file or
// directory in the tree, including root. All errors that arise visiting files
// and directories are filtered by walkFn. The files are walked in lexical
// order, which makes the output deterministic but means that for very
// large directories Walk can be inefficient.
// Walk does not follow symbolic links.
func startWalk(c *dhCreator, root string, walkFn filepath.WalkFunc) error {
	info, err := c.fs.Lstat(root)
	if err != nil {
		return walkFn(root, nil, err)
	}
	return walk(c, root, info, walkFn)
}

// walk recursively descends path, calling w.
func walk(c *dhCreator, path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
	err := walkFn(path, info, nil)
	if err != nil {
		if info.IsDir() && err == filepath.SkipDir {
			return nil
		}
		return err
	}

	if !info.IsDir() {
		return nil
	}

	names, err := readOrderedDirNames(c, path)
	if err != nil {
		return walkFn(path, info, err)
	}

	for _, name := range names {
		filename := filepath.Join(path, name)
		fileInfo, err := c.fs.Lstat(filename)
		if err != nil {
			if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
				return err
			}
		} else {
			err = walk(c, filename, fileInfo, walkFn)
			if err != nil {
				if !fileInfo.IsDir() || err != filepath.SkipDir {
					return err
				}
			}
		}
	}
	c.DH.Entries = append(c.DH.Entries, Entry{
		Name: "..",
		Type: DotDotType,
		Pos:  len(c.DH.Entries),
	})
	if c.curDir != nil {
		c.curDir = c.curDir.Parent
	}
	return nil
}

// readOrderedDirNames reads the directory and returns a sorted list of all
// entries with non-directories first, followed by directories.
func readOrderedDirNames(c *dhCreator, dirname string) ([]string, error) {
	infos, err := c.fs.Readdir(dirname)
	if err != nil {
		return nil, err
	}

	names := []string{}
	dirnames := []string{}
	for _, info := range infos {
		if info.IsDir() {
			dirnames = append(dirnames, info.Name())
			continue
		}
		names = append(names, info.Name())
	}
	sort.Strings(names)
	sort.Strings(dirnames)
	return append(names, dirnames...), nil
}

// signatureEntries is a simple helper function that returns a slice of Entry's
// that describe the metadata signature about the host. Items like date, user,
// machine, and tree (which is specified by argument `root`), are considered.
// These Entry's construct comments in the mtree specification, so if there is
// an error trying to obtain a particular metadata, we simply don't construct
// the Entry.
func signatureEntries(root string) []Entry {
	var sigEntries []Entry
	user, err := user.Current()
	if err == nil {
		userEntry := Entry{
			Type: CommentType,
			Raw:  fmt.Sprintf("#%16s%s", "user: ", user.Username),
		}
		sigEntries = append(sigEntries, userEntry)
	}

	hostname, err := os.Hostname()
	if err == nil {
		hostEntry := Entry{
			Type: CommentType,
			Raw:  fmt.Sprintf("#%16s%s", "machine: ", hostname),
		}
		sigEntries = append(sigEntries, hostEntry)
	}

	if tree := filepath.Clean(root); tree == "." || tree == ".." {
		root, err := os.Getwd()
		if err == nil {
			// use parent directory of current directory
			if tree == ".." {
				root = filepath.Dir(root)
			}
			treeEntry := Entry{
				Type: CommentType,
				Raw:  fmt.Sprintf("#%16s%s", "tree: ", filepath.Clean(root)),
			}
			sigEntries = append(sigEntries, treeEntry)
		}
	} else {
		treeEntry := Entry{
			Type: CommentType,
			Raw:  fmt.Sprintf("#%16s%s", "tree: ", filepath.Clean(root)),
		}
		sigEntries = append(sigEntries, treeEntry)
	}

	dateEntry := Entry{
		Type: CommentType,
		Raw:  fmt.Sprintf("#%16s%s", "date: ", time.Now().Format("Mon Jan 2 15:04:05 2006")),
	}
	sigEntries = append(sigEntries, dateEntry)

	return sigEntries
}

// keywordEntries returns a slice of entries including a comment of the
// keywords requested when generating this manifest.
func keywordEntries(keywords []Keyword) []Entry {
	// Convert all of the keywords to zero-value keyvals.
	return []Entry{
		{
			Type: CommentType,
			Raw:  fmt.Sprintf("#%16s%s", "keywords: ", strings.Join(FromKeywords(keywords), ",")),
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"net/http"
	"os"
	"regexp"

	"log"

	"strings"

	"time"

	"strconv"

	"github.com/justinas/alice" // Middleware chaining
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

var wikiDir string
var tagDir string
var pubDir string
var ekey []byte

var encryptionFlag = []byte("ENCRYPTED")

var specialDir []string

type basePage struct {
	Title string
	Nav   nav
}
type wikiPage struct {
	Body      template.HTML
	Tags      string
	TagArray  []string
	Created   string
	Modified  string
	Published bool
	Encrypted bool
	basePage
}
type searchPage struct {
	basePage
	Results []QueryResults
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}
func getPDFFilename(folder, name string) string {
	return folder + name
}

func getWikiFilename(folder, name string) string {
	return folder + name + ".md"
}

func getWikiTagsFilename(name string) string {
	return tagDir + name
}
func getWikiPubFilename(name string) string {
	return pubDir + name
}
func (p *wikiPage) save(s storage) error {
	var err error
	filename := getWikiFilename(wikiDir, p.Title)
	body := []byte(p.Body)
	if p.Encrypted {
		body, err = encrypt(body, ekey)
		if err != nil {
			return err
		}
		body = append(encryptionFlag, body...)
	}
	err = s.storeFile(filename, body)
	if err != nil {
		return err
	}

	tagsfile := getWikiTagsFilename(p.Title)
	err = s.storeFile(tagsfile, []byte(p.Tags))
	if err != nil {
		return err
	}

	log.Printf("Pub flag %v\n", p.Published)
	pubfile := getWikiPubFilename(p.Title)
	if p.Published {
		log.Printf("Saving %v\n", pubfile)
		err = s.storeFile(pubfile, nil)
		if err != nil {
			return err
		}

	} else {
		log.Printf("Removing %v\n", pubfile)
		// Only return an error if something other than file doesnt exit
		// We expect this fail to not exist most of the time but we dont know if
		// we don't try
		if err = s.deleteFile(pubfile); !os.IsNotExist(err) {
			return err
		}
	}

	return nil
}

func convertMarkdown(page *wikiPage, err error) (*wikiPage, error) {
	if err != nil {
		return page, err
	}
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")

	page.Body = template.HTML(regexp.MustCompile("\r\n").ReplaceAllString(string(page.Body), "\n"))

	unsafe := blackfriday.Run([]byte(page.Body),
		blackfriday.WithExtensions(
			blackfriday.CommonExtensions|
				blackfriday.HardLineBreak|
				blackfriday.HeadingIDs|
				blackfriday.AutoHeadingIDs,
		),
	)

	page.Body = template.HTML(p.SanitizeBytes(unsafe))
	return page, nil

}
func viewHandler(w http.ResponseWriter, r *http.Request, p *wikiPage, s storage) {
	p, err := convertMarkdown(s.getPage(p))
	if err != nil {
		p, err = s.checkForPDF(p)
		if err != nil {
			http.Redirect(w, r, "/wiki/edit/"+p.Title, http.StatusFound)
			return
		}
	} else {
		p.Body = template.HTML(parseWikiWords([]byte(p.Body)))
	}

	renderTemplate(w, "view", p)
}

func vueHandler(w http.ResponseWriter, r *http.Request, p *wikiPage, s storage) {
	renderTemplate(w, "viewjs", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, p *wikiPage, s storage) {
	p, _ = s.getPage(p)
	renderTemplate(w, "edit", p)
}

func makeSearchHandler(fn navFunc, s storage) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		term := r.URL.Query().Get("term") // Get the search term
		if len(term) == 0 {
			http.NotFound(w, r)
			return
		}

		results := ParseQueryResults(s.searchPages(wikiDir, term))
		p := &searchPage{Results: results, basePage: basePage{Title: "Search", Nav: fn(s)}}

		renderTemplate(w, "search", p)
	}
}

func redirectHandler(c Config) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		host := r.Host
		var port string
		hostparts := strings.Split(host, ":")
		if len(hostparts) == 2 {
			host = hostparts[0]
			port = strconv.Itoa(c.HTTPSPort)
		}
		target := "https://" + host
		if len(port) > 0 {
			target += ":" + port

		}
		target += r.URL.Path
		if len(r.URL.RawQuery) > 0 {
			target += "?" + r.URL.RawQuery
		}
		http.Redirect(w, r, target, http.StatusTemporaryRedirect)

	}
}

func simpleHandler(page string, fn navFunc, s storage) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		renderTemplate(w, page, fn(s))
	}

}

func saveHandler(w http.ResponseWriter, r *http.Request, wiki string, s storage) string {
	body := r.FormValue("body")

	body = regexp.MustCompile("\r\n").ReplaceAllString(body, "\n")

	log.Printf("Checkbox is : %v", r.FormValue("wikipub"))
	p := wikiPage{basePage: basePage{Title: wiki}, Body: template.HTML(body), Tags: r.FormValue("wikitags")}
	if r.FormValue("wikipub") == "on" {
		p.Published = true
	}
	if r.FormValue("wikicrypt") == "on" {
		p.Encrypted = true
	}

	err := p.save(s)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return ""
	}
	http.Redirect(w, r, "/wiki/view/"+p.Title, http.StatusFound)

	return r.FormValue("wikitags")
}

var templates = template.Must(template.ParseFiles(
	"views/edit.html",
	"views/view.html",
	"views/viewjs.html",
	"views/pub.html",
	"views/pubhome.html",
	"views/login.html",
	"views/home.html",
	"views/search.html",
	"views/index.html",
	"views/footer.html",
	"views/leftnav.html"))

func renderTemplate(w http.ResponseWriter, tmpl string, p interface{}) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

var validPath = regexp.MustCompile("^/wiki/(edit|save|view|search)/([a-zA-Z0-9\\.\\-_ /]*)$")

func makeHandler(fn func(http.ResponseWriter, *http.Request, *wikiPage, storage), navfn navFunc, s storage) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		wword := r.URL.Query().Get("wword") // Get the wiki word param if available
		if len(wword) == 0 {
			log.Printf("Path is : %v", r.URL.Path)
			m := validPath.FindStringSubmatch(r.URL.Path)
			if m == nil {
				http.NotFound(w, r)
				return
			}
			wword = m[2]
		}
		p := &wikiPage{basePage: basePage{Title: wword, Nav: navfn(s)}}
		fn(w, r, p, s)
	}
}

func processSave(fn func(http.ResponseWriter, *http.Request, string, storage) string, s storage) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r, m[2], s)

	}
}

func parseWikiWords(target []byte) []byte {
	var wikiWord = regexp.MustCompile(`\{\{([^\}^#]+)[#]*(.*)\}\}`)

	return wikiWord.ReplaceAll(target, []byte("$1"))
}

func loggingHandler(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		t1 := time.Now()
		next.ServeHTTP(w, r)
		t2 := time.Now()
		log.Printf("[%s] %q %v\n", r.Method, r.URL.String(), t2.Sub(t1))
	}
	return http.HandlerFunc(fn)
}

func main() {
	specialDir = []string{"tags", "pub"}
	config, err := LoadConfig()
	if err != nil {
		log.Fatal(err)
	}

	if config.Logfile != "" {
		f, err := os.OpenFile(config.Logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			checkErr(err)
		}
		defer f.Close()

		log.SetOutput(f)
	}

	config.LoadCookieKey()

	auth := NewAuth(*config, persistUsers)

	wikiDir = config.WikiDir
	if !strings.HasSuffix(wikiDir, "/") {
		wikiDir = wikiDir + "/"
	}
	tagDir = wikiDir + "tags/" // Make sure this doesnt double up the / in the path...
	pubDir = wikiDir + "pub/"  // Make sure this doesnt double up the / in the path...

	ekey = []byte(config.EncryptionKey)

	os.Mkdir(config.WikiDir, 0755)
	os.Mkdir(config.WikiDir+"tags", 0755)

	authHandlers := alice.New(loggingHandler, auth.validate)
	noauthHandlers := alice.New(loggingHandler)

	httpmux := http.NewServeMux()
	httpsmux := httpmux
	// setup wiki on https
	if config.UseHTTPS {
		httpsmux = http.NewServeMux()
	}

	fstore := fileStorage{tagDir}

	httpsmux.Handle("/wiki", authHandlers.ThenFunc(simpleHandler("home", getNav, fstore)))
	httpsmux.Handle("/wiki/login/", noauthHandlers.ThenFunc(auth.loginHandler))
	httpsmux.Handle("/wiki/register/", noauthHandlers.ThenFunc(auth.registerHandler))
	httpsmux.Handle("/wiki/logout/", authHandlers.ThenFunc(logoutHandler))
	httpsmux.Handle("/wiki/search/", authHandlers.ThenFunc(makeSearchHandler(getNav, fstore)))
	httpsmux.Handle("/wiki/view/", authHandlers.ThenFunc(makeHandler(viewHandler, getNav, fstore)))
	//httpsmux.Handle("/wiki/view/", authHandlers.ThenFunc(makeHandler(vueHandler, getNav, fstore)))
	httpsmux.Handle("/wiki/edit/", authHandlers.ThenFunc(makeHandler(editHandler, getNav, fstore)))
	httpsmux.Handle("/wiki/save/", authHandlers.ThenFunc(processSave(saveHandler, fstore)))
	httpsmux.Handle("/wiki/raw/", http.StripPrefix("/wiki/raw/", http.FileServer(http.Dir(wikiDir))))
	httpsmux.Handle("/pub/", noauthHandlers.ThenFunc(makePubHandler(pubHandler, getNav, fstore)))
	httpsmux.Handle("/pub", noauthHandlers.ThenFunc(simpleHandler("pubhome", getPubNav, fstore)))
	httpsmux.Handle("/api", noauthHandlers.ThenFunc(apiHandler(innerAPIHandler, fstore)))

	if config.UseHTTPS {
		// Any routes that duplicate the http routing are only done here
		httpsmux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
		go http.ListenAndServeTLS(
			":"+strconv.Itoa(config.HTTPSPort),
			config.CertPath,
			config.KeyPath,
			httpsmux)

		httpmux.HandleFunc("/wiki", redirectHandler(*config))
		httpsmux.Handle("/", http.FileServer(http.Dir("wwwroot")))
	}

	// Listen for normal traffic against root
	httpmux.Handle("/", http.FileServer(http.Dir("wwwroot")))
	httpmux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
	err = http.ListenAndServe(":"+strconv.Itoa(config.HTTPPort), httpmux)
	checkErr(err)

}
<commit_msg>trigger build<commit_after>package main

import (
	"html/template"
	"net/http"
	"os"
	"regexp"

	"log"

	"strings"

	"time"

	"strconv"

	"github.com/justinas/alice" // Middleware chaining
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
)

var wikiDir string
var tagDir string
var pubDir string
var ekey []byte

var encryptionFlag = []byte("ENCRYPTED")

var specialDir []string

type basePage struct {
	Title string
	Nav   nav
}
type wikiPage struct {
	Body      template.HTML
	Tags      string
	TagArray  []string
	Created   string
	Modified  string
	Published bool
	Encrypted bool
	basePage
}
type searchPage struct {
	basePage
	Results []QueryResults
}

func checkErr(err error) {
	if err != nil {
		panic(err)
	}
}
func getPDFFilename(folder, name string) string {
	return folder + name
}

func getWikiFilename(folder, name string) string {
	return folder + name + ".md"
}

func getWikiTagsFilename(name string) string {
	return tagDir + name
}
func getWikiPubFilename(name string) string {
	return pubDir + name
}
func (p *wikiPage) save(s storage) error {
	var err error
	filename := getWikiFilename(wikiDir, p.Title)
	body := []byte(p.Body)
	if p.Encrypted {
		body, err = encrypt(body, ekey)
		if err != nil {
			return err
		}
		body = append(encryptionFlag, body...)
	}
	err = s.storeFile(filename, body)
	if err != nil {
		return err
	}

	tagsfile := getWikiTagsFilename(p.Title)
	err = s.storeFile(tagsfile, []byte(p.Tags))
	if err != nil {
		return err
	}

	log.Printf("Pub flag %v\n", p.Published)
	pubfile := getWikiPubFilename(p.Title)
	if p.Published {
		log.Printf("Saving %v\n", pubfile)
		err = s.storeFile(pubfile, nil)
		if err != nil {
			return err
		}

	} else {
		log.Printf("Removing %v\n", pubfile)
		// Only return an error if something other than file doesnt exit
		// We expect this fail to not exist most of the time but we dont know if
		// we don't try
		if err = s.deleteFile(pubfile); !os.IsNotExist(err) {
			return err
		}
	}

	return nil
}

func convertMarkdown(page *wikiPage, err error) (*wikiPage, error) {
	if err != nil {
		return page, err
	}
	p := bluemonday.UGCPolicy()
	p.AllowAttrs("class").Matching(regexp.MustCompile("^language-[a-zA-Z0-9]+$")).OnElements("code")

	page.Body = template.HTML(regexp.MustCompile("\r\n").ReplaceAllString(string(page.Body), "\n"))

	unsafe := blackfriday.Run([]byte(page.Body),
		blackfriday.WithExtensions(
			blackfriday.CommonExtensions|
				blackfriday.HardLineBreak|
				blackfriday.HeadingIDs|
				blackfriday.AutoHeadingIDs,
		),
	)

	page.Body = template.HTML(p.SanitizeBytes(unsafe))
	return page, nil

}
func viewHandler(w http.ResponseWriter, r *http.Request, p *wikiPage, s storage) {
	p, err := convertMarkdown(s.getPage(p))
	if err != nil {
		p, err = s.checkForPDF(p)
		if err != nil {
			http.Redirect(w, r, "/wiki/edit/"+p.Title, http.StatusFound)
			return
		}
	} else {
		p.Body = template.HTML(parseWikiWords([]byte(p.Body)))
	}

	renderTemplate(w, "view", p)
}

func vueHandler(w http.ResponseWriter, r *http.Request, p *wikiPage, s storage) {
	renderTemplate(w, "viewjs", p)
}

func editHandler(w http.ResponseWriter, r *http.Request, p *wikiPage, s storage) {
	p, _ = s.getPage(p)
	renderTemplate(w, "edit", p)
}

func makeSearchHandler(fn navFunc, s storage) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		term := r.URL.Query().Get("term") // Get the search term
		if len(term) == 0 {
			http.NotFound(w, r)
			return
		}

		results := ParseQueryResults(s.searchPages(wikiDir, term))
		p := &searchPage{Results: results, basePage: basePage{Title: "Search", Nav: fn(s)}}

		renderTemplate(w, "search", p)
	}
}

func redirectHandler(c Config) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		host := r.Host
		var port string
		hostparts := strings.Split(host, ":")
		if len(hostparts) == 2 {
			host = hostparts[0]
			port = strconv.Itoa(c.HTTPSPort)
		}
		target := "https://" + host
		if len(port) > 0 {
			target += ":" + port

		}
		target += r.URL.Path
		if len(r.URL.RawQuery) > 0 {
			target += "?" + r.URL.RawQuery
		}
		http.Redirect(w, r, target, http.StatusTemporaryRedirect)

	}
}

func simpleHandler(page string, fn navFunc, s storage) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		renderTemplate(w, page, fn(s))
	}

}

func saveHandler(w http.ResponseWriter, r *http.Request, wiki string, s storage) string {
	body := r.FormValue("body")

	body = regexp.MustCompile("\r\n").ReplaceAllString(body, "\n")

	log.Printf("Checkbox is : %v", r.FormValue("wikipub"))
	p := wikiPage{basePage: basePage{Title: wiki}, Body: template.HTML(body), Tags: r.FormValue("wikitags")}
	if r.FormValue("wikipub") == "on" {
		p.Published = true
	}
	if r.FormValue("wikicrypt") == "on" {
		p.Encrypted = true
	}

	err := p.save(s)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return ""
	}
	http.Redirect(w, r, "/wiki/view/"+p.Title, http.StatusFound)

	return r.FormValue("wikitags")
}

var templates = template.Must(template.ParseFiles(
	"views/edit.html",
	"views/view.html",
	"views/viewjs.html",
	"views/pub.html",
	"views/pubhome.html",
	"views/login.html",
	"views/home.html",
	"views/search.html",
	"views/index.html",
	"views/footer.html",
	"views/leftnav.html"))

func renderTemplate(w http.ResponseWriter, tmpl string, p interface{}) {
	err := templates.ExecuteTemplate(w, tmpl+".html", p)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

var validPath = regexp.MustCompile("^/wiki/(edit|save|view|search)/([a-zA-Z0-9\\.\\-_ /]*)$")

func makeHandler(fn func(http.ResponseWriter, *http.Request, *wikiPage, storage), navfn navFunc, s storage) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		wword := r.URL.Query().Get("wword") // Get the wiki word param if available
		if len(wword) == 0 {
			log.Printf("Path is : %v", r.URL.Path)
			m := validPath.FindStringSubmatch(r.URL.Path)
			if m == nil {
				http.NotFound(w, r)
				return
			}
			wword = m[2]
		}
		p := &wikiPage{basePage: basePage{Title: wword, Nav: navfn(s)}}
		fn(w, r, p, s)
	}
}

func processSave(fn func(http.ResponseWriter, *http.Request, string, storage) string, s storage) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		m := validPath.FindStringSubmatch(r.URL.Path)
		if m == nil {
			http.NotFound(w, r)
			return
		}
		fn(w, r, m[2], s)

	}
}

func parseWikiWords(target []byte) []byte {
	var wikiWord = regexp.MustCompile(`\{\{([^\}^#]+)[#]*(.*)\}\}`)

	return wikiWord.ReplaceAll(target, []byte("$1"))
}

func loggingHandler(next http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		t1 := time.Now()
		next.ServeHTTP(w, r)
		t2 := time.Now()
		log.Printf("[%s] %q %v\n", r.Method, r.URL.String(), t2.Sub(t1))
	}
	return http.HandlerFunc(fn)
}

func main() {
	specialDir = []string{"tags", "pub"}
	config, err := LoadConfig()
	if err != nil {
		log.Fatal(err)
	}

	if config.Logfile != "" {
		f, err := os.OpenFile(config.Logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			checkErr(err)
		}
		defer f.Close()

		log.SetOutput(f)
	}

	config.LoadCookieKey()

	auth := NewAuth(*config, persistUsers)

	wikiDir = config.WikiDir
	if !strings.HasSuffix(wikiDir, "/") {
		wikiDir = wikiDir + "/"
	}
	tagDir = wikiDir + "tags/" // Make sure this doesnt double up the / in the path...
	pubDir = wikiDir + "pub/"  // Make sure this doesnt double up the / in the path...

	ekey = []byte(config.EncryptionKey)

	os.Mkdir(config.WikiDir, 0755)
	os.Mkdir(config.WikiDir+"tags", 0755)

	authHandlers := alice.New(loggingHandler, auth.validate)
	noauthHandlers := alice.New(loggingHandler)

	httpmux := http.NewServeMux()
	httpsmux := httpmux
	// setup wiki on https
	if config.UseHTTPS {
		httpsmux = http.NewServeMux()
	}

	fstore := fileStorage{tagDir}

	httpsmux.Handle("/wiki", authHandlers.ThenFunc(simpleHandler("home", getNav, fstore)))
	httpsmux.Handle("/wiki/login/", noauthHandlers.ThenFunc(auth.loginHandler))
	httpsmux.Handle("/wiki/register/", noauthHandlers.ThenFunc(auth.registerHandler))
	httpsmux.Handle("/wiki/logout/", authHandlers.ThenFunc(logoutHandler))
	httpsmux.Handle("/wiki/search/", authHandlers.ThenFunc(makeSearchHandler(getNav, fstore)))
	httpsmux.Handle("/wiki/view/", authHandlers.ThenFunc(makeHandler(viewHandler, getNav, fstore)))
	httpsmux.Handle("/wiki/edit/", authHandlers.ThenFunc(makeHandler(editHandler, getNav, fstore)))
	httpsmux.Handle("/wiki/save/", authHandlers.ThenFunc(processSave(saveHandler, fstore)))
	httpsmux.Handle("/wiki/raw/", http.StripPrefix("/wiki/raw/", http.FileServer(http.Dir(wikiDir))))
	httpsmux.Handle("/pub/", noauthHandlers.ThenFunc(makePubHandler(pubHandler, getNav, fstore)))
	httpsmux.Handle("/pub", noauthHandlers.ThenFunc(simpleHandler("pubhome", getPubNav, fstore)))
	httpsmux.Handle("/api", noauthHandlers.ThenFunc(apiHandler(innerAPIHandler, fstore)))

	if config.UseHTTPS {
		// Any routes that duplicate the http routing are only done here
		httpsmux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
		go http.ListenAndServeTLS(
			":"+strconv.Itoa(config.HTTPSPort),
			config.CertPath,
			config.KeyPath,
			httpsmux)

		httpmux.HandleFunc("/wiki", redirectHandler(*config))
		httpsmux.Handle("/", http.FileServer(http.Dir("wwwroot")))
	}

	// Listen for normal traffic against root
	httpmux.Handle("/", http.FileServer(http.Dir("wwwroot")))
	httpmux.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
	err = http.ListenAndServe(":"+strconv.Itoa(config.HTTPPort), httpmux)
	checkErr(err)

}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"mahonia.googlecode.com/hg"
	"os"
	"unicode"
	"utf8"
)

// wordRune maps c into a reduced set of "word" characters.
// If c is a letter, it returns it in lowercase.
// It it it is a digit, it returns it unchanged.
// Otherwise it returns a space.
func wordRune(c int) int {
	switch {
	case c >= 'a' && c <= 'z' || c >= '0' && c <= '9':
		return c
	case c >= 'A' && c <= 'Z':
		return c + ('a' - 'A')
	case c < 128:
		return ' '
	case unicode.IsDigit(c):
		return c
	case unicode.IsLetter(c):
		return unicode.ToLower(c)
	}

	return ' '
}

// wordString applies wordRune to each character in s and removes extra spaces.
func wordString(s string) string {
	runes := make([]int, 0, 20)
	prevRune := 0

	for _, c := range s {
		c = wordRune(c)
		if c != ' ' || prevRune != ' ' {
			runes = append(runes, c)
		}
		prevRune = c
	}

	return string(runes)
}

// A wordReader applies two filters to the data from another os.Reader:
// first it decodes the character set; then it applies wordRune to each character
// and removes extra spaces.
type wordReader struct {
	buf      []byte
	rd       io.Reader
	decode   mahonia.Decoder
	r, w     int
	err      os.Error
	prevRune int
}

func newWordReader(rd io.Reader, d mahonia.Decoder) *wordReader {
	b := new(wordReader)
	b.buf = make([]byte, 4096)
	b.rd = rd
	b.decode = d
	return b
}

// fill reads a new chunk into the buffer.
func (b *wordReader) fill() {
	// Slide existing data to beginning.
	if b.r > 0 {
		copy(b.buf, b.buf[b.r:b.w])
		b.w -= b.r
		b.r = 0
	}

	// Read new data.
	n, e := b.rd.Read(b.buf[b.w:])
	b.w += n
	if e != nil {
		b.err = e
	}
}

// Read reads data into p.
// It returns the number of bytes read into p.
// It calls Read at most once on the underlying Reader,
// hence n may be less than len(p).
// At EOF, the count will be zero and err will be os.EOF.
func (b *wordReader) Read(p []byte) (n int, err os.Error) {
	n = len(p)
	filled := false
	if n == 0 {
		return 0, b.err
	}
	if b.w == b.r {
		if b.err != nil {
			return 0, b.err
		}
		if n > len(b.buf) {
			// Large read, empty buffer.
			// Allocate a larger buffer for efficiency.
			b.buf = make([]byte, n)
		}
		b.fill()
		filled = true
		if b.w == b.r {
			return 0, b.err
		}
	}

	i := 0
	prev := b.prevRune
	for i < n {
		rune, size, status := b.decode(b.buf[b.r:b.w])

		if status == mahonia.STATE_ONLY {
			b.r += size
			continue
		}

		if status == mahonia.NO_ROOM {
			if b.err != nil {
				rune = 0xfffd
				size = b.w - b.r
				status = mahonia.INVALID_CHAR
			} else if filled {
				break
			} else {
				b.fill()
				filled = true
				continue
			}
		}

		rune = wordRune(rune)
		if rune == ' ' && prev == ' ' {
			b.r += size
			continue
		}

		if i+utf8.RuneLen(rune) > n {
			break
		}

		b.r += size
		if rune < 128 {
			p[i] = byte(rune)
			i++
		} else {
			i += utf8.EncodeRune(p[i:], rune)
		}
		prev = rune
	}

	b.prevRune = prev
	return i, nil
}
<commit_msg>Add break to wordReader.Read at EOF.<commit_after>package main

import (
	"io"
	"mahonia.googlecode.com/hg"
	"os"
	"unicode"
	"utf8"
)

// wordRune maps c into a reduced set of "word" characters.
// If c is a letter, it returns it in lowercase.
// It it it is a digit, it returns it unchanged.
// Otherwise it returns a space.
func wordRune(c int) int {
	switch {
	case c >= 'a' && c <= 'z' || c >= '0' && c <= '9':
		return c
	case c >= 'A' && c <= 'Z':
		return c + ('a' - 'A')
	case c < 128:
		return ' '
	case unicode.IsDigit(c):
		return c
	case unicode.IsLetter(c):
		return unicode.ToLower(c)
	}

	return ' '
}

// wordString applies wordRune to each character in s and removes extra spaces.
func wordString(s string) string {
	runes := make([]int, 0, 20)
	prevRune := 0

	for _, c := range s {
		c = wordRune(c)
		if c != ' ' || prevRune != ' ' {
			runes = append(runes, c)
		}
		prevRune = c
	}

	return string(runes)
}

// A wordReader applies two filters to the data from another os.Reader:
// first it decodes the character set; then it applies wordRune to each character
// and removes extra spaces.
type wordReader struct {
	buf      []byte
	rd       io.Reader
	decode   mahonia.Decoder
	r, w     int
	err      os.Error
	prevRune int
}

func newWordReader(rd io.Reader, d mahonia.Decoder) *wordReader {
	b := new(wordReader)
	b.buf = make([]byte, 4096)
	b.rd = rd
	b.decode = d
	return b
}

// fill reads a new chunk into the buffer.
func (b *wordReader) fill() {
	// Slide existing data to beginning.
	if b.r > 0 {
		copy(b.buf, b.buf[b.r:b.w])
		b.w -= b.r
		b.r = 0
	}

	// Read new data.
	n, e := b.rd.Read(b.buf[b.w:])
	b.w += n
	if e != nil {
		b.err = e
	}
}

// Read reads data into p.
// It returns the number of bytes read into p.
// It calls Read at most once on the underlying Reader,
// hence n may be less than len(p).
// At EOF, the count will be zero and err will be os.EOF.
func (b *wordReader) Read(p []byte) (n int, err os.Error) {
	n = len(p)
	filled := false
	if n == 0 {
		return 0, b.err
	}
	if b.w == b.r {
		if b.err != nil {
			return 0, b.err
		}
		if n > len(b.buf) {
			// Large read, empty buffer.
			// Allocate a larger buffer for efficiency.
			b.buf = make([]byte, n)
		}
		b.fill()
		filled = true
		if b.w == b.r {
			return 0, b.err
		}
	}

	i := 0
	prev := b.prevRune
	for i < n {
		rune, size, status := b.decode(b.buf[b.r:b.w])

		if status == mahonia.STATE_ONLY {
			b.r += size
			continue
		}

		if status == mahonia.NO_ROOM {
			if b.err != nil {
				rune = 0xfffd
				size = b.w - b.r
				if size == 0 {
					break
				}
				status = mahonia.INVALID_CHAR
			} else if filled {
				break
			} else {
				b.fill()
				filled = true
				continue
			}
		}

		rune = wordRune(rune)
		if rune == ' ' && prev == ' ' {
			b.r += size
			continue
		}

		if i+utf8.RuneLen(rune) > n {
			break
		}

		b.r += size
		if rune < 128 {
			p[i] = byte(rune)
			i++
		} else {
			i += utf8.EncodeRune(p[i:], rune)
		}
		prev = rune
	}

	b.prevRune = prev
	return i, nil
}
<|endoftext|>