<commit_before>package grok

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
)

var (
	canonical = regexp.MustCompile(`%{(\w+(?::\w+(?::\w+)?)?)}`)
	normal    = regexp.MustCompile(`%{([\w-.]+(?::[\w-.]+(?::[\w-.]+)?)?)}`)
	symbolic  = regexp.MustCompile(`\W`)
)

// A Config structure is used to configure a Grok parser.
type Config struct {
	NamedCapturesOnly   bool
	SkipDefaultPatterns bool
	RemoveEmptyValues   bool
	PatternsDir         []string
	Patterns            map[string]string
}

// Grok object us used to load patterns and deconstruct strings using those
// patterns.
type Grok struct {
	rawPattern       map[string]string
	config           *Config
	aliases          map[string]string
	compiledPatterns map[string]*gRegexp
	patterns         map[string]*gPattern
	patternsGuard    *sync.RWMutex
	compiledGuard    *sync.RWMutex
}

type gPattern struct {
	expression string
	typeInfo   semanticTypes
}

type gRegexp struct {
	regexp   *regexp.Regexp
	typeInfo semanticTypes
}

type semanticTypes map[string]string

// New returns a Grok object.
func New() (*Grok, error) {
	return NewWithConfig(&Config{})
}

// NewWithConfig returns a Grok object that is configured to behave according
// to the supplied Config structure.
func NewWithConfig(config *Config) (*Grok, error) {
	g := &Grok{
		config:           config,
		aliases:          map[string]string{},
		compiledPatterns: map[string]*gRegexp{},
		patterns:         map[string]*gPattern{},
		rawPattern:       map[string]string{},
		patternsGuard:    new(sync.RWMutex),
		compiledGuard:    new(sync.RWMutex),
	}

	if !config.SkipDefaultPatterns {
		err := g.AddPatternsFromMap(patterns)
		if err != nil {
			return nil, err
		}
	}

	if len(config.PatternsDir) > 0 {
		for _, path := range config.PatternsDir {
			err := g.AddPatternsFromPath(path)
			if err != nil {
				return nil, err
			}
		}

	}

	if err := g.AddPatternsFromMap(config.Patterns); err != nil {
		return nil, err
	}

	return g, nil
}

// AddPattern adds a new pattern to the list of loaded patterns.
func (g *Grok) addPattern(name, pattern string) error {
	dnPattern, ti, err := g.denormalizePattern(pattern, g.patterns)
	if err != nil {
		return err
	}

	g.patterns[name] = &gPattern{expression: dnPattern, typeInfo: ti}
	return nil
}

// AddPattern adds a named pattern to grok
func (g *Grok) AddPattern(name, pattern string) error {
	g.patternsGuard.Lock()
	defer g.patternsGuard.Unlock()

	g.rawPattern[name] = pattern
	return g.buildPatterns()
}

// AddPatternsFromMap loads a map of named patterns
func (g *Grok) AddPatternsFromMap(m map[string]string) error {
	g.patternsGuard.Lock()
	defer g.patternsGuard.Unlock()

	for name, pattern := range m {
		g.rawPattern[name] = pattern
	}
	return g.buildPatterns()
}

// AddPatternsFromMap adds new patterns from the specified map to the list of
// loaded patterns.
func (g *Grok) addPatternsFromMap(m map[string]string) error {
	patternDeps := graph{}
	for k, v := range m {
		keys := []string{}
		for _, key := range canonical.FindAllStringSubmatch(v, -1) {
			names := strings.Split(key[1], ":")
			syntax := names[0]
			if g.patterns[syntax] == nil {
				if _, ok := m[syntax]; !ok {
					return fmt.Errorf("no pattern found for %%{%s}", syntax)
				}
			}
			keys = append(keys, syntax)
		}
		patternDeps[k] = keys
	}
	order, _ := sortGraph(patternDeps)
	for _, key := range reverseList(order) {
		err := g.addPattern(key, m[key])
		if err != nil {
			return fmt.Errorf("cannot add pattern %q: %v", key, err)
		}
	}

	return nil
}

// AddPatternsFromPath adds new patterns from the files in the specified
// directory to the list of loaded patterns.
func (g *Grok) AddPatternsFromPath(path string) error {
	if fi, err := os.Stat(path); err == nil {
		if fi.IsDir() {
			path = path + "/*"
		}
	} else {
		return fmt.Errorf("invalid path : %s", path)
	}

	// only one error can be raised, when pattern is malformed
	// pattern is hard-coded "/*" so we ignore err
	files, _ := filepath.Glob(path)

	var filePatterns = map[string]string{}
	for _, fileName := range files {
		file, err := os.Open(fileName)
		if err != nil {
			return err
		}

		scanner := bufio.NewScanner(bufio.NewReader(file))

		for scanner.Scan() {
			l := scanner.Text()
			if len(l) > 0 && l[0] != '#' {
				names := strings.SplitN(l, " ", 2)
				filePatterns[names[0]] = names[1]
			}
		}

		file.Close()
	}

	return g.AddPatternsFromMap(filePatterns)
}

// Match returns true if the specified text matches the pattern.
func (g *Grok) Match(pattern, text string) (bool, error) {
	gr, err := g.compile(pattern)
	if err != nil {
		return false, err
	}

	if ok := gr.regexp.MatchString(text); !ok {
		return false, nil
	}

	return true, nil
}

// compiledParse parses the specified text and returns a map with the results.
func (g *Grok) compiledParse(gr *gRegexp, text string) (map[string]string, error) {
	captures := make(map[string]string)
	if match := gr.regexp.FindStringSubmatch(text); len(match) > 0 {
		for i, name := range gr.regexp.SubexpNames() {
			if name != "" {
				if g.config.RemoveEmptyValues && match[i] == "" {
					continue
				}
				name = g.nameToAlias(name)
				captures[name] = match[i]
			}
		}
	}

	return captures, nil
}

// Parse the specified text and return a map with the results.
func (g *Grok) Parse(pattern, text string) (map[string]string, error) {
	gr, err := g.compile(pattern)
	if err != nil {
		return nil, err
	}

	return g.compiledParse(gr, text)
}

// ParseTyped returns a inteface{} map with typed captured fields based on provided pattern over the text
func (g *Grok) ParseTyped(pattern string, text string) (map[string]interface{}, error) {
	gr, err := g.compile(pattern)
	if err != nil {
		return nil, err
	}
	match := gr.regexp.FindStringSubmatch(text)
	captures := make(map[string]interface{})
	if len(match) > 0 {
		for i, segmentName := range gr.regexp.SubexpNames() {
			if len(segmentName) != 0 {
				if g.config.RemoveEmptyValues == true && match[i] == "" {
					continue
				}
				name := g.nameToAlias(segmentName)
				if segmentType, ok := gr.typeInfo[name]; ok {
					switch segmentType {
					case "int":
						captures[name], _ = strconv.Atoi(match[i])
					case "float":
						captures[name], _ = strconv.ParseFloat(match[i], 64)
					default:
						return nil, fmt.Errorf("ERROR the value %s cannot be converted to %s", match[i], segmentType)
					}
				} else {
					captures[name] = match[i]
				}
			}

		}
	}

	return captures, nil
}

// ParseToMultiMap parses the specified text and returns a map with the
// results. Values are stored in an string slice, so values from captures with
// the same name don't get overridden.
func (g *Grok) ParseToMultiMap(pattern, text string) (map[string][]string, error) {
	gr, err := g.compile(pattern)
	if err != nil {
		return nil, err
	}

	captures := make(map[string][]string)
	if match := gr.regexp.FindStringSubmatch(text); len(match) > 0 {
		for i, name := range gr.regexp.SubexpNames() {
			if name != "" {
				if g.config.RemoveEmptyValues == true && match[i] == "" {
					continue
				}
				name = g.nameToAlias(name)
				captures[name] = append(captures[name], match[i])
			}
		}
	}

	return captures, nil
}

func (g *Grok) buildPatterns() error {
	g.patterns = map[string]*gPattern{}
	return g.addPatternsFromMap(g.rawPattern)
}

func (g *Grok) compile(pattern string) (*gRegexp, error) {
	g.compiledGuard.RLock()
	gr, ok := g.compiledPatterns[pattern]
	g.compiledGuard.RUnlock()

	if ok {
		return gr, nil
	}

	g.patternsGuard.RLock()
	newPattern, ti, err := g.denormalizePattern(pattern, g.patterns)
	g.patternsGuard.RUnlock()
	if err != nil {
		return nil, err
	}

	compiledRegex, err := regexp.Compile(newPattern)
	if err != nil {
		return nil, err
	}
	gr = &gRegexp{regexp: compiledRegex, typeInfo: ti}

	g.compiledGuard.Lock()
	g.compiledPatterns[pattern] = gr
	g.compiledGuard.Unlock()

	return gr, nil
}

func (g *Grok) denormalizePattern(pattern string, storedPatterns map[string]*gPattern) (string, semanticTypes, error) {
	ti := semanticTypes{}
	for _, values := range normal.FindAllStringSubmatch(pattern, -1) {
		names := strings.Split(values[1], ":")

		syntax, semantic, alias := names[0], names[0], names[0]
		if len(names) > 1 {
			semantic = names[1]
			alias = g.aliasizePatternName(semantic)
		}

		// Add type cast information only if type set, and not string
		if len(names) == 3 {
			if names[2] != "string" {
				ti[semantic] = names[2]
			}
		}

		storedPattern, ok := storedPatterns[syntax]
		if !ok {
			return "", ti, fmt.Errorf("no pattern found for %%{%s}", syntax)
		}

		var buffer bytes.Buffer
		if !g.config.NamedCapturesOnly || (g.config.NamedCapturesOnly && len(names) > 1) {
			buffer.WriteString("(?P<")
			buffer.WriteString(alias)
			buffer.WriteString(">")
			buffer.WriteString(storedPattern.expression)
			buffer.WriteString(")")
		} else {
			buffer.WriteString("(")
			buffer.WriteString(storedPattern.expression)
			buffer.WriteString(")")
		}

		//Merge type Informations
		for k, v := range storedPattern.typeInfo {
			//Lastest type information is the one to keep in memory
			if _, ok := ti[k]; !ok {
				ti[k] = v
			}
		}

		pattern = strings.Replace(pattern, values[0], buffer.String(), -1)
	}

	return pattern, ti, nil

}

func (g *Grok) aliasizePatternName(name string) string {
	alias := symbolic.ReplaceAllString(name, "_")
	g.aliases[alias] = name
	return alias
}

func (g *Grok) nameToAlias(name string) string {
	alias, ok := g.aliases[name]
	if ok {
		return alias
	}
	return name
}

// ParseStream will match the given pattern on a line by line basis from the reader
// and apply the results to the process function
func (g *Grok) ParseStream(reader *bufio.Reader, pattern string, process func(map[string]string) error) error {
	gr, err := g.compile(pattern)
	if err != nil {
		return err
	}
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		values, err := g.compiledParse(gr, line)
		if err != nil {
			return err
		}
		if err = process(values); err != nil {
			return err
		}
	}
}
<commit_msg>Fix typo.<commit_after>package grok

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
)

var (
	canonical = regexp.MustCompile(`%{(\w+(?::\w+(?::\w+)?)?)}`)
	normal    = regexp.MustCompile(`%{([\w-.]+(?::[\w-.]+(?::[\w-.]+)?)?)}`)
	symbolic  = regexp.MustCompile(`\W`)
)

// A Config structure is used to configure a Grok parser.
type Config struct {
	NamedCapturesOnly   bool
	SkipDefaultPatterns bool
	RemoveEmptyValues   bool
	PatternsDir         []string
	Patterns            map[string]string
}

// Grok object us used to load patterns and deconstruct strings using those
// patterns.
type Grok struct {
	rawPattern       map[string]string
	config           *Config
	aliases          map[string]string
	compiledPatterns map[string]*gRegexp
	patterns         map[string]*gPattern
	patternsGuard    *sync.RWMutex
	compiledGuard    *sync.RWMutex
}

type gPattern struct {
	expression string
	typeInfo   semanticTypes
}

type gRegexp struct {
	regexp   *regexp.Regexp
	typeInfo semanticTypes
}

type semanticTypes map[string]string

// New returns a Grok object.
func New() (*Grok, error) {
	return NewWithConfig(&Config{})
}

// NewWithConfig returns a Grok object that is configured to behave according
// to the supplied Config structure.
func NewWithConfig(config *Config) (*Grok, error) {
	g := &Grok{
		config:           config,
		aliases:          map[string]string{},
		compiledPatterns: map[string]*gRegexp{},
		patterns:         map[string]*gPattern{},
		rawPattern:       map[string]string{},
		patternsGuard:    new(sync.RWMutex),
		compiledGuard:    new(sync.RWMutex),
	}

	if !config.SkipDefaultPatterns {
		err := g.AddPatternsFromMap(patterns)
		if err != nil {
			return nil, err
		}
	}

	if len(config.PatternsDir) > 0 {
		for _, path := range config.PatternsDir {
			err := g.AddPatternsFromPath(path)
			if err != nil {
				return nil, err
			}
		}

	}

	if err := g.AddPatternsFromMap(config.Patterns); err != nil {
		return nil, err
	}

	return g, nil
}

// AddPattern adds a new pattern to the list of loaded patterns.
func (g *Grok) addPattern(name, pattern string) error {
	dnPattern, ti, err := g.denormalizePattern(pattern, g.patterns)
	if err != nil {
		return err
	}

	g.patterns[name] = &gPattern{expression: dnPattern, typeInfo: ti}
	return nil
}

// AddPattern adds a named pattern to grok
func (g *Grok) AddPattern(name, pattern string) error {
	g.patternsGuard.Lock()
	defer g.patternsGuard.Unlock()

	g.rawPattern[name] = pattern
	return g.buildPatterns()
}

// AddPatternsFromMap loads a map of named patterns
func (g *Grok) AddPatternsFromMap(m map[string]string) error {
	g.patternsGuard.Lock()
	defer g.patternsGuard.Unlock()

	for name, pattern := range m {
		g.rawPattern[name] = pattern
	}
	return g.buildPatterns()
}

// AddPatternsFromMap adds new patterns from the specified map to the list of
// loaded patterns.
func (g *Grok) addPatternsFromMap(m map[string]string) error {
	patternDeps := graph{}
	for k, v := range m {
		keys := []string{}
		for _, key := range canonical.FindAllStringSubmatch(v, -1) {
			names := strings.Split(key[1], ":")
			syntax := names[0]
			if g.patterns[syntax] == nil {
				if _, ok := m[syntax]; !ok {
					return fmt.Errorf("no pattern found for %%{%s}", syntax)
				}
			}
			keys = append(keys, syntax)
		}
		patternDeps[k] = keys
	}
	order, _ := sortGraph(patternDeps)
	for _, key := range reverseList(order) {
		err := g.addPattern(key, m[key])
		if err != nil {
			return fmt.Errorf("cannot add pattern %q: %v", key, err)
		}
	}

	return nil
}

// AddPatternsFromPath adds new patterns from the files in the specified
// directory to the list of loaded patterns.
func (g *Grok) AddPatternsFromPath(path string) error {
	if fi, err := os.Stat(path); err == nil {
		if fi.IsDir() {
			path = path + "/*"
		}
	} else {
		return fmt.Errorf("invalid path : %s", path)
	}

	// only one error can be raised, when pattern is malformed
	// pattern is hard-coded "/*" so we ignore err
	files, _ := filepath.Glob(path)

	var filePatterns = map[string]string{}
	for _, fileName := range files {
		file, err := os.Open(fileName)
		if err != nil {
			return err
		}

		scanner := bufio.NewScanner(bufio.NewReader(file))

		for scanner.Scan() {
			l := scanner.Text()
			if len(l) > 0 && l[0] != '#' {
				names := strings.SplitN(l, " ", 2)
				filePatterns[names[0]] = names[1]
			}
		}

		file.Close()
	}

	return g.AddPatternsFromMap(filePatterns)
}

// Match returns true if the specified text matches the pattern.
func (g *Grok) Match(pattern, text string) (bool, error) {
	gr, err := g.compile(pattern)
	if err != nil {
		return false, err
	}

	if ok := gr.regexp.MatchString(text); !ok {
		return false, nil
	}

	return true, nil
}

// compiledParse parses the specified text and returns a map with the results.
func (g *Grok) compiledParse(gr *gRegexp, text string) (map[string]string, error) {
	captures := make(map[string]string)
	if match := gr.regexp.FindStringSubmatch(text); len(match) > 0 {
		for i, name := range gr.regexp.SubexpNames() {
			if name != "" {
				if g.config.RemoveEmptyValues && match[i] == "" {
					continue
				}
				name = g.nameToAlias(name)
				captures[name] = match[i]
			}
		}
	}

	return captures, nil
}

// Parse the specified text and return a map with the results.
func (g *Grok) Parse(pattern, text string) (map[string]string, error) {
	gr, err := g.compile(pattern)
	if err != nil {
		return nil, err
	}

	return g.compiledParse(gr, text)
}

// ParseTyped returns a interface{} map with typed captured fields based on provided pattern over the text
func (g *Grok) ParseTyped(pattern string, text string) (map[string]interface{}, error) {
	gr, err := g.compile(pattern)
	if err != nil {
		return nil, err
	}
	match := gr.regexp.FindStringSubmatch(text)
	captures := make(map[string]interface{})
	if len(match) > 0 {
		for i, segmentName := range gr.regexp.SubexpNames() {
			if len(segmentName) != 0 {
				if g.config.RemoveEmptyValues == true && match[i] == "" {
					continue
				}
				name := g.nameToAlias(segmentName)
				if segmentType, ok := gr.typeInfo[name]; ok {
					switch segmentType {
					case "int":
						captures[name], _ = strconv.Atoi(match[i])
					case "float":
						captures[name], _ = strconv.ParseFloat(match[i], 64)
					default:
						return nil, fmt.Errorf("ERROR the value %s cannot be converted to %s", match[i], segmentType)
					}
				} else {
					captures[name] = match[i]
				}
			}

		}
	}

	return captures, nil
}

// ParseToMultiMap parses the specified text and returns a map with the
// results. Values are stored in an string slice, so values from captures with
// the same name don't get overridden.
func (g *Grok) ParseToMultiMap(pattern, text string) (map[string][]string, error) {
	gr, err := g.compile(pattern)
	if err != nil {
		return nil, err
	}

	captures := make(map[string][]string)
	if match := gr.regexp.FindStringSubmatch(text); len(match) > 0 {
		for i, name := range gr.regexp.SubexpNames() {
			if name != "" {
				if g.config.RemoveEmptyValues == true && match[i] == "" {
					continue
				}
				name = g.nameToAlias(name)
				captures[name] = append(captures[name], match[i])
			}
		}
	}

	return captures, nil
}

func (g *Grok) buildPatterns() error {
	g.patterns = map[string]*gPattern{}
	return g.addPatternsFromMap(g.rawPattern)
}

func (g *Grok) compile(pattern string) (*gRegexp, error) {
	g.compiledGuard.RLock()
	gr, ok := g.compiledPatterns[pattern]
	g.compiledGuard.RUnlock()

	if ok {
		return gr, nil
	}

	g.patternsGuard.RLock()
	newPattern, ti, err := g.denormalizePattern(pattern, g.patterns)
	g.patternsGuard.RUnlock()
	if err != nil {
		return nil, err
	}

	compiledRegex, err := regexp.Compile(newPattern)
	if err != nil {
		return nil, err
	}
	gr = &gRegexp{regexp: compiledRegex, typeInfo: ti}

	g.compiledGuard.Lock()
	g.compiledPatterns[pattern] = gr
	g.compiledGuard.Unlock()

	return gr, nil
}

func (g *Grok) denormalizePattern(pattern string, storedPatterns map[string]*gPattern) (string, semanticTypes, error) {
	ti := semanticTypes{}
	for _, values := range normal.FindAllStringSubmatch(pattern, -1) {
		names := strings.Split(values[1], ":")

		syntax, semantic, alias := names[0], names[0], names[0]
		if len(names) > 1 {
			semantic = names[1]
			alias = g.aliasizePatternName(semantic)
		}

		// Add type cast information only if type set, and not string
		if len(names) == 3 {
			if names[2] != "string" {
				ti[semantic] = names[2]
			}
		}

		storedPattern, ok := storedPatterns[syntax]
		if !ok {
			return "", ti, fmt.Errorf("no pattern found for %%{%s}", syntax)
		}

		var buffer bytes.Buffer
		if !g.config.NamedCapturesOnly || (g.config.NamedCapturesOnly && len(names) > 1) {
			buffer.WriteString("(?P<")
			buffer.WriteString(alias)
			buffer.WriteString(">")
			buffer.WriteString(storedPattern.expression)
			buffer.WriteString(")")
		} else {
			buffer.WriteString("(")
			buffer.WriteString(storedPattern.expression)
			buffer.WriteString(")")
		}

		//Merge type Informations
		for k, v := range storedPattern.typeInfo {
			//Lastest type information is the one to keep in memory
			if _, ok := ti[k]; !ok {
				ti[k] = v
			}
		}

		pattern = strings.Replace(pattern, values[0], buffer.String(), -1)
	}

	return pattern, ti, nil

}

func (g *Grok) aliasizePatternName(name string) string {
	alias := symbolic.ReplaceAllString(name, "_")
	g.aliases[alias] = name
	return alias
}

func (g *Grok) nameToAlias(name string) string {
	alias, ok := g.aliases[name]
	if ok {
		return alias
	}
	return name
}

// ParseStream will match the given pattern on a line by line basis from the reader
// and apply the results to the process function
func (g *Grok) ParseStream(reader *bufio.Reader, pattern string, process func(map[string]string) error) error {
	gr, err := g.compile(pattern)
	if err != nil {
		return err
	}
	for {
		line, err := reader.ReadString('\n')
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
		values, err := g.compiledParse(gr, line)
		if err != nil {
			return err
		}
		if err = process(values); err != nil {
			return err
		}
	}
}
<|endoftext|>


<commit_before>// © 2014, Robert Hülle

package main

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"os"
	"regexp"
	"strings"
)

const (
	bufferSize = 0x4000
)

type patternID struct {
	pattern *regexp.Regexp
	prefix  string
	suffix  string
}

var patternIDs = []patternID{
	{
		regexp.MustCompile("([0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12})"),
		"#",
		"^EBOK",
	},
	{
		regexp.MustCompile("\x12(B[0-9][0-9A-Z]{5,})"),
		"#",
		"^EBOK",
	},
}

func bookHash(path string) string {
	switch {
	case strings.HasSuffix(path, ".mobi"):
		return mobiHash(path)
	default:
		return pathHash(path)
	}
}

func pathHash(path string) string {
	if !strings.HasPrefix(path, "/mnt/us/") {
		path = "/mnt/us/" + path
	}
	h := sha1.Sum([]byte(path))
	hs := hex.EncodeToString(h[:])
	return "*" + hs
}

func mobiHash(path string) string {
	data := make([]byte, bufferSize)
	n := readMax(data, path)
	data = data[:n]
	hash := findPattern(data)
	if len(hash) > 0 {
		return hash
	}
	return pathHash(path)
}

func findPattern(data []byte) string {
	for _, pid := range patternIDs {
		res := pid.pattern.FindSubmatch(data)
		if len(res) > 1 {
			return pid.prefix + string(res[1]) + pid.suffix
		}
	}
	return ""
}

func readMax(data []byte, path string) int {
	f, err := os.Open(path)
	n := 0
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not open file: %v\n", err)
		return 0
	}
	defer f.Close()
	for {
		nn, _ := f.Read(data)
		data = data[nn:]
		n += nn
		if nn == 0 {
			break
		}
	}
	return n
}
<commit_msg>fix filepath when running outside /mnt/us<commit_after>// © 2014, Robert Hülle

package main

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"os"
	"regexp"
	"strings"
)

const (
	bufferSize = 0x4000
)

type patternID struct {
	pattern *regexp.Regexp
	prefix  string
	suffix  string
}

var patternIDs = []patternID{
	{
		regexp.MustCompile("([0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12})"),
		"#",
		"^EBOK",
	},
	{
		regexp.MustCompile("\x12(B[0-9][0-9A-Z]{5,})"),
		"#",
		"^EBOK",
	},
}

func bookHash(path string) string {
	switch {
	case strings.HasSuffix(path, ".mobi"):
		return mobiHash(path)
	default:
		return pathHash(path)
	}
}

func pathHash(path string) string {
	if !strings.HasPrefix(path, "/mnt/us/") {
		// strip non-standard prefix from path and replace it with /mnt/us
		i := len(KindleDir) + 1
		path = "/mnt/us/" + path[i:]
	}
	h := sha1.Sum([]byte(path))
	hs := hex.EncodeToString(h[:])
	return "*" + hs
}

func mobiHash(path string) string {
	data := make([]byte, bufferSize)
	n := readMax(data, path)
	data = data[:n]
	hash := findPattern(data)
	if len(hash) > 0 {
		return hash
	}
	return pathHash(path)
}

func findPattern(data []byte) string {
	for _, pid := range patternIDs {
		res := pid.pattern.FindSubmatch(data)
		if len(res) > 1 {
			return pid.prefix + string(res[1]) + pid.suffix
		}
	}
	return ""
}

func readMax(data []byte, path string) int {
	f, err := os.Open(path)
	n := 0
	if err != nil {
		fmt.Fprintf(os.Stderr, "can not open file: %v\n", err)
		return 0
	}
	defer f.Close()
	for {
		nn, _ := f.Read(data)
		data = data[nn:]
		n += nn
		if nn == 0 {
			break
		}
	}
	return n
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
	"text/tabwriter"
	"text/template"
)

// The text template for the Default help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...]

VERSION:
   {{.Version}}{{if or .Author .Email}}

AUTHOR:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
   {{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`

// The text template for the command help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var CommandHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   command {{.Name}}{{if .Flags}} [command options]{{end}} [arguments...]{{if .Description}}

DESCRIPTION:
   {{.Description}}{{end}}{{if .Flags}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{ end }}
`

// The text template for the subcommand help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var SubcommandHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} command{{if .Flags}} [command options]{{end}} [arguments...]

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
   {{end}}{{if .Flags}}
OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`

var helpCommand = Command{
	Name:      "help",
	ShortName: "h",
	Usage:     "Shows a list of commands or help for one command",
	Action: func(c *Context) {
		args := c.Args()
		if args.Present() {
			ShowCommandHelp(c, args.First())
		} else {
			ShowAppHelp(c)
		}
	},
}

var helpSubcommand = Command{
	Name:      "help",
	ShortName: "h",
	Usage:     "Shows a list of commands or help for one command",
	Action: func(c *Context) {
		args := c.Args()
		if args.Present() {
			ShowCommandHelp(c, args.First())
		} else {
			ShowSubcommandHelp(c)
		}
	},
}

// Prints help for the App
var HelpPrinter = printHelp

// Prints version for the App
var VersionPrinter = printVersion

func ShowAppHelp(c *Context) {
	HelpPrinter(AppHelpTemplate, c.App)
}

// Prints the list of subcommands as the default app completion method
func DefaultAppComplete(c *Context) {
	for _, command := range c.App.Commands {
		fmt.Println(command.Name)
		if command.ShortName != "" {
			fmt.Println(command.ShortName)
		}
	}
}

// Prints help for the given command
func ShowCommandHelp(c *Context, command string) {
	for _, c := range c.App.Commands {
		if c.HasName(command) {
			HelpPrinter(CommandHelpTemplate, c)
			return
		}
	}

	if c.App.CommandNotFound != nil {
		c.App.CommandNotFound(c, command)
	} else {
		fmt.Printf("No help topic for '%v'\n", command)
	}
}

// Prints help for the given subcommand
func ShowSubcommandHelp(c *Context) {
	HelpPrinter(SubcommandHelpTemplate, c.App)
}

// Prints the version number of the App
func ShowVersion(c *Context) {
	VersionPrinter(c)
}

func printVersion(c *Context) {
	fmt.Printf("%v version %v\n", c.App.Name, c.App.Version)
}

// Prints the lists of commands within a given context
func ShowCompletions(c *Context) {
	a := c.App
	if a != nil && a.BashComplete != nil {
		a.BashComplete(c)
	}
}

// Prints the custom completions for a given command
func ShowCommandCompletions(ctx *Context, command string) {
	c := ctx.App.Command(command)
	if c != nil && c.BashComplete != nil {
		c.BashComplete(ctx)
	}
}

func printHelp(templ string, data interface{}) {
	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Parse(templ))
	err := t.Execute(w, data)
	if err != nil {
		panic(err)
	}
	w.Flush()
}

func checkVersion(c *Context) bool {
	if c.GlobalBool("version") {
		ShowVersion(c)
		return true
	}

	return false
}

func checkHelp(c *Context) bool {
	if c.GlobalBool("h") || c.GlobalBool("help") {
		ShowAppHelp(c)
		return true
	}

	return false
}

func checkCommandHelp(c *Context, name string) bool {
	if c.Bool("h") || c.Bool("help") {
		ShowCommandHelp(c, name)
		return true
	}

	return false
}

func checkSubcommandHelp(c *Context) bool {
	if c.GlobalBool("h") || c.GlobalBool("help") {
		ShowSubcommandHelp(c)
		return true
	}

	return false
}

func checkCompletions(c *Context) bool {
	if c.GlobalBool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
		ShowCompletions(c)
		return true
	}

	return false
}

func checkCommandCompletions(c *Context, name string) bool {
	if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
		ShowCommandCompletions(c, name)
		return true
	}

	return false
}
<commit_msg>Fixed ShowSubcommandHelp<commit_after>package cli

import (
	"fmt"
	"os"
	"text/tabwriter"
	"text/template"
)

// The text template for the Default help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...]

VERSION:
   {{.Version}}{{if or .Author .Email}}

AUTHOR:{{if .Author}}
  {{.Author}}{{if .Email}} - <{{.Email}}>{{end}}{{else}}
  {{.Email}}{{end}}{{end}}

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
   {{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`

// The text template for the command help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var CommandHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   command {{.Name}}{{if .Flags}} [command options]{{end}} [arguments...]{{if .Description}}

DESCRIPTION:
   {{.Description}}{{end}}{{if .Flags}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{ end }}
`

// The text template for the subcommand help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var SubcommandHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} command{{if .Flags}} [command options]{{end}} [arguments...]

COMMANDS:
   {{range .Commands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}}
   {{end}}{{if .Flags}}
OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`

var helpCommand = Command{
	Name:      "help",
	ShortName: "h",
	Usage:     "Shows a list of commands or help for one command",
	Action: func(c *Context) {
		args := c.Args()
		if args.Present() {
			ShowCommandHelp(c, args.First())
		} else {
			ShowAppHelp(c)
		}
	},
}

var helpSubcommand = Command{
	Name:      "help",
	ShortName: "h",
	Usage:     "Shows a list of commands or help for one command",
	Action: func(c *Context) {
		args := c.Args()
		if args.Present() {
			ShowCommandHelp(c, args.First())
		} else {
			ShowSubcommandHelp(c)
		}
	},
}

// Prints help for the App
var HelpPrinter = printHelp

// Prints version for the App
var VersionPrinter = printVersion

func ShowAppHelp(c *Context) {
	HelpPrinter(AppHelpTemplate, c.App)
}

// Prints the list of subcommands as the default app completion method
func DefaultAppComplete(c *Context) {
	for _, command := range c.App.Commands {
		fmt.Println(command.Name)
		if command.ShortName != "" {
			fmt.Println(command.ShortName)
		}
	}
}

// Prints help for the given command
func ShowCommandHelp(c *Context, command string) {
	for _, c := range c.App.Commands {
		if c.HasName(command) {
			HelpPrinter(CommandHelpTemplate, c)
			return
		}
	}

	if c.App.CommandNotFound != nil {
		c.App.CommandNotFound(c, command)
	} else {
		fmt.Printf("No help topic for '%v'\n", command)
	}
}

// Prints help for the given subcommand
func ShowSubcommandHelp(c *Context) {
	ShowCommandHelp(c, c.Command.Name)
}

// Prints the version number of the App
func ShowVersion(c *Context) {
	VersionPrinter(c)
}

func printVersion(c *Context) {
	fmt.Printf("%v version %v\n", c.App.Name, c.App.Version)
}

// Prints the lists of commands within a given context
func ShowCompletions(c *Context) {
	a := c.App
	if a != nil && a.BashComplete != nil {
		a.BashComplete(c)
	}
}

// Prints the custom completions for a given command
func ShowCommandCompletions(ctx *Context, command string) {
	c := ctx.App.Command(command)
	if c != nil && c.BashComplete != nil {
		c.BashComplete(ctx)
	}
}

func printHelp(templ string, data interface{}) {
	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Parse(templ))
	err := t.Execute(w, data)
	if err != nil {
		panic(err)
	}
	w.Flush()
}

func checkVersion(c *Context) bool {
	if c.GlobalBool("version") {
		ShowVersion(c)
		return true
	}

	return false
}

func checkHelp(c *Context) bool {
	if c.GlobalBool("h") || c.GlobalBool("help") {
		ShowAppHelp(c)
		return true
	}

	return false
}

func checkCommandHelp(c *Context, name string) bool {
	if c.Bool("h") || c.Bool("help") {
		ShowCommandHelp(c, name)
		return true
	}

	return false
}

func checkSubcommandHelp(c *Context) bool {
	if c.GlobalBool("h") || c.GlobalBool("help") {
		ShowSubcommandHelp(c)
		return true
	}

	return false
}

func checkCompletions(c *Context) bool {
	if c.GlobalBool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
		ShowCompletions(c)
		return true
	}

	return false
}

func checkCommandCompletions(c *Context, name string) bool {
	if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
		ShowCommandCompletions(c, name)
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"os"
	"text/template"

	"github.com/flynn/flynn-controller/client"
)

var cmdHelp = &Command{
	NoClient: true,
	Usage:    "help [topic]",
	Long:     `Help shows usage for a command or other topic.`,
}

func init() {
	cmdHelp.Run = runHelp // break init loop
}

func runHelp(cmd *Command, args []string, client *controller.Client) error {
	if len(args) == 0 {
		printUsage()
		return nil // not os.Exit(2); success
	}
	if len(args) != 1 {
		return errors.New("too many arguments")
	}

	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.printUsage(false)
			return nil
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic: %q. Run 'flynn help'.\n", args[0])
	os.Exit(2)
	return nil
}

var cmdVersion = &Command{
	Run:      runVersion,
	NoClient: true,
	Usage:    "version",
	Short:    "show flynn version",
	Long:     `Version shows the flynn client version string.`,
}

func runVersion(cmd *Command, args []string, client *controller.Client) error {
	fmt.Println(Version)
	return nil
}

var usageTemplate = template.Must(template.New("usage").Parse(`
Usage: flynn [-a app] [command] [options] [arguments]

Commands:
{{range .Commands}}{{if .Runnable}}
    {{.Name | printf "%-8s"}}  {{.Short}}{{end}}{{end}}
{{range .Plugins}}
    {{.Name | printf "%-8s"}}  {{.Short}} (plugin){{end}}

Run 'flynn help [command]' for details.`[1:]))

func printUsage() {
	usageTemplate.Execute(os.Stdout, struct {
		Commands []*Command
	}{
		commands,
	})
}

func usage() {
	printUsage()
	os.Exit(2)
}
<commit_msg>Remove plugins from usage template<commit_after>package main

import (
	"errors"
	"fmt"
	"os"
	"text/template"

	"github.com/flynn/flynn-controller/client"
)

var cmdHelp = &Command{
	NoClient: true,
	Usage:    "help [topic]",
	Long:     `Help shows usage for a command or other topic.`,
}

func init() {
	cmdHelp.Run = runHelp // break init loop
}

func runHelp(cmd *Command, args []string, client *controller.Client) error {
	if len(args) == 0 {
		printUsage()
		return nil // not os.Exit(2); success
	}
	if len(args) != 1 {
		return errors.New("too many arguments")
	}

	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.printUsage(false)
			return nil
		}
	}

	fmt.Fprintf(os.Stderr, "Unknown help topic: %q. Run 'flynn help'.\n", args[0])
	os.Exit(2)
	return nil
}

var cmdVersion = &Command{
	Run:      runVersion,
	NoClient: true,
	Usage:    "version",
	Short:    "show flynn version",
	Long:     `Version shows the flynn client version string.`,
}

func runVersion(cmd *Command, args []string, client *controller.Client) error {
	fmt.Println(Version)
	return nil
}

var usageTemplate = template.Must(template.New("usage").Parse(`
Usage: flynn [-a app] [command] [options] [arguments]

Commands:
{{range .Commands}}{{if .Runnable}}
    {{.Name | printf "%-8s"}}  {{.Short}}{{end}}{{end}}

Run 'flynn help [command]' for details.`[1:]))

func printUsage() {
	usageTemplate.Execute(os.Stdout, struct {
		Commands []*Command
	}{
		commands,
	})
}

func usage() {
	printUsage()
	os.Exit(2)
}
<|endoftext|>


<commit_before>package logrusly

import (
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	// Use this fork until the fixes are accepted upstream https://github.com/segmentio/go-loggly/pull/6
	"github.com/sebest/go-loggly"
)

// LogglyHook to send log messages to the Loggly API. You must set:
type LogglyHook struct {
	client *loggly.Client
	host   string
}

func NewLogglyHook(token string, host string, tags ...string) *LogglyHook {
	client := loggly.New(token, tags...)
	client.Defaults = loggly.Message{}

	// sigc := make(chan os.Signal, 1)
	// signal.Notify(sigc,
	// 	syscall.SIGHUP,
	// 	syscall.SIGINT,
	// 	syscall.SIGTERM,
	// 	syscall.SIGQUIT)
	// go func() {
	// 	s := <-sigc
	// 	if s != nil {
	// 		client.Flush()
	// 	}
	// }()

	return &LogglyHook{
		client: client,
		host:   host,
	}
}

func (hook *LogglyHook) Fire(entry *logrus.Entry) error {
	level := entry.Level.String()
	logglyMessage := loggly.Message{
		"timestamp": entry.Time.UTC().Format(time.RFC3339Nano),
		"level":     strings.ToUpper(level),
		"message":   entry.Message,
		"host":      hook.host,
		"data":      entry.Data,
	}

	err := hook.client.Send(logglyMessage)
	if err != nil {
		log := logrus.New()
		log.WithFields(logrus.Fields{
			"source": "loggly",
			"error":  err,
		}).Warn("Failed to send error to Loggly")
		return err
	}

	if level == "fatal" || level == "panic" {
		hook.Flush()
	}

	return nil
}

func (hook *LogglyHook) Flush() {
	hook.client.Flush()
}

func (hook *LogglyHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.WarnLevel,
		logrus.ErrorLevel,
		logrus.FatalLevel,
		logrus.PanicLevel,
	}
}
<commit_msg>send all levels to loggly<commit_after>package logrusly

import (
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	// Use this fork until the fixes are accepted upstream https://github.com/segmentio/go-loggly/pull/6
	"github.com/sebest/go-loggly"
)

// LogglyHook to send log messages to the Loggly API. You must set:
type LogglyHook struct {
	client *loggly.Client
	host   string
}

func NewLogglyHook(token string, host string, tags ...string) *LogglyHook {
	client := loggly.New(token, tags...)
	client.Defaults = loggly.Message{}

	// sigc := make(chan os.Signal, 1)
	// signal.Notify(sigc,
	// 	syscall.SIGHUP,
	// 	syscall.SIGINT,
	// 	syscall.SIGTERM,
	// 	syscall.SIGQUIT)
	// go func() {
	// 	s := <-sigc
	// 	if s != nil {
	// 		client.Flush()
	// 	}
	// }()

	return &LogglyHook{
		client: client,
		host:   host,
	}
}

func (hook *LogglyHook) Fire(entry *logrus.Entry) error {
	level := entry.Level.String()
	logglyMessage := loggly.Message{
		"timestamp": entry.Time.UTC().Format(time.RFC3339Nano),
		"level":     strings.ToUpper(level),
		"message":   entry.Message,
		"host":      hook.host,
		"data":      entry.Data,
	}

	err := hook.client.Send(logglyMessage)
	if err != nil {
		log := logrus.New()
		log.WithFields(logrus.Fields{
			"source": "loggly",
			"error":  err,
		}).Warn("Failed to send error to Loggly")
		return err
	}

	if level == "fatal" || level == "panic" {
		hook.Flush()
	}

	return nil
}

func (hook *LogglyHook) Flush() {
	hook.client.Flush()
}

func (hook *LogglyHook) Levels() []logrus.Level {
	return []logrus.Level{
		logrus.DebugLevel,
		logrus.InfoLevel,
		logrus.WarnLevel,
		logrus.ErrorLevel,
		logrus.FatalLevel,
		logrus.PanicLevel,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
)

func pageHTML(title string, content string) string {
	var html bytes.Buffer
	html.WriteString(`<!DOCTYPE html>

<title>`)
	html.WriteString(title)
	html.WriteString(`</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400,400i,600,700" rel="stylesheet">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<link rel="stylesheet" href="/static/style.css">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">

			<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
			Toggle navigation

			</button>
			BookBrowser

Books

Authors

Series

Random Book

				<form class="navbar-form" role="search" method="GET" action="/search/">

					<input type="text" class="form-control" placeholder="Search" name="q" id="q">

						<button class="btn btn-default" type="submit"></button>

				</form>

`)
	html.WriteString(title)
	html.WriteString(`

`)
	html.WriteString(content)
	html.WriteString(`

BookBrowser ` + curversion + `

Copyright 2017 Patrick G

`)
	return html.String()
}

func bookHTML(b *Book, isCard bool) string {
	var html bytes.Buffer
	if isCard {
		html.WriteString(`<div class="book card">`)
	} else {
		html.WriteString(`<div class="book info">`)
	}

	html.WriteString(`<a class="cover" href="/books/` + b.ID + `">`)
	if b.HasCover {
		if isCard {
			html.WriteString(`<img alt="cover" src="/covers/` + b.ID + `_thumb.jpg" />`)
		} else {
			html.WriteString(`<img alt="cover" src="/covers/` + b.ID + `.jpg" />`)
		}
	} else {
		html.WriteString(`<img alt="cover" src="/static/nocover.jpg" />`)
	}
	html.WriteString(`</a>`)

	html.WriteString(`<div class="meta">`)
	html.WriteString(`<a class="title" href="/books/` + b.ID + `">`)
	html.WriteString(b.Title)
	html.WriteString(`</a>`)
	if b.Author != "" {
		html.WriteString(`<a class="author" href="/authors/` + b.AuthorID + `">`)
		html.WriteString(b.Author)
		html.WriteString(`</a>`)
	}
	if b.Series.Name != "" {
		html.WriteString(`<div class="series">`)
		html.WriteString(`<a class="name" href="/series/` + b.Series.ID + `">`)
		html.WriteString(b.Series.Name)
		html.WriteString(`</a> - <span class="index">`)
		html.WriteString(fmt.Sprintf("%v", b.Series.Index))
		html.WriteString(`</span>`)
		html.WriteString(`</div>`)
	}
	if !isCard && b.Description != "" {
		html.WriteString(`<div class="description">`)
		html.WriteString(b.Description)
		html.WriteString(`</div>`)
	}
	html.WriteString(`<a class="download btn btn-default" href="/download/` + b.ID + `.epub">Download EPUB</a>`)
	html.WriteString(`<a class="reader btn btn-default" href="/static/reader/#!/download/` + b.ID + `.epub">Read</a>`)
	html.WriteString(`</div>`)

	html.WriteString(`</div>`)
	return html.String()
}

func bookListPageHTML(books []Book, title string, notfoundtext string) (html string, notfound bool) {
	if len(books) == 0 {
		return pageHTML("Not Found", notfoundtext), true
	}

	var booksHTML bytes.Buffer
	booksHTML.WriteString(`<div class="books cards">`)
	for _, b := range books {
		booksHTML.WriteString(bookHTML(&b, true))
	}
	booksHTML.WriteString(`</div>`)

	return pageHTML(title, booksHTML.String()), false
}

func itemCardHTML(title string, description string, link string) string {
	var html bytes.Buffer
	html.WriteString(`<a class="item card" href="` + link + `">`)
	html.WriteString(`<div class="title">`)
	html.WriteString(title)
	html.WriteString(`</div>`)
	html.WriteString(`<div class="description">`)
	html.WriteString(description)
	html.WriteString(`</div>`)
	html.WriteString(`</a>`)
	return html.String()
}
<commit_msg>Added update notifications (#2)<commit_after>package main

import (
	"bytes"
	"fmt"
)

func pageHTML(title string, content string) string {
	var html bytes.Buffer
	html.WriteString(`<!DOCTYPE html>

<title>`)
	html.WriteString(title)
	html.WriteString(`</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400,400i,600,700" rel="stylesheet">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<link rel="stylesheet" href="/static/style.css">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">

			<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
			Toggle navigation

			</button>
			BookBrowser

Books

Authors

Series

Random Book

				<form class="navbar-form" role="search" method="GET" action="/search/">

					<input type="text" class="form-control" placeholder="Search" name="q" id="q">

						<button class="btn btn-default" type="submit"></button>

				</form>

`)
	html.WriteString(title)
	html.WriteString(`

`)
	html.WriteString(content)
	html.WriteString(`

BookBrowser ` + curversion + `

Copyright 2017 Patrick G

`)
	return html.String()
}

func bookHTML(b *Book, isCard bool) string {
	var html bytes.Buffer
	if isCard {
		html.WriteString(`<div class="book card">`)
	} else {
		html.WriteString(`<div class="book info">`)
	}

	html.WriteString(`<a class="cover" href="/books/` + b.ID + `">`)
	if b.HasCover {
		if isCard {
			html.WriteString(`<img alt="cover" src="/covers/` + b.ID + `_thumb.jpg" />`)
		} else {
			html.WriteString(`<img alt="cover" src="/covers/` + b.ID + `.jpg" />`)
		}
	} else {
		html.WriteString(`<img alt="cover" src="/static/nocover.jpg" />`)
	}
	html.WriteString(`</a>`)

	html.WriteString(`<div class="meta">`)
	html.WriteString(`<a class="title" href="/books/` + b.ID + `">`)
	html.WriteString(b.Title)
	html.WriteString(`</a>`)
	if b.Author != "" {
		html.WriteString(`<a class="author" href="/authors/` + b.AuthorID + `">`)
		html.WriteString(b.Author)
		html.WriteString(`</a>`)
	}
	if b.Series.Name != "" {
		html.WriteString(`<div class="series">`)
		html.WriteString(`<a class="name" href="/series/` + b.Series.ID + `">`)
		html.WriteString(b.Series.Name)
		html.WriteString(`</a> - <span class="index">`)
		html.WriteString(fmt.Sprintf("%v", b.Series.Index))
		html.WriteString(`</span>`)
		html.WriteString(`</div>`)
	}
	if !isCard && b.Description != "" {
		html.WriteString(`<div class="description">`)
		html.WriteString(b.Description)
		html.WriteString(`</div>`)
	}
	html.WriteString(`<a class="download btn btn-default" href="/download/` + b.ID + `.epub">Download EPUB</a>`)
	html.WriteString(`<a class="reader btn btn-default" href="/static/reader/#!/download/` + b.ID + `.epub">Read</a>`)
	html.WriteString(`</div>`)

	html.WriteString(`</div>`)
	return html.String()
}

func bookListPageHTML(books []Book, title string, notfoundtext string) (html string, notfound bool) {
	if len(books) == 0 {
		return pageHTML("Not Found", notfoundtext), true
	}

	var booksHTML bytes.Buffer
	booksHTML.WriteString(`<div class="books cards">`)
	for _, b := range books {
		booksHTML.WriteString(bookHTML(&b, true))
	}
	booksHTML.WriteString(`</div>`)

	return pageHTML(title, booksHTML.String()), false
}

func itemCardHTML(title string, description string, link string) string {
	var html bytes.Buffer
	html.WriteString(`<a class="item card" href="` + link + `">`)
	html.WriteString(`<div class="title">`)
	html.WriteString(title)
	html.WriteString(`</div>`)
	html.WriteString(`<div class="description">`)
	html.WriteString(description)
	html.WriteString(`</div>`)
	html.WriteString(`</a>`)
	return html.String()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"sort"
	"strings"
	"sync"

	"github.com/pocke/gfm-viewer/env"
	"github.com/yosssi/ace"
)

type Server struct {
	pages map[string]string
	mu    *sync.RWMutex
}

func NewServer() *Server {
	s := &Server{
		pages: make(map[string]string),
		mu:    &sync.RWMutex{},
	}

	go func() {
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			path := r.URL.Path
			if path == "/" || path == "/index.html" {
				s.indexHandler(w, r)
			} else if strings.HasPrefix(path, "/files") {
				s.ServeFile(w, r)
			} else {
				http.Error(w, "404 Not Found", http.StatusNotFound)
				return
			}
		})
		// TODO: port
		http.ListenAndServe(":1124", nil)
	}()

	return s
}

func (s *Server) Add(path, html string) {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.pages[path] = html
}

// Update same as Add.
func (s *Server) Update(path, html string) {
	s.Add(path, html)
}

func (s *Server) Get(path string) (string, bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	html, ok := s.pages[path]
	if ok {
		return html, ok
	} else {
		html, ok := s.pages["/"+path]
		return html, ok
	}
}

func (s *Server) Index() []string {
	s.mu.RLock()
	defer s.mu.RUnlock()

	res := make([]string, 0, len(s.pages))

	for path := range s.pages {
		res = append(res, path)
	}

	sort.Strings(res)
	return res
}

func (s *Server) ServeFile(w http.ResponseWriter, r *http.Request) {
	path := strings.TrimLeft(r.URL.Path, "/files")
	html, ok := s.Get(path)
	if !ok {
		http.Error(w, fmt.Sprintf("%s page not found", path), http.StatusNotFound)
		return
	}
	w.Write([]byte(html))
}

func (s *Server) indexHandler(w http.ResponseWriter, r *http.Request) {
	tpl, err := ace.Load("assets/index", "", &ace.Options{
		DynamicReload: env.DEBUG,
		Asset:         Asset,
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	err = tpl.Execute(w, s.Index())
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}
<commit_msg>loadAce func<commit_after>package main

import (
	"fmt"
	"net/http"
	"sort"
	"strings"
	"sync"

	"github.com/pocke/gfm-viewer/env"
	"github.com/yosssi/ace"
)

type Server struct {
	pages map[string]string
	mu    *sync.RWMutex
}

func NewServer() *Server {
	s := &Server{
		pages: make(map[string]string),
		mu:    &sync.RWMutex{},
	}

	go func() {
		http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
			path := r.URL.Path
			if path == "/" || path == "/index.html" {
				s.indexHandler(w, r)
			} else if strings.HasPrefix(path, "/files") {
				s.ServeFile(w, r)
			} else {
				http.Error(w, "404 Not Found", http.StatusNotFound)
				return
			}
		})
		// TODO: port
		http.ListenAndServe(":1124", nil)
	}()

	return s
}

func (s *Server) Add(path, html string) {
	s.mu.Lock()
	defer s.mu.Unlock()

	s.pages[path] = html
}

// Update same as Add.
func (s *Server) Update(path, html string) {
	s.Add(path, html)
}

func (s *Server) Get(path string) (string, bool) {
	s.mu.RLock()
	defer s.mu.RUnlock()

	html, ok := s.pages[path]
	if ok {
		return html, ok
	} else {
		html, ok := s.pages["/"+path]
		return html, ok
	}
}

func (s *Server) Index() []string {
	s.mu.RLock()
	defer s.mu.RUnlock()

	res := make([]string, 0, len(s.pages))

	for path := range s.pages {
		res = append(res, path)
	}

	sort.Strings(res)
	return res
}

func (s *Server) ServeFile(w http.ResponseWriter, r *http.Request) {
	path := strings.TrimLeft(r.URL.Path, "/files")
	html, ok := s.Get(path)
	if !ok {
		http.Error(w, fmt.Sprintf("%s page not found", path), http.StatusNotFound)
		return
	}
	w.Write([]byte(html))
}

func (s *Server) indexHandler(w http.ResponseWriter, r *http.Request) {
	loadAce(w, "index", s.Index())
}

func (s *Server) beforeAuthHandler(w http.ResponseWriter, r *http.Request) {
	loadAce(w, "before_auth", nil)
}

func loadAce(w http.ResponseWriter, action string, data interface{}) {
	tpl, err := ace.Load("assets/"+action, "", &ace.Options{
		DynamicReload: env.DEBUG,
		Asset:         Asset,
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	err = tpl.Execute(w, data)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
}
<|endoftext|>


<commit_before>package htcat

import (
	"net/http"
	"net/url"
	"sync"
)

type HtCat struct {
	defrag

	u     *url.URL
	cl    http.Client
	tasks chan *httpFrag

	// Protect httpFragGen with a Mutex.
	httpFragGenMu sync.Mutex
	httpFragGen
}

func New(client http.Client,
	u *url.URL, parallelism int, chunkSize, totalSize int64) *HtCat {
	cat := HtCat{
		u: u,
	}

	cat.initDefrag()
	cat.targetFragSize = chunkSize
	cat.totalSize = totalSize

	for i := 0; i < parallelism; i += 1 {
		go cat.get()
	}

	return &cat
}

func (cat *HtCat) nextFragment() *httpFrag {
	cat.httpFragGenMu.Lock()
	defer cat.httpFragGenMu.Unlock()

	var hf *httpFrag

	if cat.httpFragGen.hasNext() {
		f := cat.defrag.nextFragment()
		hf = cat.httpFragGen.nextFragment(f)
	} else {
		cat.defrag.setLast(cat.defrag.lastAllocated())
	}

	return hf
}

func (cat *HtCat) get() {
	for {
		hf := cat.nextFragment()
		if hf == nil {
			return
		}

		req := http.Request{
			Method:     "GET",
			URL:        cat.u,
			Proto:      "HTTP/1.1",
			ProtoMajor: 1,
			ProtoMinor: 1,
			Header:     hf.header,
			Body:       nil,
			Host:       cat.u.Host,
		}

		resp, err := cat.cl.Do(&req)
		if err != nil {
			resp.Body.Close()
			cat.defrag.cancel(err)
		}

		er := newEagerReader(resp.Body, hf.size)
		hf.fragment.contents = er
		cat.register(hf.fragment)
		er.WaitClosed()
	}
}
<commit_msg>Fix crash bug: .Close on nil pointer<commit_after>package htcat

import (
	"net/http"
	"net/url"
	"sync"
)

type HtCat struct {
	defrag

	u     *url.URL
	cl    http.Client
	tasks chan *httpFrag

	// Protect httpFragGen with a Mutex.
	httpFragGenMu sync.Mutex
	httpFragGen
}

func New(client http.Client,
	u *url.URL, parallelism int, chunkSize, totalSize int64) *HtCat {
	cat := HtCat{
		u: u,
	}

	cat.initDefrag()
	cat.targetFragSize = chunkSize
	cat.totalSize = totalSize

	for i := 0; i < parallelism; i += 1 {
		go cat.get()
	}

	return &cat
}

func (cat *HtCat) nextFragment() *httpFrag {
	cat.httpFragGenMu.Lock()
	defer cat.httpFragGenMu.Unlock()

	var hf *httpFrag

	if cat.httpFragGen.hasNext() {
		f := cat.defrag.nextFragment()
		hf = cat.httpFragGen.nextFragment(f)
	} else {
		cat.defrag.setLast(cat.defrag.lastAllocated())
	}

	return hf
}

func (cat *HtCat) get() {
	for {
		hf := cat.nextFragment()
		if hf == nil {
			return
		}

		req := http.Request{
			Method:     "GET",
			URL:        cat.u,
			Proto:      "HTTP/1.1",
			ProtoMajor: 1,
			ProtoMinor: 1,
			Header:     hf.header,
			Body:       nil,
			Host:       cat.u.Host,
		}

		resp, err := cat.cl.Do(&req)
		if err != nil {
			cat.defrag.cancel(err)
		}

		er := newEagerReader(resp.Body, hf.size)
		hf.fragment.contents = er
		cat.register(hf.fragment)
		er.WaitClosed()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"errors"
	"io"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/http/httputil"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/flynn/go-discoverd"
)

type HTTPFrontend struct {
	Addr      string
	TLSAddr   string
	TLSConfig *tls.Config

	mtx      sync.RWMutex
	domains  map[string]*httpServer
	services map[string]*httpServer

	etcdPrefix string

	etcd      EtcdClient
	discoverd DiscoverdClient
}

type EtcdClient interface {
	Create(key string, value string, ttl uint64) (*etcd.Response, error)
	Get(key string, sort, recursive bool) (*etcd.Response, error)
	Watch(prefix string, waitIndex uint64, recursive bool, receiver chan *etcd.Response, stop chan bool) (*etcd.Response, error)
}

type DiscoverdClient interface {
	NewServiceSet(string) (discoverd.ServiceSet, error)
}

func NewHTTPFrontend(addr string, etcdc EtcdClient, discoverdc DiscoverdClient) *HTTPFrontend {
	return &HTTPFrontend{
		Addr:       addr,
		etcd:       etcdc,
		etcdPrefix: "/strowger/http/",
		discoverd:  discoverdc,
		domains:    make(map[string]*httpServer),
		services:   make(map[string]*httpServer),
	}
}

func (s *HTTPFrontend) AddHTTPDomain(domain string, service string, certs [][]byte, key []byte) error {
	return s.addDomain(domain, service, true)
}

var ErrDomainExists = errors.New("strowger: domain exists with different service")

func (s *HTTPFrontend) addDomain(domain string, service string, persist bool) error {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	if server, ok := s.domains[domain]; ok {
		if server.name != service {
			return ErrDomainExists
		}
		return nil
	}

	server := s.services[service]
	if server == nil {
		services, err := s.discoverd.NewServiceSet(service)
		if err != nil {
			return err
		}
		server = &httpServer{name: service, services: services}
	}
	if persist {
		if _, err := s.etcd.Create(s.etcdPrefix+domain+"/service", service, 0); err != nil {
			return err
		}
	}
	// TODO: set cert/key data if provided

	server.refs++
	s.domains[domain] = server
	s.services[service] = server
	// TODO: TLS config

	log.Println("Add service", service, "to domain", domain)

	return nil
}

func (s *HTTPFrontend) RemoveHTTPDomain(domain string) {
	s.mtx.Lock()
	defer s.mtx.Unlock()
	server := s.domains[domain]
	if server == nil {
		return
	}
	delete(s.domains, domain)
	server.refs++
	if server.refs <= 0 {
		// TODO: close service set stream
		delete(s.services, server.name)
	}
	// TODO: persist
}

func (s *HTTPFrontend) syncDatabase() {
	var since uint64
	data, err := s.etcd.Get(s.etcdPrefix, false, true)
	if e, ok := err.(*etcd.EtcdError); ok && e.ErrorCode == 100 {
		// key not found, ignore
		goto watch
	}
	if err != nil {
		log.Fatal(err)
		return
	}
	since = data.EtcdIndex
	for _, node := range data.Node.Nodes {
		if !node.Dir {
			continue
		}
		domain := path.Base(node.Key)
		serviceRes, err := s.etcd.Get(node.Key+"/service", false, false)
		if err != nil {
			log.Fatal(err)
		}
		if err := s.addDomain(domain, serviceRes.Node.Value, false); err != nil {
			log.Fatal(err)
		}
	}

watch:
	stream := make(chan *etcd.Response)
	stop := make(chan bool)
	// TODO: store stop
	go s.etcd.Watch(s.etcdPrefix, since, false, stream, stop)
	for res := range stream {
		if res.Node.Dir || path.Base(res.Node.Key) == "service" {
			continue
		}
		domain := path.Base(path.Dir(res.Node.Key))
		s.mtx.Lock()
		_, exists := s.domains[domain]
		s.mtx.Unlock()
		if !exists {
			if err := s.addDomain(domain, res.Node.Value, false); err != nil {
				// TODO: log error
			}
		}
	}
	// TODO: handle delete
	log.Println("done watching etcd")
}

func (s *HTTPFrontend) serve() {
	l, err := net.Listen("tcp", s.Addr)
	if err != nil {
		// TODO: log error
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			// TODO: log error
			break
		}
		go s.handle(conn)
	}
}

func (s *HTTPFrontend) serveTLS() {
	l, err := net.Listen("tcp", s.TLSAddr)
	if err != nil {
		// TODO: log error
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			// TODO: log error
			break
		}
		go s.handle(conn)
	}
}

func (s *HTTPFrontend) handle(conn net.Conn) {
	defer conn.Close()
	sc := httputil.NewServerConn(conn, nil)
	req, err := sc.Read()
	if err != nil {
		if err != httputil.ErrPersistEOF {
			// TODO: log error
		}
		return
	}

	s.mtx.RLock()
	// TODO: handle wildcard domains
	backend := s.domains[req.Host]
	s.mtx.RUnlock()
	log.Println(req, backend)
	if backend == nil {
		// TODO: return 404
		return
	}
	_, tls := conn.(*tls.Conn)
	backend.handle(req, sc, tls)
}

type httpServer struct {
	name     string
	services discoverd.ServiceSet
	refs     int
}

func (s *httpServer) getBackend() *httputil.ClientConn {
	for _, addr := range shuffle(s.services.Addrs()) {
		// TODO: set connection timeout
		backend, err := net.Dial("tcp", addr)
		if err != nil {
			// TODO: log error
			// TODO: limit number of backends tried
			// TODO: temporarily quarantine failing backends
			log.Println("backend error", err)
			continue
		}
		return httputil.NewClientConn(backend, nil)
	}
	// TODO: log no backends found error
	return nil
}

func (s *httpServer) handle(req *http.Request, sc *httputil.ServerConn, tls bool) {
	req.Header.Set("X-Request-Start", strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
	backend := s.getBackend()
	if backend == nil {
		// TODO: Return 503
		log.Println("no backend found")
		return
	}
	defer backend.Close()

	for {
		if req.Method != "GET" && req.Method != "POST" && req.Method != "HEAD" &&
			req.Method != "OPTIONS" && req.Method != "PUT" && req.Method != "DELETE" && req.Method != "TRACE" {
			// TODO: return 405
		}

		req.Proto = "HTTP/1.1"
		req.ProtoMajor = 1
		req.ProtoMinor = 1
		delete(req.Header, "Te")
		delete(req.Header, "Transfer-Encoding")

		if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
			// If we aren't the first proxy retain prior
			// X-Forwarded-For information as a comma+space
			// separated list and fold multiple headers into one.
			if prior, ok := req.Header["X-Forwarded-For"]; ok {
				clientIP = strings.Join(prior, ", ") + ", " + clientIP
			}
			req.Header.Set("X-Forwarded-For", clientIP)
		}
		if tls {
			req.Header.Set("X-Forwarded-Proto", "https")
		} else {
			req.Header.Set("X-Forwarded-Proto", "http")
		}
		// TODO: Set X-Forwarded-Port

		if err := backend.Write(req); err != nil {
			log.Println("server write err:", err)
			return
		}
		res, err := backend.Read(req)
		if res != nil {
			if err := sc.Write(req, res); err != nil {
				if err != io.EOF && err != httputil.ErrPersistEOF {
					log.Println("client write err:", err)
					// TODO: log error
				}
				return
			}
		}
		if err != nil {
			if err != io.EOF && err != httputil.ErrPersistEOF {
				log.Println("server read err:", err)
				// TODO: log error
				// TODO: Return 502
			}
			return
		}

		// TODO: Proxy HTTP CONNECT? (example: Go RPC over HTTP)
		if res.StatusCode == http.StatusSwitchingProtocols {
			serverW, serverR := backend.Hijack()
			clientW, clientR := sc.Hijack()
			defer serverW.Close()
			done := make(chan struct{})
			go func() {
				serverR.WriteTo(clientW)
				close(done)
			}()
			clientR.WriteTo(serverW)
			<-done
			return
		}

		// TODO: http pipelining
		req, err = sc.Read()
		if err != nil {
			if err != io.EOF && err != httputil.ErrPersistEOF {
				log.Println("client read err:", err)
			}
			return
		}
		req.Header.Set("X-Request-Start", strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
	}
}

func shuffle(s []string) []string {
	for i := len(s) - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		s[i], s[j] = s[j], s[i]
	}
	return s
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<commit_msg>Add http error responses.<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/http/httputil"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/coreos/go-etcd/etcd"
	"github.com/flynn/go-discoverd"
)

type HTTPFrontend struct {
	Addr      string
	TLSAddr   string
	TLSConfig *tls.Config

	mtx      sync.RWMutex
	domains  map[string]*httpServer
	services map[string]*httpServer

	etcdPrefix string

	etcd      EtcdClient
	discoverd DiscoverdClient
}

type EtcdClient interface {
	Create(key string, value string, ttl uint64) (*etcd.Response, error)
	Get(key string, sort, recursive bool) (*etcd.Response, error)
	Watch(prefix string, waitIndex uint64, recursive bool, receiver chan *etcd.Response, stop chan bool) (*etcd.Response, error)
}

type DiscoverdClient interface {
	NewServiceSet(string) (discoverd.ServiceSet, error)
}

func NewHTTPFrontend(addr string, etcdc EtcdClient, discoverdc DiscoverdClient) *HTTPFrontend {
	return &HTTPFrontend{
		Addr:       addr,
		etcd:       etcdc,
		etcdPrefix: "/strowger/http/",
		discoverd:  discoverdc,
		domains:    make(map[string]*httpServer),
		services:   make(map[string]*httpServer),
	}
}

func (s *HTTPFrontend) AddHTTPDomain(domain string, service string, certs [][]byte, key []byte) error {
	return s.addDomain(domain, service, true)
}

var ErrDomainExists = errors.New("strowger: domain exists with different service")

func (s *HTTPFrontend) addDomain(domain string, service string, persist bool) error {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	if server, ok := s.domains[domain]; ok {
		if server.name != service {
			return ErrDomainExists
		}
		return nil
	}

	server := s.services[service]
	if server == nil {
		services, err := s.discoverd.NewServiceSet(service)
		if err != nil {
			return err
		}
		server = &httpServer{name: service, services: services}
	}
	if persist {
		if _, err := s.etcd.Create(s.etcdPrefix+domain+"/service", service, 0); err != nil {
			return err
		}
	}
	// TODO: set cert/key data if provided

	server.refs++
	s.domains[domain] = server
	s.services[service] = server
	// TODO: TLS config

	log.Println("Add service", service, "to domain", domain)

	return nil
}

func (s *HTTPFrontend) RemoveHTTPDomain(domain string) {
	s.mtx.Lock()
	defer s.mtx.Unlock()
	server := s.domains[domain]
	if server == nil {
		return
	}
	delete(s.domains, domain)
	server.refs++
	if server.refs <= 0 {
		// TODO: close service set stream
		delete(s.services, server.name)
	}
	// TODO: persist
}

func (s *HTTPFrontend) syncDatabase() {
	var since uint64
	data, err := s.etcd.Get(s.etcdPrefix, false, true)
	if e, ok := err.(*etcd.EtcdError); ok && e.ErrorCode == 100 {
		// key not found, ignore
		goto watch
	}
	if err != nil {
		log.Fatal(err)
		return
	}
	since = data.EtcdIndex
	for _, node := range data.Node.Nodes {
		if !node.Dir {
			continue
		}
		domain := path.Base(node.Key)
		serviceRes, err := s.etcd.Get(node.Key+"/service", false, false)
		if err != nil {
			log.Fatal(err)
		}
		if err := s.addDomain(domain, serviceRes.Node.Value, false); err != nil {
			log.Fatal(err)
		}
	}

watch:
	stream := make(chan *etcd.Response)
	stop := make(chan bool)
	// TODO: store stop
	go s.etcd.Watch(s.etcdPrefix, since, false, stream, stop)
	for res := range stream {
		if res.Node.Dir || path.Base(res.Node.Key) == "service" {
			continue
		}
		domain := path.Base(path.Dir(res.Node.Key))
		s.mtx.Lock()
		_, exists := s.domains[domain]
		s.mtx.Unlock()
		if !exists {
			if err := s.addDomain(domain, res.Node.Value, false); err != nil {
				// TODO: log error
			}
		}
	}
	// TODO: handle delete
	log.Println("done watching etcd")
}

func (s *HTTPFrontend) serve() {
	l, err := net.Listen("tcp", s.Addr)
	if err != nil {
		// TODO: log error
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			// TODO: log error
			break
		}
		go s.handle(conn)
	}
}

func (s *HTTPFrontend) serveTLS() {
	l, err := net.Listen("tcp", s.TLSAddr)
	if err != nil {
		// TODO: log error
		return
	}
	for {
		conn, err := l.Accept()
		if err != nil {
			// TODO: log error
			break
		}
		go s.handle(conn)
	}
}

func fail(sc *httputil.ServerConn, req *http.Request, code int, msg string) {
	resp := &http.Response{
		StatusCode:    code,
		ProtoMajor:    1,
		ProtoMinor:    0,
		Request:       req,
		Body:          ioutil.NopCloser(bytes.NewBufferString(msg)),
		ContentLength: int64(len(msg)),
	}
	sc.Write(req, resp)
}

func (s *HTTPFrontend) handle(conn net.Conn) {
	defer conn.Close()
	sc := httputil.NewServerConn(conn, nil)
	req, err := sc.Read()
	if err != nil {
		if err != httputil.ErrPersistEOF {
			// TODO: log error
		}
		return
	}

	s.mtx.RLock()
	// TODO: handle wildcard domains
	backend := s.domains[req.Host]
	s.mtx.RUnlock()
	log.Println(req, backend)
	if backend == nil {
		fail(sc, req, 404, "Not Found")
		return
	}
	_, tls := conn.(*tls.Conn)
	backend.handle(req, sc, tls)
}

type httpServer struct {
	name     string
	services discoverd.ServiceSet
	refs     int
}

func (s *httpServer) getBackend() *httputil.ClientConn {
	for _, addr := range shuffle(s.services.Addrs()) {
		// TODO: set connection timeout
		backend, err := net.Dial("tcp", addr)
		if err != nil {
			// TODO: log error
			// TODO: limit number of backends tried
			// TODO: temporarily quarantine failing backends
			log.Println("backend error", err)
			continue
		}
		return httputil.NewClientConn(backend, nil)
	}
	// TODO: log no backends found error
	return nil
}

func (s *httpServer) handle(req *http.Request, sc *httputil.ServerConn, tls bool) {
	req.Header.Set("X-Request-Start", strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
	backend := s.getBackend()
	if backend == nil {
		log.Println("no backend found")
		fail(sc, req, 503, "Service Unavailable")
		return
	}
	defer backend.Close()

	for {
		if req.Method != "GET" && req.Method != "POST" && req.Method != "HEAD" &&
			req.Method != "OPTIONS" && req.Method != "PUT" && req.Method != "DELETE" && req.Method != "TRACE" {
			fail(sc, req, 405, "Method not allowed")
			return
		}

		req.Proto = "HTTP/1.1"
		req.ProtoMajor = 1
		req.ProtoMinor = 1
		delete(req.Header, "Te")
		delete(req.Header, "Transfer-Encoding")

		if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
			// If we aren't the first proxy retain prior
			// X-Forwarded-For information as a comma+space
			// separated list and fold multiple headers into one.
			if prior, ok := req.Header["X-Forwarded-For"]; ok {
				clientIP = strings.Join(prior, ", ") + ", " + clientIP
			}
			req.Header.Set("X-Forwarded-For", clientIP)
		}
		if tls {
			req.Header.Set("X-Forwarded-Proto", "https")
		} else {
			req.Header.Set("X-Forwarded-Proto", "http")
		}
		// TODO: Set X-Forwarded-Port

		if err := backend.Write(req); err != nil {
			log.Println("server write err:", err)
			return
		}
		res, err := backend.Read(req)
		if res != nil {
			if err := sc.Write(req, res); err != nil {
				if err != io.EOF && err != httputil.ErrPersistEOF {
					log.Println("client write err:", err)
					// TODO: log error
				}
				return
			}
		}
		if err != nil {
			if err != io.EOF && err != httputil.ErrPersistEOF {
				log.Println("server read err:", err)
				// TODO: log error
				fail(sc, req, 502, "Bad Gateway")
			}
			return
		}

		// TODO: Proxy HTTP CONNECT? (example: Go RPC over HTTP)
		if res.StatusCode == http.StatusSwitchingProtocols {
			serverW, serverR := backend.Hijack()
			clientW, clientR := sc.Hijack()
			defer serverW.Close()
			done := make(chan struct{})
			go func() {
				serverR.WriteTo(clientW)
				close(done)
			}()
			clientR.WriteTo(serverW)
			<-done
			return
		}

		// TODO: http pipelining
		req, err = sc.Read()
		if err != nil {
			if err != io.EOF && err != httputil.ErrPersistEOF {
				log.Println("client read err:", err)
			}
			return
		}
		req.Header.Set("X-Request-Start", strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
	}
}

func shuffle(s []string) []string {
	for i := len(s) - 1; i > 0; i-- {
		j := rand.Intn(i + 1)
		s[i], s[j] = s[j], s[i]
	}
	return s
}

func init() {
	rand.Seed(time.Now().UnixNano())
}
<|endoftext|>


<commit_before>package hush

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
)

type Tree struct {
	items yaml.MapSlice
}

func Main() {
	tree, err := LoadTree()
	if err != nil {
		die("%s\n", err.Error())
	}
	//warn("initial tree = %#v\n", tree)

	if len(os.Args) == 1 {
		err = tree.Print()
		if err != nil {
			die("%s\n", err.Error())
		}
		os.Exit(0)
	}

	switch os.Args[1] {
	case "set": // hush set paypal.com/personal/user john@example.com
		mainSetValue(tree)
	case "import": // hush import
		mainImport(tree)
	default:
		die("Usage: hum ...\n")
	}
}

func mainSetValue(tree *Tree) {
	pattern, value := os.Args[2], os.Args[3]
	value, err := captureValue(value)
	if err != nil {
		die("%s\n", err.Error())
	}
	paths, err := tree.Match(pattern)
	if err != nil {
		die("%s\n", err.Error())
	}

	var path []string
	switch len(paths) {
	case 0:
		path = strings.Split(pattern, "/")
	case 1:
		path = paths[0]
	default:
		die("pattern %q matches multiple paths: %s", paths)
	}

	tree.SetPath(path, value)
	tree.Print()
	err = tree.Save()
	if err != nil {
		die("%s\n", err.Error())
	}
}

func mainImport(tree *Tree) {
	scanner := bufio.NewScanner(os.Stdin)
	for n := 1; scanner.Scan(); n++ {
		txt := scanner.Text()
		if txt == "" {
			continue
		}
		parts := strings.SplitN(txt, "\t", 2)
		if len(parts) < 2 {
			warn("line %d missing tab delimiter\n", n)
			continue
		}
		path := strings.Split(parts[0], "/")
		val := parts[1]
		tree.SetPath(path, val)
	}
	tree.Print()
	err := tree.Save()
	if err != nil {
		die("%s\n", err.Error())
	}
}

func isTerminal(file *os.File) bool {
	return terminal.IsTerminal(int(os.Stdin.Fd()))
}

func captureValue(value string) (string, error) {
	if value == "-" {
		if isTerminal(os.Stdout) {
			editor := editor()
			warn("would launch %s to capture value\n", editor)
			return "", nil
		}

		all, err := ioutil.ReadAll(os.Stdin)
		return string(all), err
	}
	return value, nil
}

func die(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
	os.Exit(1)
}

func warn(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
}

func LoadTree() (*Tree, error) {
	hushPath, err := hushPath()
	if err != nil {
		return nil, err
	}

	_, err = os.Stat(hushPath)
	if os.IsNotExist(err) {
		warn("hush file does not exist. assuming an empty one\n")
		return &Tree{}, nil
	}
	if err != nil {
		return nil, errors.Wrap(err, "can't stat hush file")
	}
	// TODO reduce file permissions if they're too loose

	file, err := os.Open(hushPath)
	if err != nil {
		return nil, errors.Wrap(err, "opening hush file")
	}
	hushData, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, errors.Wrap(err, "can't read hush file")
	}

	keys := make(yaml.MapSlice, 0)
	err = yaml.Unmarshal(hushData, &keys)
	if err != nil {
		return nil, errors.Wrap(err, "can't parse hush file")
	}
	tree := &Tree{keys}
	tree.decrypt()
	return tree, nil
}

func (tree *Tree) Match(pattern string) ([][]string, error) {
	var matches [][]string
	// TODO perform pattern matching
	return matches, nil
}

func (tree *Tree) Get(needle string) (interface{}, bool) {
	for _, item := range tree.items {
		if key, ok := item.Key.(string); ok {
			if key == needle {
				return item.Value, true
			}
		} else {
			die("all keys should be strings not %#v\n", item.Key)
		}
	}
	return nil, false
}

func (tree *Tree) Set(needle string, val interface{}) {
	//warn("Set: %s %s\n", needle, val)
	for i, item := range tree.items {
		if key, ok := item.Key.(string); ok {
			if key == needle {
				tree.items[i].Value = val
				return
			}
		} else {
			die("all keys should be strings not %#v\n", item.Key)
		}
	}

	tree.items = append(tree.items, yaml.MapItem{
		Key:   needle,
		Value: val,
	})
}

func (tree *Tree) SetPath(path []string, val interface{}) {
	//warn("SetPath: %s %s\n", path, val)
	//defer warn("after Set(): %#v\n", tree)
	switch len(path) {
	case 0:
		die("path should not have 0 length")
	case 1:
		tree.Set(path[0], val)
		return
	}

	t := &Tree{}
	key := path[0]
	x, found := tree.Get(key)
	if items, ok := x.(yaml.MapSlice); found && ok {
		//warn("descending into: %s\n", key)
		t.items = items
	} else {
		//warn("creating subtree: %s\n", key)
	}
	t.SetPath(path[1:], val)
	tree.Set(key, t.items)
}

// Print displays a tree for human consumption.
func (tree *Tree) Print() error {
	data, err := yaml.Marshal(tree.items)
	if err != nil {
		return errors.Wrap(err, "printing tree")
	}

	_, err = os.Stdout.Write(data)
	return err
}

// Save stores a tree to disk for permanent, private archival.
func (tree *Tree) Save() error {
	tree.encrypt()

	data, err := yaml.Marshal(tree.items)
	if err != nil {
		return errors.Wrap(err, "saving tree")
	}

	// save to temporary file
	file, err := ioutil.TempFile("", "hush-")
	if err != nil {
		return errors.Wrap(err, "saving tree")
	}
	_, err = file.Write(data)
	file.Close()
	if err != nil {
		return errors.Wrap(err, "saving tree")
	}

	// move temporary file over top of permanent file
	hushPath, err := hushPath()
	if err != nil {
		return errors.Wrap(err, "saving tree")
	}
	err = os.Rename(file.Name(), hushPath)
	return errors.Wrap(err, "saving tree")
}

func (tree *Tree) encrypt() {
	/*
		block, err := aes.NewCipher(encryptionKey)
		if err != nil {
			panic(err)
		}
		gcm, err := cipher.NewGCM(block)
		if err != nil {
			panic(err)
		}
		nonce := make([]byte, 12)
		_, err = rand.Read(nonce)
		if err != nil {
			panic(err)
		}

		mapLeaves(tree.items, func(leaf string) string {
			plaintext := make([]byte, 1+len(leaf))
			plaintext[0] = 1 // version number
			copy(plaintext[1:], []byte(leaf))
			ciphertext := gcm.Seal(nil, nonce, plaintext, nil)
			return base64.StdEncoding.EncodeToString(ciphertext)
		})
	*/
}

var encryptionKey = []byte(`0123456789abcdef`)

func (tree *Tree) decrypt() {
	/*
		mapLeaves(tree.items, func(leaf string) string {
			data, err := base64.StdEncoding.DecodeString(leaf)
			if err != nil {
				// it must be decrypted already
				return leaf
			}
			if len(data) < 1 {
				panic("too little encrypted data")
			}
			if data[0] != 1 {
				panic("I only understand version 1")
			}
			return string(data[1:])
		})
	*/
}

/*
func mapLeaves(items yaml.MapSlice, f func(string) string) {
	for i := range items {
		item := &items[i]
		val := item.Value
		if items, ok := val.(yaml.MapSlice); ok {
			mapLeaves(items, f)
		} else if str, ok := val.(string); ok {
			item.Value = f(str)
		} else {
			panic("unexpected leaf type")
		}
	}
}
*/

func home() (string, error) {
	home := os.Getenv("HOME")
	if home == "" {
		return "", errors.New("Point $HOME at your home directory")
	}
	return home, nil
}

func hushPath() (string, error) {
	home, err := home()
	if err != nil {
		return "", err
	}
	return path.Join(home, ".hush"), nil
}

var editorVarNames = []string{
	"HUSH_EDITOR",
	"VISUAL",
	"EDITOR",
}

func editor() string {
	for _, varName := range editorVarNames {
		ed := os.Getenv(varName)
		if ed != "" {
			return ed
		}
	}

	ed := "vi"
	warn("environment configures no editor. defaulting to %s", ed)
	return ed
}
<commit_msg>sort commands alphabetically<commit_after>package hush

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
)

type Tree struct {
	items yaml.MapSlice
}

func Main() {
	tree, err := LoadTree()
	if err != nil {
		die("%s\n", err.Error())
	}
	//warn("initial tree = %#v\n", tree)

	if len(os.Args) == 1 {
		err = tree.Print()
		if err != nil {
			die("%s\n", err.Error())
		}
		os.Exit(0)
	}

	switch os.Args[1] {
	case "import": // hush import
		mainImport(tree)
	case "set": // hush set paypal.com/personal/user john@example.com
		mainSetValue(tree)
	default:
		die("Usage: hum ...\n")
	}
}

func mainSetValue(tree *Tree) {
	pattern, value := os.Args[2], os.Args[3]
	value, err := captureValue(value)
	if err != nil {
		die("%s\n", err.Error())
	}
	paths, err := tree.Match(pattern)
	if err != nil {
		die("%s\n", err.Error())
	}

	var path []string
	switch len(paths) {
	case 0:
		path = strings.Split(pattern, "/")
	case 1:
		path = paths[0]
	default:
		die("pattern %q matches multiple paths: %s", paths)
	}

	tree.SetPath(path, value)
	tree.Print()
	err = tree.Save()
	if err != nil {
		die("%s\n", err.Error())
	}
}

func mainImport(tree *Tree) {
	scanner := bufio.NewScanner(os.Stdin)
	for n := 1; scanner.Scan(); n++ {
		txt := scanner.Text()
		if txt == "" {
			continue
		}
		parts := strings.SplitN(txt, "\t", 2)
		if len(parts) < 2 {
			warn("line %d missing tab delimiter\n", n)
			continue
		}
		path := strings.Split(parts[0], "/")
		val := parts[1]
		tree.SetPath(path, val)
	}
	tree.Print()
	err := tree.Save()
	if err != nil {
		die("%s\n", err.Error())
	}
}

func isTerminal(file *os.File) bool {
	return terminal.IsTerminal(int(os.Stdin.Fd()))
}

func captureValue(value string) (string, error) {
	if value == "-" {
		if isTerminal(os.Stdout) {
			editor := editor()
			warn("would launch %s to capture value\n", editor)
			return "", nil
		}

		all, err := ioutil.ReadAll(os.Stdin)
		return string(all), err
	}
	return value, nil
}

func die(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
	os.Exit(1)
}

func warn(format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
}

func LoadTree() (*Tree, error) {
	hushPath, err := hushPath()
	if err != nil {
		return nil, err
	}

	_, err = os.Stat(hushPath)
	if os.IsNotExist(err) {
		warn("hush file does not exist. assuming an empty one\n")
		return &Tree{}, nil
	}
	if err != nil {
		return nil, errors.Wrap(err, "can't stat hush file")
	}
	// TODO reduce file permissions if they're too loose

	file, err := os.Open(hushPath)
	if err != nil {
		return nil, errors.Wrap(err, "opening hush file")
	}
	hushData, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, errors.Wrap(err, "can't read hush file")
	}

	keys := make(yaml.MapSlice, 0)
	err = yaml.Unmarshal(hushData, &keys)
	if err != nil {
		return nil, errors.Wrap(err, "can't parse hush file")
	}
	tree := &Tree{keys}
	tree.decrypt()
	return tree, nil
}

func (tree *Tree) Match(pattern string) ([][]string, error) {
	var matches [][]string
	// TODO perform pattern matching
	return matches, nil
}

func (tree *Tree) Get(needle string) (interface{}, bool) {
	for _, item := range tree.items {
		if key, ok := item.Key.(string); ok {
			if key == needle {
				return item.Value, true
			}
		} else {
			die("all keys should be strings not %#v\n", item.Key)
		}
	}
	return nil, false
}

func (tree *Tree) Set(needle string, val interface{}) {
	//warn("Set: %s %s\n", needle, val)
	for i, item := range tree.items {
		if key, ok := item.Key.(string); ok {
			if key == needle {
				tree.items[i].Value = val
				return
			}
		} else {
			die("all keys should be strings not %#v\n", item.Key)
		}
	}

	tree.items = append(tree.items, yaml.MapItem{
		Key:   needle,
		Value: val,
	})
}

func (tree *Tree) SetPath(path []string, val interface{}) {
	//warn("SetPath: %s %s\n", path, val)
	//defer warn("after Set(): %#v\n", tree)
	switch len(path) {
	case 0:
		die("path should not have 0 length")
	case 1:
		tree.Set(path[0], val)
		return
	}

	t := &Tree{}
	key := path[0]
	x, found := tree.Get(key)
	if items, ok := x.(yaml.MapSlice); found && ok {
		//warn("descending into: %s\n", key)
		t.items = items
	} else {
		//warn("creating subtree: %s\n", key)
	}
	t.SetPath(path[1:], val)
	tree.Set(key, t.items)
}

// Print displays a tree for human consumption.
func (tree *Tree) Print() error {
	data, err := yaml.Marshal(tree.items)
	if err != nil {
		return errors.Wrap(err, "printing tree")
	}

	_, err = os.Stdout.Write(data)
	return err
}

// Save stores a tree to disk for permanent, private archival.
func (tree *Tree) Save() error {
	tree.encrypt()

	data, err := yaml.Marshal(tree.items)
	if err != nil {
		return errors.Wrap(err, "saving tree")
	}

	// save to temporary file
	file, err := ioutil.TempFile("", "hush-")
	if err != nil {
		return errors.Wrap(err, "saving tree")
	}
	_, err = file.Write(data)
	file.Close()
	if err != nil {
		return errors.Wrap(err, "saving tree")
	}

	// move temporary file over top of permanent file
	hushPath, err := hushPath()
	if err != nil {
		return errors.Wrap(err, "saving tree")
	}
	err = os.Rename(file.Name(), hushPath)
	return errors.Wrap(err, "saving tree")
}

func (tree *Tree) encrypt() {
	/*
		block, err := aes.NewCipher(encryptionKey)
		if err != nil {
			panic(err)
		}
		gcm, err := cipher.NewGCM(block)
		if err != nil {
			panic(err)
		}
		nonce := make([]byte, 12)
		_, err = rand.Read(nonce)
		if err != nil {
			panic(err)
		}

		mapLeaves(tree.items, func(leaf string) string {
			plaintext := make([]byte, 1+len(leaf))
			plaintext[0] = 1 // version number
			copy(plaintext[1:], []byte(leaf))
			ciphertext := gcm.Seal(nil, nonce, plaintext, nil)
			return base64.StdEncoding.EncodeToString(ciphertext)
		})
	*/
}

var encryptionKey = []byte(`0123456789abcdef`)

func (tree *Tree) decrypt() {
	/*
		mapLeaves(tree.items, func(leaf string) string {
			data, err := base64.StdEncoding.DecodeString(leaf)
			if err != nil {
				// it must be decrypted already
				return leaf
			}
			if len(data) < 1 {
				panic("too little encrypted data")
			}
			if data[0] != 1 {
				panic("I only understand version 1")
			}
			return string(data[1:])
		})
	*/
}

/*
func mapLeaves(items yaml.MapSlice, f func(string) string) {
	for i := range items {
		item := &items[i]
		val := item.Value
		if items, ok := val.(yaml.MapSlice); ok {
			mapLeaves(items, f)
		} else if str, ok := val.(string); ok {
			item.Value = f(str)
		} else {
			panic("unexpected leaf type")
		}
	}
}
*/

func home() (string, error) {
	home := os.Getenv("HOME")
	if home == "" {
		return "", errors.New("Point $HOME at your home directory")
	}
	return home, nil
}

func hushPath() (string, error) {
	home, err := home()
	if err != nil {
		return "", err
	}
	return path.Join(home, ".hush"), nil
}

var editorVarNames = []string{
	"HUSH_EDITOR",
	"VISUAL",
	"EDITOR",
}

func editor() string {
	for _, varName := range editorVarNames {
		ed := os.Getenv(varName)
		if ed != "" {
			return ed
		}
	}

	ed := "vi"
	warn("environment configures no editor. defaulting to %s", ed)
	return ed
}
<|endoftext|>


<commit_before>package ipdb

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"strings"
	//"fmt"
)

const METALEN = 20

// Header struct
type Header struct {
	version                                                       uint32
	continentLen, countryLen, areaLen, regionLen, cityLen, ispLen uint16
	netLen                                                        uint32
}

// Record struct
type Record struct {
	ip                                                      uint32 //ip
	mask                                                    uint8  //mask
	continentID, countryID, areaID, regionID, cityID, ispID uint16
}

// Hdata struct
type Hdata struct {
	id   uint16
	name string
}

// DB struct
type DB struct {
	Head                                        Header
	Continent, Country, Area, Region, City, Isp map[uint16]string
	Idx                                         map[int]uint32
	Rstart                                      int
	Data                                        []byte
}

// Result structs
type Result struct {
	Cidr, Continent, Country, Area, Region, City, Isp string
}

func Load(dataFile string) (db *DB, err error) {
	data, err := ioutil.ReadFile(dataFile)
	if err != nil {
		return
	}
	db = new(DB)
	db.Init(data)
	return
}

func IP2long(ipv4 string) (ret uint32, err error) {
	ret = 0
	ip := net.ParseIP(ipv4)
	if ip == nil {
		err = errors.New("Invalid ip format")
		return
	}
	return binary.BigEndian.Uint32(ip.To4()), nil
}

func Long2Ip(a uint32) net.IP {
	return net.IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a))
}

func (db *DB) Init(data []byte) {
	db.Head = Header{
		binary.BigEndian.Uint32(data[:4]),
		binary.BigEndian.Uint16(data[4 : 4+2]),
		binary.BigEndian.Uint16(data[6 : 6+2]),
		binary.BigEndian.Uint16(data[8 : 8+2]),
		binary.BigEndian.Uint16(data[10 : 10+2]),
		binary.BigEndian.Uint16(data[12 : 12+2]),
		binary.BigEndian.Uint16(data[14 : 14+2]),
		binary.BigEndian.Uint32(data[16 : 16+4]),
	}
	//fmt.Printf("head = %+v", db.Head)
	db.Continent = make(map[uint16]string, db.Head.continentLen)
	for i := 0; i < int(db.Head.continentLen); i++ {
		pos := METALEN + i*4
		pack := data[pos : pos+4]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+2]),
		}
		db.Continent[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Country = make(map[uint16]string, db.Head.countryLen)
	for i := 0; i < int(db.Head.countryLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + i*4
		pack := data[pos : pos+4]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+2]),
		}
		db.Country[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Area = make(map[uint16]string, db.Head.areaLen)
	for i := 0; i < int(db.Head.areaLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Area[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Region = make(map[uint16]string, db.Head.regionLen)
	for i := 0; i < int(db.Head.regionLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Region[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.City = make(map[uint16]string, db.Head.cityLen)
	for i := 0; i < int(db.Head.cityLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.City[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Isp = make(map[uint16]string, db.Head.ispLen)
	for i := 0; i < int(db.Head.ispLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + int(db.Head.cityLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Isp[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Rstart = METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
		int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + int(db.Head.cityLen)*66 +
		int(db.Head.ispLen)*66

	db.Idx = make(map[int]uint32, 256)
	for i := 0; i < 256; i++ {
		off := db.Rstart + int(db.Head.netLen)*17 + i*4
		db.Idx[i] = binary.BigEndian.Uint32(data[off : off+4])
	}
	//fmt.Printf("idx = %+v", db.Idx)
	db.Data = data
	return
}

func (db *DB) Find(ipv4 string) (result *Result, err error) {
	var ip32 uint32
	if ip32, err = IP2long(ipv4); err != nil {
		return nil, err
	}
	return db.FindByUint(ip32)
}

func (db *DB) findStartIdxOffset(ip uint32) (start int) {
	ipFirst := int(ip >> 24)
	start = 0
	for i := ipFirst; i > 0; i-- {
		if db.Idx[i] != 0 {
			start = int(db.Idx[i])
			break
		}
	}
	return
}

func (db *DB) findEndIdxOffset(ip uint32) (end int) {
	ipFirst := int(ip >> 24)
	end = int(db.Head.netLen - 1)
	for i := ipFirst + 1; i < 256; i++ {
		if db.Idx[i] != 0 {
			end = int(db.Idx[i])
			break
		}
	}
	return
}

func (db *DB) FindByUint(ip uint32) (result *Result, err error) {
	f := db.findStartIdxOffset(ip)
	n := 0
	l := db.findEndIdxOffset(ip)
	for f <= l {
		m := int((f + l) / 2)
		n = n + 1
		p := db.Rstart + m*17
		pack := db.Data[p : p+17]
		r := Record{
			binary.BigEndian.Uint32(pack[:4]),
			uint8(pack[4 : 4+1][0]),
			binary.BigEndian.Uint16(pack[5 : 5+2]),
			binary.BigEndian.Uint16(pack[7 : 7+2]),
			binary.BigEndian.Uint16(pack[9 : 9+2]),
			binary.BigEndian.Uint16(pack[11 : 11+2]),
			binary.BigEndian.Uint16(pack[13 : 13+2]),
			binary.BigEndian.Uint16(pack[15 : 15+2]),
		}
		rs := r.ip
		re := rs + uint32(math.Pow(2, float64(32-r.mask))) - 1
		if ip >= rs && ip <= re {
			return &Result{
				Cidr:      fmt.Sprintf("%s/%d", Long2Ip(rs).To4(), r.mask),
				Continent: db.Continent[r.continentID],
				Country:   db.Country[r.countryID],
				Area:      db.Area[r.areaID],
				Region:    db.Region[r.regionID],
				City:      db.City[r.cityID],
				Isp:       db.Isp[r.ispID],
			}, nil
		}
		if ip > re {
			f = m + 1
		}
		if ip < rs {
			l = m - 1
		}
	}
	return nil, errors.New("Not Found")
}
<commit_msg>fix unload db err<commit_after>package ipdb

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"net"
	"strings"
	//"fmt"
)

const METALEN = 20

// Header struct
type Header struct {
	version                                                       uint32
	continentLen, countryLen, areaLen, regionLen, cityLen, ispLen uint16
	netLen                                                        uint32
}

// Record struct
type Record struct {
	ip                                                      uint32 //ip
	mask                                                    uint8  //mask
	continentID, countryID, areaID, regionID, cityID, ispID uint16
}

// Hdata struct
type Hdata struct {
	id   uint16
	name string
}

// DB struct
type DB struct {
	Head                                        Header
	Continent, Country, Area, Region, City, Isp map[uint16]string
	Idx                                         map[int]uint32
	Rstart                                      int
	Data                                        []byte
}

// Result structs
type Result struct {
	Cidr, Continent, Country, Area, Region, City, Isp string
}

func Load(dataFile string) (db *DB, err error) {
	data, err := ioutil.ReadFile(dataFile)
	if err != nil {
		return
	}
	db = new(DB)
	db.Init(data)
	return
}

func IP2long(ipv4 string) (ret uint32, err error) {
	ret = 0
	ip := net.ParseIP(ipv4)
	if ip == nil {
		err = errors.New("Invalid ip format")
		return
	}
	return binary.BigEndian.Uint32(ip.To4()), nil
}

func Long2Ip(a uint32) net.IP {
	return net.IPv4(byte(a>>24), byte(a>>16), byte(a>>8), byte(a))
}

func (db *DB) Init(data []byte) {
	db.Head = Header{
		binary.BigEndian.Uint32(data[:4]),
		binary.BigEndian.Uint16(data[4 : 4+2]),
		binary.BigEndian.Uint16(data[6 : 6+2]),
		binary.BigEndian.Uint16(data[8 : 8+2]),
		binary.BigEndian.Uint16(data[10 : 10+2]),
		binary.BigEndian.Uint16(data[12 : 12+2]),
		binary.BigEndian.Uint16(data[14 : 14+2]),
		binary.BigEndian.Uint32(data[16 : 16+4]),
	}
	//fmt.Printf("head = %+v", db.Head)
	db.Continent = make(map[uint16]string, db.Head.continentLen)
	for i := 0; i < int(db.Head.continentLen); i++ {
		pos := METALEN + i*4
		pack := data[pos : pos+4]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+2]),
		}
		db.Continent[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Country = make(map[uint16]string, db.Head.countryLen)
	for i := 0; i < int(db.Head.countryLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + i*4
		pack := data[pos : pos+4]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+2]),
		}
		db.Country[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Area = make(map[uint16]string, db.Head.areaLen)
	for i := 0; i < int(db.Head.areaLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Area[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Region = make(map[uint16]string, db.Head.regionLen)
	for i := 0; i < int(db.Head.regionLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Region[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.City = make(map[uint16]string, db.Head.cityLen)
	for i := 0; i < int(db.Head.cityLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.City[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Isp = make(map[uint16]string, db.Head.ispLen)
	for i := 0; i < int(db.Head.ispLen); i++ {
		pos := METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
			int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + int(db.Head.cityLen)*66 + i*66
		pack := data[pos : pos+66]
		dc := Hdata{
			binary.BigEndian.Uint16(pack[:2]),
			string(pack[2 : 2+64]),
		}
		db.Isp[dc.id] = strings.Trim(dc.name, "\u0000")
		//fmt.Printf("dc = %+v", dc)
	}
	db.Rstart = METALEN + int(db.Head.continentLen)*4 + int(db.Head.countryLen)*4 +
		int(db.Head.areaLen)*66 + int(db.Head.regionLen)*66 + int(db.Head.cityLen)*66 +
		int(db.Head.ispLen)*66

	db.Idx = make(map[int]uint32, 256)
	for i := 0; i < 256; i++ {
		off := db.Rstart + int(db.Head.netLen)*17 + i*4
		db.Idx[i] = binary.BigEndian.Uint32(data[off : off+4])
	}
	//fmt.Printf("idx = %+v", db.Idx)
	db.Data = data
	return
}

func (db *DB) Find(ipv4 string) (result *Result, err error) {
	var ip32 uint32
	if ip32, err = IP2long(ipv4); err != nil {
		return nil, err
	}
	return db.FindByUint(ip32)
}

func (db *DB) findStartIdxOffset(ip uint32) (start int) {
	ipFirst := int(ip >> 24)
	start = 0
	for i := ipFirst; i > 0; i-- {
		if db.Idx[i] != 0 {
			start = int(db.Idx[i])
			break
		}
	}
	return
}

func (db *DB) findEndIdxOffset(ip uint32) (end int) {
	ipFirst := int(ip >> 24)
	end = int(db.Head.netLen - 1)
	for i := ipFirst + 1; i < 256; i++ {
		if db.Idx[i] != 0 {
			end = int(db.Idx[i])
			break
		}
	}
	return
}

func (db *DB) FindByUint(ip uint32) (result *Result, err error) {
	if db == nil || db.Rstart == 0 {
		return nil, errors.New("unload db first")
	}
	f := db.findStartIdxOffset(ip)
	n := 0
	l := db.findEndIdxOffset(ip)
	for f <= l {
		m := int((f + l) / 2)
		n = n + 1
		p := db.Rstart + m*17
		pack := db.Data[p : p+17]
		r := Record{
			binary.BigEndian.Uint32(pack[:4]),
			uint8(pack[4 : 4+1][0]),
			binary.BigEndian.Uint16(pack[5 : 5+2]),
			binary.BigEndian.Uint16(pack[7 : 7+2]),
			binary.BigEndian.Uint16(pack[9 : 9+2]),
			binary.BigEndian.Uint16(pack[11 : 11+2]),
			binary.BigEndian.Uint16(pack[13 : 13+2]),
			binary.BigEndian.Uint16(pack[15 : 15+2]),
		}
		rs := r.ip
		re := rs + uint32(math.Pow(2, float64(32-r.mask))) - 1
		if ip >= rs && ip <= re {
			return &Result{
				Cidr:      fmt.Sprintf("%s/%d", Long2Ip(rs).To4(), r.mask),
				Continent: db.Continent[r.continentID],
				Country:   db.Country[r.countryID],
				Area:      db.Area[r.areaID],
				Region:    db.Region[r.regionID],
				City:      db.City[r.cityID],
				Isp:       db.Isp[r.ispID],
			}, nil
		}
		if ip > re {
			f = m + 1
		}
		if ip < rs {
			l = m - 1
		}
	}
	return nil, errors.New("Not Found")
}
<|endoftext|>