<commit_before>// +build windows

package gopsutil

import (
	"os"
	"syscall"
	"unsafe"
)

var (
	procGetSystemTimeAsFileTime = modkernel32.NewProc("GetSystemTimeAsFileTime")
	procGetTickCount            = modkernel32.NewProc("GetTickCount")
)

func HostInfo() (*HostInfoStat, error) {
	ret := &HostInfoStat{}
	hostname, err := os.Hostname()
	if err != nil {
		return ret, err
	}

	ret.Hostname = hostname
	uptimemsec, _, err := procGetTickCount.Call()
	if uptimemsec == 0 {
		return ret, syscall.GetLastError()
	}

	ret.Uptime = uint64(uptimemsec) / 1000

	procs, err := Pids()
	if err != nil {
		return ret, err
	}

	ret.Procs = uint64(len(procs))

	return ret, nil
}

func BootTime() (int64, error) {
	var lpSystemTimeAsFileTime FILETIME

	r, _, _ := procGetSystemTimeAsFileTime.Call(uintptr(unsafe.Pointer(&lpSystemTimeAsFileTime)))
	if r == 0 {
		return 0, syscall.GetLastError()
	}

	// TODO: This calc is wrong.
	ll := (uint32(lpSystemTimeAsFileTime.DwHighDateTime))<<32 + lpSystemTimeAsFileTime.DwLowDateTime
	pt := (uint64(ll) - 116444736000000000) / 10000000

	u, _, _ := procGetTickCount.Call()
	if u == 0 {
		return 0, syscall.GetLastError()
	}
	uptime := uint64(u) / 1000

	return uint64(pt - uptime), nil
}
func Users() ([]UserStat, error) {

	var ret []UserStat

	return ret, nil
}
<commit_msg>fix the type declaration on Windows<commit_after>// +build windows

package gopsutil

import (
	"os"
	"syscall"
	"unsafe"
)

var (
	procGetSystemTimeAsFileTime = modkernel32.NewProc("GetSystemTimeAsFileTime")
	procGetTickCount            = modkernel32.NewProc("GetTickCount")
)

func HostInfo() (*HostInfoStat, error) {
	ret := &HostInfoStat{}
	hostname, err := os.Hostname()
	if err != nil {
		return ret, err
	}

	ret.Hostname = hostname
	uptimemsec, _, err := procGetTickCount.Call()
	if uptimemsec == 0 {
		return ret, syscall.GetLastError()
	}

	ret.Uptime = uint64(uptimemsec) / 1000

	procs, err := Pids()
	if err != nil {
		return ret, err
	}

	ret.Procs = uint64(len(procs))

	return ret, nil
}

func BootTime() (uint64, error) {
	var lpSystemTimeAsFileTime FILETIME

	r, _, _ := procGetSystemTimeAsFileTime.Call(uintptr(unsafe.Pointer(&lpSystemTimeAsFileTime)))
	if r == 0 {
		return 0, syscall.GetLastError()
	}

	// TODO: This calc is wrong.
	ll := (uint32(lpSystemTimeAsFileTime.DwHighDateTime))<<32 + lpSystemTimeAsFileTime.DwLowDateTime
	pt := (uint64(ll) - 116444736000000000) / 10000000

	u, _, _ := procGetTickCount.Call()
	if u == 0 {
		return 0, syscall.GetLastError()
	}
	uptime := uint64(u) / 1000

	return uint64(pt - uptime), nil
}
func Users() ([]UserStat, error) {

	var ret []UserStat

	return ret, nil
}
<|endoftext|>


<commit_before>// Package plugin enables plugins to register with Abot and connect to the
// database.
package plugin

import (
	"bufio"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"

	"github.com/itsabot/abot/core"
	"github.com/itsabot/abot/core/log"
	"github.com/itsabot/abot/shared/datatypes"
	"github.com/itsabot/abot/shared/nlp"
	_ "github.com/lib/pq" // Import the pq PostgreSQL driver
)

// ErrMissingPluginName is returned when a plugin name is expected, but
// but a blank name is provided.
var ErrMissingPluginName = errors.New("missing plugin name")

// ErrMissingTrigger is returned when a trigger is expected but none
// were found.
var ErrMissingTrigger = errors.New("missing plugin trigger")

// ErrMissingPluginFns is returned when plugin functions are expected but none
// were found.
var ErrMissingPluginFns = errors.New("missing plugin functions")

// New builds a Plugin with its trigger, RPC, and configuration settings from
// its plugin.json.
func New(url string, trigger *nlp.StructuredInput,
	fns *dt.PluginFns) (*dt.Plugin, error) {

	if trigger == nil {
		return nil, ErrMissingTrigger
	}
	if fns == nil || fns.Run == nil || fns.FollowUp == nil {
		return nil, ErrMissingPluginFns
	}

	// Read plugin.json data from within plugins.go, unmarshal into struct
	p := filepath.Join(os.Getenv("GOPATH"), "src", "github.com", "itsabot",
		"abot", "plugins.go")
	fi, err := os.Open(p)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := fi.Close(); err != nil {
			log.Info("failed to close file", fi.Name())
			return
		}
	}()
	var found bool
	var data string
	scn := bufio.NewScanner(fi)
	for scn.Scan() {
		t := scn.Text()
		if !found && t != url {
			continue
		} else if t == url {
			found = true
			continue
		} else if len(t) >= 1 && t[0] == '}' {
			data += t
			break
		}
		data += t
	}
	if err := scn.Err(); err != nil {
		return nil, err
	}

	c := dt.PluginConfig{}
	if err = json.Unmarshal([]byte(data), &c); err != nil {
		return nil, err
	}
	if len(c.Name) == 0 {
		return nil, ErrMissingPluginName
	}
	db, err := core.ConnectDB()
	if err != nil {
		return nil, err
	}
	l := log.New(c.Name)
	l.SetDebug(os.Getenv("ABOT_DEBUG") == "true")
	plg := &dt.Plugin{
		Config:    c,
		Trigger:   trigger,
		DB:        db,
		Log:       log.New(c.Name),
		PluginFns: fns,
		Events: &dt.PluginEvents{
			PostReceive:    func(cmd *string) {},
			PreProcessing:  func(cmd *string, u *dt.User) {},
			PostProcessing: func(in *dt.Msg) {},
			PostResponse:   func(in *dt.Msg, resp *string) {},
		},
	}
	if err = RegisterPlugin(plg); err != nil {
		return nil, err
	}
	return plg, nil
}

// RegisterPlugin enables Abot to notify plugins when specific StructuredInput
// is encountered matching triggers set in the plugins themselves. Note that
// plugins will only listen when ALL criteria are met and that there's no
// support currently for duplicate routes (e.g. "find_restaurant" leading to
// either one of two plugins).
func RegisterPlugin(p *dt.Plugin) error {
	log.Debug("registering", p.Config.Name)
	for _, c := range p.Trigger.Commands {
		for _, o := range p.Trigger.Objects {
			s := strings.ToLower(c + "_" + o)
			if core.RegPlugins.Get(s) != nil {
				log.Info("found duplicate plugin or trigger",
					p.Config.Name, "on", s)
			}
			core.RegPlugins.Set(s, p)
		}
	}
	core.AllPlugins = append(core.AllPlugins, p)
	return nil
}
<commit_msg>Fix missing plugins.go bug<commit_after>// Package plugin enables plugins to register with Abot and connect to the
// database.
package plugin

import (
	"bufio"
	"encoding/json"
	"errors"
	"os"
	"path/filepath"
	"strings"

	"github.com/itsabot/abot/core"
	"github.com/itsabot/abot/core/log"
	"github.com/itsabot/abot/shared/datatypes"
	"github.com/itsabot/abot/shared/nlp"
	_ "github.com/lib/pq" // Import the pq PostgreSQL driver
)

// ErrMissingPluginName is returned when a plugin name is expected, but
// but a blank name is provided.
var ErrMissingPluginName = errors.New("missing plugin name")

// ErrMissingTrigger is returned when a trigger is expected but none
// were found.
var ErrMissingTrigger = errors.New("missing plugin trigger")

// ErrMissingPluginFns is returned when plugin functions are expected but none
// were found.
var ErrMissingPluginFns = errors.New("missing plugin functions")

// New builds a Plugin with its trigger, RPC, and configuration settings from
// its plugin.json.
func New(url string, trigger *nlp.StructuredInput,
	fns *dt.PluginFns) (*dt.Plugin, error) {

	if trigger == nil {
		return nil, ErrMissingTrigger
	}
	if fns == nil || fns.Run == nil || fns.FollowUp == nil {
		return nil, ErrMissingPluginFns
	}

	// Read plugin.json data from within plugins.go, unmarshal into struct
	p := filepath.Join(os.Getenv("GOPATH"), "src", "github.com", "itsabot",
		"abot", "plugins.go")
	fi, err := os.OpenFile(p, os.O_RDONLY|os.O_CREATE, 0666)
	if err != nil {
		return nil, err
	}
	defer func() {
		if err := fi.Close(); err != nil {
			log.Info("failed to close file", fi.Name())
			return
		}
	}()
	var found bool
	var data string
	scn := bufio.NewScanner(fi)
	for scn.Scan() {
		t := scn.Text()
		if !found && t != url {
			continue
		} else if t == url {
			found = true
			continue
		} else if len(t) >= 1 && t[0] == '}' {
			data += t
			break
		}
		data += t
	}
	if err := scn.Err(); err != nil {
		return nil, err
	}

	c := dt.PluginConfig{}
	if err = json.Unmarshal([]byte(data), &c); err != nil {
		return nil, err
	}
	if len(c.Name) == 0 {
		return nil, ErrMissingPluginName
	}
	db, err := core.ConnectDB()
	if err != nil {
		return nil, err
	}
	l := log.New(c.Name)
	l.SetDebug(os.Getenv("ABOT_DEBUG") == "true")
	plg := &dt.Plugin{
		Config:    c,
		Trigger:   trigger,
		DB:        db,
		Log:       log.New(c.Name),
		PluginFns: fns,
		Events: &dt.PluginEvents{
			PostReceive:    func(cmd *string) {},
			PreProcessing:  func(cmd *string, u *dt.User) {},
			PostProcessing: func(in *dt.Msg) {},
			PostResponse:   func(in *dt.Msg, resp *string) {},
		},
	}
	if err = RegisterPlugin(plg); err != nil {
		return nil, err
	}
	return plg, nil
}

// RegisterPlugin enables Abot to notify plugins when specific StructuredInput
// is encountered matching triggers set in the plugins themselves. Note that
// plugins will only listen when ALL criteria are met and that there's no
// support currently for duplicate routes (e.g. "find_restaurant" leading to
// either one of two plugins).
func RegisterPlugin(p *dt.Plugin) error {
	log.Debug("registering", p.Config.Name)
	for _, c := range p.Trigger.Commands {
		for _, o := range p.Trigger.Objects {
			s := strings.ToLower(c + "_" + o)
			if core.RegPlugins.Get(s) != nil {
				log.Info("found duplicate plugin or trigger",
					p.Config.Name, "on", s)
			}
			core.RegPlugins.Set(s, p)
		}
	}
	core.AllPlugins = append(core.AllPlugins, p)
	return nil
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"io"
	"strconv"
	"text/scanner"

	"github.com/zimmski/tavor/token"
	"github.com/zimmski/tavor/token/lists"
	"github.com/zimmski/tavor/token/primitives"
)

//TODO remove this
var DEBUG = false

const zeroRune = 0

type tavorParser struct {
	scan scanner.Scanner

	err string

	lookup map[string]token.Token
	used   map[string]struct{}
}

func (p *tavorParser) expectRune(expect rune, got rune) (rune, error) {
	if got != expect {
		return got, &ParserError{
			Message: fmt.Sprintf("Expected \"%c\" but got \"%c\"", expect, got),
			Type:    ParseErrorExpectRune,
		}
	}

	return got, nil
}

func (p *tavorParser) expectScanRune(expect rune) (rune, error) {
	got := p.scan.Scan()
	if DEBUG {
		fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(got), p.scan.TokenText())
	}

	return p.expectRune(expect, got)
}

func (p *tavorParser) parseTerm(c rune) (token.Token, error) {
	switch c {
	case scanner.Ident:
		n := p.scan.TokenText()

		if _, ok := p.lookup[n]; !ok {
			return nil, &ParserError{
				Message: fmt.Sprintf("Token %s does not exists", n),
				Type:    ParseErrorTokenDoesNotExists,
			}
		}

		p.used[n] = struct{}{}

		return p.lookup[n].Clone(), nil
	case scanner.Int:
		v, _ := strconv.Atoi(p.scan.TokenText())

		return primitives.NewConstantInt(v), nil
	case scanner.String:
		s := p.scan.TokenText()

		if s[0] != '"' {
			panic("unknown " + s) // TODO remove this
		}

		if s[len(s)-1] != '"' {
			return nil, &ParserError{
				Message: "String is not terminated",
				Type:    ParseErrorNonTerminatedString,
			}
		}

		return primitives.NewConstantString(s[1 : len(s)-1]), nil
	}

	return nil, nil
}

func (p *tavorParser) parseTokenDefinition() (rune, error) {
	name := p.scan.TokenText()

	if _, ok := p.lookup[name]; ok {
		return zeroRune, &ParserError{
			Message: "Token already exists",
			Type:    ParseErrorTokenExists,
		}
	}

	// do an empty definition to allow loops
	p.lookup[name] = nil

	if c, err := p.expectScanRune('='); err != nil {
		// unexpected new line?
		if c == '\n' {
			return zeroRune, &ParserError{
				Message: "New line inside single line token definitions is not allowed",
				Type:    ParseErrorEarlyNewLine,
			}
		}

		return zeroRune, err
	}

	tokens := make([]token.Token, 0)

	c := p.scan.Scan()
	if DEBUG {
		fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
	}

	for {
		tok, err := p.parseTerm(c)
		if err != nil {
			return zeroRune, err
		} else if tok != nil {
			tokens = append(tokens, tok)
		} else {
			switch c {
			case scanner.EOF:
				return zeroRune, &ParserError{
					Message: "New line at end of token definition needed",
					Type:    ParseErrorNewLineNeeded,
				}
			case ',': // multi line token
				if _, err := p.expectScanRune('\n'); err != nil {
					return zeroRune, err
				}

				c = p.scan.Scan()
				if DEBUG {
					fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
				}

				if c == '\n' {
					return zeroRune, &ParserError{
						Message: "Multi line token definition unexpectedly terminated",
						Type:    ParseErrorUnexpectedTokenDefinitionTermination,
					}
				}

				continue
			case '\n':
				switch len(tokens) {
				case 0:
					return zeroRune, &ParserError{
						Message: "Empty token definition",
						Type:    ParseErrorEmptyTokenDefinition,
					}
				case 1:
					p.lookup[name] = tokens[0]
				default:
					p.lookup[name] = lists.NewAll(tokens...)
				}

				c = p.scan.Scan()
				if DEBUG {
					fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
				}

				return c, nil
			default:
				panic("now what?") // TODO remove this
			}
		}

		c = p.scan.Scan()
		if DEBUG {
			fmt.Printf("aa%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
		}
	}

	return c, nil
}

func ParseTavor(src io.Reader) (token.Token, error) {
	var err error

	p := &tavorParser{
		lookup: make(map[string]token.Token),
		used:   make(map[string]struct{}),
	}

	if DEBUG {
		fmt.Println("INIT")
	}

	p.scan.Init(src)

	p.scan.Error = func(s *scanner.Scanner, msg string) {
		p.err = msg
	}
	p.scan.Whitespace = 1<<'\t' | 1<<' ' | 1<<'\r'

	c := p.scan.Scan()
	if DEBUG {
		fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
	}

	for c != scanner.EOF {
		switch c {
		case '\n':
			// ignore new lines in the global scope
		case scanner.Ident:
			c, err = p.parseTokenDefinition()
			if err != nil {
				return nil, err
			}

			continue
		case scanner.Int:
			return nil, &ParserError{
				Message: "Token names have to start with a letter",
				Type:    ParseErrorInvalidTokenName,
			}
		default:
			panic("what am i to do now") // TODO remove this
		}

		c = p.scan.Scan()
		if DEBUG {
			fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
		}
	}

	if _, ok := p.lookup["START"]; !ok {
		return nil, &ParserError{
			Message: "No START token defined",
			Type:    ParseErrorNoStart,
		}
	}

	p.used["START"] = struct{}{}

	for key := range p.lookup {
		if _, ok := p.used[key]; !ok {
			return nil, &ParserError{
				Message: fmt.Sprintf("Token %s declared but not used", key),
				Type:    ParseErrorUnusedToken,
			}
		}
	}

	return p.lookup["START"], nil
}
<commit_msg>refactor global scope parsing into own method<commit_after>package parser

import (
	"fmt"
	"io"
	"strconv"
	"text/scanner"

	"github.com/zimmski/tavor/token"
	"github.com/zimmski/tavor/token/lists"
	"github.com/zimmski/tavor/token/primitives"
)

//TODO remove this
var DEBUG = false

const zeroRune = 0

type tavorParser struct {
	scan scanner.Scanner

	err string

	lookup map[string]token.Token
	used   map[string]struct{}
}

func (p *tavorParser) expectRune(expect rune, got rune) (rune, error) {
	if got != expect {
		return got, &ParserError{
			Message: fmt.Sprintf("Expected \"%c\" but got \"%c\"", expect, got),
			Type:    ParseErrorExpectRune,
		}
	}

	return got, nil
}

func (p *tavorParser) expectScanRune(expect rune) (rune, error) {
	got := p.scan.Scan()
	if DEBUG {
		fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(got), p.scan.TokenText())
	}

	return p.expectRune(expect, got)
}

func (p *tavorParser) parseGlobalScope() error {
	var err error

	c := p.scan.Scan()
	if DEBUG {
		fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
	}

	for c != scanner.EOF {
		switch c {
		case '\n':
			// ignore new lines in the global scope
		case scanner.Ident:
			c, err = p.parseTokenDefinition()
			if err != nil {
				return err
			}

			continue
		case scanner.Int:
			return &ParserError{
				Message: "Token names have to start with a letter",
				Type:    ParseErrorInvalidTokenName,
			}
		default:
			panic("what am i to do now") // TODO remove this
		}

		c = p.scan.Scan()
		if DEBUG {
			fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
		}
	}

	return nil
}

func (p *tavorParser) parseTerm(c rune) (token.Token, error) {
	switch c {
	case scanner.Ident:
		n := p.scan.TokenText()

		if _, ok := p.lookup[n]; !ok {
			return nil, &ParserError{
				Message: fmt.Sprintf("Token %s does not exists", n),
				Type:    ParseErrorTokenDoesNotExists,
			}
		}

		p.used[n] = struct{}{}

		return p.lookup[n].Clone(), nil
	case scanner.Int:
		v, _ := strconv.Atoi(p.scan.TokenText())

		return primitives.NewConstantInt(v), nil
	case scanner.String:
		s := p.scan.TokenText()

		if s[0] != '"' {
			panic("unknown " + s) // TODO remove this
		}

		if s[len(s)-1] != '"' {
			return nil, &ParserError{
				Message: "String is not terminated",
				Type:    ParseErrorNonTerminatedString,
			}
		}

		return primitives.NewConstantString(s[1 : len(s)-1]), nil
	}

	return nil, nil
}

func (p *tavorParser) parseTokenDefinition() (rune, error) {
	name := p.scan.TokenText()

	if _, ok := p.lookup[name]; ok {
		return zeroRune, &ParserError{
			Message: "Token already exists",
			Type:    ParseErrorTokenExists,
		}
	}

	// do an empty definition to allow loops
	p.lookup[name] = nil

	if c, err := p.expectScanRune('='); err != nil {
		// unexpected new line?
		if c == '\n' {
			return zeroRune, &ParserError{
				Message: "New line inside single line token definitions is not allowed",
				Type:    ParseErrorEarlyNewLine,
			}
		}

		return zeroRune, err
	}

	tokens := make([]token.Token, 0)

	c := p.scan.Scan()
	if DEBUG {
		fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
	}

	for {
		tok, err := p.parseTerm(c)
		if err != nil {
			return zeroRune, err
		} else if tok != nil {
			tokens = append(tokens, tok)
		} else {
			switch c {
			case scanner.EOF:
				return zeroRune, &ParserError{
					Message: "New line at end of token definition needed",
					Type:    ParseErrorNewLineNeeded,
				}
			case ',': // multi line token
				if _, err := p.expectScanRune('\n'); err != nil {
					return zeroRune, err
				}

				c = p.scan.Scan()
				if DEBUG {
					fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
				}

				if c == '\n' {
					return zeroRune, &ParserError{
						Message: "Multi line token definition unexpectedly terminated",
						Type:    ParseErrorUnexpectedTokenDefinitionTermination,
					}
				}

				continue
			case '\n':
				switch len(tokens) {
				case 0:
					return zeroRune, &ParserError{
						Message: "Empty token definition",
						Type:    ParseErrorEmptyTokenDefinition,
					}
				case 1:
					p.lookup[name] = tokens[0]
				default:
					p.lookup[name] = lists.NewAll(tokens...)
				}

				c = p.scan.Scan()
				if DEBUG {
					fmt.Printf("%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
				}

				return c, nil
			default:
				panic("now what?") // TODO remove this
			}
		}

		c = p.scan.Scan()
		if DEBUG {
			fmt.Printf("aa%d:%v -> %v\n", p.scan.Line, scanner.TokenString(c), p.scan.TokenText())
		}
	}

	return c, nil
}

func ParseTavor(src io.Reader) (token.Token, error) {
	p := &tavorParser{
		lookup: make(map[string]token.Token),
		used:   make(map[string]struct{}),
	}

	if DEBUG {
		fmt.Println("INIT")
	}

	p.scan.Init(src)

	p.scan.Error = func(s *scanner.Scanner, msg string) {
		p.err = msg
	}
	p.scan.Whitespace = 1<<'\t' | 1<<' ' | 1<<'\r'

	if err := p.parseGlobalScope(); err != nil {
		return nil, err
	}

	if _, ok := p.lookup["START"]; !ok {
		return nil, &ParserError{
			Message: "No START token defined",
			Type:    ParseErrorNoStart,
		}
	}

	p.used["START"] = struct{}{}

	for key := range p.lookup {
		if _, ok := p.used[key]; !ok {
			return nil, &ParserError{
				Message: fmt.Sprintf("Token %s declared but not used", key),
				Type:    ParseErrorUnusedToken,
			}
		}
	}

	return p.lookup["START"], nil
}
<|endoftext|>


<commit_before>package filter

import (
	"github.com/pa001024/MoeCron/source"
	"regexp"
)

type FilterMoegirlwiki struct { // 萌娘百科
	IFilter
}

var (
	rep_mw_round1 = regexp.MustCompile(
		`^[\s\S]+?\s==? ?(?:基本介绍|简介|簡介|.{2,4}设定|.{2,4}介绍) ?==\n|` + // 去除第一个区块前的所有内容 替换为空 如果没有任何区块则不会替换
			`{{[\s\S]+?}}|` + // 去除所有{{}}标签
			// `\[\[(?:File|分类):[\s\S]+?\]\]|` + // 提取并替换所有WIKI标签(提取图片和分类)
			`\[\[([^|\s]+?)\]\]|` + // 替换[[明文|词条名]]词条链接为明文 即$1
			`\[\[[\s\S]+?\]\]|` + // 替换[[明文]]词条链接为明文
			`'''(.+?)'''|` + // 替换粗体为明文 即$2
			`<s>.+?</s>|` + // 去除所有<s></s>标签
			`<\w+>(.+?)</\w+>` + // 替换所有HTML标签为明文 即$3
			``)
	rep_mw_round2 = regexp.MustCompile(
		`(?:\n|^).{1,12}[：:].+|` + // 去除说明体文字 如 姓名: xxx  `\n(?:本名|姓名|身高|机体|声优|种别|种类|萌点|年龄)[：:].+` +
			`== (?:基本信息|登场作品|人物档案|人物信息|外部链接) ==[\s\S]+?(==|$)|` + // 去除无用区块 $1
			`==+? ?.+? ?==+|` + // 去除标题
			`\* ?|` + // 去除列表
			`\[\[([^|\s]+?)\]\]|` + // 第二次替换[[明文|词条名]]词条链接为明文 即$2
			`<[\w "%%=/\-:]+?>` + // 去除HTML标签如
<references />
			``)
	rep_mw_round3 = regexp.MustCompile(
		`===+ (?:基本资料) ===+[\s\S]+?(==|$)|` + // 去除无用小区块 $1
			`(\n)\n+|` + // 去除连续两个以上的换行符 $2
			`^\s+|` + // 去除开头空白
			`（待补完）|` + // 去除（待补完）
			`[^\S\n]\s+` + // 去除两个以上的空白符 行文中分词的单个空格不会被替换 中文空格不会被替换
			``)
	flp_mw = regexp.MustCompile(`R18`)

	// {{[\s\S]+?}}|\[\[[\s\S]+?\]\]|'''(.+?)'''|== (.+?) ==|<references />|\s+
)

func (this *FilterMoegirlwiki) FilterContent(src string) (dst string) {
	const LEN_LIMIT = 100
	dst = rep_mw_round3.ReplaceAllString(
		rep_mw_round2.ReplaceAllString(
			rep_mw_round1.ReplaceAllString(src, "$1$2$3"), "$1$2"), "$1$2")
	ds := []rune(dst)
	if len(ds) > LEN_LIMIT {
		dst = string(ds[0:LEN_LIMIT]) + "..."
	}
	return dst
}

func (this *FilterMoegirlwiki) Process(src []*source.FeedInfo) (dst []*source.FeedInfo) {
	dst = make([]*source.FeedInfo, 0, 10)
	for _, v := range src {
		if !flp_mw.MatchString(v.Title) && !flp_mw.MatchString(v.Content) {
			nv := *v
			nv.Content = this.FilterContent(nv.Content)
			dst = append(dst, &nv)
		}
	}
	return
}
<commit_msg>Update FilterMoegirlwiki<commit_after>package filter

import (
	"github.com/pa001024/MoeCron/source"
	"regexp"
)

type FilterMoegirlwiki struct { // 萌娘百科
	IFilter
}

var (
	rep_mw_round1 = regexp.MustCompile(
		`^[\s\S]+?\s==? ?(?:基本介绍|简介|簡介|.{2,4}设定|.{2,4}介绍) ?==\n|` + // 去除第一个区块前的所有内容 替换为空 如果没有任何区块则不会替换
			`{{[\s\S]+?}}|` + // 去除所有{{}}标签
			`{\|[\s\S]+?\|}|` + // 去除所有{||}标签
			// `\[\[(?:File|分类):[\s\S]+?\]\]|` + // 提取并替换所有WIKI标签(提取图片和分类)
			`\[\[([^|\s]+?)\]\]|` + // 替换[[明文|词条名]]词条链接为明文 即$1
			`\[\[[\s\S]+?\]\]|` + // 替换[[明文]]词条链接为明文
			`'''(.+?)'''|` + // 替换粗体为明文 即$2
			`<s>.+?</s>|` + // 去除所有<s></s>标签
			`<\w+>(.+?)</\w+>` + // 替换所有HTML标签为明文 即$3
			``)
	rep_mw_round2 = regexp.MustCompile(
		`(?:\n|^).{1,12}[：:].+|` + // 去除说明体文字 如 姓名: xxx  `\n(?:本名|姓名|身高|机体|声优|种别|种类|萌点|年龄)[：:].+` +
			`== (?:基本信息|登场作品|人物档案|人物信息|外部链接) ==[\s\S]+?(==|$)|` + // 去除无用区块 $1
			`==+? ?.+? ?==+|` + // 去除标题
			`\* ?|` + // 去除列表
			`\[\[([^|\s]+?)\]\]|` + // 第二次替换[[明文|词条名]]词条链接为明文 即$2
			`<[\w "%%=/\-:]+?>` + // 去除HTML标签如
<references />
			``)
	rep_mw_round3 = regexp.MustCompile(
		`===+ (?:基本资料) ===+[\s\S]+?(==|$)|` + // 去除无用小区块 $1
			`(\n)\n+|` + // 去除连续两个以上的换行符 $2
			`^\s+|` + // 去除开头空白
			`（待补完）|` + // 去除（待补完）
			`[^\S\n]\s+` + // 去除两个以上的空白符 行文中分词的单个空格不会被替换 中文空格不会被替换
			``)
	flp_mw = regexp.MustCompile(`R18`)

	// {{[\s\S]+?}}|\[\[[\s\S]+?\]\]|'''(.+?)'''|== (.+?) ==|<references />|\s+
)

func (this *FilterMoegirlwiki) FilterContent(src string) (dst string) {
	const LEN_LIMIT = 100
	dst = rep_mw_round3.ReplaceAllString(
		rep_mw_round2.ReplaceAllString(
			rep_mw_round1.ReplaceAllString(src, "$1$2$3"), "$1$2"), "$1$2")
	ds := []rune(dst)
	if len(ds) > LEN_LIMIT {
		dst = string(ds[0:LEN_LIMIT]) + "..."
	}
	return dst
}

func (this *FilterMoegirlwiki) Process(src []*source.FeedInfo) (dst []*source.FeedInfo) {
	dst = make([]*source.FeedInfo, 0, 10)
	for _, v := range src {
		if !flp_mw.MatchString(v.Title) && !flp_mw.MatchString(v.Content) {
			nv := *v
			nv.Content = this.FilterContent(nv.Content)
			dst = append(dst, &nv)
		}
	}
	return
}
<|endoftext|>


<commit_before>package metrics

import (
	"reflect"
	"runtime"
	"testing"
	"time"
)

func mockMetric() (*MockSink, *Metrics) {
	m := &MockSink{}
	met := &Metrics{sink: m}
	return m, met
}

func TestMetrics_SetGauge(t *testing.T) {
	m, met := mockMetric()
	met.SetGauge([]string{"key"}, float32(1))
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.HostName = "test"
	met.EnableHostname = true
	met.SetGauge([]string{"key"}, float32(1))
	if m.keys[0][0] != "test" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.EnableTypePrefix = true
	met.SetGauge([]string{"key"}, float32(1))
	if m.keys[0][0] != "gauge" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.ServiceName = "service"
	met.SetGauge([]string{"key"}, float32(1))
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}
}

func TestMetrics_EmitKey(t *testing.T) {
	m, met := mockMetric()
	met.EmitKey([]string{"key"}, float32(1))
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.EnableTypePrefix = true
	met.EmitKey([]string{"key"}, float32(1))
	if m.keys[0][0] != "kv" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.ServiceName = "service"
	met.EmitKey([]string{"key"}, float32(1))
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}
}

func TestMetrics_IncrCounter(t *testing.T) {
	m, met := mockMetric()
	met.IncrCounter([]string{"key"}, float32(1))
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.EnableTypePrefix = true
	met.IncrCounter([]string{"key"}, float32(1))
	if m.keys[0][0] != "counter" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.ServiceName = "service"
	met.IncrCounter([]string{"key"}, float32(1))
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}
}

func TestMetrics_AddSample(t *testing.T) {
	m, met := mockMetric()
	met.AddSample([]string{"key"}, float32(1))
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.EnableTypePrefix = true
	met.AddSample([]string{"key"}, float32(1))
	if m.keys[0][0] != "sample" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.ServiceName = "service"
	met.AddSample([]string{"key"}, float32(1))
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}
}

func TestMetrics_MeasureSince(t *testing.T) {
	m, met := mockMetric()
	met.TimerGranularity = time.Millisecond
	n := time.Now()
	met.MeasureSince([]string{"key"}, n)
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] > 0.1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.TimerGranularity = time.Millisecond
	met.EnableTypePrefix = true
	met.MeasureSince([]string{"key"}, n)
	if m.keys[0][0] != "timer" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] > 0.1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.TimerGranularity = time.Millisecond
	met.ServiceName = "service"
	met.MeasureSince([]string{"key"}, n)
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] > 0.1 {
		t.Fatalf("")
	}
}

func TestMetrics_EmitRuntimeStats(t *testing.T) {
	runtime.GC()
	m, met := mockMetric()
	met.emitRuntimeStats()

	if m.keys[0][0] != "runtime" || m.keys[0][1] != "num_goroutines" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[0] <= 1 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[1][0] != "runtime" || m.keys[1][1] != "alloc_bytes" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[1] <= 40000 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[2][0] != "runtime" || m.keys[2][1] != "sys_bytes" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[2] <= 100000 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[3][0] != "runtime" || m.keys[3][1] != "malloc_count" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[3] <= 100 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[4][0] != "runtime" || m.keys[4][1] != "free_count" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[4] <= 100 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[5][0] != "runtime" || m.keys[5][1] != "heap_objects" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[5] <= 100 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[6][0] != "runtime" || m.keys[6][1] != "total_gc_pause_ns" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[6] <= 100000 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[7][0] != "runtime" || m.keys[7][1] != "total_gc_runs" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[7] <= 1 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[8][0] != "runtime" || m.keys[8][1] != "gc_pause_ns" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[8] <= 1000 {
		t.Fatalf("bad val: %v", m.vals)
	}
}

func TestInsert(t *testing.T) {
	k := []string{"hi", "bob"}
	exp := []string{"hi", "there", "bob"}
	out := insert(1, "there", k)
	if !reflect.DeepEqual(exp, out) {
		t.Fatalf("bad insert %v %v", exp, out)
	}
}
<commit_msg>Updating test for Go 1.5<commit_after>package metrics

import (
	"reflect"
	"runtime"
	"testing"
	"time"
)

func mockMetric() (*MockSink, *Metrics) {
	m := &MockSink{}
	met := &Metrics{sink: m}
	return m, met
}

func TestMetrics_SetGauge(t *testing.T) {
	m, met := mockMetric()
	met.SetGauge([]string{"key"}, float32(1))
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.HostName = "test"
	met.EnableHostname = true
	met.SetGauge([]string{"key"}, float32(1))
	if m.keys[0][0] != "test" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.EnableTypePrefix = true
	met.SetGauge([]string{"key"}, float32(1))
	if m.keys[0][0] != "gauge" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.ServiceName = "service"
	met.SetGauge([]string{"key"}, float32(1))
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}
}

func TestMetrics_EmitKey(t *testing.T) {
	m, met := mockMetric()
	met.EmitKey([]string{"key"}, float32(1))
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.EnableTypePrefix = true
	met.EmitKey([]string{"key"}, float32(1))
	if m.keys[0][0] != "kv" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.ServiceName = "service"
	met.EmitKey([]string{"key"}, float32(1))
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}
}

func TestMetrics_IncrCounter(t *testing.T) {
	m, met := mockMetric()
	met.IncrCounter([]string{"key"}, float32(1))
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.EnableTypePrefix = true
	met.IncrCounter([]string{"key"}, float32(1))
	if m.keys[0][0] != "counter" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.ServiceName = "service"
	met.IncrCounter([]string{"key"}, float32(1))
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}
}

func TestMetrics_AddSample(t *testing.T) {
	m, met := mockMetric()
	met.AddSample([]string{"key"}, float32(1))
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.EnableTypePrefix = true
	met.AddSample([]string{"key"}, float32(1))
	if m.keys[0][0] != "sample" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.ServiceName = "service"
	met.AddSample([]string{"key"}, float32(1))
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] != 1 {
		t.Fatalf("")
	}
}

func TestMetrics_MeasureSince(t *testing.T) {
	m, met := mockMetric()
	met.TimerGranularity = time.Millisecond
	n := time.Now()
	met.MeasureSince([]string{"key"}, n)
	if m.keys[0][0] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] > 0.1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.TimerGranularity = time.Millisecond
	met.EnableTypePrefix = true
	met.MeasureSince([]string{"key"}, n)
	if m.keys[0][0] != "timer" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] > 0.1 {
		t.Fatalf("")
	}

	m, met = mockMetric()
	met.TimerGranularity = time.Millisecond
	met.ServiceName = "service"
	met.MeasureSince([]string{"key"}, n)
	if m.keys[0][0] != "service" || m.keys[0][1] != "key" {
		t.Fatalf("")
	}
	if m.vals[0] > 0.1 {
		t.Fatalf("")
	}
}

func TestMetrics_EmitRuntimeStats(t *testing.T) {
	runtime.GC()
	m, met := mockMetric()
	met.emitRuntimeStats()

	if m.keys[0][0] != "runtime" || m.keys[0][1] != "num_goroutines" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[0] <= 1 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[1][0] != "runtime" || m.keys[1][1] != "alloc_bytes" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[1] <= 40000 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[2][0] != "runtime" || m.keys[2][1] != "sys_bytes" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[2] <= 100000 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[3][0] != "runtime" || m.keys[3][1] != "malloc_count" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[3] <= 100 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[4][0] != "runtime" || m.keys[4][1] != "free_count" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[4] <= 100 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[5][0] != "runtime" || m.keys[5][1] != "heap_objects" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[5] <= 100 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[6][0] != "runtime" || m.keys[6][1] != "total_gc_pause_ns" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[6] <= 100000 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[7][0] != "runtime" || m.keys[7][1] != "total_gc_runs" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[7] < 1 {
		t.Fatalf("bad val: %v", m.vals)
	}

	if m.keys[8][0] != "runtime" || m.keys[8][1] != "gc_pause_ns" {
		t.Fatalf("bad key %v", m.keys)
	}
	if m.vals[8] <= 1000 {
		t.Fatalf("bad val: %v", m.vals)
	}
}

func TestInsert(t *testing.T) {
	k := []string{"hi", "bob"}
	exp := []string{"hi", "there", "bob"}
	out := insert(1, "there", k)
	if !reflect.DeepEqual(exp, out) {
		t.Fatalf("bad insert %v %v", exp, out)
	}
}
<|endoftext|>


<commit_before>package instances

import (
	"encoding/json"
	"net/http"
	"strconv"

	"github.com/cozy/cozy-stack/model/instance/lifecycle"
	"github.com/cozy/cozy-stack/model/job"
	"github.com/cozy/cozy-stack/model/sharing"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/labstack/echo/v4"
)

func fsckHandler(c echo.Context) (err error) {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	indexIntegrityCheck, _ := strconv.ParseBool(c.QueryParam("IndexIntegrity"))
	filesConsistencyCheck, _ := strconv.ParseBool(c.QueryParam("FilesConsistency"))
	failFast, _ := strconv.ParseBool(c.QueryParam("FailFast"))

	logCh := make(chan *vfs.FsckLog)
	go func() {
		fs := i.VFS()
		if indexIntegrityCheck {
			err = fs.CheckIndexIntegrity(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else if filesConsistencyCheck {
			err = fs.CheckFilesConsistency(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else {
			err = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		}
		close(logCh)
	}()

	w := c.Response().Writer
	w.WriteHeader(200)
	encoder := json.NewEncoder(w)
	for log := range logCh {
		// XXX do not serialize to JSON the children and the cozyMetadata, as
		// it can take more than 64ko and scanner will ignore such lines.
		if log.FileDoc != nil {
			log.FileDoc.DirsChildren = nil  // It can be filled on type mismatch
			log.FileDoc.FilesChildren = nil // Idem
			log.FileDoc.FilesChildrenSize = 0
			log.FileDoc.Metadata = nil
		}
		if log.DirDoc != nil {
			log.DirDoc.DirsChildren = nil
			log.DirDoc.FilesChildren = nil
			log.DirDoc.FilesChildrenSize = 0
			log.DirDoc.Metadata = nil
		}
		if log.VersionDoc != nil {
			log.VersionDoc.Metadata = nil
		}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	if err != nil {
		log := map[string]string{"error": err.Error()}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	return nil
}

func checkTriggers(c echo.Context) error {
	domain := c.Param("domain")
	inst, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	triggers, err := job.System().GetAllTriggers(inst)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"error": err.Error()},
			})
		}
		return wrapError(err)
	}

	results := []map[string]interface{}{}
	for i, left := range triggers {
		for j, right := range triggers {
			if i >= j {
				continue
			}
			if left.Type() != right.Type() {
				continue
			}
			lInfos := left.Infos()
			rInfos := right.Infos()
			if lInfos.WorkerType != rInfos.WorkerType {
				continue
			}
			if lInfos.Arguments != rInfos.Arguments {
				continue
			}
			if lInfos.Debounce != rInfos.Debounce {
				continue
			}
			results = append(results, map[string]interface{}{
				"type":      "duplicate",
				"_id":       lInfos.ID(),
				"other_id":  rInfos.ID(),
				"trigger":   left.Type(),
				"worker":    lInfos.WorkerType,
				"arguments": lInfos.Arguments,
				"debounce":  lInfos.Debounce,
			})
		}
	}

	return c.JSON(http.StatusOK, results)
}

func checkShared(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckShared(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"error": err.Error()},
			})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}

func checkSharings(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckSharings(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"error": err.Error()},
			})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}
<commit_msg>Fix check triggers for banks (#2823)<commit_after>package instances

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"

	"github.com/cozy/cozy-stack/model/instance/lifecycle"
	"github.com/cozy/cozy-stack/model/job"
	"github.com/cozy/cozy-stack/model/sharing"
	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/labstack/echo/v4"
)

func fsckHandler(c echo.Context) (err error) {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	indexIntegrityCheck, _ := strconv.ParseBool(c.QueryParam("IndexIntegrity"))
	filesConsistencyCheck, _ := strconv.ParseBool(c.QueryParam("FilesConsistency"))
	failFast, _ := strconv.ParseBool(c.QueryParam("FailFast"))

	logCh := make(chan *vfs.FsckLog)
	go func() {
		fs := i.VFS()
		if indexIntegrityCheck {
			err = fs.CheckIndexIntegrity(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else if filesConsistencyCheck {
			err = fs.CheckFilesConsistency(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		} else {
			err = fs.Fsck(func(log *vfs.FsckLog) { logCh <- log }, failFast)
		}
		close(logCh)
	}()

	w := c.Response().Writer
	w.WriteHeader(200)
	encoder := json.NewEncoder(w)
	for log := range logCh {
		// XXX do not serialize to JSON the children and the cozyMetadata, as
		// it can take more than 64ko and scanner will ignore such lines.
		if log.FileDoc != nil {
			log.FileDoc.DirsChildren = nil  // It can be filled on type mismatch
			log.FileDoc.FilesChildren = nil // Idem
			log.FileDoc.FilesChildrenSize = 0
			log.FileDoc.Metadata = nil
		}
		if log.DirDoc != nil {
			log.DirDoc.DirsChildren = nil
			log.DirDoc.FilesChildren = nil
			log.DirDoc.FilesChildrenSize = 0
			log.DirDoc.Metadata = nil
		}
		if log.VersionDoc != nil {
			log.VersionDoc.Metadata = nil
		}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	if err != nil {
		log := map[string]string{"error": err.Error()}
		if errenc := encoder.Encode(log); errenc != nil {
			i.Logger().WithField("nspace", "fsck").
				Warnf("Cannot encode to JSON: %s (%v)", errenc, log)
		}
		if f, ok := w.(http.Flusher); ok {
			f.Flush()
		}
	}
	return nil
}

func checkTriggers(c echo.Context) error {
	domain := c.Param("domain")
	inst, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	triggers, err := job.System().GetAllTriggers(inst)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"error": err.Error()},
			})
		}
		return wrapError(err)
	}

	results := []map[string]interface{}{}
	for i, left := range triggers {
		for j, right := range triggers {
			if i >= j {
				continue
			}
			if left.Type() == "@in" {
				// It doesn't make sense to compare @in triggers as they can
				// have scheduled at different times
				continue
			}
			if left.Type() != right.Type() {
				continue
			}
			lInfos := left.Infos()
			rInfos := right.Infos()
			if lInfos.WorkerType != rInfos.WorkerType {
				continue
			}
			if lInfos.Arguments != rInfos.Arguments {
				continue
			}
			if lInfos.Debounce != rInfos.Debounce {
				continue
			}
			lHasMessage := lInfos.Message != nil
			rHasMessage := rInfos.Message != nil
			if lHasMessage != rHasMessage {
				continue
			}
			if lHasMessage && rHasMessage {
				if !bytes.Equal(lInfos.Message, rInfos.Message) {
					continue
				}
			}
			results = append(results, map[string]interface{}{
				"type":      "duplicate",
				"_id":       lInfos.ID(),
				"other_id":  rInfos.ID(),
				"trigger":   left.Type(),
				"worker":    lInfos.WorkerType,
				"arguments": lInfos.Arguments,
				"debounce":  lInfos.Debounce,
				"message":   fmt.Sprintf("%s", lInfos.Message),
			})
		}
	}

	return c.JSON(http.StatusOK, results)
}

func checkShared(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckShared(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"error": err.Error()},
			})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}

func checkSharings(c echo.Context) error {
	domain := c.Param("domain")
	i, err := lifecycle.GetInstance(domain)
	if err != nil {
		return wrapError(err)
	}

	results, err := sharing.CheckSharings(i)
	if err != nil {
		if couchdb.IsNotFoundError(err) {
			return c.JSON(http.StatusOK, []map[string]interface{}{
				{"error": err.Error()},
			})
		}
		return wrapError(err)
	}
	return c.JSON(http.StatusOK, results)
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package routing

import (
	"encoding/json"
	"net/http"
	"time"

	clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
	"github.com/matrix-org/dendrite/clientapi/jsonerror"
	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
	"github.com/matrix-org/dendrite/keyserver/api"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/gomatrixserverlib"
	"github.com/matrix-org/util"
	"github.com/sirupsen/logrus"
	"golang.org/x/crypto/ed25519"
)

type queryKeysRequest struct {
	DeviceKeys map[string][]string `json:"device_keys"`
}

// QueryDeviceKeys returns device keys for users on this server.
// https://matrix.org/docs/spec/server_server/latest#post-matrix-federation-v1-user-keys-query
func QueryDeviceKeys(
	httpReq *http.Request, request *gomatrixserverlib.FederationRequest, keyAPI api.FederationKeyAPI, thisServer gomatrixserverlib.ServerName,
) util.JSONResponse {
	var qkr queryKeysRequest
	err := json.Unmarshal(request.Content(), &qkr)
	if err != nil {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
		}
	}
	// make sure we only query users on our domain
	for userID := range qkr.DeviceKeys {
		_, serverName, err := gomatrixserverlib.SplitID('@', userID)
		if err != nil {
			delete(qkr.DeviceKeys, userID)
			continue // ignore invalid users
		}
		if serverName != thisServer {
			delete(qkr.DeviceKeys, userID)
			continue
		}
	}

	var queryRes api.QueryKeysResponse
	if err := keyAPI.QueryKeys(httpReq.Context(), &api.QueryKeysRequest{
		UserToDevices: qkr.DeviceKeys,
	}, &queryRes); err != nil {
		return jsonerror.InternalAPIError(httpReq.Context(), err)
	}
	if queryRes.Error != nil {
		util.GetLogger(httpReq.Context()).WithError(queryRes.Error).Error("Failed to QueryKeys")
		return jsonerror.InternalServerError()
	}
	return util.JSONResponse{
		Code: 200,
		JSON: struct {
			DeviceKeys      interface{} `json:"device_keys"`
			MasterKeys      interface{} `json:"master_keys"`
			SelfSigningKeys interface{} `json:"self_signing_keys"`
		}{
			queryRes.DeviceKeys,
			queryRes.MasterKeys,
			queryRes.SelfSigningKeys,
		},
	}
}

type claimOTKsRequest struct {
	OneTimeKeys map[string]map[string]string `json:"one_time_keys"`
}

// ClaimOneTimeKeys claims OTKs for users on this server.
// https://matrix.org/docs/spec/server_server/latest#post-matrix-federation-v1-user-keys-claim
func ClaimOneTimeKeys(
	httpReq *http.Request, request *gomatrixserverlib.FederationRequest, keyAPI api.FederationKeyAPI, thisServer gomatrixserverlib.ServerName,
) util.JSONResponse {
	var cor claimOTKsRequest
	err := json.Unmarshal(request.Content(), &cor)
	if err != nil {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
		}
	}
	// make sure we only claim users on our domain
	for userID := range cor.OneTimeKeys {
		_, serverName, err := gomatrixserverlib.SplitID('@', userID)
		if err != nil {
			delete(cor.OneTimeKeys, userID)
			continue // ignore invalid users
		}
		if serverName != thisServer {
			delete(cor.OneTimeKeys, userID)
			continue
		}
	}

	var claimRes api.PerformClaimKeysResponse
	if err := keyAPI.PerformClaimKeys(httpReq.Context(), &api.PerformClaimKeysRequest{
		OneTimeKeys: cor.OneTimeKeys,
	}, &claimRes); err != nil {
		return jsonerror.InternalAPIError(httpReq.Context(), err)
	}
	if claimRes.Error != nil {
		util.GetLogger(httpReq.Context()).WithError(claimRes.Error).Error("Failed to PerformClaimKeys")
		return jsonerror.InternalServerError()
	}
	return util.JSONResponse{
		Code: 200,
		JSON: struct {
			OneTimeKeys interface{} `json:"one_time_keys"`
		}{claimRes.OneTimeKeys},
	}
}

// LocalKeys returns the local keys for the server.
// See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys
func LocalKeys(cfg *config.FederationAPI, serverName gomatrixserverlib.ServerName) util.JSONResponse {
	keys, err := localKeys(cfg, serverName)
	if err != nil {
		return util.MessageResponse(http.StatusNotFound, err.Error())
	}
	return util.JSONResponse{Code: http.StatusOK, JSON: keys}
}

func localKeys(cfg *config.FederationAPI, serverName gomatrixserverlib.ServerName) (*gomatrixserverlib.ServerKeys, error) {
	var keys gomatrixserverlib.ServerKeys
	var virtualHost *config.VirtualHost
loop:
	for _, v := range cfg.Matrix.VirtualHosts {
		if v.ServerName == serverName {
			virtualHost = v
			break loop
		}
		for _, httpHost := range v.MatchHTTPHosts {
			if httpHost == serverName {
				virtualHost = v
				break loop
			}
		}
	}

	identity, err := cfg.Matrix.SigningIdentityFor(serverName)
	if err != nil {
		identity, _ = cfg.Matrix.SigningIdentityFor(cfg.Matrix.ServerName)
	}

	if identity.ServerName == serverName {
		publicKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)
		keys.ServerName = cfg.Matrix.ServerName
		keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(cfg.Matrix.KeyValidityPeriod))
		keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
			cfg.Matrix.KeyID: {
				Key: gomatrixserverlib.Base64Bytes(publicKey),
			},
		}
		keys.OldVerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.OldVerifyKey{}
		for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys {
			keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{
				VerifyKey: gomatrixserverlib.VerifyKey{
					Key: oldVerifyKey.PublicKey,
				},
				ExpiredTS: oldVerifyKey.ExpiredAt,
			}
		}
	} else {
		publicKey := virtualHost.PrivateKey.Public().(ed25519.PublicKey)
		keys.ServerName = virtualHost.ServerName
		keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(virtualHost.KeyValidityPeriod))
		keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
			virtualHost.KeyID: {
				Key: gomatrixserverlib.Base64Bytes(publicKey),
			},
		}
		// TODO: Virtual hosts probably want to be able to specify old signing
		// keys too, just in case
	}

	toSign, err := json.Marshal(keys.ServerKeyFields)
	if err != nil {
		return nil, err
	}

	keys.Raw, err = gomatrixserverlib.SignJSON(
		string(identity.ServerName), identity.KeyID, identity.PrivateKey, toSign,
	)
	return &keys, err
}

func NotaryKeys(
	httpReq *http.Request, cfg *config.FederationAPI,
	fsAPI federationAPI.FederationInternalAPI,
	req *gomatrixserverlib.PublicKeyNotaryLookupRequest,
) util.JSONResponse {
	serverName := gomatrixserverlib.ServerName(httpReq.Host) // TODO: this is not ideal
	if !cfg.Matrix.IsLocalServerName(serverName) {
		return util.JSONResponse{
			Code: http.StatusNotFound,
			JSON: jsonerror.NotFound("Server name not known"),
		}
	}

	if req == nil {
		req = &gomatrixserverlib.PublicKeyNotaryLookupRequest{}
		if reqErr := clienthttputil.UnmarshalJSONRequest(httpReq, &req); reqErr != nil {
			return *reqErr
		}
	}

	var response struct {
		ServerKeys []json.RawMessage `json:"server_keys"`
	}
	response.ServerKeys = []json.RawMessage{}

	for serverName, kidToCriteria := range req.ServerKeys {
		var keyList []gomatrixserverlib.ServerKeys
		if serverName == cfg.Matrix.ServerName {
			if k, err := localKeys(cfg, serverName); err == nil {
				keyList = append(keyList, *k)
			} else {
				return util.ErrorResponse(err)
			}
		} else {
			var resp federationAPI.QueryServerKeysResponse
			err := fsAPI.QueryServerKeys(httpReq.Context(), &federationAPI.QueryServerKeysRequest{
				ServerName:      serverName,
				KeyIDToCriteria: kidToCriteria,
			}, &resp)
			if err != nil {
				return util.ErrorResponse(err)
			}
			keyList = append(keyList, resp.ServerKeys...)
		}
		if len(keyList) == 0 {
			continue
		}

		for _, keys := range keyList {
			j, err := json.Marshal(keys)
			if err != nil {
				logrus.WithError(err).Errorf("Failed to marshal %q response", serverName)
				return jsonerror.InternalServerError()
			}

			js, err := gomatrixserverlib.SignJSON(
				string(cfg.Matrix.ServerName), cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, j,
			)
			if err != nil {
				logrus.WithError(err).Errorf("Failed to sign %q response", serverName)
				return jsonerror.InternalServerError()
			}

			response.ServerKeys = append(response.ServerKeys, js)
		}
	}

	return util.JSONResponse{
		Code: http.StatusOK,
		JSON: response,
	}
}
<commit_msg>Try that again<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package routing

import (
	"encoding/json"
	"net/http"
	"time"

	clienthttputil "github.com/matrix-org/dendrite/clientapi/httputil"
	"github.com/matrix-org/dendrite/clientapi/jsonerror"
	federationAPI "github.com/matrix-org/dendrite/federationapi/api"
	"github.com/matrix-org/dendrite/keyserver/api"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/matrix-org/gomatrixserverlib"
	"github.com/matrix-org/util"
	"github.com/sirupsen/logrus"
	"golang.org/x/crypto/ed25519"
)

type queryKeysRequest struct {
	DeviceKeys map[string][]string `json:"device_keys"`
}

// QueryDeviceKeys returns device keys for users on this server.
// https://matrix.org/docs/spec/server_server/latest#post-matrix-federation-v1-user-keys-query
func QueryDeviceKeys(
	httpReq *http.Request, request *gomatrixserverlib.FederationRequest, keyAPI api.FederationKeyAPI, thisServer gomatrixserverlib.ServerName,
) util.JSONResponse {
	var qkr queryKeysRequest
	err := json.Unmarshal(request.Content(), &qkr)
	if err != nil {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
		}
	}
	// make sure we only query users on our domain
	for userID := range qkr.DeviceKeys {
		_, serverName, err := gomatrixserverlib.SplitID('@', userID)
		if err != nil {
			delete(qkr.DeviceKeys, userID)
			continue // ignore invalid users
		}
		if serverName != thisServer {
			delete(qkr.DeviceKeys, userID)
			continue
		}
	}

	var queryRes api.QueryKeysResponse
	if err := keyAPI.QueryKeys(httpReq.Context(), &api.QueryKeysRequest{
		UserToDevices: qkr.DeviceKeys,
	}, &queryRes); err != nil {
		return jsonerror.InternalAPIError(httpReq.Context(), err)
	}
	if queryRes.Error != nil {
		util.GetLogger(httpReq.Context()).WithError(queryRes.Error).Error("Failed to QueryKeys")
		return jsonerror.InternalServerError()
	}
	return util.JSONResponse{
		Code: 200,
		JSON: struct {
			DeviceKeys      interface{} `json:"device_keys"`
			MasterKeys      interface{} `json:"master_keys"`
			SelfSigningKeys interface{} `json:"self_signing_keys"`
		}{
			queryRes.DeviceKeys,
			queryRes.MasterKeys,
			queryRes.SelfSigningKeys,
		},
	}
}

type claimOTKsRequest struct {
	OneTimeKeys map[string]map[string]string `json:"one_time_keys"`
}

// ClaimOneTimeKeys claims OTKs for users on this server.
// https://matrix.org/docs/spec/server_server/latest#post-matrix-federation-v1-user-keys-claim
func ClaimOneTimeKeys(
	httpReq *http.Request, request *gomatrixserverlib.FederationRequest, keyAPI api.FederationKeyAPI, thisServer gomatrixserverlib.ServerName,
) util.JSONResponse {
	var cor claimOTKsRequest
	err := json.Unmarshal(request.Content(), &cor)
	if err != nil {
		return util.JSONResponse{
			Code: http.StatusBadRequest,
			JSON: jsonerror.BadJSON("The request body could not be decoded into valid JSON. " + err.Error()),
		}
	}
	// make sure we only claim users on our domain
	for userID := range cor.OneTimeKeys {
		_, serverName, err := gomatrixserverlib.SplitID('@', userID)
		if err != nil {
			delete(cor.OneTimeKeys, userID)
			continue // ignore invalid users
		}
		if serverName != thisServer {
			delete(cor.OneTimeKeys, userID)
			continue
		}
	}

	var claimRes api.PerformClaimKeysResponse
	if err := keyAPI.PerformClaimKeys(httpReq.Context(), &api.PerformClaimKeysRequest{
		OneTimeKeys: cor.OneTimeKeys,
	}, &claimRes); err != nil {
		return jsonerror.InternalAPIError(httpReq.Context(), err)
	}
	if claimRes.Error != nil {
		util.GetLogger(httpReq.Context()).WithError(claimRes.Error).Error("Failed to PerformClaimKeys")
		return jsonerror.InternalServerError()
	}
	return util.JSONResponse{
		Code: 200,
		JSON: struct {
			OneTimeKeys interface{} `json:"one_time_keys"`
		}{claimRes.OneTimeKeys},
	}
}

// LocalKeys returns the local keys for the server.
// See https://matrix.org/docs/spec/server_server/unstable.html#publishing-keys
func LocalKeys(cfg *config.FederationAPI, serverName gomatrixserverlib.ServerName) util.JSONResponse {
	keys, err := localKeys(cfg, serverName)
	if err != nil {
		return util.MessageResponse(http.StatusNotFound, err.Error())
	}
	return util.JSONResponse{Code: http.StatusOK, JSON: keys}
}

func localKeys(cfg *config.FederationAPI, serverName gomatrixserverlib.ServerName) (*gomatrixserverlib.ServerKeys, error) {
	var keys gomatrixserverlib.ServerKeys
	var virtualHost *config.VirtualHost
loop:
	for _, v := range cfg.Matrix.VirtualHosts {
		if v.ServerName == serverName {
			virtualHost = v
			break loop
		}
		for _, httpHost := range v.MatchHTTPHosts {
			if httpHost == serverName {
				virtualHost = v
				break loop
			}
		}
	}

	var identity *gomatrixserverlib.SigningIdentity
	var err error
	if virtualHost == nil {
		if identity, err = cfg.Matrix.SigningIdentityFor(cfg.Matrix.ServerName); err != nil {
			return nil, err
		}
		publicKey := cfg.Matrix.PrivateKey.Public().(ed25519.PublicKey)
		keys.ServerName = cfg.Matrix.ServerName
		keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(cfg.Matrix.KeyValidityPeriod))
		keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
			cfg.Matrix.KeyID: {
				Key: gomatrixserverlib.Base64Bytes(publicKey),
			},
		}
		keys.OldVerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.OldVerifyKey{}
		for _, oldVerifyKey := range cfg.Matrix.OldVerifyKeys {
			keys.OldVerifyKeys[oldVerifyKey.KeyID] = gomatrixserverlib.OldVerifyKey{
				VerifyKey: gomatrixserverlib.VerifyKey{
					Key: oldVerifyKey.PublicKey,
				},
				ExpiredTS: oldVerifyKey.ExpiredAt,
			}
		}
	} else {
		if identity, err = cfg.Matrix.SigningIdentityFor(virtualHost.ServerName); err != nil {
			return nil, err
		}
		publicKey := virtualHost.PrivateKey.Public().(ed25519.PublicKey)
		keys.ServerName = virtualHost.ServerName
		keys.ValidUntilTS = gomatrixserverlib.AsTimestamp(time.Now().Add(virtualHost.KeyValidityPeriod))
		keys.VerifyKeys = map[gomatrixserverlib.KeyID]gomatrixserverlib.VerifyKey{
			virtualHost.KeyID: {
				Key: gomatrixserverlib.Base64Bytes(publicKey),
			},
		}
		// TODO: Virtual hosts probably want to be able to specify old signing
		// keys too, just in case
	}

	toSign, err := json.Marshal(keys.ServerKeyFields)
	if err != nil {
		return nil, err
	}

	keys.Raw, err = gomatrixserverlib.SignJSON(
		string(identity.ServerName), identity.KeyID, identity.PrivateKey, toSign,
	)
	return &keys, err
}

func NotaryKeys(
	httpReq *http.Request, cfg *config.FederationAPI,
	fsAPI federationAPI.FederationInternalAPI,
	req *gomatrixserverlib.PublicKeyNotaryLookupRequest,
) util.JSONResponse {
	serverName := gomatrixserverlib.ServerName(httpReq.Host) // TODO: this is not ideal
	if !cfg.Matrix.IsLocalServerName(serverName) {
		return util.JSONResponse{
			Code: http.StatusNotFound,
			JSON: jsonerror.NotFound("Server name not known"),
		}
	}

	if req == nil {
		req = &gomatrixserverlib.PublicKeyNotaryLookupRequest{}
		if reqErr := clienthttputil.UnmarshalJSONRequest(httpReq, &req); reqErr != nil {
			return *reqErr
		}
	}

	var response struct {
		ServerKeys []json.RawMessage `json:"server_keys"`
	}
	response.ServerKeys = []json.RawMessage{}

	for serverName, kidToCriteria := range req.ServerKeys {
		var keyList []gomatrixserverlib.ServerKeys
		if serverName == cfg.Matrix.ServerName {
			if k, err := localKeys(cfg, serverName); err == nil {
				keyList = append(keyList, *k)
			} else {
				return util.ErrorResponse(err)
			}
		} else {
			var resp federationAPI.QueryServerKeysResponse
			err := fsAPI.QueryServerKeys(httpReq.Context(), &federationAPI.QueryServerKeysRequest{
				ServerName:      serverName,
				KeyIDToCriteria: kidToCriteria,
			}, &resp)
			if err != nil {
				return util.ErrorResponse(err)
			}
			keyList = append(keyList, resp.ServerKeys...)
		}
		if len(keyList) == 0 {
			continue
		}

		for _, keys := range keyList {
			j, err := json.Marshal(keys)
			if err != nil {
				logrus.WithError(err).Errorf("Failed to marshal %q response", serverName)
				return jsonerror.InternalServerError()
			}

			js, err := gomatrixserverlib.SignJSON(
				string(cfg.Matrix.ServerName), cfg.Matrix.KeyID, cfg.Matrix.PrivateKey, j,
			)
			if err != nil {
				logrus.WithError(err).Errorf("Failed to sign %q response", serverName)
				return jsonerror.InternalServerError()
			}

			response.ServerKeys = append(response.ServerKeys, js)
		}
	}

	return util.JSONResponse{
		Code: http.StatusOK,
		JSON: response,
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Perform a hermetic build of gcsfuse at a particular git tag specifying the
// version name, producing a release tarball. On Linux, also produce release
// packages.
//
// Usage:
//
//     package_gcsfuse dst_dir version [commit]
//
// This will cause the gcsfuse git repo to be cloned to a temporary location
// and a build performed, embedding the given version name. The build will be
// performed at the given commit (or branch or tag), which defaults to
// `v<version>`.
//
// A tarball will be written to dst_dir, along with .deb and .rpm files if
// building on Linux
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
)

func run(args []string) (err error) {
	osys := runtime.GOOS
	arch := runtime.GOARCH

	// Extract arguments.
	if len(args) < 2 || len(args) > 3 {
		err = fmt.Errorf("Usage: %s dst_dir version [commit]", os.Args[0])
		return
	}

	dstDir := args[0]
	version := args[1]

	commit := fmt.Sprintf("v%s", version)
	if len(args) >= 3 {
		commit = args[2]
	}

	log.Printf("Using settings:")
	log.Printf("  dstDir:  %s", dstDir)
	log.Printf("  commit:  %s", commit)
	log.Printf("  version: %s", version)

	// Ensure that all of the tools we need are present.
	err = checkForTools()
	if err != nil {
		return
	}

	// Assemble binaries, mount(8) helper scripts, etc.
	buildDir, err := build(commit, version, osys)
	if err != nil {
		err = fmt.Errorf("build: %v", err)
		return
	}

	defer os.RemoveAll(buildDir)

	// Write out a tarball.
	err = packageTarball(buildDir, version, osys, arch, dstDir)
	if err != nil {
		err = fmt.Errorf("packageTarball: %v", err)
		return
	}

	// Write out .deb and .rpm files if we're building for Linux.
	if osys == "linux" {
		err = packageDeb(buildDir, version, osys, arch, dstDir)
		if err != nil {
			err = fmt.Errorf("packageDeb: %v", err)
			return
		}

		err = packageRpm(buildDir, version, osys, arch, dstDir)
		if err != nil {
			err = fmt.Errorf("packageDeb: %v", err)
			return
		}
	}

	return
}

func main() {
	log.SetFlags(log.Lmicroseconds)
	flag.Parse()

	err := run(flag.Args())
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<commit_msg>Updated package comments.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Perform a hermetic build of gcsfuse at a particular git tag specifying the
// version name, producing .deb and .rpm files for Linux.
//
// Usage:
//
//     package_gcsfuse dst_dir version [commit]
//
// This will cause the gcsfuse git repo to be cloned to a temporary location
// and a build performed, embedding the given version name. The build will be
// performed at the given commit (or branch or tag), which defaults to
// `v<version>`.
//
// .deb and .rpm files will be written to dst_dir.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
)

func run(args []string) (err error) {
	osys := runtime.GOOS
	arch := runtime.GOARCH

	// Extract arguments.
	if len(args) < 2 || len(args) > 3 {
		err = fmt.Errorf("Usage: %s dst_dir version [commit]", os.Args[0])
		return
	}

	dstDir := args[0]
	version := args[1]

	commit := fmt.Sprintf("v%s", version)
	if len(args) >= 3 {
		commit = args[2]
	}

	log.Printf("Using settings:")
	log.Printf("  dstDir:  %s", dstDir)
	log.Printf("  commit:  %s", commit)
	log.Printf("  version: %s", version)

	// Ensure that all of the tools we need are present.
	err = checkForTools()
	if err != nil {
		return
	}

	// Assemble binaries, mount(8) helper scripts, etc.
	buildDir, err := build(commit, version, osys)
	if err != nil {
		err = fmt.Errorf("build: %v", err)
		return
	}

	defer os.RemoveAll(buildDir)

	// Write out a tarball.
	err = packageTarball(buildDir, version, osys, arch, dstDir)
	if err != nil {
		err = fmt.Errorf("packageTarball: %v", err)
		return
	}

	// Write out .deb and .rpm files if we're building for Linux.
	if osys == "linux" {
		err = packageDeb(buildDir, version, osys, arch, dstDir)
		if err != nil {
			err = fmt.Errorf("packageDeb: %v", err)
			return
		}

		err = packageRpm(buildDir, version, osys, arch, dstDir)
		if err != nil {
			err = fmt.Errorf("packageDeb: %v", err)
			return
		}
	}

	return
}

func main() {
	log.SetFlags(log.Lmicroseconds)
	flag.Parse()

	err := run(flag.Args())
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Package geoindex provides in memory geoindex implementation. It works by splitting the earth surface
// into grid with fixed size cells and storing data in each cell. The data can be points, count of points,
// and expiring points/counts. Has Range and K-Nearest queries.
package geoindex

import (
	"math"
	"sort"
)

// A geoindex that stores points.
type PointsIndex struct {
	index           *geoIndex
	currentPosition map[string]Point
}

// NewPointsIndex creates new PointsIndex that maintains the points in each cell.
func NewPointsIndex(resolution Meters) *PointsIndex {
	newSet := func() interface{} {
		return newSet()
	}

	return &PointsIndex{newGeoIndex(resolution, newSet), make(map[string]Point)}
}

// NewExpiringPointsIndex creates new PointIndex that expires the points in each cell after expiration minutes.
func NewExpiringPointsIndex(resolution Meters, expiration Minutes) *PointsIndex {
	currentPosition := make(map[string]Point)

	newExpiringSet := func() interface{} {
		set := newExpiringSet(expiration)

		set.OnExpire(func(id string, value interface{}) {
			point := value.(Point)
			delete(currentPosition, point.Id())
		})

		return set
	}

	return &PointsIndex{newGeoIndex(resolution, newExpiringSet), currentPosition}
}

// Get gets a point from the index given an id.
func (points *PointsIndex) Get(id string) Point {
	if point, ok := points.currentPosition[id]; ok {
		// first it gets the set of the currentPosition and then gets the point from the set
		// this is done so it triggers expiration on expiringSet, and returns nil if a point has expired
		if result, resultOk := points.index.GetEntryAt(point).(set).Get(id); resultOk {
			return result.(Point)
		}
	}
	return nil
}

// GetAll get all Points from the index as a map from id to point
func (points *PointsIndex) GetAll() map[string]Point {
	newpoints := make(map[string]Point, 0)
	for i, p := range points.currentPosition {
		newpoints[i] = p
	}
	return newpoints
}

// Add adds a point to the index. If a point with the same Id already exists it gets replaced.
func (points *PointsIndex) Add(point Point) {
	points.Remove(point.Id())
	newSet := points.index.AddEntryAt(point).(set)
	newSet.Add(point.Id(), point)
	points.currentPosition[point.Id()] = point
}

// Remove removes a point from the index.
func (points *PointsIndex) Remove(id string) {
	if prevPoint, ok := points.currentPosition[id]; ok {
		set := points.index.GetEntryAt(prevPoint).(set)
		set.Remove(prevPoint.Id())
		delete(points.currentPosition, prevPoint.Id())
	}
}

func between(value float64, min float64, max float64) bool {
	return value >= min && value <= max
}

func getPoints(entries []interface{}, accept func(point Point) bool) []Point {
	result := make([]Point, 0)
	result = getPointsAppend(result, entries, accept)
	return result
}

func getPointsAppend(s []Point, entries []interface{}, accept func(point Point) bool) []Point {
	for _, entry := range entries {
		pointsSetEntry := (entry).(set)

		for _, value := range pointsSetEntry.Values() {
			point := value.(Point)
			if accept(point) {
				s = append(s, point)
			}
		}
	}
	return s
}

// Range returns the points within the range defined by top left and bottom right.
func (points *PointsIndex) Range(topLeft Point, bottomRight Point) []Point {
	entries := points.index.Range(topLeft, bottomRight)
	accept := func(point Point) bool {
		return between(point.Lat(), bottomRight.Lat(), topLeft.Lat()) &&
			between(point.Lon(), topLeft.Lon(), bottomRight.Lon())
	}

	return getPoints(entries, accept)
}

type sortedPoints struct {
	points []Point
	point  Point
}

func (p *sortedPoints) Len() int {
	return len(p.points)
}

func (p *sortedPoints) Swap(i, j int) {
	p.points[i], p.points[j] = p.points[j], p.points[i]
}

func (p *sortedPoints) Less(i, j int) bool {
	return approximateSquareDistance(p.points[i], p.point) < approximateSquareDistance(p.points[j], p.point)
}

func min(a, b int) int {
	if a < b {
		return a
	} else {
		return b
	}
}

// KNearest returns the k nearest points near point within maxDistance that match the accept criteria.
func (points *PointsIndex) KNearest(point Point, k int, maxDistance Meters, accept func(p Point) bool) []Point {
	nearbyPoints := make([]Point, 0)
	pointEntry := points.index.GetEntryAt(point).(set)
	nearbyPoints = append(nearbyPoints, getPoints([]interface{}{pointEntry}, accept)...)

	totalCount := 0
	idx := cellOf(point, points.index.resolution)
	// Explicitely assign a greater max distance so that we definitely return enough points
	// and make sure it searches at least one square away.
	coarseMaxDistance := math.Max(float64(maxDistance)*2.0, float64(points.index.resolution)*2.0+0.01)

	for d := 1; float64(d)*float64(points.index.resolution) <= coarseMaxDistance; d++ {
		oldCount := len(nearbyPoints)

		nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x-d, idx.x+d, idx.y+d, idx.y+d), accept)
		nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x-d, idx.x+d, idx.y-d, idx.y-d), accept)
		nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x-d, idx.x-d, idx.y-d+1, idx.y+d-1), accept)
		nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x+d, idx.x+d, idx.y-d+1, idx.y+d-1), accept)

		totalCount += len(nearbyPoints) - oldCount

		if totalCount > k {
			break
		}
	}

	sortedPoints := &sortedPoints{nearbyPoints, point}
	sort.Sort(sortedPoints)

	k = min(k, len(sortedPoints.points))

	// filter points which longer than maxDistance away from point.
	for i, nearbyPoint := range sortedPoints.points {
		if Distance(point, nearbyPoint) > maxDistance || i == k {
			k = i
			break
		}
	}

	return sortedPoints.points[0:k]
}
<commit_msg>Added a PointsWithin method to the geoindex.<commit_after>// Package geoindex provides in memory geoindex implementation. It works by splitting the earth surface
// into grid with fixed size cells and storing data in each cell. The data can be points, count of points,
// and expiring points/counts. Has Range and K-Nearest queries.
package geoindex

import (
	"math"
	"sort"
)

// A geoindex that stores points.
type PointsIndex struct {
	index           *geoIndex
	currentPosition map[string]Point
}

// NewPointsIndex creates new PointsIndex that maintains the points in each cell.
func NewPointsIndex(resolution Meters) *PointsIndex {
	newSet := func() interface{} {
		return newSet()
	}

	return &PointsIndex{newGeoIndex(resolution, newSet), make(map[string]Point)}
}

// NewExpiringPointsIndex creates new PointIndex that expires the points in each cell after expiration minutes.
func NewExpiringPointsIndex(resolution Meters, expiration Minutes) *PointsIndex {
	currentPosition := make(map[string]Point)

	newExpiringSet := func() interface{} {
		set := newExpiringSet(expiration)

		set.OnExpire(func(id string, value interface{}) {
			point := value.(Point)
			delete(currentPosition, point.Id())
		})

		return set
	}

	return &PointsIndex{newGeoIndex(resolution, newExpiringSet), currentPosition}
}

// Get gets a point from the index given an id.
func (points *PointsIndex) Get(id string) Point {
	if point, ok := points.currentPosition[id]; ok {
		// first it gets the set of the currentPosition and then gets the point from the set
		// this is done so it triggers expiration on expiringSet, and returns nil if a point has expired
		if result, resultOk := points.index.GetEntryAt(point).(set).Get(id); resultOk {
			return result.(Point)
		}
	}
	return nil
}

// GetAll get all Points from the index as a map from id to point
func (points *PointsIndex) GetAll() map[string]Point {
	newpoints := make(map[string]Point, 0)
	for i, p := range points.currentPosition {
		newpoints[i] = p
	}
	return newpoints
}

// Add adds a point to the index. If a point with the same Id already exists it gets replaced.
func (points *PointsIndex) Add(point Point) {
	points.Remove(point.Id())
	newSet := points.index.AddEntryAt(point).(set)
	newSet.Add(point.Id(), point)
	points.currentPosition[point.Id()] = point
}

// Remove removes a point from the index.
func (points *PointsIndex) Remove(id string) {
	if prevPoint, ok := points.currentPosition[id]; ok {
		set := points.index.GetEntryAt(prevPoint).(set)
		set.Remove(prevPoint.Id())
		delete(points.currentPosition, prevPoint.Id())
	}
}

func between(value float64, min float64, max float64) bool {
	return value >= min && value <= max
}

func getPoints(entries []interface{}, accept func(point Point) bool) []Point {
	result := make([]Point, 0)
	result = getPointsAppend(result, entries, accept)
	return result
}

func getPointsAppend(s []Point, entries []interface{}, accept func(point Point) bool) []Point {
	for _, entry := range entries {
		pointsSetEntry := (entry).(set)

		for _, value := range pointsSetEntry.Values() {
			point := value.(Point)
			if accept(point) {
				s = append(s, point)
			}
		}
	}
	return s
}

// Range returns the points within the range defined by top left and bottom right.
func (points *PointsIndex) Range(topLeft Point, bottomRight Point) []Point {
	entries := points.index.Range(topLeft, bottomRight)
	accept := func(point Point) bool {
		return between(point.Lat(), bottomRight.Lat(), topLeft.Lat()) &&
			between(point.Lon(), topLeft.Lon(), bottomRight.Lon())
	}

	return getPoints(entries, accept)
}

type sortedPoints struct {
	points []Point
	point  Point
}

func (p *sortedPoints) Len() int {
	return len(p.points)
}

func (p *sortedPoints) Swap(i, j int) {
	p.points[i], p.points[j] = p.points[j], p.points[i]
}

func (p *sortedPoints) Less(i, j int) bool {
	return approximateSquareDistance(p.points[i], p.point) < approximateSquareDistance(p.points[j], p.point)
}

func min(a, b int) int {
	if a < b {
		return a
	} else {
		return b
	}
}

// KNearest returns the k nearest points near point within maxDistance that match the accept criteria.
func (points *PointsIndex) KNearest(point Point, k int, maxDistance Meters, accept func(p Point) bool) []Point {
	nearbyPoints := make([]Point, 0)
	pointEntry := points.index.GetEntryAt(point).(set)
	nearbyPoints = append(nearbyPoints, getPoints([]interface{}{pointEntry}, accept)...)

	totalCount := 0
	idx := cellOf(point, points.index.resolution)
	// Explicitely assign a greater max distance so that we definitely return enough points
	// and make sure it searches at least one square away.
	coarseMaxDistance := math.Max(float64(maxDistance)*2.0, float64(points.index.resolution)*2.0+0.01)

	for d := 1; float64(d)*float64(points.index.resolution) <= coarseMaxDistance; d++ {
		oldCount := len(nearbyPoints)

		nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x-d, idx.x+d, idx.y+d, idx.y+d), accept)
		nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x-d, idx.x+d, idx.y-d, idx.y-d), accept)
		nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x-d, idx.x-d, idx.y-d+1, idx.y+d-1), accept)
		nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x+d, idx.x+d, idx.y-d+1, idx.y+d-1), accept)

		totalCount += len(nearbyPoints) - oldCount

		if totalCount > k {
			break
		}
	}

	sortedPoints := &sortedPoints{nearbyPoints, point}
	sort.Sort(sortedPoints)

	k = min(k, len(sortedPoints.points))

	// filter points which longer than maxDistance away from point.
	for i, nearbyPoint := range sortedPoints.points {
		if Distance(point, nearbyPoint) > maxDistance || i == k {
			k = i
			break
		}
	}

	return sortedPoints.points[0:k]
}

// PointsWithin returns all points with distance of point that match the accept criteria.
func (points *PointsIndex) PointsWithin(point Point, distance Meters, accept func(p Point) bool) []Point {

	d := int(distance / points.index.resolution)

	if d == 0 {
		d = 1
	}

	idx := cellOf(point, points.index.resolution)

	nearbyPoints := make([]Point, 0)
	nearbyPoints = getPointsAppend(nearbyPoints, points.index.get(idx.x-d, idx.x+d, idx.y-d, idx.y+d), accept)

	// filter points which longer than maxDistance away from point.
	withinPoints := make([]Point, 0)
	for _, nearbyPoint := range nearbyPoints {
		if Distance(point, nearbyPoint) < distance {
			withinPoints = append(withinPoints, nearbyPoint)
		}
	}

	return withinPoints
}
<|endoftext|>


<commit_before>//  ---------------------------------------------------------------------------
//
//  BufferedLog.go
//
//  Copyright (c) 2014, Jared Chavez.
//  All rights reserved.
//
//  Use of this source code is governed by a BSD-style
//  license that can be found in the LICENSE file.
//
//  -----------

package flog

import (
	xlog "github.com/xaevman/log"
	"github.com/xaevman/shutdown"

	"bytes"
	"io"
	"log"
	"os"
	"sync"
	"sync/atomic"
	"time"
)

const blMaxBufferSize = 64 * 1024 // 64KB

// BufferedLog represents a buffered, file-backed logger and enforces a standardized
// logging format. New logging entries are sent to a memory buffer and
// periodically flushed to the backing file at configurable intervals
// by a seperate goroutine.
type BufferedLog struct {
	baseDir   string
	buffer    bytes.Buffer
	shutdown  *shutdown.Sync
	enabled   bool
	file      *os.File
	flushSec  int32
	flushChan chan interface{}
	hasClosed bool
	lock      sync.RWMutex
	logger    *log.Logger
	name      string
}

// BaseDir returns the base directory of the file backing this BufferedLog instance.
func (this *BufferedLog) BaseDir() string {
	this.lock.RLock()
	defer this.lock.RUnlock()

	return this.baseDir
}

// BufferCap returns the current capacity of the underlying memory buffer.
func (this *BufferedLog) BufferCap() int {
	this.lock.RLock()
	defer this.lock.RUnlock()

	return this.buffer.Cap()
}

// Close disables the BufferedLog instance, flushes any remaining entries to disk, and
// then closes the backing log file.
func (this *BufferedLog) Close() {
	if this.hasClosed {
		return
	}

	this.hasClosed = true

	this.print(xlog.NewLogMsg(this.name, "==== Close log ====", 2))

	// stop flush routine
	this.shutdown.Start()

	if this.shutdown.WaitForTimeout() {
		this.print(xlog.NewLogMsg(this.name, "Timeout waiting on shutdown", 2))
	}

	// flush logs
	this.flushLogs()

	// close file
	this.file.Close()
}

// SetEnabled temporarily enables/disables the log instance.
func (this *BufferedLog) SetEnabled(enabled bool) {
	this.lock.Lock()
	defer this.lock.Unlock()

	this.enabled = enabled
}

// FlushInterval returns the interval between log flushes in seconds.
func (this *BufferedLog) FlushIntervalSec() int32 {
	return atomic.LoadInt32(&this.flushSec)
}

// Name returns the friendly name of the log.
func (this *BufferedLog) Name() string {
	this.lock.RLock()
	defer this.lock.RUnlock()

	return this.name
}

// Print formats and buffers a new log entry as long as the BufferedLog instance
// is enabled.
func (this *BufferedLog) Print(msg *xlog.LogMsg) {
	this.lock.RLock()
	if !this.enabled {
		this.lock.RUnlock()
		return
	}
	this.lock.RUnlock()

	this.print(msg)
}

// SetFlushIntervalSec sets the interval at which the log buffer worker
// will attempt to flush new entries into the backing log file.
func (this *BufferedLog) SetFlushIntervalSec(interval int32) {
	atomic.StoreInt32(&this.flushSec, interval)
}

// asyncFlush is run in a separate goroutine and periodically flushes
// buffered entries to the backing file.
func (this *BufferedLog) asyncFlush() {
	defer this.shutdown.Complete()

	for {
		flushSec := atomic.LoadInt32(&this.flushSec)

		select {
		case <-this.shutdown.Signal:
			this.print(xlog.NewLogMsg(
				this.name,
				"Async log shutdown",
				3,
			))
			return
		case <-this.flushChan:
			this.flushLogs()
		case <-time.After(time.Duration(flushSec) * time.Second):
			this.flushLogs()
		}
	}
}

// flushLogs copies the contents of the log buffer into the open log file.
func (this *BufferedLog) flushLogs() {
	this.lock.Lock()
	defer this.lock.Unlock()

	// flush may have just happened, so check
	// the buffer len again before blocking on the
	// disk
	if this.buffer.Len() < blMaxBufferSize {
		return
	}

	_, err := io.Copy(this.file, &this.buffer)
	if err != nil {
		panic(err)
	}

	err = this.file.Sync()
	if err != nil {
		panic(err)
	}
}

func (this *BufferedLog) print(msg *xlog.LogMsg) {
	this.lock.Lock()

	log.Print(msg)
	this.logger.Print(msg)

	if this.buffer.Len() > blMaxBufferSize {
		this.lock.Unlock()

		select {
		case this.flushChan <- nil:
			return
		case <-time.After(1 * time.Second):
			return
		}
	} else {
		this.lock.Unlock()
	}
}
<commit_msg>add tracing markup to BufferedLog<commit_after>//  ---------------------------------------------------------------------------
//
//  BufferedLog.go
//
//  Copyright (c) 2014, Jared Chavez.
//  All rights reserved.
//
//  Use of this source code is governed by a BSD-style
//  license that can be found in the LICENSE file.
//
//  -----------

package flog

import (
	xlog "github.com/xaevman/log"
	"github.com/xaevman/shutdown"

	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"os"
	"runtime/trace"
	"sync"
	"sync/atomic"
	"time"
)

const blMaxBufferSize = 64 * 1024 // 64KB

// BufferedLog represents a buffered, file-backed logger and enforces a standardized
// logging format. New logging entries are sent to a memory buffer and
// periodically flushed to the backing file at configurable intervals
// by a seperate goroutine.
type BufferedLog struct {
	baseDir   string
	buffer    bytes.Buffer
	shutdown  *shutdown.Sync
	enabled   bool
	file      *os.File
	flushSec  int32
	flushChan chan interface{}
	hasClosed bool
	lock      sync.RWMutex
	logger    *log.Logger
	name      string
}

// BaseDir returns the base directory of the file backing this BufferedLog instance.
func (this *BufferedLog) BaseDir() string {
	ctx := context.Background()

	trace.WithRegion(ctx, "BaseDir().acquireReadLock", this.lock.RLock)
	defer this.lock.RUnlock()

	return this.baseDir
}

// BufferCap returns the current capacity of the underlying memory buffer.
func (this *BufferedLog) BufferCap() int {
	ctx := context.Background()

	trace.WithRegion(ctx, "BufferCap().acquireReadLock", this.lock.RLock)
	defer this.lock.RUnlock()

	return this.buffer.Cap()
}

// Close disables the BufferedLog instance, flushes any remaining entries to disk, and
// then closes the backing log file.
func (this *BufferedLog) Close() {
	if this.hasClosed {
		return
	}

	this.hasClosed = true

	ctx := context.Background()
	this.print(ctx, xlog.NewLogMsg(this.name, "==== Close log ====", 2))

	// stop flush routine
	this.shutdown.Start()

	if this.shutdown.WaitForTimeout() {
		this.print(ctx, xlog.NewLogMsg(this.name, "Timeout waiting on shutdown", 2))
	}

	// flush logs
	this.flushLogs()

	// close file
	this.file.Close()
}

// SetEnabled temporarily enables/disables the log instance.
func (this *BufferedLog) SetEnabled(enabled bool) {
	ctx := context.Background()

	trace.WithRegion(ctx, "SetEnabled().acquireLock", this.lock.Lock)
	defer this.lock.Unlock()

	this.enabled = enabled
}

// FlushInterval returns the interval between log flushes in seconds.
func (this *BufferedLog) FlushIntervalSec() int32 {
	return atomic.LoadInt32(&this.flushSec)
}

// Name returns the friendly name of the log.
func (this *BufferedLog) Name() string {
	ctx := context.Background()

	trace.WithRegion(ctx, "Name().acquireReadLock", this.lock.RLock)
	defer this.lock.RUnlock()

	return this.name
}

// Print formats and buffers a new log entry as long as the BufferedLog instance
// is enabled.
func (this *BufferedLog) Print(msg *xlog.LogMsg) {
	ctx := context.Background()

	trace.WithRegion(ctx, "Print().acquireReadLock", this.lock.RLock)
	if !this.enabled {
		this.lock.RUnlock()
		return
	}
	this.lock.RUnlock()

	this.print(ctx, msg)
}

// SetFlushIntervalSec sets the interval at which the log buffer worker
// will attempt to flush new entries into the backing log file.
func (this *BufferedLog) SetFlushIntervalSec(interval int32) {
	atomic.StoreInt32(&this.flushSec, interval)
}

// asyncFlush is run in a separate goroutine and periodically flushes
// buffered entries to the backing file.
func (this *BufferedLog) asyncFlush() {
	defer this.shutdown.Complete()

	ctx := context.Background()

	for {
		flushSec := atomic.LoadInt32(&this.flushSec)

		select {
		case <-this.shutdown.Signal:
			this.print(ctx, xlog.NewLogMsg(
				this.name,
				"Async log shutdown",
				3,
			))
			return
		case <-this.flushChan:
			trace.Log(ctx, "flushChan triggered", "")
			trace.WithRegion(ctx, "flushLogs()", this.flushLogs)
		case <-time.After(time.Duration(flushSec) * time.Second):
			trace.Log(ctx, "flushSecElapsed", fmt.Sprintf("%d", flushSec))
			this.flushLogs()
		}
	}
}

// flushLogs copies the contents of the log buffer into the open log file.
func (this *BufferedLog) flushLogs() {
	ctx := context.Background()

	trace.WithRegion(ctx, "flushLogs().acquireLock", this.lock.Lock)
	defer this.lock.Unlock()

	// flush may have just happened, so check
	// the buffer len again before blocking on the
	// disk
	if this.buffer.Len() < blMaxBufferSize {
		return
	}

	r := trace.StartRegion(ctx, "flushLogs().write")
	_, err := io.Copy(this.file, &this.buffer)
	r.End()
	if err != nil {
		panic(err)
	}

	r = trace.StartRegion(ctx, "flushLogs().sync")
	err = this.file.Sync()
	r.End()
	if err != nil {
		panic(err)
	}
}

func (this *BufferedLog) print(ctx context.Context, msg *xlog.LogMsg) {
	trace.WithRegion(ctx, "print().acquireLock", this.lock.Lock)

	r := trace.StartRegion(ctx, "print().delegates")
	log.Print(msg)
	this.logger.Print(msg)
	r.End()

	if this.buffer.Len() > blMaxBufferSize {
		this.lock.Unlock()

		r = trace.StartRegion(ctx, "print().requestFlush")
		select {
		case this.flushChan <- nil:
			return
		case <-time.After(1 * time.Second):
			return
		}
		r.End()
	} else {
		this.lock.Unlock()
	}
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/lang"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"net/http"
)

func (api *coreAPI) readLang(request *http.Request) []lang.Base {
	result := make([]lang.Base, 0)

	exists := make(map[string]bool, len(result))
	for _, obj := range api.contentType.Read(request) {
		langObj, ok := obj.(lang.Base)

		if !ok {
			panic(fmt.Sprintf("Trying to read lang objects while non-lang ones found: %s", obj.GetKind()))
		}

		key := runtime.KeyForStorable(langObj)
		if exists[key] {
			panic(fmt.Sprintf("Duplicate objects with key %s detected in the request", key))
		}
		exists[key] = true

		result = append(result, langObj)
	}

	return result
}
<commit_msg>Avoid shawowing in api package<commit_after>package api

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/lang"
	"github.com/Aptomi/aptomi/pkg/runtime"
	"net/http"
)

func (api *coreAPI) readLang(request *http.Request) []lang.Base {
	result := make([]lang.Base, 0)

	exists := make(map[string]bool, len(result))
	for _, obj := range api.contentType.Read(request) {
		langObj, ok := obj.(lang.Base)

		if !ok {
			panic(fmt.Sprintf("Trying to read lang objects while non-lang ones found: %s", obj.GetKind()))
		}

		objKey := runtime.KeyForStorable(langObj)
		if exists[objKey] {
			panic(fmt.Sprintf("Duplicate objects with key %s detected in the request", objKey))
		}
		exists[objKey] = true

		result = append(result, langObj)
	}

	return result
}
<|endoftext|>


<commit_before>package ip

import (
	"errors"
	"net"
	"syscall"

	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
)

func GetIfaceIP4Addr(iface *net.Interface) (net.IP, error) {
	addrs, err := iface.Addrs()
	if err != nil {
		return nil, err
	}

	for _, addr := range addrs {
		// Attempt to parse the address in CIDR notation
		// and assert it is IPv4
		ip, _, err := net.ParseCIDR(addr.String())
		if err == nil && ip.To4() != nil {
			return ip.To4(), nil
		}
	}

	return nil, errors.New("No IPv4 address found for given interface")
}

func GetIfaceIP4AddrMatch(iface *net.Interface, matchAddr net.IP) error {
	addrs, err := iface.Addrs()
	if err != nil {
		return err
	}

	for _, addr := range addrs {
		// Attempt to parse the address in CIDR notation
		// and assert it is IPv4
		ip, _, err := net.ParseCIDR(addr.String())
		if err == nil && ip.To4() != nil {
			if ip.To4().Equal(matchAddr) {
				return nil
			}
		}
	}

	return errors.New("No IPv4 address found for given interface")
}

func GetDefaultGatewayIface() (*net.Interface, error) {
	routes, err := netlink.RouteList(nil, syscall.AF_INET)
	if err != nil {
		return nil, err
	}

	for _, route := range routes {
		if route.Dst == nil || route.Dst.String() == "0.0.0.0/0" {
			if route.LinkIndex <= 0 {
				return nil, errors.New("Found default route but could not determine interface")
			}
			return net.InterfaceByIndex(route.LinkIndex)
		}
	}

	return nil, errors.New("Unable to find default route")
}

func GetInterfaceByIP(ip net.IP) (*net.Interface, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		err := GetIfaceIP4AddrMatch(&iface, ip)
		if err == nil {
			return &iface, nil
		}
	}

	return nil, errors.New("No interface with given IP found")
}
<commit_msg>Prefer global instead link-local iface addr<commit_after>package ip

import (
	"errors"
	"net"
	"syscall"

	"github.com/coreos/flannel/Godeps/_workspace/src/github.com/vishvananda/netlink"
)

func GetIfaceIP4Addr(iface *net.Interface) (net.IP, error) {
	addrs, err := iface.Addrs()
	if err != nil {
		return nil, err
	}

	// prefer non link-local addr
	var ll net.IP

	for _, addr := range addrs {
		// Attempt to parse the address in CIDR notation
		// and assert it is IPv4
		ip, _, err := net.ParseCIDR(addr.String())
		if err != nil || ip.To4() == nil {
			continue
		}

		if ip.IsGlobalUnicast() {
			return ip, nil
		}

		if ip.IsLinkLocalUnicast() {
			ll = ip
		}
	}

	if ll != nil {
		// didn't find global but found link-local. it'll do.
		return ll, nil
	}

	return nil, errors.New("No IPv4 address found for given interface")
}

func GetIfaceIP4AddrMatch(iface *net.Interface, matchAddr net.IP) error {
	addrs, err := iface.Addrs()
	if err != nil {
		return err
	}

	for _, addr := range addrs {
		// Attempt to parse the address in CIDR notation
		// and assert it is IPv4
		ip, _, err := net.ParseCIDR(addr.String())
		if err == nil && ip.To4() != nil {
			if ip.To4().Equal(matchAddr) {
				return nil
			}
		}
	}

	return errors.New("No IPv4 address found for given interface")
}

func GetDefaultGatewayIface() (*net.Interface, error) {
	routes, err := netlink.RouteList(nil, syscall.AF_INET)
	if err != nil {
		return nil, err
	}

	for _, route := range routes {
		if route.Dst == nil || route.Dst.String() == "0.0.0.0/0" {
			if route.LinkIndex <= 0 {
				return nil, errors.New("Found default route but could not determine interface")
			}
			return net.InterfaceByIndex(route.LinkIndex)
		}
	}

	return nil, errors.New("Unable to find default route")
}

func GetInterfaceByIP(ip net.IP) (*net.Interface, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return nil, err
	}

	for _, iface := range ifaces {
		err := GetIfaceIP4AddrMatch(&iface, ip)
		if err == nil {
			return &iface, nil
		}
	}

	return nil, errors.New("No interface with given IP found")
}
<|endoftext|>