<commit_before>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kv

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"strings"

	"github.com/juju/errors"
)

var (
	_ Index         = (*kvIndex)(nil)
	_ IndexIterator = (*IndexIter)(nil)
)

func encodeHandle(h int64) []byte {
	buf := &bytes.Buffer{}
	err := binary.Write(buf, binary.BigEndian, h)
	if err != nil {
		panic(err)
	}
	return buf.Bytes()
}

func decodeHandle(data []byte) (int64, error) {
	var h int64
	buf := bytes.NewBuffer(data)
	err := binary.Read(buf, binary.BigEndian, &h)
	return h, err
}

// IndexIter is for KV store index iterator.
type IndexIter struct {
	it     Iterator
	idx    *kvIndex
	prefix string
}

// Close does the clean up works when KV store index iterator is closed.
func (c *IndexIter) Close() {
	if c.it != nil {
		c.it.Close()
		c.it = nil
	}
}

// Next returns current key and moves iterator to the next step.
func (c *IndexIter) Next() (k []interface{}, h int64, err error) {
	if !c.it.Valid() {
		return nil, 0, io.EOF
	}
	if !strings.HasPrefix(c.it.Key(), c.prefix) {
		return nil, 0, io.EOF
	}
	// get indexedValues
	buf := []byte(c.it.Key())[len(c.prefix):]
	vv, err := DecodeValue(buf)
	if err != nil {
		return nil, 0, err
	}
	// if index is *not* unique, the handle is in keybuf
	if !c.idx.unique {
		h = vv[len(vv)-1].(int64)
		k = vv[0 : len(vv)-1]
	} else {
		// otherwise handle is value
		h, err = decodeHandle(c.it.Value())
		if err != nil {
			return nil, 0, err
		}
		k = vv
	}
	// update new iter to next
	newIt, err := c.it.Next(hasPrefix([]byte(c.prefix)))
	if err != nil {
		return nil, 0, err
	}
	c.it = newIt
	return
}

// kvIndex is the data structure for index data in the KV store.
type kvIndex struct {
	indexName string
	unique    bool
	dbName    string
	storage   Storage
	prefix    string
}

func genIndexPrefix(indexPrefix, indexName string) string {
	// Adding \xFF\xFF seperator to guarantee not iterating same prefix index
	// e.g, two indices c1 and c with index prefix p, if no \xFF\xFF,
	// the index format looks p_c and p_c1, if c has a index value which the first encoded byte is '1',
	// we will meet a error, because p_c1 is for index c1.
	// Why \xFF\xFF? We guarantee that all the encoded value with util/codec EncodeKey is less than \xFF\xFF.
	// TODO: check indexName whether has \xFF\xFF suffix?
	return fmt.Sprintf("%s_%s\xFF\xFF", indexPrefix, indexName)
}

// NewKVIndex builds a new kvIndex object.
func NewKVIndex(indexPrefix, indexName string, unique bool) Index {
	return &kvIndex{
		indexName: indexName,
		unique:    unique,
		prefix:    genIndexPrefix(indexPrefix, indexName),
	}
}

func (c *kvIndex) genIndexKey(indexedValues []interface{}, h int64) ([]byte, error) {
	var (
		encVal []byte
		err    error
	)
	// only support single value index
	if !c.unique {
		encVal, err = EncodeValue(append(indexedValues, h)...)
	} else {
		encVal, err = EncodeValue(indexedValues...)
	}
	if err != nil {
		return nil, err
	}
	buf := append([]byte(nil), []byte(c.prefix)...)
	buf = append(buf, encVal...)
	return buf, nil
}

// Create creates a new entry in the kvIndex data.
// If the index is unique and there already exists an entry with the same key, Create will return ErrKeyExists
func (c *kvIndex) Create(txn Transaction, indexedValues []interface{}, h int64) error {
	keyBuf, err := c.genIndexKey(indexedValues, h)
	if err != nil {
		return err
	}
	if !c.unique {
		// TODO: reconsider value
		err = txn.Set(keyBuf, []byte("timestamp?"))
		return errors.Trace(err)
	}

	// unique index
	_, err = txn.Get(keyBuf)
	if IsErrNotFound(err) {
		err = txn.Set(keyBuf, encodeHandle(h))
		return errors.Trace(err)
	}

	return errors.Trace(ErrKeyExists)
}

// Delete removes the entry for handle h and indexdValues from KV index.
func (c *kvIndex) Delete(txn Transaction, indexedValues []interface{}, h int64) error {
	keyBuf, err := c.genIndexKey(indexedValues, h)
	if err != nil {
		return err
	}
	err = txn.Delete(keyBuf)
	return err
}

func hasPrefix(prefix []byte) FnKeyCmp {
	return func(k []byte) bool {
		return bytes.HasPrefix(k, prefix)
	}
}

// Drop removes the KV index from store.
func (c *kvIndex) Drop(txn Transaction) error {
	prefix := []byte(c.prefix)
	it, err := txn.Seek(prefix, hasPrefix(prefix))
	if err != nil {
		return err
	}
	defer it.Close()
	// remove all indices
	for it.Valid() {
		if !strings.HasPrefix(it.Key(), c.prefix) {
			break
		}
		err := txn.Delete([]byte(it.Key()))
		if err != nil {
			return err
		}
		it, err = it.Next(hasPrefix(prefix))
		if err != nil {
			return err
		}
	}
	return nil
}

// Seek searches KV index for the entry with indexedValues.
func (c *kvIndex) Seek(txn Transaction, indexedValues []interface{}) (iter IndexIterator, hit bool, err error) {
	keyBuf, err := c.genIndexKey(indexedValues, 0)
	if err != nil {
		return nil, false, err
	}
	it, err := txn.Seek(keyBuf, hasPrefix([]byte(c.prefix)))
	if err != nil {
		return nil, false, err
	}
	// check if hit
	hit = false
	if it.Valid() && it.Key() == string(keyBuf) {
		hit = true
	}
	return &IndexIter{it: it, idx: c, prefix: c.prefix}, hit, nil
}

// SeekFirst returns an iterator which points to the first entry of the KV index.
func (c *kvIndex) SeekFirst(txn Transaction) (iter IndexIterator, err error) {
	prefix := []byte(c.prefix)
	it, err := txn.Seek([]byte(c.prefix), hasPrefix(prefix))
	if err != nil {
		return nil, err
	}
	return &IndexIter{it: it, idx: c, prefix: c.prefix}, nil
}
<commit_msg>kv: use EncodeBytes instead<commit_after>// Copyright 2015 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package kv

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"strings"

	"github.com/juju/errors"
	"github.com/pingcap/tidb/util/codec"
)

var (
	_ Index         = (*kvIndex)(nil)
	_ IndexIterator = (*IndexIter)(nil)
)

func encodeHandle(h int64) []byte {
	buf := &bytes.Buffer{}
	err := binary.Write(buf, binary.BigEndian, h)
	if err != nil {
		panic(err)
	}
	return buf.Bytes()
}

func decodeHandle(data []byte) (int64, error) {
	var h int64
	buf := bytes.NewBuffer(data)
	err := binary.Read(buf, binary.BigEndian, &h)
	return h, err
}

// IndexIter is for KV store index iterator.
type IndexIter struct {
	it     Iterator
	idx    *kvIndex
	prefix string
}

// Close does the clean up works when KV store index iterator is closed.
func (c *IndexIter) Close() {
	if c.it != nil {
		c.it.Close()
		c.it = nil
	}
}

// Next returns current key and moves iterator to the next step.
func (c *IndexIter) Next() (k []interface{}, h int64, err error) {
	if !c.it.Valid() {
		return nil, 0, io.EOF
	}
	if !strings.HasPrefix(c.it.Key(), c.prefix) {
		return nil, 0, io.EOF
	}
	// get indexedValues
	buf := []byte(c.it.Key())[len(c.prefix):]
	vv, err := DecodeValue(buf)
	if err != nil {
		return nil, 0, err
	}
	// if index is *not* unique, the handle is in keybuf
	if !c.idx.unique {
		h = vv[len(vv)-1].(int64)
		k = vv[0 : len(vv)-1]
	} else {
		// otherwise handle is value
		h, err = decodeHandle(c.it.Value())
		if err != nil {
			return nil, 0, err
		}
		k = vv
	}
	// update new iter to next
	newIt, err := c.it.Next(hasPrefix([]byte(c.prefix)))
	if err != nil {
		return nil, 0, err
	}
	c.it = newIt
	return
}

// kvIndex is the data structure for index data in the KV store.
type kvIndex struct {
	indexName string
	unique    bool
	dbName    string
	storage   Storage
	prefix    string
}

func genIndexPrefix(indexPrefix, indexName string) string {
	// Use EncodeBytes to guarantee generating different index prefix.
	// e.g, two indices c1 and c with index prefix p, if no EncodeBytes,
	// the index format looks p_c and p_c1, if c has a index value which the first encoded byte is '1',
	// we will meet a error, because p_c1 is for index c1.
	// If EncodeBytes, c1 -> c1\x00\x01 and c -> c\x00\x01, the prefixs are different.
	key := fmt.Sprintf("%s_%s", indexPrefix, indexName)
	return string(codec.EncodeBytes(nil, []byte(key)))
}

// NewKVIndex builds a new kvIndex object.
func NewKVIndex(indexPrefix, indexName string, unique bool) Index {
	return &kvIndex{
		indexName: indexName,
		unique:    unique,
		prefix:    genIndexPrefix(indexPrefix, indexName),
	}
}

func (c *kvIndex) genIndexKey(indexedValues []interface{}, h int64) ([]byte, error) {
	var (
		encVal []byte
		err    error
	)
	// only support single value index
	if !c.unique {
		encVal, err = EncodeValue(append(indexedValues, h)...)
	} else {
		encVal, err = EncodeValue(indexedValues...)
	}
	if err != nil {
		return nil, err
	}
	buf := append([]byte(nil), []byte(c.prefix)...)
	buf = append(buf, encVal...)
	return buf, nil
}

// Create creates a new entry in the kvIndex data.
// If the index is unique and there already exists an entry with the same key, Create will return ErrKeyExists
func (c *kvIndex) Create(txn Transaction, indexedValues []interface{}, h int64) error {
	keyBuf, err := c.genIndexKey(indexedValues, h)
	if err != nil {
		return err
	}
	if !c.unique {
		// TODO: reconsider value
		err = txn.Set(keyBuf, []byte("timestamp?"))
		return errors.Trace(err)
	}

	// unique index
	_, err = txn.Get(keyBuf)
	if IsErrNotFound(err) {
		err = txn.Set(keyBuf, encodeHandle(h))
		return errors.Trace(err)
	}

	return errors.Trace(ErrKeyExists)
}

// Delete removes the entry for handle h and indexdValues from KV index.
func (c *kvIndex) Delete(txn Transaction, indexedValues []interface{}, h int64) error {
	keyBuf, err := c.genIndexKey(indexedValues, h)
	if err != nil {
		return err
	}
	err = txn.Delete(keyBuf)
	return err
}

func hasPrefix(prefix []byte) FnKeyCmp {
	return func(k []byte) bool {
		return bytes.HasPrefix(k, prefix)
	}
}

// Drop removes the KV index from store.
func (c *kvIndex) Drop(txn Transaction) error {
	prefix := []byte(c.prefix)
	it, err := txn.Seek(prefix, hasPrefix(prefix))
	if err != nil {
		return err
	}
	defer it.Close()
	// remove all indices
	for it.Valid() {
		if !strings.HasPrefix(it.Key(), c.prefix) {
			break
		}
		err := txn.Delete([]byte(it.Key()))
		if err != nil {
			return err
		}
		it, err = it.Next(hasPrefix(prefix))
		if err != nil {
			return err
		}
	}
	return nil
}

// Seek searches KV index for the entry with indexedValues.
func (c *kvIndex) Seek(txn Transaction, indexedValues []interface{}) (iter IndexIterator, hit bool, err error) {
	keyBuf, err := c.genIndexKey(indexedValues, 0)
	if err != nil {
		return nil, false, err
	}
	it, err := txn.Seek(keyBuf, hasPrefix([]byte(c.prefix)))
	if err != nil {
		return nil, false, err
	}
	// check if hit
	hit = false
	if it.Valid() && it.Key() == string(keyBuf) {
		hit = true
	}
	return &IndexIter{it: it, idx: c, prefix: c.prefix}, hit, nil
}

// SeekFirst returns an iterator which points to the first entry of the KV index.
func (c *kvIndex) SeekFirst(txn Transaction) (iter IndexIterator, err error) {
	prefix := []byte(c.prefix)
	it, err := txn.Seek([]byte(c.prefix), hasPrefix(prefix))
	if err != nil {
		return nil, err
	}
	return &IndexIter{it: it, idx: c, prefix: c.prefix}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/Zac-Garby/pluto/bytecode"
	"github.com/Zac-Garby/pluto/compiler"
	"github.com/Zac-Garby/pluto/object"
	"github.com/Zac-Garby/pluto/parser"
	"github.com/Zac-Garby/pluto/vm"

	"github.com/fatih/color"
)

func main() {
	store := vm.NewStore()

	for {
		reader := bufio.NewReader(os.Stdin)
		fmt.Print(">> ")
		text, _ := reader.ReadString('\n')
		text = strings.TrimRight(text, "\n")

		if obj, err := execute(text, store); err != nil {
			color.Red("  %s", err)
		} else if obj != nil {
			color.Cyan("  %s", obj)
		}
	}
}

func execute(text string, store *vm.Store) (object.Object, error) {
	var (
		cmp   = compiler.New()
		parse = parser.New(text)
		prog  = parse.Parse()
	)

	if len(parse.Errors) > 0 {
		parse.PrintErrors()
		os.Exit(1)
	}

	err := cmp.CompileProgram(prog)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	code, err := bytecode.Read(cmp.Bytes)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	store.Names = cmp.Names
	store.Functions.Functions = cmp.Functions
	store.Patterns = cmp.Patterns

	machine := vm.New()
	machine.Run(code, store, cmp.Constants)

	if machine.Error != nil {
		return nil, machine.Error
	}

	return machine.ExtractValue(), nil
}
<commit_msg>Keep functions defined in the REPL<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"strings"

	"github.com/Zac-Garby/pluto/bytecode"
	"github.com/Zac-Garby/pluto/compiler"
	"github.com/Zac-Garby/pluto/object"
	"github.com/Zac-Garby/pluto/parser"
	"github.com/Zac-Garby/pluto/vm"

	"github.com/fatih/color"
)

func main() {
	store := vm.NewStore()

	for {
		reader := bufio.NewReader(os.Stdin)
		fmt.Print(">> ")
		text, _ := reader.ReadString('\n')
		text = strings.TrimRight(text, "\n")

		if obj, err := execute(text, store); err != nil {
			color.Red("  %s", err)
		} else if obj != nil {
			color.Cyan("  %s", obj)
		}
	}
}

func execute(text string, store *vm.Store) (object.Object, error) {
	var (
		cmp   = compiler.New()
		parse = parser.New(text)
		prog  = parse.Parse()
	)

	if len(parse.Errors) > 0 {
		parse.PrintErrors()
		os.Exit(1)
	}

	err := cmp.CompileProgram(prog)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	code, err := bytecode.Read(cmp.Bytes)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	store.Names = cmp.Names
	store.Functions.Functions = append(cmp.Functions, store.Functions.Functions...)
	store.Patterns = cmp.Patterns

	machine := vm.New()
	machine.Run(code, store, cmp.Constants)

	if machine.Error != nil {
		return nil, machine.Error
	}

	return machine.ExtractValue(), nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"plaid/lang"
	"plaid/lib"
)

func main() {
	if len(os.Args[1:]) >= 1 {
		if errs := run(os.Args[1]); len(errs) > 0 {
			for _, err := range errs {
				fmt.Fprintln(os.Stderr, err.Error())
			}
			os.Exit(1)
		}
	}
}

func read(filename string) (src string, errs []error) {
	if contents, err := ioutil.ReadFile(filename); err == nil {
		return string(contents), nil
	} else {
		return "", []error{err}
	}
}

func run(filename string) (errs []error) {
	var src string
	var ast *lang.RootNode
	var mod *lang.VirtualModule

	fmt.Println("\n=== SOURCE CODE")
	if src, errs = read(filename); len(errs) > 0 {
		return errs
	} else {
		fmt.Println(src)
	}

	fmt.Println("\n=== SYNTAX TREE")
	if ast, errs = lang.Parse(filename, src); len(errs) > 0 {
		return errs
	} else {
		fmt.Println(ast)
	}

	stdlib := make(map[string]*lang.Library)
	stdlib["io"] = lib.IO()

	if mod, errs = lang.Link(filename, ast, stdlib); len(errs) > 0 {
		return errs
	}

	fmt.Println("\n=== MODULE")
	if _, errs = lang.Check(mod); len(errs) > 0 {
		return errs
	} else {
		fmt.Println(mod)
	}

	return nil
}
<commit_msg>include compilation and execution<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"plaid/lang"
	"plaid/lib"
)

func main() {
	if len(os.Args[1:]) >= 1 {
		if errs := run(os.Args[1]); len(errs) > 0 {
			for _, err := range errs {
				fmt.Fprintln(os.Stderr, err.Error())
			}
			os.Exit(1)
		}
	}
}

func read(filename string) (src string, errs []error) {
	if contents, err := ioutil.ReadFile(filename); err == nil {
		return string(contents), nil
	} else {
		return "", []error{err}
	}
}

func run(filename string) (errs []error) {
	var src string
	var ast *lang.RootNode
	var mod *lang.VirtualModule
	var btc lang.Bytecode

	fmt.Println("\n=== SOURCE CODE")
	if src, errs = read(filename); len(errs) > 0 {
		return errs
	} else {
		fmt.Println(src)
	}

	fmt.Println("\n=== SYNTAX TREE")
	if ast, errs = lang.Parse(filename, src); len(errs) > 0 {
		return errs
	} else {
		fmt.Println(ast)
	}

	stdlib := make(map[string]*lang.Library)
	stdlib["io"] = lib.IO()

	if mod, errs = lang.Link(filename, ast, stdlib); len(errs) > 0 {
		return errs
	}

	fmt.Println("\n=== MODULE")
	if _, errs = lang.Check(mod); len(errs) > 0 {
		return errs
	} else {
		fmt.Println(mod)
	}

	fmt.Println("\n=== BYTECODE")
	btc = lang.Compile(mod)
	fmt.Println(btc.String())

	fmt.Println("\n=== VM")
	lang.Run(btc)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/dustin/go-humanize"
	"github.com/go-errors/errors"
	"github.com/itchio/butler/comm"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/pwr"
)

func verify(signaturePath string, dir string, woundsPath string, healPath string) {
	must(doVerify(signaturePath, dir, woundsPath, healPath))
}

func doVerify(signaturePath string, dir string, woundsPath string, healPath string) error {
	if woundsPath == "" {
		if healPath == "" {
			comm.Opf("Verifying %s", dir)
		} else {
			comm.Opf("Verifying %s, healing as we go", dir)
		}
	} else {
		if healPath == "" {
			comm.Opf("Verifying %s, writing wounds to %s", dir, woundsPath)
		} else {
			comm.Dief("Options --wounds and --heal cannot be used at the same time")
		}
	}
	startTime := time.Now()

	signatureReader, err := eos.Open(signaturePath)
	if err != nil {
		return errors.Wrap(err, 1)
	}
	defer signatureReader.Close()

	signature, err := pwr.ReadSignature(signatureReader)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	vc := &pwr.ValidatorContext{
		Consumer:   comm.NewStateConsumer(),
		WoundsPath: woundsPath,
		HealPath:   healPath,
	}

	comm.StartProgress()

	err = vc.Validate(dir, signature)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	comm.EndProgress()

	prettySize := humanize.IBytes(uint64(signature.Container.Size))
	perSecond := humanize.IBytes(uint64(float64(signature.Container.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s/s\n", prettySize, signature.Container.Stats(), perSecond)

	if vc.WoundsConsumer.HasWounds() {
		comm.Dief("%s corrupted data found", humanize.IBytes(uint64(vc.WoundsConsumer.TotalCorrupted())))
	}

	return nil
}
<commit_msg>Output bps for verify<commit_after>package main

import (
	"time"

	"github.com/dustin/go-humanize"
	"github.com/go-errors/errors"
	"github.com/itchio/butler/comm"
	"github.com/itchio/wharf/eos"
	"github.com/itchio/wharf/pwr"
)

func verify(signaturePath string, dir string, woundsPath string, healPath string) {
	must(doVerify(signaturePath, dir, woundsPath, healPath))
}

func doVerify(signaturePath string, dir string, woundsPath string, healPath string) error {
	if woundsPath == "" {
		if healPath == "" {
			comm.Opf("Verifying %s", dir)
		} else {
			comm.Opf("Verifying %s, healing as we go", dir)
		}
	} else {
		if healPath == "" {
			comm.Opf("Verifying %s, writing wounds to %s", dir, woundsPath)
		} else {
			comm.Dief("Options --wounds and --heal cannot be used at the same time")
		}
	}
	startTime := time.Now()

	signatureReader, err := eos.Open(signaturePath)
	if err != nil {
		return errors.Wrap(err, 1)
	}
	defer signatureReader.Close()

	signature, err := pwr.ReadSignature(signatureReader)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	vc := &pwr.ValidatorContext{
		Consumer:   comm.NewStateConsumer(),
		WoundsPath: woundsPath,
		HealPath:   healPath,
	}

	comm.StartProgressWithTotalBytes(signature.Container.Size)

	err = vc.Validate(dir, signature)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	comm.EndProgress()

	prettySize := humanize.IBytes(uint64(signature.Container.Size))
	perSecond := humanize.IBytes(uint64(float64(signature.Container.Size) / time.Since(startTime).Seconds()))
	comm.Statf("%s (%s) @ %s/s\n", prettySize, signature.Container.Stats(), perSecond)

	if vc.WoundsConsumer.HasWounds() {
		comm.Dief("%s corrupted data found", humanize.IBytes(uint64(vc.WoundsConsumer.TotalCorrupted())))
	}

	return nil
}
<|endoftext|>


<commit_before>package rabbithole

import (
	"encoding/json"
	"net/http"
	"net/url"
)

//
// GET /api/vhosts
//

// Example response:

// [
//   {
//     "message_stats": {
//       "publish": 78,
//       "publish_details": {
//         "rate": 0
//       }
//     },
//     "messages": 0,
//     "messages_details": {
//       "rate": 0
//     },
//     "messages_ready": 0,
//     "messages_ready_details": {
//       "rate": 0
//     },
//     "messages_unacknowledged": 0,
//     "messages_unacknowledged_details": {
//       "rate": 0
//     },
//     "recv_oct": 16653,
//     "recv_oct_details": {
//       "rate": 0
//     },
//     "send_oct": 40495,
//     "send_oct_details": {
//       "rate": 0
//     },
//     "name": "\/",
//     "tracing": false
//   },
//   {
//     "name": "29dd51888b834698a8b5bc3e7f8623aa1c9671f5",
//     "tracing": false
//   }
// ]

type VhostInfo struct {
	// Virtual host name
	Name string `json:"name"`
	// True if tracing is enabled for this virtual host
	Tracing bool `json:"tracing"`

	// Total number of messages in queues of this virtual host
	Messages        int         `json:"messages"`
	MessagesDetails RateDetails `json:"messages_details"`

	// Total number of messages ready to be delivered in queues of this virtual host
	MessagesReady        int         `json:"messages_ready"`
	MessagesReadyDetails RateDetails `json:"messages_ready_details"`

	// Total number of messages pending acknowledgement from consumers in this virtual host
	MessagesUnacknowledged        int         `json:"messages_unacknowledged"`
	MessagesUnacknowledgedDetails RateDetails `json:"messages_unacknowledged_details"`

	// Octets received
	RecvOct uint64 `json:"recv_oct"`
	// Octets sent
	SendOct        uint64      `json:"send_oct"`
	RecvCount      uint64      `json:"recv_cnt"`
	SendCount      uint64      `json:"send_cnt"`
	SendPending    uint64      `json:"send_pend"`
	RecvOctDetails RateDetails `json:"recv_oct_details"`
	SendOctDetails RateDetails `json:"send_oct_details"`
}

// Returns a list of virtual hosts.
func (c *Client) ListVhosts() (rec []VhostInfo, err error) {
	req, err := newGETRequest(c, "vhosts/")
	if err != nil {
		return []VhostInfo{}, err
	}

	if err = executeAndParseRequest(c, req, &rec); err != nil {
		return []VhostInfo{}, err
	}

	return rec, nil
}

//
// GET /api/vhosts/{name}
//

// Returns information about a specific virtual host.
func (c *Client) GetVhost(vhostname string) (rec *VhostInfo, err error) {
	req, err := newGETRequest(c, "vhosts/"+url.QueryEscape(vhostname))
	if err != nil {
		return nil, err
	}

	if err = executeAndParseRequest(c, req, &rec); err != nil {
		return nil, err
	}

	return rec, nil
}

//
// PUT /api/vhosts/{name}
//

// Settings used to create or modify virtual hosts.
type VhostSettings struct {
	// True if tracing should be enabled.
	Tracing bool `json:"tracing"`
}

// Creates or updates a virtual host.
func (c *Client) PutVhost(vhostname string, settings VhostSettings) (res *http.Response, err error) {
	body, err := json.Marshal(settings)
	if err != nil {
		return nil, err
	}

	req, err := newRequestWithBody(c, "PUT", "vhosts/"+url.QueryEscape(vhostname), body)
	if err != nil {
		return nil, err
	}

	res, err = executeRequest(c, req)
	if err != nil {
		return nil, err
	}

	return res, nil
}

//
// DELETE /api/vhosts/{name}
//

// Deletes a virtual host.
func (c *Client) DeleteVhost(vhostname string) (res *http.Response, err error) {
	req, err := newRequestWithBody(c, "DELETE", "vhosts/"+url.QueryEscape(vhostname), nil)
	if err != nil {
		return nil, err
	}

	res, err = executeRequest(c, req)
	if err != nil {
		return nil, err
	}

	return res, nil
}
<commit_msg>Update vhosts.go<commit_after>package rabbithole

import (
	"encoding/json"
	"net/http"
	"net/url"
)

//
// GET /api/vhosts
//

// Example response:

// [
//   {
//     "message_stats": {
//       "publish": 78,
//       "publish_details": {
//         "rate": 0
//       }
//     },
//     "messages": 0,
//     "messages_details": {
//       "rate": 0
//     },
//     "messages_ready": 0,
//     "messages_ready_details": {
//       "rate": 0
//     },
//     "messages_unacknowledged": 0,
//     "messages_unacknowledged_details": {
//       "rate": 0
//     },
//     "recv_oct": 16653,
//     "recv_oct_details": {
//       "rate": 0
//     },
//     "send_oct": 40495,
//     "send_oct_details": {
//       "rate": 0
//     },
//     "name": "\/",
//     "tracing": false
//   },
//   {
//     "name": "29dd51888b834698a8b5bc3e7f8623aa1c9671f5",
//     "tracing": false
//   }
// ]

type VhostInfo struct {
	// Virtual host name
	Name string `json:"name"`
	// True if tracing is enabled for this virtual host
	Tracing bool `json:"tracing"`

	// Total number of messages in queues of this virtual host
	Messages        int         `json:"messages"`
	MessagesDetails RateDetails `json:"messages_details"`

	// Total number of messages ready to be delivered in queues of this virtual host
	MessagesReady        int         `json:"messages_ready"`
	MessagesReadyDetails RateDetails `json:"messages_ready_details"`

	// Total number of messages pending acknowledgement from consumers in this virtual host
	MessagesUnacknowledged        int         `json:"messages_unacknowledged"`
	MessagesUnacknowledgedDetails RateDetails `json:"messages_unacknowledged_details"`

	// Octets received
	RecvOct uint64 `json:"recv_oct"`
	// Octets sent
	SendOct        uint64      `json:"send_oct"`
	RecvCount      uint64      `json:"recv_cnt"`
	SendCount      uint64      `json:"send_cnt"`
	SendPending    uint64      `json:"send_pend"`
	RecvOctDetails RateDetails `json:"recv_oct_details"`
	SendOctDetails RateDetails `json:"send_oct_details"`
}

// Returns a list of virtual hosts.
func (c *Client) ListVhosts() (rec []VhostInfo, err error) {
	req, err := newGETRequest(c, "vhosts")
	if err != nil {
		return []VhostInfo{}, err
	}

	if err = executeAndParseRequest(c, req, &rec); err != nil {
		return []VhostInfo{}, err
	}

	return rec, nil
}

//
// GET /api/vhosts/{name}
//

// Returns information about a specific virtual host.
func (c *Client) GetVhost(vhostname string) (rec *VhostInfo, err error) {
	req, err := newGETRequest(c, "vhosts/"+url.QueryEscape(vhostname))
	if err != nil {
		return nil, err
	}

	if err = executeAndParseRequest(c, req, &rec); err != nil {
		return nil, err
	}

	return rec, nil
}

//
// PUT /api/vhosts/{name}
//

// Settings used to create or modify virtual hosts.
type VhostSettings struct {
	// True if tracing should be enabled.
	Tracing bool `json:"tracing"`
}

// Creates or updates a virtual host.
func (c *Client) PutVhost(vhostname string, settings VhostSettings) (res *http.Response, err error) {
	body, err := json.Marshal(settings)
	if err != nil {
		return nil, err
	}

	req, err := newRequestWithBody(c, "PUT", "vhosts/"+url.QueryEscape(vhostname), body)
	if err != nil {
		return nil, err
	}

	res, err = executeRequest(c, req)
	if err != nil {
		return nil, err
	}

	return res, nil
}

//
// DELETE /api/vhosts/{name}
//

// Deletes a virtual host.
func (c *Client) DeleteVhost(vhostname string) (res *http.Response, err error) {
	req, err := newRequestWithBody(c, "DELETE", "vhosts/"+url.QueryEscape(vhostname), nil)
	if err != nil {
		return nil, err
	}

	res, err = executeRequest(c, req)
	if err != nil {
		return nil, err
	}

	return res, nil
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2015 Alex Yatskov <alex@foosoft.net>
 * Author: Alex Yatskov <alex@foosoft.net>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 * the Software, and to permit persons to whom the Software is furnished to do so,
 * subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package main

import (
	"bytes"
	"fmt"
	"log"
	"path"
	"regexp"
	"strconv"
	"strings"

	"github.com/russross/blackfriday"
)

const (
	defNumCols = 80
	defTabSize = 4
)

const (
	flagNoToc = 1 << iota
	flagNoRules
	flagPascal
)

type list struct {
	index int
}

type heading struct {
	text  []byte
	level int
}

type vimDoc struct {
	filename string
	title    string
	desc     string
	cols     int
	tabs     int
	flags    int
	tocPos   int
	lists    []*list
	headings []*heading
}

func VimDocRenderer(filename, desc string, cols, tabs, flags int) blackfriday.Renderer {
	filename = path.Base(filename)
	title := filename

	if index := strings.LastIndex(filename, "."); index > -1 {
		title = filename[:index]
		if flags&flagPascal == 0 {
			title = strings.ToLower(title)
		}
	}

	return &vimDoc{
		filename: filename,
		title:    title,
		desc:     desc,
		cols:     cols,
		tabs:     tabs,
		flags:    flags,
		tocPos:   -1}
}

func (v *vimDoc) fixupCodeTags(input []byte) []byte {
	r := regexp.MustCompile(`(?m)^\s*([<>])$`)
	return r.ReplaceAll(input, []byte("$1"))
}

func (v *vimDoc) buildHelpTag(text []byte) []byte {
	if v.flags&flagPascal == 0 {
		text = bytes.ToLower(text)
		text = bytes.Replace(text, []byte{' '}, []byte{'_'}, -1)
	} else {
		text = bytes.Title(text)
		text = bytes.Replace(text, []byte{' '}, []byte{}, -1)
	}

	return []byte(fmt.Sprintf("%s-%s", v.title, text))
}

func (v *vimDoc) buildChapters(h *heading) []byte {
	index := -1
	{
		for i, curr := range v.headings {
			if curr == h {
				index = i
				break
			}
		}

		if index < 0 {
			log.Fatal("heading not found")
		}
	}

	var chapters []int
	{
		level := h.level
		siblings := 1

		for i := index - 1; i >= 0; i-- {
			curr := v.headings[i]

			if curr.level == level {
				siblings++
			} else if curr.level < level {
				chapters = append(chapters, siblings)
				level = curr.level
				siblings = 1
			}
		}

		chapters = append(chapters, siblings)
	}

	var out bytes.Buffer
	for i := len(chapters) - 1; i >= 0; i-- {
		out.WriteString(strconv.Itoa(chapters[i]))
		out.WriteString(".")
	}

	return out.Bytes()
}

func (v *vimDoc) writeSplitText(out *bytes.Buffer, left, right []byte, repeat string, trim int) {
	padding := v.cols - (len(left) + len(right)) + trim
	if padding <= 0 {
		padding = 1
	}

	out.Write(left)
	out.WriteString(strings.Repeat(repeat, padding))
	out.Write(right)
	out.WriteString("\n")
}

func (v *vimDoc) writeRule(out *bytes.Buffer, repeat string) {
	out.WriteString(strings.Repeat(repeat, v.cols))
	out.WriteString("\n")
}

func (v *vimDoc) writeToc(out *bytes.Buffer) {
	for _, h := range v.headings {
		title := fmt.Sprintf("%s%s %s", strings.Repeat(" ", (h.level-1)*v.tabs), v.buildChapters(h), h.text)
		link := fmt.Sprintf("|%s|", v.buildHelpTag(h.text))
		v.writeSplitText(out, []byte(title), []byte(link), ".", 2)
	}
}

func (v *vimDoc) writeIndent(out *bytes.Buffer, text string, trim int) {
	lines := strings.Split(text, "\n")

	for index, line := range lines {
		width := v.tabs
		if width >= trim && index == 0 {
			width -= trim
		}

		if len(line) > 0 {
			out.WriteString(strings.Repeat(" ", width))
			out.WriteString(line)
			out.WriteString("\n")
		}
	}
}

// Block-level callbacks
func (v *vimDoc) BlockCode(out *bytes.Buffer, text []byte, lang string) {
	out.WriteString(">\n")
	v.writeIndent(out, string(text), 0)
	out.WriteString("<\n\n")
}

func (v *vimDoc) BlockQuote(out *bytes.Buffer, text []byte) {
	out.WriteString(">\n")
	v.writeIndent(out, string(text), 0)
	out.WriteString("<\n\n")
}

func (v *vimDoc) BlockHtml(out *bytes.Buffer, text []byte) {
	out.WriteString(">\n")
	v.writeIndent(out, string(text), 0)
	out.WriteString("<\n\n")
}

func (v *vimDoc) Header(out *bytes.Buffer, text func() bool, level int, id string) {
	initPos := out.Len()
	if v.flags&flagNoRules == 0 {
		switch level {
		case 1:
			v.writeRule(out, "=")
		case 2:
			v.writeRule(out, "-")
		}
	}

	headingPos := out.Len()
	if !text() {
		out.Truncate(initPos)
		return
	}

	if v.tocPos == -1 && len(v.headings) > 0 {
		v.tocPos = initPos
	}

	var temp []byte
	temp = append(temp, out.Bytes()[headingPos:]...)

	h := &heading{temp, level}
	v.headings = append(v.headings, h)

	out.Truncate(headingPos)
	tag := fmt.Sprintf("*%s*", v.buildHelpTag(h.text))
	title := fmt.Sprintf("%s %s", v.buildChapters(h), bytes.ToUpper(h.text))
	v.writeSplitText(out, []byte(title), []byte(tag), " ", 2)
	out.WriteString("\n")
}

func (v *vimDoc) HRule(out *bytes.Buffer) {
	v.writeRule(out, "-")
}

func (v *vimDoc) List(out *bytes.Buffer, text func() bool, flags int) {
	v.lists = append(v.lists, &list{1})
	text()
	v.lists = v.lists[:len(v.lists)-1]
}

func (v *vimDoc) ListItem(out *bytes.Buffer, text []byte, flags int) {
	marker := out.Len()

	list := v.lists[len(v.lists)-1]
	if flags&blackfriday.LIST_TYPE_ORDERED == blackfriday.LIST_TYPE_ORDERED {
		out.WriteString(fmt.Sprintf("%d. ", list.index))
		list.index++
	} else {
		out.WriteString("* ")
	}

	v.writeIndent(out, string(text), out.Len()-marker)

	if flags&blackfriday.LIST_ITEM_END_OF_LIST != 0 {
		out.WriteString("\n")
	}
}

func (*vimDoc) Paragraph(out *bytes.Buffer, text func() bool) {
	marker := out.Len()

	if !text() {
		out.Truncate(marker)
		return
	}

	out.WriteString("\n\n")
}

func (*vimDoc) Table(out *bytes.Buffer, heading []byte, body []byte, columnData []int) {
	// unimplemented
	log.Println("Table is a stub")
}

func (*vimDoc) TableRow(out *bytes.Buffer, text []byte) {
	// unimplemented
	log.Println("TableRow is a stub")
}

func (*vimDoc) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {
	// unimplemented
	log.Println("TableHeaderCell is a stub")
}

func (*vimDoc) TableCell(out *bytes.Buffer, text []byte, flags int) {
	// unimplemented
	log.Println("TableCell is a stub")
}

func (*vimDoc) Footnotes(out *bytes.Buffer, text func() bool) {
	// unimplemented
	log.Println("Footnotes is a stub")
}

func (*vimDoc) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {
	// unimplemented
	log.Println("FootnoteItem is a stub")
}

func (*vimDoc) TitleBlock(out *bytes.Buffer, text []byte) {
	// unimplemented
	log.Println("TitleBlock is a stub")
}

// Span-level callbacks
func (*vimDoc) AutoLink(out *bytes.Buffer, link []byte, kind int) {
	out.Write(link)
}

func (*vimDoc) CodeSpan(out *bytes.Buffer, text []byte) {
	r := regexp.MustCompile(`\s`)

	// vim does not correctly highlight space-delimited words in code spans
	if !r.Match(text) {
		out.WriteString("`")
		out.Write(text)
		out.WriteString("`")
	}
}

func (*vimDoc) DoubleEmphasis(out *bytes.Buffer, text []byte) {
	out.Write(text)
}

func (*vimDoc) Emphasis(out *bytes.Buffer, text []byte) {
	out.Write(text)
}

func (*vimDoc) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {
	// cannot view images in vim
}

func (*vimDoc) LineBreak(out *bytes.Buffer) {
	out.WriteString("\n")
}

func (*vimDoc) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
	out.WriteString(fmt.Sprintf("%s (%s)", content, link))
}

func (*vimDoc) RawHtmlTag(out *bytes.Buffer, tag []byte) {
	// unimplemented
	log.Println("StrikeThrough is a stub")
}

func (*vimDoc) TripleEmphasis(out *bytes.Buffer, text []byte) {
	out.Write(text)
}

func (*vimDoc) StrikeThrough(out *bytes.Buffer, text []byte) {
	// unimplemented
	log.Println("StrikeThrough is a stub")
}

func (*vimDoc) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {
	// unimplemented
	log.Println("FootnoteRef is a stub")
}

// Low-level callbacks
func (v *vimDoc) Entity(out *bytes.Buffer, entity []byte) {
	out.Write(entity)
}

func (v *vimDoc) NormalText(out *bytes.Buffer, text []byte) {
	out.Write(text)
}

// Header and footer
func (v *vimDoc) DocumentHeader(out *bytes.Buffer) {
	if len(v.desc) > 0 {
		v.writeSplitText(out, []byte(v.filename), []byte(v.desc), " ", 0)
	} else {
		out.WriteString(v.filename)
		out.WriteString("\n")
	}

	out.WriteString("\n")
}

func (v *vimDoc) DocumentFooter(out *bytes.Buffer) {
	var temp bytes.Buffer

	if v.tocPos > 0 && v.flags&flagNoToc == 0 {
		temp.Write(out.Bytes()[:v.tocPos])
		v.writeToc(&temp)
		temp.WriteString("\n")
		temp.Write(out.Bytes()[v.tocPos:])
	} else {
		temp.ReadFrom(out)
	}

	out.Reset()
	out.Write(v.fixupCodeTags(temp.Bytes()))
}

func (v *vimDoc) GetFlags() int {
	return v.flags
}
<commit_msg>Update log messages<commit_after>/*
 * Copyright (c) 2015 Alex Yatskov <alex@foosoft.net>
 * Author: Alex Yatskov <alex@foosoft.net>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of
 * this software and associated documentation files (the "Software"), to deal in
 * the Software without restriction, including without limitation the rights to
 * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 * the Software, and to permit persons to whom the Software is furnished to do so,
 * subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

package main

import (
	"bytes"
	"fmt"
	"log"
	"path"
	"regexp"
	"strconv"
	"strings"

	"github.com/russross/blackfriday"
)

const (
	defNumCols = 80
	defTabSize = 4
)

const (
	flagNoToc = 1 << iota
	flagNoRules
	flagPascal
)

type list struct {
	index int
}

type heading struct {
	text  []byte
	level int
}

type vimDoc struct {
	filename string
	title    string
	desc     string
	cols     int
	tabs     int
	flags    int
	tocPos   int
	lists    []*list
	headings []*heading
}

func VimDocRenderer(filename, desc string, cols, tabs, flags int) blackfriday.Renderer {
	filename = path.Base(filename)
	title := filename

	if index := strings.LastIndex(filename, "."); index > -1 {
		title = filename[:index]
		if flags&flagPascal == 0 {
			title = strings.ToLower(title)
		}
	}

	return &vimDoc{
		filename: filename,
		title:    title,
		desc:     desc,
		cols:     cols,
		tabs:     tabs,
		flags:    flags,
		tocPos:   -1}
}

func (v *vimDoc) fixupCodeTags(input []byte) []byte {
	r := regexp.MustCompile(`(?m)^\s*([<>])$`)
	return r.ReplaceAll(input, []byte("$1"))
}

func (v *vimDoc) buildHelpTag(text []byte) []byte {
	if v.flags&flagPascal == 0 {
		text = bytes.ToLower(text)
		text = bytes.Replace(text, []byte{' '}, []byte{'_'}, -1)
	} else {
		text = bytes.Title(text)
		text = bytes.Replace(text, []byte{' '}, []byte{}, -1)
	}

	return []byte(fmt.Sprintf("%s-%s", v.title, text))
}

func (v *vimDoc) buildChapters(h *heading) []byte {
	index := -1
	{
		for i, curr := range v.headings {
			if curr == h {
				index = i
				break
			}
		}

		if index < 0 {
			log.Fatal("heading not found")
		}
	}

	var chapters []int
	{
		level := h.level
		siblings := 1

		for i := index - 1; i >= 0; i-- {
			curr := v.headings[i]

			if curr.level == level {
				siblings++
			} else if curr.level < level {
				chapters = append(chapters, siblings)
				level = curr.level
				siblings = 1
			}
		}

		chapters = append(chapters, siblings)
	}

	var out bytes.Buffer
	for i := len(chapters) - 1; i >= 0; i-- {
		out.WriteString(strconv.Itoa(chapters[i]))
		out.WriteString(".")
	}

	return out.Bytes()
}

func (v *vimDoc) writeSplitText(out *bytes.Buffer, left, right []byte, repeat string, trim int) {
	padding := v.cols - (len(left) + len(right)) + trim
	if padding <= 0 {
		padding = 1
	}

	out.Write(left)
	out.WriteString(strings.Repeat(repeat, padding))
	out.Write(right)
	out.WriteString("\n")
}

func (v *vimDoc) writeRule(out *bytes.Buffer, repeat string) {
	out.WriteString(strings.Repeat(repeat, v.cols))
	out.WriteString("\n")
}

func (v *vimDoc) writeToc(out *bytes.Buffer) {
	for _, h := range v.headings {
		title := fmt.Sprintf("%s%s %s", strings.Repeat(" ", (h.level-1)*v.tabs), v.buildChapters(h), h.text)
		link := fmt.Sprintf("|%s|", v.buildHelpTag(h.text))
		v.writeSplitText(out, []byte(title), []byte(link), ".", 2)
	}
}

func (v *vimDoc) writeIndent(out *bytes.Buffer, text string, trim int) {
	lines := strings.Split(text, "\n")

	for index, line := range lines {
		width := v.tabs
		if width >= trim && index == 0 {
			width -= trim
		}

		if len(line) > 0 {
			out.WriteString(strings.Repeat(" ", width))
			out.WriteString(line)
			out.WriteString("\n")
		}
	}
}

// Block-level callbacks
func (v *vimDoc) BlockCode(out *bytes.Buffer, text []byte, lang string) {
	out.WriteString(">\n")
	v.writeIndent(out, string(text), 0)
	out.WriteString("<\n\n")
}

func (v *vimDoc) BlockQuote(out *bytes.Buffer, text []byte) {
	out.WriteString(">\n")
	v.writeIndent(out, string(text), 0)
	out.WriteString("<\n\n")
}

func (v *vimDoc) BlockHtml(out *bytes.Buffer, text []byte) {
	out.WriteString(">\n")
	v.writeIndent(out, string(text), 0)
	out.WriteString("<\n\n")
}

func (v *vimDoc) Header(out *bytes.Buffer, text func() bool, level int, id string) {
	initPos := out.Len()
	if v.flags&flagNoRules == 0 {
		switch level {
		case 1:
			v.writeRule(out, "=")
		case 2:
			v.writeRule(out, "-")
		}
	}

	headingPos := out.Len()
	if !text() {
		out.Truncate(initPos)
		return
	}

	if v.tocPos == -1 && len(v.headings) > 0 {
		v.tocPos = initPos
	}

	var temp []byte
	temp = append(temp, out.Bytes()[headingPos:]...)

	h := &heading{temp, level}
	v.headings = append(v.headings, h)

	out.Truncate(headingPos)
	tag := fmt.Sprintf("*%s*", v.buildHelpTag(h.text))
	title := fmt.Sprintf("%s %s", v.buildChapters(h), bytes.ToUpper(h.text))
	v.writeSplitText(out, []byte(title), []byte(tag), " ", 2)
	out.WriteString("\n")
}

func (v *vimDoc) HRule(out *bytes.Buffer) {
	v.writeRule(out, "-")
}

func (v *vimDoc) List(out *bytes.Buffer, text func() bool, flags int) {
	v.lists = append(v.lists, &list{1})
	text()
	v.lists = v.lists[:len(v.lists)-1]
}

func (v *vimDoc) ListItem(out *bytes.Buffer, text []byte, flags int) {
	marker := out.Len()

	list := v.lists[len(v.lists)-1]
	if flags&blackfriday.LIST_TYPE_ORDERED == blackfriday.LIST_TYPE_ORDERED {
		out.WriteString(fmt.Sprintf("%d. ", list.index))
		list.index++
	} else {
		out.WriteString("* ")
	}

	v.writeIndent(out, string(text), out.Len()-marker)

	if flags&blackfriday.LIST_ITEM_END_OF_LIST != 0 {
		out.WriteString("\n")
	}
}

func (*vimDoc) Paragraph(out *bytes.Buffer, text func() bool) {
	marker := out.Len()

	if !text() {
		out.Truncate(marker)
		return
	}

	out.WriteString("\n\n")
}

func (*vimDoc) Table(out *bytes.Buffer, heading []byte, body []byte, columnData []int) {
	// unimplemented
	log.Println("Table is unimplemented")
}

func (*vimDoc) TableRow(out *bytes.Buffer, text []byte) {
	// unimplemented
	log.Println("TableRow is unimplemented")
}

func (*vimDoc) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {
	// unimplemented
	log.Println("TableHeaderCell is unimplemented")
}

func (*vimDoc) TableCell(out *bytes.Buffer, text []byte, flags int) {
	// unimplemented
	log.Println("TableCell is unimplemented")
}

func (*vimDoc) Footnotes(out *bytes.Buffer, text func() bool) {
	// unimplemented
	log.Println("Footnotes is unimplemented")
}

func (*vimDoc) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {
	// unimplemented
	log.Println("FootnoteItem is unimplemented")
}

func (*vimDoc) TitleBlock(out *bytes.Buffer, text []byte) {
	// unimplemented
	log.Println("TitleBlock is unimplemented")
}

// Span-level callbacks
func (*vimDoc) AutoLink(out *bytes.Buffer, link []byte, kind int) {
	out.Write(link)
}

func (*vimDoc) CodeSpan(out *bytes.Buffer, text []byte) {
	r := regexp.MustCompile(`\s`)

	// vim does not correctly highlight space-delimited words in code spans
	if !r.Match(text) {
		out.WriteString("`")
		out.Write(text)
		out.WriteString("`")
	}
}

func (*vimDoc) DoubleEmphasis(out *bytes.Buffer, text []byte) {
	out.Write(text)
}

func (*vimDoc) Emphasis(out *bytes.Buffer, text []byte) {
	out.Write(text)
}

func (*vimDoc) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {
	// cannot view images in vim
}

func (*vimDoc) LineBreak(out *bytes.Buffer) {
	out.WriteString("\n")
}

func (*vimDoc) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) {
	out.WriteString(fmt.Sprintf("%s (%s)", content, link))
}

func (*vimDoc) RawHtmlTag(out *bytes.Buffer, tag []byte) {
	// unimplemented
	log.Println("StrikeThrough is unimplemented")
}

func (*vimDoc) TripleEmphasis(out *bytes.Buffer, text []byte) {
	out.Write(text)
}

func (*vimDoc) StrikeThrough(out *bytes.Buffer, text []byte) {
	// unimplemented
	log.Println("StrikeThrough is unimplemented")
}

func (*vimDoc) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {
	// unimplemented
	log.Println("FootnoteRef is unimplemented")
}

// Low-level callbacks
func (v *vimDoc) Entity(out *bytes.Buffer, entity []byte) {
	out.Write(entity)
}

func (v *vimDoc) NormalText(out *bytes.Buffer, text []byte) {
	out.Write(text)
}

// Header and footer
func (v *vimDoc) DocumentHeader(out *bytes.Buffer) {
	if len(v.desc) > 0 {
		v.writeSplitText(out, []byte(v.filename), []byte(v.desc), " ", 0)
	} else {
		out.WriteString(v.filename)
		out.WriteString("\n")
	}

	out.WriteString("\n")
}

func (v *vimDoc) DocumentFooter(out *bytes.Buffer) {
	var temp bytes.Buffer

	if v.tocPos > 0 && v.flags&flagNoToc == 0 {
		temp.Write(out.Bytes()[:v.tocPos])
		v.writeToc(&temp)
		temp.WriteString("\n")
		temp.Write(out.Bytes()[v.tocPos:])
	} else {
		temp.ReadFrom(out)
	}

	out.Reset()
	out.Write(v.fixupCodeTags(temp.Bytes()))
}

func (v *vimDoc) GetFlags() int {
	return v.flags
}
<|endoftext|>


<commit_before>// Copyright 2014 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author: polaris	polaris@studygolang.com

package logic

import (
	"encoding/json"
	"errors"
	"net/http"
	"net/url"
	"strconv"
	"util"

	. "db"

	"github.com/polaris1119/config"
	"github.com/polaris1119/goutils"
	"github.com/polaris1119/logger"

	"model"
)

type SearcherLogic struct {
	maxRows int

	engineUrl string
}

var DefaultSearcher = SearcherLogic{maxRows: 100, engineUrl: config.ConfigFile.MustValue("search", "engine_url")}

// 准备索引数据，post 给 solr
// isAll: 是否全量
func (self SearcherLogic) Indexing(isAll bool) {
	self.IndexingArticle(isAll)
	self.IndexingTopic(isAll)
	self.IndexingResource(isAll)
	self.IndexingOpenProject(isAll)
}

// IndexingArticle 索引博文
func (self SearcherLogic) IndexingArticle(isAll bool) {
	solrClient := NewSolrClient()

	var (
		articleList []*model.Article
		err         error
	)

	if isAll {
		id := 0
		for {
			articleList = make([]*model.Article, 0)
			err = MasterDB.Where("id>?", id).Limit(self.maxRows).Find(&articleList)
			if err != nil {
				logger.Errorln("IndexingArticle error:", err)
				break
			}

			if len(articleList) == 0 {
				break
			}

			for _, article := range articleList {
				if id < article.Id {
					id = article.Id
				}

				document := model.NewDocument(article, nil)
				if article.Status != model.ArticleStatusOffline {
					solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
				} else {
					solrClient.PushDel(model.NewDelCommand(document))
				}
			}

			solrClient.Post()
		}
	}
}

// 索引主题
func (self SearcherLogic) IndexingTopic(isAll bool) {
	solrClient := NewSolrClient()

	var (
		topicList   []*model.Topic
		topicExList map[int]*model.TopicEx

		err error
	)

	if isAll {
		id := 0
		for {
			topicList = make([]*model.Topic, 0)
			topicExList = make(map[int]*model.TopicEx)

			err = MasterDB.Where("tid>?", id).Limit(self.maxRows).Find(&topicList)
			if err != nil {
				logger.Errorln("IndexingTopic error:", err)
				break
			}

			if len(topicList) == 0 {
				break
			}

			tids := util.Models2Intslice(topicList, "Tid")

			err = MasterDB.In("tid", tids).Find(&topicExList)
			if err != nil {
				logger.Errorln("IndexingTopic error:", err)
				break
			}

			for _, topic := range topicList {
				if id < topic.Tid {
					id = topic.Tid
				}

				topicEx := topicExList[topic.Tid]

				document := model.NewDocument(topic, topicEx)
				addCommand := model.NewDefaultArgsAddCommand(document)

				solrClient.PushAdd(addCommand)
			}

			solrClient.Post()
		}
	}
}

// 索引资源
func (self SearcherLogic) IndexingResource(isAll bool) {
	solrClient := NewSolrClient()

	var (
		resourceList   []*model.Resource
		resourceExList map[int]*model.ResourceEx
		err            error
	)

	if isAll {
		id := 0
		for {
			resourceList = make([]*model.Resource, 0)
			err = MasterDB.Where("id>?", id).Limit(self.maxRows).Find(&resourceList)
			if err != nil {
				logger.Errorln("IndexingResource error:", err)
				break
			}

			if len(resourceList) == 0 {
				break
			}

			ids := util.Models2Intslice(resourceList, "Id")

			err = MasterDB.In("id", ids).Find(&resourceExList)
			if err != nil {
				logger.Errorln("IndexingResource error:", err)
				break
			}

			for _, resource := range resourceList {
				if id < resource.Id {
					id = resource.Id
				}

				resourceEx := resourceExList[resource.Id]

				document := model.NewDocument(resource, resourceEx)
				addCommand := model.NewDefaultArgsAddCommand(document)

				solrClient.PushAdd(addCommand)
			}

			solrClient.Post()
		}
	}
}

// IndexingOpenProject 索引博文
func (self SearcherLogic) IndexingOpenProject(isAll bool) {
	solrClient := NewSolrClient()

	var (
		projectList []*model.OpenProject
		err         error
	)

	if isAll {
		id := 0
		for {
			projectList = make([]*model.OpenProject, 0)
			err = MasterDB.Where("id>?", id).Limit(self.maxRows).Find(&projectList)
			if err != nil {
				logger.Errorln("IndexingArticle error:", err)
				break
			}

			if len(projectList) == 0 {
				break
			}

			for _, project := range projectList {
				if id < project.Id {
					id = project.Id
				}

				document := model.NewDocument(project, nil)
				if project.Status != model.ProjectStatusOffline {
					solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
				} else {
					solrClient.PushDel(model.NewDelCommand(document))
				}
			}

			solrClient.Post()
		}
	}
}

const searchContentLen = 350

// DoSearch 搜索
func (self SearcherLogic) DoSearch(q, field string, start, rows int) (*model.ResponseBody, error) {
	selectUrl := self.engineUrl + "/select?"

	var values = url.Values{
		"wt":             []string{"json"},
		"hl":             []string{"true"},
		"hl.fl":          []string{"title,content"},
		"hl.simple.pre":  []string{""},
		"hl.simple.post": []string{""},
		"hl.fragsize":    []string{strconv.Itoa(searchContentLen)},
		"start":          []string{strconv.Itoa(start)},
		"rows":           []string{strconv.Itoa(rows)},
	}

	if q == "" {
		values.Add("q", "*:*")
	} else {
		searchStat := &model.SearchStat{}
		MasterDB.Where("keyword=?", q).Get(searchStat)
		if searchStat.Id > 0 {
			MasterDB.Where("keyword=?", q).Incr("times", 1).Update(new(model.SearchStat))
		} else {
			searchStat.Keyword = q
			searchStat.Times = 1
			_, err := MasterDB.Insert(searchStat)
			if err != nil {
				MasterDB.Where("keyword=?", q).Incr("times", 1).Update(new(model.SearchStat))
			}
		}
	}

	isTag := false
	// TODO: 目前大部分都没有tag，因此，对tag特殊处理
	if field == "text" || field == "tag" {
		if field == "tag" {
			isTag = true
		}
		field = ""
	}

	if field != "" {
		values.Add("df", field)
		if q != "" {
			values.Add("q", q)
		}
	} else {
		// 全文检索
		if q != "" {
			if isTag {
				values.Add("q", "title:"+q+"^2"+" OR tags:"+q+"^4 OR content:"+q+"^0.2")
			} else {
				values.Add("q", "title:"+q+"^2"+" OR content:"+q+"^0.2")
			}
		}
	}

	resp, err := http.Get(selectUrl + values.Encode())
	if err != nil {
		logger.Errorln("search error:", err)
		return &model.ResponseBody{}, err
	}

	defer resp.Body.Close()

	var searchResponse model.SearchResponse
	err = json.NewDecoder(resp.Body).Decode(&searchResponse)
	if err != nil {
		logger.Errorln("parse response error:", err)
		return &model.ResponseBody{}, err
	}

	if len(searchResponse.Highlight) > 0 {
		for _, doc := range searchResponse.RespBody.Docs {
			highlighting, ok := searchResponse.Highlight[doc.Id]
			if ok {
				if len(highlighting.Title) > 0 {
					doc.HlTitle = highlighting.Title[0]
				}

				if len(highlighting.Content) > 0 {
					doc.HlContent = highlighting.Content[0]
				}
			}

			if doc.HlTitle == "" {
				doc.HlTitle = doc.Title
			}

			if doc.HlContent == "" && doc.Content != "" {
				maxLen := len(doc.Content) - 1
				if maxLen > searchContentLen {
					maxLen = searchContentLen
				}
				doc.HlContent = doc.Content[:maxLen]
			}

			doc.HlContent += "..."
		}
	}

	if searchResponse.RespBody == nil {
		searchResponse.RespBody = &model.ResponseBody{}
	}

	return searchResponse.RespBody, nil
}

type SolrClient struct {
	addCommands []*model.AddCommand
	delCommands []*model.DelCommand
}

func NewSolrClient() *SolrClient {
	return &SolrClient{
		addCommands: make([]*model.AddCommand, 0, 100),
		delCommands: make([]*model.DelCommand, 0, 100),
	}
}

func (this *SolrClient) PushAdd(addCommand *model.AddCommand) {
	this.addCommands = append(this.addCommands, addCommand)
}

func (this *SolrClient) PushDel(delCommand *model.DelCommand) {
	this.delCommands = append(this.delCommands, delCommand)
}

func (this *SolrClient) Post() error {
	stringBuilder := goutils.NewBuffer().Append("{")

	needComma := false
	for _, addCommand := range this.addCommands {
		commandJson, err := json.Marshal(addCommand)
		if err != nil {
			continue
		}

		if stringBuilder.Len() == 1 {
			needComma = false
		} else {
			needComma = true
		}

		if needComma {
			stringBuilder.Append(",")
		}

		stringBuilder.Append(`"add":`).Append(commandJson)
	}

	for _, delCommand := range this.delCommands {
		commandJson, err := json.Marshal(delCommand)
		if err != nil {
			continue
		}

		if stringBuilder.Len() == 1 {
			needComma = false
		} else {
			needComma = true
		}

		if needComma {
			stringBuilder.Append(",")
		}

		stringBuilder.Append(`"delete":`).Append(commandJson)
	}

	if stringBuilder.Len() == 1 {
		logger.Errorln("post docs:no right addcommand")
		return errors.New("no right addcommand")
	}

	stringBuilder.Append("}")

	logger.Infoln("start post data to solr...")

	resp, err := http.Post(config.ConfigFile.MustValue("search", "engine_url")+"/update?wt=json&commit=true", "application/json", stringBuilder)
	if err != nil {
		logger.Errorln("post error:", err)
		return err
	}

	defer resp.Body.Close()

	var result map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&result)
	if err != nil {
		logger.Errorln("parse response error:", err)
		return err
	}

	logger.Infoln("post data result:", result)

	return nil
}
<commit_msg>bugfix<commit_after>// Copyright 2014 The StudyGolang Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// http://studygolang.com
// Author: polaris	polaris@studygolang.com

package logic

import (
	"encoding/json"
	"errors"
	"net/http"
	"net/url"
	"strconv"
	"util"

	. "db"

	"github.com/polaris1119/config"
	"github.com/polaris1119/goutils"
	"github.com/polaris1119/logger"

	"model"
)

type SearcherLogic struct {
	maxRows int

	engineUrl string
}

var DefaultSearcher = SearcherLogic{maxRows: 100, engineUrl: config.ConfigFile.MustValue("search", "engine_url")}

// 准备索引数据，post 给 solr
// isAll: 是否全量
func (self SearcherLogic) Indexing(isAll bool) {
	self.IndexingArticle(isAll)
	self.IndexingTopic(isAll)
	self.IndexingResource(isAll)
	self.IndexingOpenProject(isAll)
}

// IndexingArticle 索引博文
func (self SearcherLogic) IndexingArticle(isAll bool) {
	solrClient := NewSolrClient()

	var (
		articleList []*model.Article
		err         error
	)

	if isAll {
		id := 0
		for {
			articleList = make([]*model.Article, 0)
			err = MasterDB.Where("id>?", id).Limit(self.maxRows).OrderBy("id ASC").Find(&articleList)
			if err != nil {
				logger.Errorln("IndexingArticle error:", err)
				break
			}

			if len(articleList) == 0 {
				break
			}

			for _, article := range articleList {
				if id < article.Id {
					id = article.Id
				}

				document := model.NewDocument(article, nil)
				if article.Status != model.ArticleStatusOffline {
					solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
				} else {
					solrClient.PushDel(model.NewDelCommand(document))
				}
			}

			solrClient.Post()
		}
	}
}

// 索引主题
func (self SearcherLogic) IndexingTopic(isAll bool) {
	solrClient := NewSolrClient()

	var (
		topicList   []*model.Topic
		topicExList map[int]*model.TopicEx

		err error
	)

	if isAll {
		id := 0
		for {
			topicList = make([]*model.Topic, 0)
			topicExList = make(map[int]*model.TopicEx)

			err = MasterDB.Where("tid>?", id).OrderBy("tid ASC").Limit(self.maxRows).Find(&topicList)
			if err != nil {
				logger.Errorln("IndexingTopic error:", err)
				break
			}

			if len(topicList) == 0 {
				break
			}

			tids := util.Models2Intslice(topicList, "Tid")

			err = MasterDB.In("tid", tids).Find(&topicExList)
			if err != nil {
				logger.Errorln("IndexingTopic error:", err)
				break
			}

			for _, topic := range topicList {
				if id < topic.Tid {
					id = topic.Tid
				}

				topicEx := topicExList[topic.Tid]

				document := model.NewDocument(topic, topicEx)
				addCommand := model.NewDefaultArgsAddCommand(document)

				solrClient.PushAdd(addCommand)
			}

			solrClient.Post()
		}
	}
}

// 索引资源
func (self SearcherLogic) IndexingResource(isAll bool) {
	solrClient := NewSolrClient()

	var (
		resourceList   []*model.Resource
		resourceExList map[int]*model.ResourceEx
		err            error
	)

	if isAll {
		id := 0
		for {
			resourceList = make([]*model.Resource, 0)
			err = MasterDB.Where("id>?", id).OrderBy("id ASC").Limit(self.maxRows).Find(&resourceList)
			if err != nil {
				logger.Errorln("IndexingResource error:", err)
				break
			}

			if len(resourceList) == 0 {
				break
			}

			ids := util.Models2Intslice(resourceList, "Id")

			err = MasterDB.In("id", ids).Find(&resourceExList)
			if err != nil {
				logger.Errorln("IndexingResource error:", err)
				break
			}

			for _, resource := range resourceList {
				if id < resource.Id {
					id = resource.Id
				}

				resourceEx := resourceExList[resource.Id]

				document := model.NewDocument(resource, resourceEx)
				addCommand := model.NewDefaultArgsAddCommand(document)

				solrClient.PushAdd(addCommand)
			}

			solrClient.Post()
		}
	}
}

// IndexingOpenProject 索引博文
func (self SearcherLogic) IndexingOpenProject(isAll bool) {
	solrClient := NewSolrClient()

	var (
		projectList []*model.OpenProject
		err         error
	)

	if isAll {
		id := 0
		for {
			projectList = make([]*model.OpenProject, 0)
			err = MasterDB.Where("id>?", id).OrderBy("id ASC").Limit(self.maxRows).Find(&projectList)
			if err != nil {
				logger.Errorln("IndexingArticle error:", err)
				break
			}

			if len(projectList) == 0 {
				break
			}

			for _, project := range projectList {
				if id < project.Id {
					id = project.Id
				}

				document := model.NewDocument(project, nil)
				if project.Status != model.ProjectStatusOffline {
					solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
				} else {
					solrClient.PushDel(model.NewDelCommand(document))
				}
			}

			solrClient.Post()
		}
	}
}

const searchContentLen = 350

// DoSearch 搜索
func (self SearcherLogic) DoSearch(q, field string, start, rows int) (*model.ResponseBody, error) {
	selectUrl := self.engineUrl + "/select?"

	var values = url.Values{
		"wt":             []string{"json"},
		"hl":             []string{"true"},
		"hl.fl":          []string{"title,content"},
		"hl.simple.pre":  []string{""},
		"hl.simple.post": []string{""},
		"hl.fragsize":    []string{strconv.Itoa(searchContentLen)},
		"start":          []string{strconv.Itoa(start)},
		"rows":           []string{strconv.Itoa(rows)},
	}

	if q == "" {
		values.Add("q", "*:*")
	} else {
		searchStat := &model.SearchStat{}
		MasterDB.Where("keyword=?", q).Get(searchStat)
		if searchStat.Id > 0 {
			MasterDB.Where("keyword=?", q).Incr("times", 1).Update(new(model.SearchStat))
		} else {
			searchStat.Keyword = q
			searchStat.Times = 1
			_, err := MasterDB.Insert(searchStat)
			if err != nil {
				MasterDB.Where("keyword=?", q).Incr("times", 1).Update(new(model.SearchStat))
			}
		}
	}

	isTag := false
	// TODO: 目前大部分都没有tag，因此，对tag特殊处理
	if field == "text" || field == "tag" {
		if field == "tag" {
			isTag = true
		}
		field = ""
	}

	if field != "" {
		values.Add("df", field)
		if q != "" {
			values.Add("q", q)
		}
	} else {
		// 全文检索
		if q != "" {
			if isTag {
				values.Add("q", "title:"+q+"^2"+" OR tags:"+q+"^4 OR content:"+q+"^0.2")
			} else {
				values.Add("q", "title:"+q+"^2"+" OR content:"+q+"^0.2")
			}
		}
	}

	resp, err := http.Get(selectUrl + values.Encode())
	if err != nil {
		logger.Errorln("search error:", err)
		return &model.ResponseBody{}, err
	}

	defer resp.Body.Close()

	var searchResponse model.SearchResponse
	err = json.NewDecoder(resp.Body).Decode(&searchResponse)
	if err != nil {
		logger.Errorln("parse response error:", err)
		return &model.ResponseBody{}, err
	}

	if len(searchResponse.Highlight) > 0 {
		for _, doc := range searchResponse.RespBody.Docs {
			highlighting, ok := searchResponse.Highlight[doc.Id]
			if ok {
				if len(highlighting.Title) > 0 {
					doc.HlTitle = highlighting.Title[0]
				}

				if len(highlighting.Content) > 0 {
					doc.HlContent = highlighting.Content[0]
				}
			}

			if doc.HlTitle == "" {
				doc.HlTitle = doc.Title
			}

			if doc.HlContent == "" && doc.Content != "" {
				maxLen := len(doc.Content) - 1
				if maxLen > searchContentLen {
					maxLen = searchContentLen
				}
				doc.HlContent = doc.Content[:maxLen]
			}

			doc.HlContent += "..."
		}
	}

	if searchResponse.RespBody == nil {
		searchResponse.RespBody = &model.ResponseBody{}
	}

	return searchResponse.RespBody, nil
}

type SolrClient struct {
	addCommands []*model.AddCommand
	delCommands []*model.DelCommand
}

func NewSolrClient() *SolrClient {
	return &SolrClient{
		addCommands: make([]*model.AddCommand, 0, 100),
		delCommands: make([]*model.DelCommand, 0, 100),
	}
}

func (this *SolrClient) PushAdd(addCommand *model.AddCommand) {
	this.addCommands = append(this.addCommands, addCommand)
}

func (this *SolrClient) PushDel(delCommand *model.DelCommand) {
	this.delCommands = append(this.delCommands, delCommand)
}

func (this *SolrClient) Post() error {
	stringBuilder := goutils.NewBuffer().Append("{")

	needComma := false
	for _, addCommand := range this.addCommands {
		commandJson, err := json.Marshal(addCommand)
		if err != nil {
			continue
		}

		if stringBuilder.Len() == 1 {
			needComma = false
		} else {
			needComma = true
		}

		if needComma {
			stringBuilder.Append(",")
		}

		stringBuilder.Append(`"add":`).Append(commandJson)
	}

	for _, delCommand := range this.delCommands {
		commandJson, err := json.Marshal(delCommand)
		if err != nil {
			continue
		}

		if stringBuilder.Len() == 1 {
			needComma = false
		} else {
			needComma = true
		}

		if needComma {
			stringBuilder.Append(",")
		}

		stringBuilder.Append(`"delete":`).Append(commandJson)
	}

	if stringBuilder.Len() == 1 {
		logger.Errorln("post docs:no right addcommand")
		return errors.New("no right addcommand")
	}

	stringBuilder.Append("}")

	logger.Infoln("start post data to solr...")

	resp, err := http.Post(config.ConfigFile.MustValue("search", "engine_url")+"/update?wt=json&commit=true", "application/json", stringBuilder)
	if err != nil {
		logger.Errorln("post error:", err)
		return err
	}

	defer resp.Body.Close()

	var result map[string]interface{}
	err = json.NewDecoder(resp.Body).Decode(&result)
	if err != nil {
		logger.Errorln("parse response error:", err)
		return err
	}

	logger.Infoln("post data result:", result)

	return nil
}
<|endoftext|>


<commit_before>package timestamp

import (
	"fmt"
	"net/http"
)

const JSON_TEMPLATE = `{
	"timestampSeconds": %v,
	"timestampMilliSeconds": %v,
	"timestampNanoseconds": %v,
	"date": %v
}`

func init() {
	// Root = howto
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		fmt.Fprintln(w, "/ : display this message")
		fmt.Fprintln(w, "/raw?q=123 : try to decode 123, and prints plain text result")
		fmt.Fprintln(w, "/json?q=123 : try to decode 123, and prints JSON result")
	})

	// Raw: spits text answer only
	http.HandleFunc("/raw", func(w http.ResponseWriter, r *http.Request) {
		q := r.FormValue("q")
		t, err := parseUnknown(q)

		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, err.Error())
			return
		}
		fmt.Fprintf(w, "%v\n", t.Unix())
		fmt.Fprintf(w, "%v\n", t.UnixNano()/1000000)
		fmt.Fprintf(w, "%v\n", t.UnixNano())
		fmt.Fprintf(w, "%v\n", t)
	})

	// JSON
	http.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		q := r.FormValue("q")
		t, err := parseUnknown(q)

		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "{ error:%q }", err.Error())
			return
		}
		fmt.Fprintf(w, JSON_TEMPLATE,
			t.Unix(), t.UnixNano()/1000000, t.UnixNano(), t)
	})

}
<commit_msg>Nicer HTML home with textfield for right-click > Add search keyword...<commit_after>package timestamp

import (
	"fmt"
	"net/http"
)

func init() {
	// Root = howto
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, HOME_TEMPLATE)
	})

	// Raw: spits text answer only
	http.HandleFunc("/raw", func(w http.ResponseWriter, r *http.Request) {
		q := r.FormValue("q")
		t, err := parseUnknown(q)

		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, err.Error())
			return
		}
		fmt.Fprintf(w, "%v\n", t.Unix())
		fmt.Fprintf(w, "%v\n", t.UnixNano()/1000000)
		fmt.Fprintf(w, "%v\n", t.UnixNano())
		fmt.Fprintf(w, "%v\n", t)
	})

	// JSON
	http.HandleFunc("/json", func(w http.ResponseWriter, r *http.Request) {
		q := r.FormValue("q")
		t, err := parseUnknown(q)

		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			fmt.Fprintf(w, "{ error:%q }", err.Error())
			return
		}
		fmt.Fprintf(w, JSON_TEMPLATE,
			t.Unix(), t.UnixNano()/1000000, t.UnixNano(), t)
	})
}

const JSON_TEMPLATE = `{
	"timestampSeconds": %v,
	"timestampMilliSeconds": %v,
	"timestampNanoseconds": %v,
	"date": %v
}`

const HOME_TEMPLATE = `

	<title>Online timestamp stuff</title>

Online timestamp & date (approximate) codec

/

display this message

/raw?q=123

try to decode 123, and gives plain text result

/json?q=123

try to decode 123, and gives JSON result

		<form action="/raw">
			Raw <input type="text" name="q" value="123" /> <input type="submit" value=">" />
		</form>
		<form action="/json">
			JSON <input type="text" name="q" value="123" /> <input type="submit" value=">" />
		</form>

`<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"fmt"
	"github.com/Comcast/webpa-common/concurrent"
	"github.com/Comcast/webpa-common/secure"
	"github.com/Comcast/webpa-common/secure/handler"
	"github.com/Comcast/webpa-common/secure/key"
	"github.com/Comcast/webpa-common/server"
	"github.com/Comcast/webpa-common/webhook"
	"github.com/SermoDigital/jose/jwt"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"time"
)

const (
	applicationName = "caduceus"
	DEFAULT_KEY_ID  = "current"
)

// getValidator returns validator for JWT tokens
func getValidator(v *viper.Viper) (validator secure.Validator, err error) {
	default_validators := make(secure.Validators, 0, 0)
	var jwtVals []JWTValidator

	v.UnmarshalKey("jwtValidators", &jwtVals)

	// make sure there is at least one jwtValidator supplied
	if len(jwtVals) < 1 {
		validator = default_validators
		return
	}

	// if a JWTKeys section was supplied, configure a JWS validator
	// and append it to the chain of validators
	validators := make(secure.Validators, 0, len(jwtVals))

	for _, validatorDescriptor := range jwtVals {
		var keyResolver key.Resolver
		keyResolver, err = validatorDescriptor.Keys.NewResolver()
		if err != nil {
			validator = validators
			return
		}

		validators = append(
			validators,
			secure.JWSValidator{
				DefaultKeyId:  DEFAULT_KEY_ID,
				Resolver:      keyResolver,
				JWTValidators: []*jwt.Validator{validatorDescriptor.Custom.New()},
			},
		)
	}

	validator = validators

	return
}

// caduceus is the driver function for Caduceus.  It performs everything main() would do,
// except for obtaining the command-line arguments (which are passed to it).
func caduceus(arguments []string) int {
	var (
		f = pflag.NewFlagSet(applicationName, pflag.ContinueOnError)
		v = viper.New()

		logger, webPA, err = server.Initialize(applicationName, arguments, f, v)
	)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to initialize Viper environment: %s\n", err)
		return 1
	}

	logger.Info("Using configuration file: %s", v.ConfigFileUsed())

	caduceusConfig := new(CaduceusConfig)
	err = v.Unmarshal(caduceusConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to unmarshal configuration data into struct: %s\n", err)
		return 1
	}

	workerPool := WorkerPoolFactory{
		NumWorkers: caduceusConfig.NumWorkerThreads,
		QueueSize:  caduceusConfig.JobQueueSize,
	}.New()

	caduceusProfilerFactory := ServerProfilerFactory{
		Frequency: caduceusConfig.ProfilerFrequency,
		Duration:  caduceusConfig.ProfilerDuration,
		QueueSize: caduceusConfig.ProfilerQueueSize,
	}

	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	timeout := time.Duration(caduceusConfig.SenderClientTimeout) * time.Second

	// declare a new sender wrapper and pass it a profiler factory so that it can create
	// unique profilers on a per outboundSender basis
	caduceusSenderWrapper, err := SenderWrapperFactory{
		NumWorkersPerSender: caduceusConfig.SenderNumWorkersPerSender,
		QueueSizePerSender:  caduceusConfig.SenderQueueSizePerSender,
		CutOffPeriod:        time.Duration(caduceusConfig.SenderCutOffPeriod) * time.Second,
		Linger:              time.Duration(caduceusConfig.SenderLinger) * time.Second,
		ProfilerFactory:     caduceusProfilerFactory,
		Logger:              logger,
		Client:              &http.Client{Transport: tr, Timeout: timeout},
	}.New()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to initialize new caduceus sender wrapper: %s\n", err)
		return 1
	}

	// here we create a profiler specifically for our main server handler
	caduceusHandlerProfiler, err := caduceusProfilerFactory.New()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to profiler for main caduceus handler: %s\n", err)
		return 1
	}

	serverWrapper := &ServerHandler{
		Logger: logger,
		caduceusHandler: &CaduceusHandler{
			handlerProfiler: caduceusHandlerProfiler,
			senderWrapper:   caduceusSenderWrapper,
			Logger:          logger,
		},
		doJob: workerPool.Send,
	}

	profileWrapper := &ProfileHandler{
		profilerData: caduceusHandlerProfiler,
		Logger:       logger,
	}

	validator, err := getValidator(v)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Validator error: %v\n", err)
		return 1
	}

	authHandler := handler.AuthorizationHandler{
		HeaderName:          "Authorization",
		ForbiddenStatusCode: 403,
		Validator:           validator,
		Logger:              logger,
	}

	caduceusHandler := alice.New(authHandler.Decorate)

	mux := mux.NewRouter()
	mux.Handle("/api/v1/run", caduceusHandler.Then(serverWrapper))
	mux.Handle("/api/v1/profile", caduceusHandler.Then(profileWrapper))

	webhookFactory, err := webhook.NewFactory(v)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating new webhook factory: %s\n", err)
		return 1
	}
	webhookRegistry, webhookHandler := webhookFactory.NewRegistryAndHandler()
	webhookFactory.SetExternalUpdate(caduceusSenderWrapper.Update)

	// register webhook end points for api
	mux.Handle("/hook", caduceusHandler.ThenFunc(webhookRegistry.UpdateRegistry))
	mux.Handle("/hooks", caduceusHandler.ThenFunc(webhookRegistry.GetRegistry))

	selfURL := &url.URL{
		Scheme: "https",
		Host:   v.GetString("fqdn") + v.GetString("primary.address"),
	}

	webhookFactory.Initialize(mux, selfURL, webhookHandler, logger)

	caduceusHealth := &CaduceusHealth{}
	var runnable concurrent.Runnable

	caduceusHealth.Monitor, runnable = webPA.Prepare(logger, mux)
	serverWrapper.caduceusHealth = caduceusHealth

	waitGroup, shutdown, err := concurrent.Execute(runnable)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to start device manager: %s\n", err)
		return 1
	}

	webhookFactory.PrepareAndStart()

	logger.Info("Caduceus is up and running!")

	// Attempt to obtain the current listener list from current system without having to wait for listener reregistration.
	startChan := make(chan webhook.Result, 1)
	webhookFactory.Start.GetCurrentSystemsHooks(startChan)
	var webhookStartResults webhook.Result = <-startChan
	if webhookStartResults.Error != nil {
		logger.Error(webhookStartResults.Error)
	} else {
		webhookFactory.SetList(webhook.NewList(webhookStartResults.Hooks))
		caduceusSenderWrapper.Update(webhookStartResults.Hooks)
	}

	var (
		signals = make(chan os.Signal, 1)
	)

	signal.Notify(signals)
	<-signals
	close(shutdown)
	waitGroup.Wait()

	// shutdown the sender wrapper gently so that all queued messages get serviced
	caduceusSenderWrapper.Shutdown(true)

	return 0
}

func main() {
	os.Exit(caduceus(os.Args))
}
<commit_msg>Add a bridge to the old endpoint as well.<commit_after>package main

import (
	"crypto/tls"
	"fmt"
	"github.com/Comcast/webpa-common/concurrent"
	"github.com/Comcast/webpa-common/secure"
	"github.com/Comcast/webpa-common/secure/handler"
	"github.com/Comcast/webpa-common/secure/key"
	"github.com/Comcast/webpa-common/server"
	"github.com/Comcast/webpa-common/webhook"
	"github.com/SermoDigital/jose/jwt"
	"github.com/gorilla/mux"
	"github.com/justinas/alice"
	"github.com/spf13/pflag"
	"github.com/spf13/viper"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"time"
)

const (
	applicationName = "caduceus"
	DEFAULT_KEY_ID  = "current"
)

// getValidator returns validator for JWT tokens
func getValidator(v *viper.Viper) (validator secure.Validator, err error) {
	default_validators := make(secure.Validators, 0, 0)
	var jwtVals []JWTValidator

	v.UnmarshalKey("jwtValidators", &jwtVals)

	// make sure there is at least one jwtValidator supplied
	if len(jwtVals) < 1 {
		validator = default_validators
		return
	}

	// if a JWTKeys section was supplied, configure a JWS validator
	// and append it to the chain of validators
	validators := make(secure.Validators, 0, len(jwtVals))

	for _, validatorDescriptor := range jwtVals {
		var keyResolver key.Resolver
		keyResolver, err = validatorDescriptor.Keys.NewResolver()
		if err != nil {
			validator = validators
			return
		}

		validators = append(
			validators,
			secure.JWSValidator{
				DefaultKeyId:  DEFAULT_KEY_ID,
				Resolver:      keyResolver,
				JWTValidators: []*jwt.Validator{validatorDescriptor.Custom.New()},
			},
		)
	}

	validator = validators

	return
}

// caduceus is the driver function for Caduceus.  It performs everything main() would do,
// except for obtaining the command-line arguments (which are passed to it).
func caduceus(arguments []string) int {
	var (
		f = pflag.NewFlagSet(applicationName, pflag.ContinueOnError)
		v = viper.New()

		logger, webPA, err = server.Initialize(applicationName, arguments, f, v)
	)

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to initialize Viper environment: %s\n", err)
		return 1
	}

	logger.Info("Using configuration file: %s", v.ConfigFileUsed())

	caduceusConfig := new(CaduceusConfig)
	err = v.Unmarshal(caduceusConfig)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to unmarshal configuration data into struct: %s\n", err)
		return 1
	}

	workerPool := WorkerPoolFactory{
		NumWorkers: caduceusConfig.NumWorkerThreads,
		QueueSize:  caduceusConfig.JobQueueSize,
	}.New()

	caduceusProfilerFactory := ServerProfilerFactory{
		Frequency: caduceusConfig.ProfilerFrequency,
		Duration:  caduceusConfig.ProfilerDuration,
		QueueSize: caduceusConfig.ProfilerQueueSize,
	}

	tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	timeout := time.Duration(caduceusConfig.SenderClientTimeout) * time.Second

	// declare a new sender wrapper and pass it a profiler factory so that it can create
	// unique profilers on a per outboundSender basis
	caduceusSenderWrapper, err := SenderWrapperFactory{
		NumWorkersPerSender: caduceusConfig.SenderNumWorkersPerSender,
		QueueSizePerSender:  caduceusConfig.SenderQueueSizePerSender,
		CutOffPeriod:        time.Duration(caduceusConfig.SenderCutOffPeriod) * time.Second,
		Linger:              time.Duration(caduceusConfig.SenderLinger) * time.Second,
		ProfilerFactory:     caduceusProfilerFactory,
		Logger:              logger,
		Client:              &http.Client{Transport: tr, Timeout: timeout},
	}.New()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to initialize new caduceus sender wrapper: %s\n", err)
		return 1
	}

	// here we create a profiler specifically for our main server handler
	caduceusHandlerProfiler, err := caduceusProfilerFactory.New()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to profiler for main caduceus handler: %s\n", err)
		return 1
	}

	serverWrapper := &ServerHandler{
		Logger: logger,
		caduceusHandler: &CaduceusHandler{
			handlerProfiler: caduceusHandlerProfiler,
			senderWrapper:   caduceusSenderWrapper,
			Logger:          logger,
		},
		doJob: workerPool.Send,
	}

	profileWrapper := &ProfileHandler{
		profilerData: caduceusHandlerProfiler,
		Logger:       logger,
	}

	validator, err := getValidator(v)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Validator error: %v\n", err)
		return 1
	}

	authHandler := handler.AuthorizationHandler{
		HeaderName:          "Authorization",
		ForbiddenStatusCode: 403,
		Validator:           validator,
		Logger:              logger,
	}

	caduceusHandler := alice.New(authHandler.Decorate)

	mux := mux.NewRouter()
	mux.Handle("/api/v1/notify", caduceusHandler.Then(serverWrapper))
	mux.Handle("/api/v1/profile", caduceusHandler.Then(profileWrapper))

	// Support the old endpoint too.
	mux.Handle("/api/v2/notify", caduceusHandler.Then(serverWrapper))

	webhookFactory, err := webhook.NewFactory(v)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating new webhook factory: %s\n", err)
		return 1
	}
	webhookRegistry, webhookHandler := webhookFactory.NewRegistryAndHandler()
	webhookFactory.SetExternalUpdate(caduceusSenderWrapper.Update)

	// register webhook end points for api
	mux.Handle("/hook", caduceusHandler.ThenFunc(webhookRegistry.UpdateRegistry))
	mux.Handle("/hooks", caduceusHandler.ThenFunc(webhookRegistry.GetRegistry))

	selfURL := &url.URL{
		Scheme: "https",
		Host:   v.GetString("fqdn") + v.GetString("primary.address"),
	}

	webhookFactory.Initialize(mux, selfURL, webhookHandler, logger)

	caduceusHealth := &CaduceusHealth{}
	var runnable concurrent.Runnable

	caduceusHealth.Monitor, runnable = webPA.Prepare(logger, mux)
	serverWrapper.caduceusHealth = caduceusHealth

	waitGroup, shutdown, err := concurrent.Execute(runnable)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to start device manager: %s\n", err)
		return 1
	}

	webhookFactory.PrepareAndStart()

	logger.Info("Caduceus is up and running!")

	// Attempt to obtain the current listener list from current system without having to wait for listener reregistration.
	startChan := make(chan webhook.Result, 1)
	webhookFactory.Start.GetCurrentSystemsHooks(startChan)
	var webhookStartResults webhook.Result = <-startChan
	if webhookStartResults.Error != nil {
		logger.Error(webhookStartResults.Error)
	} else {
		webhookFactory.SetList(webhook.NewList(webhookStartResults.Hooks))
		caduceusSenderWrapper.Update(webhookStartResults.Hooks)
	}

	var (
		signals = make(chan os.Signal, 1)
	)

	signal.Notify(signals)
	<-signals
	close(shutdown)
	waitGroup.Wait()

	// shutdown the sender wrapper gently so that all queued messages get serviced
	caduceusSenderWrapper.Shutdown(true)

	return 0
}

func main() {
	os.Exit(caduceus(os.Args))
}
<|endoftext|>


<commit_before>// Warren is a program to act as part of a monitoring system on a home network.
// It exports data for external programs to acquire and log to timeseries
// databases. Currently, Warren exports data in a way that is intended for
// scraping by Prometheus - http://prometheus.io/.
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"syscall"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/huin/warren/cc"
	"github.com/huin/warren/httpexport"
	"github.com/huin/warren/linux"
	"github.com/huin/warren/streammatch"
	"github.com/huin/warren/systemd"
	promm "github.com/prometheus/client_golang/prometheus"
)

var (
	configFile = flag.String("config", "", "Path to configuration file")
)

type Config struct {
	LogPath     string
	Prometheus  PrometheusConfig
	CurrentCost []cc.Config
	File        []streammatch.FileCfg
	Proc        []streammatch.ProcCfg
	System      *linux.Config
	Systemd     *systemd.Config
	HTTPExport  []httpexport.Config
}

type PrometheusConfig struct {
	HandlerPath string
	// TODO: Deprecate ServeAddr and move into Config - it's not really specific
	// to the Prometheus part of things.
	ServeAddr string
}

func initLogging(logpath string) error {
	if logpath == "" {
		// Leave default logging as STDERR.
		return nil
	}

	f, err := os.OpenFile(logpath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, syscall.S_IWUSR|syscall.S_IRUSR)
	if err != nil {
		return fmt.Errorf("failed to open log file: %v", err)
	}
	log.SetOutput(f)
	return nil
}

func readConfig(filename string) (*Config, error) {
	config := new(Config)
	_, err := toml.DecodeFile(filename, &config)
	if err != nil {
		return nil, err
	}
	return config, nil
}

func monitorLoop(name string, fn func() error) {
	for {
		if err := fn(); err != nil {
			log.Printf("%s monitoring error (restarting): %v", name, err)
		} else {
			log.Printf("%s returned without error (restarting)", name)
		}
		restartCounter.With(promm.Labels{"name": name}).Inc()
		// Avoid tightlooping on recurring failure.
		time.Sleep(5 * time.Second)
	}
}

var restartCounter *promm.CounterVec

func init() {
	restartCounter = promm.NewCounterVec(
		promm.CounterOpts{
			Namespace: "warren", Name: "running_monitor_restarts_total",
			Help: "Number of times a running monitor has restarted. (count)",
		},
		[]string{"name"},
	)
	promm.MustRegister(restartCounter)
}

func main() {
	flag.Parse()
	if *configFile == "" {
		log.Fatal("--config is required with a filename")
	}
	config, err := readConfig(*configFile)
	if err != nil {
		log.Fatal("Failed to read configuration: ", err)
	}
	initLogging(config.LogPath)

	if len(config.CurrentCost) > 0 {
		log.Printf("Starting %d CurrentCost collectors", len(config.CurrentCost))
	}
	for i, cfg := range config.CurrentCost {
		c, err := cc.New(cfg)
		if err != nil {
			log.Fatalf("Error in CurrentCost[%d]: %v", i, err)
		}
		promm.MustRegister(c)
		go monitorLoop("currentcost", c.Run)
	}

	if len(config.File) > 0 {
		log.Printf("Starting %d File collectors", len(config.File))
	}
	for i, cfg := range config.File {
		fc, err := streammatch.NewFileCollector(cfg)
		if err != nil {
			log.Fatalf("Error in File[%d]: %v", i, err)
		}
		promm.MustRegister(fc)
	}

	if len(config.Proc) > 0 {
		log.Printf("Starting %d Proc collectors", len(config.Proc))
	}
	for i, cfg := range config.Proc {
		c, err := streammatch.NewProcCollector(cfg)
		if err != nil {
			log.Fatalf("Error in Proc[%d]: %v", i, err)
		}
		promm.MustRegister(c)
	}

	if config.System != nil {
		log.Print("Starting local system monitoring")
		c, err := linux.New(*config.System)
		if err != nil {
			log.Fatalf("Error in System: %v", err)
		}
		promm.MustRegister(c)
	}

	if config.Systemd != nil {
		log.Print("Starting local systemd monitoring")
		c, err := systemd.New(*config.Systemd)
		if err != nil {
			log.Fatalf("Error in Systemd: %v", err)
		}
		promm.MustRegister(c)
	}

	if len(config.HTTPExport) > 0 {
		log.Printf("Starting %d HTTPExport collectors", len(config.HTTPExport))
	}
	for i, hec := range config.HTTPExport {
		c, err := httpexport.New(hec)
		if err != nil {
			log.Fatalf("Error in HTTPExport[%d]: %v", i, err)
		}
		promm.MustRegister(c)
	}

	log.Print("Starting Prometheus metrics handler")
	http.Handle(config.Prometheus.HandlerPath, promm.Handler())
	http.ListenAndServe(config.Prometheus.ServeAddr, nil)
}
<commit_msg>Check for unknown keys in configuration.<commit_after>// Warren is a program to act as part of a monitoring system on a home network.
// It exports data for external programs to acquire and log to timeseries
// databases. Currently, Warren exports data in a way that is intended for
// scraping by Prometheus - http://prometheus.io/.
package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"syscall"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/huin/warren/cc"
	"github.com/huin/warren/httpexport"
	"github.com/huin/warren/linux"
	"github.com/huin/warren/streammatch"
	"github.com/huin/warren/systemd"
	promm "github.com/prometheus/client_golang/prometheus"
)

var (
	configFile = flag.String("config", "", "Path to configuration file")
)

type Config struct {
	// Ignore unknown TOML keys found during parsing configuration.
	// (unknown keys will be an error in future, for now they are simply logged)
	IgnoreUnknownKeys bool `toml:"ignore_unknown_keys"`
	LogPath           string
	Prometheus        PrometheusConfig
	CurrentCost       []cc.Config
	File              []streammatch.FileCfg
	Proc              []streammatch.ProcCfg
	System            *linux.Config
	Systemd           *systemd.Config
	HTTPExport        []httpexport.Config
}

type PrometheusConfig struct {
	HandlerPath string
	// TODO: Deprecate ServeAddr and move into Config - it's not really specific
	// to the Prometheus part of things.
	ServeAddr string
}

func initLogging(logpath string) error {
	if logpath == "" {
		// Leave default logging as STDERR.
		return nil
	}

	f, err := os.OpenFile(logpath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, syscall.S_IWUSR|syscall.S_IRUSR)
	if err != nil {
		return fmt.Errorf("failed to open log file: %v", err)
	}
	log.SetOutput(f)
	return nil
}

func readConfig(filename string) (*Config, error) {
	config := new(Config)
	md, err := toml.DecodeFile(filename, &config)
	if err != nil {
		return nil, err
	}
	keys := md.Undecoded()
	if !config.IgnoreUnknownKeys && len(keys) > 0 {
		log.Printf("Found %d unknown keys in configuration file %q. This will be a fatal error in future, set `ignore_unknown_keys = true` to prevent this message or errors.", len(keys))
		for _, key := range keys {
			log.Printf("Unknown key: %q", key)
		}
	}
	return config, nil
}

func monitorLoop(name string, fn func() error) {
	for {
		if err := fn(); err != nil {
			log.Printf("%s monitoring error (restarting): %v", name, err)
		} else {
			log.Printf("%s returned without error (restarting)", name)
		}
		restartCounter.With(promm.Labels{"name": name}).Inc()
		// Avoid tightlooping on recurring failure.
		time.Sleep(5 * time.Second)
	}
}

var restartCounter *promm.CounterVec

func init() {
	restartCounter = promm.NewCounterVec(
		promm.CounterOpts{
			Namespace: "warren", Name: "running_monitor_restarts_total",
			Help: "Number of times a running monitor has restarted. (count)",
		},
		[]string{"name"},
	)
	promm.MustRegister(restartCounter)
}

func main() {
	flag.Parse()
	if *configFile == "" {
		log.Fatal("--config is required with a filename")
	}
	config, err := readConfig(*configFile)
	if err != nil {
		log.Fatal("Failed to read configuration: ", err)
	}
	initLogging(config.LogPath)

	if len(config.CurrentCost) > 0 {
		log.Printf("Starting %d CurrentCost collectors", len(config.CurrentCost))
	}
	for i, cfg := range config.CurrentCost {
		c, err := cc.New(cfg)
		if err != nil {
			log.Fatalf("Error in CurrentCost[%d]: %v", i, err)
		}
		promm.MustRegister(c)
		go monitorLoop("currentcost", c.Run)
	}

	if len(config.File) > 0 {
		log.Printf("Starting %d File collectors", len(config.File))
	}
	for i, cfg := range config.File {
		fc, err := streammatch.NewFileCollector(cfg)
		if err != nil {
			log.Fatalf("Error in File[%d]: %v", i, err)
		}
		promm.MustRegister(fc)
	}

	if len(config.Proc) > 0 {
		log.Printf("Starting %d Proc collectors", len(config.Proc))
	}
	for i, cfg := range config.Proc {
		c, err := streammatch.NewProcCollector(cfg)
		if err != nil {
			log.Fatalf("Error in Proc[%d]: %v", i, err)
		}
		promm.MustRegister(c)
	}

	if config.System != nil {
		log.Print("Starting local system monitoring")
		c, err := linux.New(*config.System)
		if err != nil {
			log.Fatalf("Error in System: %v", err)
		}
		promm.MustRegister(c)
	}

	if config.Systemd != nil {
		log.Print("Starting local systemd monitoring")
		c, err := systemd.New(*config.Systemd)
		if err != nil {
			log.Fatalf("Error in Systemd: %v", err)
		}
		promm.MustRegister(c)
	}

	if len(config.HTTPExport) > 0 {
		log.Printf("Starting %d HTTPExport collectors", len(config.HTTPExport))
	}
	for i, hec := range config.HTTPExport {
		c, err := httpexport.New(hec)
		if err != nil {
			log.Fatalf("Error in HTTPExport[%d]: %v", i, err)
		}
		promm.MustRegister(c)
	}

	log.Print("Starting Prometheus metrics handler")
	http.Handle(config.Prometheus.HandlerPath, promm.Handler())
	http.ListenAndServe(config.Prometheus.ServeAddr, nil)
}
<|endoftext|>


<commit_before>package terminal

import (
	"fmt"
	"regexp"
	"runtime"
)

type Color uint

const (
	red    Color = 31
	green        = 32
	yellow       = 33
	//	blue          = 34
	magenta = 35
	cyan    = 36
	grey    = 37
	white   = 38
)

func colorize(message string, color Color, bold bool) string {
	if runtime.GOOS == "windows" {
		return message
	}

	attr := 0
	if bold {
		attr = 1
	}

	return fmt.Sprintf("\033[%d;%dm%s\033[0m", attr, color, message)
}

func decolorize(message string) string {
	reg, err := regexp.Compile(`\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]`)
	if err != nil {
		panic(err)
	}
	return string(reg.ReplaceAll([]byte(message), []byte("")))
}

func HeaderColor(message string) string {
	return colorize(message, white, true)
}

func TableContentColor(message string) string {
	return colorize(message, grey, false)
}

func CommandColor(message string) string {
	return colorize(message, yellow, true)
}

func StartedColor(message string) string {
	return colorize(message, grey, true)
}

func StoppedColor(message string) string {
	return colorize(message, grey, true)
}

func AdvisoryColor(message string) string {
	return colorize(message, yellow, true)
}

func CrashedColor(message string) string {
	return colorize(message, red, true)
}

func FailureColor(message string) string {
	return colorize(message, red, true)
}

func SuccessColor(message string) string {
	return colorize(message, green, true)
}

func EntityNameColor(message string) string {
	return colorize(message, cyan, true)
}

func PromptColor(message string) string {
	return colorize(message, cyan, true)
}

func TableContentHeaderColor(message string) string {
	return colorize(message, cyan, true)
}

func WarningColor(message string) string {
	return colorize(message, magenta, true)
}

func LogStdoutColor(message string) string {
	return colorize(message, white, false)
}

func LogStderrColor(message string) string {
	return colorize(message, magenta, false)
}

func LogAppHeaderColor(message string) string {
	return colorize(message, yellow, true)
}

func LogSysHeaderColor(message string) string {
	return colorize(message, cyan, true)
}
<commit_msg>changing stderr color to red in order to match all other error conditions<commit_after>package terminal

import (
	"fmt"
	"regexp"
	"runtime"
)

type Color uint

const (
	red    Color = 31
	green        = 32
	yellow       = 33
	//	blue          = 34
	magenta = 35
	cyan    = 36
	grey    = 37
	white   = 38
)

func colorize(message string, color Color, bold bool) string {
	if runtime.GOOS == "windows" {
		return message
	}

	attr := 0
	if bold {
		attr = 1
	}

	return fmt.Sprintf("\033[%d;%dm%s\033[0m", attr, color, message)
}

func decolorize(message string) string {
	reg, err := regexp.Compile(`\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]`)
	if err != nil {
		panic(err)
	}
	return string(reg.ReplaceAll([]byte(message), []byte("")))
}

func HeaderColor(message string) string {
	return colorize(message, white, true)
}

func TableContentColor(message string) string {
	return colorize(message, grey, false)
}

func CommandColor(message string) string {
	return colorize(message, yellow, true)
}

func StartedColor(message string) string {
	return colorize(message, grey, true)
}

func StoppedColor(message string) string {
	return colorize(message, grey, true)
}

func AdvisoryColor(message string) string {
	return colorize(message, yellow, true)
}

func CrashedColor(message string) string {
	return colorize(message, red, true)
}

func FailureColor(message string) string {
	return colorize(message, red, true)
}

func SuccessColor(message string) string {
	return colorize(message, green, true)
}

func EntityNameColor(message string) string {
	return colorize(message, cyan, true)
}

func PromptColor(message string) string {
	return colorize(message, cyan, true)
}

func TableContentHeaderColor(message string) string {
	return colorize(message, cyan, true)
}

func WarningColor(message string) string {
	return colorize(message, magenta, true)
}

func LogStdoutColor(message string) string {
	return colorize(message, white, false)
}

func LogStderrColor(message string) string {
	return colorize(message, red, false)
}

func LogAppHeaderColor(message string) string {
	return colorize(message, yellow, true)
}

func LogSysHeaderColor(message string) string {
	return colorize(message, cyan, true)
}
<|endoftext|>