<commit_before>package couch

import (
    "strings"
    "fmt"
    "os"
    "json"
    "bytes"
    "http"
    "io/ioutil"
)

var (
    CouchDBHost = "localhost"
    CouchDBPort = "5984"
    CouchDBName = "exampledb"
    CouchDBURL  = ""
)

func init() {
    CouchDBURL = fmt.Sprintf("http://%s:%s/%s/", CouchDBHost, CouchDBPort, CouchDBName)
}

//
// Helper and utility functions (private)
//

// Replaces all instances of from with to in s (quite inefficient right now)
func replace(s, from, to string) string {
    toks := strings.SplitAfter(s, from, 0)
    newstr := ""
    for i, tok := range toks {
        if i < len(toks)-1 {
            if !strings.HasSuffix(tok, from) {
                panic("problem in replace")
            }
            newtok := tok[0 : len(tok)-len(from)]
            newstr = newstr + newtok + to
        } else {
            newstr = newstr + tok
        }
    }
    return newstr
}

// Converts given URL to string containing the body of the response.
func url_to_string(url string) string {
    if r, _, err := http.Get(url); err == nil {
        b, err := ioutil.ReadAll(r.Body)
        r.Body.Close()
        if err == nil {
            return string(b)
        }
    }
    return ""
}

// Marshal given interface to JSON string
func to_JSON(p interface{}) (string, os.Error) {
    buf := new(bytes.Buffer)
    if err := json.Marshal(buf, p); err != nil {
        return "", err
    }
    return buf.String(), nil
}

// Unmarshal JSON string to given interface
func from_JSON(s string, p interface{}) os.Error {
    if ok, errtok := json.Unmarshal(s, p); !ok {
        return os.NewError(fmt.Sprintf("error unmarshaling: %s", errtok))
    }
    return nil
}

// Since the json pkg doesn't handle fields beginning with _, we need to
// convert "_id" and "_rev" to "Id" and "Rev" to extract that data.
func temp_hack_json_id_rev(json_str string) string {
    json_str = replace(json_str, `"Id"`, `"_id"`)
    json_str = replace(json_str, `"Rev"`, `"_rev"`)
    return json_str
}

type IdAndRev struct {
    Id  string
    Rev string
}

// Simply extract id and rev from a given JSON string (typically a document)
func extract_id_and_rev(json_str string) (string, string, os.Error) {
    // this assumes the temp replacement hack has already been applied
    id_rev := new(IdAndRev)
    if err := from_JSON(json_str, id_rev); err != nil {
        return "", "", err
    }
    return id_rev.Id, id_rev.Rev, nil
}

//
// Interface functions (public)
//

type InsertResponse struct {
    Ok  bool
    Id  string
    Rev string
}

// Inserts document to CouchDB, returning id and rev on success. The document
// interface may optionally specify an "Id" field.
func Insert(p interface{}) (string, string, os.Error) {
    body_type := "application/json"
    json_str, err := to_JSON(p)
    if err != nil {
        return "", "", err
    }
    json_str = temp_hack_json_id_rev(json_str)

    r, err := http.Post(CouchDBURL, body_type, bytes.NewBufferString(json_str))
    if err != nil {
        return "", "", err
    }

    b, err := ioutil.ReadAll(r.Body)
    r.Body.Close()
    if err != nil {
        return "", "", err
    }

    ir := new(InsertResponse)
    if err := from_JSON(string(b), ir); err != nil {
        return "", "", err
    }

    if !ir.Ok {
        return "", "", os.NewError("CouchDB returned not-OK")
    }

    return ir.Id, ir.Rev, nil
}

// Unmarshals the document matching id to the given interface, returning rev.
func Retrieve(id string, p interface{}) (string, os.Error) {
    if len(id) <= 0 {
        return "", os.NewError("no id specified")
    }

    json_str := url_to_string(fmt.Sprintf("%s%s", CouchDBURL, id))
    json_str = temp_hack_json_id_rev(json_str)
    _, rev, err := extract_id_and_rev(json_str)
    if err != nil {
        return "", err
    }

    return rev, from_JSON(json_str, p)
}

// Edits the given document, which must specify both id and rev fields (as "Id"
// and "Rev"), and returns the new rev.
func Edit(p interface{}) (string, os.Error) {
    _, rev, err := Insert(p)
    return rev, err
}

<commit_msg>Add RetrieveIds for keyed view response, and dummy Delete()<commit_after>package couch

import (
    "strings"
    "fmt"
    "os"
    "json"
    "bytes"
    "http"
    "io/ioutil"
)

var (
    CouchDBHost = "localhost"
    CouchDBPort = "5984"
    CouchDBName = "exampledb"
    CouchDBURL  = ""
)

func init() {
    CouchDBURL = fmt.Sprintf("http://%s:%s/%s/", CouchDBHost, CouchDBPort, CouchDBName)
}

//
// Helper and utility functions (private)
//

// Replaces all instances of from with to in s (quite inefficient right now)
func replace(s, from, to string) string {
    toks := strings.SplitAfter(s, from, 0)
    newstr := ""
    for i, tok := range toks {
        if i < len(toks)-1 {
            if !strings.HasSuffix(tok, from) {
                panic("problem in replace")
            }
            newtok := tok[0 : len(tok)-len(from)]
            newstr = newstr + newtok + to
        } else {
            newstr = newstr + tok
        }
    }
    return newstr
}

// Converts given URL to string containing the body of the response.
func url_to_string(url string) string {
    if r, _, err := http.Get(url); err == nil {
        b, err := ioutil.ReadAll(r.Body)
        r.Body.Close()
        if err == nil {
            return string(b)
        }
    }
    return ""
}

// Marshal given interface to JSON string
func to_JSON(p interface{}) (string, os.Error) {
    buf := new(bytes.Buffer)
    if err := json.Marshal(buf, p); err != nil {
        return "", err
    }
    return buf.String(), nil
}

// Unmarshal JSON string to given interface
func from_JSON(s string, p interface{}) os.Error {
    if ok, errtok := json.Unmarshal(s, p); !ok {
        return os.NewError(fmt.Sprintf("error unmarshaling: %s", errtok))
    }
    return nil
}

// Since the json pkg doesn't handle fields beginning with _, we need to
// convert "_id" and "_rev" to "Id" and "Rev" to extract that data.
func temp_hack_json_id_rev(json_str string) string {
    json_str = replace(json_str, `"Id"`, `"_id"`)
    json_str = replace(json_str, `"Rev"`, `"_rev"`)
    return json_str
}

type IdAndRev struct {
    Id  string
    Rev string
}

// Simply extract id and rev from a given JSON string (typically a document)
func extract_id_and_rev(json_str string) (string, string, os.Error) {
    // this assumes the temp replacement hack has already been applied
    id_rev := new(IdAndRev)
    if err := from_JSON(json_str, id_rev); err != nil {
        return "", "", err
    }
    return id_rev.Id, id_rev.Rev, nil
}

//
// Interface functions (public)
//

type InsertResponse struct {
    Ok  bool
    Id  string
    Rev string
}

// Inserts document to CouchDB, returning id and rev on success. The document
// interface may optionally specify an "Id" field.
func Insert(p interface{}) (string, string, os.Error) {
    body_type := "application/json"
    json_str, err := to_JSON(p)
    if err != nil {
        return "", "", err
    }
    json_str = temp_hack_json_id_rev(json_str)

    r, err := http.Post(CouchDBURL, body_type, bytes.NewBufferString(json_str))
    if err != nil {
        return "", "", err
    }

    b, err := ioutil.ReadAll(r.Body)
    r.Body.Close()
    if err != nil {
        return "", "", err
    }

    ir := new(InsertResponse)
    if err := from_JSON(string(b), ir); err != nil {
        return "", "", err
    }

    if !ir.Ok {
        return "", "", os.NewError("CouchDB returned not-OK")
    }

    return ir.Id, ir.Rev, nil
}

// Unmarshals the document matching id to the given interface, returning rev.
func Retrieve(id string, p interface{}) (string, os.Error) {
    if len(id) <= 0 {
        return "", os.NewError("no id specified")
    }

    json_str := url_to_string(fmt.Sprintf("%s%s", CouchDBURL, id))
    json_str = temp_hack_json_id_rev(json_str)
    _, rev, err := extract_id_and_rev(json_str)
    if err != nil {
        return "", err
    }

    return rev, from_JSON(json_str, p)
}

// Edits the given document, which must specify both id and rev fields (as "Id"
// and "Rev"), and returns the new rev.
func Edit(p interface{}) (string, os.Error) {
    _, rev, err := Insert(p)
    return rev, err
}

//
func Delete(id, rev string) os.Error {
    return os.NewError("delete not yet implemented")
}

type Row struct {
    Id  string
    Key string
}

type KeyedViewResponse struct {
    Total_rows uint64
    Offset     uint64
    Rows       []Row
}

// Return array of document ids as returned by the given view, by given key.
func RetrieveIds(view, key string) []string {
    // view should be eg. "_design/my_foo/_view/my_bar"
    if len(view) <= 0 || len(key) <= 0 {
        return make([]string, 0)
    }

    full_url := fmt.Sprintf(`%s%s?key="%s"`, CouchDBURL, view, key)
    json_str := url_to_string(full_url)
    kvr := new(KeyedViewResponse)
    if err := from_JSON(json_str, kvr); err != nil {
        return make([]string, 0)
    }

    ids := make([]string, len(kvr.Rows))
    for i, row := range kvr.Rows {
        ids[i] = row.Id
    }
    return ids
}
<|endoftext|>


<commit_before>package commitlog

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"

	"github.com/pkg/errors"
)

type CommitLog struct {
	Options
	name           string
	mu             sync.RWMutex
	segments       []*Segment
	vActiveSegment atomic.Value
}

type Options struct {
	Path         string
	SegmentBytes int64
}

func New(opts Options) (*CommitLog, error) {
	if opts.Path == "" {
		return nil, errors.New("path is empty")
	}

	if opts.SegmentBytes == 0 {
		// TODO default here
	}

	path, _ := filepath.Abs(opts.Path)
	l := &CommitLog{
		Options: opts,
		name:    filepath.Base(path),
	}

	return l, nil
}

func (l *CommitLog) Init() error {
	err := os.MkdirAll(l.Path, 0755)
	if err != nil {
		return errors.Wrap(err, "mkdir failed")
	}
	return nil
}

func (l *CommitLog) Open() error {
	_, err := ioutil.ReadDir(l.Path)
	if err != nil {
		return errors.Wrap(err, "read dir failed")
	}

	activeSegment, err := NewSegment(l.Path, 0, l.SegmentBytes)
	if err != nil {
		return err
	}
	l.vActiveSegment.Store(activeSegment)

	l.segments = append(l.segments, activeSegment)

	return nil
}

func (l *CommitLog) DeleteAll() error {
	return os.RemoveAll(l.Path)
}

func (l *CommitLog) Append(ms MessageSet) error {
	l.mu.Lock()
	defer l.mu.Unlock()
	if l.checkSplit() {
		if err := l.split(); err != nil {
			return err
		}
	}
	position := l.activeSegment().Position
	if _, err := l.activeSegment().Write(ms); err != nil {
		return err
	}
	e := Entry{
		Offset:   ms.Offset(),
		Position: position,
	}
	if err := l.activeSegment().Index.WriteEntry(e); err != nil {
		return err
	}
	return nil
}

func (l *CommitLog) Read(p []byte) (n int, err error) {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.activeSegment().Read(p)
}

func (l *CommitLog) checkSplit() bool {
	return l.activeSegment().IsFull()
}

func (l *CommitLog) split() error {
	seg, err := NewSegment(l.Path, l.newestOffset(), l.SegmentBytes)
	if err != nil {
		return err
	}
	l.segments = append(l.segments, seg)
	l.vActiveSegment.Store(seg)
	return nil
}

func (l *CommitLog) newestOffset() int64 {
	return l.activeSegment().NextOffset
}

func (l *CommitLog) activeSegment() *Segment {
	return l.vActiveSegment.Load().(*Segment)
}
<commit_msg>segment message set offset on write<commit_after>package commitlog

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"sync/atomic"

	"github.com/pkg/errors"
)

type CommitLog struct {
	Options
	name           string
	mu             sync.RWMutex
	segments       []*Segment
	vActiveSegment atomic.Value
}

type Options struct {
	Path         string
	SegmentBytes int64
}

func New(opts Options) (*CommitLog, error) {
	if opts.Path == "" {
		return nil, errors.New("path is empty")
	}

	if opts.SegmentBytes == 0 {
		// TODO default here
	}

	path, _ := filepath.Abs(opts.Path)
	l := &CommitLog{
		Options: opts,
		name:    filepath.Base(path),
	}

	return l, nil
}

func (l *CommitLog) Init() error {
	err := os.MkdirAll(l.Path, 0755)
	if err != nil {
		return errors.Wrap(err, "mkdir failed")
	}
	return nil
}

func (l *CommitLog) Open() error {
	_, err := ioutil.ReadDir(l.Path)
	if err != nil {
		return errors.Wrap(err, "read dir failed")
	}

	activeSegment, err := NewSegment(l.Path, 0, l.SegmentBytes)
	if err != nil {
		return err
	}
	l.vActiveSegment.Store(activeSegment)

	l.segments = append(l.segments, activeSegment)

	return nil
}

func (l *CommitLog) DeleteAll() error {
	return os.RemoveAll(l.Path)
}

func (l *CommitLog) Append(ms MessageSet) error {
	l.mu.Lock()
	defer l.mu.Unlock()
	if l.checkSplit() {
		if err := l.split(); err != nil {
			return err
		}
	}
	position := l.activeSegment().Position
	offset := l.activeSegment().NextOffset
	ms.PutOffset(offset)
	if _, err := l.activeSegment().Write(ms); err != nil {
		return err
	}
	e := Entry{
		Offset:   offset,
		Position: position,
	}
	if err := l.activeSegment().Index.WriteEntry(e); err != nil {
		return err
	}
	return nil
}

func (l *CommitLog) Read(p []byte) (n int, err error) {
	l.mu.Lock()
	defer l.mu.Unlock()
	return l.activeSegment().Read(p)
}

func (l *CommitLog) checkSplit() bool {
	return l.activeSegment().IsFull()
}

func (l *CommitLog) split() error {
	seg, err := NewSegment(l.Path, l.newestOffset(), l.SegmentBytes)
	if err != nil {
		return err
	}
	l.segments = append(l.segments, seg)
	l.vActiveSegment.Store(seg)
	return nil
}

func (l *CommitLog) newestOffset() int64 {
	return l.activeSegment().NextOffset
}

func (l *CommitLog) activeSegment() *Segment {
	return l.vActiveSegment.Load().(*Segment)
}
<|endoftext|>


<commit_before>package commonlib

// secretshare - share secrets securely
// Copyright (C) 2016  Alexander Wauck
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

import (
	"io"

	"crypto/aes"
	"crypto/cipher"
)

type Decrypter struct {
	stream io.Reader
	key []byte
	cbc cipher.BlockMode
	paddingRemaining byte
	paddingLen byte
	nextBlock []byte
	blockPos int
	messageSize int64
	totalRead int64
}

func NewDecrypter(stream io.Reader, messageSize int64, key []byte) (*Decrypter, error) {
	paddingLen := messageSize % aes.BlockSize
	DEBUGPrintf("Decrypter: Number of bytes in last block: %d\n", paddingLen)
	if paddingLen > 0 {
		paddingLen = aes.BlockSize - paddingLen
		DEBUGPrintf("Decrypter: Calculated padding length of %d\n", paddingLen)
	}
	if paddingLen > 255 {
		return nil, BadBlockSizeError
	}

	headerData := make([]byte, 1 + aes.BlockSize)
	bytesRead, err := stream.Read(headerData)
	if bytesRead < 1 + aes.BlockSize {
		return nil, io.ErrUnexpectedEOF
	}
	if headerData[0] != byte(paddingLen) {
		return nil, DataCorruptionError
	}
	iv := headerData[1:]

	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	cbc := cipher.NewCBCDecrypter(block, iv)

	decrypter := &Decrypter{
		stream: stream,
		key: key,
		cbc: cbc,
		paddingRemaining: byte(paddingLen),
		paddingLen: byte(paddingLen),
		nextBlock: nil,
		blockPos: 0,
		messageSize: messageSize,
		totalRead: 0,
	}
	err = decrypter.readBlock()
	if err != nil && err != io.EOF {
		return nil, err
	}
	return decrypter, nil

}

func (self *Decrypter) readBlock() error {
	self.nextBlock = make([]byte, aes.BlockSize)
	//DEBUGPrintf("Made a new block of size %d\n", aes.BlockSize)
	self.blockPos = 0
	target := self.nextBlock
	totalBytesRead := 0
	readAll := false
	defer func() {
		if readAll {
			self.cbc.CryptBlocks(self.nextBlock, self.nextBlock)
		} else {
			self.nextBlock = nil
		}
	}()

	for !readAll {
		bytesRead, err := self.stream.Read(target)
		if err != nil && err != io.EOF {
			return err
		}
		target = target[bytesRead:]
		totalBytesRead += bytesRead
		if err == io.EOF {
			if totalBytesRead == 0 {
				DEBUGPrintln("Decrypter: readblock: EOF!")
				return io.EOF
			}
			if totalBytesRead == len(self.nextBlock) {
				DEBUGPrintln("Decrypter: readblock: EOF!")
				readAll = true
				return io.EOF
			}
			return DataCorruptionError
		}
		if totalBytesRead == aes.BlockSize {
			readAll = true
		}
	}

	return nil
}

func (self *Decrypter) Read(p []byte) (int, error) {
	var eof error = nil
	bytesWritten := 0
	defer func() {
		if bytesWritten == 0 {
			self.nextBlock = nil
		}
	}()

	for bytesWritten < len(p) && self.nextBlock != nil {
		for bytesWritten < len(p) && self.blockPos < len(self.nextBlock) {
			p[bytesWritten] = self.nextBlock[self.blockPos]
			bytesWritten++
			self.blockPos++
			self.totalRead++
			//DEBUGPrintf("Wrote %d/%d bytes (block)\n", bytesWritten, len(p))
			if self.totalRead == self.messageSize {
				self.nextBlock = nil
				return bytesWritten, io.EOF
			}
		}
		if self.blockPos >= len(self.nextBlock) {
			err := self.readBlock()
			if err != nil {
				if err != io.EOF {
					return bytesWritten, err
				} else {
					eof = err
				}
			}
		}
	}

	DEBUGPrintf("Decrypter: Wrote %d bytes total\n", bytesWritten)
	if eof != nil {
		DEBUGPrintf("Decrypter: Hit EOF!\n")
	} else if bytesWritten == 0 {
		return bytesWritten, DecrypterWeirdEOFError
	}
	return bytesWritten, eof
}
<commit_msg>More debug output<commit_after>package commonlib

// secretshare - share secrets securely
// Copyright (C) 2016  Alexander Wauck
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

import (
	"io"

	"crypto/aes"
	"crypto/cipher"
)

type Decrypter struct {
	stream io.Reader
	key []byte
	cbc cipher.BlockMode
	paddingRemaining byte
	paddingLen byte
	nextBlock []byte
	blockPos int
	messageSize int64
	totalRead int64
}

func NewDecrypter(stream io.Reader, messageSize int64, key []byte) (*Decrypter, error) {
	paddingLen := messageSize % aes.BlockSize
	DEBUGPrintf("Decrypter: Number of bytes in last block: %d\n", paddingLen)
	if paddingLen > 0 {
		paddingLen = aes.BlockSize - paddingLen
		DEBUGPrintf("Decrypter: Calculated padding length of %d\n", paddingLen)
	}
	if paddingLen > 255 {
		return nil, BadBlockSizeError
	}

	headerData := make([]byte, 1 + aes.BlockSize)
	bytesRead, err := stream.Read(headerData)
	if bytesRead < 1 + aes.BlockSize {
		return nil, io.ErrUnexpectedEOF
	}
	if headerData[0] != byte(paddingLen) {
		DEBUGPrintf("Decrypter: header data says padding is %d, but we say it's %d\n", headerData[0], paddingLen)
		return nil, DataCorruptionError
	}
	iv := headerData[1:]

	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	cbc := cipher.NewCBCDecrypter(block, iv)

	decrypter := &Decrypter{
		stream: stream,
		key: key,
		cbc: cbc,
		paddingRemaining: byte(paddingLen),
		paddingLen: byte(paddingLen),
		nextBlock: nil,
		blockPos: 0,
		messageSize: messageSize,
		totalRead: 0,
	}
	err = decrypter.readBlock()
	if err != nil && err != io.EOF {
		return nil, err
	}
	return decrypter, nil

}

func (self *Decrypter) readBlock() error {
	self.nextBlock = make([]byte, aes.BlockSize)
	//DEBUGPrintf("Made a new block of size %d\n", aes.BlockSize)
	self.blockPos = 0
	target := self.nextBlock
	totalBytesRead := 0
	readAll := false
	defer func() {
		if readAll {
			self.cbc.CryptBlocks(self.nextBlock, self.nextBlock)
		} else {
			self.nextBlock = nil
		}
	}()

	for !readAll {
		bytesRead, err := self.stream.Read(target)
		if err != nil && err != io.EOF {
			return err
		}
		target = target[bytesRead:]
		totalBytesRead += bytesRead
		if err == io.EOF {
			if totalBytesRead == 0 {
				DEBUGPrintln("Decrypter: readblock: EOF!")
				return io.EOF
			}
			if totalBytesRead == len(self.nextBlock) {
				DEBUGPrintln("Decrypter: readblock: EOF!")
				readAll = true
				return io.EOF
			}
			return DataCorruptionError
		}
		if totalBytesRead == aes.BlockSize {
			readAll = true
		}
	}

	return nil
}

func (self *Decrypter) Read(p []byte) (int, error) {
	var eof error = nil
	bytesWritten := 0
	defer func() {
		if bytesWritten == 0 {
			self.nextBlock = nil
		}
	}()

	for bytesWritten < len(p) && self.nextBlock != nil {
		for bytesWritten < len(p) && self.blockPos < len(self.nextBlock) {
			p[bytesWritten] = self.nextBlock[self.blockPos]
			bytesWritten++
			self.blockPos++
			self.totalRead++
			//DEBUGPrintf("Wrote %d/%d bytes (block)\n", bytesWritten, len(p))
			if self.totalRead == self.messageSize {
				self.nextBlock = nil
				return bytesWritten, io.EOF
			}
		}
		if self.blockPos >= len(self.nextBlock) {
			err := self.readBlock()
			if err != nil {
				if err != io.EOF {
					return bytesWritten, err
				} else {
					eof = err
				}
			}
		}
	}

	DEBUGPrintf("Decrypter: Wrote %d bytes total\n", bytesWritten)
	if eof != nil {
		DEBUGPrintf("Decrypter: Hit EOF!\n")
	} else if bytesWritten == 0 {
		return bytesWritten, DecrypterWeirdEOFError
	}
	return bytesWritten, eof
}
<|endoftext|>


<commit_before>package gparselib

import (
	"testing"
)

func MakeParseLiteral(literal string) SubparserOp {
	return func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseLiteral(pd, ctx, nil, literal)
	}
}

func TestParseOptional(t *testing.T) {
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseOptional(pd, ctx, MakeParseLiteral("flow"), nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("no match", 0, " flow"),
			expectedResult:   newResult(0, "", nil, -1),
			expectedSrcPos:   0,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("1 match", 0, "flow"),
			expectedResult:   newResult(0, "flow", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("2 matches", 0, "flowflow"),
			expectedResult:   newResult(0, "flow", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		},
	})
}

func TestParseMulti0(t *testing.T) {
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti0(pd, ctx, MakeParseLiteral("flow"), nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("no match", 0, " flow"),
			expectedResult:   newResult(0, "", []interface{}{}, -1),
			expectedSrcPos:   0,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("1 match", 0, "flow"),
			expectedResult:   newResult(0, "flow", []interface{}{nil}, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("2 matches", 0, "flowflow"),
			expectedResult:   newResult(0, "flowflow", []interface{}{nil, nil}, -1),
			expectedSrcPos:   8,
			expectedErrCount: 0,
		},
	})
}

func TestParseMulti1(t *testing.T) {
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti1(pd, ctx, MakeParseLiteral("flow"), nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("no match", 0, " flow"),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 2,
		}, {
			givenParseData:   newData("1 match", 0, "flow"),
			expectedResult:   newResult(0, "flow", []interface{}{nil}, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("2 matches", 0, "flowflow"),
			expectedResult:   newResult(0, "flowflow", []interface{}{nil, nil}, -1),
			expectedSrcPos:   8,
			expectedErrCount: 0,
		},
	})
}

func TestParseMulti(t *testing.T) {
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti(pd, ctx, MakeParseLiteral("flow"), nil, 2, 3)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("multi(2-3): 1 match", 0, "flow flow"),
			expectedResult:   newResult(0, "", nil, 4),
			expectedSrcPos:   0,
			expectedErrCount: 2,
		}, {
			givenParseData:   newData("multi(2-3): 2 matches", 0, "flowflow"),
			expectedResult:   newResult(0, "flowflow", []interface{}{nil, nil}, -1),
			expectedSrcPos:   8,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("multi(2-3): 3 matches", 0, "flowflowflow"),
			expectedResult:   newResult(0, "flowflowflow", []interface{}{nil, nil, nil}, -1),
			expectedSrcPos:   12,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("multi(2-3): 4 matches", 0, "flowflowflowflow"),
			expectedResult:   newResult(0, "flowflowflow", []interface{}{nil, nil, nil}, -1),
			expectedSrcPos:   12,
			expectedErrCount: 0,
		},
	})
}

func TestParseMultiNested(t *testing.T) {
	pm2to3 := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti(pd, ctx, MakeParseLiteral("flow"), nil, 2, 3)
	}
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti1(pd, ctx, pm2to3, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData: newData(
				"multi(2-3) in multi(1-n): 6 match",
				0,
				"flowflowflowflowflowflowflow",
			),
			expectedResult: newResult(
				0,
				"flowflowflowflowflowflow",
				[]interface{}{
					[]interface{}{nil, nil, nil},
					[]interface{}{nil, nil, nil}},
				-1,
			),
			expectedSrcPos:   24,
			expectedErrCount: 0,
		},
	})
}

func TestParseAll_NormalFunctionality(t *testing.T) {
	plFlow := MakeParseLiteral("flow")
	plNo := MakeParseLiteral("no")
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAll(pd, ctx, []SubparserOp{plFlow, plNo}, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("empty", 0, ""),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 1,
		}, {
			givenParseData:   newData("no match", 0, " flow no"),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 1,
		}, {
			givenParseData:   newData("match flow", 0, "flowabc"),
			expectedResult:   newResult(0, "", nil, 4),
			expectedSrcPos:   0,
			expectedErrCount: 1,
		}, {
			givenParseData:   newData("match no", 3, "123noabc"),
			expectedResult:   newResult(3, "", nil, 3),
			expectedSrcPos:   3,
			expectedErrCount: 1,
		}, {
			givenParseData:   newData("match all", 3, "123flownoabc"),
			expectedResult:   newResult(3, "flowno", []interface{}{nil, nil}, -1),
			expectedSrcPos:   9,
			expectedErrCount: 0,
		},
	})
}

func TestParseAll_ManySubs(t *testing.T) {
	pl1 := MakeParseLiteral("1")
	pl2 := MakeParseLiteral("2")
	pl3 := MakeParseLiteral("3")
	pl4 := MakeParseLiteral("4")
	pl5 := MakeParseLiteral("5")
	pl6 := MakeParseLiteral("6")
	pl7 := MakeParseLiteral("7")
	pl8 := MakeParseLiteral("8")
	pl9 := MakeParseLiteral("9")
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAll(pd, ctx,
			[]SubparserOp{pl1, pl2, pl3, pl4, pl5, pl6, pl7, pl8, pl9}, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData: newData("match 9", 0, "1234567890"),
			expectedResult: newResult(
				0,
				"123456789",
				[]interface{}{nil, nil, nil, nil, nil, nil, nil, nil, nil},
				-1,
			),
			expectedSrcPos:   9,
			expectedErrCount: 0,
		},
	})
}

func TestParseAll_Nested(t *testing.T) {
	plFlow := MakeParseLiteral("flow")
	plNo := MakeParseLiteral("no")
	pInner := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAll(pd, ctx, []SubparserOp{plFlow, plNo}, nil)
	}
	plFun := MakeParseLiteral("fun")
	pOuter := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAll(pd, ctx, []SubparserOp{plFun, pInner}, nil)
	}

	runTests(t, pOuter, []parseTestData{
		{
			givenParseData: newData("match nested", 3, "123funflownoabc"),
			expectedResult: newResult(
				3,
				"funflowno",
				[]interface{}{nil, []interface{}{nil, nil}},
				-1,
			),
			expectedSrcPos:   12,
			expectedErrCount: 0,
		},
	})
}

func TestParseAny_NormalFunctionality(t *testing.T) {
	plFlow := MakeParseLiteral("flow")
	plNo := MakeParseLiteral("no")
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAny(pd, ctx, []SubparserOp{plFlow, plNo}, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("empty", 0, ""),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 3,
		}, {
			givenParseData:   newData("no match", 0, " flow 3"),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 3,
		}, {
			givenParseData:   newData("match flow", 0, "flowabc"),
			expectedResult:   newResult(0, "flow", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match no", 2, "12noabc"),
			expectedResult:   newResult(2, "no", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match both", 3, "123flownoabc"),
			expectedResult:   newResult(3, "flow", nil, -1),
			expectedSrcPos:   7,
			expectedErrCount: 0,
		},
	})
}

func TestParseAny_Nested(t *testing.T) {
	plFlow := MakeParseLiteral("flow")
	plNo := MakeParseLiteral("no")
	pInner := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAny(pd, ctx, []SubparserOp{plFlow, plNo}, nil)
	}

	plFun := MakeParseLiteral("fun")
	pOuter := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAny(pd, ctx, []SubparserOp{plFun, pInner}, nil)
	}

	runTests(t, pOuter, []parseTestData{
		{
			givenParseData:   newData("match flow", 3, "123flowabc"),
			expectedResult:   newResult(3, "flow", nil, -1),
			expectedSrcPos:   7,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match fun", 3, "123funabc"),
			expectedResult:   newResult(3, "fun", nil, -1),
			expectedSrcPos:   6,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match no", 3, "123noabc"),
			expectedResult:   newResult(3, "no", nil, -1),
			expectedSrcPos:   5,
			expectedErrCount: 0,
		},
	})
}
<commit_msg>Ensure full test coverage<commit_after>package gparselib

import (
	"testing"
)

func MakeParseLiteral(literal string) SubparserOp {
	return func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseLiteral(pd, ctx, nil, literal)
	}
}

func TestParseOptional(t *testing.T) {
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseOptional(pd, ctx, MakeParseLiteral("flow"), nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("no match", 0, " flow"),
			expectedResult:   newResult(0, "", nil, -1),
			expectedSrcPos:   0,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("1 match", 0, "flow"),
			expectedResult:   newResult(0, "flow", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("2 matches", 0, "flowflow"),
			expectedResult:   newResult(0, "flow", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		},
	})
}

func TestParseMulti0(t *testing.T) {
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti0(pd, ctx, MakeParseLiteral("flow"), nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("no match", 0, " flow"),
			expectedResult:   newResult(0, "", []interface{}{}, -1),
			expectedSrcPos:   0,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("1 match", 0, "flow"),
			expectedResult:   newResult(0, "flow", []interface{}{nil}, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("2 matches", 0, "flowflow"),
			expectedResult:   newResult(0, "flowflow", []interface{}{nil, nil}, -1),
			expectedSrcPos:   8,
			expectedErrCount: 0,
		},
	})
}

func TestParseMulti1(t *testing.T) {
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti1(pd, ctx, MakeParseLiteral("flow"), nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("no match", 0, " flow"),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 2,
		}, {
			givenParseData:   newData("1 match", 0, "flow"),
			expectedResult:   newResult(0, "flow", []interface{}{nil}, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("2 matches", 0, "flowflow"),
			expectedResult:   newResult(0, "flowflow", []interface{}{nil, nil}, -1),
			expectedSrcPos:   8,
			expectedErrCount: 0,
		},
	})
}

func TestParseMulti(t *testing.T) {
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti(pd, ctx, MakeParseLiteral("flow"), nil, 2, 3)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("multi(2-3): 1 match", 0, "flow flow"),
			expectedResult:   newResult(0, "", nil, 4),
			expectedSrcPos:   0,
			expectedErrCount: 2,
		}, {
			givenParseData:   newData("multi(2-3): 2 matches", 0, "flowflow"),
			expectedResult:   newResult(0, "flowflow", []interface{}{nil, nil}, -1),
			expectedSrcPos:   8,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("multi(2-3): 3 matches", 0, "flowflowflow"),
			expectedResult:   newResult(0, "flowflowflow", []interface{}{nil, nil, nil}, -1),
			expectedSrcPos:   12,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("multi(2-3): 4 matches", 0, "flowflowflowflow"),
			expectedResult:   newResult(0, "flowflowflow", []interface{}{nil, nil, nil}, -1),
			expectedSrcPos:   12,
			expectedErrCount: 0,
		},
	})
}

func TestParseMultiNested(t *testing.T) {
	pm2to3 := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti(pd, ctx, MakeParseLiteral("flow"), nil, 2, 3)
	}
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseMulti1(pd, ctx, pm2to3, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData: newData(
				"multi(2-3) in multi(1-n): 6 match",
				0,
				"flowflowflowflowflowflowflow",
			),
			expectedResult: newResult(
				0,
				"flowflowflowflowflowflow",
				[]interface{}{
					[]interface{}{nil, nil, nil},
					[]interface{}{nil, nil, nil}},
				-1,
			),
			expectedSrcPos:   24,
			expectedErrCount: 0,
		},
	})
}

func TestParseAll_NormalFunctionality(t *testing.T) {
	plFlow := MakeParseLiteral("flow")
	plNo := MakeParseLiteral("no")
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAll(pd, ctx, []SubparserOp{plFlow, plNo}, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("empty", 0, ""),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 1,
		}, {
			givenParseData:   newData("no match", 0, " flow no"),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 1,
		}, {
			givenParseData:   newData("match flow", 0, "flowabc"),
			expectedResult:   newResult(0, "", nil, 4),
			expectedSrcPos:   0,
			expectedErrCount: 1,
		}, {
			givenParseData:   newData("match no", 3, "123noabc"),
			expectedResult:   newResult(3, "", nil, 3),
			expectedSrcPos:   3,
			expectedErrCount: 1,
		}, {
			givenParseData:   newData("match all", 3, "123flownoabc"),
			expectedResult:   newResult(3, "flowno", []interface{}{nil, nil}, -1),
			expectedSrcPos:   9,
			expectedErrCount: 0,
		},
	})
}

func TestParseAll_ManySubs(t *testing.T) {
	pl1 := MakeParseLiteral("1")
	pl2 := MakeParseLiteral("2")
	pl3 := MakeParseLiteral("3")
	pl4 := MakeParseLiteral("4")
	pl5 := MakeParseLiteral("5")
	pl6 := MakeParseLiteral("6")
	pl7 := MakeParseLiteral("7")
	pl8 := MakeParseLiteral("8")
	pl9 := MakeParseLiteral("9")
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAll(pd, ctx,
			[]SubparserOp{pl1, pl2, pl3, pl4, pl5, pl6, pl7, pl8, pl9}, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData: newData("match 9", 0, "1234567890"),
			expectedResult: newResult(
				0,
				"123456789",
				[]interface{}{nil, nil, nil, nil, nil, nil, nil, nil, nil},
				-1,
			),
			expectedSrcPos:   9,
			expectedErrCount: 0,
		},
	})
}

func TestParseAll_Nested(t *testing.T) {
	plFlow := MakeParseLiteral("flow")
	plNo := MakeParseLiteral("no")
	pInner := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAll(pd, ctx, []SubparserOp{plFlow, plNo}, nil)
	}
	plFun := MakeParseLiteral("fun")
	pOuter := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAll(pd, ctx, []SubparserOp{plFun, pInner}, nil)
	}

	runTests(t, pOuter, []parseTestData{
		{
			givenParseData: newData("match nested", 3, "123funflownoabc"),
			expectedResult: newResult(
				3,
				"funflowno",
				[]interface{}{nil, []interface{}{nil, nil}},
				-1,
			),
			expectedSrcPos:   12,
			expectedErrCount: 0,
		},
	})
}

func TestParseAny_NormalFunctionality(t *testing.T) {
	plFlow := MakeParseLiteral("flow")
	plNo := MakeParseLiteral("no")
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAny(pd, ctx, []SubparserOp{plFlow, plNo}, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("empty", 0, ""),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 3,
		}, {
			givenParseData:   newData("no match", 0, " flow 3"),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 3,
		}, {
			givenParseData:   newData("match flow", 0, "flowabc"),
			expectedResult:   newResult(0, "flow", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match no", 2, "12noabc"),
			expectedResult:   newResult(2, "no", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match both", 3, "123flownoabc"),
			expectedResult:   newResult(3, "flow", nil, -1),
			expectedSrcPos:   7,
			expectedErrCount: 0,
		},
	})
}

func TestParseAny_Nested(t *testing.T) {
	plFlow := MakeParseLiteral("flow")
	plNo := MakeParseLiteral("no")
	pInner := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAny(pd, ctx, []SubparserOp{plFlow, plNo}, nil)
	}

	plFun := MakeParseLiteral("fun")
	pOuter := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseAny(pd, ctx, []SubparserOp{plFun, pInner}, nil)
	}

	runTests(t, pOuter, []parseTestData{
		{
			givenParseData:   newData("match flow", 3, "123flowabc"),
			expectedResult:   newResult(3, "flow", nil, -1),
			expectedSrcPos:   7,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match fun", 3, "123funabc"),
			expectedResult:   newResult(3, "fun", nil, -1),
			expectedSrcPos:   6,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match no", 3, "123noabc"),
			expectedResult:   newResult(3, "no", nil, -1),
			expectedSrcPos:   5,
			expectedErrCount: 0,
		},
	})
}

func TestParseBest_NormalFunctionality(t *testing.T) {
	plFlo := MakeParseLiteral("flo")
	plFlow := MakeParseLiteral("flow")
	p := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseBest(pd, ctx, []SubparserOp{plFlo, plFlow}, nil)
	}

	runTests(t, p, []parseTestData{
		{
			givenParseData:   newData("empty", 0, ""),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 3,
		}, {
			givenParseData:   newData("no match", 0, " flow 3"),
			expectedResult:   newResult(0, "", nil, 0),
			expectedSrcPos:   0,
			expectedErrCount: 3,
		}, {
			givenParseData:   newData("match flo", 2, "12floabc"),
			expectedResult:   newResult(2, "flo", nil, -1),
			expectedSrcPos:   5,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match flow", 0, "flowabc"),
			expectedResult:   newResult(0, "flow", nil, -1),
			expectedSrcPos:   4,
			expectedErrCount: 0,
		},
	})
}

func TestParseBest_Nested(t *testing.T) {
	plFlo := MakeParseLiteral("flo")
	plFlow := MakeParseLiteral("flow")
	pInner := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseBest(pd, ctx, []SubparserOp{plFlo, plFlow}, nil)
	}

	plFl := MakeParseLiteral("fl")
	pOuter := func(pd *ParseData, ctx interface{}) (*ParseData, interface{}) {
		return ParseBest(pd, ctx, []SubparserOp{plFl, pInner}, nil)
	}

	runTests(t, pOuter, []parseTestData{
		{
			givenParseData:   newData("match flow", 3, "123flowabc"),
			expectedResult:   newResult(3, "flow", nil, -1),
			expectedSrcPos:   7,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match flo", 3, "123floabc"),
			expectedResult:   newResult(3, "flo", nil, -1),
			expectedSrcPos:   6,
			expectedErrCount: 0,
		}, {
			givenParseData:   newData("match fl", 3, "123flabc"),
			expectedResult:   newResult(3, "fl", nil, -1),
			expectedSrcPos:   5,
			expectedErrCount: 0,
		},
	})
}
<|endoftext|>


<commit_before>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "strconv"
  "strings"
  "net/textproto"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store *ArticleStore
  database Database
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  // http frontend
  frontend Frontend

  // nntp feeds map, feed, isoutbound
  feeds map[NNTPConnection]bool
  infeed chan *NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their message id
  send_all_feeds chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  feed := NNTPConnection{conn, textproto.NewConn(conn), inbound, self.debug, new(ConnectionInfo), policy, make(chan *NNTPMessage, 64),  make(chan string, 512)}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// sync every article to all feeds
func (self *NNTPDaemon) syncAll() {
  log.Println("sync all feeds")
  chnl := make(chan string, 128)
  go self.store.IterateAllArticles(chnl)
  for {
    select {
    case msgid := <- chnl:
      for feed, use := range self.feeds {
        if use {
          feed.sync <- msgid
        }
      }
    }
  }
}

// run daemon
func (self *NNTPDaemon) Run() {
  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return
  }
  defer self.listener.Close()

  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  // start accepting incoming connections
  go self.acceptloop()

  if self.sync_on_start {
    go self.syncAll()
  }
  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  self.pollfeeds()

}

func (self *NNTPDaemon) pollfrontend() {
  for {

    select {
    case nntp := <- self.frontend.NewPostsChan():
      // new post from frontend
      // ammend path
      nntp.Path = self.instance_name + "!" + nntp.Path
      // tell infeed that we got one
      self.infeed <- nntp
      break
    }
  }
}

func (self *NNTPDaemon) pollfeeds() {
  for {
    select {
    case msgid := <- self.send_all_feeds:
      // send all feeds
      nntp := self.store.GetMessage(msgid)
      if nntp == nil {
        log.Printf("failed to load %s for federation", msgid)
      } else {
        for feed , use := range self.feeds {
          if use && feed.policy != nil {
            if feed.policy.AllowsNewsgroup(nntp.Newsgroup) {
              feed.send <- nntp
            }
          }
        }
      }
      break
    case nntp := <- self.infeed:
      // register article
      self.database.RegisterArticle(nntp)
      if nntp.VerifySignature() {
        // store article
        self.store.StorePost(nntp)
        // queue to all outfeeds
        self.send_all_feeds <- nntp.MessageID
      } else {
        log.Printf("%s has invalid signature", nntp.MessageID)
      }

      break
    case msgid := <- self.infeed_load:
      // load temp message
      // this deletes the temp file
      nntp := self.store.ReadTempMessage(msgid)
      // rewrite path header
      nntp.Path = self.instance_name +"!" + nntp.Path
      // offer infeed
      self.infeed <- nntp
      break
    }
  }
}

func (self *NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunInbound(nntp)
  }
}

func (self *NNTPDaemon) RunInbound(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Println("SRNd NNTPD bound at", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {
  CheckConfig()
  log.Println("load config")
  self.conf = ReadConf()
  if self.conf == nil {
    log.Println("cannot load config")
    return false
  }
  self.infeed = make(chan *NNTPMessage, 64)
  self.infeed_load = make(chan string, 256)
  self.send_all_feeds = make(chan string, 64)
  self.feeds = make(map[NNTPConnection]bool)

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  self.database.CreateTables()

  self.store = new(ArticleStore)
  self.store.directory = self.conf.store["store_dir"]
  self.store.temp = self.conf.store["incoming_dir"]
  self.store.database = self.database
  self.store.Init()

  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  if self.sync_on_start {
    log.Println("sync on start")
  }
  self.bind_addr = self.conf.daemon["bind"]
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    self.frontend = NewHTTPFrontend(self, self.conf.frontend)
    go self.frontend.Mainloop()
  }

  return true
}
<commit_msg>move infeed offering to the same select block<commit_after>//
// daemon.go
//
package srnd
import (
  "log"
  "net"
  "strconv"
  "strings"
  "net/textproto"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store *ArticleStore
  database Database
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  // http frontend
  frontend Frontend

  // nntp feeds map, feed, isoutbound
  feeds map[NNTPConnection]bool
  infeed chan *NNTPMessage
  // channel to load messages to infeed given their message id
  infeed_load chan string
  // channel for broadcasting a message to all feeds given their message id
  send_all_feeds chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  feed := NNTPConnection{conn, textproto.NewConn(conn), inbound, self.debug, new(ConnectionInfo), policy, make(chan *NNTPMessage, 64),  make(chan string, 512)}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
					time.Sleep(5)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
					time.Sleep(5)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// sync every article to all feeds
func (self *NNTPDaemon) syncAll() {
  log.Println("sync all feeds")
  chnl := make(chan string, 128)
  go self.store.IterateAllArticles(chnl)
  for {
    select {
    case msgid := <- chnl:
      for feed, use := range self.feeds {
        if use {
          feed.sync <- msgid
        }
      }
    }
  }
}

// run daemon
func (self *NNTPDaemon) Run() {
  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return
  }
  defer self.listener.Close()

  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  // start accepting incoming connections
  go self.acceptloop()

  if self.sync_on_start {
    go self.syncAll()
  }
  // if we have no frontend this does nothing
  if self.frontend != nil {
    go self.pollfrontend()
  }
  self.pollfeeds()

}

func (self *NNTPDaemon) pollfrontend() {
  for {
    select {
    case nntp := <- self.frontend.NewPostsChan():
      // new post from frontend
      // ammend path
      nntp.Path = self.instance_name + "!" + nntp.Path
      // tell infeed that we got one
      self.infeed <- nntp
      break
    case msgid := <- self.infeed_load:
      // load temp message
      // this deletes the temp file
      nntp := self.store.ReadTempMessage(msgid)
      // rewrite path header
      nntp.Path = self.instance_name +"!" + nntp.Path
      // offer infeed
      self.infeed <- nntp
      break

    }
  }
}

func (self *NNTPDaemon) pollfeeds() {
  for {
    select {
    case msgid := <- self.send_all_feeds:
      // send all feeds
      nntp := self.store.GetMessage(msgid)
      if nntp == nil {
        log.Printf("failed to load %s for federation", msgid)
      } else {
        for feed , use := range self.feeds {
          if use && feed.policy != nil {
            if feed.policy.AllowsNewsgroup(nntp.Newsgroup) {
              feed.send <- nntp
            }
          }
        }
      }
      break
    case nntp := <- self.infeed:
      // register article
      self.database.RegisterArticle(nntp)
      if nntp.VerifySignature() {
        // store article
        self.store.StorePost(nntp)
        // queue to all outfeeds
        self.send_all_feeds <- nntp.MessageID
      } else {
        log.Printf("%s has invalid signature", nntp.MessageID)
      }

      break
    }
  }
}

func (self *NNTPDaemon) acceptloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunInbound(nntp)
  }
}

func (self *NNTPDaemon) RunInbound(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Println("SRNd NNTPD bound at", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {
  CheckConfig()
  log.Println("load config")
  self.conf = ReadConf()
  if self.conf == nil {
    log.Println("cannot load config")
    return false
  }
  self.infeed = make(chan *NNTPMessage, 64)
  self.infeed_load = make(chan string, 256)
  self.send_all_feeds = make(chan string, 64)
  self.feeds = make(map[NNTPConnection]bool)

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  self.database = NewDatabase(self.conf.database["type"], self.conf.database["schema"], db_host, db_port, db_user, db_passwd)
  self.database.CreateTables()

  self.store = new(ArticleStore)
  self.store.directory = self.conf.store["store_dir"]
  self.store.temp = self.conf.store["incoming_dir"]
  self.store.database = self.database
  self.store.Init()

  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  if self.sync_on_start {
    log.Println("sync on start")
  }
  self.bind_addr = self.conf.daemon["bind"]
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  // do we enable the frontend?
  if self.conf.frontend["enable"] == "1" {
    log.Printf("frontend %s enabled", self.conf.frontend["name"])
    self.frontend = NewHTTPFrontend(self, self.conf.frontend)
    go self.frontend.Mainloop()
  }

  return true
}
<|endoftext|>


<commit_before>//
// markup.go
// memeposting markup parser
//
package srnd

import (
	"github.com/mvdan/xurls"
	"html"
	"regexp"
	"strings"
)

// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
// var re_external_link = regexp.MustCompile(`((?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?)`);
var re_external_link = xurls.Strict
var re_backlink = regexp.MustCompile(`>> ?([0-9a-f]+)`)

// parse backlink
func backlink(word string) (markup string) {
	re := regexp.MustCompile(`>> ?([0-9a-f]+)`)
	link := re.FindString(word)
	if len(link) > 2 {
		link = strings.Trim(link[2:], " ")
		if len(link) > 2 {
			url := template.findLink(link)
			if len(url) == 0 {
				return ">>" + link + ""
			}
			// backlink exists
			parts := strings.Split(url, "#")
			longhash := ""
			if len(parts) > 1 {
				longhash = parts[1]
			}
			return `<a class='backlink' backlinkhash="` + longhash + `" href="` + url + `">>>` + link + ""
		} else {
			return escapeline(word)
		}
	}
	return escapeline(word)
}

func escapeline(line string) (markup string) {
	markup = html.EscapeString(line)
	return
}

func formatline(line string) (markup string) {
	line = strings.Trim(line, "\t\r\n ")
	if len(line) > 0 {
		if strings.HasPrefix(line, ">") && !(strings.HasPrefix(line, ">>") && re_backlink.MatchString(strings.Split(line, " ")[0])) {
			// le ebin meme arrows
			markup += ""
			markup += escapeline(line)
			markup += ""
		} else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
			// redtext
			markup += ""
			markup += escapeline(line[2 : len(line)-2])
			markup += ""
		} else if strings.HasPrefix(line, "@@") && strings.HasSuffix(line, "@@") {
			// psytext
			markup += ""
			markup += escapeline(line[2 : len(line)-2])
			markup += ""
		} else {
			// regular line
			// for each word
			for _, word := range strings.Split(line, " ") {
				// check for backlink
				if re_backlink.MatchString(word) {
					markup += backlink(word)
				} else {
					// linkify as needed
					word = escapeline(word)
					markup += re_external_link.ReplaceAllString(word, `<a href="$1">$1</a>`)
				}
				markup += " "
			}
		}
	}
	return
}

// old parse function
func memeposting(src string) (markup string) {
	for _, line := range strings.Split(src, "\n") {
		markup += formatline(line)
	}
	return
}
<commit_msg>make each line have \n<commit_after>//
// markup.go
// memeposting markup parser
//
package srnd

import (
	"github.com/mvdan/xurls"
	"html"
	"regexp"
	"strings"
)

// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
// var re_external_link = regexp.MustCompile(`((?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?)`);
var re_external_link = xurls.Strict
var re_backlink = regexp.MustCompile(`>> ?([0-9a-f]+)`)

// parse backlink
func backlink(word string) (markup string) {
	re := regexp.MustCompile(`>> ?([0-9a-f]+)`)
	link := re.FindString(word)
	if len(link) > 2 {
		link = strings.Trim(link[2:], " ")
		if len(link) > 2 {
			url := template.findLink(link)
			if len(url) == 0 {
				return ">>" + link + ""
			}
			// backlink exists
			parts := strings.Split(url, "#")
			longhash := ""
			if len(parts) > 1 {
				longhash = parts[1]
			}
			return `<a class='backlink' backlinkhash="` + longhash + `" href="` + url + `">>>` + link + ""
		} else {
			return escapeline(word)
		}
	}
	return escapeline(word)
}

func escapeline(line string) (markup string) {
	markup = html.EscapeString(line)
	return
}

func formatline(line string) (markup string) {
	line = strings.Trim(line, "\t\r\n ")
	if len(line) > 0 {
		if strings.HasPrefix(line, ">") && !(strings.HasPrefix(line, ">>") && re_backlink.MatchString(strings.Split(line, " ")[0])) {
			// le ebin meme arrows
			markup += ""
			markup += escapeline(line)
			markup += ""
		} else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
			// redtext
			markup += ""
			markup += escapeline(line[2 : len(line)-2])
			markup += ""
		} else if strings.HasPrefix(line, "@@") && strings.HasSuffix(line, "@@") {
			// psytext
			markup += ""
			markup += escapeline(line[2 : len(line)-2])
			markup += ""
		} else {
			// regular line
			// for each word
			for _, word := range strings.Split(line, " ") {
				// check for backlink
				if re_backlink.MatchString(word) {
					markup += backlink(word)
				} else {
					// linkify as needed
					word = escapeline(word)
					markup += re_external_link.ReplaceAllString(word, `<a href="$1">$1</a>`)
				}
				markup += " "
			}
		}
	}
	return
}

// old parse function
func memeposting(src string) (markup string) {
	for _, line := range strings.Split(src, "\n") {
		markup += formatline(line) + "\n"
	}
	return
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"fmt"
	"os"
	"strings"

	_ "github.com/mattn/go-sqlite3"

	log "code.google.com/p/log4go"
)

const (
	dbName = "db.sqlite"
)

// The SQL database.
type DB struct {
	dbConn *sql.DB
}

// Query result types
type RowResult map[string]string
type RowResults []map[string]string

// New creates a new database. Deletes any existing database.
func New(dbPath string) *DB {
	os.Remove(dbPath)
	return Open(dbPath)
}

// Open an existing database, creating it if it does not exist.
func Open(dbPath string) *DB {
	log.Trace("SQLite database path is %s", dbPath)
	dbc, err := sql.Open("sqlite3", dbPath)
	if err != nil {
		log.Error(err)
		return nil
	}
	return &DB{
		dbConn: dbc,
	}
}

// Close closes the underlying database connection.
func (db *DB) Close() error {
	return db.dbConn.Close()
}

// Query runs the supplied query against the sqlite database. It returns a slice of
// RowResults.
func (db *DB) Query(query string) (RowResults, error) {
	if !strings.HasPrefix(strings.ToUpper(query), "SELECT ") {
		log.Warn("Query \"%s\" may modify the database", query)
	}
	rows, err := db.dbConn.Query(query)
	if err != nil {
		log.Error("failed to execute SQLite query", err.Error())
		return nil, err
	}
	defer rows.Close()

	results := make(RowResults, 0)

	columns, _ := rows.Columns()
	rawResult := make([][]byte, len(columns))
	dest := make([]interface{}, len(columns))
	for i, _ := range rawResult {
		dest[i] = &rawResult[i] // Pointers to each string in the interface slice
	}

	for rows.Next() {
		err = rows.Scan(dest...)
		if err != nil {
			log.Error("failed to scan SQLite row", err.Error())
			return nil, err
		}

		r := make(RowResult)
		for i, raw := range rawResult {
			if raw == nil {
				r[columns[i]] = "null"
			} else {
				r[columns[i]] = string(raw)
			}
		}
		results = append(results, r)
	}
	log.Debug(func() string { return "Executed query successfully: " + query })
	return results, nil
}

// Execute executes the given sqlite statement, of a type that doesn't return rows.
func (db *DB) Execute(stmt string) error {
	_, err := db.dbConn.Exec(stmt)
	log.Debug(func() string {
		if err != nil {
			return fmt.Sprintf("Error executing \"%s\", error: %s", stmt, err.Error())
		}
		return fmt.Sprintf("Successfully executed \"%s\"", stmt)
	})
	return err
}

// StartTransaction starts an explicit transaction.
func (db *DB) StartTransaction() error {
	_, err := db.dbConn.Exec("BEGIN")
	log.Debug(func() string {
		if err != nil {
			return "Error starting transaction"
		}
		return "Successfully started transaction"
	})
	return err
}

// CommitTransaction commits all changes made since StartTraction was called.
func (db *DB) CommitTransaction() error {
	_, err := db.dbConn.Exec("END")
	log.Debug(func() string {
		if err != nil {
			return "Error ending transaction"
		}
		return "Successfully ended transaction"
	})
	return err
}

// RollbackTransaction aborts the transaction. No statement issued since
// StartTransaction was called will take effect.
func (db *DB) RollbackTransaction() error {
	_, err := db.dbConn.Exec("ROLLBACK")
	log.Debug(func() string {
		if err != nil {
			return "Error rolling back transaction"
		}
		return "Successfully rolled back transaction"
	})
	return err
}
<commit_msg>Enhance logging<commit_after>package db

import (
	"database/sql"
	"fmt"
	"os"
	"strings"

	_ "github.com/mattn/go-sqlite3"

	log "code.google.com/p/log4go"
)

const (
	dbName = "db.sqlite"
)

// The SQL database.
type DB struct {
	dbConn *sql.DB
}

// Query result types
type RowResult map[string]string
type RowResults []map[string]string

// New creates a new database. Deletes any existing database.
func New(dbPath string) *DB {
	log.Trace("Removing any existing SQLite database at %s", dbPath)
	os.Remove(dbPath)
	return Open(dbPath)
}

// Open an existing database, creating it if it does not exist.
func Open(dbPath string) *DB {
	log.Trace("Opening SQLite database path at %s", dbPath)
	dbc, err := sql.Open("sqlite3", dbPath)
	if err != nil {
		log.Error(err)
		return nil
	}
	return &DB{
		dbConn: dbc,
	}
}

// Close closes the underlying database connection.
func (db *DB) Close() error {
	return db.dbConn.Close()
}

// Query runs the supplied query against the sqlite database. It returns a slice of
// RowResults.
func (db *DB) Query(query string) (RowResults, error) {
	if !strings.HasPrefix(strings.ToUpper(query), "SELECT ") {
		log.Warn("Query \"%s\" may modify the database", query)
	}
	rows, err := db.dbConn.Query(query)
	if err != nil {
		log.Error("failed to execute SQLite query", err.Error())
		return nil, err
	}
	defer rows.Close()

	results := make(RowResults, 0)

	columns, _ := rows.Columns()
	rawResult := make([][]byte, len(columns))
	dest := make([]interface{}, len(columns))
	for i, _ := range rawResult {
		dest[i] = &rawResult[i] // Pointers to each string in the interface slice
	}

	for rows.Next() {
		err = rows.Scan(dest...)
		if err != nil {
			log.Error("failed to scan SQLite row", err.Error())
			return nil, err
		}

		r := make(RowResult)
		for i, raw := range rawResult {
			if raw == nil {
				r[columns[i]] = "null"
			} else {
				r[columns[i]] = string(raw)
			}
		}
		results = append(results, r)
	}
	log.Debug(func() string { return "Executed query successfully: " + query })
	return results, nil
}

// Execute executes the given sqlite statement, of a type that doesn't return rows.
func (db *DB) Execute(stmt string) error {
	_, err := db.dbConn.Exec(stmt)
	log.Debug(func() string {
		if err != nil {
			return fmt.Sprintf("Error executing \"%s\", error: %s", stmt, err.Error())
		}
		return fmt.Sprintf("Successfully executed \"%s\"", stmt)
	})
	return err
}

// StartTransaction starts an explicit transaction.
func (db *DB) StartTransaction() error {
	_, err := db.dbConn.Exec("BEGIN")
	log.Debug(func() string {
		if err != nil {
			return "Error starting transaction"
		}
		return "Successfully started transaction"
	})
	return err
}

// CommitTransaction commits all changes made since StartTraction was called.
func (db *DB) CommitTransaction() error {
	_, err := db.dbConn.Exec("END")
	log.Debug(func() string {
		if err != nil {
			return "Error ending transaction"
		}
		return "Successfully ended transaction"
	})
	return err
}

// RollbackTransaction aborts the transaction. No statement issued since
// StartTransaction was called will take effect.
func (db *DB) RollbackTransaction() error {
	_, err := db.dbConn.Exec("ROLLBACK")
	log.Debug(func() string {
		if err != nil {
			return "Error rolling back transaction"
		}
		return "Successfully rolled back transaction"
	})
	return err
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"strconv"
)

func InitDB(path string) (*sql.DB, error) {
	db, err := sql.Open("sqlite3", path)
	if err != nil {
		return db, err
	}
	err = createTables(db)
	return db, err
}

func createTables(db *sql.DB) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	version := getMigrationVersion(tx)
	switch version {
	case 0:
		query := `
		CREATE TABLE states(
			user_id INTEGER PRIMARY KEY NOT NULL,
			action TEXT
		);
		CREATE TABLE votes(
			id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
			user_id INTEGER,
			author TEXT,
			permalink TEXT,
			percent INTEGER,
			completed BOOLEAN NOT NULL CHECK (completed IN (0,1)) DEFAULT 0,
			date DATETIME DEFAULT CURRENT_TIMESTAMP
		);
		CREATE TABLE credentials(
			user_id INTEGER PRIMARY KEY NOT NULL,
			user_name TEXT,
			power INTEGER NOT NULL DEFAULT 100,
			rating INTEGER NOT NULL DEFAULT 10,
			active BOOLEAN NOT NULL CHECK (active IN (0,1)) DEFAULT 0
		);
		CREATE TABLE responses(
			id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
			user_id INTEGER,
			vote_id INTEGER,
			result BOOLEAN NOT NULL CHECK (result IN (0,1)),
			date DATETIME DEFAULT CURRENT_TIMESTAMP
		);
		CREATE UNIQUE INDEX idx_user_vote ON responses(user_id, vote_id);
		CREATE UNIQUE INDEX idx_author_permalink ON votes(author, permalink);
		`
		_, err = tx.Exec(query)
		if err != nil {
			tx.Rollback()
			return err
		}
		setMigrationVersion(tx, 1)
		fallthrough
	case 1:
		query := `
		CREATE TABLE referrals(
			id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
			user_id INTEGER UNIQUE NOT NULL,
			referrer TEXT NOT NULL,
			completed BOOLEAN NOT NULL CHECK (completed IN (0,1))
		);
		`
		_, err = tx.Exec(query)
		if err != nil {
			tx.Rollback()
			return err
		}
		setMigrationVersion(tx, 2)
		fallthrough
	case 3:
		query := `
		CREATE TABLE cred_temp AS (SELECT user_id, user_name, power, active FROM credentials);
		DROP credentials;
		ALTER TABLE cred_temp RENAME TO credentials;
		`
		_, err = tx.Exec(query)
		if err != nil {
			tx.Rollback()
			return err
		}
		setMigrationVersion(tx, 3)
		//fallthrough
	}
	tx.Commit()
	return nil
}

func getMigrationVersion(tx *sql.Tx) int {
	var version int
	row := tx.QueryRow("PRAGMA user_version")
	err := row.Scan(&version)
	if err != nil {
		log.Panic(err.Error())
	}
	return version
}

func setMigrationVersion(tx *sql.Tx, version int) {
	query := "PRAGMA user_version = " + strconv.Itoa(version)
	_, err := tx.Exec(query)
	if err != nil {
		log.Panic(err.Error())
	}
}
<commit_msg>Update SQL synthax in new migration<commit_after>package db

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"log"
	"strconv"
)

func InitDB(path string) (*sql.DB, error) {
	db, err := sql.Open("sqlite3", path)
	if err != nil {
		return db, err
	}
	err = createTables(db)
	return db, err
}

func createTables(db *sql.DB) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	version := getMigrationVersion(tx)
	switch version {
	case 0:
		query := `
		CREATE TABLE states(
			user_id INTEGER PRIMARY KEY NOT NULL,
			action TEXT
		);
		CREATE TABLE votes(
			id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
			user_id INTEGER,
			author TEXT,
			permalink TEXT,
			percent INTEGER,
			completed BOOLEAN NOT NULL CHECK (completed IN (0,1)) DEFAULT 0,
			date DATETIME DEFAULT CURRENT_TIMESTAMP
		);
		CREATE TABLE credentials(
			user_id INTEGER PRIMARY KEY NOT NULL,
			user_name TEXT,
			power INTEGER NOT NULL DEFAULT 100,
			rating INTEGER NOT NULL DEFAULT 10,
			active BOOLEAN NOT NULL CHECK (active IN (0,1)) DEFAULT 0
		);
		CREATE TABLE responses(
			id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
			user_id INTEGER,
			vote_id INTEGER,
			result BOOLEAN NOT NULL CHECK (result IN (0,1)),
			date DATETIME DEFAULT CURRENT_TIMESTAMP
		);
		CREATE UNIQUE INDEX idx_user_vote ON responses(user_id, vote_id);
		CREATE UNIQUE INDEX idx_author_permalink ON votes(author, permalink);
		`
		_, err = tx.Exec(query)
		if err != nil {
			tx.Rollback()
			return err
		}
		setMigrationVersion(tx, 1)
		fallthrough
	case 1:
		query := `
		CREATE TABLE referrals(
			id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
			user_id INTEGER UNIQUE NOT NULL,
			referrer TEXT NOT NULL,
			completed BOOLEAN NOT NULL CHECK (completed IN (0,1))
		);
		`
		_, err = tx.Exec(query)
		if err != nil {
			tx.Rollback()
			return err
		}
		setMigrationVersion(tx, 2)
		fallthrough
	case 2:
		query := `
		CREATE TABLE cred_temp AS SELECT user_id, user_name, power, active FROM credentials;
		DROP TABLE credentials;
		ALTER TABLE cred_temp RENAME TO credentials;
		`
		_, err = tx.Exec(query)
		if err != nil {
			tx.Rollback()
			return err
		}
		setMigrationVersion(tx, 3)
		//fallthrough
	}
	tx.Commit()
	return nil
}

func getMigrationVersion(tx *sql.Tx) int {
	var version int
	row := tx.QueryRow("PRAGMA user_version")
	err := row.Scan(&version)
	if err != nil {
		log.Panic(err.Error())
	}
	return version
}

func setMigrationVersion(tx *sql.Tx, version int) {
	query := "PRAGMA user_version = " + strconv.Itoa(version)
	_, err := tx.Exec(query)
	if err != nil {
		log.Panic(err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zanzibar

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
	"go.uber.org/multierr"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

// ClientHTTPResponse is the struct managing the client response
// when making outbound http calls.
type ClientHTTPResponse struct {
	req              *ClientHTTPRequest
	finishTime       time.Time
	finished         bool
	rawResponse      *http.Response
	rawResponseBytes []byte
	StatusCode       int
	Header           http.Header
}

// NewClientHTTPResponse allocates a client http response object
// to track http response.
func NewClientHTTPResponse(
	req *ClientHTTPRequest,
) *ClientHTTPResponse {
	return &ClientHTTPResponse{
		req: req,
	}
}

func (res *ClientHTTPResponse) setRawHTTPResponse(httpRes *http.Response) {
	res.rawResponse = httpRes
	res.StatusCode = httpRes.StatusCode
	res.Header = httpRes.Header
}

// ReadAll reads bytes from response.
func (res *ClientHTTPResponse) ReadAll() ([]byte, error) {
	rawBody, err := ioutil.ReadAll(res.rawResponse.Body)

	cerr := res.rawResponse.Body.Close()
	if cerr != nil {
		/* coverage ignore next line */
		res.req.Logger.Error("Could not close response body", zap.Error(cerr))
	}

	if err != nil {
		res.req.Logger.Error("Could not read response body", zap.Error(err))
		res.finish()
		return nil, errors.Wrapf(
			err, "Could not read %s.%s response body",
			res.req.ClientID, res.req.MethodName,
		)
	}

	res.rawResponseBytes = rawBody
	res.finish()
	return rawBody, nil
}

// GetRawBody returns the body as byte array if it has been read.
func (res *ClientHTTPResponse) GetRawBody() []byte {
	return res.rawResponseBytes
}

// ReadAndUnmarshalBody will try to unmarshal non pointer value or fail
func (res *ClientHTTPResponse) ReadAndUnmarshalBody(v interface{}) error {
	rawBody, err := res.ReadAll()
	if err != nil {
		/* coverage ignore next line */
		return err
	}

	err = json.Unmarshal(rawBody, v)
	if err != nil {
		res.req.Logger.Warn("Could not parse response json", zap.Error(err))
		return errors.Wrapf(
			err, "Could not parse %s.%s response json",
			res.req.ClientID, res.req.MethodName,
		)
	}

	return nil
}

// ReadAndUnmarshalBodyMultipleOptions will try to unmarshal non pointer value to one of the provided types or fail
// It will return the deserialized struct (if any) that succeeded
func (res *ClientHTTPResponse) ReadAndUnmarshalBodyMultipleOptions(vs []interface{}) (interface{}, error) {
	rawBody, err := res.ReadAll()
	if err != nil {
		/* coverage ignore next line */
		return nil, err
	}

	var merr error
	for _, v := range vs {
		err = json.Unmarshal(rawBody, v)
		if err == nil {
			// All done -- successfully deserialized
			return v, nil
		}
		merr = multierr.Append(merr, err)
	}

	err = fmt.Errorf("all json serialization errors: %s", merr.Error())

	res.req.Logger.Warn("Could not parse response json into any of provided interfaces", zap.Error(err))
	return nil, errors.Wrapf(
		err, "Could not parse %s.%s response json into any of provided interfaces",
		res.req.ClientID, res.req.MethodName,
	)
}

// CheckOKResponse checks if the status code is OK.
func (res *ClientHTTPResponse) CheckOKResponse(okResponses []int) {
	for _, okResponse := range okResponses {
		if res.rawResponse.StatusCode == okResponse {
			return
		}
	}

	res.req.Logger.Warn("Unknown response status code",
		zap.Int("UnknownStatusCode", res.rawResponse.StatusCode),
	)
}

// finish will handle final logic, like metrics
func (res *ClientHTTPResponse) finish() {
	if !res.req.started {
		/* coverage ignore next line */
		res.req.Logger.Error("Forgot to start client request")
		/* coverage ignore next line */
		return
	}
	if res.finished {
		/* coverage ignore next line */
		res.req.Logger.Error("Finished a client response multiple times")
		/* coverage ignore next line */
		return
	}
	res.finished = true
	res.finishTime = time.Now()

	logFn := res.req.ContextLogger.Debug

	// emit metrics
	delta := res.finishTime.Sub(res.req.startTime)
	res.req.metrics.RecordTimer(res.req.ctx, clientLatency, delta)
	res.req.metrics.RecordHistogramDuration(res.req.ctx, clientLatencyHist, delta)
	_, known := knownStatusCodes[res.StatusCode]
	if !known {
		res.req.Logger.Error(
			"Could not emit statusCode metric",
			zap.Int("UnknownStatusCode", res.StatusCode),
		)
	} else {
		scopeTags := map[string]string{scopeTagStatus: fmt.Sprintf("%d", res.StatusCode)}
		res.req.ctx = WithScopeTags(res.req.ctx, scopeTags)
		res.req.metrics.IncCounter(res.req.ctx, clientStatus, 1)
	}
	if !known || res.StatusCode >= 400 && res.StatusCode < 600 {
		res.req.metrics.IncCounter(res.req.ctx, clientErrors, 1)
		logFn = res.req.ContextLogger.Warn
	}

	// write logs
	logFn(
		res.req.ctx,
		"Finished an outgoing client HTTP request",
		clientHTTPLogFields(res.req, res)...,
	)
}

func clientHTTPLogFields(req *ClientHTTPRequest, res *ClientHTTPResponse) []zapcore.Field {
	fields := []zapcore.Field{
		zap.Time(logFieldRequestFinishedTime, res.finishTime),
		zap.Int(logFieldResponseStatusCode, res.StatusCode),
	}
	for k, v := range req.httpReq.Header {
		if len(v) > 0 {
			fields = append(fields, zap.String(
				fmt.Sprintf("%s-%s", logFieldClientRequestHeaderPrefix, k),
				strings.Join(v, ", "),
			))
		}
	}
	for k, v := range req.res.Header {
		if len(v) > 0 {
			fields = append(fields, zap.String(
				fmt.Sprintf("%s-%s", logFieldClientResponseHeaderPrefix, k),
				strings.Join(v, ", "),
			))
		}
	}

	// TODO: log jaeger trace span

	return fields
}
<commit_msg>Refactor UnmarshalBody (#694)<commit_after>// Copyright (c) 2020 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package zanzibar

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/pkg/errors"
	"go.uber.org/multierr"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

// ClientHTTPResponse is the struct managing the client response
// when making outbound http calls.
type ClientHTTPResponse struct {
	req              *ClientHTTPRequest
	finishTime       time.Time
	finished         bool
	rawResponse      *http.Response
	rawResponseBytes []byte
	StatusCode       int
	Header           http.Header
}

// NewClientHTTPResponse allocates a client http response object
// to track http response.
func NewClientHTTPResponse(
	req *ClientHTTPRequest,
) *ClientHTTPResponse {
	return &ClientHTTPResponse{
		req: req,
	}
}

func (res *ClientHTTPResponse) setRawHTTPResponse(httpRes *http.Response) {
	res.rawResponse = httpRes
	res.StatusCode = httpRes.StatusCode
	res.Header = httpRes.Header
}

// ReadAll reads bytes from response.
func (res *ClientHTTPResponse) ReadAll() ([]byte, error) {
	rawBody, err := ioutil.ReadAll(res.rawResponse.Body)

	cerr := res.rawResponse.Body.Close()
	if cerr != nil {
		/* coverage ignore next line */
		res.req.Logger.Error("Could not close response body", zap.Error(cerr))
	}

	if err != nil {
		res.req.Logger.Error("Could not read response body", zap.Error(err))
		res.finish()
		return nil, errors.Wrapf(
			err, "Could not read %s.%s response body",
			res.req.ClientID, res.req.MethodName,
		)
	}

	res.rawResponseBytes = rawBody
	res.finish()
	return rawBody, nil
}

// GetRawBody returns the body as byte array if it has been read.
func (res *ClientHTTPResponse) GetRawBody() []byte {
	return res.rawResponseBytes
}

// ReadAndUnmarshalBody will try to unmarshal non pointer value or fail
func (res *ClientHTTPResponse) ReadAndUnmarshalBody(v interface{}) error {
	rawBody, err := res.ReadAll()
	if err != nil {
		/* coverage ignore next line */
		return err
	}
	return res.UnmarshalBody(v, rawBody)
}

// UnmarshalBody helper to unmarshal body into struct
func (res *ClientHTTPResponse) UnmarshalBody(v interface{}, rawBody []byte) error {
	err := json.Unmarshal(rawBody, v)
	if err != nil {
		res.req.Logger.Warn("Could not parse response json", zap.Error(err))
		return errors.Wrapf(
			err, "Could not parse %s.%s response json",
			res.req.ClientID, res.req.MethodName,
		)
	}
	return nil
}

// ReadAndUnmarshalBodyMultipleOptions will try to unmarshal non pointer value to one of the provided types or fail
// It will return the deserialized struct (if any) that succeeded
func (res *ClientHTTPResponse) ReadAndUnmarshalBodyMultipleOptions(vs []interface{}) (interface{}, error) {
	rawBody, err := res.ReadAll()
	if err != nil {
		/* coverage ignore next line */
		return nil, err
	}

	var merr error
	for _, v := range vs {
		err = json.Unmarshal(rawBody, v)
		if err == nil {
			// All done -- successfully deserialized
			return v, nil
		}
		merr = multierr.Append(merr, err)
	}

	err = fmt.Errorf("all json serialization errors: %s", merr.Error())

	res.req.Logger.Warn("Could not parse response json into any of provided interfaces", zap.Error(err))
	return nil, errors.Wrapf(
		err, "Could not parse %s.%s response json into any of provided interfaces",
		res.req.ClientID, res.req.MethodName,
	)
}

// CheckOKResponse checks if the status code is OK.
func (res *ClientHTTPResponse) CheckOKResponse(okResponses []int) {
	for _, okResponse := range okResponses {
		if res.rawResponse.StatusCode == okResponse {
			return
		}
	}

	res.req.Logger.Warn("Unknown response status code",
		zap.Int("UnknownStatusCode", res.rawResponse.StatusCode),
	)
}

// finish will handle final logic, like metrics
func (res *ClientHTTPResponse) finish() {
	if !res.req.started {
		/* coverage ignore next line */
		res.req.Logger.Error("Forgot to start client request")
		/* coverage ignore next line */
		return
	}
	if res.finished {
		/* coverage ignore next line */
		res.req.Logger.Error("Finished a client response multiple times")
		/* coverage ignore next line */
		return
	}
	res.finished = true
	res.finishTime = time.Now()

	logFn := res.req.ContextLogger.Debug

	// emit metrics
	delta := res.finishTime.Sub(res.req.startTime)
	res.req.metrics.RecordTimer(res.req.ctx, clientLatency, delta)
	res.req.metrics.RecordHistogramDuration(res.req.ctx, clientLatencyHist, delta)
	_, known := knownStatusCodes[res.StatusCode]
	if !known {
		res.req.Logger.Error(
			"Could not emit statusCode metric",
			zap.Int("UnknownStatusCode", res.StatusCode),
		)
	} else {
		scopeTags := map[string]string{scopeTagStatus: fmt.Sprintf("%d", res.StatusCode)}
		res.req.ctx = WithScopeTags(res.req.ctx, scopeTags)
		res.req.metrics.IncCounter(res.req.ctx, clientStatus, 1)
	}
	if !known || res.StatusCode >= 400 && res.StatusCode < 600 {
		res.req.metrics.IncCounter(res.req.ctx, clientErrors, 1)
		logFn = res.req.ContextLogger.Warn
	}

	// write logs
	logFn(
		res.req.ctx,
		"Finished an outgoing client HTTP request",
		clientHTTPLogFields(res.req, res)...,
	)
}

func clientHTTPLogFields(req *ClientHTTPRequest, res *ClientHTTPResponse) []zapcore.Field {
	fields := []zapcore.Field{
		zap.Time(logFieldRequestFinishedTime, res.finishTime),
		zap.Int(logFieldResponseStatusCode, res.StatusCode),
	}
	for k, v := range req.httpReq.Header {
		if len(v) > 0 {
			fields = append(fields, zap.String(
				fmt.Sprintf("%s-%s", logFieldClientRequestHeaderPrefix, k),
				strings.Join(v, ", "),
			))
		}
	}
	for k, v := range req.res.Header {
		if len(v) > 0 {
			fields = append(fields, zap.String(
				fmt.Sprintf("%s-%s", logFieldClientResponseHeaderPrefix, k),
				strings.Join(v, ", "),
			))
		}
	}

	// TODO: log jaeger trace span

	return fields
}
<|endoftext|>


<commit_before>package httpimg_test

import (
	"fmt"
	"github.com/jung-kurt/gofpdf"
	"github.com/jung-kurt/gofpdf/contrib/httpimg"
	"path/filepath"
)

const (
	cnGofpdfDir  = "./"
	cnExampleDir = cnGofpdfDir + "/pdf"
)

func exampleFilename(baseStr string) string {
	return filepath.Join(cnExampleDir, baseStr+".pdf")
}

func summary(err error, fileStr string) {
	if err == nil {
		fileStr = filepath.ToSlash(fileStr)
		fmt.Printf("Successfully generated %s\n", fileStr)
	} else {
		fmt.Println(err)
	}
}

func ExampleRegister() {
	pdf := gofpdf.New("", "", "", "")
	pdf.SetFont("Helvetica", "", 12)
	pdf.SetFillColor(200, 200, 220)
	pdf.AddPage()

	url := "https://github.com/jung-kurt/gofpdf/raw/master/image/logo_gofpdf.jpg?raw=true"
	httpimg.Register(pdf, url, "")
	pdf.Image(url, 100, 100, 20, 20, false, "", 0, "")

	fileStr := exampleFilename("contrib_httpimg_Register")
	err := pdf.OutputFileAndClose(fileStr)
	summary(err, fileStr)
	// Output:
	// Successfully generated pdf/contrib_httpimg_Register.pdf
}
<commit_msg>Httpimg: cleanup pdf path on with tests.<commit_after>package httpimg_test

import (
	"fmt"
	"github.com/jung-kurt/gofpdf"
	"github.com/jung-kurt/gofpdf/contrib/httpimg"
	"os"
	"path/filepath"
)

const (
	cnGofpdfDir  = "./"
	cnExampleDir = cnGofpdfDir + "/pdf"
)

func init() {
	cleanup()
}

func cleanup() {
	filepath.Walk(cnExampleDir,
		func(path string, info os.FileInfo, err error) (reterr error) {
			if path[len(path)-4:] == ".pdf" {
				os.Remove(path)
			}
			return
		})
}

func exampleFilename(baseStr string) string {
	return filepath.Join(cnExampleDir, baseStr+".pdf")
}

func summary(err error, fileStr string) {
	if err == nil {
		fileStr = filepath.ToSlash(fileStr)
		fmt.Printf("Successfully generated %s\n", fileStr)
	} else {
		fmt.Println(err)
	}
}

func ExampleRegister() {
	pdf := gofpdf.New("", "", "", "")
	pdf.SetFont("Helvetica", "", 12)
	pdf.SetFillColor(200, 200, 220)
	pdf.AddPage()

	url := "https://github.com/jung-kurt/gofpdf/raw/master/image/logo_gofpdf.jpg?raw=true"
	httpimg.Register(pdf, url, "")
	pdf.Image(url, 100, 100, 20, 20, false, "", 0, "")

	fileStr := exampleFilename("contrib_httpimg_Register")
	err := pdf.OutputFileAndClose(fileStr)
	summary(err, fileStr)
	// Output:
	// Successfully generated pdf/contrib_httpimg_Register.pdf
}
<|endoftext|>