<commit_before>package lrcompress

import (
	"bufio"
	"encoding/binary"
	"errors"
	"io"
)

type compRing [1 << CompHistBits]byte
type compHtbl [1 << hBits]int64

// Compressor is a Writer into which you can dump content.
type Compressor struct {
	pos        int64     // count of bytes ever written
	ring       compRing  // the bytes
	h          uint64    // current rolling hash
	matchMin   int64     // first matchable byte (for Reset)
	matchPos   int64     // current match start or 0
	matchLen   int64     // current match length or 0
	cursor     int64     // "expected" match start
	w          io.Writer // compressed output
	literalLen int64     // current literal length or 0
	encodeBuf  [16]byte  // for varints
	hTbl       compHtbl  // hashtable holding offsets into source file
}

const CompHistBits = 22    // log2 bytes of history for compression
const hBits = 18           // log2 hashtable size
const window = 64          // bytes that must overlap to match
const maxLiteral = 1 << 16 // we'll write this size literal
const maxMatch = 1 << 18   // max match we output (we'll read larger)
// note ring size during *compression* is baked in at compile time
const rMask = 1<<CompHistBits - 1 // &rMask turns offset into ring pos
// c.hTbl[h>>hShift&hMask] is current hashtable entry
const hMask = 1<<hBits - 1
const hShift = 64 - hBits

// only access hTbl if (h&fMask == fMask)
const fBits uint = CompHistBits - hBits + 1 // 1/2 fill the table
const fMask uint64 = 1<<fBits - 1

// Make a compressor with 1<<CompHistBits of memory, writing output to w.
func NewCompressor(w io.Writer) *Compressor {
	return &Compressor{w: w, pos: 1, cursor: 1}
}

func (c *Compressor) putInt(i int64) (err error) {
	n := binary.PutVarint(c.encodeBuf[:], i)
	_, err = c.w.Write(c.encodeBuf[:n])
	return
}

func (c *Compressor) putMatch(matchPos, matchLen int64) (err error) {
	err = c.putInt(matchLen)
	if err != nil {
		return
	}
	err = c.putInt(matchPos - c.cursor)
	c.cursor = matchPos + matchLen
	return
}

func (c *Compressor) putLiteral(pos, literalLen int64) (err error) {
	if literalLen == 0 {
		return
	}
	err = c.putInt(-literalLen)
	if err != nil {
		return
	}
	if literalLen > pos&rMask {
		_, err = c.w.Write(c.ring[(pos-literalLen)&rMask:])
		if err != nil {
			return
		}
		_, err = c.w.Write(c.ring[:pos&rMask])
	} else {
		_, err = c.w.Write(c.ring[(pos-literalLen)&rMask : pos&rMask])
	}
	if err != nil {
		return
	}
	c.cursor += literalLen
	return
}

func (c *Compressor) Write(p []byte) (n int, err error) {
	h, ring, hTbl, pos, matchPos, matchLen, literalLen, matchMin := c.h, &c.ring, &c.hTbl, c.pos, c.matchPos, c.matchLen, c.literalLen, c.matchMin
	for _, b := range p {
		// can use any 32-bit const with least sig. bits=10b and some higher
		// bits set; even *=6 eventually mixes lower bits into the top ones
		h *= ((0x703a03ac|1)*2)&(1<<32-1) | 1<<31
		h ^= uint64(b)
		// if we're in a match, extend or end it
		if matchLen > 0 {
			// try to extend it
			if ring[(matchPos+matchLen)&rMask] == b &&
				matchLen < maxMatch {
				matchLen++
			} else {
				// can't extend it, flush out what we have
				if err = c.putMatch(matchPos, matchLen); err != nil {
					return
				}
				matchPos, matchLen = 0, 0
			}
		} else if literalLen > window && h&fMask == fMask {
			// see if we can *start* a match here.
			// get the hashtable entry
			match := hTbl[h>>hShift&hMask]
			// check if it's in the usable range and cur. byte matches
			if match > matchMin && b == ring[match&rMask] && match > pos-rMask+maxLiteral {
				matchPos, matchLen = match, 1 // 1 because cur. byte matched
				// extend backwards
				for literalLen > 0 &&
					matchPos-1 > pos-rMask+maxLiteral &&
					matchPos-1 > matchMin &&
					ring[(pos-matchLen)&rMask] == ring[(matchPos-1)&rMask] {
					literalLen--
					matchPos--
					matchLen++
				}
				if matchLen < window { // short match, ignore
					literalLen += matchLen - 1
					matchLen, matchPos = 0, 0
				} else { // match was long enough
					// this literal ends before pos-matchLen+1, not pos
					if err = c.putLiteral(pos-matchLen+1, literalLen); err != nil {
						return
					}
					literalLen = 0
				}
			}
		}
		// (still) not in a match, so just extend the literal
		if matchLen == 0 {
			if literalLen == maxLiteral {
				if err = c.putLiteral(pos, literalLen); err != nil {
					return
				}
				literalLen = 0
			}
			literalLen++
		}

		// update hashtable and ring
		ring[pos&rMask] = b
		if h&fMask == fMask {
			hTbl[h>>hShift&hMask] = pos
		}
		pos++
	}
	c.h, c.pos, c.matchPos, c.matchLen, c.literalLen = h, pos, matchPos, matchLen, literalLen
	return len(p), nil
}

func (c *Compressor) Flush() (err error) {
	if c.matchLen > 0 {
		err = c.putMatch(c.matchPos, c.matchLen)
		c.matchPos, c.matchLen = 0, 0
	} else {
		err = c.putLiteral(c.pos, c.literalLen)
		c.literalLen = 0
	}
	return
}

// Flush if needed, then write an end marker to the diff output.
func (c *Compressor) Close() (err error) {
	if err = c.Flush(); err != nil {
		return
	}
	return c.putInt(0)
}

// Clear the history and other state. If you've written data, Flush() it first.
func (c *Compressor) Reset() {
	if c.matchPos != 0 || c.literalLen != 0 || c.matchLen != 0 {
		panic("Call Flush() before Reset()")
	}
	c.matchMin = c.pos
	c.cursor = c.pos
	c.pos++
}

// Load data as if it had been written. If you've written real data, Flush() it first.
func (c *Compressor) LoadHistory(d []byte) {
	if c.matchPos != 0 || c.literalLen != 0 || c.matchLen != 0 {
		panic("Call Flush() before LoadHistory()")
	}
	h, pos, hTbl, ring := c.h, c.pos, &c.hTbl, &c.ring
	for _, b := range d {
		h *= ((0x703a03ac|1)*2)&(1<<32-1) | 1<<31
		h ^= uint64(b)
		// update hashtable and ring
		ring[pos&rMask] = b
		if h&fMask == fMask {
			hTbl[h>>hShift&hMask] = pos
		}
		pos++
	}
	c.cursor += int64(len(d))
	c.h, c.pos = h, pos
}

// Change where compressed data is written. If you've written data, Flush() it first.
func (c *Compressor) SetWriter(w io.Writer) {
	c.w = w
}

const maxReadLiteral = 1 << 16

// ring is a ring-buffer of bytes with Copy and Write operations. Writes and
// copies are teed to an io.Writer provided on initialization.
type ring struct {
	pos  int64     // count of bytes ever written
	mask int64     // &mask turns pos into a ring offset
	w    io.Writer // output of writes/copies goes here as well as ring
	ring []byte    // the bytes
}

func newRing(sizeBits uint, w io.Writer) ring {
	return ring{
		pos:  0,
		mask: 1<<sizeBits - 1,
		w:    w,
		ring: make([]byte, 1<<sizeBits),
	}
}

func (r *ring) Write(p []byte) (n int, err error) {
	n, err = r.w.Write(p)
	if err != nil {
		return
	}
	for len(p) > 0 && err == nil {
		l := len(p)
		pos := int(r.pos & r.mask)
		if pos+l > len(r.ring) {
			l = len(r.ring) - pos
		}
		copy(r.ring[pos:pos+l], p[:l])
		p = p[l:]
		r.pos += int64(l)
	}
	return len(p), nil
}

// ring.Copy copies old content to the current position. If the copy source
// overlaps the destination, Copy will produce repeats.
func (r *ring) Copy(start int64, n int) (err error) {
	N := n
	for N > 0 && err == nil {
		n = N
		q := int(start & r.mask)
		// lower piece size (n) if needed
		if start == r.pos { // unsupported but don't hang forever
			return errors.New("zero offset for copy unsupported")
		} else if n > len(r.ring) {
			return errors.New("copies limited to length of hist buffer")
		} else if start+int64(n) > r.pos { // src overlaps dest
			n = int(r.pos - start)
		}
		if q+n > len(r.ring) { // source wraps around
			n = len(r.ring) - q
		}
		// do the copy and any write
		start += int64(n)
		if _, err := r.Write(r.ring[q : q+n]); err != nil {
			return err
		}
		N -= n
	}
	return
}

// Decompress input from rd to w in one shot. Does not handle framing format.
func Decompress(historyBits uint, rd io.Reader, w io.Writer) error {
	br := bufio.NewReader(rd)
	r := newRing(historyBits, w)
	cursor := int64(0)
	literal := [maxReadLiteral]byte{}
	for {
		instr, err := binary.ReadVarint(br)
		if err != nil {
			return err
		}
		if instr > 0 { // copy!
			l := instr
			cursorMove, err := binary.ReadVarint(br)
			if err != nil {
				return err
			}
			cursor += cursorMove
			if err = r.Copy(cursor, int(l)); err != nil {
				return err
			}
			cursor += l
		}
		if instr == 0 { // end of stream!
			return io.EOF
		}
		if instr < 0 { // literal!
			l := -instr
			cursor += l
			if _, err := io.ReadFull(br, literal[:l]); err != nil {
				return err
			}
			if _, err = r.Write(literal[:l]); err != nil {
				return err
			}
		}
	}
}

func NewDecompressor(historyBits uint, rd io.Reader) io.Reader {
	pr, pw := io.Pipe()
	go func() {
		pw.CloseWithError(Decompress(historyBits, rd, pw))
	}()
	return pr
}
<commit_msg>shorter hash and out-of-lining "is this match legit" test are each little wins (and out-of-lining is nicer anyway)<commit_after>package lrcompress

import (
	"bufio"
	"encoding/binary"
	"errors"
	"io"
)

// ring buffer (yep, size baked in)
const CompHistBits = 22           // log2 bytes of history for compression
const rMask = 1<<CompHistBits - 1 // &rMask turns offset into ring pos

// compression hashtable
const hBits = 18           // log2 hashtable size
const hMask = 1<<hBits - 1 // c.hTbl[h>>hShift&hMask] is current hashtable entry
const hShift = 32 - hBits
const fMask = 1<<fBits - 1             // hit hashtable if fBits are 1111...
const fBits = CompHistBits - hBits + 1 // 1/2 fill the table

// output format choices
const window = 64          // bytes that must overlap to match
const maxLiteral = 1 << 16 // we'll write this size literal
const maxMatch = 1 << 18   // max match we output (we'll read larger)

type compRing [1 << CompHistBits]byte
type compHtbl [1 << hBits]int64

// Compressor is a Writer into which you can dump content.
type Compressor struct {
	pos        int64     // count of bytes ever written
	ring       compRing  // the bytes
	h          uint32    // current rolling hash
	matchMin   int64     // first matchable byte (for Reset)
	matchPos   int64     // current match start or 0
	matchLen   int64     // current match length or 0
	cursor     int64     // "expected" match start
	w          io.Writer // compressed output
	literalLen int64     // current literal length or 0
	encodeBuf  [16]byte  // for varints
	hTbl       compHtbl  // hashtable holding offsets into source file
}

// Make a compressor with 1<<CompHistBits of memory, writing output to w.
func NewCompressor(w io.Writer) *Compressor {
	return &Compressor{w: w, pos: 1, cursor: 1}
}

func (c *Compressor) putInt(i int64) (err error) {
	n := binary.PutVarint(c.encodeBuf[:], i)
	_, err = c.w.Write(c.encodeBuf[:n])
	return
}

func (c *Compressor) putMatch(matchPos, matchLen int64) (err error) {
	err = c.putInt(matchLen)
	if err != nil {
		return
	}
	err = c.putInt(matchPos - c.cursor)
	c.cursor = matchPos + matchLen
	return
}

func (c *Compressor) putLiteral(pos, literalLen int64) (err error) {
	if literalLen == 0 {
		return
	}
	err = c.putInt(-literalLen)
	if err != nil {
		return
	}
	if literalLen > pos&rMask {
		_, err = c.w.Write(c.ring[(pos-literalLen)&rMask:])
		if err != nil {
			return
		}
		_, err = c.w.Write(c.ring[:pos&rMask])
	} else {
		_, err = c.w.Write(c.ring[(pos-literalLen)&rMask : pos&rMask])
	}
	if err != nil {
		return
	}
	c.cursor += literalLen
	return
}

// found a potential match; see if it checks out and use it if so
func (c *Compressor) tryMatch(ring *compRing, pos, literalLen, match int64) (matchLen_ int64, err error) {
	matchPos, matchLen := match, int64(1) // 1 because cur. byte matched
	min := pos - rMask + maxLiteral
	if min < c.matchMin {
		min = c.matchMin
	}
	// extend backwards
	for literalLen > 0 &&
		matchPos-1 > min &&
		ring[(pos-matchLen)&rMask] == ring[(matchPos-1)&rMask] {
		literalLen--
		matchPos--
		matchLen++
	}
	if matchLen >= window { // long enough match, flush literal and use it
		// this literal ends before pos-matchLen+1, not pos
		if err = c.putLiteral(pos-matchLen+1, literalLen); err != nil {
			return
		}
		return matchLen, error(nil)
	} else { // short match, ignore
		return 0, error(nil)
	}
}

func (c *Compressor) Write(p []byte) (n int, err error) {
	h, ring, hTbl, pos, matchPos, matchLen, literalLen, matchMin := c.h, &c.ring, &c.hTbl, c.pos, c.matchPos, c.matchLen, c.literalLen, c.matchMin
	for _, b := range p {
		// can use any 32-bit const with least sig. bits=10b and some higher
		// bits set; even *=6 eventually mixes lower bits into the top ones
		h *= ((0x703a03ac|1)*2)&(1<<32-1) | 1<<31
		h ^= uint32(b)
		// if we're in a match, extend or end it
		if matchLen > 0 {
			// try to extend it
			if ring[(matchPos+matchLen)&rMask] == b &&
				matchLen < maxMatch {
				matchLen++
			} else {
				// can't extend it, flush out what we have
				if err = c.putMatch(matchPos, matchLen); err != nil {
					return
				}
				matchPos, matchLen = 0, 0
			}
		} else if literalLen > window && h&fMask == fMask {
			match := hTbl[h>>hShift&hMask]
			// check if it's in usable range and cur. byte matches, then tryMatch
			if match > matchMin && b == ring[match&rMask] && match > pos-rMask+maxLiteral {
				matchLen, err = c.tryMatch(ring, pos, literalLen, match)
				if matchLen > 0 {
					literalLen = 0
					matchPos = match - matchLen + 1
				} else if err != nil {
					return
				}
			}
		}
		// (still) not in a match, so just extend the literal
		if matchLen == 0 {
			if literalLen == maxLiteral {
				if err = c.putLiteral(pos, literalLen); err != nil {
					return
				}
				literalLen = 0
			}
			literalLen++
		}

		// update hashtable and ring
		ring[pos&rMask] = b
		if h&fMask == fMask {
			hTbl[h>>hShift&hMask] = pos
		}
		pos++
	}
	c.h, c.pos, c.matchPos, c.matchLen, c.literalLen = h, pos, matchPos, matchLen, literalLen
	return len(p), nil
}

func (c *Compressor) Flush() (err error) {
	if c.matchLen > 0 {
		err = c.putMatch(c.matchPos, c.matchLen)
		c.matchPos, c.matchLen = 0, 0
	} else {
		err = c.putLiteral(c.pos, c.literalLen)
		c.literalLen = 0
	}
	return
}

// Flush if needed, then write an end marker to the diff output.
func (c *Compressor) Close() (err error) {
	if err = c.Flush(); err != nil {
		return
	}
	return c.putInt(0)
}

// Clear the history and other state. If you've written data, Flush() it first.
func (c *Compressor) Reset() {
	if c.matchPos != 0 || c.literalLen != 0 || c.matchLen != 0 {
		panic("Call Flush() before Reset()")
	}
	c.matchMin = c.pos
	c.cursor = c.pos
	c.pos++
}

// Load data as if it had been written. If you've written real data, Flush() it first.
func (c *Compressor) LoadHistory(d []byte) {
	if c.matchPos != 0 || c.literalLen != 0 || c.matchLen != 0 {
		panic("Call Flush() before LoadHistory()")
	}
	h, pos, hTbl, ring := c.h, c.pos, &c.hTbl, &c.ring
	for _, b := range d {
		h *= ((0x703a03ac|1)*2)&(1<<32-1) | 1<<31
		h ^= uint32(b)
		// update hashtable and ring
		ring[pos&rMask] = b
		if h&fMask == fMask {
			hTbl[h>>hShift&hMask] = pos
		}
		pos++
	}
	c.cursor += int64(len(d))
	c.h, c.pos = h, pos
}

// Change where compressed data is written. If you've written data, Flush() it first.
func (c *Compressor) SetWriter(w io.Writer) {
	if c.matchPos != 0 || c.literalLen != 0 || c.matchLen != 0 {
		panic("Call Flush() before SetWriter()")
	}
	c.w = w
}

const maxReadLiteral = 1 << 16

// ring is a ring-buffer of bytes with Copy and Write operations. Writes and
// copies are teed to an io.Writer provided on initialization.
type ring struct {
	pos  int64     // count of bytes ever written
	mask int64     // &mask turns pos into a ring offset
	w    io.Writer // output of writes/copies goes here as well as ring
	ring []byte    // the bytes
}

func newRing(sizeBits uint, w io.Writer) ring {
	return ring{
		pos:  0,
		mask: 1<<sizeBits - 1,
		w:    w,
		ring: make([]byte, 1<<sizeBits),
	}
}

func (r *ring) Write(p []byte) (n int, err error) {
	n, err = r.w.Write(p)
	if err != nil {
		return
	}
	for len(p) > 0 && err == nil {
		l := len(p)
		pos := int(r.pos & r.mask)
		if pos+l > len(r.ring) {
			l = len(r.ring) - pos
		}
		copy(r.ring[pos:pos+l], p[:l])
		p = p[l:]
		r.pos += int64(l)
	}
	return len(p), nil
}

// ring.Copy copies old content to the current position. If the copy source
// overlaps the destination, Copy will produce repeats.
func (r *ring) Copy(start int64, n int) (err error) {
	N := n
	for N > 0 && err == nil {
		n = N
		q := int(start & r.mask)
		// lower piece size (n) if needed
		if start == r.pos { // unsupported but don't hang forever
			return errors.New("zero offset for copy unsupported")
		} else if n > len(r.ring) {
			return errors.New("copies limited to length of hist buffer")
		} else if start+int64(n) > r.pos { // src overlaps dest
			n = int(r.pos - start)
		}
		if q+n > len(r.ring) { // source wraps around
			n = len(r.ring) - q
		}
		// do the copy and any write
		start += int64(n)
		if _, err := r.Write(r.ring[q : q+n]); err != nil {
			return err
		}
		N -= n
	}
	return
}

// Decompress input from rd to w in one shot. Does not handle framing format.
func Decompress(historyBits uint, rd io.Reader, w io.Writer) error {
	br := bufio.NewReader(rd)
	r := newRing(historyBits, w)
	cursor := int64(0)
	literal := [maxReadLiteral]byte{}
	for {
		instr, err := binary.ReadVarint(br)
		if err != nil {
			return err
		}
		if instr > 0 { // copy!
			l := instr
			cursorMove, err := binary.ReadVarint(br)
			if err != nil {
				return err
			}
			cursor += cursorMove
			if err = r.Copy(cursor, int(l)); err != nil {
				return err
			}
			cursor += l
		}
		if instr == 0 { // end of stream!
			return io.EOF
		}
		if instr < 0 { // literal!
			l := -instr
			cursor += l
			if _, err := io.ReadFull(br, literal[:l]); err != nil {
				return err
			}
			if _, err = r.Write(literal[:l]); err != nil {
				return err
			}
		}
	}
}

func NewDecompressor(historyBits uint, rd io.Reader) io.Reader {
	pr, pw := io.Pipe()
	go func() {
		pw.CloseWithError(Decompress(historyBits, rd, pw))
	}()
	return pr
}
<|endoftext|>


<commit_before>package address

import (
	"googlemaps.github.io/maps"
)

type componentType string

const (
	StreetNumber             componentType = "street_number"
	Route                    componentType = "route"
	Subpremise               componentType = "subpremise"
	AdministrativeAreaLevel1 componentType = "administrative_area_level_1"
	Country                  componentType = "country"
	PostalCode               componentType = "postal_code"
)

//FromGoogleAddressComponents creates an Addresss from a slice of components, using the AddressCompoment.Types to discriminate.
func FromGoogleAddressComponents(addressComponents []maps.AddressComponent, whitelist ...componentType) (address Address) {
	var street struct{ name, number string }
	for _, component := range addressComponents {
		val := component.ShortName
		for _, label := range component.Types {
			if !isWhitelisted(whitelist, label) {
				continue
			}
			switch label {
			case "street_number":
				street.number = val
			case "route":
				street.name = val
			case "subpremise":
				address.Extension = val
			case "administrative_area_level_1":
				address.Region = val
			case "country":
				address.Country = val
			case "postal_code":
				address.PostalCode = val
			default: //pass
			}
			//note: PO box doesn't seem to be handled.
		}
	}

	if street.name != "" {
		if street.number != "" {
			address.Street = street.number + " " + street.name
		} else {
			address.Street = street.name
		}
	}

	return address
}

func isWhitelisted(whitelist []componentType, componentLabel string) bool {
	if len(whitelist) == 0 {
		return true
	}

	for _, wl := range whitelist {
		if string(wl) == componentLabel {
			return true
		}
	}

	return false
}<commit_msg>Add locality address component<commit_after>package address

import (
	"googlemaps.github.io/maps"
)

type componentType string

const (
	StreetNumber             componentType = "street_number"
	Route                    componentType = "route"                       //street
	Subpremise               componentType = "subpremise"                  //unit number
	Locality                 componentType = "Locality"                    //city
	AdministrativeAreaLevel1 componentType = "administrative_area_level_1" //state
	PostalCode               componentType = "postal_code"                 //zip
	Country                  componentType = "country"
)

//FromGoogleAddressComponents creates an Addresss from a slice of components, using the AddressCompoment.Types to discriminate.
func FromGoogleAddressComponents(addressComponents []maps.AddressComponent, whitelist ...componentType) (address Address) {
	var street struct{ name, number string }
	for _, component := range addressComponents {
		val := component.ShortName
		for _, label := range component.Types {
			if !isWhitelisted(whitelist, label) {
				continue
			}
			switch label {
			case "street_number":
				street.number = val
			case "route":
				street.name = val
			case "subpremise":
				address.Extension = val
			case "locality":
				address.Locality = val
			case "administrative_area_level_1":
				address.Region = val
			case "country":
				address.Country = val
			case "postal_code":
				address.PostalCode = val
			default: //pass
			}
			//note: PO box doesn't seem to be handled.
		}
	}

	if street.name != "" {
		if street.number != "" {
			address.Street = street.number + " " + street.name
		} else {
			address.Street = street.name
		}
	}

	return address
}

func isWhitelisted(whitelist []componentType, componentLabel string) bool {
	if len(whitelist) == 0 {
		return true
	}

	for _, wl := range whitelist {
		if string(wl) == componentLabel {
			return true
		}
	}

	return false
}
<|endoftext|>


<commit_before>package response

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

var debug bool

// Init sets the debug variable to the provided value.
func Init(d bool) {
	debug = d
}

// Response represents an API response
type Response interface {
	Render(w http.ResponseWriter) error
	String() string
}

// Sync response
type syncResponse struct {
	success   bool
	etag      interface{}
	metadata  interface{}
	location  string
	code      int
	headers   map[string]string
	plaintext bool
}

// EmptySyncResponse represents an empty syncResponse.
var EmptySyncResponse = &syncResponse{success: true, metadata: make(map[string]interface{})}

// SyncResponse returns a new syncResponse with the success and metadata fields
// set to the provided values.
func SyncResponse(success bool, metadata interface{}) Response {
	return &syncResponse{success: success, metadata: metadata}
}

// SyncResponseETag returns a new syncResponse with an etag.
func SyncResponseETag(success bool, metadata interface{}, etag interface{}) Response {
	return &syncResponse{success: success, metadata: metadata, etag: etag}
}

// SyncResponseLocation returns a new syncResponse with a location.
func SyncResponseLocation(success bool, metadata interface{}, location string) Response {
	return &syncResponse{success: success, metadata: metadata, location: location}
}

// SyncResponseRedirect returns a new syncResponse with a location, indicating
// a permanent redirect.
func SyncResponseRedirect(address string) Response {
	return &syncResponse{success: true, location: address, code: http.StatusPermanentRedirect}
}

// SyncResponseHeaders returns a new syncResponse with headers.
func SyncResponseHeaders(success bool, metadata interface{}, headers map[string]string) Response {
	return &syncResponse{success: success, metadata: metadata, headers: headers}
}

// SyncResponsePlain return a new syncResponse with plaintext.
func SyncResponsePlain(success bool, metadata string) Response {
	return &syncResponse{success: success, metadata: metadata, plaintext: true}
}

func (r *syncResponse) Render(w http.ResponseWriter) error {
	// Set an appropriate ETag header
	if r.etag != nil {
		etag, err := util.EtagHash(r.etag)
		if err == nil {
			w.Header().Set("ETag", fmt.Sprintf("\"%s\"", etag))
		}
	}

	// Prepare the JSON response
	status := api.Success
	if !r.success {
		status = api.Failure
	}

	if r.headers != nil {
		for h, v := range r.headers {
			w.Header().Set(h, v)
		}
	}

	code := r.code

	if r.location != "" {
		w.Header().Set("Location", r.location)
		if code == 0 {
			code = 201
		}
	}

	// Handle plain text headers.
	if r.plaintext {
		w.Header().Set("Content-Type", "text/plain")
	}

	// Write header and status code.
	if code == 0 {
		code = http.StatusOK
	}

	w.WriteHeader(code)

	// Handle plain text responses.
	if r.plaintext {
		if r.metadata != nil {
			_, err := w.Write([]byte(r.metadata.(string)))
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Handle JSON responses.
	resp := api.ResponseRaw{
		Type:       api.SyncResponse,
		Status:     status.String(),
		StatusCode: int(status),
		Metadata:   r.metadata,
	}

	var debugLogger logger.Logger
	if debug {
		debugLogger = logging.AddContext(logger.Log, log.Ctx{"http_code": code})
	}

	return util.WriteJSON(w, resp, debugLogger)
}

func (r *syncResponse) String() string {
	if r.success {
		return "success"
	}

	return "failure"
}

// Error response
type errorResponse struct {
	code int
	msg  string
}

// ErrorResponse returns an error response with the given code and msg.
func ErrorResponse(code int, msg string) Response {
	return &errorResponse{code, msg}
}

// BadRequest returns a bad request response (400) with the given error.
func BadRequest(err error) Response {
	return &errorResponse{http.StatusBadRequest, err.Error()}
}

// Conflict returns a conflict response (409) with the given error.
func Conflict(err error) Response {
	message := "already exists"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusConflict, message}
}

// Forbidden returns a forbidden response (403) with the given error.
func Forbidden(err error) Response {
	message := "not authorized"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusForbidden, message}
}

// InternalError returns an internal error response (500) with the given error.
func InternalError(err error) Response {
	return &errorResponse{http.StatusInternalServerError, err.Error()}
}

// NotFound returns a not found response (404) with the given error.
func NotFound(err error) Response {
	message := "not found"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusNotFound, message}
}

// NotImplemented returns a not implemented response (501) with the given error.
func NotImplemented(err error) Response {
	message := "not implemented"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusNotImplemented, message}
}

// PreconditionFailed returns a precondition failed response (412) with the
// given error.
func PreconditionFailed(err error) Response {
	return &errorResponse{http.StatusPreconditionFailed, err.Error()}
}

// Unavailable return an unavailable response (503) with the given error.
func Unavailable(err error) Response {
	message := "unavailable"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusServiceUnavailable, message}
}

func (r *errorResponse) String() string {
	return r.msg
}

func (r *errorResponse) Render(w http.ResponseWriter) error {
	var output io.Writer

	buf := &bytes.Buffer{}
	output = buf
	var captured *bytes.Buffer
	if debug {
		captured = &bytes.Buffer{}
		output = io.MultiWriter(buf, captured)
	}

	err := json.NewEncoder(output).Encode(shared.Jmap{"type": api.ErrorResponse, "error": r.msg, "error_code": r.code})

	if err != nil {
		return err
	}

	if debug {
		debugLogger := logging.AddContext(logger.Log, log.Ctx{"http_code": r.code})
		util.DebugJSON("Error Response", captured, debugLogger)
	}

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("X-Content-Type-Options", "nosniff")
	w.WriteHeader(r.code)
	fmt.Fprintln(w, buf.String())

	return nil
}

// FileResponseEntry represents a file response entry.
type FileResponseEntry struct {
	Identifier string
	Path       string
	Filename   string
	Buffer     []byte /* either a path or a buffer must be provided */
}

type fileResponse struct {
	req              *http.Request
	files            []FileResponseEntry
	headers          map[string]string
	removeAfterServe bool
}

// FileResponse returns a new file response.
func FileResponse(r *http.Request, files []FileResponseEntry, headers map[string]string, removeAfterServe bool) Response {
	return &fileResponse{r, files, headers, removeAfterServe}
}

func (r *fileResponse) Render(w http.ResponseWriter) error {
	if r.headers != nil {
		for k, v := range r.headers {
			w.Header().Set(k, v)
		}
	}

	// No file, well, it's easy then
	if len(r.files) == 0 {
		return nil
	}

	// For a single file, return it inline
	if len(r.files) == 1 {
		var rs io.ReadSeeker
		var mt time.Time
		var sz int64

		if r.files[0].Path == "" {
			rs = bytes.NewReader(r.files[0].Buffer)
			mt = time.Now()
			sz = int64(len(r.files[0].Buffer))
		} else {
			f, err := os.Open(r.files[0].Path)
			if err != nil {
				return err
			}
			defer f.Close()

			fi, err := f.Stat()
			if err != nil {
				return err
			}

			mt = fi.ModTime()
			sz = fi.Size()
			rs = f
		}

		w.Header().Set("Content-Type", "application/octet-stream")
		w.Header().Set("Content-Length", fmt.Sprintf("%d", sz))
		w.Header().Set("Content-Disposition", fmt.Sprintf("inline;filename=%s", r.files[0].Filename))

		http.ServeContent(w, r.req, r.files[0].Filename, mt, rs)
		if r.files[0].Path != "" && r.removeAfterServe {
			err := os.Remove(r.files[0].Path)
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Now the complex multipart answer.
	mw := multipart.NewWriter(w)
	defer mw.Close()

	w.Header().Set("Content-Type", mw.FormDataContentType())
	w.Header().Set("Transfer-Encoding", "chunked")

	for _, entry := range r.files {
		var rd io.Reader
		if entry.Path != "" {
			fd, err := os.Open(entry.Path)
			if err != nil {
				return err
			}
			defer fd.Close()

			rd = fd
		} else {
			rd = bytes.NewReader(entry.Buffer)
		}

		fw, err := mw.CreateFormFile(entry.Identifier, entry.Filename)
		if err != nil {
			return err
		}

		_, err = io.Copy(fw, rd)
		if err != nil {
			return err
		}
	}

	return nil
}

func (r *fileResponse) String() string {
	return fmt.Sprintf("%d files", len(r.files))
}

type forwardedResponse struct {
	client  lxd.InstanceServer
	request *http.Request
}

// ForwardedResponse takes a request directed to a node and forwards it to
// another node, writing back the response it gegs.
func ForwardedResponse(client lxd.InstanceServer, request *http.Request) Response {
	return &forwardedResponse{
		client:  client,
		request: request,
	}
}

func (r *forwardedResponse) Render(w http.ResponseWriter) error {
	info, err := r.client.GetConnectionInfo()
	if err != nil {
		return err
	}

	url := fmt.Sprintf("%s%s", info.Addresses[0], r.request.URL.RequestURI())
	forwarded, err := http.NewRequest(r.request.Method, url, r.request.Body)
	if err != nil {
		return err
	}

	for key := range r.request.Header {
		forwarded.Header.Set(key, r.request.Header.Get(key))
	}

	httpClient, err := r.client.GetHTTPClient()
	if err != nil {
		return err
	}

	response, err := httpClient.Do(forwarded)
	if err != nil {
		return err
	}

	for key := range response.Header {
		w.Header().Set(key, response.Header.Get(key))
	}

	w.WriteHeader(response.StatusCode)
	_, err = io.Copy(w, response.Body)
	return err
}

func (r *forwardedResponse) String() string {
	return fmt.Sprintf("request to %s", r.request.URL)
}
<commit_msg>lxd/response/response: Use api.ResponseRaw in error response<commit_after>package response

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"time"

	"github.com/lxc/lxd/client"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

var debug bool

// Init sets the debug variable to the provided value.
func Init(d bool) {
	debug = d
}

// Response represents an API response
type Response interface {
	Render(w http.ResponseWriter) error
	String() string
}

// Sync response
type syncResponse struct {
	success   bool
	etag      interface{}
	metadata  interface{}
	location  string
	code      int
	headers   map[string]string
	plaintext bool
}

// EmptySyncResponse represents an empty syncResponse.
var EmptySyncResponse = &syncResponse{success: true, metadata: make(map[string]interface{})}

// SyncResponse returns a new syncResponse with the success and metadata fields
// set to the provided values.
func SyncResponse(success bool, metadata interface{}) Response {
	return &syncResponse{success: success, metadata: metadata}
}

// SyncResponseETag returns a new syncResponse with an etag.
func SyncResponseETag(success bool, metadata interface{}, etag interface{}) Response {
	return &syncResponse{success: success, metadata: metadata, etag: etag}
}

// SyncResponseLocation returns a new syncResponse with a location.
func SyncResponseLocation(success bool, metadata interface{}, location string) Response {
	return &syncResponse{success: success, metadata: metadata, location: location}
}

// SyncResponseRedirect returns a new syncResponse with a location, indicating
// a permanent redirect.
func SyncResponseRedirect(address string) Response {
	return &syncResponse{success: true, location: address, code: http.StatusPermanentRedirect}
}

// SyncResponseHeaders returns a new syncResponse with headers.
func SyncResponseHeaders(success bool, metadata interface{}, headers map[string]string) Response {
	return &syncResponse{success: success, metadata: metadata, headers: headers}
}

// SyncResponsePlain return a new syncResponse with plaintext.
func SyncResponsePlain(success bool, metadata string) Response {
	return &syncResponse{success: success, metadata: metadata, plaintext: true}
}

func (r *syncResponse) Render(w http.ResponseWriter) error {
	// Set an appropriate ETag header
	if r.etag != nil {
		etag, err := util.EtagHash(r.etag)
		if err == nil {
			w.Header().Set("ETag", fmt.Sprintf("\"%s\"", etag))
		}
	}

	// Prepare the JSON response
	status := api.Success
	if !r.success {
		status = api.Failure
	}

	if r.headers != nil {
		for h, v := range r.headers {
			w.Header().Set(h, v)
		}
	}

	code := r.code

	if r.location != "" {
		w.Header().Set("Location", r.location)
		if code == 0 {
			code = 201
		}
	}

	// Handle plain text headers.
	if r.plaintext {
		w.Header().Set("Content-Type", "text/plain")
	}

	// Write header and status code.
	if code == 0 {
		code = http.StatusOK
	}

	w.WriteHeader(code)

	// Handle plain text responses.
	if r.plaintext {
		if r.metadata != nil {
			_, err := w.Write([]byte(r.metadata.(string)))
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Handle JSON responses.
	resp := api.ResponseRaw{
		Type:       api.SyncResponse,
		Status:     status.String(),
		StatusCode: int(status),
		Metadata:   r.metadata,
	}

	var debugLogger logger.Logger
	if debug {
		debugLogger = logging.AddContext(logger.Log, log.Ctx{"http_code": code})
	}

	return util.WriteJSON(w, resp, debugLogger)
}

func (r *syncResponse) String() string {
	if r.success {
		return "success"
	}

	return "failure"
}

// Error response
type errorResponse struct {
	code int    // Code to return in both the HTTP header and Code field of the response body.
	msg  string // Message to return in the Error field of the response body.
}

// ErrorResponse returns an error response with the given code and msg.
func ErrorResponse(code int, msg string) Response {
	return &errorResponse{code, msg}
}

// BadRequest returns a bad request response (400) with the given error.
func BadRequest(err error) Response {
	return &errorResponse{http.StatusBadRequest, err.Error()}
}

// Conflict returns a conflict response (409) with the given error.
func Conflict(err error) Response {
	message := "already exists"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusConflict, message}
}

// Forbidden returns a forbidden response (403) with the given error.
func Forbidden(err error) Response {
	message := "not authorized"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusForbidden, message}
}

// InternalError returns an internal error response (500) with the given error.
func InternalError(err error) Response {
	return &errorResponse{http.StatusInternalServerError, err.Error()}
}

// NotFound returns a not found response (404) with the given error.
func NotFound(err error) Response {
	message := "not found"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusNotFound, message}
}

// NotImplemented returns a not implemented response (501) with the given error.
func NotImplemented(err error) Response {
	message := "not implemented"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusNotImplemented, message}
}

// PreconditionFailed returns a precondition failed response (412) with the
// given error.
func PreconditionFailed(err error) Response {
	return &errorResponse{http.StatusPreconditionFailed, err.Error()}
}

// Unavailable return an unavailable response (503) with the given error.
func Unavailable(err error) Response {
	message := "unavailable"
	if err != nil {
		message = err.Error()
	}

	return &errorResponse{http.StatusServiceUnavailable, message}
}

func (r *errorResponse) String() string {
	return r.msg
}

func (r *errorResponse) Render(w http.ResponseWriter) error {
	var output io.Writer

	buf := &bytes.Buffer{}
	output = buf
	var captured *bytes.Buffer
	if debug {
		captured = &bytes.Buffer{}
		output = io.MultiWriter(buf, captured)
	}

	resp := api.ResponseRaw{
		Type:  api.ErrorResponse,
		Error: r.msg,
		Code:  r.code, // Set the error code in the Code field of the response body.
	}

	err := json.NewEncoder(output).Encode(resp)

	if err != nil {
		return err
	}

	if debug {
		debugLogger := logging.AddContext(logger.Log, log.Ctx{"http_code": r.code})
		util.DebugJSON("Error Response", captured, debugLogger)
	}

	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("X-Content-Type-Options", "nosniff")

	w.WriteHeader(r.code) // Set the error code in the HTTP header response.

	fmt.Fprintln(w, buf.String())

	return nil
}

// FileResponseEntry represents a file response entry.
type FileResponseEntry struct {
	Identifier string
	Path       string
	Filename   string
	Buffer     []byte /* either a path or a buffer must be provided */
}

type fileResponse struct {
	req              *http.Request
	files            []FileResponseEntry
	headers          map[string]string
	removeAfterServe bool
}

// FileResponse returns a new file response.
func FileResponse(r *http.Request, files []FileResponseEntry, headers map[string]string, removeAfterServe bool) Response {
	return &fileResponse{r, files, headers, removeAfterServe}
}

func (r *fileResponse) Render(w http.ResponseWriter) error {
	if r.headers != nil {
		for k, v := range r.headers {
			w.Header().Set(k, v)
		}
	}

	// No file, well, it's easy then
	if len(r.files) == 0 {
		return nil
	}

	// For a single file, return it inline
	if len(r.files) == 1 {
		var rs io.ReadSeeker
		var mt time.Time
		var sz int64

		if r.files[0].Path == "" {
			rs = bytes.NewReader(r.files[0].Buffer)
			mt = time.Now()
			sz = int64(len(r.files[0].Buffer))
		} else {
			f, err := os.Open(r.files[0].Path)
			if err != nil {
				return err
			}
			defer f.Close()

			fi, err := f.Stat()
			if err != nil {
				return err
			}

			mt = fi.ModTime()
			sz = fi.Size()
			rs = f
		}

		w.Header().Set("Content-Type", "application/octet-stream")
		w.Header().Set("Content-Length", fmt.Sprintf("%d", sz))
		w.Header().Set("Content-Disposition", fmt.Sprintf("inline;filename=%s", r.files[0].Filename))

		http.ServeContent(w, r.req, r.files[0].Filename, mt, rs)
		if r.files[0].Path != "" && r.removeAfterServe {
			err := os.Remove(r.files[0].Path)
			if err != nil {
				return err
			}
		}

		return nil
	}

	// Now the complex multipart answer.
	mw := multipart.NewWriter(w)
	defer mw.Close()

	w.Header().Set("Content-Type", mw.FormDataContentType())
	w.Header().Set("Transfer-Encoding", "chunked")

	for _, entry := range r.files {
		var rd io.Reader
		if entry.Path != "" {
			fd, err := os.Open(entry.Path)
			if err != nil {
				return err
			}
			defer fd.Close()

			rd = fd
		} else {
			rd = bytes.NewReader(entry.Buffer)
		}

		fw, err := mw.CreateFormFile(entry.Identifier, entry.Filename)
		if err != nil {
			return err
		}

		_, err = io.Copy(fw, rd)
		if err != nil {
			return err
		}
	}

	return nil
}

func (r *fileResponse) String() string {
	return fmt.Sprintf("%d files", len(r.files))
}

type forwardedResponse struct {
	client  lxd.InstanceServer
	request *http.Request
}

// ForwardedResponse takes a request directed to a node and forwards it to
// another node, writing back the response it gegs.
func ForwardedResponse(client lxd.InstanceServer, request *http.Request) Response {
	return &forwardedResponse{
		client:  client,
		request: request,
	}
}

func (r *forwardedResponse) Render(w http.ResponseWriter) error {
	info, err := r.client.GetConnectionInfo()
	if err != nil {
		return err
	}

	url := fmt.Sprintf("%s%s", info.Addresses[0], r.request.URL.RequestURI())
	forwarded, err := http.NewRequest(r.request.Method, url, r.request.Body)
	if err != nil {
		return err
	}

	for key := range r.request.Header {
		forwarded.Header.Set(key, r.request.Header.Get(key))
	}

	httpClient, err := r.client.GetHTTPClient()
	if err != nil {
		return err
	}

	response, err := httpClient.Do(forwarded)
	if err != nil {
		return err
	}

	for key := range response.Header {
		w.Header().Set(key, response.Header.Get(key))
	}

	w.WriteHeader(response.StatusCode)
	_, err = io.Copy(w, response.Body)
	return err
}

func (r *forwardedResponse) String() string {
	return fmt.Sprintf("request to %s", r.request.URL)
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"path"
	"strconv"
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
)

const (
	streamTypeMessage  streamType = "message"
	streamTypeMsgAppV2 streamType = "msgappv2"
	streamTypeMsgApp   streamType = "msgapp"

	streamBufSize = 4096
)

type streamType string

func (t streamType) endpoint() string {
	switch t {
	case streamTypeMsgApp: // for backward compatibility of v2.0
		return RaftStreamPrefix
	case streamTypeMsgAppV2:
		return path.Join(RaftStreamPrefix, "msgapp")
	case streamTypeMessage:
		return path.Join(RaftStreamPrefix, "message")
	default:
		log.Panicf("rafthttp: unhandled stream type %v", t)
		return ""
	}
}

var (
	// linkHeartbeatMessage is a special message used as heartbeat message in
	// link layer. It never conflicts with messages from raft because raft
	// doesn't send out messages without From and To fields.
	linkHeartbeatMessage = raftpb.Message{Type: raftpb.MsgHeartbeat}
)

func isLinkHeartbeatMessage(m raftpb.Message) bool {
	return m.Type == raftpb.MsgHeartbeat && m.From == 0 && m.To == 0
}

type outgoingConn struct {
	t       streamType
	termStr string
	io.Writer
	http.Flusher
	io.Closer
}

// streamWriter is a long-running go-routine that writes messages into the
// attached outgoingConn.
type streamWriter struct {
	id types.ID
	fs *stats.FollowerStats
	r  Raft

	mu      sync.Mutex // guard field working and closer
	closer  io.Closer
	working bool

	msgc  chan raftpb.Message
	connc chan *outgoingConn
	stopc chan struct{}
	done  chan struct{}
}

func startStreamWriter(id types.ID, fs *stats.FollowerStats, r Raft) *streamWriter {
	w := &streamWriter{
		id:    id,
		fs:    fs,
		r:     r,
		msgc:  make(chan raftpb.Message, streamBufSize),
		connc: make(chan *outgoingConn),
		stopc: make(chan struct{}),
		done:  make(chan struct{}),
	}
	go w.run()
	return w
}

func (cw *streamWriter) run() {
	var msgc chan raftpb.Message
	var heartbeatc <-chan time.Time
	var t streamType
	var msgAppTerm uint64
	var enc encoder
	var flusher http.Flusher
	tickc := time.Tick(ConnReadTimeout / 3)

	for {
		select {
		case <-heartbeatc:
			start := time.Now()
			if err := enc.encode(linkHeartbeatMessage); err != nil {
				reportSentFailure(string(t), linkHeartbeatMessage)

				log.Printf("rafthttp: failed to heartbeat on stream %s due to %v. waiting for a new stream to be established.", t, err)
				cw.resetCloser()
				heartbeatc, msgc = nil, nil
				continue
			}
			flusher.Flush()
			reportSentDuration(string(t), linkHeartbeatMessage, time.Since(start))
		case m := <-msgc:
			if t == streamTypeMsgApp && m.Term != msgAppTerm {
				// TODO: reasonable retry logic
				if m.Term > msgAppTerm {
					cw.resetCloser()
					heartbeatc, msgc = nil, nil
					// TODO: report to raft at peer level
					cw.r.ReportUnreachable(m.To)
				}
				continue
			}
			start := time.Now()
			if err := enc.encode(m); err != nil {
				reportSentFailure(string(t), m)

				log.Printf("rafthttp: failed to send message on stream %s due to %v. waiting for a new stream to be established.", t, err)
				cw.resetCloser()
				heartbeatc, msgc = nil, nil
				cw.r.ReportUnreachable(m.To)
				continue
			}
			flusher.Flush()
			reportSentDuration(string(t), m, time.Since(start))
		case conn := <-cw.connc:
			cw.resetCloser()
			t = conn.t
			switch conn.t {
			case streamTypeMsgApp:
				var err error
				msgAppTerm, err = strconv.ParseUint(conn.termStr, 10, 64)
				if err != nil {
					log.Panicf("rafthttp: unexpected parse term %s error: %v", conn.termStr, err)
				}
				enc = &msgAppEncoder{w: conn.Writer, fs: cw.fs}
			case streamTypeMsgAppV2:
				enc = newMsgAppV2Encoder(conn.Writer, cw.fs)
			case streamTypeMessage:
				enc = &messageEncoder{w: conn.Writer}
			default:
				log.Panicf("rafthttp: unhandled stream type %s", conn.t)
			}
			flusher = conn.Flusher
			cw.mu.Lock()
			cw.closer = conn.Closer
			cw.working = true
			cw.mu.Unlock()
			heartbeatc, msgc = tickc, cw.msgc
		case <-cw.stopc:
			cw.resetCloser()
			close(cw.done)
			return
		}
	}
}

func (cw *streamWriter) writec() (chan<- raftpb.Message, bool) {
	cw.mu.Lock()
	defer cw.mu.Unlock()
	return cw.msgc, cw.working
}

func (cw *streamWriter) resetCloser() {
	cw.mu.Lock()
	defer cw.mu.Unlock()
	if !cw.working {
		return
	}
	cw.closer.Close()
	if len(cw.msgc) > 0 {
		cw.r.ReportUnreachable(uint64(cw.id))
	}
	cw.msgc = make(chan raftpb.Message, streamBufSize)
	cw.working = false
}

func (cw *streamWriter) attach(conn *outgoingConn) bool {
	select {
	case cw.connc <- conn:
		return true
	case <-cw.done:
		return false
	}
}

func (cw *streamWriter) stop() {
	close(cw.stopc)
	<-cw.done
}

// streamReader is a long-running go-routine that dials to the remote stream
// endponit and reads messages from the response body returned.
type streamReader struct {
	tr       http.RoundTripper
	picker   *urlPicker
	t        streamType
	from, to types.ID
	cid      types.ID
	recvc    chan<- raftpb.Message
	propc    chan<- raftpb.Message
	errorc   chan<- error

	mu         sync.Mutex
	msgAppTerm uint64
	req        *http.Request
	closer     io.Closer
	stopc      chan struct{}
	done       chan struct{}
}

func startStreamReader(tr http.RoundTripper, picker *urlPicker, t streamType, from, to, cid types.ID, recvc chan<- raftpb.Message, propc chan<- raftpb.Message, errorc chan<- error) *streamReader {
	r := &streamReader{
		tr:     tr,
		picker: picker,
		t:      t,
		from:   from,
		to:     to,
		cid:    cid,
		recvc:  recvc,
		propc:  propc,
		errorc: errorc,
		stopc:  make(chan struct{}),
		done:   make(chan struct{}),
	}
	go r.run()
	return r
}

func (cr *streamReader) run() {
	for {
		rc, err := cr.dial()
		if err != nil {
			log.Printf("rafthttp: roundtripping error: %v", err)
		} else {
			err := cr.decodeLoop(rc)
			if err != io.EOF && !isClosedConnectionError(err) {
				log.Printf("rafthttp: failed to read message on stream %s due to %v", cr.t, err)
			}
		}
		select {
		// Wait 100ms to create a new stream, so it doesn't bring too much
		// overhead when retry.
		case <-time.After(100 * time.Millisecond):
		case <-cr.stopc:
			close(cr.done)
			return
		}
	}
}

func (cr *streamReader) decodeLoop(rc io.ReadCloser) error {
	var dec decoder
	cr.mu.Lock()
	switch cr.t {
	case streamTypeMsgApp:
		dec = &msgAppDecoder{r: rc, local: cr.from, remote: cr.to, term: cr.msgAppTerm}
	case streamTypeMsgAppV2:
		dec = newMsgAppV2Decoder(rc, cr.from, cr.to)
	case streamTypeMessage:
		dec = &messageDecoder{r: rc}
	default:
		log.Panicf("rafthttp: unhandled stream type %s", cr.t)
	}
	cr.closer = rc
	cr.mu.Unlock()

	for {
		m, err := dec.decode()
		switch {
		case err != nil:
			cr.mu.Lock()
			cr.resetCloser()
			cr.mu.Unlock()
			return err
		case isLinkHeartbeatMessage(m):
			// do nothing for linkHeartbeatMessage
		default:
			recvc := cr.recvc
			if m.Type == raftpb.MsgProp {
				recvc = cr.propc
			}
			select {
			case recvc <- m:
			default:
				log.Printf("rafthttp: dropping %s from %x because receive buffer is blocked",
					m.Type, m.From)
			}
		}
	}
}

func (cr *streamReader) updateMsgAppTerm(term uint64) {
	cr.mu.Lock()
	defer cr.mu.Unlock()
	if cr.msgAppTerm == term {
		return
	}
	cr.msgAppTerm = term
	cr.resetCloser()
}

// TODO: always cancel in-flight dial and decode
func (cr *streamReader) stop() {
	close(cr.stopc)
	cr.mu.Lock()
	cr.cancelRequest()
	cr.resetCloser()
	cr.mu.Unlock()
	<-cr.done
}

func (cr *streamReader) isWorking() bool {
	cr.mu.Lock()
	defer cr.mu.Unlock()
	return cr.closer != nil
}

func (cr *streamReader) dial() (io.ReadCloser, error) {
	u := cr.picker.pick()
	cr.mu.Lock()
	term := cr.msgAppTerm
	cr.mu.Unlock()

	uu := u
	uu.Path = path.Join(cr.t.endpoint(), cr.from.String())
	req, err := http.NewRequest("GET", uu.String(), nil)
	if err != nil {
		cr.picker.unreachable(u)
		return nil, fmt.Errorf("new request to %s error: %v", u, err)
	}
	req.Header.Set("X-Etcd-Cluster-ID", cr.cid.String())
	req.Header.Set("X-Raft-To", cr.to.String())
	if cr.t == streamTypeMsgApp {
		req.Header.Set("X-Raft-Term", strconv.FormatUint(term, 10))
	}
	cr.mu.Lock()
	cr.req = req
	cr.mu.Unlock()
	resp, err := cr.tr.RoundTrip(req)
	if err != nil {
		cr.picker.unreachable(u)
		return nil, fmt.Errorf("error roundtripping to %s: %v", req.URL, err)
	}
	switch resp.StatusCode {
	case http.StatusGone:
		resp.Body.Close()
		err := fmt.Errorf("the member has been permanently removed from the cluster")
		select {
		case cr.errorc <- err:
		default:
		}
		return nil, err
	case http.StatusOK:
		return resp.Body, nil
	default:
		resp.Body.Close()
		return nil, fmt.Errorf("unhandled http status %d", resp.StatusCode)
	}
}

func (cr *streamReader) cancelRequest() {
	if canceller, ok := cr.tr.(*http.Transport); ok {
		canceller.CancelRequest(cr.req)
	}
}

func (cr *streamReader) resetCloser() {
	if cr.closer != nil {
		cr.closer.Close()
	}
	cr.closer = nil
}

func canUseMsgAppStream(m raftpb.Message) bool {
	return m.Type == raftpb.MsgApp && m.Term == m.LogTerm
}

func isClosedConnectionError(err error) bool {
	operr, ok := err.(*net.OpError)
	return ok && operr.Err.Error() == "use of closed network connection"
}
<commit_msg>rafthttp: resetCloser -> close<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rafthttp

import (
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"path"
	"strconv"
	"sync"
	"time"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
)

const (
	streamTypeMessage  streamType = "message"
	streamTypeMsgAppV2 streamType = "msgappv2"
	streamTypeMsgApp   streamType = "msgapp"

	streamBufSize = 4096
)

type streamType string

func (t streamType) endpoint() string {
	switch t {
	case streamTypeMsgApp: // for backward compatibility of v2.0
		return RaftStreamPrefix
	case streamTypeMsgAppV2:
		return path.Join(RaftStreamPrefix, "msgapp")
	case streamTypeMessage:
		return path.Join(RaftStreamPrefix, "message")
	default:
		log.Panicf("rafthttp: unhandled stream type %v", t)
		return ""
	}
}

var (
	// linkHeartbeatMessage is a special message used as heartbeat message in
	// link layer. It never conflicts with messages from raft because raft
	// doesn't send out messages without From and To fields.
	linkHeartbeatMessage = raftpb.Message{Type: raftpb.MsgHeartbeat}
)

func isLinkHeartbeatMessage(m raftpb.Message) bool {
	return m.Type == raftpb.MsgHeartbeat && m.From == 0 && m.To == 0
}

type outgoingConn struct {
	t       streamType
	termStr string
	io.Writer
	http.Flusher
	io.Closer
}

// streamWriter is a long-running go-routine that writes messages into the
// attached outgoingConn.
type streamWriter struct {
	id types.ID
	fs *stats.FollowerStats
	r  Raft

	mu      sync.Mutex // guard field working and closer
	closer  io.Closer
	working bool

	msgc  chan raftpb.Message
	connc chan *outgoingConn
	stopc chan struct{}
	done  chan struct{}
}

func startStreamWriter(id types.ID, fs *stats.FollowerStats, r Raft) *streamWriter {
	w := &streamWriter{
		id:    id,
		fs:    fs,
		r:     r,
		msgc:  make(chan raftpb.Message, streamBufSize),
		connc: make(chan *outgoingConn),
		stopc: make(chan struct{}),
		done:  make(chan struct{}),
	}
	go w.run()
	return w
}

func (cw *streamWriter) run() {
	var msgc chan raftpb.Message
	var heartbeatc <-chan time.Time
	var t streamType
	var msgAppTerm uint64
	var enc encoder
	var flusher http.Flusher
	tickc := time.Tick(ConnReadTimeout / 3)

	for {
		select {
		case <-heartbeatc:
			start := time.Now()
			if err := enc.encode(linkHeartbeatMessage); err != nil {
				reportSentFailure(string(t), linkHeartbeatMessage)

				log.Printf("rafthttp: failed to heartbeat on stream %s due to %v. waiting for a new stream to be established.", t, err)
				cw.close()
				heartbeatc, msgc = nil, nil
				continue
			}
			flusher.Flush()
			reportSentDuration(string(t), linkHeartbeatMessage, time.Since(start))
		case m := <-msgc:
			if t == streamTypeMsgApp && m.Term != msgAppTerm {
				// TODO: reasonable retry logic
				if m.Term > msgAppTerm {
					cw.close()
					heartbeatc, msgc = nil, nil
					// TODO: report to raft at peer level
					cw.r.ReportUnreachable(m.To)
				}
				continue
			}
			start := time.Now()
			if err := enc.encode(m); err != nil {
				reportSentFailure(string(t), m)

				log.Printf("rafthttp: failed to send message on stream %s due to %v. waiting for a new stream to be established.", t, err)
				cw.close()
				heartbeatc, msgc = nil, nil
				cw.r.ReportUnreachable(m.To)
				continue
			}
			flusher.Flush()
			reportSentDuration(string(t), m, time.Since(start))
		case conn := <-cw.connc:
			cw.close()
			t = conn.t
			switch conn.t {
			case streamTypeMsgApp:
				var err error
				msgAppTerm, err = strconv.ParseUint(conn.termStr, 10, 64)
				if err != nil {
					log.Panicf("rafthttp: unexpected parse term %s error: %v", conn.termStr, err)
				}
				enc = &msgAppEncoder{w: conn.Writer, fs: cw.fs}
			case streamTypeMsgAppV2:
				enc = newMsgAppV2Encoder(conn.Writer, cw.fs)
			case streamTypeMessage:
				enc = &messageEncoder{w: conn.Writer}
			default:
				log.Panicf("rafthttp: unhandled stream type %s", conn.t)
			}
			flusher = conn.Flusher
			cw.mu.Lock()
			cw.closer = conn.Closer
			cw.working = true
			cw.mu.Unlock()
			heartbeatc, msgc = tickc, cw.msgc
		case <-cw.stopc:
			cw.close()
			close(cw.done)
			return
		}
	}
}

func (cw *streamWriter) writec() (chan<- raftpb.Message, bool) {
	cw.mu.Lock()
	defer cw.mu.Unlock()
	return cw.msgc, cw.working
}

func (cw *streamWriter) close() {
	cw.mu.Lock()
	defer cw.mu.Unlock()
	if !cw.working {
		return
	}
	cw.closer.Close()
	if len(cw.msgc) > 0 {
		cw.r.ReportUnreachable(uint64(cw.id))
	}
	cw.msgc = make(chan raftpb.Message, streamBufSize)
	cw.working = false
}

func (cw *streamWriter) attach(conn *outgoingConn) bool {
	select {
	case cw.connc <- conn:
		return true
	case <-cw.done:
		return false
	}
}

func (cw *streamWriter) stop() {
	close(cw.stopc)
	<-cw.done
}

// streamReader is a long-running go-routine that dials to the remote stream
// endponit and reads messages from the response body returned.
type streamReader struct {
	tr       http.RoundTripper
	picker   *urlPicker
	t        streamType
	from, to types.ID
	cid      types.ID
	recvc    chan<- raftpb.Message
	propc    chan<- raftpb.Message
	errorc   chan<- error

	mu         sync.Mutex
	msgAppTerm uint64
	req        *http.Request
	closer     io.Closer
	stopc      chan struct{}
	done       chan struct{}
}

func startStreamReader(tr http.RoundTripper, picker *urlPicker, t streamType, from, to, cid types.ID, recvc chan<- raftpb.Message, propc chan<- raftpb.Message, errorc chan<- error) *streamReader {
	r := &streamReader{
		tr:     tr,
		picker: picker,
		t:      t,
		from:   from,
		to:     to,
		cid:    cid,
		recvc:  recvc,
		propc:  propc,
		errorc: errorc,
		stopc:  make(chan struct{}),
		done:   make(chan struct{}),
	}
	go r.run()
	return r
}

func (cr *streamReader) run() {
	for {
		rc, err := cr.dial()
		if err != nil {
			log.Printf("rafthttp: roundtripping error: %v", err)
		} else {
			err := cr.decodeLoop(rc)
			if err != io.EOF && !isClosedConnectionError(err) {
				log.Printf("rafthttp: failed to read message on stream %s due to %v", cr.t, err)
			}
		}
		select {
		// Wait 100ms to create a new stream, so it doesn't bring too much
		// overhead when retry.
		case <-time.After(100 * time.Millisecond):
		case <-cr.stopc:
			close(cr.done)
			return
		}
	}
}

func (cr *streamReader) decodeLoop(rc io.ReadCloser) error {
	var dec decoder
	cr.mu.Lock()
	switch cr.t {
	case streamTypeMsgApp:
		dec = &msgAppDecoder{r: rc, local: cr.from, remote: cr.to, term: cr.msgAppTerm}
	case streamTypeMsgAppV2:
		dec = newMsgAppV2Decoder(rc, cr.from, cr.to)
	case streamTypeMessage:
		dec = &messageDecoder{r: rc}
	default:
		log.Panicf("rafthttp: unhandled stream type %s", cr.t)
	}
	cr.closer = rc
	cr.mu.Unlock()

	for {
		m, err := dec.decode()
		switch {
		case err != nil:
			cr.mu.Lock()
			cr.close()
			cr.mu.Unlock()
			return err
		case isLinkHeartbeatMessage(m):
			// do nothing for linkHeartbeatMessage
		default:
			recvc := cr.recvc
			if m.Type == raftpb.MsgProp {
				recvc = cr.propc
			}
			select {
			case recvc <- m:
			default:
				log.Printf("rafthttp: dropping %s from %x because receive buffer is blocked",
					m.Type, m.From)
			}
		}
	}
}

func (cr *streamReader) updateMsgAppTerm(term uint64) {
	cr.mu.Lock()
	defer cr.mu.Unlock()
	if cr.msgAppTerm == term {
		return
	}
	cr.msgAppTerm = term
	cr.close()
}

// TODO: always cancel in-flight dial and decode
func (cr *streamReader) stop() {
	close(cr.stopc)
	cr.mu.Lock()
	cr.cancelRequest()
	cr.close()
	cr.mu.Unlock()
	<-cr.done
}

func (cr *streamReader) isWorking() bool {
	cr.mu.Lock()
	defer cr.mu.Unlock()
	return cr.closer != nil
}

func (cr *streamReader) dial() (io.ReadCloser, error) {
	u := cr.picker.pick()
	cr.mu.Lock()
	term := cr.msgAppTerm
	cr.mu.Unlock()

	uu := u
	uu.Path = path.Join(cr.t.endpoint(), cr.from.String())
	req, err := http.NewRequest("GET", uu.String(), nil)
	if err != nil {
		cr.picker.unreachable(u)
		return nil, fmt.Errorf("new request to %s error: %v", u, err)
	}
	req.Header.Set("X-Etcd-Cluster-ID", cr.cid.String())
	req.Header.Set("X-Raft-To", cr.to.String())
	if cr.t == streamTypeMsgApp {
		req.Header.Set("X-Raft-Term", strconv.FormatUint(term, 10))
	}
	cr.mu.Lock()
	cr.req = req
	cr.mu.Unlock()
	resp, err := cr.tr.RoundTrip(req)
	if err != nil {
		cr.picker.unreachable(u)
		return nil, fmt.Errorf("error roundtripping to %s: %v", req.URL, err)
	}
	switch resp.StatusCode {
	case http.StatusGone:
		resp.Body.Close()
		err := fmt.Errorf("the member has been permanently removed from the cluster")
		select {
		case cr.errorc <- err:
		default:
		}
		return nil, err
	case http.StatusOK:
		return resp.Body, nil
	default:
		resp.Body.Close()
		return nil, fmt.Errorf("unhandled http status %d", resp.StatusCode)
	}
}

func (cr *streamReader) cancelRequest() {
	if canceller, ok := cr.tr.(*http.Transport); ok {
		canceller.CancelRequest(cr.req)
	}
}

func (cr *streamReader) close() {
	if cr.closer != nil {
		cr.closer.Close()
	}
	cr.closer = nil
}

func canUseMsgAppStream(m raftpb.Message) bool {
	return m.Type == raftpb.MsgApp && m.Term == m.LogTerm
}

func isClosedConnectionError(err error) bool {
	operr, ok := err.(*net.OpError)
	return ok && operr.Err.Error() == "use of closed network connection"
}
<|endoftext|>


<commit_before>package management

import (
	"fmt"
	"net/http"
	"time"

	rate "github.com/beefsack/go-rate"
	"github.com/ingaged/sling"
)

const baseURL = "https://api.contentful.com"

type Client struct {
	AccessToken string

	sling *sling.Sling
	rl    *rate.RateLimiter
}

////////////////////
// Initialization //
////////////////////

// NewClient creates a new Contentful API client
func NewClient(accessToken string, version string, httpClient *http.Client) *Client {
	client := &Client{
		AccessToken: accessToken,
		sling: sling.New().Client(httpClient).Base(baseURL).
			Set("Content-Type", contentTypeHeader(version)).
			Set("Authorization", authorizationHeader(accessToken)),
	}

	client.rl = rate.New(10, time.Second*1)

	return client
}

func contentTypeHeader(version string) string {
	return fmt.Sprintf("application/vnd.contentful.management.%v+json", version)
}

func authorizationHeader(accessToken string) string {
	return fmt.Sprintf("Bearer %v", accessToken)
}

////////////////
// Base Types //
////////////////

type System struct {
	ID        string
	CreatedAt time.Time
	UpdatedAt time.Time

	Type    string
	Version int
}

type Pagination struct {
	Total int
	Skip  int
	Limit int
}

type ContentfulError struct {
	RequestID string `json:"requestId"`
	Message   string `json:"message"`
	Sys       struct {
		Type string `json:"type"`
		ID   string `json:"id"`
	} `json:"sys"`
}

func (e *ContentfulError) Error() string {
	return e.Message
}
<commit_msg>Tweaked System to also return the space if it's available<commit_after>package management

import (
	"fmt"
	"net/http"
	"time"

	rate "github.com/beefsack/go-rate"
	"github.com/ingaged/sling"
)

const baseURL = "https://api.contentful.com"

type Client struct {
	AccessToken string

	sling *sling.Sling
	rl    *rate.RateLimiter
}

////////////////////
// Initialization //
////////////////////

// NewClient creates a new Contentful API client
func NewClient(accessToken string, version string, httpClient *http.Client) *Client {
	client := &Client{
		AccessToken: accessToken,
		sling: sling.New().Client(httpClient).Base(baseURL).
			Set("Content-Type", contentTypeHeader(version)).
			Set("Authorization", authorizationHeader(accessToken)),
	}

	client.rl = rate.New(10, time.Second*1)

	return client
}

func contentTypeHeader(version string) string {
	return fmt.Sprintf("application/vnd.contentful.management.%v+json", version)
}

func authorizationHeader(accessToken string) string {
	return fmt.Sprintf("Bearer %v", accessToken)
}

////////////////
// Base Types //
////////////////

type System struct {
	ID        string
	CreatedAt time.Time
	UpdatedAt time.Time

	Type    string
	Version int

	Space *struct {
		*Link `json:"sys"`
	} `json:"space, omitempty"`
}

type Pagination struct {
	Total int
	Skip  int
	Limit int
}

type ContentfulError struct {
	RequestID string `json:"requestId"`
	Message   string `json:"message"`
	Sys       struct {
		Type string `json:"type"`
		ID   string `json:"id"`
	} `json:"sys"`
}

func (e *ContentfulError) Error() string {
	return e.Message
}
<|endoftext|>


<commit_before>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"sync"
	"testing"

	"github.com/spf13/cobra"

	"k8s.io/helm/pkg/helm"
	"k8s.io/helm/pkg/helm/helmpath"
	"k8s.io/helm/pkg/repo"
	"k8s.io/helm/pkg/repo/repotest"
)

var testName = "test-name"

func TestRepoAddCmd(t *testing.T) {
	srv, thome, err := repotest.NewTempServer("testdata/testserver/*.*")
	if err != nil {
		t.Fatal(err)
	}

	cleanup := resetEnv()
	defer func() {
		srv.Stop()
		os.RemoveAll(thome.String())
		cleanup()
	}()
	if err := ensureTestHome(thome, t); err != nil {
		t.Fatal(err)
	}

	settings.Home = thome

	tests := []releaseCase{
		{
			name:     "add a repository",
			args:     []string{testName, srv.URL()},
			expected: "\"" + testName + "\" has been added to your repositories",
		},
	}

	runReleaseCases(t, tests, func(c *helm.FakeClient, out io.Writer) *cobra.Command {
		return newRepoAddCmd(out)
	})
}

func TestRepoAdd(t *testing.T) {
	ts, thome, err := repotest.NewTempServer("testdata/testserver/*.*")
	if err != nil {
		t.Fatal(err)
	}

	cleanup := resetEnv()
	hh := thome
	defer func() {
		ts.Stop()
		os.RemoveAll(thome.String())
		cleanup()
	}()
	if err := ensureTestHome(hh, t); err != nil {
		t.Fatal(err)
	}

	settings.Home = thome

	if err := addRepository(testName, ts.URL(), "", "", hh, "", "", "", true); err != nil {
		t.Error(err)
	}

	f, err := repo.LoadRepositoriesFile(hh.RepositoryFile())
	if err != nil {
		t.Error(err)
	}

	if !f.Has(testName) {
		t.Errorf("%s was not successfully inserted into %s", testName, hh.RepositoryFile())
	}

	if err := addRepository(testName, ts.URL(), "", "", hh, "", "", "", false); err != nil {
		t.Errorf("Repository was not updated: %s", err)
	}

	if err := addRepository(testName, ts.URL(), "", "", hh, "", "", "", false); err != nil {
		t.Errorf("Duplicate repository name was added")
	}
}
func TestRepoAddConcurrentGoRoutines(t *testing.T) {
	ts, thome, err := repotest.NewTempServer("testdata/testserver/*.*")
	if err != nil {
		t.Fatal(err)
	}

	cleanup := resetEnv()
	defer func() {
		ts.Stop()
		os.RemoveAll(thome.String())
		cleanup()
	}()

	settings.Home = thome
	if err := ensureTestHome(settings.Home, t); err != nil {
		t.Fatal(err)
	}

	var wg sync.WaitGroup
	wg.Add(3)
	for i := 0; i < 3; i++ {
		go func(name string) {
			// TODO: launch repository additions in sub-processes as file locks are bound to processes, not file descriptors
			defer wg.Done()
			if err := addRepository(name, ts.URL(), "", "", settings.Home, "", "", "", true); err != nil {
				t.Error(err)
			}
		}(fmt.Sprintf("%s-%d", testName, i))
	}
	wg.Wait()

	f, err := repo.LoadRepositoriesFile(settings.Home.RepositoryFile())
	if err != nil {
		t.Error(err)
	}

	var name string
	for i := 0; i < 3; i++ {
		name = fmt.Sprintf("%s-%d", testName, i)
		if !f.Has(name) {
			t.Errorf("%s was not successfully inserted into %s", name, settings.Home.RepositoryFile())
		}
	}
}

// Same as TestRepoAddConcurrentGoRoutines but with repository additions in sub-processes
func TestRepoAddConcurrentSubProcesses(t *testing.T) {
	goWantHelperProcess := os.Getenv("GO_WANT_HELPER_PROCESS")
	if goWantHelperProcess == "" {
		// parent

		ts, thome, err := repotest.NewTempServer("testdata/testserver/*.*")
		if err != nil {
			t.Fatal(err)
		}

		settings.Home = thome

		cleanup := resetEnv()
		defer func() {
			ts.Stop()
			os.RemoveAll(thome.String())
			cleanup()
		}()
		if err := ensureTestHome(settings.Home, t); err != nil {
			t.Fatal(err)
		}

		var wg sync.WaitGroup
		wg.Add(3)
		for i := 0; i < 3; i++ {
			go func(name string) {
				defer wg.Done()

				cmd := exec.Command(os.Args[0], "-test.run=^TestRepoAddConcurrentSubProcesses$")
				cmd.Env = append(os.Environ(), fmt.Sprintf("GO_WANT_HELPER_PROCESS=%s,%s", name, ts.URL()), fmt.Sprintf("HELM_HOME=%s", settings.Home))
				out, err := cmd.CombinedOutput()
				if len(out) > 0 || err != nil {
					t.Fatalf("child process: %q, %v", out, err)
				}
			}(fmt.Sprintf("%s-%d", testName, i))
		}
		wg.Wait()

		f, err := repo.LoadRepositoriesFile(settings.Home.RepositoryFile())
		if err != nil {
			t.Error(err)
		}

		var name string
		for i := 0; i < 3; i++ {
			name = fmt.Sprintf("%s-%d", testName, i)
			if !f.Has(name) {
				t.Errorf("%s was not successfully inserted into %s", name, settings.Home.RepositoryFile())
			}
		}
	} else {
		// child
		s := strings.Split(goWantHelperProcess, ",")
		settings.Home = helmpath.Home(os.Getenv("HELM_HOME"))
		repoName := s[0]
		tsURL := s[1]
		if err := addRepository(repoName, tsURL, "", "", settings.Home, "", "", "", true); err != nil {
			t.Fatal(err)
		}

		os.Exit(0)
	}
}
<commit_msg>Remove obsolete comment from early file locking implementation<commit_after>/*
Copyright The Helm Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
	"sync"
	"testing"

	"github.com/spf13/cobra"

	"k8s.io/helm/pkg/helm"
	"k8s.io/helm/pkg/helm/helmpath"
	"k8s.io/helm/pkg/repo"
	"k8s.io/helm/pkg/repo/repotest"
)

var testName = "test-name"

func TestRepoAddCmd(t *testing.T) {
	srv, thome, err := repotest.NewTempServer("testdata/testserver/*.*")
	if err != nil {
		t.Fatal(err)
	}

	cleanup := resetEnv()
	defer func() {
		srv.Stop()
		os.RemoveAll(thome.String())
		cleanup()
	}()
	if err := ensureTestHome(thome, t); err != nil {
		t.Fatal(err)
	}

	settings.Home = thome

	tests := []releaseCase{
		{
			name:     "add a repository",
			args:     []string{testName, srv.URL()},
			expected: "\"" + testName + "\" has been added to your repositories",
		},
	}

	runReleaseCases(t, tests, func(c *helm.FakeClient, out io.Writer) *cobra.Command {
		return newRepoAddCmd(out)
	})
}

func TestRepoAdd(t *testing.T) {
	ts, thome, err := repotest.NewTempServer("testdata/testserver/*.*")
	if err != nil {
		t.Fatal(err)
	}

	cleanup := resetEnv()
	hh := thome
	defer func() {
		ts.Stop()
		os.RemoveAll(thome.String())
		cleanup()
	}()
	if err := ensureTestHome(hh, t); err != nil {
		t.Fatal(err)
	}

	settings.Home = thome

	if err := addRepository(testName, ts.URL(), "", "", hh, "", "", "", true); err != nil {
		t.Error(err)
	}

	f, err := repo.LoadRepositoriesFile(hh.RepositoryFile())
	if err != nil {
		t.Error(err)
	}

	if !f.Has(testName) {
		t.Errorf("%s was not successfully inserted into %s", testName, hh.RepositoryFile())
	}

	if err := addRepository(testName, ts.URL(), "", "", hh, "", "", "", false); err != nil {
		t.Errorf("Repository was not updated: %s", err)
	}

	if err := addRepository(testName, ts.URL(), "", "", hh, "", "", "", false); err != nil {
		t.Errorf("Duplicate repository name was added")
	}
}
func TestRepoAddConcurrentGoRoutines(t *testing.T) {
	ts, thome, err := repotest.NewTempServer("testdata/testserver/*.*")
	if err != nil {
		t.Fatal(err)
	}

	cleanup := resetEnv()
	defer func() {
		ts.Stop()
		os.RemoveAll(thome.String())
		cleanup()
	}()

	settings.Home = thome
	if err := ensureTestHome(settings.Home, t); err != nil {
		t.Fatal(err)
	}

	var wg sync.WaitGroup
	wg.Add(3)
	for i := 0; i < 3; i++ {
		go func(name string) {
			defer wg.Done()
			if err := addRepository(name, ts.URL(), "", "", settings.Home, "", "", "", true); err != nil {
				t.Error(err)
			}
		}(fmt.Sprintf("%s-%d", testName, i))
	}
	wg.Wait()

	f, err := repo.LoadRepositoriesFile(settings.Home.RepositoryFile())
	if err != nil {
		t.Error(err)
	}

	var name string
	for i := 0; i < 3; i++ {
		name = fmt.Sprintf("%s-%d", testName, i)
		if !f.Has(name) {
			t.Errorf("%s was not successfully inserted into %s", name, settings.Home.RepositoryFile())
		}
	}
}

// Same as TestRepoAddConcurrentGoRoutines but with repository additions in sub-processes
func TestRepoAddConcurrentSubProcesses(t *testing.T) {
	goWantHelperProcess := os.Getenv("GO_WANT_HELPER_PROCESS")
	if goWantHelperProcess == "" {
		// parent

		ts, thome, err := repotest.NewTempServer("testdata/testserver/*.*")
		if err != nil {
			t.Fatal(err)
		}

		settings.Home = thome

		cleanup := resetEnv()
		defer func() {
			ts.Stop()
			os.RemoveAll(thome.String())
			cleanup()
		}()
		if err := ensureTestHome(settings.Home, t); err != nil {
			t.Fatal(err)
		}

		var wg sync.WaitGroup
		wg.Add(3)
		for i := 0; i < 3; i++ {
			go func(name string) {
				defer wg.Done()

				cmd := exec.Command(os.Args[0], "-test.run=^TestRepoAddConcurrentSubProcesses$")
				cmd.Env = append(os.Environ(), fmt.Sprintf("GO_WANT_HELPER_PROCESS=%s,%s", name, ts.URL()), fmt.Sprintf("HELM_HOME=%s", settings.Home))
				out, err := cmd.CombinedOutput()
				if len(out) > 0 || err != nil {
					t.Fatalf("child process: %q, %v", out, err)
				}
			}(fmt.Sprintf("%s-%d", testName, i))
		}
		wg.Wait()

		f, err := repo.LoadRepositoriesFile(settings.Home.RepositoryFile())
		if err != nil {
			t.Error(err)
		}

		var name string
		for i := 0; i < 3; i++ {
			name = fmt.Sprintf("%s-%d", testName, i)
			if !f.Has(name) {
				t.Errorf("%s was not successfully inserted into %s", name, settings.Home.RepositoryFile())
			}
		}
	} else {
		// child
		s := strings.Split(goWantHelperProcess, ",")
		settings.Home = helmpath.Home(os.Getenv("HELM_HOME"))
		repoName := s[0]
		tsURL := s[1]
		if err := addRepository(repoName, tsURL, "", "", settings.Home, "", "", "", true); err != nil {
			t.Fatal(err)
		}

		os.Exit(0)
	}
}
<|endoftext|>


<commit_before>// Package regexes exposes pre compiled reqular expressions that are used by
// ngorm.
package regexes

import "regexp"

var (
	//DistinctSQL matches distict sql query
	DistinctSQL = regexp.MustCompile(`(?i)distinct[^a-z]+[a-z]+`)

	//Column matches database column
	// only match string like `name`, `users.name`
	Column = regexp.MustCompile("^[a-zA-Z]+(\\.[a-zA-Z]+)*$")

	//IsNumber matches if the string is a number.
	IsNumber = regexp.MustCompile("^\\s*\\d+\\s*$")

	//Comparison matches comparison in sql query
	Comparison = regexp.MustCompile("(?i) (=|<>|>|<|LIKE|IS|IN) ")

	//CcountingQuery matches cound query.
	CcountingQuery = regexp.MustCompile("(?i)^count(.+)$")
)
<commit_msg>[regexes] Fix typo<commit_after>// Package regexes exposes pre compiled reqular expressions that are used by
// ngorm.
package regexes

import "regexp"

var (
	//DistinctSQL matches distict sql query
	DistinctSQL = regexp.MustCompile(`(?i)distinct[^a-z]+[a-z]+`)

	//Column matches database column
	// only match string like `name`, `users.name`
	Column = regexp.MustCompile("^[a-zA-Z]+(\\.[a-zA-Z]+)*$")

	//IsNumber matches if the string is a number.
	IsNumber = regexp.MustCompile("^\\s*\\d+\\s*$")

	//Comparison matches comparison in sql query
	Comparison = regexp.MustCompile("(?i) (=|<>|>|<|LIKE|IS|IN) ")

	//CountingQuery matches cound query.
	CountingQuery = regexp.MustCompile("(?i)^count(.+)$")
)
<|endoftext|>


<commit_before>/**
 * Functionality to persist information about feeds that
 * have been subscribed to and any data we might want to
 * keep about things like the number of items retrieved.
 */

package main

import (
	"database/sql"
	rss "github.com/jteeuwen/go-pkg-rss"
	//"github.com/jteeuwen/go-pkg-xmlx"
	_ "github.com/mattn/go-sqlite3"
	"time"
)

type Item struct {
	Id          int
	FeedId      int
	Title       string
	WasInserted bool
	InsertedOn  time.Time
}

const createFeedsTable = `create table if not exists feeds(
    id integer primary key,
    url varchar(255) unique,
    type varchar(8),
    charset varchar(64)
);`

const createItemsTable = `create table if not exists items(
    id integer primary key,
    feed_id integer,
    title varchar(255),
    was_inserted boolean,
    inserted_on date,
    foreign key(feed_id) references feeds(id)
);`

var tableInitializers = []string{
	createFeedsTable,
	createItemsTable,
}

/**
 * Repeatedly test a condition until it passes, allowing a thread to block
 * on the condition.
 * @param {func() bool} condition - A closure that will test the condition
 * @param {time.Duration} testRate - The frequency at which to invoke the condition function
 * @return A channel from which the number of calls to the condition were made when it passes
 */
func WaitUntilPass(condition func() bool, testRate time.Duration) chan int {
	reportAttempts := make(chan int, 1)
	go func() {
		attempts := 0
		passed := false
		for !passed {
			attempts++
			passed = condition()
			<-time.After(testRate)
		}
		reportAttempts <- attempts
	}()
	return reportAttempts
}

/**
 * Create a database connection to SQLite3 and initialize (if not already)
 * the tables used to store information about RSS feeds being followed etc.
 * @param {string} dbFileName - The name of the file to keep the database info in
 */
func InitDBConnection(dbFileName string) (*sql.DB, error) {
	db, err := sql.Open("sqlite3", dbFileName)
	if err != nil {
		return nil, err
	}
	for _, initializer := range tableInitializers {
		_, err = db.Exec(initializer)
		if err != nil {
			break
		}
	}
	return db, err
}

/**
 * Persist information about a feed to the storage medium.
 * @param {*sql.DB} db - The database connection to use
 * @param {Feed} feed - Information describing the new feed to save
 */
func SaveNewFeed(db *sql.DB, feed Feed) error {
    tx, err1 := db.Begin()
    if err1 != nil {
        return err1
    }
    stmt, err2 := tx.Prepare("insert into feeds(url, type, charset) values(?,?,?)")
    if err2 != nil {
        return err2
    }
    defer stmt.Close()
    _, err3 := stmt.Exec(feed.Url, feed.Type, feed.Charset)
    if err3 != nil {
        return err3
    }
    tx.Commit()
    return nil
}

/**
 * Get a collection of all the feeds subscribed to.
 * @param {*sql.DB} db - The database connection to use
 */
func AllFeeds(db *sql.DB) ([]Feed, error) {
	var feeds []Feed
    tx, err1 := db.Begin()
    if err1 != nil {
        return feeds, err1
    }
    rows, err2 := tx.Query("select id, url, type, charset from feeds")
    if err2 != nil {
        return feeds, err2
    }
    for rows.Next() {
        var url, _type, charset string
        var id int
        rows.Scan(&id, &url, &_type, &charset)
        feeds = append(feeds, Feed{id, url, _type, charset})
    }
    rows.Close()
    return feeds, nil
}

/**
 * Get the basic information about a persisted feed from its URL.
 * @param {*sql.DB} db - The database connection to use
 * @param {string} url - The URL to search for
 */
func GetFeedByUrl(db *sql.DB, url string) (Feed, error) {
	var feed Feed
    tx, err1 := db.Begin()
    if err1 != nil {
        return feed, err1
    }
    stmt, err2 := tx.Prepare("select id, url, type, charset from feeds where url=?")
    if err2 != nil {
        return feed, err2
    }
    defer stmt.Close()
    rows, err3 := stmt.Query(url)
    if err3 != nil {
        return feed, err3
    }
	var id int
	var _type, charset string
	rows.Scan(&id, &url, &_type, &charset)
	rows.Close()
	return Feed{id, url, _type, charset}, nil
}

/**
 * Delete a feed by referencing its URL.
 * @param {*sql.DB} db - The database connection to use
 * @param {string} url - The URL of the feed
 */
func DeleteFeedByUrl(db *sql.DB, url string) error {
    tx, err1 := db.Begin()
    if err1 != nil {
        return err1
    }
    stmt, err2 := tx.Prepare("delete from feeds where url=?")
    if err2 != nil {
        return err2
    }
    defer stmt.Close()
    _, err3 := stmt.Exec(url)
    if err3 != nil {
        return err3
    }
    tx.Commit()
    return nil
}

/**
 * Store information about an item in a channel from a particular feed
 * @param {*sql.DB} db - the database connection to use
 * @param {string} feedUrl - The URL of the RSS/Atom feed
 * @param {*rss.Item} item - The item to store the content of
 */
func SaveNewItem(db *sql.DB, feedUrl string, item *rss.Item) error {
    tx, err1 := db.Begin()
    if err1 != nil {
        return err1
    }
    stmt, err2 := tx.Prepare(`insert into items(feed_id, title, was_inserted)
                              values((select id from feeds where url=?), ?, ?)`)
    if err2 != nil {
        return err2
    }
    defer stmt.Close()
    _, err3 := stmt.Exec(feedUrl, item.Title, false)
    if err3 != nil {
        return err3
    }
    tx.Commit()
    return nil
}

/**
 * Get the items stored for a particular feed in reference to its URL.
 * @param {*sql.DB} db - the database connection to use
 * @param {string} url - The URL of the feed to get items from
 */
func GetItemsByFeedUrl(db *sql.DB, url string) ([]Item, error) {
    var items []Item
    tx, err1 := db.Begin()
    if err1 != nil {
        return items, err1
    }
    stmt, err2 := tx.Prepare(`select id, feed_id, was_inserted, inserted_on
                              from items where feed_id=(select id from feeds where url=?)`)
    if err2 != nil {
        return items, err2
    }
    defer stmt.Close()
    rows, err3 := stmt.Query(url)
    if err3 != nil {
        return items, err3
    }
	for rows.Next() {
		var id, feedId int
		var title string
		var wasInserted bool
		var insertedOn time.Time
		rows.Scan(&id, &feedId, &title, &wasInserted, &insertedOn)
		items = append(items, Item{id, feedId, title, wasInserted, insertedOn})
	}
	rows.Close()
	return items, nil
}

/**
 * Delete a particular item.
 * @param {*sql.DB} db - The database connection to use
 * @param {int} id - The identifier of the item to delete
 */
func DeleteItem(db *sql.DB, id int) error {
    tx, err1 := db.Begin()
    if err1 != nil {
        return err1
    }
    stmt, err2 := tx.Prepare("delete from items where id=?")
    if err2 != nil {
        return err2
    }
    defer stmt.Close()
    _, err3 := stmt.Exec(id)
    if err3 != nil {
        return err3
    }
    tx.Commit()
    return nil
}
<commit_msg>Trying to figure out why articles don't end up with the right ID<commit_after>/**
 * Functionality to persist information about feeds that
 * have been subscribed to and any data we might want to
 * keep about things like the number of items retrieved.
 */

package main

import (
	"database/sql"
	rss "github.com/jteeuwen/go-pkg-rss"
	//"github.com/jteeuwen/go-pkg-xmlx"
	"errors"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"github.com/nicksnyder/go-i18n/i18n"
	"os"
	"time"
)

type Item struct {
	Id          int
	FeedId      int
	Title       string
	WasInserted bool
	InsertedOn  time.Time
}

const createFeedsTable = `create table if not exists feeds(
    id integer primary key,
    url varchar(255) unique,
    type varchar(8),
    charset varchar(64)
);`

const createItemsTable = `create table if not exists items(
    id integer primary key,
    feed_id integer,
    title varchar(255),
    was_inserted boolean,
    inserted_on date,
    foreign key(feed_id) references feeds(id)
);`

var tableInitializers = []string{
	createFeedsTable,
	createItemsTable,
}

/**
 * Repeatedly test a condition until it passes, allowing a thread to block
 * on the condition.
 * @param {func() bool} condition - A closure that will test the condition
 * @param {time.Duration} testRate - The frequency at which to invoke the condition function
 * @return A channel from which the number of calls to the condition were made when it passes
 */
func WaitUntilPass(condition func() bool, testRate time.Duration) chan int {
	reportAttempts := make(chan int, 1)
	go func() {
		attempts := 0
		passed := false
		for !passed {
			attempts++
			passed = condition()
			<-time.After(testRate)
		}
		reportAttempts <- attempts
	}()
	return reportAttempts
}

/**
 * Create a database connection to SQLite3 and initialize (if not already)
 * the tables used to store information about RSS feeds being followed etc.
 * @param {string} dbFileName - The name of the file to keep the database info in
 */
func InitDBConnection(dbFileName string) (*sql.DB, error) {
	db, err := sql.Open("sqlite3", dbFileName)
	if err != nil {
		return nil, err
	}
	for _, initializer := range tableInitializers {
		_, err = db.Exec(initializer)
		if err != nil {
			break
		}
	}
	return db, err
}

/**
 * Persist information about a feed to the storage medium.
 * @param {*sql.DB} db - The database connection to use
 * @param {Feed} feed - Information describing the new feed to save
 */
func SaveNewFeed(db *sql.DB, feed Feed) error {
	tx, err1 := db.Begin()
	if err1 != nil {
		return err1
	}
	stmt, err2 := tx.Prepare("insert into feeds(url, type, charset) values(?,?,?)")
	if err2 != nil {
		return err2
	}
	defer stmt.Close()
	_, err3 := stmt.Exec(feed.Url, feed.Type, feed.Charset)
	if err3 != nil {
		return err3
	}
	tx.Commit()
	return nil
}

/**
 * Get a collection of all the feeds subscribed to.
 * @param {*sql.DB} db - The database connection to use
 */
func AllFeeds(db *sql.DB) ([]Feed, error) {
	var feeds []Feed
	tx, err1 := db.Begin()
	if err1 != nil {
		return feeds, err1
	}
	rows, err2 := tx.Query("select id, url, type, charset from feeds")
	if err2 != nil {
		return feeds, err2
	}
	for rows.Next() {
		var url, _type, charset string
		var id int
		rows.Scan(&id, &url, &_type, &charset)
		feeds = append(feeds, Feed{id, url, _type, charset})
	}
	rows.Close()
	return feeds, nil
}

/**
 * Get the basic information about a persisted feed from its URL.
 * @param {*sql.DB} db - The database connection to use
 * @param {string} url - The URL to search for
 */
func GetFeedByUrl(db *sql.DB, url string) (Feed, error) {
	var feed Feed
	tx, err1 := db.Begin()
	if err1 != nil {
		return feed, err1
	}
	stmt, err2 := tx.Prepare("select id, url, type, charset from feeds where url=?")
	if err2 != nil {
		return feed, err2
	}
	defer stmt.Close()
	rows, err3 := stmt.Query(url)
	if err3 != nil {
		return feed, err3
	}
	var id int
	var _type, charset string
	rows.Scan(&id, &url, &_type, &charset)
	if url == "" {
		id = -1
	}
	fmt.Printf("Feed URL %s has ID %d\n", url, id)
	rows.Close()
	return Feed{id, url, _type, charset}, nil
}

/**
 * Delete a feed by referencing its URL.
 * @param {*sql.DB} db - The database connection to use
 * @param {string} url - The URL of the feed
 */
func DeleteFeedByUrl(db *sql.DB, url string) error {
	tx, err1 := db.Begin()
	if err1 != nil {
		return err1
	}
	stmt, err2 := tx.Prepare("delete from feeds where url=?")
	if err2 != nil {
		return err2
	}
	defer stmt.Close()
	_, err3 := stmt.Exec(url)
	if err3 != nil {
		return err3
	}
	tx.Commit()
	return nil
}

/**
 * Store information about an item in a channel from a particular feed
 * @param {*sql.DB} db - the database connection to use
 * @param {string} feedUrl - The URL of the RSS/Atom feed
 * @param {*rss.Item} item - The item to store the content of
 */
func SaveNewItem(db *sql.DB, feedUrl string, item *rss.Item) error {
	feed, err := GetFeedByUrl(db, feedUrl)
	T, _ := i18n.Tfunc(os.Getenv(LANG_ENVVAR), DEFAULT_LANG)
	if err != nil || feed.Url == "" {
		return errors.New(T("not_followed_feed_err", map[string]interface{}{
			"URL": feedUrl,
		}))
	}
	tx, err1 := db.Begin()
	if err1 != nil {
		return err1
	}
	stmt, err2 := tx.Prepare(`insert into items(feed_id, title, was_inserted)
                              values(?, ?, ?)`)
	if err2 != nil {
		return err2
	}
	defer stmt.Close()
	_, err3 := stmt.Exec(feed.Id, item.Title, false)
	if err3 != nil {
		return err3
	}
	tx.Commit()
	return nil
}

/**
 * Get the items stored for a particular feed in reference to its URL.
 * @param {*sql.DB} db - the database connection to use
 * @param {string} url - The URL of the feed to get items from
 */
func GetItemsByFeedUrl(db *sql.DB, url string) ([]Item, error) {
	var items []Item
	tx, err1 := db.Begin()
	if err1 != nil {
		return items, err1
	}
	stmt, err2 := tx.Prepare(`select id, feed_id, was_inserted, inserted_on
                              from items where feed_id=(select id from feeds where url=?)`)
	if err2 != nil {
		return items, err2
	}
	defer stmt.Close()
	rows, err3 := stmt.Query(url)
	if err3 != nil {
		return items, err3
	}
	for rows.Next() {
		var id, feedId int
		var title string
		var wasInserted bool
		var insertedOn time.Time
		rows.Scan(&id, &feedId, &title, &wasInserted, &insertedOn)
		items = append(items, Item{id, feedId, title, wasInserted, insertedOn})
	}
	rows.Close()
	return items, nil
}

/**
 * Delete a particular item.
 * @param {*sql.DB} db - The database connection to use
 * @param {int} id - The identifier of the item to delete
 */
func DeleteItem(db *sql.DB, id int) error {
	tx, err1 := db.Begin()
	if err1 != nil {
		return err1
	}
	stmt, err2 := tx.Prepare("delete from items where id=?")
	if err2 != nil {
		return err2
	}
	defer stmt.Close()
	_, err3 := stmt.Exec(id)
	if err3 != nil {
		return err3
	}
	tx.Commit()
	return nil
}
<|endoftext|>