<commit_before>package farm

// This file provides a 32-bit hash equivalent to CityHash32 (v1.1.1)
// and a 128-bit hash equivalent to CityHash128 (v1.1.1).  It also provides
// a seeded 32-bit hash function similar to CityHash32.

func hash32Len13to24Seed(s []byte, seed uint32) uint32 {
	slen := len(s)
	a := fetch32(s, -4+(slen>>1))
	b := fetch32(s, 4)
	c := fetch32(s, slen-8)
	d := fetch32(s, (slen >> 1))
	e := fetch32(s, 0)
	f := fetch32(s, slen-4)
	h := d*c1 + uint32(slen) + seed
	a = rotate32(a, 12) + f
	h = mur(c, h) + a
	a = rotate32(a, 3) + c
	h = mur(e, h) + a
	a = rotate32(a+f, 12) + d
	h = mur(b^seed, h) + a
	return fmix(h)
}

func hash32Len0to4(s []byte, seed uint32) uint32 {
	slen := len(s)
	b := seed
	c := uint32(9)
	for i := 0; i < slen; i++ {
		v := int8(s[i])
		b = uint32(b*c1) + uint32(v)
		c ^= b
	}
	return fmix(mur(b, mur(uint32(slen), c)))
}

func hash128to64(x uint128) uint64 {
	// Murmur-inspired hashing.
	const kMul uint64 = 0x9ddfea08eb382d69
	a := (x.lo ^ x.hi) * kMul
	a ^= (a >> 47)
	b := (x.hi ^ a) * kMul
	b ^= (b >> 47)
	b *= kMul
	return b
}

type uint128 struct {
	hi uint64
	lo uint64
}

// A subroutine for CityHash128().  Returns a decent 128-bit hash for strings
// of any length representable in signed long.  Based on City and Murmur.
func CityMurmur(s []byte, seed uint128) uint128 {
	slen := uint32(len(s))
	a := seed.lo
	b := seed.hi
	c := uint64(0)
	d := uint64(0)
	l := slen - 16
	if l <= 0 { // len <= 16
		a = shiftMix(a*k1) * k1
		c = b*k1 + hashLen0to16(s)
		if slen >= 8 {
			d = shiftMix(a + fetch64(s, 0))
		} else {
			d = shiftMix(a + c)
		}
	} else { // len > 16
		c = hashLen16(fetch64(s, int(slen-8))+k1, a)
		d = hashLen16(b+uint64(slen), c+fetch64(s, int(slen-16)))
		a += d
		for {
			a ^= shiftMix(fetch64(s, 0)*k1) * k1
			a *= k1
			b ^= a
			c ^= shiftMix(fetch64(s, 8)*k1) * k1
			c *= k1
			d ^= c
			s = s[16:]
			l -= 16
			if l <= 0 {
				break
			}
		}
	}
	a = hashLen16(a, c)
	b = hashLen16(d, b)
	return uint128{a ^ b, hashLen16(b, a)}
}

func CityHash128WithSeed(s []byte, seed uint128) uint128 {
	slen := len(s)
	if slen < 128 {
		return CityMurmur(s, seed)
	}

	// We expect len >= 128 to be the common case.  Keep 56 bytes of state:
	// v, w, x, y, and z.
	var v1, v2 uint64
	var w1, w2 uint64
	x := seed.lo
	y := seed.hi
	z := uint64(slen) * k1
	v1 = rotate64(y^k1, 49)*k1 + fetch64(s, 0)
	v2 = rotate64(v1, 42)*k1 + fetch64(s, 8)
	w1 = rotate64(y+z, 35)*k1 + x
	w2 = rotate64(x+fetch64(s, 88), 53) * k1

	// This is the same inner loop as CityHash64(), manually unrolled.
	for {
		x = rotate64(x+y+v1+fetch64(s, 8), 37) * k1
		y = rotate64(y+v2+fetch64(s, 48), 42) * k1
		x ^= w2
		y += v1 + fetch64(s, 40)
		z = rotate64(z+w1, 33) * k1
		v1, v2 = weakHashLen32WithSeeds(s, v2*k1, x+w1)
		w1, w2 = weakHashLen32WithSeeds(s[32:], z+w2, y+fetch64(s, 16))
		z, x = x, z
		s = s[64:]
		x = rotate64(x+y+v1+fetch64(s, 8), 37) * k1
		y = rotate64(y+v2+fetch64(s, 48), 42) * k1
		x ^= w2
		y += v1 + fetch64(s, 40)
		z = rotate64(z+w1, 33) * k1
		v1, v2 = weakHashLen32WithSeeds(s, v2*k1, x+w1)
		w1, w2 = weakHashLen32WithSeeds(s[32:], z+w2, y+fetch64(s, 16))
		z, x = x, z
		s = s[64:]
		slen -= 128
		if slen < 128 {
			break
		}
	}
	x += rotate64(v1+z, 49) * k0
	y = y*k0 + rotate64(w2, 37)
	z = z*k0 + rotate64(w1, 27)
	w1 *= 9
	v1 *= k0
	// If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s.
	for tail_done := 0; tail_done < slen; {
		tail_done += 32
		y = rotate64(x+y, 42)*k0 + v2
		w1 += fetch64(s, slen-tail_done+16)
		x = x*k0 + w1
		z += w2 + fetch64(s, slen-tail_done)
		w2 += v1
		v1, v2 = weakHashLen32WithSeeds(s[slen-tail_done:], v1+z, v2)
		v1 *= k0
	}
	// At this point our 56 bytes of state should contain more than
	// enough information for a strong 128-bit hash.  We use two
	// different 56-byte-to-8-byte hashes to get a 16-byte final result.
	x = hashLen16(x, v1)
	y = hashLen16(y+z, w1)
	return uint128{hashLen16(x+v2, w2) + y,
		hashLen16(x+w2, y+v2)}
}

func CityHash128(s []byte) uint128 {
	slen := len(s)
	if slen >= 16 {
		CityHash128WithSeed(s, uint128{fetch64(s, 0), fetch64(s, 8) + k0})
	}
	return CityHash128WithSeed(s, uint128{k0, k1})
}

func Fingerprint128(s []byte) uint128 {
	return CityHash128(s)
}
<commit_msg>unexport cityhash internals<commit_after>package farm

// This file provides a 32-bit hash equivalent to CityHash32 (v1.1.1)
// and a 128-bit hash equivalent to CityHash128 (v1.1.1).  It also provides
// a seeded 32-bit hash function similar to CityHash32.

func hash32Len13to24Seed(s []byte, seed uint32) uint32 {
	slen := len(s)
	a := fetch32(s, -4+(slen>>1))
	b := fetch32(s, 4)
	c := fetch32(s, slen-8)
	d := fetch32(s, (slen >> 1))
	e := fetch32(s, 0)
	f := fetch32(s, slen-4)
	h := d*c1 + uint32(slen) + seed
	a = rotate32(a, 12) + f
	h = mur(c, h) + a
	a = rotate32(a, 3) + c
	h = mur(e, h) + a
	a = rotate32(a+f, 12) + d
	h = mur(b^seed, h) + a
	return fmix(h)
}

func hash32Len0to4(s []byte, seed uint32) uint32 {
	slen := len(s)
	b := seed
	c := uint32(9)
	for i := 0; i < slen; i++ {
		v := int8(s[i])
		b = uint32(b*c1) + uint32(v)
		c ^= b
	}
	return fmix(mur(b, mur(uint32(slen), c)))
}

func hash128to64(x uint128) uint64 {
	// Murmur-inspired hashing.
	const kMul uint64 = 0x9ddfea08eb382d69
	a := (x.lo ^ x.hi) * kMul
	a ^= (a >> 47)
	b := (x.hi ^ a) * kMul
	b ^= (b >> 47)
	b *= kMul
	return b
}

type uint128 struct {
	hi uint64
	lo uint64
}

// A subroutine for CityHash128().  Returns a decent 128-bit hash for strings
// of any length representable in signed long.  Based on City and Murmur.
func cityMurmur(s []byte, seed uint128) uint128 {
	slen := uint32(len(s))
	a := seed.lo
	b := seed.hi
	c := uint64(0)
	d := uint64(0)
	l := slen - 16
	if l <= 0 { // len <= 16
		a = shiftMix(a*k1) * k1
		c = b*k1 + hashLen0to16(s)
		if slen >= 8 {
			d = shiftMix(a + fetch64(s, 0))
		} else {
			d = shiftMix(a + c)
		}
	} else { // len > 16
		c = hashLen16(fetch64(s, int(slen-8))+k1, a)
		d = hashLen16(b+uint64(slen), c+fetch64(s, int(slen-16)))
		a += d
		for {
			a ^= shiftMix(fetch64(s, 0)*k1) * k1
			a *= k1
			b ^= a
			c ^= shiftMix(fetch64(s, 8)*k1) * k1
			c *= k1
			d ^= c
			s = s[16:]
			l -= 16
			if l <= 0 {
				break
			}
		}
	}
	a = hashLen16(a, c)
	b = hashLen16(d, b)
	return uint128{a ^ b, hashLen16(b, a)}
}

func cityHash128WithSeed(s []byte, seed uint128) uint128 {
	slen := len(s)
	if slen < 128 {
		return cityMurmur(s, seed)
	}

	// We expect len >= 128 to be the common case.  Keep 56 bytes of state:
	// v, w, x, y, and z.
	var v1, v2 uint64
	var w1, w2 uint64
	x := seed.lo
	y := seed.hi
	z := uint64(slen) * k1
	v1 = rotate64(y^k1, 49)*k1 + fetch64(s, 0)
	v2 = rotate64(v1, 42)*k1 + fetch64(s, 8)
	w1 = rotate64(y+z, 35)*k1 + x
	w2 = rotate64(x+fetch64(s, 88), 53) * k1

	// This is the same inner loop as CityHash64(), manually unrolled.
	for {
		x = rotate64(x+y+v1+fetch64(s, 8), 37) * k1
		y = rotate64(y+v2+fetch64(s, 48), 42) * k1
		x ^= w2
		y += v1 + fetch64(s, 40)
		z = rotate64(z+w1, 33) * k1
		v1, v2 = weakHashLen32WithSeeds(s, v2*k1, x+w1)
		w1, w2 = weakHashLen32WithSeeds(s[32:], z+w2, y+fetch64(s, 16))
		z, x = x, z
		s = s[64:]
		x = rotate64(x+y+v1+fetch64(s, 8), 37) * k1
		y = rotate64(y+v2+fetch64(s, 48), 42) * k1
		x ^= w2
		y += v1 + fetch64(s, 40)
		z = rotate64(z+w1, 33) * k1
		v1, v2 = weakHashLen32WithSeeds(s, v2*k1, x+w1)
		w1, w2 = weakHashLen32WithSeeds(s[32:], z+w2, y+fetch64(s, 16))
		z, x = x, z
		s = s[64:]
		slen -= 128
		if slen < 128 {
			break
		}
	}
	x += rotate64(v1+z, 49) * k0
	y = y*k0 + rotate64(w2, 37)
	z = z*k0 + rotate64(w1, 27)
	w1 *= 9
	v1 *= k0
	// If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s.
	for tail_done := 0; tail_done < slen; {
		tail_done += 32
		y = rotate64(x+y, 42)*k0 + v2
		w1 += fetch64(s, slen-tail_done+16)
		x = x*k0 + w1
		z += w2 + fetch64(s, slen-tail_done)
		w2 += v1
		v1, v2 = weakHashLen32WithSeeds(s[slen-tail_done:], v1+z, v2)
		v1 *= k0
	}
	// At this point our 56 bytes of state should contain more than
	// enough information for a strong 128-bit hash.  We use two
	// different 56-byte-to-8-byte hashes to get a 16-byte final result.
	x = hashLen16(x, v1)
	y = hashLen16(y+z, w1)
	return uint128{hashLen16(x+v2, w2) + y,
		hashLen16(x+w2, y+v2)}
}

func cityHash128(s []byte) uint128 {
	slen := len(s)
	if slen >= 16 {
		cityHash128WithSeed(s, uint128{fetch64(s, 0), fetch64(s, 8) + k0})
	}
	return cityHash128WithSeed(s, uint128{k0, k1})
}

func Fingerprint128(s []byte) uint128 {
	return cityHash128(s)
}
<|endoftext|>


<commit_before>package recycleme

import (
	"testing"
)

func TestAmazonFetcher(t *testing.T) {
	amazonFetcher, err := NewAmazonURLFetcher()

	if amazonFetcher.SecretKey == "" || amazonFetcher.AccessKey == "" || amazonFetcher.AssociateTag == "" {
		t.Log("Missing either AccessKey, SecretKey or AssociateTag. AmazonFetcher will not be tested")
		return
	}
	_, err = amazonFetcher.Fetch("4006381333634")
	if err != nil {
		if err.(*ProductError).err != errTooManyProducts {
			t.Fatal(err)
		}
	}
	p, err := amazonFetcher.Fetch("5021991938818")
	if err != nil {
		t.Fatal(err)
	}
	if p.Name != "Clipper Thé Vert Biologique 20 infusettes" ||
		p.EAN != "5021991938818" ||
		p.URL != "webservices.amazon.fr" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/517qE9owUDL.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}
}

func TestDefaultFetchers(t *testing.T) {
	p, err := UpcItemDbFetcher.Fetch("5029053038896")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Kleenex tissues in a Christmas House box" ||
		p.EAN != "5029053038896" ||
		p.URL != "http://www.upcitemdb.com/upc/5029053038896" ||
		p.ImageURL != "http://www.staples.co.uk/content/images/product/428056_1_xnl.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = UpcItemDbFetcher.Fetch("4006381333634")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Stabilo Boss Original Highlighter Blue" ||
		p.EAN != "4006381333634" ||
		p.URL != "http://www.upcitemdb.com/upc/4006381333634" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/41SfgGjtcpL._SL160_.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = OpenFoodFactsFetcher.Fetch("7613034383808")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Four à Pierre Royale" || p.EAN != "7613034383808" ||
		p.URL != "http://fr.openfoodfacts.org/api/v0/produit/7613034383808.json" ||
		p.ImageURL != "http://static.openfoodfacts.org/images/products/761/303/438/3808/front.8.400.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = IsbnSearchFetcher.Fetch("9782501104265")
	if err != nil {
		t.Error(err)
	} else if p.Name != "le rugby c'est pas sorcier" || p.EAN != "9782501104265" ||
		p.URL != "http://www.isbnsearch.org/isbn/9782501104265" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/51V4iimUfML._SL194_.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}
}

func TestScrap(t *testing.T) {
	fetcher, _ := NewDefaultFetcher()
	_, err := fetcher.Fetch("5029053038896")
	if err != nil {
		t.Error(err)
	} else {
		_, err = fetcher.Fetch("7613034383808")
		if err != nil {
			t.Error(err)
		} else {
			_, err = fetcher.Fetch("7640140337517")
			if err == nil {
				t.Error(err)
			}
		}
	}
}
<commit_msg>Update Scrap test<commit_after>package recycleme

import (
	"testing"
)

func TestAmazonFetcher(t *testing.T) {
	amazonFetcher, err := NewAmazonURLFetcher()

	if amazonFetcher.SecretKey == "" || amazonFetcher.AccessKey == "" || amazonFetcher.AssociateTag == "" {
		t.Log("Missing either AccessKey, SecretKey or AssociateTag. AmazonFetcher will not be tested")
		return
	}
	_, err = amazonFetcher.Fetch("4006381333634")
	if err != nil {
		if err.(*ProductError).err != errTooManyProducts {
			t.Fatal(err)
		}
	}
	p, err := amazonFetcher.Fetch("5021991938818")
	if err != nil {
		t.Fatal(err)
	}
	if p.Name != "Clipper Thé Vert Biologique 20 infusettes" ||
		p.EAN != "5021991938818" ||
		p.URL != "webservices.amazon.fr" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/517qE9owUDL.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}
}

func TestDefaultFetchers(t *testing.T) {
	p, err := UpcItemDbFetcher.Fetch("5029053038896")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Kleenex tissues in a Christmas House box" ||
		p.EAN != "5029053038896" ||
		p.URL != "http://www.upcitemdb.com/upc/5029053038896" ||
		p.ImageURL != "http://www.staples.co.uk/content/images/product/428056_1_xnl.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = UpcItemDbFetcher.Fetch("4006381333634")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Stabilo Boss Original Highlighter Blue" ||
		p.EAN != "4006381333634" ||
		p.URL != "http://www.upcitemdb.com/upc/4006381333634" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/41SfgGjtcpL._SL160_.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = OpenFoodFactsFetcher.Fetch("7613034383808")
	if err != nil {
		t.Error(err)
	} else if p.Name != "Four à Pierre Royale" || p.EAN != "7613034383808" ||
		p.URL != "http://fr.openfoodfacts.org/api/v0/produit/7613034383808.json" ||
		p.ImageURL != "http://static.openfoodfacts.org/images/products/761/303/438/3808/front.8.400.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}

	p, err = IsbnSearchFetcher.Fetch("9782501104265")
	if err != nil {
		t.Error(err)
	} else if p.Name != "le rugby c'est pas sorcier" || p.EAN != "9782501104265" ||
		p.URL != "http://www.isbnsearch.org/isbn/9782501104265" ||
		p.ImageURL != "http://ecx.images-amazon.com/images/I/51V4iimUfML._SL194_.jpg" {
		t.Errorf("Some attributes are invalid for: %v", p)
	}
}

func TestDefaultFetcher(t *testing.T) {
	fetcher, _ := NewDefaultFetcher()
	_, err := fetcher.Fetch("5029053038896")
	if err != nil {
		t.Fatal(err)
	}

	_, err = fetcher.Fetch("7613034383808")
	if err != nil {
		t.Fatal(err)
	}

	_, err = fetcher.Fetch("7640140337517")
	if err == nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Package httpgzip is a simple wrapper around http.FileServer that looks for
// a compressed version of a file and serves that if the client requested
// compressed content
package httpgzip // import "vimagination.zapto.org/httpgzip"

import (
	"mime"
	"net/http"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"vimagination.zapto.org/httpencoding"
)

const (
	contentEncoding = "Content-Encoding"
	contentType     = "Content-Type"
	contentLength   = "Content-Length"
	indexPage       = "index.html"
)

var encodings = map[string]string{
	"gzip":    ".gz",
	"x-gzip":  ".gz",
	"br":      ".br",
	"deflate": ".fl",
}

type overlay []http.FileSystem

func (o overlay) Open(name string) (f http.File, err error) {
	for _, fs := range o {
		f, err = fs.Open(name)
		if err == nil {
			return f, nil
		}
	}
	return nil, err
}

type fileServer struct {
	root http.FileSystem
	h    http.Handler
}

// FileServer creates a wrapper around http.FileServer using the given
// http.FileSystem
//
// Additional http.FileSystem's can be specified and will be turned into a
// Handler that checks each in order, stopping at the first
func FileServer(root http.FileSystem, roots ...http.FileSystem) http.Handler {
	if len(roots) > 0 {
		overlays := make(overlay, 1, len(roots)+1)
		overlays[0] = root
		overlays = append(overlays, roots...)
		root = overlays
	}
	return &fileServer{
		root,
		http.FileServer(root),
	}
}

func (f *fileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fsh := fileserverHandler{
		fileServer: f,
		w:          w,
		r:          r,
	}
	if !httpencoding.HandleEncoding(r, &fsh) {
		httpencoding.InvalidEncoding(w)
	}
}

type fileserverHandler struct {
	*fileServer
	w http.ResponseWriter
	r *http.Request
}

func (f *fileserverHandler) Handle(encoding string) bool {
	if encoding == "" {
		httpencoding.ClearEncoding(f.r)
		f.h.ServeHTTP(f.w, f.r)
		return true
	}
	ext, ok := encodings[encoding]
	if !ok {
		return false
	}
	p := path.Clean(f.r.URL.Path)
	m := p
	nf, err := f.root.Open(p + ext)
	if strings.HasSuffix(p, "/") {
		m += indexPage
		if err != nil {
			nf, err = f.root.Open(p + ext)
			p += indexPage
		}
	}
	if err == nil {
		if ctype := mime.TypeByExtension(filepath.Ext(m)); ctype != "" {
			s, err := nf.Stat()
			if err == nil {
				f.w.Header().Set(contentType, ctype)
				f.w.Header().Set(contentLength, strconv.FormatInt(s.Size(), 10))
				f.w.Header().Set(contentEncoding, encoding)
				f.r.URL.Path = p + ext
				httpencoding.ClearEncoding(f.r)
				f.h.ServeHTTP(f.w, f.r)
				return true
			}
		}
	}
	return false
}
<commit_msg>added byte checking for content-type<commit_after>// Package httpgzip is a simple wrapper around http.FileServer that looks for
// a compressed version of a file and serves that if the client requested
// compressed content
package httpgzip // import "vimagination.zapto.org/httpgzip"

import (
	"io"
	"mime"
	"net/http"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"vimagination.zapto.org/httpencoding"
)

const (
	contentEncoding = "Content-Encoding"
	contentType     = "Content-Type"
	contentLength   = "Content-Length"
	indexPage       = "index.html"
)

var encodings = map[string]string{
	"gzip":    ".gz",
	"x-gzip":  ".gz",
	"br":      ".br",
	"deflate": ".fl",
}

type overlay []http.FileSystem

func (o overlay) Open(name string) (f http.File, err error) {
	for _, fs := range o {
		f, err = fs.Open(name)
		if err == nil {
			return f, nil
		}
	}
	return nil, err
}

type fileServer struct {
	root http.FileSystem
	h    http.Handler
}

// FileServer creates a wrapper around http.FileServer using the given
// http.FileSystem
//
// Additional http.FileSystem's can be specified and will be turned into a
// Handler that checks each in order, stopping at the first
func FileServer(root http.FileSystem, roots ...http.FileSystem) http.Handler {
	if len(roots) > 0 {
		overlays := make(overlay, 1, len(roots)+1)
		overlays[0] = root
		overlays = append(overlays, roots...)
		root = overlays
	}
	return &fileServer{
		root,
		http.FileServer(root),
	}
}

func (f *fileServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	fsh := fileserverHandler{
		fileServer: f,
		w:          w,
		r:          r,
	}
	if !httpencoding.HandleEncoding(r, &fsh) {
		httpencoding.InvalidEncoding(w)
	}
}

type fileserverHandler struct {
	*fileServer
	w http.ResponseWriter
	r *http.Request
}

func (f *fileserverHandler) Handle(encoding string) bool {
	if encoding == "" {
		httpencoding.ClearEncoding(f.r)
		f.h.ServeHTTP(f.w, f.r)
		return true
	}
	ext, ok := encodings[encoding]
	if !ok {
		return false
	}
	p := path.Clean(f.r.URL.Path)
	m := p
	nf, err := f.root.Open(p + ext)
	if strings.HasSuffix(p, "/") {
		m += indexPage
		if err != nil {
			nf, err = f.root.Open(p + ext)
			p += indexPage
		}
	}
	if err == nil {
		ctype := mime.TypeByExtension(filepath.Ext(m))
		if ctype == "" {
			df, err := f.root.Open(m)
			if err == nil {
				var buf [512]byte
				n, _ := io.ReadFull(df, buf[:])
				ctype = http.DetectContentType(buf[:n])
				nf.Seek(0, io.SeekStart)
				df.Close()
			}
		}
		if ctype != "" {
			s, err := nf.Stat()
			if err == nil {
				f.w.Header().Set(contentType, ctype)
				f.w.Header().Set(contentLength, strconv.FormatInt(s.Size(), 10))
				f.w.Header().Set(contentEncoding, encoding)
				f.r.URL.Path = p + ext
				httpencoding.ClearEncoding(f.r)
				f.h.ServeHTTP(f.w, f.r)
				return true
			}
		}
	}
	return false
}
<|endoftext|>


<commit_before>package hello

import (
	"fmt"
	"net/http"
)

func init() {
	http.HandleFunc("/", handler)

}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Hello, world!")

}
<commit_msg>Change of the package name<commit_after>package polling

import (
	"fmt"
	"net/http"
)

func init() {
	http.HandleFunc("/", handler)

}

func handler(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, "Hello, world!")

}
<|endoftext|>


<commit_before>package slackboard

import (
	"github.com/juju/ratelimit"
	"time"
)

// QPSPerSlackEndpoint controls rate limiting.
type QPSPerSlackEndpoint struct {
	bucket  *ratelimit.Bucket
	maxWait *time.Duration
}

// NewQPSPerSlackEndpoint initializes QPSPerSlackEndpoint.
func NewQPSPerSlackEndpoint(conf ConfToml) *QPSPerSlackEndpoint {
	qps := conf.Core.QPS
	if qps <= 0 {
		return nil
	}

	var maxWait *time.Duration
	duration := conf.Core.MaxDelayDuration
	if 0 <= duration {
		sec := time.Duration(duration) * time.Second
		maxWait = &sec
	}

	return &QPSPerSlackEndpoint{
		ratelimit.NewBucketWithRate(float64(qps), int64(qps)),
		maxWait,
	}
}

// Available takes count from the bucket.
// If it is not available immediately, do nothing and return false.
func (qpsend QPSPerSlackEndpoint) Available() bool {
	if qpsend.bucket.TakeAvailable(1) == 0 {
		return false
	}

	return true
}

// WaitAndAvailable waits until the bucket becomes available
func (qpsend QPSPerSlackEndpoint) WaitAndAvailable() bool {
	maxWait := qpsend.maxWait
	if maxWait == nil {
		// disable rate limiting
		return true
	}
	return qpsend.bucket.WaitMaxDuration(1, *maxWait)
}
<commit_msg>style: run goimports.<commit_after>package slackboard

import (
	"time"

	"github.com/juju/ratelimit"
)

// QPSPerSlackEndpoint controls rate limiting.
type QPSPerSlackEndpoint struct {
	bucket  *ratelimit.Bucket
	maxWait *time.Duration
}

// NewQPSPerSlackEndpoint initializes QPSPerSlackEndpoint.
func NewQPSPerSlackEndpoint(conf ConfToml) *QPSPerSlackEndpoint {
	qps := conf.Core.QPS
	if qps <= 0 {
		return nil
	}

	var maxWait *time.Duration
	duration := conf.Core.MaxDelayDuration
	if 0 <= duration {
		sec := time.Duration(duration) * time.Second
		maxWait = &sec
	}

	return &QPSPerSlackEndpoint{
		ratelimit.NewBucketWithRate(float64(qps), int64(qps)),
		maxWait,
	}
}

// Available takes count from the bucket.
// If it is not available immediately, do nothing and return false.
func (qpsend QPSPerSlackEndpoint) Available() bool {
	if qpsend.bucket.TakeAvailable(1) == 0 {
		return false
	}

	return true
}

// WaitAndAvailable waits until the bucket becomes available
func (qpsend QPSPerSlackEndpoint) WaitAndAvailable() bool {
	maxWait := qpsend.maxWait
	if maxWait == nil {
		// disable rate limiting
		return true
	}
	return qpsend.bucket.WaitMaxDuration(1, *maxWait)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
)

//loadFiles creates a GistFile each of which holding the contents of a file
func loadFiles(fileNames []string, c chan *GistFile) {
	for _, name := range fileNames {
		content, err := ioutil.ReadFile(name)
		if err == nil {
			c <- &GistFile{Name: name, Content: string(content)}
		}
	}
	close(c)
}

func main() {
	userConfig, err := LoadConfig()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(userConfig.Gist.Username)
	fmt.Println(userConfig.Gist.Token)

	description := os.Args[1]
	public := os.Args[2]
	files := os.Args[3:]
	fmt.Println(description)
	fmt.Println(public)
	fmt.Println(files)

	c := make(chan *GistFile)
	go loadFiles(files, c)

	gistFiles := make(map[string]GistFile)
	for g := range c {
		gistFiles[g.Name] = *g
	}

	gist := &Gist{Description: description, Public: false, Files: gistFiles}

	gistURL, err := gist.Post(userConfig.Gist.Username, userConfig.Gist.Token)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(gistURL)
}
<commit_msg>Implement CLI with built-in flag lib<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
)

//loadFiles creates a GistFile each of which holding the contents of a file
func loadFiles(fileNames []string, c chan *GistFile) {
	for _, name := range fileNames {
		content, err := ioutil.ReadFile(name)
		if err == nil {
			c <- &GistFile{Name: name, Content: string(content)}
		}
	}
	close(c)
}

//createGist create a Gist based on the given data
func createGist(fileNames []string, description string, public bool) *Gist {
	c := make(chan *GistFile)
	go loadFiles(fileNames, c)

	gistFiles := make(map[string]GistFile)
	for g := range c {
		gistFiles[g.Name] = *g
	}

	gist := &Gist{Description: description, Public: public, Files: gistFiles}
	return gist
}

func main() {
	description := flag.String(
		"description",
		"Floof Gist",
		"A description of the gist.")

	public := flag.Bool(
		"public",
		false,
		"Indicates whether the gist is public. (default false)")

	flag.Parse()

	files := flag.Args()
	if len(files) == 0 {
		log.Fatal("No files given")
		return
	}

	userConfig, err := LoadConfig()
	if err != nil {
		log.Fatal(err)
		return
	}

	gist := createGist(files, *description, *public)

	gistURL, err := gist.Post(userConfig.Gist.Username, userConfig.Gist.Token)
	if err != nil {
		log.Fatal(err)
		return
	}
	fmt.Println(gistURL)
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// Command wordoffset sends audio data to the Google Speech API
// and prints word offset information.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"golang.org/x/net/context"

	speech "cloud.google.com/go/speech/apiv1"
	speechpb "google.golang.org/genproto/googleapis/cloud/speech/v1"
)

const usage = `Usage: wordoffset <audiofile>

Audio file must be a 16-bit signed little-endian encoded
with a sample rate of 16000.

The path to the audio file may be a GCS URI (gs://...).
`

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(2)
	}

	var sendFunc func(*speech.Client, io.Writer, string) error

	path := os.Args[1]
	if strings.Contains(path, "://") {
		sendFunc = asyncWords
	} else {
		sendFunc = syncWords
	}

	ctx := context.Background()
	client, err := speech.NewClient(ctx)
	if err != nil {
		log.Fatal(err)
	}

	if err := sendFunc(client, os.Stdout, os.Args[1]); err != nil {
		log.Fatal(err)
	}
}

// [START speech_transcribe_async_time_offsets_gcs]

func asyncWords(client *speech.Client, out io.Writer, gcsURI string) error {
	ctx := context.Background()

	// Send the contents of the audio file with the encoding and
	// and sample rate information to be transcripted.
	req := &speechpb.LongRunningRecognizeRequest{
		Config: &speechpb.RecognitionConfig{
			Encoding:              speechpb.RecognitionConfig_LINEAR16,
			SampleRateHertz:       16000,
			LanguageCode:          "en-US",
			EnableWordTimeOffsets: true,
		},
		Audio: &speechpb.RecognitionAudio{
			AudioSource: &speechpb.RecognitionAudio_Uri{Uri: gcsURI},
		},
	}

	op, err := client.LongRunningRecognize(ctx, req)
	if err != nil {
		return err
	}
	resp, err := op.Wait(ctx)
	if err != nil {
		return err
	}

	// Print the results.
	for _, result := range resp.Results {
		for _, alt := range result.Alternatives {
			fmt.Fprintf(out, "\"%v\" (confidence=%3f)\n", alt.Transcript, alt.Confidence)
			for _, w := range alt.Words {
				fmt.Fprintf(out,
					"Word: \"%v\" (startTime=%3f, endTime=%3f)\n",
					w.Word,
					float64(w.StartTime.Seconds)+float64(w.StartTime.Nanos)*1e-9,
					float64(w.EndTime.Seconds)+float64(w.EndTime.Nanos)*1e-9,
				)
			}
		}
	}
	return nil
}

// [END speech_transcribe_async_time_offsets_gcs]

func syncWords(client *speech.Client, out io.Writer, file string) error {
	ctx := context.Background()

	data, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatal(err)
	}

	// Send the contents of the audio file with the encoding and
	// and sample rate information to be transcripted.
	req := &speechpb.RecognizeRequest{
		Config: &speechpb.RecognitionConfig{
			Encoding:              speechpb.RecognitionConfig_LINEAR16,
			SampleRateHertz:       16000,
			LanguageCode:          "en-US",
			EnableWordTimeOffsets: true,
		},
		Audio: &speechpb.RecognitionAudio{
			AudioSource: &speechpb.RecognitionAudio_Content{Content: data},
		},
	}

	resp, err := client.Recognize(ctx, req)
	if err != nil {
		return err
	}

	// Print the results.
	for _, result := range resp.Results {
		for _, alt := range result.Alternatives {
			fmt.Fprintf(out, "\"%v\" (confidence=%3f)\n", alt.Transcript, alt.Confidence)
			for _, w := range alt.Words {
				fmt.Fprintf(out,
					"Word: \"%v\" (startTime=%3f, endTime=%3f)\n",
					w.Word,
					float64(w.StartTime.Seconds)+float64(w.StartTime.Nanos)*1e-9,
					float64(w.EndTime.Seconds)+float64(w.EndTime.Nanos)*1e-9,
				)
			}
		}
	}
	return nil
}
<commit_msg>speech: update word time offsets region tag (#585)<commit_after>// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

// Command wordoffset sends audio data to the Google Speech API
// and prints word offset information.
package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"golang.org/x/net/context"

	speech "cloud.google.com/go/speech/apiv1"
	speechpb "google.golang.org/genproto/googleapis/cloud/speech/v1"
)

const usage = `Usage: wordoffset <audiofile>

Audio file must be a 16-bit signed little-endian encoded
with a sample rate of 16000.

The path to the audio file may be a GCS URI (gs://...).
`

func main() {
	if len(os.Args) < 2 {
		fmt.Fprintln(os.Stderr, usage)
		os.Exit(2)
	}

	var sendFunc func(*speech.Client, io.Writer, string) error

	path := os.Args[1]
	if strings.Contains(path, "://") {
		sendFunc = asyncWords
	} else {
		sendFunc = syncWords
	}

	ctx := context.Background()
	client, err := speech.NewClient(ctx)
	if err != nil {
		log.Fatal(err)
	}

	if err := sendFunc(client, os.Stdout, os.Args[1]); err != nil {
		log.Fatal(err)
	}
}

// [START speech_transcribe_async_word_time_offsets_gcs]

func asyncWords(client *speech.Client, out io.Writer, gcsURI string) error {
	ctx := context.Background()

	// Send the contents of the audio file with the encoding and
	// and sample rate information to be transcripted.
	req := &speechpb.LongRunningRecognizeRequest{
		Config: &speechpb.RecognitionConfig{
			Encoding:              speechpb.RecognitionConfig_LINEAR16,
			SampleRateHertz:       16000,
			LanguageCode:          "en-US",
			EnableWordTimeOffsets: true,
		},
		Audio: &speechpb.RecognitionAudio{
			AudioSource: &speechpb.RecognitionAudio_Uri{Uri: gcsURI},
		},
	}

	op, err := client.LongRunningRecognize(ctx, req)
	if err != nil {
		return err
	}
	resp, err := op.Wait(ctx)
	if err != nil {
		return err
	}

	// Print the results.
	for _, result := range resp.Results {
		for _, alt := range result.Alternatives {
			fmt.Fprintf(out, "\"%v\" (confidence=%3f)\n", alt.Transcript, alt.Confidence)
			for _, w := range alt.Words {
				fmt.Fprintf(out,
					"Word: \"%v\" (startTime=%3f, endTime=%3f)\n",
					w.Word,
					float64(w.StartTime.Seconds)+float64(w.StartTime.Nanos)*1e-9,
					float64(w.EndTime.Seconds)+float64(w.EndTime.Nanos)*1e-9,
				)
			}
		}
	}
	return nil
}

// [END speech_transcribe_async_word_time_offsets_gcs]

func syncWords(client *speech.Client, out io.Writer, file string) error {
	ctx := context.Background()

	data, err := ioutil.ReadFile(file)
	if err != nil {
		log.Fatal(err)
	}

	// Send the contents of the audio file with the encoding and
	// and sample rate information to be transcripted.
	req := &speechpb.RecognizeRequest{
		Config: &speechpb.RecognitionConfig{
			Encoding:              speechpb.RecognitionConfig_LINEAR16,
			SampleRateHertz:       16000,
			LanguageCode:          "en-US",
			EnableWordTimeOffsets: true,
		},
		Audio: &speechpb.RecognitionAudio{
			AudioSource: &speechpb.RecognitionAudio_Content{Content: data},
		},
	}

	resp, err := client.Recognize(ctx, req)
	if err != nil {
		return err
	}

	// Print the results.
	for _, result := range resp.Results {
		for _, alt := range result.Alternatives {
			fmt.Fprintf(out, "\"%v\" (confidence=%3f)\n", alt.Transcript, alt.Confidence)
			for _, w := range alt.Words {
				fmt.Fprintf(out,
					"Word: \"%v\" (startTime=%3f, endTime=%3f)\n",
					w.Word,
					float64(w.StartTime.Seconds)+float64(w.StartTime.Nanos)*1e-9,
					float64(w.EndTime.Seconds)+float64(w.EndTime.Nanos)*1e-9,
				)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package drivers

import "testing"

func Test_ceph_getRBDVolumeName(t *testing.T) {
	type args struct {
		vol          Volume
		snapName     string
		zombie       bool
		withPoolName bool
	}
	tests := []struct {
		name string
		args args
		want string
	}{
		{
			"Volume without pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "",
				zombie:       false,
				withPoolName: false,
			},
			"container_testvol",
		},
		{
			"Volume with unknown type and without pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeType("unknown"), ContentTypeFS, "testvol", nil, nil),
				snapName:     "",
				zombie:       false,
				withPoolName: false,
			},
			"unknown_testvol",
		},
		{
			"Volume without pool name in zombie mode",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "",
				zombie:       true,
				withPoolName: false,
			},
			"zombie_container_testvol",
		},
		{
			"Volume with pool name in zombie mode",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "",
				zombie:       true,
				withPoolName: true,
			},
			"testosdpool/zombie_container_testvol",
		},
		{
			"Volume snapshot with dedicated snapshot name and without pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "snapshot_testsnap",
				zombie:       false,
				withPoolName: false,
			},
			"container_testvol@snapshot_testsnap",
		},
		{
			"Volume snapshot with dedicated snapshot name and pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "snapshot_testsnap",
				zombie:       false,
				withPoolName: true,
			},
			"testosdpool/container_testvol@snapshot_testsnap",
		},
		{
			"Volume snapshot with pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol/testsnap", nil, nil),
				snapName:     "",
				zombie:       false,
				withPoolName: true,
			},
			"testosdpool/container_testvol@snapshot_testsnap",
		},
		{
			"Volume snapshot with additional dedicated snapshot name and pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol/testsnap", nil, nil),
				snapName:     "testsnap1",
				zombie:       false,
				withPoolName: true,
			},
			"testosdpool/container_testvol@testsnap1",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			d := &ceph{
				common{
					config: map[string]string{
						"ceph.osd.pool_name": "testosdpool",
					},
				},
			}
			if got := d.getRBDVolumeName(tt.args.vol, tt.args.snapName, tt.args.zombie, tt.args.withPoolName); got != tt.want {
				t.Errorf("ceph.getRBDVolumeName() = %v, want %v", got, tt.want)
			}
		})
	}
}
<commit_msg>lxd/storage/drivers/driver/ceph/utils: Adds tests for parseParent<commit_after>package drivers

import (
	"fmt"
	"testing"
)

func Test_ceph_getRBDVolumeName(t *testing.T) {
	type args struct {
		vol          Volume
		snapName     string
		zombie       bool
		withPoolName bool
	}
	tests := []struct {
		name string
		args args
		want string
	}{
		{
			"Volume without pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "",
				zombie:       false,
				withPoolName: false,
			},
			"container_testvol",
		},
		{
			"Volume with unknown type and without pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeType("unknown"), ContentTypeFS, "testvol", nil, nil),
				snapName:     "",
				zombie:       false,
				withPoolName: false,
			},
			"unknown_testvol",
		},
		{
			"Volume without pool name in zombie mode",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "",
				zombie:       true,
				withPoolName: false,
			},
			"zombie_container_testvol",
		},
		{
			"Volume with pool name in zombie mode",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "",
				zombie:       true,
				withPoolName: true,
			},
			"testosdpool/zombie_container_testvol",
		},
		{
			"Volume snapshot with dedicated snapshot name and without pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "snapshot_testsnap",
				zombie:       false,
				withPoolName: false,
			},
			"container_testvol@snapshot_testsnap",
		},
		{
			"Volume snapshot with dedicated snapshot name and pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol", nil, nil),
				snapName:     "snapshot_testsnap",
				zombie:       false,
				withPoolName: true,
			},
			"testosdpool/container_testvol@snapshot_testsnap",
		},
		{
			"Volume snapshot with pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol/testsnap", nil, nil),
				snapName:     "",
				zombie:       false,
				withPoolName: true,
			},
			"testosdpool/container_testvol@snapshot_testsnap",
		},
		{
			"Volume snapshot with additional dedicated snapshot name and pool name",
			args{
				vol:          NewVolume(nil, "testpool", VolumeTypeContainer, ContentTypeFS, "testvol/testsnap", nil, nil),
				snapName:     "testsnap1",
				zombie:       false,
				withPoolName: true,
			},
			"testosdpool/container_testvol@testsnap1",
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			d := &ceph{
				common{
					config: map[string]string{
						"ceph.osd.pool_name": "testosdpool",
					},
				},
			}
			if got := d.getRBDVolumeName(tt.args.vol, tt.args.snapName, tt.args.zombie, tt.args.withPoolName); got != tt.want {
				t.Errorf("ceph.getRBDVolumeName() = %v, want %v", got, tt.want)
			}
		})
	}
}
func Example_ceph_parseParent() {
	d := &ceph{}

	parents := []string{
		"pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block@readonly",
		"pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block",
		"pool/image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4.block@readonly",
		"pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4@readonly",
		"pool/zombie_image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4",
		"pool/image_9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb_ext4@readonly",
		"pool/zombie_image_2cfc5a5567b8d74c0986f3d8a77a2a78e58fe22ea9abd2693112031f85afa1a1_xfs@zombie_snapshot_7f6d679b-ee25-419e-af49-bb805cb32088",
		"pool/container_bar@zombie_snapshot_ce77e971-6c1b-45c0-b193-dba9ec5e7d82",
		"pool/container_test-project_c4.block",
		"pool/zombie_container_test-project_c1_28e7a7ab-740a-490c-8118-7caf7810f83b@zombie_snapshot_1027f4ab-de11-4cee-8015-bd532a1fed76",
	}

	for _, parent := range parents {
		vol, snapName, err := d.parseParent(parent)
		fmt.Println(vol.pool, vol.volType, vol.name, vol.config["block.filesystem"], vol.contentType, snapName, err)
	}

	// Output: pool zombie_image 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb ext4 block readonly <nil>
	// pool zombie_image 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb ext4 block  <nil>
	// pool image 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb ext4 block readonly <nil>
	// pool zombie_image 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb ext4 fs readonly <nil>
	// pool zombie_image 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb ext4 fs  <nil>
	// pool image 9e90b7b9ccdd7a671a987fadcf07ab92363be57e7f056d18d42af452cdaf95bb ext4 fs readonly <nil>
	// pool zombie_image 2cfc5a5567b8d74c0986f3d8a77a2a78e58fe22ea9abd2693112031f85afa1a1 xfs fs zombie_snapshot_7f6d679b-ee25-419e-af49-bb805cb32088 <nil>
	// pool container bar  fs zombie_snapshot_ce77e971-6c1b-45c0-b193-dba9ec5e7d82 <nil>
	// pool container test-project_c4  block  <nil>
	// pool zombie_container test-project_c1_28e7a7ab-740a-490c-8118-7caf7810f83b  fs zombie_snapshot_1027f4ab-de11-4cee-8015-bd532a1fed76 <nil>
}
<|endoftext|>


<commit_before>package pdb

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/BurntSushi/bcbgo/seq"
)

// AminoThreeToOne is a map from three letter amino acids to their
// corresponding single letter representation.
var AminoThreeToOne = map[string]byte{
	"ALA": 'A', "ARG": 'R', "ASN": 'N', "ASP": 'D', "CYS": 'C',
	"GLU": 'E', "GLN": 'Q', "GLY": 'G', "HIS": 'H', "ILE": 'I',
	"LEU": 'L', "LYS": 'K', "MET": 'M', "PHE": 'F', "PRO": 'P',
	"SER": 'S', "THR": 'T', "TRP": 'W', "TYR": 'Y', "VAL": 'V',
	"SEC": 'U', "PYL": 'O',
	"UNK": 'X', "ACE": 'X', "NH2": 'X',
	"ASX": 'X', "GLX": 'X',
}

// AminoOneToThree is the reverse of AminoThreeToOne. It is created in
// this packages 'init' function.
var AminoOneToThree = map[byte]string{}

func init() {
	// Create a reverse map of AminoThreeToOne.
	for k, v := range AminoThreeToOne {
		AminoOneToThree[v] = k
	}
}

// Entry represents all information known about a particular PDB file (that
// has been implemented in this package).
//
// Currently, a PDB entry is simply a file path and a map of protein chains.
type Entry struct {
	Path           string
	IdCode         string
	Classification string
	Chains         []*Chain
}

// New creates a new PDB Entry from a file. If the file cannot be read, or there
// is an error parsing the PDB file, an error is returned.
//
// If the file name ends with ".gz", gzip decompression will be used.
func New(fileName string) (*Entry, error) {
	var reader io.Reader
	var err error

	reader, err = os.Open(fileName)
	if err != nil {
		return nil, err
	}

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fileName) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	entry := &Entry{
		Path:   fileName,
		Chains: make([]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	// Note that it is imperative that we preserve the order of ATOM records
	// as we read them. We are currently trying to replicate Fragbag, and this
	// is what Fragbag does. (A more stable approach would require more
	// information from the PDB file; like differentiating models, since
	// sorting on ATOM serial number isn't good enough.)
	breader := bufio.NewReaderSize(reader, 1000)
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		// The record name is always in the fix six columns.
		switch strings.TrimSpace(string(line[0:6])) {
		case "HEADER":
			if err := entry.parseHeader(line); err != nil {
				return nil, err
			}
		case "SEQRES":
			entry.parseSeqres(line)
		case "ATOM":
			entry.parseAtom(line)
		}
	}

	// If we didn't pick up any chains, this probably isn't a valid PDB file.
	if len(entry.Chains) == 0 {
		return nil, fmt.Errorf("The file '%s' does not appear to be a valid "+
			"PDB file.", fileName)
	}

	// If we couldn't find an Id code, inspect the base name of the file path.
	if len(entry.IdCode) == 0 {
		name := path.Base(fileName)
		switch {
		case len(name) >= 7 && name[0:3] == "pdb":
			entry.IdCode = name[3:7]
		case len(name) == 7: // cath
			entry.IdCode = name[0:4]
		}
	}

	return entry, nil
}

// Chain looks for the chain with identifier ident and returns it. 'nil' is
// returned if the chain could not be found.
func (e *Entry) Chain(ident byte) *Chain {
	for _, chain := range e.Chains {
		if chain.Ident == ident {
			return chain
		}
	}
	return nil
}

// OneChain returns a single chain in the PDB file. If there is more than one
// chain, OneChain will panic. This is convenient when you expect a PDB file to
// have only a single chain, but don't know the name.
func (e *Entry) OneChain() *Chain {
	if len(e.Chains) != 1 {
		panic(fmt.Sprintf("OneChain can only be called on PDB entries with "+
			"ONE chain. But the '%s' PDB entry has %d chains.",
			e.Path, len(e.Chains)))
	}
	return e.Chains[0]
}

// Name returns the base name of the path of this PDB entry.
func (e *Entry) Name() string {
	return path.Base(e.Path)
}

// String returns a list of all chains, their residue start/stop indices,
// and the amino acid sequence.
func (e *Entry) String() string {
	lines := make([]string, 0)
	for _, chain := range e.Chains {
		lines = append(lines, chain.String())
	}
	return strings.Join(lines, "\n")
}

// getOrMakeChain looks for a chain in the 'Chains' slice corresponding to the
// chain indentifier. If one exists, it is returned. If one doesn't exist,
// it is created, memory is allocated and it is returned.
func (e *Entry) getOrMakeChain(ident byte) *Chain {
	if ident == ' ' {
		ident = '_'
	}

	chain := e.Chain(ident)
	if chain != nil {
		return chain
	}
	newChain := &Chain{
		Entry:            e,
		Ident:            ident,
		Sequence:         make([]seq.Residue, 0, 30),
		AtomResidueStart: 0,
		AtomResidueEnd:   0,
		CaAtoms:          make(Atoms, 0, 30),
		CaSequence:       make([]seq.Residue, 0, 30),
		CaSeqRes:         make([]*Atom, 0, 30),
	}
	e.Chains = append(e.Chains, newChain)
	return newChain
}

// parseHeader loads the "idCode" and "classification" fields from the
// header record.
//
// If the fields are already filled, then we've seen a second header record
// and therefore report an error.
func (e *Entry) parseHeader(line []byte) error {
	if len(e.Classification) > 0 || len(e.IdCode) > 0 {
		return fmt.Errorf("More than one HEADER record was found.")
	}
	e.Classification = strings.TrimSpace(string(line[10:50]))
	e.IdCode = strings.TrimSpace(string(line[62:66]))
	return nil
}

// parseSeqres loads all pertinent information from SEQRES records in a PDB
// file. In particular, amino acid resides are read and added to the chain's
// "Sequence" field. If a residue isn't a valid amino acid, it is simply
// ignored.
//
// N.B. This assumes that the SEQRES records are in order in the PDB file.
func (e *Entry) parseSeqres(line []byte) {
	chain := e.getOrMakeChain(line[11])

	// Residues are in columns 19-21, 23-25, 27-29, ..., 67-69
	for i := 19; i <= 67; i += 4 {
		end := i + 3

		// If we're passed the end of this line, quit.
		if end >= len(line) {
			break
		}

		// Get the residue. If it's not in our sequence map, skip it.
		residue := strings.TrimSpace(string(line[i:end]))
		if single, ok := AminoThreeToOne[residue]; ok {
			chain.Sequence = append(chain.Sequence, seq.Residue(single))
			chain.CaSeqRes = append(chain.CaSeqRes, nil)
		}
	}
}

// parseAtom loads all pertinent information from ATOM records in a PDB file.
// Currently, this only includes deducing the amino acid residue start and
// stop indices. (Note that the length of the range is not necessarily
// equivalent to the length of the amino acid sequence found in the SEQRES
// records.)
//
// ATOM records without a valid amino acid residue in columns 18-20 are ignored.
func (e *Entry) parseAtom(line []byte) {
	chain := e.getOrMakeChain(line[21])

	// An ATOM record is only processed if it corresponds to an amino acid
	// residue. (Which is in columns 17-19.)
	residue := strings.TrimSpace(string(line[17:20]))
	if _, ok := AminoThreeToOne[residue]; !ok {
		// Sanity check. I'm pretty sure that only amino acids have three
		// letter abbreviations.
		if len(residue) == 3 {
			panic(fmt.Sprintf("The residue '%s' found in PDB file '%s' has "+
				"length 3, but is not in my amino acid map.",
				residue, e.Path))
		}
		return
	}

	// The residue sequence number is in columns 22-25. Grab it, trim it,
	// and look for an integer.
	snum := strings.TrimSpace(string(line[22:26]))
	inum := int(0)
	if num, err := strconv.ParseInt(snum, 10, 32); err == nil {
		inum = int(num)
		switch {
		case chain.AtomResidueStart == 0 || inum < chain.AtomResidueStart:
			chain.AtomResidueStart = inum
		case chain.AtomResidueEnd == 0 || inum > chain.AtomResidueEnd:
			chain.AtomResidueEnd = inum
		}
	}

	// Build an Atom value. We need the serial number from columns 6-10,
	// the atom name from columns 12-15, the amino acid residue from
	// columns 17-19 (we already have that: 'residue'), the residue sequence
	// number from columns 22-25 (already have that too: 'inum'), and the
	// three dimension coordinates in columns 30-37 (x), 38-45 (y), and
	// 46-53 (z).
	atom := Atom{
		Name:       strings.TrimSpace(string(line[12:16])),
		Residue:    residue,
		ResidueInd: inum,
		Coords:     [3]float64{},
	}

	serialStr := strings.TrimSpace(string(line[6:11]))
	if serial64, err := strconv.ParseInt(serialStr, 10, 32); err == nil {
		atom.Serial = int(serial64)
	}

	xstr := strings.TrimSpace(string(line[30:38]))
	ystr := strings.TrimSpace(string(line[38:46]))
	zstr := strings.TrimSpace(string(line[46:54]))
	if x64, err := strconv.ParseFloat(xstr, 64); err == nil {
		atom.Coords[0] = x64
	}
	if y64, err := strconv.ParseFloat(ystr, 64); err == nil {
		atom.Coords[1] = y64
	}
	if z64, err := strconv.ParseFloat(zstr, 64); err == nil {
		atom.Coords[2] = z64
	}

	// Now add our atom to the chain.
	chain.Atoms = append(chain.Atoms, atom)
	if atom.Name == "CA" {
		chain.CaAtoms = append(chain.CaAtoms, atom)
		chain.CaSequence = append(chain.CaSequence,
			seq.Residue(AminoThreeToOne[residue]))

		// If we have a valid residue number, then add this atom into our
		// CaSeqRes list. Which is a correspondence between residues and
		// *maybe* atoms.
		if inum > 0 {
			chain.CaSeqRes[inum-1] = &atom
		}
	}
}

// Chain represents a protein chain or subunit in a PDB file. Each chain has
// its own identifier, amino acid sequence (if its a protein sequence), and
// the start and stop residue indices of the ATOM coordinates.
//
// It also contains a slice of all carbon-alpha ATOM records corresponding
// to an amino acid.
type Chain struct {
	Entry                            *Entry
	Ident                            byte
	Sequence                         []seq.Residue
	AtomResidueStart, AtomResidueEnd int
	Atoms                            Atoms
	CaAtoms                          Atoms
	CaSequence                       []seq.Residue
	CaSeqRes                         []*Atom
}

// ValidProtein returns true when there are ATOM records corresponding to
// a protein backbone.
func (c *Chain) ValidProtein() bool {
	return c.AtomResidueStart != c.AtomResidueEnd
}

// String returns a FASTA-like formatted string of this chain and all of its
// related information.
func (c *Chain) String() string {
	return strings.TrimSpace(
		fmt.Sprintf("> Chain %c (%d, %d) :: length %d\n%s",
			c.Ident, c.AtomResidueStart, c.AtomResidueEnd,
			len(c.Sequence), c.Sequence))
}

// Atom contains information about an ATOM record, including the serial
// number, the residue (and residue sequence number), the atom name and the
// three dimensional coordinates.
type Atom struct {
	Serial     int
	Name       string
	ResidueInd int
	Residue    string

	// Coords is a triple where the first element is X, the second is Y and
	// the third is Z.
	Coords [3]float64
}

func (a Atom) String() string {
	return fmt.Sprintf("(%d, %s, %d, %s, [%0.4f %0.4f %0.4f])",
		a.Serial, a.Name, a.ResidueInd, a.Residue,
		a.Coords[0], a.Coords[1], a.Coords[2])
}

// Atoms names a slice of Atom for sorting.
type Atoms []Atom

func (as Atoms) String() string {
	lines := make([]string, len(as))
	for i, atom := range as {
		lines[i] = atom.String()
	}
	return strings.Join(lines, "\n")
}
<commit_msg>Commit before reorg<commit_after>package pdb

import (
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/BurntSushi/bcbgo/seq"
)

// AminoThreeToOne is a map from three letter amino acids to their
// corresponding single letter representation.
var AminoThreeToOne = map[string]byte{
	"ALA": 'A', "ARG": 'R', "ASN": 'N', "ASP": 'D', "CYS": 'C',
	"GLU": 'E', "GLN": 'Q', "GLY": 'G', "HIS": 'H', "ILE": 'I',
	"LEU": 'L', "LYS": 'K', "MET": 'M', "PHE": 'F', "PRO": 'P',
	"SER": 'S', "THR": 'T', "TRP": 'W', "TYR": 'Y', "VAL": 'V',
	"SEC": 'U', "PYL": 'O',
	"UNK": 'X', "ACE": 'X', "NH2": 'X',
	"ASX": 'X', "GLX": 'X',
	"MSE": 'M',
}

// AminoOneToThree is the reverse of AminoThreeToOne. It is created in
// this packages 'init' function.
var AminoOneToThree = map[byte]string{}

func init() {
	// Create a reverse map of AminoThreeToOne.
	for k, v := range AminoThreeToOne {
		AminoOneToThree[v] = k
	}
}

// Entry represents all information known about a particular PDB file (that
// has been implemented in this package).
//
// Currently, a PDB entry is simply a file path and a map of protein chains.
type Entry struct {
	Path           string
	IdCode         string
	Classification string
	Chains         []*Chain
}

// New creates a new PDB Entry from a file. If the file cannot be read, or there
// is an error parsing the PDB file, an error is returned.
//
// If the file name ends with ".gz", gzip decompression will be used.
func New(fileName string) (*Entry, error) {
	var reader io.Reader
	var err error

	reader, err = os.Open(fileName)
	if err != nil {
		return nil, err
	}

	// If the file is gzipped, use the gzip decompressor.
	if path.Ext(fileName) == ".gz" {
		reader, err = gzip.NewReader(reader)
		if err != nil {
			return nil, err
		}
	}

	entry := &Entry{
		Path:   fileName,
		Chains: make([]*Chain, 0),
	}

	// Now traverse each line, and process it according to the record name.
	// Note that it is imperative that we preserve the order of ATOM records
	// as we read them. We are currently trying to replicate Fragbag, and this
	// is what Fragbag does. (A more stable approach would require more
	// information from the PDB file; like differentiating models, since
	// sorting on ATOM serial number isn't good enough.)
	breader := bufio.NewReaderSize(reader, 1000)
	for {
		// We ignore 'isPrefix' here, since we never care about lines longer
		// than 1000 characters, which is the size of our buffer.
		line, _, err := breader.ReadLine()
		if err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}

		// The record name is always in the fix six columns.
		switch strings.TrimSpace(string(line[0:6])) {
		case "HEADER":
			if err := entry.parseHeader(line); err != nil {
				return nil, err
			}
		case "SEQRES":
			entry.parseSeqres(line)
		case "ATOM":
			entry.parseAtom(line)
		}
	}

	// If we didn't pick up any chains, this probably isn't a valid PDB file.
	if len(entry.Chains) == 0 {
		return nil, fmt.Errorf("The file '%s' does not appear to be a valid "+
			"PDB file.", fileName)
	}

	// If we couldn't find an Id code, inspect the base name of the file path.
	if len(entry.IdCode) == 0 {
		name := path.Base(fileName)
		switch {
		case len(name) >= 7 && name[0:3] == "pdb":
			entry.IdCode = name[3:7]
		case len(name) == 7: // cath
			entry.IdCode = name[0:4]
		}
	}

	return entry, nil
}

// Chain looks for the chain with identifier ident and returns it. 'nil' is
// returned if the chain could not be found.
func (e *Entry) Chain(ident byte) *Chain {
	for _, chain := range e.Chains {
		if chain.Ident == ident {
			return chain
		}
	}
	return nil
}

// OneChain returns a single chain in the PDB file. If there is more than one
// chain, OneChain will panic. This is convenient when you expect a PDB file to
// have only a single chain, but don't know the name.
func (e *Entry) OneChain() *Chain {
	if len(e.Chains) != 1 {
		panic(fmt.Sprintf("OneChain can only be called on PDB entries with "+
			"ONE chain. But the '%s' PDB entry has %d chains.",
			e.Path, len(e.Chains)))
	}
	return e.Chains[0]
}

// Name returns the base name of the path of this PDB entry.
func (e *Entry) Name() string {
	return path.Base(e.Path)
}

// String returns a list of all chains, their residue start/stop indices,
// and the amino acid sequence.
func (e *Entry) String() string {
	lines := make([]string, 0)
	for _, chain := range e.Chains {
		lines = append(lines, chain.String())
	}
	return strings.Join(lines, "\n")
}

// getOrMakeChain looks for a chain in the 'Chains' slice corresponding to the
// chain indentifier. If one exists, it is returned. If one doesn't exist,
// it is created, memory is allocated and it is returned.
func (e *Entry) getOrMakeChain(ident byte) *Chain {
	if ident == ' ' {
		ident = '_'
	}

	chain := e.Chain(ident)
	if chain != nil {
		return chain
	}
	newChain := &Chain{
		Entry:            e,
		Ident:            ident,
		Sequence:         make([]seq.Residue, 0, 30),
		AtomResidueStart: 0,
		AtomResidueEnd:   0,
		CaAtoms:          make(Atoms, 0, 30),
		CaSequence:       make([]seq.Residue, 0, 30),
		CaSeqRes:         make([]*Atom, 0, 30),
	}
	e.Chains = append(e.Chains, newChain)
	return newChain
}

// parseHeader loads the "idCode" and "classification" fields from the
// header record.
//
// If the fields are already filled, then we've seen a second header record
// and therefore report an error.
func (e *Entry) parseHeader(line []byte) error {
	if len(e.Classification) > 0 || len(e.IdCode) > 0 {
		return fmt.Errorf("More than one HEADER record was found.")
	}
	e.Classification = strings.TrimSpace(string(line[10:50]))
	e.IdCode = strings.TrimSpace(string(line[62:66]))
	return nil
}

// parseSeqres loads all pertinent information from SEQRES records in a PDB
// file. In particular, amino acid resides are read and added to the chain's
// "Sequence" field. If a residue isn't a valid amino acid, it is simply
// ignored.
//
// N.B. This assumes that the SEQRES records are in order in the PDB file.
func (e *Entry) parseSeqres(line []byte) {
	chain := e.getOrMakeChain(line[11])

	// Residues are in columns 19-21, 23-25, 27-29, ..., 67-69
	for i := 19; i <= 67; i += 4 {
		end := i + 3

		// If we're passed the end of this line, quit.
		if end >= len(line) {
			break
		}

		// Get the residue. If it's not in our sequence map, skip it.
		residue := strings.TrimSpace(string(line[i:end]))
		if len(residue) == 0 {
			break
		}
		if single, ok := AminoThreeToOne[residue]; ok {
			chain.Sequence = append(chain.Sequence, seq.Residue(single))
			chain.CaSeqRes = append(chain.CaSeqRes, nil)
		} else {
			panic(fmt.Sprintf("Unknown residue '%s'.", residue))
		}
	}
}

// parseAtom loads all pertinent information from ATOM records in a PDB file.
// Currently, this only includes deducing the amino acid residue start and
// stop indices. (Note that the length of the range is not necessarily
// equivalent to the length of the amino acid sequence found in the SEQRES
// records.)
//
// ATOM records without a valid amino acid residue in columns 18-20 are ignored.
func (e *Entry) parseAtom(line []byte) {
	chain := e.getOrMakeChain(line[21])

	// An ATOM record is only processed if it corresponds to an amino acid
	// residue. (Which is in columns 17-19.)
	residue := strings.TrimSpace(string(line[17:20]))
	if _, ok := AminoThreeToOne[residue]; !ok {
		// Sanity check. I'm pretty sure that only amino acids have three
		// letter abbreviations.
		if len(residue) == 3 {
			panic(fmt.Sprintf("The residue '%s' found in PDB file '%s' has "+
				"length 3, but is not in my amino acid map.",
				residue, e.Path))
		}
		return
	}

	// The residue sequence number is in columns 22-25. Grab it, trim it,
	// and look for an integer.
	snum := strings.TrimSpace(string(line[22:26]))
	inum := int(0)
	if num, err := strconv.ParseInt(snum, 10, 32); err == nil {
		inum = int(num)
		switch {
		case chain.AtomResidueStart == 0 || inum < chain.AtomResidueStart:
			chain.AtomResidueStart = inum
		case chain.AtomResidueEnd == 0 || inum > chain.AtomResidueEnd:
			chain.AtomResidueEnd = inum
		}
	}

	// Build an Atom value. We need the serial number from columns 6-10,
	// the atom name from columns 12-15, the amino acid residue from
	// columns 17-19 (we already have that: 'residue'), the residue sequence
	// number from columns 22-25 (already have that too: 'inum'), and the
	// three dimension coordinates in columns 30-37 (x), 38-45 (y), and
	// 46-53 (z).
	atom := Atom{
		Name:       strings.TrimSpace(string(line[12:16])),
		Residue:    residue,
		ResidueInd: inum,
		Coords:     [3]float64{},
	}

	serialStr := strings.TrimSpace(string(line[6:11]))
	if serial64, err := strconv.ParseInt(serialStr, 10, 32); err == nil {
		atom.Serial = int(serial64)
	}

	xstr := strings.TrimSpace(string(line[30:38]))
	ystr := strings.TrimSpace(string(line[38:46]))
	zstr := strings.TrimSpace(string(line[46:54]))
	if x64, err := strconv.ParseFloat(xstr, 64); err == nil {
		atom.Coords[0] = x64
	}
	if y64, err := strconv.ParseFloat(ystr, 64); err == nil {
		atom.Coords[1] = y64
	}
	if z64, err := strconv.ParseFloat(zstr, 64); err == nil {
		atom.Coords[2] = z64
	}

	// Now add our atom to the chain.
	chain.Atoms = append(chain.Atoms, atom)
	if atom.Name == "CA" {
		chain.CaAtoms = append(chain.CaAtoms, atom)
		chain.CaSequence = append(chain.CaSequence,
			seq.Residue(AminoThreeToOne[residue]))

		// If we have a valid residue number, then add this atom into our
		// CaSeqRes list. Which is a correspondence between residues and
		// *maybe* atoms.
		if inum > 0 {
			if inum-1 >= len(chain.CaSeqRes) {
				println(inum-1, len(chain.CaSeqRes))
				fmt.Printf("%d :: %s\n", len(chain.Sequence), chain.Sequence)
			}
			chain.CaSeqRes[inum-1] = &atom
		}
	}
}

// Chain represents a protein chain or subunit in a PDB file. Each chain has
// its own identifier, amino acid sequence (if its a protein sequence), and
// the start and stop residue indices of the ATOM coordinates.
//
// It also contains a slice of all carbon-alpha ATOM records corresponding
// to an amino acid.
type Chain struct {
	Entry                            *Entry
	Ident                            byte
	Sequence                         []seq.Residue
	AtomResidueStart, AtomResidueEnd int
	Atoms                            Atoms
	CaAtoms                          Atoms
	CaSequence                       []seq.Residue
	CaSeqRes                         []*Atom
}

// ValidProtein returns true when there are ATOM records corresponding to
// a protein backbone.
func (c *Chain) ValidProtein() bool {
	return c.AtomResidueStart != c.AtomResidueEnd
}

// String returns a FASTA-like formatted string of this chain and all of its
// related information.
func (c *Chain) String() string {
	return strings.TrimSpace(
		fmt.Sprintf("> Chain %c (%d, %d) :: length %d\n%s",
			c.Ident, c.AtomResidueStart, c.AtomResidueEnd,
			len(c.Sequence), c.Sequence))
}

// CaAtomSlice attempts to extract a contiguous slice of alpha-carbon ATOM
// records based on *residue* index. Namely, if a contiguous slice cannot be
// found, nil is returned.
func (c *Chain) CaAtomSlice(start, end int) Atoms {
	atoms := make(Atoms, end-start)
	for i, cai := 0, start; cai < end; i, cai = i+1, cai+1 {
		if c.CaSeqRes[cai] == nil {
			return nil
		}
		atoms[i] = *c.CaSeqRes[cai]
	}
	return atoms
}

// Atom contains information about an ATOM record, including the serial
// number, the residue (and residue sequence number), the atom name and the
// three dimensional coordinates.
type Atom struct {
	Serial     int
	Name       string
	ResidueInd int
	Residue    string

	// Coords is a triple where the first element is X, the second is Y and
	// the third is Z.
	Coords [3]float64
}

func (a Atom) String() string {
	return fmt.Sprintf("(%d, %s, %d, %s, [%0.4f %0.4f %0.4f])",
		a.Serial, a.Name, a.ResidueInd, a.Residue,
		a.Coords[0], a.Coords[1], a.Coords[2])
}

// Atoms names a slice of Atom for sorting.
type Atoms []Atom

func (as Atoms) String() string {
	lines := make([]string, len(as))
	for i, atom := range as {
		lines[i] = atom.String()
	}
	return strings.Join(lines, "\n")
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/moov-io/ach"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/gorilla/mux"
)

var (
	bugReportHelp = "please report this as a bug -- https://github.com/moov-io/ach/issues/new"

	// ErrBadRouting is returned when an expected path variable is missing, which is always programmer error.
	ErrBadRouting = fmt.Errorf("inconsistent mapping between route and handler, %s", bugReportHelp)
	ErrFoundABug  = fmt.Errorf("Snuck into encodeError with err == nil, %s", bugReportHelp)

	MaxContentLength = 1 * 1024 * 1024 // bytes
)

// contextKey is a unique (and compariable) type we use
// to store and retrieve additional information in the
// go-kit context.
type contextKey int

const (
	accessControlAllowOrigin contextKey = iota
	accessControlAllowMethods
	accessControlAllowHeaders
	accessControlAllowCredentials
)

// saveCORSHeadersIntoContext saves CORS headers into the go-kit context.
//
// This is designed to be added as a ServerOption in our main http handler.
func saveCORSHeadersIntoContext() httptransport.RequestFunc {
	return func(ctx context.Context, r *http.Request) context.Context {
		if v := r.Header.Get("Access-Control-Allow-Origin"); v != "" {
			ctx = context.WithValue(ctx, accessControlAllowOrigin, v)

			v = r.Header.Get("Access-Control-Allow-Methods")
			ctx = context.WithValue(ctx, accessControlAllowMethods, v)

			v = r.Header.Get("Access-Control-Allow-Headers")
			ctx = context.WithValue(ctx, accessControlAllowHeaders, v)

			v = r.Header.Get("Access-Control-Allow-Credentials")
			ctx = context.WithValue(ctx, accessControlAllowCredentials, v)
		}
		return ctx
	}
}

// respondWithSavedCORSHeaders looks in the go-kit request context
// for our own CORS headers. (Stored with our context key in
// saveCORSHeadersIntoContext.)
//
// This is designed to be added as a ServerOption in our main http handler.
func respondWithSavedCORSHeaders() httptransport.ServerResponseFunc {
	return func(ctx context.Context, w http.ResponseWriter) context.Context {
		if v, ok := ctx.Value(accessControlAllowOrigin).(string); ok && v != "" {
			w.Header().Set("Access-Control-Allow-Origin", v)
		}
		if v, ok := ctx.Value(accessControlAllowMethods).(string); ok && v != "" {
			w.Header().Set("Access-Control-Allow-Methods", v)
		}
		if v, ok := ctx.Value(accessControlAllowHeaders).(string); ok && v != "" {
			w.Header().Set("Access-Control-Allow-Headers", v)
		}
		if v, ok := ctx.Value(accessControlAllowCredentials).(string); ok && v != "" {
			w.Header().Set("Access-Control-Allow-Credentials", v)
		}
		return ctx
	}
}

// preflightHandler captures Corss Origin Resource Sharing (CORS) requests
// by looking at all OPTIONS requests for the Origin header, parsing that
// and responding back with the other Access-Control-Allow-* headers.
//
// Docs: https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS
func preflightHandler(options []httptransport.ServerOption) http.Handler {
	return httptransport.NewServer(
		endpoint.Nop,
		httptransport.NopRequestDecoder,
		func(_ context.Context, _ http.ResponseWriter, _ interface{}) error {
			return nil
		},
		options...,
	)
}

func MakeHTTPHandler(s Service, repo Repository, logger log.Logger) http.Handler {
	r := mux.NewRouter()
	e := MakeServerEndpoints(s, repo)
	options := []httptransport.ServerOption{
		httptransport.ServerErrorLogger(logger),
		httptransport.ServerErrorEncoder(encodeError),
		httptransport.ServerBefore(saveCORSHeadersIntoContext()),
		httptransport.ServerAfter(respondWithSavedCORSHeaders()),
	}

	// HTTP Methods
	r.Methods("OPTIONS").Handler(preflightHandler(options)) // CORS pre-flight handler
	r.Methods("GET").Path("/ping").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte("PONG"))
	})
	r.Methods("GET").Path("/files").Handler(httptransport.NewServer(
		e.GetFilesEndpoint,
		decodeGetFilesRequest,
		encodeResponse,
		options...,
	))
	r.Methods("POST").Path("/files/create").Handler(httptransport.NewServer(
		e.CreateFileEndpoint,
		decodeCreateFileRequest,
		encodeResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{id}").Handler(httptransport.NewServer(
		e.GetFileEndpoint,
		decodeGetFileRequest,
		encodeResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{id}/contents").Handler(httptransport.NewServer(
		e.GetFileContentsEndpoint,
		decodeGetFileContentsRequest,
		encodeTextResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{id}/validate").Handler(httptransport.NewServer(
		e.ValidateFileEndpoint,
		decodeValidateFileRequest,
		encodeResponse,
		options...,
	))
	r.Methods("DELETE").Path("/files/{id}").Handler(httptransport.NewServer(
		e.DeleteFileEndpoint,
		decodeDeleteFileRequest,
		encodeResponse,
		options...,
	))
	r.Methods("POST").Path("/files/{fileID}/batches/").Handler(httptransport.NewServer(
		e.CreateBatchEndpoint,
		decodeCreateBatchRequest,
		encodeResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{fileID}/batches/").Handler(httptransport.NewServer(
		e.GetBatchesEndpoint,
		decodeGetBatchesRequest,
		encodeResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{fileID}/batches/{batchID}").Handler(httptransport.NewServer(
		e.GetBatchEndpoint,
		decodeGetBatchRequest,
		encodeResponse,
		options...,
	))
	r.Methods("DELETE").Path("/files/{fileID}/batches/{batchID}").Handler(httptransport.NewServer(
		e.DeleteBatchEndpoint,
		decodeDeleteBatchRequest,
		encodeResponse,
		options...,
	))
	return r
}

//** FILES ** //
func decodeCreateFileRequest(_ context.Context, request *http.Request) (interface{}, error) {
	// Make sure content-length is small enough
	if !acceptableContentLength(request.Header) {
		return nil, errors.New("request body is too large")
	}

	var r io.Reader
	var req createFileRequest

	// Sets default values
	req.File = &ach.File{
		Header: ach.NewFileHeader(),
	}

	bs, err := ioutil.ReadAll(request.Body)
	if err != nil {
		return nil, err
	}

	h := request.Header.Get("Content-Type")
	if strings.Contains(h, "application/json") {
		// Read body as ACH file in JSON
		f, err := ach.FileFromJson(bs)
		if err != nil {
			return nil, err
		}
		req.File = f
	} else {
		// Attempt parsing body as an ACH File
		r = bytes.NewReader(bs)
		f, err := ach.NewReader(r).Read()
		if err != nil {
			return nil, err
		}
		req.File = &f
	}
	return req, nil
}

func decodeGetFileRequest(_ context.Context, r *http.Request) (interface{}, error) {
	vars := mux.Vars(r)
	id, ok := vars["id"]
	if !ok {
		return nil, ErrBadRouting
	}
	return getFileRequest{ID: id}, nil
}

func decodeDeleteFileRequest(_ context.Context, r *http.Request) (interface{}, error) {
	vars := mux.Vars(r)
	id, ok := vars["id"]
	if !ok {
		return nil, ErrBadRouting
	}
	return deleteFileRequest{ID: id}, nil
}

func decodeGetFilesRequest(_ context.Context, r *http.Request) (interface{}, error) {
	return getFilesRequest{}, nil
}

func decodeGetFileContentsRequest(_ context.Context, r *http.Request) (interface{}, error) {
	vars := mux.Vars(r)
	id, ok := vars["id"]
	if !ok {
		return nil, ErrBadRouting
	}
	return getFileContentsRequest{ID: id}, nil
}

func decodeValidateFileRequest(_ context.Context, r *http.Request) (interface{}, error) {
	vars := mux.Vars(r)
	id, ok := vars["id"]
	if !ok {
		return nil, ErrBadRouting
	}
	return validateFileRequest{ID: id}, nil
}

//** BATCHES **//

func decodeCreateBatchRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var req createBatchRequest
	vars := mux.Vars(r)
	id, ok := vars["fileID"]
	if !ok {
		return nil, ErrBadRouting
	}
	req.FileID = id
	req.BatchHeader = *ach.NewBatchHeader()
	if e := json.NewDecoder(r.Body).Decode(&req.BatchHeader); e != nil {
		return nil, e
	}
	return req, nil
}

func decodeGetBatchesRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var req getBatchesRequest
	vars := mux.Vars(r)
	id, ok := vars["fileID"]
	if !ok {
		return nil, ErrBadRouting
	}
	req.fileID = id
	return req, nil
}

func decodeGetBatchRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var req getBatchRequest
	vars := mux.Vars(r)
	fileID, ok := vars["fileID"]
	if !ok {
		return nil, ErrBadRouting
	}
	batchID, ok := vars["batchID"]
	if !ok {
		return nil, ErrBadRouting
	}

	req.fileID = fileID
	req.batchID = batchID
	return req, nil
}

func decodeDeleteBatchRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var req deleteBatchRequest
	vars := mux.Vars(r)
	fileID, ok := vars["fileID"]
	if !ok {
		return nil, ErrBadRouting
	}
	batchID, ok := vars["batchID"]
	if !ok {
		return nil, ErrBadRouting
	}

	req.fileID = fileID
	req.batchID = batchID
	return req, nil
}

// errorer is implemented by all concrete response types that may contain
// errors. There are a few well-known values which are used to change the
// HTTP response code without needing to trigger an endpoint (transport-level)
// error.
type errorer interface {
	error() error
}

// counter is implemented by any concrete response types that may contain
// some arbitrary count information.
type counter interface {
	count() int
}

// encodeResponse is the common method to encode all response types to the
// client. I chose to do it this way because, since we're using JSON, there's no
// reason to provide anything more specific. It's certainly possible to
// specialize on a per-response (per-method) basis.
func encodeResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
	if e, ok := response.(errorer); ok && e.error() != nil {
		// Not a Go kit transport error, but a business-logic error.
		// Provide those as HTTP errors.
		encodeError(ctx, e.error(), w)
		return nil
	}

	// Used for pagination
	if e, ok := response.(counter); ok {
		w.Header().Set("X-Total-Count", strconv.Itoa(e.count()))
	}

	// Don't overwrite a header (i.e. called from encodeTextResponse)
	if v := w.Header().Get("Content-Type"); v == "" {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		// Only write json body if we're setting response as json
		return json.NewEncoder(w).Encode(response)
	}
	return nil
}

// encodeTextResponse will marshal response into the HTTP Response
// This method is designed text/plain content-types and expects response
// to be an io.Reader.
func encodeTextResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
	if r, ok := response.(io.Reader); ok {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusOK)
		_, err := io.Copy(w, r)
		return err
	}
	return nil
}

// encodeError JSON encodes the supplied error
func encodeError(_ context.Context, err error, w http.ResponseWriter) {
	if err == nil {
		err = ErrFoundABug
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(codeFrom(err))
	json.NewEncoder(w).Encode(map[string]interface{}{
		"error": err.Error(),
	})
}

func codeFrom(err error) int {
	if err == nil {
		return http.StatusOK
	}

	switch err {
	case ErrNotFound:
		return http.StatusNotFound
	case ErrAlreadyExists:
		return http.StatusBadRequest
	}
	// TODO(adam): this should really probably be a 4xx error
	// TODO(adam): on GET /files/:id/validate a "bad" file returns 500
	return http.StatusInternalServerError
}

func acceptableContentLength(headers http.Header) bool {
	h := headers.Get("Content-Length")
	if v, err := strconv.Atoi(h); err == nil {
		return v <= MaxContentLength
	}
	return false
}
<commit_msg>server: don't expect trailing slash on endpoints<commit_after>package server

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"

	"github.com/moov-io/ach"

	"github.com/go-kit/kit/endpoint"
	"github.com/go-kit/kit/log"
	httptransport "github.com/go-kit/kit/transport/http"
	"github.com/gorilla/mux"
)

var (
	bugReportHelp = "please report this as a bug -- https://github.com/moov-io/ach/issues/new"

	// ErrBadRouting is returned when an expected path variable is missing, which is always programmer error.
	ErrBadRouting = fmt.Errorf("inconsistent mapping between route and handler, %s", bugReportHelp)
	ErrFoundABug  = fmt.Errorf("Snuck into encodeError with err == nil, %s", bugReportHelp)

	MaxContentLength = 1 * 1024 * 1024 // bytes
)

// contextKey is a unique (and compariable) type we use
// to store and retrieve additional information in the
// go-kit context.
type contextKey int

const (
	accessControlAllowOrigin contextKey = iota
	accessControlAllowMethods
	accessControlAllowHeaders
	accessControlAllowCredentials
)

// saveCORSHeadersIntoContext saves CORS headers into the go-kit context.
//
// This is designed to be added as a ServerOption in our main http handler.
func saveCORSHeadersIntoContext() httptransport.RequestFunc {
	return func(ctx context.Context, r *http.Request) context.Context {
		if v := r.Header.Get("Access-Control-Allow-Origin"); v != "" {
			ctx = context.WithValue(ctx, accessControlAllowOrigin, v)

			v = r.Header.Get("Access-Control-Allow-Methods")
			ctx = context.WithValue(ctx, accessControlAllowMethods, v)

			v = r.Header.Get("Access-Control-Allow-Headers")
			ctx = context.WithValue(ctx, accessControlAllowHeaders, v)

			v = r.Header.Get("Access-Control-Allow-Credentials")
			ctx = context.WithValue(ctx, accessControlAllowCredentials, v)
		}
		return ctx
	}
}

// respondWithSavedCORSHeaders looks in the go-kit request context
// for our own CORS headers. (Stored with our context key in
// saveCORSHeadersIntoContext.)
//
// This is designed to be added as a ServerOption in our main http handler.
func respondWithSavedCORSHeaders() httptransport.ServerResponseFunc {
	return func(ctx context.Context, w http.ResponseWriter) context.Context {
		if v, ok := ctx.Value(accessControlAllowOrigin).(string); ok && v != "" {
			w.Header().Set("Access-Control-Allow-Origin", v)
		}
		if v, ok := ctx.Value(accessControlAllowMethods).(string); ok && v != "" {
			w.Header().Set("Access-Control-Allow-Methods", v)
		}
		if v, ok := ctx.Value(accessControlAllowHeaders).(string); ok && v != "" {
			w.Header().Set("Access-Control-Allow-Headers", v)
		}
		if v, ok := ctx.Value(accessControlAllowCredentials).(string); ok && v != "" {
			w.Header().Set("Access-Control-Allow-Credentials", v)
		}
		return ctx
	}
}

// preflightHandler captures Corss Origin Resource Sharing (CORS) requests
// by looking at all OPTIONS requests for the Origin header, parsing that
// and responding back with the other Access-Control-Allow-* headers.
//
// Docs: https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS
func preflightHandler(options []httptransport.ServerOption) http.Handler {
	return httptransport.NewServer(
		endpoint.Nop,
		httptransport.NopRequestDecoder,
		func(_ context.Context, _ http.ResponseWriter, _ interface{}) error {
			return nil
		},
		options...,
	)
}

func MakeHTTPHandler(s Service, repo Repository, logger log.Logger) http.Handler {
	r := mux.NewRouter()
	e := MakeServerEndpoints(s, repo)
	options := []httptransport.ServerOption{
		httptransport.ServerErrorLogger(logger),
		httptransport.ServerErrorEncoder(encodeError),
		httptransport.ServerBefore(saveCORSHeadersIntoContext()),
		httptransport.ServerAfter(respondWithSavedCORSHeaders()),
	}

	// HTTP Methods
	r.Methods("OPTIONS").Handler(preflightHandler(options)) // CORS pre-flight handler
	r.Methods("GET").Path("/ping").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		w.Write([]byte("PONG"))
	})
	r.Methods("GET").Path("/files").Handler(httptransport.NewServer(
		e.GetFilesEndpoint,
		decodeGetFilesRequest,
		encodeResponse,
		options...,
	))
	r.Methods("POST").Path("/files/create").Handler(httptransport.NewServer(
		e.CreateFileEndpoint,
		decodeCreateFileRequest,
		encodeResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{id}").Handler(httptransport.NewServer(
		e.GetFileEndpoint,
		decodeGetFileRequest,
		encodeResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{id}/contents").Handler(httptransport.NewServer(
		e.GetFileContentsEndpoint,
		decodeGetFileContentsRequest,
		encodeTextResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{id}/validate").Handler(httptransport.NewServer(
		e.ValidateFileEndpoint,
		decodeValidateFileRequest,
		encodeResponse,
		options...,
	))
	r.Methods("DELETE").Path("/files/{id}").Handler(httptransport.NewServer(
		e.DeleteFileEndpoint,
		decodeDeleteFileRequest,
		encodeResponse,
		options...,
	))
	r.Methods("POST").Path("/files/{fileID}/batches").Handler(httptransport.NewServer(
		e.CreateBatchEndpoint,
		decodeCreateBatchRequest,
		encodeResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{fileID}/batches").Handler(httptransport.NewServer(
		e.GetBatchesEndpoint,
		decodeGetBatchesRequest,
		encodeResponse,
		options...,
	))
	r.Methods("GET").Path("/files/{fileID}/batches/{batchID}").Handler(httptransport.NewServer(
		e.GetBatchEndpoint,
		decodeGetBatchRequest,
		encodeResponse,
		options...,
	))
	r.Methods("DELETE").Path("/files/{fileID}/batches/{batchID}").Handler(httptransport.NewServer(
		e.DeleteBatchEndpoint,
		decodeDeleteBatchRequest,
		encodeResponse,
		options...,
	))
	return r
}

//** FILES ** //
func decodeCreateFileRequest(_ context.Context, request *http.Request) (interface{}, error) {
	// Make sure content-length is small enough
	if !acceptableContentLength(request.Header) {
		return nil, errors.New("request body is too large")
	}

	var r io.Reader
	var req createFileRequest

	// Sets default values
	req.File = &ach.File{
		Header: ach.NewFileHeader(),
	}

	bs, err := ioutil.ReadAll(request.Body)
	if err != nil {
		return nil, err
	}

	h := request.Header.Get("Content-Type")
	if strings.Contains(h, "application/json") {
		// Read body as ACH file in JSON
		f, err := ach.FileFromJson(bs)
		if err != nil {
			return nil, err
		}
		req.File = f
	} else {
		// Attempt parsing body as an ACH File
		r = bytes.NewReader(bs)
		f, err := ach.NewReader(r).Read()
		if err != nil {
			return nil, err
		}
		req.File = &f
	}
	return req, nil
}

func decodeGetFileRequest(_ context.Context, r *http.Request) (interface{}, error) {
	vars := mux.Vars(r)
	id, ok := vars["id"]
	if !ok {
		return nil, ErrBadRouting
	}
	return getFileRequest{ID: id}, nil
}

func decodeDeleteFileRequest(_ context.Context, r *http.Request) (interface{}, error) {
	vars := mux.Vars(r)
	id, ok := vars["id"]
	if !ok {
		return nil, ErrBadRouting
	}
	return deleteFileRequest{ID: id}, nil
}

func decodeGetFilesRequest(_ context.Context, r *http.Request) (interface{}, error) {
	return getFilesRequest{}, nil
}

func decodeGetFileContentsRequest(_ context.Context, r *http.Request) (interface{}, error) {
	vars := mux.Vars(r)
	id, ok := vars["id"]
	if !ok {
		return nil, ErrBadRouting
	}
	return getFileContentsRequest{ID: id}, nil
}

func decodeValidateFileRequest(_ context.Context, r *http.Request) (interface{}, error) {
	vars := mux.Vars(r)
	id, ok := vars["id"]
	if !ok {
		return nil, ErrBadRouting
	}
	return validateFileRequest{ID: id}, nil
}

//** BATCHES **//

func decodeCreateBatchRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var req createBatchRequest
	vars := mux.Vars(r)
	id, ok := vars["fileID"]
	if !ok {
		return nil, ErrBadRouting
	}
	req.FileID = id
	req.BatchHeader = *ach.NewBatchHeader()
	if e := json.NewDecoder(r.Body).Decode(&req.BatchHeader); e != nil {
		return nil, e
	}
	return req, nil
}

func decodeGetBatchesRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var req getBatchesRequest
	vars := mux.Vars(r)
	id, ok := vars["fileID"]
	if !ok {
		return nil, ErrBadRouting
	}
	req.fileID = id
	return req, nil
}

func decodeGetBatchRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var req getBatchRequest
	vars := mux.Vars(r)
	fileID, ok := vars["fileID"]
	if !ok {
		return nil, ErrBadRouting
	}
	batchID, ok := vars["batchID"]
	if !ok {
		return nil, ErrBadRouting
	}

	req.fileID = fileID
	req.batchID = batchID
	return req, nil
}

func decodeDeleteBatchRequest(_ context.Context, r *http.Request) (interface{}, error) {
	var req deleteBatchRequest
	vars := mux.Vars(r)
	fileID, ok := vars["fileID"]
	if !ok {
		return nil, ErrBadRouting
	}
	batchID, ok := vars["batchID"]
	if !ok {
		return nil, ErrBadRouting
	}

	req.fileID = fileID
	req.batchID = batchID
	return req, nil
}

// errorer is implemented by all concrete response types that may contain
// errors. There are a few well-known values which are used to change the
// HTTP response code without needing to trigger an endpoint (transport-level)
// error.
type errorer interface {
	error() error
}

// counter is implemented by any concrete response types that may contain
// some arbitrary count information.
type counter interface {
	count() int
}

// encodeResponse is the common method to encode all response types to the
// client. I chose to do it this way because, since we're using JSON, there's no
// reason to provide anything more specific. It's certainly possible to
// specialize on a per-response (per-method) basis.
func encodeResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
	if e, ok := response.(errorer); ok && e.error() != nil {
		// Not a Go kit transport error, but a business-logic error.
		// Provide those as HTTP errors.
		encodeError(ctx, e.error(), w)
		return nil
	}

	// Used for pagination
	if e, ok := response.(counter); ok {
		w.Header().Set("X-Total-Count", strconv.Itoa(e.count()))
	}

	// Don't overwrite a header (i.e. called from encodeTextResponse)
	if v := w.Header().Get("Content-Type"); v == "" {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		// Only write json body if we're setting response as json
		return json.NewEncoder(w).Encode(response)
	}
	return nil
}

// encodeTextResponse will marshal response into the HTTP Response
// This method is designed text/plain content-types and expects response
// to be an io.Reader.
func encodeTextResponse(ctx context.Context, w http.ResponseWriter, response interface{}) error {
	if r, ok := response.(io.Reader); ok {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusOK)
		_, err := io.Copy(w, r)
		return err
	}
	return nil
}

// encodeError JSON encodes the supplied error
func encodeError(_ context.Context, err error, w http.ResponseWriter) {
	if err == nil {
		err = ErrFoundABug
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteHeader(codeFrom(err))
	json.NewEncoder(w).Encode(map[string]interface{}{
		"error": err.Error(),
	})
}

func codeFrom(err error) int {
	if err == nil {
		return http.StatusOK
	}

	switch err {
	case ErrNotFound:
		return http.StatusNotFound
	case ErrAlreadyExists:
		return http.StatusBadRequest
	}
	// TODO(adam): this should really probably be a 4xx error
	// TODO(adam): on GET /files/:id/validate a "bad" file returns 500
	return http.StatusInternalServerError
}

func acceptableContentLength(headers http.Header) bool {
	h := headers.Get("Content-Length")
	if v, err := strconv.Atoi(h); err == nil {
		return v <= MaxContentLength
	}
	return false
}
<|endoftext|>