<commit_before>package rest

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

var (
	// ErrJsonPayloadEmpty is returned when the JSON payload is empty.
	ErrJsonPayloadEmpty = errors.New("JSON payload is empty")
)

// Request inherits from http.Request, and provides additional methods.
type Request struct {
	*http.Request

	// Map of parameters that have been matched in the URL Path.
	PathParams map[string]string

	// Environment used by middlewares to communicate.
	Env map[string]interface{}
}

// PathParam provides a convenient access to the PathParams map.
func (r *Request) PathParam(name string) string {
	return r.PathParams[name]
}

// DecodeJsonPayload reads the request body and decodes the JSON using json.Unmarshal.
func (r *Request) DecodeJsonPayload(v interface{}) error {
	content, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		return err
	}
	if len(content) == 0 {
		return ErrJsonPayloadEmpty
	}
	err = json.Unmarshal(content, v)
	if err != nil {
		return err
	}
	return nil
}

// BaseUrl returns a new URL object with the Host and Scheme taken from the request.
// (without the trailing slash in the host)
func (r *Request) BaseUrl() *url.URL {
	scheme := r.URL.Scheme
	if scheme == "" {
		scheme = "http"
	}

	// HTTP/2.0 gives the default scheme as HTTP even when used with TLS
	// Check if version 2.0 and TLS is not nil and given back https scheme
	if scheme == "http" && r.ProtoMajor >= 2 && r.TLS != nil {
		scheme = "https"
	}

	host := r.Host
	if len(host) > 0 && host[len(host)-1] == '/' {
		host = host[:len(host)-1]
	}

	return &url.URL{
		Scheme: scheme,
		Host:   host,
	}
}

// UrlFor returns the URL object from UriBase with the Path set to path, and the query
// string built with queryParams.
func (r *Request) UrlFor(path string, queryParams map[string][]string) *url.URL {
	baseUrl := r.BaseUrl()
	baseUrl.Path = path
	if queryParams != nil {
		query := url.Values{}
		for k, v := range queryParams {
			for _, vv := range v {
				query.Add(k, vv)
			}
		}
		baseUrl.RawQuery = query.Encode()
	}
	return baseUrl
}

// CorsInfo contains the CORS request info derived from a rest.Request.
type CorsInfo struct {
	IsCors      bool
	IsPreflight bool
	Origin      string
	OriginUrl   *url.URL

	// The header value is converted to uppercase to avoid common mistakes.
	AccessControlRequestMethod string

	// The header values are normalized with http.CanonicalHeaderKey.
	AccessControlRequestHeaders []string
}

// GetCorsInfo derives CorsInfo from Request.
func (r *Request) GetCorsInfo() *CorsInfo {

	origin := r.Header.Get("Origin")

	var originUrl *url.URL
	var isCors bool

	if origin == "" {
		isCors = false
	} else if origin == "null" {
		isCors = true
	} else {
		var err error
		originUrl, err = url.ParseRequestURI(origin)
		isCors = err == nil && r.Host != originUrl.Host
	}

	reqMethod := r.Header.Get("Access-Control-Request-Method")

	reqHeaders := []string{}
	rawReqHeaders := r.Header[http.CanonicalHeaderKey("Access-Control-Request-Headers")]
	for _, rawReqHeader := range rawReqHeaders {
		if len(rawReqHeader) == 0 {
			continue
		}
		// net/http does not handle comma delimited headers for us
		for _, reqHeader := range strings.Split(rawReqHeader, ",") {
			reqHeaders = append(reqHeaders, http.CanonicalHeaderKey(strings.TrimSpace(reqHeader)))
		}
	}

	isPreflight := isCors && r.Method == "OPTIONS" && reqMethod != ""

	return &CorsInfo{
		IsCors:                      isCors,
		IsPreflight:                 isPreflight,
		Origin:                      origin,
		OriginUrl:                   originUrl,
		AccessControlRequestMethod:  strings.ToUpper(reqMethod),
		AccessControlRequestHeaders: reqHeaders,
	}
}
<commit_msg>Remove the check for HTTP2<commit_after>package rest

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

var (
	// ErrJsonPayloadEmpty is returned when the JSON payload is empty.
	ErrJsonPayloadEmpty = errors.New("JSON payload is empty")
)

// Request inherits from http.Request, and provides additional methods.
type Request struct {
	*http.Request

	// Map of parameters that have been matched in the URL Path.
	PathParams map[string]string

	// Environment used by middlewares to communicate.
	Env map[string]interface{}
}

// PathParam provides a convenient access to the PathParams map.
func (r *Request) PathParam(name string) string {
	return r.PathParams[name]
}

// DecodeJsonPayload reads the request body and decodes the JSON using json.Unmarshal.
func (r *Request) DecodeJsonPayload(v interface{}) error {
	content, err := ioutil.ReadAll(r.Body)
	r.Body.Close()
	if err != nil {
		return err
	}
	if len(content) == 0 {
		return ErrJsonPayloadEmpty
	}
	err = json.Unmarshal(content, v)
	if err != nil {
		return err
	}
	return nil
}

// BaseUrl returns a new URL object with the Host and Scheme taken from the request.
// (without the trailing slash in the host)
func (r *Request) BaseUrl() *url.URL {
	scheme := r.URL.Scheme
	if scheme == "" {
		scheme = "http"
	}

	// HTTP sometimes gives the default scheme as HTTP even when used with TLS
	// Check if TLS is not nil and given back https scheme
	if scheme == "http" && r.TLS != nil {
		scheme = "https"
	}

	host := r.Host
	if len(host) > 0 && host[len(host)-1] == '/' {
		host = host[:len(host)-1]
	}

	return &url.URL{
		Scheme: scheme,
		Host:   host,
	}
}

// UrlFor returns the URL object from UriBase with the Path set to path, and the query
// string built with queryParams.
func (r *Request) UrlFor(path string, queryParams map[string][]string) *url.URL {
	baseUrl := r.BaseUrl()
	baseUrl.Path = path
	if queryParams != nil {
		query := url.Values{}
		for k, v := range queryParams {
			for _, vv := range v {
				query.Add(k, vv)
			}
		}
		baseUrl.RawQuery = query.Encode()
	}
	return baseUrl
}

// CorsInfo contains the CORS request info derived from a rest.Request.
type CorsInfo struct {
	IsCors      bool
	IsPreflight bool
	Origin      string
	OriginUrl   *url.URL

	// The header value is converted to uppercase to avoid common mistakes.
	AccessControlRequestMethod string

	// The header values are normalized with http.CanonicalHeaderKey.
	AccessControlRequestHeaders []string
}

// GetCorsInfo derives CorsInfo from Request.
func (r *Request) GetCorsInfo() *CorsInfo {

	origin := r.Header.Get("Origin")

	var originUrl *url.URL
	var isCors bool

	if origin == "" {
		isCors = false
	} else if origin == "null" {
		isCors = true
	} else {
		var err error
		originUrl, err = url.ParseRequestURI(origin)
		isCors = err == nil && r.Host != originUrl.Host
	}

	reqMethod := r.Header.Get("Access-Control-Request-Method")

	reqHeaders := []string{}
	rawReqHeaders := r.Header[http.CanonicalHeaderKey("Access-Control-Request-Headers")]
	for _, rawReqHeader := range rawReqHeaders {
		if len(rawReqHeader) == 0 {
			continue
		}
		// net/http does not handle comma delimited headers for us
		for _, reqHeader := range strings.Split(rawReqHeader, ",") {
			reqHeaders = append(reqHeaders, http.CanonicalHeaderKey(strings.TrimSpace(reqHeader)))
		}
	}

	isPreflight := isCors && r.Method == "OPTIONS" && reqMethod != ""

	return &CorsInfo{
		IsCors:                      isCors,
		IsPreflight:                 isPreflight,
		Origin:                      origin,
		OriginUrl:                   originUrl,
		AccessControlRequestMethod:  strings.ToUpper(reqMethod),
		AccessControlRequestHeaders: reqHeaders,
	}
}
<|endoftext|>


<commit_before>//
// Rudimentary package for examining DEX files. See:
//
//   https://source.android.com/devices/tech/dalvik/dex-format.html
//
// for a specification of the DEX file format.
//
// This package focuses on the classes and methods in a DEX file; you
// pass it a visitor object and it will invoke interfaces on the
// visitor for each DEX class and DEX method in the DEX file of
// interest.
//
package dexread

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/thanm/go-read-a-dex/dexapkvisit"
)

type dexState struct {
	apk        *string
	dexName    string
	b          bytes.Buffer
	rdr        *bytes.Reader
	methodIds  []dexMethodIdItem
	typeIds    []uint32
	strings    []string
	fileHeader dexFileHeader
	visitor    dexapkvisit.DexApkVisitor
}

func mkError(state *dexState, fmtstring string, a ...interface{}) error {
	gripe := fmt.Sprintf(fmtstring, a...)
	apkPre := ""
	if state.apk != nil {
		apkPre = fmt.Sprintf("apk %s ", state.apk)
	}
	msg := fmt.Sprintf("reading %sdex %s: %s", apkPre, state.dexName, gripe)
	return errors.New(msg)
}

// Examine the contents of the DEX file 'dexFilePath', invoking callbacks
// within the visitor object 'visitor.
func ReadDEXFile(dexFilePath string, visitor dexapkvisit.DexApkVisitor) error {
	state := dexState{dexName: dexFilePath, visitor: visitor}
	fi, err := os.Stat(dexFilePath)
	if err != nil {
		return mkError(&state, "os.Stat failed(): %v", err)
	}
	dfile, err := os.Open(dexFilePath)
	if err != nil {
		return mkError(&state, "os.Open() failed(): %v", err)
	}
	defer dfile.Close()
	return ReadDEX(nil, dexFilePath, dfile, uint64(fi.Size()), visitor)
}

// Examine the contents of the DEX file that that is pointed to by the
// reader 'reader'. In the case that the DEX file is embedded within an
// APK file, 'apk' will point to the APK name (for error reporting
// purposes); if 'apk' is nil the assumption is that we're looking at
// a stand-alone DEX file.
func ReadDEX(apk *string, dexName string, reader io.Reader, expectedSize uint64, visitor dexapkvisit.DexApkVisitor) error {
	state := dexState{apk: apk, dexName: dexName, visitor: visitor}

	// NB: the following seems clunky/inelegant (reading in entire
	// contents of DEX and then creating a new bytes.Reader to muck
	// around within it).  Is there a more elegant or efficient way to
	// do this?  Maybe io.SectionReader?

	// Read in the whole enchilada
	nread, err := io.Copy(&state.b, reader)
	if err != nil {
		return mkError(&state, "reading dex data: %v", err)
	}
	if uint64(nread) != expectedSize {
		return mkError(&state, "expected %d bytes read %d", expectedSize, nread)
	}
	state.rdr = bytes.NewReader(state.b.Bytes())

	// Unpack file header and verify magic string
	state.fileHeader, err = unpackDexFileHeader(&state)
	if err != nil {
		return err
	}

	// Invoke visitor callback
	visitor.VisitDEX(dexName, state.fileHeader.Sha1Sig)

	// Read method ids
	state.methodIds, err = unpackMethodIds(&state)
	if err != nil {
		return err
	}

	// Read type ids
	state.typeIds, err = unpackTypeIds(&state)
	if err != nil {
		return err
	}

	// Read strings
	state.strings, err = unpackStringIds(&state)
	if err != nil {
		return err
	}

	// Dive into each class
	numClasses := state.fileHeader.ClassDefsSize
	off := state.fileHeader.ClassDefsOff
	for cl := uint32(0); cl < numClasses; cl++ {
		classHeader, err := unpackDexClass(&state, off)
		if err != nil {
			return err
		}
		visitor.Verbose(1, "class %d type idx is %d", cl, classHeader.ClassIdx)
		examineClass(&state, &classHeader)
		off += dexClassHeaderSize
	}
	return err
}

func unpackDexFileHeader(state *dexState) (retval dexFileHeader, err error) {

	// NB: do I really need a loop here? it would be nice to
	// compare slices using a single operation -- wondering if
	// there is some more idiomatic way to do this
	headerBytes := state.b.Bytes()
	DexFileMagic := [8]byte{0x64, 0x65, 0x78, 0x0a, 0x30, 0x33, 0x35, 0x00}
	for i := 0; i < 8; i++ {
		if DexFileMagic[i] != headerBytes[i] {
			return retval, mkError(state, "not a DEX file")
		}
	}

	// Populate the header file struct
	err = binary.Read(state.rdr, binary.LittleEndian, &retval)
	if err != nil {
		return retval, mkError(state, "unable to decode DEX header: %v", err)
	}

	return
}

// Can't use io.SeekStart with gccgo (gccgo libgo version doesn't include it)
const ioSeekStart = 0

func seekReader(state *dexState, off uint32) error {
	_, err := state.rdr.Seek(int64(off), ioSeekStart)
	if err != nil {
		return mkError(state, "unable to seek to offset %d: %v", off, err)
	}
	return nil
}

func unpackDexClass(state *dexState, off uint32) (retval dexClassHeader, err error) {
	err = seekReader(state, off)
	if err != nil {
		return
	}
	err = binary.Read(state.rdr, binary.LittleEndian, &retval)
	if err != nil {
		return retval, mkError(state, "unable to unpack class header: %v", err)
	}
	return
}

type ulebHelper struct {
	data []byte
}

func (a *ulebHelper) grabULEB128() uint64 {
	v, size := binary.Uvarint(a.data)
	a.data = a.data[size:]
	return v
}

//
// For the rules on how type descriptors are encoded, see
// https://source.android.com/devices/tech/dalvik/dex-format.html#typedescriptor
//
func decodeDescriptor(d string) string {
	// count array dimensions
	var dims int = 0
	pos := 0
	c := '0'
	for pos, c = range d {
		if c == '[' {
			dims++
		} else {
			break
		}
	}

	var base string
	if c == 'L' {
		// reference
		base = strings.Replace(d[pos+1:], "/", ".", -1)
		base = strings.Replace(base, ";", "", 1)
	} else {
		// primitive
		switch c {
		case 'B':
			base = "byte"
		case 'C':
			base = "char"
		case 'D':
			base = "double"
		case 'F':
			base = "float"
		case 'I':
			base = "int"
		case 'J':
			base = "long"
		case 'S':
			base = "short"
		case 'Z':
			base = "boolean"
		case 'V':
			base = "void"
		default:
			// something went wrong, punt...
			return d
		}
	}

	for i := 0; i < dims; i++ {
		base += "[]"
	}

	return base
}

func getClassName(state *dexState, ci *dexClassHeader) string {
	typeidx := state.typeIds[ci.ClassIdx]
	typename := state.strings[typeidx]
	return decodeDescriptor(typename)
}

func examineClass(state *dexState, ci *dexClassHeader) {

	// No class data? In theory this can happen
	if ci.ClassDataOff == 0 {
		state.visitor.VisitClass(getClassName(state, ci), 0)
		return
	}

	// Create new slice pointing to correct spot in buffer for class data
	content := state.b.Bytes()
	cldata := content[ci.ClassDataOff:]
	helper := ulebHelper{cldata}

	// Read four ULEB128 encoded values into struct
	var clh dexClassContents
	clh.numStaticFields = uint32(helper.grabULEB128())
	clh.numInstanceFields = uint32(helper.grabULEB128())
	clh.numDirectMethods = uint32(helper.grabULEB128())
	clh.numVirtualMethods = uint32(helper.grabULEB128())
	numMethods := clh.numDirectMethods + clh.numVirtualMethods

	// invoke visitor callback
	state.visitor.VisitClass(getClassName(state, ci), numMethods)

	state.visitor.Verbose(1, "num static fields is %d", clh.numStaticFields)
	state.visitor.Verbose(1, "num instance fields is %d", clh.numInstanceFields)
	state.visitor.Verbose(1, "num direct methods is %d", clh.numDirectMethods)
	state.visitor.Verbose(1, "num virtual methods is %d", clh.numVirtualMethods)

	// Not interested in field info, but we have to get by that
	// information to get to the interesting stuff that follows (since
	// it's ULEB, we can't skip over it directly)
	numFields := clh.numStaticFields + clh.numInstanceFields
	for i := uint32(0); i < numFields; i++ {
		helper.grabULEB128() // field_idx
		helper.grabULEB128() // access_flags
	}

	// Examine the methods. Note that method ID value read is a
	// difference from the index of the previous element in the list.
	var methodIdx uint64 = 0
	for i := uint32(0); i < numMethods; i++ {
		methodDelta := helper.grabULEB128()
		if i == 0 || i == clh.numDirectMethods {
			methodIdx = methodDelta
		} else {
			methodIdx = methodIdx + methodDelta
		}
		_ = helper.grabULEB128() // access flags currently unused
		methodCodeOffset := helper.grabULEB128()
		state.visitor.Verbose(1, "method %d idx %d off %d",
			i, methodIdx, methodCodeOffset)

		examineMethod(state, methodIdx, methodCodeOffset)
	}
}

func unpackStringIds(state *dexState) (retval []string, err error) {
	nStringIds := int(state.fileHeader.StringIdsSize)
	stringOffsets := make([]uint32, nStringIds, nStringIds)

	// position the reader at the right spot
	err = seekReader(state, state.fileHeader.StringIdsOff)
	if err != nil {
		return
	}

	// read offsets
	for i := 0; i < nStringIds; i++ {
		err := binary.Read(state.rdr, binary.LittleEndian, &stringOffsets[i])
		if err != nil {
			return []string{}, mkError(state, "string ID %d unpack failed: %v", i, err)
		}
	}

	// now read in string data
	retval = make([]string, nStringIds, nStringIds)
	for i := 0; i < nStringIds; i++ {
		retval[i] = unpackModUTFString(state, stringOffsets[i])
	}
	return retval, err
}

func zLen(sd []byte) int {
	for i := 0; i < len(sd); i++ {
		if sd[i] == 0 {
			return i
		}
	}
	return len(sd)
}

//
// DEX file strings use a somewhat peculiar "Modified" UTF-8 encoding, details
// in https://source.android.com/devices/tech/dalvik/dex-format.html#mutf-8
//
func unpackModUTFString(state *dexState, off uint32) string {
	content := state.b.Bytes()
	sdata := content[off:]
	helper := ulebHelper{sdata}

	// unpack len and then string
	sl := helper.grabULEB128()
	return string(helper.data[:sl])
}

func unpackMethodIds(state *dexState) (retval []dexMethodIdItem, err error) {

	// position the reader at the right spot
	err = seekReader(state, state.fileHeader.MethodIdsOff)
	if err != nil {
		return retval, err
	}

	// read in the array of method id items
	nMethods := int(state.fileHeader.MethodIdsSize)
	retval = make([]dexMethodIdItem, nMethods, nMethods)
	for i := 0; i < nMethods; i++ {
		err = binary.Read(state.rdr, binary.LittleEndian, &retval[i])
		if err != nil {
			return retval, mkError(state, "method ID %d unpack failed: %v", i, err)
		}
	}

	state.visitor.Verbose(1, "read %d methodids", nMethods)

	return retval, err
}

// NB: this function has a lot in common with the one above it-- what
// would be a good way to common them up? Generics or something like
// them would be useful here(?).

func unpackTypeIds(state *dexState) (retval []uint32, err error) {

	// position the reader at the right spot
	err = seekReader(state, state.fileHeader.TypeIdsOff)
	if err != nil {
		return retval, err
	}

	// read in the array of type id items
	nTypeIds := int(state.fileHeader.TypeIdsSize)
	retval = make([]uint32, nTypeIds, nTypeIds)
	for i := 0; i < nTypeIds; i++ {
		err := binary.Read(state.rdr, binary.LittleEndian, &retval[i])
		if err != nil {
			return retval, mkError(state, "type ID %d unpack failed: %v", i, err)
		}
	}

	state.visitor.Verbose(1, "read %d typeids", nTypeIds)

	return retval, err
}

func examineMethod(state *dexState, methodIdx, methodCodeOffset uint64) {

	// Look up method name from method ID
	nameIdx := state.methodIds[methodIdx].NameIdx

	name := state.strings[nameIdx]

	state.visitor.VisitMethod(name, methodIdx, methodCodeOffset)
}
<commit_msg>More compact error handling code.<commit_after>//
// Rudimentary package for examining DEX files. See:
//
//   https://source.android.com/devices/tech/dalvik/dex-format.html
//
// for a specification of the DEX file format.
//
// This package focuses on the classes and methods in a DEX file; you
// pass it a visitor object and it will invoke interfaces on the
// visitor for each DEX class and DEX method in the DEX file of
// interest.
//
package dexread

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/thanm/go-read-a-dex/dexapkvisit"
)

type dexState struct {
	apk        *string
	dexName    string
	b          bytes.Buffer
	rdr        *bytes.Reader
	methodIds  []dexMethodIdItem
	typeIds    []uint32
	strings    []string
	fileHeader dexFileHeader
	visitor    dexapkvisit.DexApkVisitor
}

func mkError(state *dexState, fmtstring string, a ...interface{}) error {
	gripe := fmt.Sprintf(fmtstring, a...)
	apkPre := ""
	if state.apk != nil {
		apkPre = fmt.Sprintf("apk %s ", state.apk)
	}
	msg := fmt.Sprintf("reading %sdex %s: %s", apkPre, state.dexName, gripe)
	return errors.New(msg)
}

// Examine the contents of the DEX file 'dexFilePath', invoking callbacks
// within the visitor object 'visitor.
func ReadDEXFile(dexFilePath string, visitor dexapkvisit.DexApkVisitor) error {
	state := dexState{dexName: dexFilePath, visitor: visitor}
	fi, err := os.Stat(dexFilePath)
	if err != nil {
		return mkError(&state, "os.Stat failed(): %v", err)
	}
	dfile, err := os.Open(dexFilePath)
	if err != nil {
		return mkError(&state, "os.Open() failed(): %v", err)
	}
	defer dfile.Close()
	return ReadDEX(nil, dexFilePath, dfile, uint64(fi.Size()), visitor)
}

// Examine the contents of the DEX file that that is pointed to by the
// reader 'reader'. In the case that the DEX file is embedded within an
// APK file, 'apk' will point to the APK name (for error reporting
// purposes); if 'apk' is nil the assumption is that we're looking at
// a stand-alone DEX file.
func ReadDEX(apk *string, dexName string, reader io.Reader, expectedSize uint64, visitor dexapkvisit.DexApkVisitor) error {
	state := dexState{apk: apk, dexName: dexName, visitor: visitor}

	// NB: the following seems clunky/inelegant (reading in entire
	// contents of DEX and then creating a new bytes.Reader to muck
	// around within it).  Is there a more elegant or efficient way to
	// do this?  Maybe io.SectionReader?

	// Read in the whole enchilada
	var nread int64
	var err error
	if nread, err = io.Copy(&state.b, reader); err != nil {
		return mkError(&state, "reading dex data: %v", err)
	}
	if uint64(nread) != expectedSize {
		return mkError(&state, "expected %d bytes read %d", expectedSize, nread)
	}
	state.rdr = bytes.NewReader(state.b.Bytes())

	// Unpack file header and verify magic string
	if state.fileHeader, err = unpackDexFileHeader(&state); err != nil {
		return err
	}

	// Invoke visitor callback
	visitor.VisitDEX(dexName, state.fileHeader.Sha1Sig)

	// Read method ids
	if state.methodIds, err = unpackMethodIds(&state); err != nil {
		return err
	}

	// Read type ids
	if state.typeIds, err = unpackTypeIds(&state); err != nil {
		return err
	}

	// Read strings
	if state.strings, err = unpackStringIds(&state); err != nil {
		return err
	}

	// Dive into each class
	numClasses := state.fileHeader.ClassDefsSize
	off := state.fileHeader.ClassDefsOff
	for cl := uint32(0); cl < numClasses; cl++ {
		var classHeader dexClassHeader
		if classHeader, err = unpackDexClass(&state, off); err != nil {
			return err
		}
		visitor.Verbose(1, "class %d type idx is %d", cl, classHeader.ClassIdx)
		examineClass(&state, &classHeader)
		off += dexClassHeaderSize
	}
	return err
}

func unpackDexFileHeader(state *dexState) (retval dexFileHeader, err error) {

	// NB: do I really need a loop here? it would be nice to
	// compare slices using a single operation -- wondering if
	// there is some more idiomatic way to do this
	headerBytes := state.b.Bytes()
	DexFileMagic := [8]byte{0x64, 0x65, 0x78, 0x0a, 0x30, 0x33, 0x35, 0x00}
	for i := 0; i < 8; i++ {
		if DexFileMagic[i] != headerBytes[i] {
			return retval, mkError(state, "not a DEX file")
		}
	}

	// Populate the header file struct
	if err = binary.Read(state.rdr, binary.LittleEndian, &retval); err != nil {
		return retval, mkError(state, "unable to decode DEX header: %v", err)
	}

	return
}

// NB: can't use io.SeekStart with gccgo (gccgo has an older version of
// libgo that doesn't include this constant). Is this a common issue?
const ioSeekStart = 0

func seekReader(state *dexState, off uint32) error {
	if _, err := state.rdr.Seek(int64(off), ioSeekStart); err != nil {
		return mkError(state, "unable to seek to offset %d: %v", off, err)
	}
	return nil
}

func unpackDexClass(state *dexState, off uint32) (retval dexClassHeader, err error) {
	if err = seekReader(state, off); err != nil {
		return
	}
	if err = binary.Read(state.rdr, binary.LittleEndian, &retval); err != nil {
		return retval, mkError(state, "unable to unpack class header: %v", err)
	}
	return
}

type ulebHelper struct {
	data []byte
}

func (a *ulebHelper) grabULEB128() uint64 {
	v, size := binary.Uvarint(a.data)
	a.data = a.data[size:]
	return v
}

//
// For the rules on how type descriptors are encoded, see
// https://source.android.com/devices/tech/dalvik/dex-format.html#typedescriptor
//
func decodeDescriptor(d string) string {
	// count array dimensions
	var dims int = 0
	pos := 0
	c := '0'
	for pos, c = range d {
		if c == '[' {
			dims++
		} else {
			break
		}
	}

	var base string
	if c == 'L' {
		// reference: replace "/" with "." and remove trailing ";"
		base = strings.Replace(d[pos+1:], "/", ".", -1)
		base = strings.Replace(base, ";", "", 1)
	} else {
		// primitive
		switch c {
		case 'B':
			base = "byte"
		case 'C':
			base = "char"
		case 'D':
			base = "double"
		case 'F':
			base = "float"
		case 'I':
			base = "int"
		case 'J':
			base = "long"
		case 'S':
			base = "short"
		case 'Z':
			base = "boolean"
		case 'V':
			base = "void"
		default:
			// something went wrong, punt...
			return d
		}
	}

	for i := 0; i < dims; i++ {
		base += "[]"
	}

	return base
}

func getClassName(state *dexState, ci *dexClassHeader) string {
	typeidx := state.typeIds[ci.ClassIdx]
	typename := state.strings[typeidx]
	return decodeDescriptor(typename)
}

func examineClass(state *dexState, ci *dexClassHeader) {

	// No class data? In theory this can happen
	if ci.ClassDataOff == 0 {
		state.visitor.VisitClass(getClassName(state, ci), 0)
		return
	}

	// Create new slice pointing to correct spot in buffer for class data
	content := state.b.Bytes()
	cldata := content[ci.ClassDataOff:]
	helper := ulebHelper{cldata}

	// Read four ULEB128 encoded values into struct
	var clh dexClassContents
	clh.numStaticFields = uint32(helper.grabULEB128())
	clh.numInstanceFields = uint32(helper.grabULEB128())
	clh.numDirectMethods = uint32(helper.grabULEB128())
	clh.numVirtualMethods = uint32(helper.grabULEB128())
	numMethods := clh.numDirectMethods + clh.numVirtualMethods

	// invoke visitor callback
	state.visitor.VisitClass(getClassName(state, ci), numMethods)

	// debugging
	state.visitor.Verbose(1, "num static fields is %d", clh.numStaticFields)
	state.visitor.Verbose(1, "num instance fields is %d", clh.numInstanceFields)
	state.visitor.Verbose(1, "num direct methods is %d", clh.numDirectMethods)
	state.visitor.Verbose(1, "num virtual methods is %d", clh.numVirtualMethods)

	// Not interested in field info, but we have to get by that
	// information to get to the interesting stuff that follows (since
	// it's ULEB, we can't skip over it directly)
	numFields := clh.numStaticFields + clh.numInstanceFields
	for i := uint32(0); i < numFields; i++ {
		helper.grabULEB128() // field_idx
		helper.grabULEB128() // access_flags
	}

	// Examine the methods. Note that method ID value read is a
	// difference from the index of the previous element in the list.
	var methodIdx uint64 = 0
	for i := uint32(0); i < numMethods; i++ {
		methodDelta := helper.grabULEB128()
		if i == 0 || i == clh.numDirectMethods {
			methodIdx = methodDelta
		} else {
			methodIdx = methodIdx + methodDelta
		}
		_ = helper.grabULEB128() // access flags currently unused
		methodCodeOffset := helper.grabULEB128()
		state.visitor.Verbose(1, "method %d idx %d off %d",
			i, methodIdx, methodCodeOffset)

		examineMethod(state, methodIdx, methodCodeOffset)
	}
}

func unpackStringIds(state *dexState) (retval []string, err error) {
	nStringIds := int(state.fileHeader.StringIdsSize)
	stringOffsets := make([]uint32, nStringIds, nStringIds)

	// position the reader at the right spot
	if err = seekReader(state, state.fileHeader.StringIdsOff); err != nil {
		return
	}

	// read offsets
	for i := 0; i < nStringIds; i++ {
		err := binary.Read(state.rdr, binary.LittleEndian, &stringOffsets[i])
		if err != nil {
			return []string{}, mkError(state, "string ID %d unpack failed: %v", i, err)
		}
	}

	// now read in string data
	retval = make([]string, nStringIds, nStringIds)
	for i := 0; i < nStringIds; i++ {
		retval[i] = unpackModUTFString(state, stringOffsets[i])
	}
	return retval, err
}

// NB: this locally scoped function was left over from a previous
// version of the code -- when I got rid of the last call to it,
// I forgot to remove the function itself. Will the compiler
// remove it for me?
func zLen(sd []byte) int {
	for i := 0; i < len(sd); i++ {
		if sd[i] == 0 {
			return i
		}
	}
	return len(sd)
}

//
// DEX file strings use a somewhat peculiar "Modified" UTF-8 encoding, details
// in https://source.android.com/devices/tech/dalvik/dex-format.html#mutf-8
//
func unpackModUTFString(state *dexState, off uint32) string {
	content := state.b.Bytes()
	sdata := content[off:]
	helper := ulebHelper{sdata}

	// unpack len and then string
	sl := helper.grabULEB128()
	return string(helper.data[:sl])
}

func unpackMethodIds(state *dexState) (retval []dexMethodIdItem, err error) {

	// position the reader at the right spot
	if err = seekReader(state, state.fileHeader.MethodIdsOff); err != nil {
		return retval, err
	}

	// read in the array of method id items
	nMethods := int(state.fileHeader.MethodIdsSize)
	retval = make([]dexMethodIdItem, nMethods, nMethods)
	for i := 0; i < nMethods; i++ {
		err = binary.Read(state.rdr, binary.LittleEndian, &retval[i])
		if err != nil {
			return retval, mkError(state, "method ID %d unpack failed: %v", i, err)
		}
	}

	state.visitor.Verbose(1, "read %d methodids", nMethods)

	return retval, err
}

// NB: this function has a lot in common with the one above it-- what
// would be a good way to common them up? Generics or something like
// them would be useful here(?).  Maybe I could do the same thing
// with interfaces?

func unpackTypeIds(state *dexState) (retval []uint32, err error) {

	// position the reader at the right spot
	err = seekReader(state, state.fileHeader.TypeIdsOff)
	if err != nil {
		return retval, err
	}

	// read in the array of type id items
	nTypeIds := int(state.fileHeader.TypeIdsSize)
	retval = make([]uint32, nTypeIds, nTypeIds)
	for i := 0; i < nTypeIds; i++ {
		err := binary.Read(state.rdr, binary.LittleEndian, &retval[i])
		if err != nil {
			return retval, mkError(state, "type ID %d unpack:: %v", i, err)
		}
	}

	state.visitor.Verbose(1, "read %d typeids", nTypeIds)

	return retval, err
}

func examineMethod(state *dexState, methodIdx, methodCodeOffset uint64) {

	// Look up method name from method ID
	nameIdx := state.methodIds[methodIdx].NameIdx

	name := state.strings[nameIdx]

	state.visitor.VisitMethod(name, methodIdx, methodCodeOffset)
}
<|endoftext|>


<commit_before>package dht

import (
	"context"
	"sync"
	"testing"
	"time"

	peer "github.com/libp2p/go-libp2p-peer"
	queue "github.com/libp2p/go-libp2p-peerstore/queue"
)

func init() {
	DialQueueScalingMutePeriod = 0
	DialQueueMaxIdle = 1 * time.Second
}

func TestDialQueueErrorsWithTooManyConsumers(t *testing.T) {
	var calls int
	defer func() {
		if e := recover(); e == nil {
			t.Error("expected a panic, got none")
		} else if calls != 4 {
			t.Errorf("expected a panic on the 4th call to Consume(); got it on call number %d", calls)
		}
	}()

	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})
	dialFn := func(ctx context.Context, p peer.ID) error {
		<-hang
		return nil
	}

	dq := newDialQueue(context.Background(), "test", in, dialFn, 3)
	for ; calls < 3; calls++ {
		dq.Consume()
	}
	calls++
	dq.Consume()
}

func TestDialQueueGrowsOnSlowDials(t *testing.T) {
	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})

	var wg sync.WaitGroup
	wg.Add(19) // we expect 19 workers
	dialFn := func(ctx context.Context, p peer.ID) error {
		wg.Done()
		<-hang
		return nil
	}

	// Enqueue 20 jobs.
	for i := 0; i < 20; i++ {
		in.EnqChan <- peer.ID(i)
	}

	// remove the mute period to grow faster.
	dq := newDialQueue(context.Background(), "test", in, dialFn, 4)

	for i := 0; i < 4; i++ {
		_ = dq.Consume()
		time.Sleep(100 * time.Millisecond)
	}

	doneCh := make(chan struct{})

	// wait in a goroutine in case the test fails and we block.
	go func() {
		defer close(doneCh)
		wg.Wait()
	}()

	select {
	case <-doneCh:
	case <-time.After(2 * time.Second):
		t.Error("expected 19 concurrent dials, got less")
	}
}

func TestDialQueueShrinksWithNoConsumers(t *testing.T) {
	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})

	var wg sync.WaitGroup
	wg.Add(13)
	dialFn := func(ctx context.Context, p peer.ID) error {
		wg.Done()
		<-hang
		return nil
	}

	dq := newDialQueue(context.Background(), "test", in, dialFn, 3)

	// Enqueue 13 jobs, one per worker we'll grow to.
	for i := 0; i < 13; i++ {
		in.EnqChan <- peer.ID(i)
	}

	// acquire 3 consumers, everytime we acquire a consumer, we will grow the pool because no dial job is completed
	// and immediately returnable.
	for i := 0; i < 3; i++ {
		_ = dq.Consume()
		time.Sleep(100 * time.Millisecond)
	}

	waitForWg(t, &wg, 2*time.Second)

	// Release a few dialFn, but not all of them because downscaling happens when workers detect there are no
	// consumers to consume their values. So the other three will be these witnesses.
	for i := 0; i < 10; i++ {
		hang <- struct{}{}
	}

	// allow enough time for signalling and dispatching values to outstanding consumers.
	time.Sleep(500 * time.Millisecond)

	// unblock the other three.
	hang <- struct{}{}
	hang <- struct{}{}
	hang <- struct{}{}

	// we should now only have 6 workers, because all the shrink events will have been honoured.
	wg.Add(6)

	// enqueue more jobs
	for i := 0; i < 20; i++ {
		in.EnqChan <- peer.ID(i)
	}

	// let's check we have 6 workers hanging.
	waitForWg(t, &wg, 2*time.Second)
}

// Inactivity = workers are idle because the DHT query is progressing slow and is producing too few peers to dial.
func TestDialQueueShrinksWithInactivity(t *testing.T) {
	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})

	var wg sync.WaitGroup
	wg.Add(13)
	dialFn := func(ctx context.Context, p peer.ID) error {
		wg.Done()
		<-hang
		return nil
	}

	// Enqueue 13 jobs.
	for i := 0; i < 13; i++ {
		in.EnqChan <- peer.ID(i)
	}

	dq := newDialQueue(context.Background(), "test", in, dialFn, 3)

	// keep up to speed with backlog by releasing the dial function every time we acquire a channel.
	for i := 0; i < 13; i++ {
		ch := dq.Consume()
		hang <- struct{}{}
		<-ch
		time.Sleep(100 * time.Millisecond)
	}

	// wait for MaxIdlePeriod.
	time.Sleep(1500 * time.Millisecond)

	// we should now only have 6 workers, because all the shrink events will have been honoured.
	wg.Add(6)

	// enqueue more jobs
	for i := 0; i < 10; i++ {
		in.EnqChan <- peer.ID(i)
	}

	// let's check we have 6 workers hanging.
	waitForWg(t, &wg, 2*time.Second)
}

func TestDialQueueMutePeriodHonored(t *testing.T) {
	DialQueueScalingMutePeriod = 2 * time.Second

	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})
	var wg sync.WaitGroup
	wg.Add(6)
	dialFn := func(ctx context.Context, p peer.ID) error {
		wg.Done()
		<-hang
		return nil
	}

	// Enqueue a bunch of jobs.
	for i := 0; i < 20; i++ {
		in.EnqChan <- peer.ID(i)
	}

	dq := newDialQueue(context.Background(), "test", in, dialFn, 3)

	// pick up three consumers.
	for i := 0; i < 3; i++ {
		_ = dq.Consume()
		time.Sleep(100 * time.Millisecond)
	}

	time.Sleep(500 * time.Millisecond)

	// we'll only have 6 workers because the grow signals have been ignored.
	waitForWg(t, &wg, 2*time.Second)
}

func waitForWg(t *testing.T, wg *sync.WaitGroup, wait time.Duration) {
	t.Helper()

	done := make(chan struct{})
	go func() {
		defer close(done)
		wg.Wait()
	}()

	select {
	case <-time.After(wait):
		t.Error("timeout while waiting for WaitGroup")
	case <-done:
	}
}
<commit_msg>harden tests.<commit_after>package dht

import (
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	peer "github.com/libp2p/go-libp2p-peer"
	queue "github.com/libp2p/go-libp2p-peerstore/queue"
)

func init() {
	DialQueueScalingMutePeriod = 0
}

func TestDialQueueErrorsWithTooManyConsumers(t *testing.T) {
	var calls int
	defer func() {
		if e := recover(); e == nil {
			t.Error("expected a panic, got none")
		} else if calls != 4 {
			t.Errorf("expected a panic on the 4th call to Consume(); got it on call number %d", calls)
		}
	}()

	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})
	dialFn := func(ctx context.Context, p peer.ID) error {
		<-hang
		return nil
	}

	dq := newDialQueue(context.Background(), "test", in, dialFn, 3)
	for ; calls < 3; calls++ {
		dq.Consume()
	}
	calls++
	dq.Consume()
}

func TestDialQueueGrowsOnSlowDials(t *testing.T) {
	DialQueueMaxIdle = 10 * time.Minute

	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})

	var cnt int32
	dialFn := func(ctx context.Context, p peer.ID) error {
		atomic.AddInt32(&cnt, 1)
		<-hang
		return nil
	}

	// Enqueue 20 jobs.
	for i := 0; i < 20; i++ {
		in.EnqChan <- peer.ID(i)
	}

	// remove the mute period to grow faster.
	dq := newDialQueue(context.Background(), "test", in, dialFn, 4)

	for i := 0; i < 4; i++ {
		_ = dq.Consume()
		time.Sleep(100 * time.Millisecond)
	}

	for i := 0; i < 20; i++ {
		if atomic.LoadInt32(&cnt) > int32(DialQueueMinParallelism) {
			return
		}
		time.Sleep(100 * time.Millisecond)
	}

	t.Errorf("expected 19 concurrent dials, got %d", atomic.LoadInt32(&cnt))

}

func TestDialQueueShrinksWithNoConsumers(t *testing.T) {
	// reduce interference from the other shrink path.
	DialQueueMaxIdle = 10 * time.Minute

	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})

	wg := new(sync.WaitGroup)
	wg.Add(13)
	dialFn := func(ctx context.Context, p peer.ID) error {
		wg.Done()
		<-hang
		return nil
	}

	dq := newDialQueue(context.Background(), "test", in, dialFn, 3)

	defer func() {
		recover()
		fmt.Println(dq.nWorkers)
	}()

	// acquire 3 consumers, everytime we acquire a consumer, we will grow the pool because no dial job is completed
	// and immediately returnable.
	for i := 0; i < 3; i++ {
		_ = dq.Consume()
	}

	// Enqueue 13 jobs, one per worker we'll grow to.
	for i := 0; i < 13; i++ {
		in.EnqChan <- peer.ID(i)
	}

	waitForWg(t, wg, 2*time.Second)

	// Release a few dialFn, but not all of them because downscaling happens when workers detect there are no
	// consumers to consume their values. So the other three will be these witnesses.
	for i := 0; i < 3; i++ {
		hang <- struct{}{}
	}

	// allow enough time for signalling and dispatching values to outstanding consumers.
	time.Sleep(1 * time.Second)

	// unblock the rest.
	for i := 0; i < 10; i++ {
		hang <- struct{}{}
	}

	wg = new(sync.WaitGroup)
	// we should now only have 6 workers, because all the shrink events will have been honoured.
	wg.Add(6)

	// enqueue more jobs.
	for i := 0; i < 6; i++ {
		in.EnqChan <- peer.ID(i)
	}

	// let's check we have 6 workers hanging.
	waitForWg(t, wg, 2*time.Second)
}

// Inactivity = workers are idle because the DHT query is progressing slow and is producing too few peers to dial.
func TestDialQueueShrinksWithWhenIdle(t *testing.T) {
	DialQueueMaxIdle = 1 * time.Second

	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})

	var wg sync.WaitGroup
	wg.Add(13)
	dialFn := func(ctx context.Context, p peer.ID) error {
		wg.Done()
		<-hang
		return nil
	}

	// Enqueue 13 jobs.
	for i := 0; i < 13; i++ {
		in.EnqChan <- peer.ID(i)
	}

	dq := newDialQueue(context.Background(), "test", in, dialFn, 3)

	// keep up to speed with backlog by releasing the dial function every time we acquire a channel.
	for i := 0; i < 13; i++ {
		ch := dq.Consume()
		hang <- struct{}{}
		<-ch
		time.Sleep(100 * time.Millisecond)
	}

	// wait for MaxIdlePeriod.
	time.Sleep(1500 * time.Millisecond)

	// we should now only have 6 workers, because all the shrink events will have been honoured.
	wg.Add(6)

	// enqueue more jobs
	for i := 0; i < 10; i++ {
		in.EnqChan <- peer.ID(i)
	}

	// let's check we have 6 workers hanging.
	waitForWg(t, &wg, 2*time.Second)
}

func TestDialQueueMutePeriodHonored(t *testing.T) {
	DialQueueScalingMutePeriod = 2 * time.Second

	in := queue.NewChanQueue(context.Background(), queue.NewXORDistancePQ("test"))
	hang := make(chan struct{})
	var wg sync.WaitGroup
	wg.Add(6)
	dialFn := func(ctx context.Context, p peer.ID) error {
		wg.Done()
		<-hang
		return nil
	}

	// Enqueue a bunch of jobs.
	for i := 0; i < 20; i++ {
		in.EnqChan <- peer.ID(i)
	}

	dq := newDialQueue(context.Background(), "test", in, dialFn, 3)

	// pick up three consumers.
	for i := 0; i < 3; i++ {
		_ = dq.Consume()
		time.Sleep(100 * time.Millisecond)
	}

	time.Sleep(500 * time.Millisecond)

	// we'll only have 6 workers because the grow signals have been ignored.
	waitForWg(t, &wg, 2*time.Second)
}

func waitForWg(t *testing.T, wg *sync.WaitGroup, wait time.Duration) {
	t.Helper()

	done := make(chan struct{})
	go func() {
		defer close(done)
		wg.Wait()
	}()

	select {
	case <-time.After(wait):
		t.Error("timeout while waiting for WaitGroup")
	case <-done:
	}
}
<|endoftext|>


<commit_before>package dilbert

import (
	"crypto/tls"
	"fmt"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/apex/log"
	"github.com/blacktop/scifgif/database"
	"github.com/iand/microdata"
	"github.com/pkg/errors"
)

var (
	attempt int
	proxies []string
)

// MaxAttempts max number of download attempts
const MaxAttempts = 30

// Comic is the dilbert comic strip meta data
type Comic struct {
	Title      string
	Tags       []string
	ImageURL   string
	Transcript string
}

func init() {
	rand.Seed(time.Now().Unix())
}

func randomAgent() string {
	var userAgents = []string{
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Safari/604.1.38",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Safari/604.1.38",
	}
	return userAgents[rand.Int()%len(userAgents)]
}

func loadRandomProxies() error {

	var proxy string

	if len(proxies) == 0 {
		doc, err := goquery.NewDocument("https://www.ip-adress.com/proxy-list")
		if err != nil {
			return errors.Wrap(err, "failed to parse ip-adress.com")
		}

		doc.Find("table").Each(func(i int, tablehtml *goquery.Selection) {
			tablehtml.Find("tr").Each(func(indextr int, rowhtml *goquery.Selection) {
				proxy = "http://" + rowhtml.Find("td").First().Text()
				if len(proxy) > 7 {
					proxies = append(proxies, proxy)
				}
			})
		})
	}

	return nil
}

func getMicroData(destURL string) (*microdata.Microdata, error) {
	baseURL, err := url.Parse(destURL)
	if err != nil {
		return nil, fmt.Errorf("failed to parse url: %w", err)
	}

	resp, err := http.Get(baseURL.String())
	if err != nil {
		return nil, fmt.Errorf("failed to visit url: %w", err)
	}
	defer resp.Body.Close()

	html, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response data: %w", err)
	}

	p := microdata.NewParser(bytes.NewReader(html), baseURL)

	return p.Parse()
}

// GetComicMetaData gets all the comic strips meta data
func GetComicMetaData(dilbertURL, date string) (Comic, error) {

	comic := Comic{}

	if attempt > MaxAttempts {
		return comic, fmt.Errorf("attempts exceeded max attempts of %d", MaxAttempts)
	}

	// proxyURL, err := url.Parse(proxies[attempt])
	// if err != nil {
	// 	return Comic{}, errors.Wrap(err, "parsing proxy URL failed")
	// }

	// client := &http.Client{
	// 	Transport: &http.Transport{
	// 		Dial: (&net.Dialer{
	// 			Timeout:   60 * time.Second,
	// 			KeepAlive: 60 * time.Second,
	// 		}).Dial,
	// 		TLSHandshakeTimeout:   60 * time.Second,
	// 		ResponseHeaderTimeout: 60 * time.Second,
	// 		TLSClientConfig: &tls.Config{
	// 			InsecureSkipVerify: true,
	// 		},
	// 		Proxy: http.ProxyURL(proxyURL),
	// 	},
	// 	Timeout: 120 * time.Second,
	// }

	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}

	req, err := http.NewRequest("GET", dilbertURL, nil)
	if err != nil {
		return Comic{}, fmt.Errorf("failed to create GET request: %v", err)
	}
	req.Header.Set("User-Agent", randomAgent())

	res, err := client.Do(req)
	if err != nil {
		return Comic{}, fmt.Errorf("client Do request failed: %v", err)
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return Comic{}, fmt.Errorf("failed to connect to %s: %s", dilbertURL, res.Status)
	}

	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		log.WithError(err).Error("goquery NewDocumentFromResponse failed")
	}

	if doc != nil {
		// GET TITLE
		doc.Find(".comic-title-name").Each(func(i int, s *goquery.Selection) {
			comic.Title = s.Text()
		})

		// GET IMAGE URL
		doc.Find(".img-comic-container").Each(func(i int, s *goquery.Selection) {
			comic.ImageURL, _ = s.Find("img").Attr("src")
			// comic.ImageURL = "http:" + comic.ImageURL
		})

		// GET TAGS
		doc.Find(".comic-tags").Each(func(i int, s *goquery.Selection) {
			s.Find("a").Each(func(i int, a *goquery.Selection) {
				comic.Tags = append(comic.Tags, strings.TrimPrefix(a.Text(), "#"))
			})
		})

		// GET TRANSCRIPT
		id := "js-toggle-transcript-" + date
		doc.Find("div#" + id).Each(func(i int, s *goquery.Selection) {
			comic.Transcript = strings.TrimSpace(s.Text())
			comic.Transcript = strings.TrimPrefix(comic.Transcript, "Transcript")
			comic.Transcript = strings.TrimSpace(comic.Transcript)
		})

		return comic, nil
	}

	attempt++
	log.WithFields(log.Fields{
		"attempt": attempt,
		"proxy":   proxies[attempt],
	}).Info("retrying again")
	// retry url meta data parse
	return GetComicMetaData(dilbertURL, date)
}

// GetAllDilbert havest all teh comics strips
func GetAllDilbert(folder string, date string) error {

	// open database
	db, err := database.Open()
	if err != nil {
		return errors.Wrap(err, "opening database failed")
	}
	defer db.Close()

	count := 0
	attempt = 0

	if err = loadRandomProxies(); err != nil {
		return errors.Wrap(err, "getting random proxy URLs failed")
	}

	if len(date) < 1 {
		// date = "1989-04-17"
		date = "2019-01-01"
	}
	start, _ := time.Parse("2006-01-02", date)

	for d := start; time.Now().After(d); d = d.AddDate(0, 0, 1) {
		date := fmt.Sprintf("%04d-%02d-%02d", d.Year(), d.Month(), d.Day())

		comic, err := GetComicMetaData("https://dilbert.com/strip/"+date, date)
		if err != nil {
			return errors.Wrap(err, "getting comic metadata failed")
		}

		filepath := filepath.Join(folder, date+".gif")
		if _, err := os.Stat(filepath); err == nil {
			log.Warnf("dilbert comic already exists: %s", filepath)
		}

		// check for a valid download URL
		dlURL, err := url.ParseRequestURI(comic.ImageURL)
		if err != nil {
			log.WithError(err).Errorf("url parsing failed for: %s", comic.ImageURL)
			continue
		}

		// download image
		log.WithFields(log.Fields{
			"id":    date,
			"title": comic.Title,
			"url":   dlURL.String(),
		}).Debug("downloading file")

		go database.DownloadImage(dlURL.String(), filepath)

		// index into bleve database
		db.WriteImageToDatabase(database.ImageMetaData{
			Name:   comic.Title,
			ID:     date,
			Source: "dilbert",
			Title:  strings.Join(comic.Tags, " "),
			Text:   comic.Transcript,
			Path:   filepath,
		}, "dilbert")

		// incr count, reset attempts and reset backoff
		count++
		attempt = 0
	}

	log.WithFields(log.Fields{"count": count}).Info("dilbert comic complete")

	return nil
}
<commit_msg>Update dilbert.go<commit_after>package dilbert

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/apex/log"
	"github.com/blacktop/scifgif/database"
	"github.com/iand/microdata"
	"github.com/pkg/errors"
)

var (
	attempt int
	proxies []string
)

// MaxAttempts max number of download attempts
const MaxAttempts = 30

// Comic is the dilbert comic strip meta data
type Comic struct {
	Title      string
	Tags       []string
	ImageURL   string
	Transcript string
}

func init() {
	rand.Seed(time.Now().Unix())
}

func randomAgent() string {
	var userAgents = []string{
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
		"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Safari/604.1.38",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:56.0) Gecko/20100101 Firefox/56.0",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Safari/604.1.38",
	}
	return userAgents[rand.Int()%len(userAgents)]
}

func loadRandomProxies() error {

	var proxy string

	if len(proxies) == 0 {
		doc, err := goquery.NewDocument("https://www.ip-adress.com/proxy-list")
		if err != nil {
			return errors.Wrap(err, "failed to parse ip-adress.com")
		}

		doc.Find("table").Each(func(i int, tablehtml *goquery.Selection) {
			tablehtml.Find("tr").Each(func(indextr int, rowhtml *goquery.Selection) {
				proxy = "http://" + rowhtml.Find("td").First().Text()
				if len(proxy) > 7 {
					proxies = append(proxies, proxy)
				}
			})
		})
	}

	return nil
}

func getMicroData(destURL string) (*microdata.Microdata, error) {
	baseURL, err := url.Parse(destURL)
	if err != nil {
		return nil, fmt.Errorf("failed to parse url: %w", err)
	}

	resp, err := http.Get(baseURL.String())
	if err != nil {
		return nil, fmt.Errorf("failed to visit url: %w", err)
	}
	defer resp.Body.Close()

	html, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response data: %w", err)
	}

	p := microdata.NewParser(bytes.NewReader(html), baseURL)

	return p.Parse()
}

// GetComicMetaData gets all the comic strips meta data
func GetComicMetaData(dilbertURL, date string) (Comic, error) {

	comic := Comic{}

	if attempt > MaxAttempts {
		return comic, fmt.Errorf("attempts exceeded max attempts of %d", MaxAttempts)
	}

	// proxyURL, err := url.Parse(proxies[attempt])
	// if err != nil {
	// 	return Comic{}, errors.Wrap(err, "parsing proxy URL failed")
	// }

	// client := &http.Client{
	// 	Transport: &http.Transport{
	// 		Dial: (&net.Dialer{
	// 			Timeout:   60 * time.Second,
	// 			KeepAlive: 60 * time.Second,
	// 		}).Dial,
	// 		TLSHandshakeTimeout:   60 * time.Second,
	// 		ResponseHeaderTimeout: 60 * time.Second,
	// 		TLSClientConfig: &tls.Config{
	// 			InsecureSkipVerify: true,
	// 		},
	// 		Proxy: http.ProxyURL(proxyURL),
	// 	},
	// 	Timeout: 120 * time.Second,
	// }

	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}

	req, err := http.NewRequest("GET", dilbertURL, nil)
	if err != nil {
		return Comic{}, fmt.Errorf("failed to create GET request: %v", err)
	}
	req.Header.Set("User-Agent", randomAgent())

	res, err := client.Do(req)
	if err != nil {
		return Comic{}, fmt.Errorf("client Do request failed: %v", err)
	}
	defer res.Body.Close()

	if res.StatusCode != 200 {
		return Comic{}, fmt.Errorf("failed to connect to %s: %s", dilbertURL, res.Status)
	}

	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		log.WithError(err).Error("goquery NewDocumentFromResponse failed")
	}

	if doc != nil {
		// GET TITLE
		doc.Find(".comic-title-name").Each(func(i int, s *goquery.Selection) {
			comic.Title = s.Text()
		})

		// GET IMAGE URL
		doc.Find(".img-comic-container").Each(func(i int, s *goquery.Selection) {
			comic.ImageURL, _ = s.Find("img").Attr("src")
			// comic.ImageURL = "http:" + comic.ImageURL
		})

		// GET TAGS
		doc.Find(".comic-tags").Each(func(i int, s *goquery.Selection) {
			s.Find("a").Each(func(i int, a *goquery.Selection) {
				comic.Tags = append(comic.Tags, strings.TrimPrefix(a.Text(), "#"))
			})
		})

		// GET TRANSCRIPT
		id := "js-toggle-transcript-" + date
		doc.Find("div#" + id).Each(func(i int, s *goquery.Selection) {
			comic.Transcript = strings.TrimSpace(s.Text())
			comic.Transcript = strings.TrimPrefix(comic.Transcript, "Transcript")
			comic.Transcript = strings.TrimSpace(comic.Transcript)
		})

		return comic, nil
	}

	attempt++
	log.WithFields(log.Fields{
		"attempt": attempt,
		"proxy":   proxies[attempt],
	}).Info("retrying again")
	// retry url meta data parse
	return GetComicMetaData(dilbertURL, date)
}

// GetAllDilbert havest all teh comics strips
func GetAllDilbert(folder string, date string) error {

	// open database
	db, err := database.Open()
	if err != nil {
		return errors.Wrap(err, "opening database failed")
	}
	defer db.Close()

	count := 0
	attempt = 0

	if err = loadRandomProxies(); err != nil {
		return errors.Wrap(err, "getting random proxy URLs failed")
	}

	if len(date) < 1 {
		// date = "1989-04-17"
		date = "2019-01-01"
	}
	start, _ := time.Parse("2006-01-02", date)

	for d := start; time.Now().After(d); d = d.AddDate(0, 0, 1) {
		date := fmt.Sprintf("%04d-%02d-%02d", d.Year(), d.Month(), d.Day())

		comic, err := GetComicMetaData("https://dilbert.com/strip/"+date, date)
		if err != nil {
			return errors.Wrap(err, "getting comic metadata failed")
		}

		filepath := filepath.Join(folder, date+".gif")
		if _, err := os.Stat(filepath); err == nil {
			log.Warnf("dilbert comic already exists: %s", filepath)
		}

		// check for a valid download URL
		dlURL, err := url.ParseRequestURI(comic.ImageURL)
		if err != nil {
			log.WithError(err).Errorf("url parsing failed for: %s", comic.ImageURL)
			continue
		}

		// download image
		log.WithFields(log.Fields{
			"id":    date,
			"title": comic.Title,
			"url":   dlURL.String(),
		}).Debug("downloading file")

		go database.DownloadImage(dlURL.String(), filepath)

		// index into bleve database
		db.WriteImageToDatabase(database.ImageMetaData{
			Name:   comic.Title,
			ID:     date,
			Source: "dilbert",
			Title:  strings.Join(comic.Tags, " "),
			Text:   comic.Transcript,
			Path:   filepath,
		}, "dilbert")

		// incr count, reset attempts and reset backoff
		count++
		attempt = 0
	}

	log.WithFields(log.Fields{"count": count}).Info("dilbert comic complete")

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
)

var (
	api  = flag.String("api", "0.0.0.0:13370", "Address to bind the API interface to")
	help = flag.Bool("help", false, "Show this help")
)

func main() {
	flag.Parse()

	if *help {
		fmt.Println("Usage: diplomaenhancer [options]")
		flag.PrintDefaults()
		return
	}

	backup, e := backupHostsFile()
	if e != nil {
		log.Fatalf("Could not manipulate hosts file %s: %s", HOSTSFILE, e)
	}
	defer restoreHostsFile(backup)

	hosts, e := ParseString(backup)
	if e != nil {
		log.Fatalf("Could not parse hosts file %s: %s", HOSTSFILE, e)
	}
	hosts.WriteToFile(HOSTSFILE)

	log.Printf("Starting server...")
	serveAPI(*api, hosts)
}

func backupHostsFile() (string, error) {
	// Check for write permissions
	f, e := os.OpenFile(HOSTSFILE, os.O_WRONLY, os.FileMode(0644))
	if e != nil {
		return "", e
	}
	f.Close()

	data, e := ioutil.ReadFile(HOSTSFILE)
	return string(data), e
}

func restoreHostsFile(content string) {
	f, e := os.Create(HOSTSFILE)
	if e != nil {
		log.Fatalf("Could not restore host file %s: %s", HOSTSFILE, e)
	}
	defer f.Close()
	f.Write([]byte(content))
}
<commit_msg>Fix shutdown behaviour<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"syscall"
)

var (
	api  = flag.String("api", "0.0.0.0:13370", "Address to bind the API interface to")
	help = flag.Bool("help", false, "Show this help")
)

func main() {
	flag.Parse()

	if *help {
		fmt.Println("Usage: diplomaenhancer [options]")
		flag.PrintDefaults()
		return
	}

	backup, e := backupHostsFile()
	if e != nil {
		log.Fatalf("Could not manipulate hosts file %s: %s", HOSTSFILE, e)
	}
	go func() {
		c := make(chan os.Signal)
		signal.Notify(c, syscall.SIGTERM, syscall.SIGINT)
		<-c
		restoreHostsFile(backup)
		os.Exit(0)
	}()

	hosts, e := ParseString(backup)
	if e != nil {
		log.Fatalf("Could not parse hosts file %s: %s", HOSTSFILE, e)
	}
	hosts.WriteToFile(HOSTSFILE)

	log.Printf("Starting server...")
	serveAPI(*api, hosts)
}

func backupHostsFile() (string, error) {
	// Check for write permissions
	f, e := os.OpenFile(HOSTSFILE, os.O_WRONLY, os.FileMode(0644))
	if e != nil {
		return "", e
	}
	f.Close()

	data, e := ioutil.ReadFile(HOSTSFILE)
	return string(data), e
}

func restoreHostsFile(content string) {
	f, e := os.Create(HOSTSFILE)
	if e != nil {
		log.Fatalf("Could not restore host file %s: %s", HOSTSFILE, e)
	}
	defer f.Close()
	f.Write([]byte(content))
}
<|endoftext|>


<commit_before>package gorethink

import (
	test "launchpad.net/gocheck"
)

type object struct {
	Id    int64  `gorethink:"id,omitempty"`
	Name  string `gorethink:"name"`
	Attrs []attr
}

type attr struct {
	Name  string
	Value interface{}
}

func (s *RethinkSuite) TestRowsScanLiteral(c *test.C) {
	row, err := Expr(5).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response interface{}
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, 5)
}

func (s *RethinkSuite) TestRowsScanSlice(c *test.C) {
	row, err := Expr([]interface{}{1, 2, 3, 4, 5}).Run(sess)
	c.Assert(err, test.IsNil)

	var response []interface{}
	err = row.ScanAll(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, []interface{}{1, 2, 3, 4, 5})
}

func (s *RethinkSuite) TestRowsScanMap(c *test.C) {
	row, err := Expr(map[string]interface{}{
		"id":   2,
		"name": "Object 1",
	}).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response map[string]interface{}
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, map[string]interface{}{
		"id":   2,
		"name": "Object 1",
	})
}

func (s *RethinkSuite) TestRowsScanMapIntoInterface(c *test.C) {
	row, err := Expr(map[string]interface{}{
		"id":   2,
		"name": "Object 1",
	}).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response interface{}
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, map[string]interface{}{
		"id":   2,
		"name": "Object 1",
	})
}

func (s *RethinkSuite) TestRowsScanMapNested(c *test.C) {
	row, err := Expr(map[string]interface{}{
		"id":   2,
		"name": "Object 1",
		"attr": []interface{}{map[string]interface{}{
			"name":  "attr 1",
			"value": "value 1",
		}},
	}).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response interface{}
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, map[string]interface{}{
		"id":   2,
		"name": "Object 1",
		"attr": []interface{}{map[string]interface{}{
			"name":  "attr 1",
			"value": "value 1",
		}},
	})
}

func (s *RethinkSuite) TestRowsScanStruct(c *test.C) {
	row, err := Expr(map[string]interface{}{
		"id":   2,
		"name": "Object 1",
		"Attrs": []interface{}{map[string]interface{}{
			"Name":  "attr 1",
			"Value": "value 1",
		}},
	}).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response object
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, test.DeepEquals, object{
		Id:   2,
		Name: "Object 1",
		Attrs: []attr{attr{
			Name:  "attr 1",
			Value: "value 1",
		}},
	})
}

func (s *RethinkSuite) TestRowsAtomString(c *test.C) {
	row, err := Expr("a").RunRow(sess)
	c.Assert(err, test.IsNil)

	var response string
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, test.Equals, "a")
}

func (s *RethinkSuite) TestRowsAtomArray(c *test.C) {
	row, err := Expr([]interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).Run(sess)
	c.Assert(err, test.IsNil)

	var response []int
	err = row.ScanAll(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, test.DeepEquals, []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 0})
}

func (s *RethinkSuite) TestEmptyResults(c *test.C) {
	DbCreate("test").Exec(sess)
	Db("test").TableCreate("test").Exec(sess)
	row, err := Db("test").Table("test").Get("missing value").RunRow(sess)
	c.Assert(err, test.IsNil)
	c.Assert(row.IsNil(), test.Equals, true)

	row, err = Db("test").Table("test").Get("missing value").RunRow(sess)
	c.Assert(err, test.IsNil)
	var response interface{}
	row.Scan(response)
	c.Assert(row.IsNil(), test.Equals, true)

	rows, err := Db("test").Table("test").Get("missing value").Run(sess)
	c.Assert(err, test.IsNil)
	rows.Next()
	c.Assert(rows.IsNil(), test.Equals, true)

	rows, err = Db("test").Table("test").GetAll("missing value", "another missing value").Run(sess)
	c.Assert(err, test.IsNil)
	c.Assert(rows.Next(), test.Equals, false)

	var obj object
	obj.Name = "missing value"
	row, err = Db("test").Table("test").Filter(obj).RunRow(sess)
	c.Assert(err, test.IsNil)
	c.Assert(row.IsNil(), test.Equals, true)
}

func (s *RethinkSuite) TestRowsScanAll(c *test.C) {
	// Ensure table + database exist
	DbCreate("test").Exec(sess)
	Db("test").TableDrop("Table3").Exec(sess)
	Db("test").TableCreate("Table3").Exec(sess)
	Db("test").Table("Table3").IndexCreate("num").Exec(sess)

	// Insert rows
	Db("test").Table("Table3").Insert([]interface{}{
		map[string]interface{}{
			"id":   2,
			"name": "Object 1",
			"Attrs": []interface{}{map[string]interface{}{
				"Name":  "attr 1",
				"Value": "value 1",
			}},
		},
		map[string]interface{}{
			"id":   3,
			"name": "Object 2",
			"Attrs": []interface{}{map[string]interface{}{
				"Name":  "attr 1",
				"Value": "value 1",
			}},
		},
	}).Exec(sess)

	// Test query
	query := Db("test").Table("Table3").OrderBy("id")
	rows, err := query.Run(sess)
	c.Assert(err, test.IsNil)

	var response []object
	err = rows.ScanAll(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, test.HasLen, 2)
	c.Assert(response, test.DeepEquals, []object{
		object{
			Id:   2,
			Name: "Object 1",
			Attrs: []attr{attr{
				Name:  "attr 1",
				Value: "value 1",
			}},
		},
		object{
			Id:   3,
			Name: "Object 2",
			Attrs: []attr{attr{
				Name:  "attr 1",
				Value: "value 1",
			}},
		},
	})
}

func (s *RethinkSuite) TestRowsCount(c *test.C) {
	rows, err := Expr([]interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).Run(sess)
	c.Assert(err, test.IsNil)
	count, _ := rows.Count()
	c.Assert(count, test.Equals, 10)
}
<commit_msg>Added test for partiall nil slices<commit_after>package gorethink

import (
	test "launchpad.net/gocheck"
)

type object struct {
	Id    int64  `gorethink:"id,omitempty"`
	Name  string `gorethink:"name"`
	Attrs []attr
}

type attr struct {
	Name  string
	Value interface{}
}

func (s *RethinkSuite) TestRowsScanLiteral(c *test.C) {
	row, err := Expr(5).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response interface{}
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, 5)
}

func (s *RethinkSuite) TestRowsScanSlice(c *test.C) {
	row, err := Expr([]interface{}{1, 2, 3, 4, 5}).Run(sess)
	c.Assert(err, test.IsNil)

	var response []interface{}
	err = row.ScanAll(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, []interface{}{1, 2, 3, 4, 5})
}

func (s *RethinkSuite) TestRowsPartiallyNilSlice(c *test.C) {
	row, err := Expr([]interface{}{
		map[string]interface{}{"num": 1},
		map[string]interface{}{"num": 2},
		nil,
		nil,
		map[string]interface{}{"num": 5},
	}).Run(sess)
	c.Assert(err, test.IsNil)

	var response []interface{}
	err = row.ScanAll(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, []interface{}{1, 2, nil, nil, 5})
}

func (s *RethinkSuite) TestRowsScanMap(c *test.C) {
	row, err := Expr(map[string]interface{}{
		"id":   2,
		"name": "Object 1",
	}).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response map[string]interface{}
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, map[string]interface{}{
		"id":   2,
		"name": "Object 1",
	})
}

func (s *RethinkSuite) TestRowsScanMapIntoInterface(c *test.C) {
	row, err := Expr(map[string]interface{}{
		"id":   2,
		"name": "Object 1",
	}).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response interface{}
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, map[string]interface{}{
		"id":   2,
		"name": "Object 1",
	})
}

func (s *RethinkSuite) TestRowsScanMapNested(c *test.C) {
	row, err := Expr(map[string]interface{}{
		"id":   2,
		"name": "Object 1",
		"attr": []interface{}{map[string]interface{}{
			"name":  "attr 1",
			"value": "value 1",
		}},
	}).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response interface{}
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, JsonEquals, map[string]interface{}{
		"id":   2,
		"name": "Object 1",
		"attr": []interface{}{map[string]interface{}{
			"name":  "attr 1",
			"value": "value 1",
		}},
	})
}

func (s *RethinkSuite) TestRowsScanStruct(c *test.C) {
	row, err := Expr(map[string]interface{}{
		"id":   2,
		"name": "Object 1",
		"Attrs": []interface{}{map[string]interface{}{
			"Name":  "attr 1",
			"Value": "value 1",
		}},
	}).RunRow(sess)
	c.Assert(err, test.IsNil)

	var response object
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, test.DeepEquals, object{
		Id:   2,
		Name: "Object 1",
		Attrs: []attr{attr{
			Name:  "attr 1",
			Value: "value 1",
		}},
	})
}

func (s *RethinkSuite) TestRowsAtomString(c *test.C) {
	row, err := Expr("a").RunRow(sess)
	c.Assert(err, test.IsNil)

	var response string
	err = row.Scan(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, test.Equals, "a")
}

func (s *RethinkSuite) TestRowsAtomArray(c *test.C) {
	row, err := Expr([]interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).Run(sess)
	c.Assert(err, test.IsNil)

	var response []int
	err = row.ScanAll(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, test.DeepEquals, []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 0})
}

func (s *RethinkSuite) TestEmptyResults(c *test.C) {
	DbCreate("test").Exec(sess)
	Db("test").TableCreate("test").Exec(sess)
	row, err := Db("test").Table("test").Get("missing value").RunRow(sess)
	c.Assert(err, test.IsNil)
	c.Assert(row.IsNil(), test.Equals, true)

	row, err = Db("test").Table("test").Get("missing value").RunRow(sess)
	c.Assert(err, test.IsNil)
	var response interface{}
	row.Scan(response)
	c.Assert(row.IsNil(), test.Equals, true)

	rows, err := Db("test").Table("test").Get("missing value").Run(sess)
	c.Assert(err, test.IsNil)
	rows.Next()
	c.Assert(rows.IsNil(), test.Equals, true)

	rows, err = Db("test").Table("test").GetAll("missing value", "another missing value").Run(sess)
	c.Assert(err, test.IsNil)
	c.Assert(rows.Next(), test.Equals, false)

	var obj object
	obj.Name = "missing value"
	row, err = Db("test").Table("test").Filter(obj).RunRow(sess)
	c.Assert(err, test.IsNil)
	c.Assert(row.IsNil(), test.Equals, true)
}

func (s *RethinkSuite) TestRowsScanAll(c *test.C) {
	// Ensure table + database exist
	DbCreate("test").Exec(sess)
	Db("test").TableDrop("Table3").Exec(sess)
	Db("test").TableCreate("Table3").Exec(sess)
	Db("test").Table("Table3").IndexCreate("num").Exec(sess)

	// Insert rows
	Db("test").Table("Table3").Insert([]interface{}{
		map[string]interface{}{
			"id":   2,
			"name": "Object 1",
			"Attrs": []interface{}{map[string]interface{}{
				"Name":  "attr 1",
				"Value": "value 1",
			}},
		},
		map[string]interface{}{
			"id":   3,
			"name": "Object 2",
			"Attrs": []interface{}{map[string]interface{}{
				"Name":  "attr 1",
				"Value": "value 1",
			}},
		},
	}).Exec(sess)

	// Test query
	query := Db("test").Table("Table3").OrderBy("id")
	rows, err := query.Run(sess)
	c.Assert(err, test.IsNil)

	var response []object
	err = rows.ScanAll(&response)
	c.Assert(err, test.IsNil)
	c.Assert(response, test.HasLen, 2)
	c.Assert(response, test.DeepEquals, []object{
		object{
			Id:   2,
			Name: "Object 1",
			Attrs: []attr{attr{
				Name:  "attr 1",
				Value: "value 1",
			}},
		},
		object{
			Id:   3,
			Name: "Object 2",
			Attrs: []attr{attr{
				Name:  "attr 1",
				Value: "value 1",
			}},
		},
	})
}

func (s *RethinkSuite) TestRowsCount(c *test.C) {
	rows, err := Expr([]interface{}{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}).Run(sess)
	c.Assert(err, test.IsNil)
	count, _ := rows.Count()
	c.Assert(count, test.Equals, 10)
}
<|endoftext|>


<commit_before>package types

import (
	"database/sql/driver"
	"encoding/json"
	"net"
	"reflect"
	"strconv"
	"time"
)

var driverValuerType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
var appenderType = reflect.TypeOf((*ValueAppender)(nil)).Elem()

type AppenderFunc func([]byte, reflect.Value, int) []byte

var valueAppenders []AppenderFunc

func init() {
	valueAppenders = []AppenderFunc{
		reflect.Bool:          appendBoolValue,
		reflect.Int:           appendIntValue,
		reflect.Int8:          appendIntValue,
		reflect.Int16:         appendIntValue,
		reflect.Int32:         appendIntValue,
		reflect.Int64:         appendIntValue,
		reflect.Uint:          appendUintValue,
		reflect.Uint8:         appendUintValue,
		reflect.Uint16:        appendUintValue,
		reflect.Uint32:        appendUintValue,
		reflect.Uint64:        appendUintValue,
		reflect.Uintptr:       nil,
		reflect.Float32:       appendFloatValue,
		reflect.Float64:       appendFloatValue,
		reflect.Complex64:     nil,
		reflect.Complex128:    nil,
		reflect.Array:         nil,
		reflect.Chan:          nil,
		reflect.Func:          nil,
		reflect.Interface:     appendIfaceValue,
		reflect.Map:           appendJSONValue,
		reflect.Ptr:           nil,
		reflect.Slice:         appendJSONValue,
		reflect.String:        appendStringValue,
		reflect.Struct:        appendStructValue,
		reflect.UnsafePointer: nil,
	}
}

func Appender(typ reflect.Type) AppenderFunc {
	return appender(typ, false)
}

func appender(typ reflect.Type, pgArray bool) AppenderFunc {
	switch typ {
	case timeType:
		return appendTimeValue
	case ipType:
		return appendIPValue
	case ipNetType:
		return appendIPNetValue
	}

	if typ.Implements(appenderType) {
		return appendAppenderValue
	}

	if typ.Implements(driverValuerType) {
		return appendDriverValuerValue
	}

	kind := typ.Kind()
	switch kind {
	case reflect.Ptr:
		return ptrAppenderFunc(typ)
	case reflect.Slice:
		if typ.Elem().Kind() == reflect.Uint8 {
			return appendBytesValue
		}
		if pgArray {
			return ArrayAppender(typ)
		}
	}
	return valueAppenders[kind]
}

func ptrAppenderFunc(typ reflect.Type) AppenderFunc {
	appender := Appender(typ.Elem())
	return func(b []byte, v reflect.Value, quote int) []byte {
		if v.IsNil() {
			return AppendNull(b, quote)
		}
		return appender(b, v.Elem(), quote)
	}
}

func appendValue(b []byte, v reflect.Value, quote int) []byte {
	if v.Kind() == reflect.Ptr {
		if v.IsNil() {
			return AppendNull(b, quote)
		}
		return appendValue(b, v.Elem(), quote)
	}

	appender := Appender(v.Type())
	return appender(b, v, quote)
}

func appendIfaceValue(b []byte, v reflect.Value, quote int) []byte {
	return Append(b, v.Interface(), quote)
}

func appendBoolValue(b []byte, v reflect.Value, _ int) []byte {
	return appendBool(b, v.Bool())
}

func appendIntValue(b []byte, v reflect.Value, _ int) []byte {
	return strconv.AppendInt(b, v.Int(), 10)
}

func appendUintValue(b []byte, v reflect.Value, _ int) []byte {
	return strconv.AppendUint(b, v.Uint(), 10)
}

func appendFloatValue(b []byte, v reflect.Value, _ int) []byte {
	return appendFloat(b, v.Float())
}

func appendBytesValue(b []byte, v reflect.Value, quote int) []byte {
	return appendBytes(b, v.Bytes(), quote)
}

func appendStringValue(b []byte, v reflect.Value, quote int) []byte {
	return AppendString(b, v.String(), quote)
}

func appendStructValue(b []byte, v reflect.Value, quote int) []byte {
	if v.Type() == timeType {
		return appendTimeValue(b, v, quote)
	}
	return appendJSONValue(b, v, quote)
}

func appendJSONValue(b []byte, v reflect.Value, quote int) []byte {
	bytes, err := json.Marshal(v.Interface())
	if err != nil {
		return AppendError(b, err)
	}
	return AppendJSONB(b, bytes, quote)
}

func appendTimeValue(b []byte, v reflect.Value, quote int) []byte {
	tm := v.Interface().(time.Time)
	return AppendTime(b, tm, quote)
}

func appendIPValue(b []byte, v reflect.Value, quote int) []byte {
	ip := v.Interface().(net.IP)
	return AppendString(b, ip.String(), quote)
}

func appendIPNetValue(b []byte, v reflect.Value, quote int) []byte {
	ipnet := v.Interface().(net.IPNet)
	return AppendString(b, ipnet.String(), quote)
}

func appendAppenderValue(b []byte, v reflect.Value, quote int) []byte {
	return appendAppender(b, v.Interface().(ValueAppender), quote)
}

func appendDriverValuerValue(b []byte, v reflect.Value, quote int) []byte {
	return appendDriverValuer(b, v.Interface().(driver.Valuer), quote)
}
<commit_msg>types: define array bytes appender<commit_after>package types

import (
	"database/sql/driver"
	"encoding/json"
	"net"
	"reflect"
	"strconv"
	"time"
)

var driverValuerType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
var appenderType = reflect.TypeOf((*ValueAppender)(nil)).Elem()

type AppenderFunc func([]byte, reflect.Value, int) []byte

var valueAppenders []AppenderFunc

func init() {
	valueAppenders = []AppenderFunc{
		reflect.Bool:          appendBoolValue,
		reflect.Int:           appendIntValue,
		reflect.Int8:          appendIntValue,
		reflect.Int16:         appendIntValue,
		reflect.Int32:         appendIntValue,
		reflect.Int64:         appendIntValue,
		reflect.Uint:          appendUintValue,
		reflect.Uint8:         appendUintValue,
		reflect.Uint16:        appendUintValue,
		reflect.Uint32:        appendUintValue,
		reflect.Uint64:        appendUintValue,
		reflect.Uintptr:       nil,
		reflect.Float32:       appendFloatValue,
		reflect.Float64:       appendFloatValue,
		reflect.Complex64:     nil,
		reflect.Complex128:    nil,
		reflect.Array:         nil,
		reflect.Chan:          nil,
		reflect.Func:          nil,
		reflect.Interface:     appendIfaceValue,
		reflect.Map:           appendJSONValue,
		reflect.Ptr:           nil,
		reflect.Slice:         appendJSONValue,
		reflect.String:        appendStringValue,
		reflect.Struct:        appendStructValue,
		reflect.UnsafePointer: nil,
	}
}

func Appender(typ reflect.Type) AppenderFunc {
	return appender(typ, false)
}

func appender(typ reflect.Type, pgArray bool) AppenderFunc {
	switch typ {
	case timeType:
		return appendTimeValue
	case ipType:
		return appendIPValue
	case ipNetType:
		return appendIPNetValue
	}

	if typ.Implements(appenderType) {
		return appendAppenderValue
	}

	if typ.Implements(driverValuerType) {
		return appendDriverValuerValue
	}

	kind := typ.Kind()
	switch kind {
	case reflect.Ptr:
		return ptrAppenderFunc(typ)
	case reflect.Slice:
		if typ.Elem().Kind() == reflect.Uint8 {
			return appendBytesValue
		}
		if pgArray {
			return ArrayAppender(typ)
		}
	case reflect.Array:
		if typ.Elem().Kind() == reflect.Uint8 {
			return appendArrayBytesValue
		}
	}
	return valueAppenders[kind]
}

func ptrAppenderFunc(typ reflect.Type) AppenderFunc {
	appender := Appender(typ.Elem())
	return func(b []byte, v reflect.Value, quote int) []byte {
		if v.IsNil() {
			return AppendNull(b, quote)
		}
		return appender(b, v.Elem(), quote)
	}
}

func appendValue(b []byte, v reflect.Value, quote int) []byte {
	if v.Kind() == reflect.Ptr {
		if v.IsNil() {
			return AppendNull(b, quote)
		}
		return appendValue(b, v.Elem(), quote)
	}

	appender := Appender(v.Type())
	return appender(b, v, quote)
}

func appendIfaceValue(b []byte, v reflect.Value, quote int) []byte {
	return Append(b, v.Interface(), quote)
}

func appendBoolValue(b []byte, v reflect.Value, _ int) []byte {
	return appendBool(b, v.Bool())
}

func appendIntValue(b []byte, v reflect.Value, _ int) []byte {
	return strconv.AppendInt(b, v.Int(), 10)
}

func appendUintValue(b []byte, v reflect.Value, _ int) []byte {
	return strconv.AppendUint(b, v.Uint(), 10)
}

func appendFloatValue(b []byte, v reflect.Value, _ int) []byte {
	return appendFloat(b, v.Float())
}

func appendBytesValue(b []byte, v reflect.Value, quote int) []byte {
	return appendBytes(b, v.Bytes(), quote)
}

func appendArrayBytesValue(b []byte, v reflect.Value, quote int) []byte {
	return appendBytes(b, v.Slice(0, v.Len()).Bytes(), quote)
}

func appendStringValue(b []byte, v reflect.Value, quote int) []byte {
	return AppendString(b, v.String(), quote)
}

func appendStructValue(b []byte, v reflect.Value, quote int) []byte {
	if v.Type() == timeType {
		return appendTimeValue(b, v, quote)
	}
	return appendJSONValue(b, v, quote)
}

func appendJSONValue(b []byte, v reflect.Value, quote int) []byte {
	bytes, err := json.Marshal(v.Interface())
	if err != nil {
		return AppendError(b, err)
	}
	return AppendJSONB(b, bytes, quote)
}

func appendTimeValue(b []byte, v reflect.Value, quote int) []byte {
	tm := v.Interface().(time.Time)
	return AppendTime(b, tm, quote)
}

func appendIPValue(b []byte, v reflect.Value, quote int) []byte {
	ip := v.Interface().(net.IP)
	return AppendString(b, ip.String(), quote)
}

func appendIPNetValue(b []byte, v reflect.Value, quote int) []byte {
	ipnet := v.Interface().(net.IPNet)
	return AppendString(b, ipnet.String(), quote)
}

func appendAppenderValue(b []byte, v reflect.Value, quote int) []byte {
	return appendAppender(b, v.Interface().(ValueAppender), quote)
}

func appendDriverValuerValue(b []byte, v reflect.Value, quote int) []byte {
	return appendDriverValuer(b, v.Interface().(driver.Valuer), quote)
}
<|endoftext|>


<commit_before>package sflow

import (
	"encoding/binary"
	"io"
)

// RawPacketFlow is a raw Ethernet header flow record.
type RawPacketFlow struct {
	Protocol    uint32
	FrameLength uint32
	Stripped    uint32
	HeaderSize  uint32
	Header      []byte
}

// ExtendedSwitchFlow is an extended switch flow record.
type ExtendedSwitchFlow struct {
	SourceVlan          uint32
	SourcePriority      uint32
	DestinationVlan     uint32
	DestinationPriority uint32
}

// RecordType returns the type of flow record.
func (f RawPacketFlow) RecordType() int {
	return TypeRawPacketFlowRecord
}

func decodeRawPacketFlow(r io.Reader) (RawPacketFlow, error) {
	f := RawPacketFlow{}

	var err error

	err = binary.Read(r, binary.BigEndian, &f.Protocol)
	if err != nil {
		return f, err
	}

	err = binary.Read(r, binary.BigEndian, &f.FrameLength)
	if err != nil {
		return f, err
	}

	err = binary.Read(r, binary.BigEndian, &f.Stripped)
	if err != nil {
		return f, err
	}

	err = binary.Read(r, binary.BigEndian, &f.HeaderSize)
	if err != nil {
		return f, err
	}

	f.Header = make([]byte, f.HeaderSize+((4-f.HeaderSize)%4))

	_, err = r.Read(f.Header)
	if err != nil {
		return f, err
	}

	// We need to consume the padded length,
	// but len(Header) should still be HeaderSize.
	f.Header = f.Header[:f.HeaderSize]

	return f, err
}

func (f RawPacketFlow) encode(w io.Writer) error {
	var err error

	err = binary.Write(w, binary.BigEndian, uint32(f.RecordType()))
	if err != nil {
		return err
	}

	// We need to calculate encoded size of the record.
	encodedRecordLength := uint32(4 * 4) // 4 32-bit records

	// Add the length of the header padded to a multiple of 4 bytes.
	encodedRecordLength += f.HeaderSize + ((4 - f.HeaderSize) % 4)

	err = binary.Write(w, binary.BigEndian, encodedRecordLength)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, f.Protocol)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, f.FrameLength)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, f.Stripped)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, f.HeaderSize)
	if err != nil {
		return err
	}

	_, err = w.Write(append(f.Header, make([]byte, (4-f.HeaderSize)%4)...))

	return err
}

// RecordType returns the type of flow record.
func (f ExtendedSwitchFlow) RecordType() int {
	return TypeExtendedSwitchFlowRecord
}

func decodedExtendedSwitchFlow(r io.Reader) (ExtendedSwitchFlow, error) {
	f := ExtendedSwitchFlow{}

	err := binary.Read(r, binary.BigEndian, &f)

	return f, err
}

func (f ExtendedSwitchFlow) encode(w io.Writer) error {
	var err error

	err = binary.Write(w, binary.BigEndian, uint32(f.RecordType()))
	if err != nil {
		return err
	}

	encodedRecordLength := uint32(4 * 4) // 4 32-bit records

	err = binary.Write(w, binary.BigEndian, encodedRecordLength)
	if err != nil {
		return err
	}

	return binary.Write(w, binary.BigEndian, f)
}
<commit_msg>Fix typo with field size alignment<commit_after>package sflow

import (
	"encoding/binary"
	"io"
)

// RawPacketFlow is a raw Ethernet header flow record.
type RawPacketFlow struct {
	Protocol    uint32
	FrameLength uint32
	Stripped    uint32
	HeaderSize  uint32
	Header      []byte
}

// ExtendedSwitchFlow is an extended switch flow record.
type ExtendedSwitchFlow struct {
	SourceVlan          uint32
	SourcePriority      uint32
	DestinationVlan     uint32
	DestinationPriority uint32
}

// RecordType returns the type of flow record.
func (f RawPacketFlow) RecordType() int {
	return TypeRawPacketFlowRecord
}

func decodeRawPacketFlow(r io.Reader) (RawPacketFlow, error) {
	f := RawPacketFlow{}

	var err error

	err = binary.Read(r, binary.BigEndian, &f.Protocol)
	if err != nil {
		return f, err
	}

	err = binary.Read(r, binary.BigEndian, &f.FrameLength)
	if err != nil {
		return f, err
	}

	err = binary.Read(r, binary.BigEndian, &f.Stripped)
	if err != nil {
		return f, err
	}

	err = binary.Read(r, binary.BigEndian, &f.HeaderSize)
	if err != nil {
		return f, err
	}

	f.Header = make([]byte, f.HeaderSize+((4-f.HeaderSize)%4))

	_, err = r.Read(f.Header)
	if err != nil {
		return f, err
	}

	// We need to consume the padded length,
	// but len(Header) should still be HeaderSize.
	f.Header = f.Header[:f.HeaderSize]

	return f, err
}

func (f RawPacketFlow) encode(w io.Writer) error {
	var err error

	err = binary.Write(w, binary.BigEndian, uint32(f.RecordType()))
	if err != nil {
		return err
	}

	// We need to calculate encoded size of the record.
	encodedRecordLength := uint32(4 * 4) // 4 32-bit records

	// Add the length of the header padded to a multiple of 4 bytes.
	encodedRecordLength += f.HeaderSize + (4 - f.HeaderSize%4)

	err = binary.Write(w, binary.BigEndian, encodedRecordLength)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, f.Protocol)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, f.FrameLength)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, f.Stripped)
	if err != nil {
		return err
	}

	err = binary.Write(w, binary.BigEndian, f.HeaderSize)
	if err != nil {
		return err
	}

	_, err = w.Write(append(f.Header, make([]byte, (4-f.HeaderSize%4))...))

	return err
}

// RecordType returns the type of flow record.
func (f ExtendedSwitchFlow) RecordType() int {
	return TypeExtendedSwitchFlowRecord
}

func decodedExtendedSwitchFlow(r io.Reader) (ExtendedSwitchFlow, error) {
	f := ExtendedSwitchFlow{}

	err := binary.Read(r, binary.BigEndian, &f)

	return f, err
}

func (f ExtendedSwitchFlow) encode(w io.Writer) error {
	var err error

	err = binary.Write(w, binary.BigEndian, uint32(f.RecordType()))
	if err != nil {
		return err
	}

	encodedRecordLength := uint32(4 * 4) // 4 32-bit records

	err = binary.Write(w, binary.BigEndian, encodedRecordLength)
	if err != nil {
		return err
	}

	return binary.Write(w, binary.BigEndian, f)
}
<|endoftext|>


<commit_before><commit_msg>reduced visibility of pool interface<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"

	"github.com/Xe/middleware"
	"github.com/codegangsta/negroni"
	"github.com/drone/routes"
	"stevenbooru.cf/config"
	"stevenbooru.cf/eye"
)

var (
	c config.Config

	configFileFlag = flag.String("conf", "./cfg/stevenbooru.cfg", "configuration file to load")
)

func init() {
	flag.Parse()

	var err error
	c, err = config.ParseConfig(*configFileFlag)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	mux := routes.New()

	mux.Get("/", func(rw http.ResponseWriter, r *http.Request) {
		eye.DoTemplate("views/index", rw, r, nil)
	})

	n := negroni.Classic()

	middleware.Inject(n)
	n.UseHandler(mux)

	n.Run(fmt.Sprintf("%s:%s", c.HTTP.Bindhost, c.HTTP.Port))
}
<commit_msg>Use globals package<commit_after>package main

import (
	"fmt"
	"net/http"

	"github.com/Xe/middleware"
	"github.com/codegangsta/negroni"
	"github.com/drone/routes"
	"stevenbooru.cf/eye"
	"stevenbooru.cf/globals"
)

func main() {
	mux := routes.New()

	mux.Get("/", func(rw http.ResponseWriter, r *http.Request) {
		eye.DoTemplate("views/index", rw, r, nil)
	})

	n := negroni.Classic()

	middleware.Inject(n)
	n.UseHandler(mux)

	n.Run(fmt.Sprintf("%s:%s", globals.Config.HTTP.Bindhost, globals.Config.HTTP.Port))
}
<|endoftext|>


<commit_before>package main

import "strings"

func replaceMarkup(s string) string {
	s = strings.Replace(s, "—", "-", -1)
	s = strings.Replace(s, "“", "\"", -1)
	s = strings.Replace(s, "”", "\"", -1)
	s = strings.Replace(s, "’", "\"", -1)
	s = strings.Replace(s, "‘", "\"", -1)
	s = strings.Replace(s, ";", ";", -1)
	s = strings.Replace(s, "", "*", -1)
	s = strings.Replace(s, "", "*", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "<s>", "~", -1)
	s = strings.Replace(s, "</s>", "~", -1)
	s = strings.Replace(s, "
", "`", -1)
	s = strings.Replace(s, "
", "`", -1)
	s = strings.Replace(s, "
", "```", -1)
	s = strings.Replace(s, "</blockquote>", "```", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "
", "\n", -1)
	return s
}
<commit_msg>Removing unnecessary markup changes<commit_after>package main

import "strings"

func replaceMarkup(s string) string {
	s = strings.Replace(s, "", "*", -1)
	s = strings.Replace(s, "", "*", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "", "_", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "<s>", "~", -1)
	s = strings.Replace(s, "</s>", "~", -1)
	s = strings.Replace(s, "
", "`", -1)
	s = strings.Replace(s, "
", "`", -1)
	s = strings.Replace(s, "
", "```", -1)
	s = strings.Replace(s, "</blockquote>", "```", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "
", "\n", -1)
	s = strings.Replace(s, "
", "\n", -1)
	return s
}
<|endoftext|>


<commit_before><commit_msg>add test<commit_after><|endoftext|>


<commit_before>package generator_test

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path"
	"runtime"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/bbs/db/etcd/test/etcd_helpers"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	etcdclient "github.com/coreos/go-etcd/etcd"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"testing"
)

var etcdPort int
var etcdUrl string
var etcdSSLConfig *etcdstorerunner.SSLConfig
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var etcdClient *etcdclient.Client

var logger lager.Logger

var bbsClient bbs.Client
var bbsBinPath string
var bbsAddress string
var bbsURL *url.URL
var bbsArgs testrunner.Args
var bbsRunner *ginkgomon.Runner
var bbsProcess ifrit.Process
var consulSession *consuladapter.Session
var consulRunner *consulrunner.ClusterRunner
var etcdHelper *etcd_helpers.ETCDHelper
var auctioneerServer *ghttp.Server

func TestGenerator(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Generator Suite")
}

var _ = SynchronizedBeforeSuite(
	func() []byte {
		os.Setenv("GOMAXPROCS", strconv.Itoa(runtime.NumCPU()))
		bbsConfig, err := gexec.Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
		Expect(err).NotTo(HaveOccurred())
		return []byte(bbsConfig)
	},
	func(bbsConfig []byte) {
		bbsBinPath = string(bbsConfig)
		SetDefaultEventuallyTimeout(15 * time.Second)
	},
)

var _ = SynchronizedAfterSuite(func() {
}, func() {
	gexec.CleanupBuildArtifacts()
})

var _ = AfterEach(func() {
	ginkgomon.Kill(bbsProcess)
	etcdRunner.Stop()
	consulRunner.Stop()
	auctioneerServer.Close()
})

var _ = BeforeEach(func() {
	logger = lagertest.NewTestLogger("test")

	auctioneerServer = ghttp.NewServer()
	auctioneerServer.UnhandledRequestStatusCode = http.StatusAccepted
	auctioneerServer.AllowUnhandledRequests = true

	bbsAddress = fmt.Sprintf("127.0.0.1:%d", 6700+GinkgoParallelNode())

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsArgs = testrunner.Args{
		Address:               bbsAddress,
		AdvertiseURL:          bbsURL.String(),
		AuctioneerAddress:     auctioneerServer.URL(),
		MetricsReportInterval: 10 * time.Millisecond,

		EncryptionKeys: []string{"label:key"},
		ActiveKeyLabel: "label",
	}
})

var _ = JustBeforeEach(func() {
	etcdPort = 4001 + GinkgoParallelNode()
	etcdScheme := "http"
	if etcdSSLConfig != nil {
		etcdScheme = "https"
	}
	etcdUrl = fmt.Sprintf(etcdScheme+"://127.0.0.1:%d", etcdPort)
	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, etcdSSLConfig)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	consulRunner.Start()
	consulRunner.WaitUntilReady()
	consulRunner.Reset()

	etcdRunner.Start()
	etcdRunner.Reset()

	etcdClient = etcdRunner.Client()
	etcdClient.SetConsistency(etcdclient.STRONG_CONSISTENCY)

	bbsArgs.ConsulCluster = consulRunner.ConsulCluster()
	bbsArgs.EtcdCluster = etcdUrl

	bbsRunner = testrunner.New(bbsBinPath, bbsArgs)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	basePath := path.Join(os.Getenv("GOPATH"), "src", "github.com", "cloudfoundry-incubator", "bbs", "cmd", "bbs", "fixtures")
	caFile := path.Join(basePath, "green-certs", "server-ca.crt")
	certFile := path.Join(basePath, "green-certs", "client.crt")
	keyFile := path.Join(basePath, "green-certs", "client.key")

	var err error
	bbsClient, err = bbs.NewSecureClient(bbsURL.String(), caFile, certFile, keyFile, 1, 1)
	Expect(err).NotTo(HaveOccurred())
})
<commit_msg>Remove references to consul session<commit_after>package generator_test

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"path"
	"runtime"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/bbs/db/etcd/test/etcd_helpers"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	etcdclient "github.com/coreos/go-etcd/etcd"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/onsi/gomega/ghttp"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"testing"
)

var etcdPort int
var etcdUrl string
var etcdSSLConfig *etcdstorerunner.SSLConfig
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var etcdClient *etcdclient.Client

var logger lager.Logger

var bbsClient bbs.Client
var bbsBinPath string
var bbsAddress string
var bbsURL *url.URL
var bbsArgs testrunner.Args
var bbsRunner *ginkgomon.Runner
var bbsProcess ifrit.Process
var consulRunner *consulrunner.ClusterRunner
var etcdHelper *etcd_helpers.ETCDHelper
var auctioneerServer *ghttp.Server

func TestGenerator(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Generator Suite")
}

var _ = SynchronizedBeforeSuite(
	func() []byte {
		os.Setenv("GOMAXPROCS", strconv.Itoa(runtime.NumCPU()))
		bbsConfig, err := gexec.Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
		Expect(err).NotTo(HaveOccurred())
		return []byte(bbsConfig)
	},
	func(bbsConfig []byte) {
		bbsBinPath = string(bbsConfig)
		SetDefaultEventuallyTimeout(15 * time.Second)
	},
)

var _ = SynchronizedAfterSuite(func() {
}, func() {
	gexec.CleanupBuildArtifacts()
})

var _ = AfterEach(func() {
	ginkgomon.Kill(bbsProcess)
	etcdRunner.Stop()
	consulRunner.Stop()
	auctioneerServer.Close()
})

var _ = BeforeEach(func() {
	logger = lagertest.NewTestLogger("test")

	auctioneerServer = ghttp.NewServer()
	auctioneerServer.UnhandledRequestStatusCode = http.StatusAccepted
	auctioneerServer.AllowUnhandledRequests = true

	bbsAddress = fmt.Sprintf("127.0.0.1:%d", 6700+GinkgoParallelNode())

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsArgs = testrunner.Args{
		Address:               bbsAddress,
		AdvertiseURL:          bbsURL.String(),
		AuctioneerAddress:     auctioneerServer.URL(),
		MetricsReportInterval: 10 * time.Millisecond,

		EncryptionKeys: []string{"label:key"},
		ActiveKeyLabel: "label",
	}
})

var _ = JustBeforeEach(func() {
	etcdPort = 4001 + GinkgoParallelNode()
	etcdScheme := "http"
	if etcdSSLConfig != nil {
		etcdScheme = "https"
	}
	etcdUrl = fmt.Sprintf(etcdScheme+"://127.0.0.1:%d", etcdPort)
	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, etcdSSLConfig)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	consulRunner.Start()
	consulRunner.WaitUntilReady()
	consulRunner.Reset()

	etcdRunner.Start()
	etcdRunner.Reset()

	etcdClient = etcdRunner.Client()
	etcdClient.SetConsistency(etcdclient.STRONG_CONSISTENCY)

	bbsArgs.ConsulCluster = consulRunner.ConsulCluster()
	bbsArgs.EtcdCluster = etcdUrl

	bbsRunner = testrunner.New(bbsBinPath, bbsArgs)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	basePath := path.Join(os.Getenv("GOPATH"), "src", "github.com", "cloudfoundry-incubator", "bbs", "cmd", "bbs", "fixtures")
	caFile := path.Join(basePath, "green-certs", "server-ca.crt")
	certFile := path.Join(basePath, "green-certs", "client.crt")
	keyFile := path.Join(basePath, "green-certs", "client.key")

	var err error
	bbsClient, err = bbs.NewSecureClient(bbsURL.String(), caFile, certFile, keyFile, 1, 1)
	Expect(err).NotTo(HaveOccurred())
})
<|endoftext|>


<commit_before>package middleware

import (
	"github.com/gin-gonic/gin"
	"github.com/vifino/carbon/modules/helpers"

	"github.com/sourcegraph/syntaxhighlight"
)

var css = `
/* Pretty printing styles. Used with prettify.js. */

/* SPAN elements with the classes below are added by prettyprint. */
.pln { color: #000 }  /* plain text */

@media screen {
	.str { color: #080 }  /* string content */
	.kwd { color: #008 }  /* a keyword */
	.com { color: #800 }  /* a comment */
	.typ { color: #606 }  /* a type name */
	.lit { color: #066 }  /* a literal value */
	/* punctuation, lisp open bracket, lisp close bracket */
	.pun, .opn, .clo { color: #660 }
	.tag { color: #008 }  /* a markup tag name */
	.atn { color: #606 }  /* a markup attribute name */
	.atv { color: #080 }  /* a markup attribute value */
	.dec, .var { color: #606 }  /* a declaration; a variable name */
	.fun { color: red }  /* a function name */
}

/* Use higher contrast and text-weight for printable form. */
@media print, projection {
	.str { color: #060 }
	.kwd { color: #006; font-weight: bold }
	.com { color: #600; font-style: italic }
	.typ { color: #404; font-weight: bold }
	.lit { color: #044 }
	.pun, .opn, .clo { color: #440 }
	.tag { color: #006; font-weight: bold }
	.atn { color: #404 }
	.atv { color: #060 }
}

/* Put a border around prettyprinted code snippets. */
pre.prettyprint { padding: 2px; border: 1px solid #888 }

/* Specify class=linenums on a pre to get line numbering */
ol.linenums { margin-top: 0; margin-bottom: 0 } /* IE indents via margin-left */
li.L0,
li.L1,
li.L2,
li.L3,
li.L5,
li.L6,
li.L7,
li.L8 { list-style-type: none }
/* Alternate shading for lines */
li.L1,
li.L3,
li.L5,
li.L7,
li.L9 { background: #eee }
`

func SyntaxHL(status int, s string) func(*gin.Context) {
	src := []byte(s)
	return func(c *gin.Context) {
		highlighted, err := syntaxhighlight.AsHTML(src)
		if err != nil {
			helpers.String(c, 500, err)
		} else {
			content := `<style>` + css + `</style><pre><code>` + string(highlighted) + `</code></pre>`
			helpers.String(c, status, content)
		}
	}
}
<commit_msg>You're kidding me, right?<commit_after>package middleware

import (
	"github.com/gin-gonic/gin"
	"github.com/vifino/carbon/modules/helpers"

	"github.com/sourcegraph/syntaxhighlight"
)

var css = `
/* Pretty printing styles. Used with prettify.js. */

/* SPAN elements with the classes below are added by prettyprint. */
.pln { color: #000 }  /* plain text */

@media screen {
	.str { color: #080 }  /* string content */
	.kwd { color: #008 }  /* a keyword */
	.com { color: #800 }  /* a comment */
	.typ { color: #606 }  /* a type name */
	.lit { color: #066 }  /* a literal value */
	/* punctuation, lisp open bracket, lisp close bracket */
	.pun, .opn, .clo { color: #660 }
	.tag { color: #008 }  /* a markup tag name */
	.atn { color: #606 }  /* a markup attribute name */
	.atv { color: #080 }  /* a markup attribute value */
	.dec, .var { color: #606 }  /* a declaration; a variable name */
	.fun { color: red }  /* a function name */
}

/* Use higher contrast and text-weight for printable form. */
@media print, projection {
	.str { color: #060 }
	.kwd { color: #006; font-weight: bold }
	.com { color: #600; font-style: italic }
	.typ { color: #404; font-weight: bold }
	.lit { color: #044 }
	.pun, .opn, .clo { color: #440 }
	.tag { color: #006; font-weight: bold }
	.atn { color: #404 }
	.atv { color: #060 }
}

/* Put a border around prettyprinted code snippets. */
pre.prettyprint { padding: 2px; border: 1px solid #888 }

/* Specify class=linenums on a pre to get line numbering */
ol.linenums { margin-top: 0; margin-bottom: 0 } /* IE indents via margin-left */
li.L0,
li.L1,
li.L2,
li.L3,
li.L5,
li.L6,
li.L7,
li.L8 { list-style-type: none }
/* Alternate shading for lines */
li.L1,
li.L3,
li.L5,
li.L7,
li.L9 { background: #eee }
`

func SyntaxHL(status int, s string) func(*gin.Context) {
	src := []byte(s)
	return func(c *gin.Context) {
		highlighted, err := syntaxhighlight.AsHTML(src)
		if err != nil {
			helpers.String(c, 500, err.Error())
		} else {
			content := `<style>` + css + `</style><pre><code>` + string(highlighted) + `</code></pre>`
			helpers.String(c, status, content)
		}
	}
}
<|endoftext|>