<commit_before>package lzbase

import "io"

// Writer allows to generate a LZMA stream, while the OpCodec object can be
// reused.
type Writer struct {
	OpCodec *OpCodec
	Dict    *WriterDict
	re      *rangeEncoder
	params  *Parameters
}

// InitWriter initializes a writer object.
func InitWriter(bw *Writer, w io.Writer, oc *OpCodec, params Parameters) error {
	switch {
	case w == nil:
		return newError("InitWriter argument w is nil")
	case oc == nil:
		return newError("InitWriter argument oc is nil")
	}
	err := verifyParameters(&params)
	if err != nil {
		return err
	}
	dict, ok := oc.dict.(*WriterDict)
	if !ok {
		return newError("op codec for writer expected")
	}
	re := newRangeEncoder(w)
	*bw = Writer{OpCodec: oc, Dict: dict, re: re, params: &params}
	return nil
}

// Write moves data into the internal buffer and triggers its compression. Note
// that beside the data held back to enable a large match all data will be be
// compressed.
func (bw *Writer) Write(p []byte) (n int, err error) {
	end := bw.Dict.end + int64(len(p))
	if end < 0 {
		panic("end counter overflow")
	}
	var rerr error
	if bw.params.SizeInHeader && end > bw.params.Size {
		p = p[:bw.params.Size-end]
		rerr = newError("write exceeds unpackLen")
	}
	for n < len(p) {
		k, err := bw.Dict.Write(p[n:])
		n += k
		if err != nil && err != errAgain {
			return n, err
		}
		if err = bw.process(0); err != nil {
			return n, err
		}
	}
	return n, rerr
}

// Close terminates the LZMA stream. It doesn't close the underlying writer
// though and leaves it alone. In some scenarios explicit closing of the
// underlying writer is required.
func (bw *Writer) Close() error {
	var err error
	if err = bw.process(allData); err != nil {
		return err
	}
	if bw.params.EOS {
		if err = bw.writeEOS(); err != nil {
			return err
		}
	}
	if err = bw.re.Close(); err != nil {
		return err
	}
	return bw.Dict.Close()
}

// The allData flag tells the process method that all data must be processed.
const allData = 1

// indicates an empty buffer
var errEmptyBuf = newError("empty buffer")

// potentialOffsets creates a list of potential offsets for matches.
func (bw *Writer) potentialOffsets(p []byte) []int64 {
	oc := bw.OpCodec
	head := bw.Dict.Offset()
	start := bw.Dict.start
	offs := make([]int64, 0, 32)
	// add potential offsets with highest priority at the top
	for i := 1; i < 11; i++ {
		// distance 1 to 8
		off := head - int64(i)
		if start <= off {
			offs = append(offs, off)
		}
	}
	if len(p) == 4 {
		// distances from the hash table
		offs = append(offs, bw.Dict.Offsets(p)...)
	}
	for i := 3; i >= 0; i-- {
		// distances from the repetition for length less than 4
		dist := int64(oc.rep[i]) + minDistance
		off := head - dist
		if start <= off {
			offs = append(offs, off)
		}
	}
	return offs
}

// errNoMatch indicates that no match could be found
var errNoMatch = newError("no match found")

// bestMatch finds the best match for the given offsets.
//
// TODO: compare all possible commands for compressed bits per encoded bits.
func (bw *Writer) bestMatch(offsets []int64) (m match, err error) {
	oc := bw.OpCodec
	// creates a match for 1
	head := bw.Dict.Offset()
	off := int64(-1)
	length := 0
	for i := len(offsets) - 1; i >= 0; i-- {
		n := bw.Dict.EqualBytes(head, offsets[i], MaxLength)
		if n > length {
			off, length = offsets[i], n
		}
	}
	if off < 0 {
		err = errNoMatch
		return
	}
	if length == 1 {
		dist := int64(oc.rep[0]) + minDistance
		offRep0 := head - dist
		if off != offRep0 {
			err = errNoMatch
			return
		}
	}
	return match{distance: head - off, length: length}, nil
}

// findOp finds an operation for the head of the dictionary.
func (bw *Writer) findOp() (op operation, err error) {
	p := make([]byte, 4)
	n, err := bw.Dict.PeekHead(p)
	if err != nil && err != errAgain && err != io.EOF {
		return nil, err
	}
	if n <= 0 {
		if n < 0 {
			panic("strange n")
		}
		return nil, errEmptyBuf
	}
	offs := bw.potentialOffsets(p[:n])
	m, err := bw.bestMatch(offs)
	if err == errNoMatch {
		return lit{b: p[0]}, nil
	}
	if err != nil {
		return nil, err
	}
	return m, nil
}

// discardOp advances the head of the dictionary and writes the the bytes into
// the hash table.
func (bw *Writer) discardOp(op operation) error {
	n, err := bw.Dict.AdvanceHead(op.Len())
	if err != nil {
		return err
	}
	if n < op.Len() {
		return errAgain
	}
	return nil
}

// process encodes the data written into the dictionary buffer. The allData
// flag requires all data remaining in the buffer to be encoded.
func (bw *Writer) process(flags int) error {
	var lowMark int
	if flags&allData == 0 {
		lowMark = MaxLength - 1
	}
	for bw.Dict.Readable() > lowMark {
		op, err := bw.findOp()
		if err != nil {
			debug.Printf("findOp error %s\n", err)
			return err
		}
		if err = bw.writeOp(op); err != nil {
			return err
		}
		debug.Printf("op %s", op)
		if err = bw.discardOp(op); err != nil {
			return err
		}
	}
	return nil
}

// writeLiteral writes a literal into the operation stream
func (bw *Writer) writeLiteral(l lit) error {
	var err error
	oc := bw.OpCodec
	state, state2, _ := oc.states()
	if err = oc.isMatch[state2].Encode(bw.re, 0); err != nil {
		return err
	}
	litState := oc.litState()
	match := bw.Dict.Byte(int64(oc.rep[0]) + 1)
	err = oc.litCodec.Encode(bw.re, l.b, state, match, litState)
	if err != nil {
		return err
	}
	oc.updateStateLiteral()
	return nil
}

// writeEOS writes the explicit EOS marker
func (bw *Writer) writeEOS() error {
	return bw.writeMatch(match{distance: maxDistance, length: MinLength})
}

func iverson(ok bool) uint32 {
	if ok {
		return 1
	}
	return 0
}

// writeRep writes a repetition operation into the operation stream
func (bw *Writer) writeMatch(m match) error {
	var err error
	oc := bw.OpCodec
	if !(minDistance <= m.distance && m.distance <= maxDistance) {
		return newError("distance out of range")
	}
	dist := uint32(m.distance - minDistance)
	if !(MinLength <= m.length && m.length <= MaxLength) &&
		!(dist == oc.rep[0] && m.length == 1) {
		return newError("length out of range")
	}
	state, state2, posState := oc.states()
	if err = oc.isMatch[state2].Encode(bw.re, 1); err != nil {
		return err
	}
	var g int
	for g = 0; g < 4; g++ {
		if oc.rep[g] == dist {
			break
		}
	}
	b := iverson(g < 4)
	if err = oc.isRep[state].Encode(bw.re, b); err != nil {
		return err
	}
	n := uint32(m.length - MinLength)
	if b == 0 {
		// simple match
		oc.rep[3], oc.rep[2], oc.rep[1], oc.rep[0] = oc.rep[2],
			oc.rep[1], oc.rep[0], dist
		oc.updateStateMatch()
		if err = oc.lenCodec.Encode(bw.re, n, posState); err != nil {
			return err
		}
		return oc.distCodec.Encode(bw.re, dist, n)
	}
	b = iverson(g != 0)
	if err = oc.isRepG0[state].Encode(bw.re, b); err != nil {
		return err
	}
	if b == 0 {
		// g == 0
		b = iverson(m.length != 1)
		if err = oc.isRepG0Long[state2].Encode(bw.re, b); err != nil {
			return err
		}
		if b == 0 {
			oc.updateStateShortRep()
			return nil
		}
	} else {
		// g in {1,2,3}
		b = iverson(g != 1)
		if err = oc.isRepG1[state].Encode(bw.re, b); err != nil {
			return err
		}
		if b == 1 {
			// g in {2,3}
			b = iverson(g != 2)
			err = oc.isRepG2[state].Encode(bw.re, b)
			if err != nil {
				return err
			}
			if b == 1 {
				oc.rep[3] = oc.rep[2]
			}
			oc.rep[2] = oc.rep[1]
		}
		oc.rep[1] = oc.rep[0]
		oc.rep[0] = dist
	}
	oc.updateStateRep()
	return oc.repLenCodec.Encode(bw.re, n, posState)
}

// writeOp writes an operation value into the stream.
func (bw *Writer) writeOp(op operation) error {
	switch x := op.(type) {
	case match:
		return bw.writeMatch(x)
	case lit:
		return bw.writeLiteral(x)
	}
	panic("unknown operation type")
}
<commit_msg>lzbase: added Properties method to Writer<commit_after>package lzbase

import "io"

// Writer allows to generate a LZMA stream, while the OpCodec object can be
// reused.
type Writer struct {
	OpCodec *OpCodec
	Dict    *WriterDict
	re      *rangeEncoder
	params  *Parameters
}

// InitWriter initializes a writer object.
func InitWriter(bw *Writer, w io.Writer, oc *OpCodec, params Parameters) error {
	switch {
	case w == nil:
		return newError("InitWriter argument w is nil")
	case oc == nil:
		return newError("InitWriter argument oc is nil")
	}
	err := verifyParameters(&params)
	if err != nil {
		return err
	}
	dict, ok := oc.dict.(*WriterDict)
	if !ok {
		return newError("op codec for writer expected")
	}
	re := newRangeEncoder(w)
	*bw = Writer{OpCodec: oc, Dict: dict, re: re, params: &params}
	return nil
}

// Write moves data into the internal buffer and triggers its compression. Note
// that beside the data held back to enable a large match all data will be be
// compressed.
func (bw *Writer) Write(p []byte) (n int, err error) {
	end := bw.Dict.end + int64(len(p))
	if end < 0 {
		panic("end counter overflow")
	}
	var rerr error
	if bw.params.SizeInHeader && end > bw.params.Size {
		p = p[:bw.params.Size-end]
		rerr = newError("write exceeds unpackLen")
	}
	for n < len(p) {
		k, err := bw.Dict.Write(p[n:])
		n += k
		if err != nil && err != errAgain {
			return n, err
		}
		if err = bw.process(0); err != nil {
			return n, err
		}
	}
	return n, rerr
}

// Close terminates the LZMA stream. It doesn't close the underlying writer
// though and leaves it alone. In some scenarios explicit closing of the
// underlying writer is required.
func (bw *Writer) Close() error {
	var err error
	if err = bw.process(allData); err != nil {
		return err
	}
	if bw.params.EOS {
		if err = bw.writeEOS(); err != nil {
			return err
		}
	}
	if err = bw.re.Close(); err != nil {
		return err
	}
	return bw.Dict.Close()
}

// The allData flag tells the process method that all data must be processed.
const allData = 1

// indicates an empty buffer
var errEmptyBuf = newError("empty buffer")

// potentialOffsets creates a list of potential offsets for matches.
func (bw *Writer) potentialOffsets(p []byte) []int64 {
	oc := bw.OpCodec
	head := bw.Dict.Offset()
	start := bw.Dict.start
	offs := make([]int64, 0, 32)
	// add potential offsets with highest priority at the top
	for i := 1; i < 11; i++ {
		// distance 1 to 8
		off := head - int64(i)
		if start <= off {
			offs = append(offs, off)
		}
	}
	if len(p) == 4 {
		// distances from the hash table
		offs = append(offs, bw.Dict.Offsets(p)...)
	}
	for i := 3; i >= 0; i-- {
		// distances from the repetition for length less than 4
		dist := int64(oc.rep[i]) + minDistance
		off := head - dist
		if start <= off {
			offs = append(offs, off)
		}
	}
	return offs
}

// errNoMatch indicates that no match could be found
var errNoMatch = newError("no match found")

// bestMatch finds the best match for the given offsets.
//
// TODO: compare all possible commands for compressed bits per encoded bits.
func (bw *Writer) bestMatch(offsets []int64) (m match, err error) {
	oc := bw.OpCodec
	// creates a match for 1
	head := bw.Dict.Offset()
	off := int64(-1)
	length := 0
	for i := len(offsets) - 1; i >= 0; i-- {
		n := bw.Dict.EqualBytes(head, offsets[i], MaxLength)
		if n > length {
			off, length = offsets[i], n
		}
	}
	if off < 0 {
		err = errNoMatch
		return
	}
	if length == 1 {
		dist := int64(oc.rep[0]) + minDistance
		offRep0 := head - dist
		if off != offRep0 {
			err = errNoMatch
			return
		}
	}
	return match{distance: head - off, length: length}, nil
}

// findOp finds an operation for the head of the dictionary.
func (bw *Writer) findOp() (op operation, err error) {
	p := make([]byte, 4)
	n, err := bw.Dict.PeekHead(p)
	if err != nil && err != errAgain && err != io.EOF {
		return nil, err
	}
	if n <= 0 {
		if n < 0 {
			panic("strange n")
		}
		return nil, errEmptyBuf
	}
	offs := bw.potentialOffsets(p[:n])
	m, err := bw.bestMatch(offs)
	if err == errNoMatch {
		return lit{b: p[0]}, nil
	}
	if err != nil {
		return nil, err
	}
	return m, nil
}

// discardOp advances the head of the dictionary and writes the the bytes into
// the hash table.
func (bw *Writer) discardOp(op operation) error {
	n, err := bw.Dict.AdvanceHead(op.Len())
	if err != nil {
		return err
	}
	if n < op.Len() {
		return errAgain
	}
	return nil
}

// process encodes the data written into the dictionary buffer. The allData
// flag requires all data remaining in the buffer to be encoded.
func (bw *Writer) process(flags int) error {
	var lowMark int
	if flags&allData == 0 {
		lowMark = MaxLength - 1
	}
	for bw.Dict.Readable() > lowMark {
		op, err := bw.findOp()
		if err != nil {
			debug.Printf("findOp error %s\n", err)
			return err
		}
		if err = bw.writeOp(op); err != nil {
			return err
		}
		debug.Printf("op %s", op)
		if err = bw.discardOp(op); err != nil {
			return err
		}
	}
	return nil
}

// writeLiteral writes a literal into the operation stream
func (bw *Writer) writeLiteral(l lit) error {
	var err error
	oc := bw.OpCodec
	state, state2, _ := oc.states()
	if err = oc.isMatch[state2].Encode(bw.re, 0); err != nil {
		return err
	}
	litState := oc.litState()
	match := bw.Dict.Byte(int64(oc.rep[0]) + 1)
	err = oc.litCodec.Encode(bw.re, l.b, state, match, litState)
	if err != nil {
		return err
	}
	oc.updateStateLiteral()
	return nil
}

// writeEOS writes the explicit EOS marker
func (bw *Writer) writeEOS() error {
	return bw.writeMatch(match{distance: maxDistance, length: MinLength})
}

func iverson(ok bool) uint32 {
	if ok {
		return 1
	}
	return 0
}

// writeRep writes a repetition operation into the operation stream
func (bw *Writer) writeMatch(m match) error {
	var err error
	oc := bw.OpCodec
	if !(minDistance <= m.distance && m.distance <= maxDistance) {
		return newError("distance out of range")
	}
	dist := uint32(m.distance - minDistance)
	if !(MinLength <= m.length && m.length <= MaxLength) &&
		!(dist == oc.rep[0] && m.length == 1) {
		return newError("length out of range")
	}
	state, state2, posState := oc.states()
	if err = oc.isMatch[state2].Encode(bw.re, 1); err != nil {
		return err
	}
	var g int
	for g = 0; g < 4; g++ {
		if oc.rep[g] == dist {
			break
		}
	}
	b := iverson(g < 4)
	if err = oc.isRep[state].Encode(bw.re, b); err != nil {
		return err
	}
	n := uint32(m.length - MinLength)
	if b == 0 {
		// simple match
		oc.rep[3], oc.rep[2], oc.rep[1], oc.rep[0] = oc.rep[2],
			oc.rep[1], oc.rep[0], dist
		oc.updateStateMatch()
		if err = oc.lenCodec.Encode(bw.re, n, posState); err != nil {
			return err
		}
		return oc.distCodec.Encode(bw.re, dist, n)
	}
	b = iverson(g != 0)
	if err = oc.isRepG0[state].Encode(bw.re, b); err != nil {
		return err
	}
	if b == 0 {
		// g == 0
		b = iverson(m.length != 1)
		if err = oc.isRepG0Long[state2].Encode(bw.re, b); err != nil {
			return err
		}
		if b == 0 {
			oc.updateStateShortRep()
			return nil
		}
	} else {
		// g in {1,2,3}
		b = iverson(g != 1)
		if err = oc.isRepG1[state].Encode(bw.re, b); err != nil {
			return err
		}
		if b == 1 {
			// g in {2,3}
			b = iverson(g != 2)
			err = oc.isRepG2[state].Encode(bw.re, b)
			if err != nil {
				return err
			}
			if b == 1 {
				oc.rep[3] = oc.rep[2]
			}
			oc.rep[2] = oc.rep[1]
		}
		oc.rep[1] = oc.rep[0]
		oc.rep[0] = dist
	}
	oc.updateStateRep()
	return oc.repLenCodec.Encode(bw.re, n, posState)
}

// writeOp writes an operation value into the stream.
func (bw *Writer) writeOp(op operation) error {
	switch x := op.(type) {
	case match:
		return bw.writeMatch(x)
	case lit:
		return bw.writeLiteral(x)
	}
	panic("unknown operation type")
}

// Properties returns the properties of the writer.
func (bw *Writer) Properties() Properties {
	return bw.OpCodec.properties
}
<|endoftext|>


<commit_before>package catalog

import (
	"fmt"
	"net/http"
)

const (
	//HandlerPath - path for catalog handler to register against
	HandlerPath = "/catalog"
)

//Get - function to handle a get request
func Get() func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, req *http.Request) {
		response := `{
			"services": [{
				"id": "5a9b9f22-a08d-11e5-8062-7831c1d4f660",
				"name": "pez-haas",
				"description": "Lease on-demand hardware as a service",
				"metadata":{
            "displayName":"PEZ-HaaS",
            "imageUrl":"http://s12.postimg.org/wt91ic9pp/broker_icon.png",
						"providerDisplayName":"PEZ"
         },
				"bindable": false,
				"plans": [{
					"id": "6a977311-a08d-11e5-8062-7831c1d4f660",
					"name": "m1.small",
					"description": "A small instance of hardware as a service",
					"metadata":{
						"bullets":[
							"96gb memory (min)",
							"2.7 GHz x (4 sockets / 12 cores per)",
							"3TB NFS shared storage",
							"40 TB total local disk",
							"/24 network (on 10.65.x.x pivotal vpn)",
							"ESXi installed"
						]
					}
				}],
				"dashboard_client": {
					"id": "pez-haas-client",
          "secret": "pez-haas-secret",
					"redirect_uri": "https://www.pezapp.io"
				}
			}]
		}`
		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, response)
	}
}
<commit_msg>[#110851312] adding oauth info to catalog json<commit_after>package catalog

import (
	"fmt"
	"net/http"
	"os"

	"github.com/cloudfoundry-community/go-cfenv"
)

const (
	//HandlerPath - path for catalog handler to register against
	HandlerPath = "/catalog"
)

func getDashboardInfo() (clientID string, clientSecret string, redirectURL string) {

	var (
		app          *cfenv.App
		oauthService *cfenv.Service
		err          error
	)

	if app, err = cfenv.Current(); err == nil {
		redirectURL = app.ApplicationURIs[0]
		services := app.Services

		if oauthService, err = services.WithName(os.Getenv("OAUTH_SERVICE_NAME")); err == nil {
			clientID = oauthService.Credentials[os.Getenv("OAUTH_CLIENT_FIELD")].(string)
			clientSecret = oauthService.Credentials[os.Getenv("OAUTH_CLIENT_SECRET_FIELD")].(string)
		}
	}
	return
}

//Get - function to handle a get request
func Get() func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, req *http.Request) {
		clientID, clientSecret, redirectURL := getDashboardInfo()
		response := fmt.Sprintf(`{
			"services": [{
				"id": "5a9b9f22-a08d-11e5-8062-7831c1d4f660",
				"name": "pez-haas",
				"description": "Lease on-demand hardware as a service",
				"metadata":{
            "displayName":"PEZ-HaaS",
            "imageUrl":"http://s12.postimg.org/wt91ic9pp/broker_icon.png",
						"providerDisplayName":"PEZ"
         },
				"bindable": false,
				"plans": [{
					"id": "6a977311-a08d-11e5-8062-7831c1d4f660",
					"name": "m1.small",
					"description": "A small instance of hardware as a service",
					"metadata":{
						"bullets":[
							"96gb memory (min)",
							"2.7 GHz x (4 sockets / 12 cores per)",
							"3TB NFS shared storage",
							"40 TB total local disk",
							"/24 network (on 10.65.x.x pivotal vpn)",
							"ESXi installed"
						]
					}
				}],
				"dashboard_client": {
					"id": %s,
					"secret": %s,
					"redirect_uri": %s
				}
			}]
		}`, clientID, clientSecret, redirectURL)
		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, response)
	}
}
<|endoftext|>


<commit_before>package console

import (
	"fmt"
	"strings"

	"github.com/jasonsoft/log"
)

// Console is an instance of the console logger
type Console struct {
}

func New() *Console {
	return &Console{}
}

// Run starts the logger consuming on the returned channed
func (c *Console) Run() chan<- *log.Entry {
	// in a big high traffic app, set a higher buffer
	ch := make(chan *log.Entry, 3000)

	go func(entries <-chan *log.Entry) {
		/*
			for {
				select {
				case message := <-entries:
					println(message.Message)
					message.Consumed()
				default:
					println("no msg")
				}

			}*/

		var e *log.Entry
		for e = range entries {
			msg := FormatFunc(e)
			println(msg)
			e.Consumed()
		}
	}(ch)

	return ch
}

func FormatFunc(entry *log.Entry) string {
	time := entry.Timestamp.Format("2006-01-02T15:04:05.999")
	level := entry.Level.String()

	strFields := ""
	for key, value := range entry.Fields {
		strFields += key + "=" + value.(string) + " "
	}

	result := fmt.Sprintf("time=\"%s\" level=%s msg=\"%s\" file=%s line=%d ", time, level, entry.Message, entry.File, entry.Line)

	if len(strFields) > 0 {
		result += strings.TrimSpace(strFields)
	}

	return result
}
<commit_msg>remove file and line information<commit_after>package console

import (
	"fmt"
	"strings"

	"github.com/jasonsoft/log"
)

// Console is an instance of the console logger
type Console struct {
}

func New() *Console {
	return &Console{}
}

// Run starts the logger consuming on the returned channed
func (c *Console) Run() chan<- *log.Entry {
	// in a big high traffic app, set a higher buffer
	ch := make(chan *log.Entry, 3000)

	go func(entries <-chan *log.Entry) {
		/*
			for {
				select {
				case message := <-entries:
					println(message.Message)
					message.Consumed()
				default:
					println("no msg")
				}

			}*/

		var e *log.Entry
		for e = range entries {
			msg := FormatFunc(e)
			println(msg)
			e.Consumed()
		}
	}(ch)

	return ch
}

func FormatFunc(entry *log.Entry) string {
	time := entry.Timestamp.Format("2006-01-02T15:04:05.999")
	level := entry.Level.String()

	strFields := ""
	for key, value := range entry.Fields {
		strFields += key + "=" + value.(string) + " "
	}

	result := fmt.Sprintf("time=\"%s\" level=%s msg=\"%s\" ", time, level, entry.Message, entry.File, entry.Line)

	if len(strFields) > 0 {
		result += strings.TrimSpace(strFields)
	}

	return result
}
<|endoftext|>


<commit_before>package distributor

import (
	"net/smtp"
)

func sendEmailMsg(endpoint string, message string) error {
	auth := smtp.PlainAuth("", "tech@sellyx.com", "sTech123", "smtp.gmail.com")

	to := []string{endpoint}
	msg := []byte("To: " + endpoint + "\r\n" + message)
	err := smtp.SendMail("smtp.gmail.com:587", auth, "donotreply@sellyx.com", to, msg)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>Minor changes<commit_after>package distributor

import (
	"net/smtp"
)

func sendEmailMsg(endpoint string, message string) error {
	auth := smtp.PlainAuth("", "email", "sTech123", "smtp.gmail.com")

	to := []string{endpoint}
	msg := []byte("To: " + endpoint + "\r\n" + message)
	err := smtp.SendMail("email", auth, "toemail", to, msg)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Project Gonder.
// Author Supme
// Copyright Supme 2016
// License http://opensource.org/licenses/MIT MIT License
//
//  THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF
//  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
//  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
//  PURPOSE.
//
// Please see the License.txt file for more information.
//
package mailer

import (
	"github.com/supme/gonder/models"
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"fmt"
//	_ "github.com/eaigner/dkim"
	"golang.org/x/net/proxy"
	"io/ioutil"
	"net"
	"net/http"
	"net/smtp"
	"strings"
	"golang.org/x/net/idna"
)

type (
	MailData struct {
		Iface 	     string
		Host         string
		From         string
		From_name    string
		To           string
		To_name      string
		Extra_header string
		Subject      string
		Html         string
		Attachments  []Attachment
		s	     	 proxy.Dialer
		n            net.Dialer
	}

	Attachment struct {
		Location string
		Name     string
	}
)

func (m *MailData) Send() error {
	var smx string
	var mx []*net.MX
	var conn net.Conn

	if m.Iface == "" {
		// default interface
		m.n = net.Dialer{}
	} else {
		if m.Iface[0:8] == "socks://" {
			m.Iface = m.Iface[8:]
			var err error
			m.s, err = proxy.SOCKS5("tcp", m.Iface, nil, proxy.FromEnvironment())
			if err != nil {
				return err
			}
		} else {
			connectAddr := net.ParseIP(m.Iface)
			tcpAddr := &net.TCPAddr{
				IP: connectAddr,
			}
			m.n = net.Dialer{LocalAddr: tcpAddr}
		}
	}

	//ToDo cache MX servers
	// trim space
	m.To = strings.TrimSpace(m.To)
	// punycode convert
	domain, err := idna.ToASCII(strings.Split(m.To, "@")[1])
	if err != nil {
		return errors.New(fmt.Sprintf("Domain name failed: %v\r\n", err))
	}
	m.To = strings.Split(m.To, "@")[0] + "@" + domain

	mx, err = net.LookupMX(domain)
	if err != nil {
		return errors.New(fmt.Sprintf("LookupMX failed: %v\r\n", err))
	} else {
		for i := range mx {
			smx := net.JoinHostPort(mx[i].Host, "25")
			// Set ip (from MX records) and port mail server
			if m.s != nil {
				conn, err = m.s.Dial("tcp", smx)
			} else {
				conn, err = m.n.Dial("tcp", smx)
			}
			if err == nil {
				break
			}
		}
	}
	if err != nil {
		return err
	}
	defer conn.Close()

	host, _, _ := net.SplitHostPort(smx)
	c, err := smtp.NewClient(conn, host)
	if err != nil {
		return err
	}

	if err := c.Hello(m.Host); err != nil {
		return err
	}

	// Set the sender and recipient first
	if err := c.Mail(m.From); err != nil {
		return err
	}

	if err := c.Rcpt(m.To); err != nil {
		return err
	}

	msg := m.makeMail()

	//dkim.New()

	w, err := c.Data()
	if err != nil {
		return err
	}
	_, err = fmt.Fprint(w, msg)
	if err != nil {
		return err
	}

	err = w.Close()
	if err != nil {
		return err
	}

	// Send the QUIT command and close the connection.
	return c.Quit()
}

func (m *MailData) makeMail() (msg string) {
	marker := makeMarker()

	msg = ""

	if m.From_name == "" {
		msg += `From: ` + m.From + "\r\n"
	} else {
		msg += `From: "` + encodeRFC2047(m.From_name) + `" <` + m.From + `>` +"\r\n"
	}

	if m.To_name == "" {
		msg += `To: ` + m.To + "\r\n"
	} else {
		msg += `To: "` + encodeRFC2047(m.To_name) + `" <` + m.To + `>` + "\r\n"
	}

	// -------------- head ----------------------------------------------------------

	msg += "Subject: " + encodeRFC2047(m.Subject) + "\r\n"
	msg += "MIME-Version: 1.0\r\n"
	msg += "Content-Type: multipart/mixed;\r\n	boundary=\"" + marker + "\"\r\n"
	msg += "X-Mailer: " + models.Version + "\r\n"
	msg += m.Extra_header + "\r\n"
	// ------------- /head ---------------------------------------------------------

	// ------------- body ----------------------------------------------------------
	msg += "\r\n--" + marker + "\r\nContent-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: 8bit\r\n\r\n"
	msg += m.Html
	// ------------ /body ---------------------------------------------------------

	// ----------- attachments ----------------------------------------------------
	for _, file := range m.Attachments {

		msg += "\r\n--" + marker
		//read and encode attachment
		content, _ := ioutil.ReadFile(file.Location + file.Name)
		encoded := base64.StdEncoding.EncodeToString(content)

		//split the encoded file in lines (doesn't matter, but low enough not to hit a max limit)
		lineMaxLength := 500
		nbrLines := len(encoded) / lineMaxLength

		//append lines to buffer
		var buf bytes.Buffer
		for i := 0; i < nbrLines; i++ {
			buf.WriteString(encoded[i*lineMaxLength:(i+1)*lineMaxLength] + "\n")
		} //for

		//append last line in buffer
		buf.WriteString(encoded[nbrLines*lineMaxLength:])

		//part 3 will be the attachment
		msg += fmt.Sprintf("\r\nContent-Type: %s;\r\n	name=\"%s\"\r\nContent-Transfer-Encoding: base64\r\nContent-Disposition: attachment;\r\n	filename=\"%s\"\r\n\r\n%s\r\n", http.DetectContentType(content), file.Name, file.Name, buf.String())
	}
	// ----------- /attachments ---------------------------------------------------

	return
}

func makeMarker() string {
	b := make([]byte, 30)
	rand.Read(b)
	en := base64.StdEncoding // or URLEncoding
	d := make([]byte, en.EncodedLen(len(b)))
	en.Encode(d, b)
	return string(d)
}

func encodeRFC2047(s string) string {
	// use code from net/mail for rfc2047 encode any string
	// UTF-8 "Q" encoding
	b := bytes.NewBufferString("=?utf-8?q?")
	for i := 0; i < len(s); i++ {
		switch c := s[i]; {
		case c == ' ':
			b.WriteByte('_')
		case isVchar(c) && c != '=' && c != '?' && c != '_':
			b.WriteByte(c)
		default:
			fmt.Fprintf(b, "=%02X", c)
		}
	}
	b.WriteString("?= ")
	return b.String()
}

// isVchar returns true if c is an RFC 5322 VCHAR character.
func isVchar(c byte) bool {
	// Visible (printing) characters.
	return '!' <= c && c <= '~'
}
<commit_msg>Bug fix: crush on bad email address<commit_after>// Project Gonder.
// Author Supme
// Copyright Supme 2016
// License http://opensource.org/licenses/MIT MIT License
//
//  THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF
//  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
//  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
//  PURPOSE.
//
// Please see the License.txt file for more information.
//
package mailer

import (
	"github.com/supme/gonder/models"
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"errors"
	"fmt"
//	_ "github.com/eaigner/dkim"
	"golang.org/x/net/proxy"
	"io/ioutil"
	"net"
	"net/http"
	"net/smtp"
	"strings"
	"golang.org/x/net/idna"
)

type (
	MailData struct {
		Iface 	     string
		Host         string
		From         string
		From_name    string
		To           string
		To_name      string
		Extra_header string
		Subject      string
		Html         string
		Attachments  []Attachment
		s	     	 proxy.Dialer
		n            net.Dialer
	}

	Attachment struct {
		Location string
		Name     string
	}
)

func (m *MailData) Send() error {
	var smx string
	var mx []*net.MX
	var conn net.Conn

	if m.Iface == "" {
		// default interface
		m.n = net.Dialer{}
	} else {
		if m.Iface[0:8] == "socks://" {
			m.Iface = m.Iface[8:]
			var err error
			m.s, err = proxy.SOCKS5("tcp", m.Iface, nil, proxy.FromEnvironment())
			if err != nil {
				return err
			}
		} else {
			connectAddr := net.ParseIP(m.Iface)
			tcpAddr := &net.TCPAddr{
				IP: connectAddr,
			}
			m.n = net.Dialer{LocalAddr: tcpAddr}
		}
	}

	//ToDo cache MX servers
	// trim space
	m.To = strings.TrimSpace(m.To)
	// punycode convert
	splitEmail := strings.Split(m.To, "@")
	if len(splitEmail) != 2 {
		return errors.New(fmt.Sprintf("Bad email"))
	}
	domain, err := idna.ToASCII(splitEmail[1])
	if err != nil {
		return errors.New(fmt.Sprintf("Domain name failed: %v\r\n", err))
	}
	m.To = strings.Split(m.To, "@")[0] + "@" + domain

	mx, err = net.LookupMX(domain)
	if err != nil {
		return errors.New(fmt.Sprintf("LookupMX failed: %v\r\n", err))
	} else {
		for i := range mx {
			smx := net.JoinHostPort(mx[i].Host, "25")
			// Set ip (from MX records) and port mail server
			if m.s != nil {
				conn, err = m.s.Dial("tcp", smx)
			} else {
				conn, err = m.n.Dial("tcp", smx)
			}
			if err == nil {
				break
			}
		}
	}
	if err != nil {
		return err
	}
	defer conn.Close()

	host, _, _ := net.SplitHostPort(smx)
	c, err := smtp.NewClient(conn, host)
	if err != nil {
		return err
	}

	if err := c.Hello(m.Host); err != nil {
		return err
	}

	// Set the sender and recipient first
	if err := c.Mail(m.From); err != nil {
		return err
	}

	if err := c.Rcpt(m.To); err != nil {
		return err
	}

	msg := m.makeMail()

	//dkim.New()

	w, err := c.Data()
	if err != nil {
		return err
	}
	_, err = fmt.Fprint(w, msg)
	if err != nil {
		return err
	}

	err = w.Close()
	if err != nil {
		return err
	}

	// Send the QUIT command and close the connection.
	return c.Quit()
}

func (m *MailData) makeMail() (msg string) {
	marker := makeMarker()

	msg = ""

	if m.From_name == "" {
		msg += `From: ` + m.From + "\r\n"
	} else {
		msg += `From: "` + encodeRFC2047(m.From_name) + `" <` + m.From + `>` +"\r\n"
	}

	if m.To_name == "" {
		msg += `To: ` + m.To + "\r\n"
	} else {
		msg += `To: "` + encodeRFC2047(m.To_name) + `" <` + m.To + `>` + "\r\n"
	}

	// -------------- head ----------------------------------------------------------

	msg += "Subject: " + encodeRFC2047(m.Subject) + "\r\n"
	msg += "MIME-Version: 1.0\r\n"
	msg += "Content-Type: multipart/mixed;\r\n	boundary=\"" + marker + "\"\r\n"
	msg += "X-Mailer: " + models.Version + "\r\n"
	msg += m.Extra_header + "\r\n"
	// ------------- /head ---------------------------------------------------------

	// ------------- body ----------------------------------------------------------
	msg += "\r\n--" + marker + "\r\nContent-Type: text/html; charset=\"utf-8\"\r\nContent-Transfer-Encoding: 8bit\r\n\r\n"
	msg += m.Html
	// ------------ /body ---------------------------------------------------------

	// ----------- attachments ----------------------------------------------------
	for _, file := range m.Attachments {

		msg += "\r\n--" + marker
		//read and encode attachment
		content, _ := ioutil.ReadFile(file.Location + file.Name)
		encoded := base64.StdEncoding.EncodeToString(content)

		//split the encoded file in lines (doesn't matter, but low enough not to hit a max limit)
		lineMaxLength := 500
		nbrLines := len(encoded) / lineMaxLength

		//append lines to buffer
		var buf bytes.Buffer
		for i := 0; i < nbrLines; i++ {
			buf.WriteString(encoded[i*lineMaxLength:(i+1)*lineMaxLength] + "\n")
		} //for

		//append last line in buffer
		buf.WriteString(encoded[nbrLines*lineMaxLength:])

		//part 3 will be the attachment
		msg += fmt.Sprintf("\r\nContent-Type: %s;\r\n	name=\"%s\"\r\nContent-Transfer-Encoding: base64\r\nContent-Disposition: attachment;\r\n	filename=\"%s\"\r\n\r\n%s\r\n", http.DetectContentType(content), file.Name, file.Name, buf.String())
	}
	// ----------- /attachments ---------------------------------------------------

	return
}

func makeMarker() string {
	b := make([]byte, 30)
	rand.Read(b)
	en := base64.StdEncoding // or URLEncoding
	d := make([]byte, en.EncodedLen(len(b)))
	en.Encode(d, b)
	return string(d)
}

func encodeRFC2047(s string) string {
	// use code from net/mail for rfc2047 encode any string
	// UTF-8 "Q" encoding
	b := bytes.NewBufferString("=?utf-8?q?")
	for i := 0; i < len(s); i++ {
		switch c := s[i]; {
		case c == ' ':
			b.WriteByte('_')
		case isVchar(c) && c != '=' && c != '?' && c != '_':
			b.WriteByte(c)
		default:
			fmt.Fprintf(b, "=%02X", c)
		}
	}
	b.WriteString("?= ")
	return b.String()
}

// isVchar returns true if c is an RFC 5322 VCHAR character.
func isVchar(c byte) bool {
	// Visible (printing) characters.
	return '!' <= c && c <= '~'
}
<|endoftext|>


<commit_before>// Alexandria
//
// Copyright (C) 2015  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package alexandria

import (
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/blevesearch/bleve"
)

func touch(file string) error {
	now := time.Now()
	return os.Chtimes(file, now, now)
}

// Run index++ to generate a (new) swish++ index file.
func GenerateIndex() error {
	// Try to open an existing index or create a new one if none exists.
	index, err := openIndex()
	if err != nil {
		mapping := bleve.NewIndexMapping()
		index, err = bleve.New(Config.AlexandriaDirectory+"bleve", mapping)
		if err != nil {
			return err
		}
	}
	defer index.Close()

	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		return err
	}

	indexUpdateFile := Config.AlexandriaDirectory + "index_updated"
	indexUpdateTime, err := getModTime(indexUpdateFile)
	if err != nil {
		LogError(err)
		return nil
	}
	// Save the time of this indexing operation
	_ = touch(Config.AlexandriaDirectory + "index_updated")
	for _, file := range files {
		// Check whether the scroll is newer than the index.
		modTime, err := getModTime(Config.KnowledgeDirectory + file.Name())
		if err != nil {
			LogError(err)
			continue
		}
		id := strings.TrimSuffix(file.Name(), ".tex")
		if modTime < indexUpdateTime {
			continue
		}

		// Load and parse the scroll content
		contentBytes, err := ioutil.ReadFile(Config.KnowledgeDirectory + file.Name())
		TryLogError(err)
		content := string(contentBytes)
		metadata := ParseMetadata(content)
		scroll := Scroll{metadata, Id(id), content}

		index.Index(id, scroll)
	}

	return nil
}

// Open the bleve index
func openIndex() (bleve.Index, error) {
	return bleve.Open(Config.AlexandriaDirectory + "bleve")
}

// Search the swish index for a given query.
func searchBleve(queryString string) ([]Id, error) {
	index, err := openIndex()
	if err != nil {
		LogError(err)
		return nil, err
	}
	defer index.Close()

	query := bleve.NewQueryStringQuery(queryString)
	search := bleve.NewSearchRequest(query)
	searchResults, err := index.Search(search)
	if err != nil {
		LogError(err)
		return nil, err
	}

	results := make([]Id, Config.MaxResults)
	for i, match := range searchResults.Hits {
		results[i] = Id(match.ID)
	}

	return results[:len(searchResults.Hits)], nil
}

// Get a list of scrolls matching the query.
func FindScrolls(query string) ([]Id, error) {
	ids, err := searchBleve(query)
	if err != nil {
		return nil, err
	}
	ProcessScrolls(ids)

	return ids, nil
}

func ComputeStatistics() Statistics {
	index, err := openIndex()
	if err != nil {
		LogError(err)
	}
	defer index.Close()

	num, size := getDirSize(Config.KnowledgeDirectory)
	if err == nil {
		tmp, err := index.DocCount()
		if err != nil {
			LogError(err)
		} else {
			num = int(tmp)
		}
	}

	return Stats{num, size}
}
<commit_msg>Use batch processing to speed up indexing<commit_after>// Alexandria
//
// Copyright (C) 2015  Colin Benner
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package alexandria

import (
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/blevesearch/bleve"
)

func touch(file string) error {
	now := time.Now()
	return os.Chtimes(file, now, now)
}

// Run index++ to generate a (new) swish++ index file.
func GenerateIndex() error {
	// Try to open an existing index or create a new one if none exists.
	index, err := openIndex()
	if err != nil {
		mapping := bleve.NewIndexMapping()
		index, err = bleve.New(Config.AlexandriaDirectory+"bleve", mapping)
		if err != nil {
			return err
		}
	}
	defer index.Close()

	files, err := ioutil.ReadDir(Config.KnowledgeDirectory)
	if err != nil {
		return err
	}

	indexUpdateFile := Config.AlexandriaDirectory + "index_updated"
	indexUpdateTime, err := getModTime(indexUpdateFile)
	if err != nil {
		LogError(err)
		return nil
	}
	// Save the time of this indexing operation
	_ = touch(Config.AlexandriaDirectory + "index_updated")

	batch := index.NewBatch()
	for _, file := range files {
		// Check whether the scroll is newer than the index.
		modTime, err := getModTime(Config.KnowledgeDirectory + file.Name())
		if err != nil {
			LogError(err)
			continue
		}
		id := strings.TrimSuffix(file.Name(), ".tex")
		if modTime < indexUpdateTime {
			continue
		}

		// Load and parse the scroll content
		contentBytes, err := ioutil.ReadFile(Config.KnowledgeDirectory + file.Name())
		TryLogError(err)
		content := string(contentBytes)
		metadata := ParseMetadata(content)
		scroll := Scroll{metadata, Id(id), content}

		batch.Index(id, scroll)
	}
	index.Batch(batch)

	return nil
}

// Open the bleve index
func openIndex() (bleve.Index, error) {
	return bleve.Open(Config.AlexandriaDirectory + "bleve")
}

// Search the swish index for a given query.
func searchBleve(queryString string) ([]Id, error) {
	index, err := openIndex()
	if err != nil {
		LogError(err)
		return nil, err
	}
	defer index.Close()

	query := bleve.NewQueryStringQuery(queryString)
	search := bleve.NewSearchRequest(query)
	searchResults, err := index.Search(search)
	if err != nil {
		LogError(err)
		return nil, err
	}

	results := make([]Id, Config.MaxResults)
	for i, match := range searchResults.Hits {
		results[i] = Id(match.ID)
	}

	return results[:len(searchResults.Hits)], nil
}

// Get a list of scrolls matching the query.
func FindScrolls(query string) ([]Id, error) {
	ids, err := searchBleve(query)
	if err != nil {
		return nil, err
	}
	ProcessScrolls(ids)

	return ids, nil
}

func ComputeStatistics() Statistics {
	index, err := openIndex()
	if err != nil {
		LogError(err)
	}
	defer index.Close()

	num, size := getDirSize(Config.KnowledgeDirectory)
	if err == nil {
		tmp, err := index.DocCount()
		if err != nil {
			LogError(err)
		} else {
			num = int(tmp)
		}
	}

	return Stats{num, size}
}
<|endoftext|>


<commit_before>package matchers_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/matchers"
)

var _ = Describe("MatchJSONMatcher", func() {
	Context("When passed stringifiables", func() {
		It("should succeed if the JSON matches", func() {
			Ω("{}").Should(MatchJSON("{}"))
			Ω(`{"a":1}`).Should(MatchJSON(`{"a":1}`))
			Ω(`{
			             "a":1
			         }`).Should(MatchJSON(`{"a":1}`))
			Ω(`{"a":1, "b":2}`).Should(MatchJSON(`{"b":2, "a":1}`))
			Ω(`{"a":1}`).ShouldNot(MatchJSON(`{"b":2, "a":1}`))
		})

		It("should work with byte arrays", func() {
			Ω([]byte("{}")).Should(MatchJSON([]byte("{}")))
			Ω("{}").Should(MatchJSON([]byte("{}")))
			Ω([]byte("{}")).Should(MatchJSON("{}"))
		})
	})

	Context("when the expected is not valid JSON", func() {
		It("should error and explain why", func() {
			success, err := (&MatchJSONMatcher{JSONToMatch: `{}`}).Match(`oops`)
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("Actual 'oops' should be valid JSON"))
		})
	})

	Context("when the actual is not valid JSON", func() {
		It("should error and explain why", func() {
			success, err := (&MatchJSONMatcher{JSONToMatch: `oops`}).Match(`{}`)
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("Expected 'oops' should be valid JSON"))
		})
	})

	Context("when the expected is neither a string nor a stringer or a byte array", func() {
		It("should error", func() {
			success, err := (&MatchJSONMatcher{JSONToMatch: 2}).Match("{}")
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got expected:\n    <int>: 2"))

			success, err = (&MatchJSONMatcher{JSONToMatch: nil}).Match("{}")
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got expected:\n    <nil>: nil"))
		})
	})

	Context("when the actual is neither a string nor a stringer", func() {
		It("should error", func() {
			success, err := (&MatchJSONMatcher{JSONToMatch: "{}"}).Match(2)
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got actual:\n    <int>: 2"))

			success, err = (&MatchJSONMatcher{JSONToMatch: "{}"}).Match(nil)
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got actual:\n    <nil>: nil"))
		})
	})
})
<commit_msg>Update test description for match json tests.<commit_after>package matchers_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/matchers"
)

var _ = Describe("MatchJSONMatcher", func() {
	Context("When passed stringifiables", func() {
		It("should succeed if the JSON matches", func() {
			Ω("{}").Should(MatchJSON("{}"))
			Ω(`{"a":1}`).Should(MatchJSON(`{"a":1}`))
			Ω(`{
			             "a":1
			         }`).Should(MatchJSON(`{"a":1}`))
			Ω(`{"a":1, "b":2}`).Should(MatchJSON(`{"b":2, "a":1}`))
			Ω(`{"a":1}`).ShouldNot(MatchJSON(`{"b":2, "a":1}`))
		})

		It("should work with byte arrays", func() {
			Ω([]byte("{}")).Should(MatchJSON([]byte("{}")))
			Ω("{}").Should(MatchJSON([]byte("{}")))
			Ω([]byte("{}")).Should(MatchJSON("{}"))
		})
	})

	Context("when the expected is not valid JSON", func() {
		It("should error and explain why", func() {
			success, err := (&MatchJSONMatcher{JSONToMatch: `{}`}).Match(`oops`)
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("Actual 'oops' should be valid JSON"))
		})
	})

	Context("when the actual is not valid JSON", func() {
		It("should error and explain why", func() {
			success, err := (&MatchJSONMatcher{JSONToMatch: `oops`}).Match(`{}`)
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("Expected 'oops' should be valid JSON"))
		})
	})

	Context("when the expected is neither a string nor a stringer nor a byte array", func() {
		It("should error", func() {
			success, err := (&MatchJSONMatcher{JSONToMatch: 2}).Match("{}")
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got expected:\n    <int>: 2"))

			success, err = (&MatchJSONMatcher{JSONToMatch: nil}).Match("{}")
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got expected:\n    <nil>: nil"))
		})
	})

	Context("when the actual is neither a string nor a stringer nor a byte array", func() {
		It("should error", func() {
			success, err := (&MatchJSONMatcher{JSONToMatch: "{}"}).Match(2)
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got actual:\n    <int>: 2"))

			success, err = (&MatchJSONMatcher{JSONToMatch: "{}"}).Match(nil)
			Ω(success).Should(BeFalse())
			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(ContainSubstring("MatchJSONMatcher matcher requires a string, stringer, or []byte.  Got actual:\n    <nil>: nil"))
		})
	})
})
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package git

import (
	"context"
	"fmt"
	"strings"
)

// Log returns the top count ChangeList at HEAD.
func (g Git) Log(ctx context.Context, count int) ([]ChangeList, error) {
	return g.LogFrom(ctx, "HEAD", count)
}

// LogFrom returns the top count ChangeList starting from at.
func (g Git) LogFrom(ctx context.Context, at string, count int) ([]ChangeList, error) {
	if at == "" {
		at = "HEAD"
	}
	str, _, err := g.run(ctx, "log", at, "--pretty=format:ǁ%Hǀ%an <%ae>ǀ%sǀ%b", fmt.Sprintf("-%d", count), g.wd)
	if err != nil {
		return nil, err
	}
	return parseLog(str)
}

// Parent returns the parent ChangeList for cl.
func (g Git) Parent(ctx context.Context, cl ChangeList) (ChangeList, error) {
	str, _, err := g.run(ctx, "log", "--pretty=format:ǁ%Hǀ%an <%ae>ǀ%sǀ%b", fmt.Sprintf("%v^", cl.SHA))
	if err != nil {
		return ChangeList{}, err
	}
	cls, err := parseLog(str)
	if err != nil {
		return ChangeList{}, err
	}
	if len(cls) == 0 {
		return ChangeList{}, fmt.Errorf("Unexpected output")
	}
	return cls[0], nil
}

// HeadCL returns the ChangeList at HEAD.
func (g Git) HeadCL(ctx context.Context) (ChangeList, error) {
	cls, err := g.Log(ctx, 1)
	if err != nil {
		return ChangeList{}, err
	}
	return cls[0], nil
}

func parseLog(str string) ([]ChangeList, error) {
	msgs := strings.Split(str, "ǁ")
	cls := make([]ChangeList, 0, len(msgs))
	for _, s := range msgs {
		if parts := strings.Split(s, "ǀ"); len(parts) == 4 {
			cl := ChangeList{
				Author:      strings.TrimSpace(parts[1]),
				Subject:     strings.TrimSpace(parts[2]),
				Description: strings.TrimSpace(parts[3]),
			}
			if err := cl.SHA.Parse(parts[0]); err != nil {
				return nil, err
			}
			cls = append(cls, cl)
		}
	}
	return cls, nil
}
<commit_msg>Fix an index out of bounds panic.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package git

import (
	"context"
	"fmt"
	"strings"
)

// Log returns the top count ChangeList at HEAD.
func (g Git) Log(ctx context.Context, count int) ([]ChangeList, error) {
	return g.LogFrom(ctx, "HEAD", count)
}

// LogFrom returns the top count ChangeList starting from at.
func (g Git) LogFrom(ctx context.Context, at string, count int) ([]ChangeList, error) {
	if at == "" {
		at = "HEAD"
	}
	str, _, err := g.run(ctx, "log", at, "--pretty=format:ǁ%Hǀ%an <%ae>ǀ%sǀ%b", fmt.Sprintf("-%d", count), g.wd)
	if err != nil {
		return nil, err
	}
	return parseLog(str)
}

// Parent returns the parent ChangeList for cl.
func (g Git) Parent(ctx context.Context, cl ChangeList) (ChangeList, error) {
	str, _, err := g.run(ctx, "log", "--pretty=format:ǁ%Hǀ%an <%ae>ǀ%sǀ%b", fmt.Sprintf("%v^", cl.SHA))
	if err != nil {
		return ChangeList{}, err
	}
	cls, err := parseLog(str)
	if err != nil {
		return ChangeList{}, err
	}
	if len(cls) == 0 {
		return ChangeList{}, fmt.Errorf("Unexpected output")
	}
	return cls[0], nil
}

// HeadCL returns the ChangeList at HEAD.
func (g Git) HeadCL(ctx context.Context) (ChangeList, error) {
	cls, err := g.Log(ctx, 1)
	if err != nil {
		return ChangeList{}, err
	}
	if len(cls) == 0 {
		return ChangeList{}, fmt.Errorf("No commits found")
	}
	return cls[0], nil
}

func parseLog(str string) ([]ChangeList, error) {
	msgs := strings.Split(str, "ǁ")
	cls := make([]ChangeList, 0, len(msgs))
	for _, s := range msgs {
		if parts := strings.Split(s, "ǀ"); len(parts) == 4 {
			cl := ChangeList{
				Author:      strings.TrimSpace(parts[1]),
				Subject:     strings.TrimSpace(parts[2]),
				Description: strings.TrimSpace(parts[3]),
			}
			if err := cl.SHA.Parse(parts[0]); err != nil {
				return nil, err
			}
			cls = append(cls, cl)
		}
	}
	return cls, nil
}
<|endoftext|>


<commit_before>package mapper

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"reflect"
	"strings"
	"unsafe"

	"github.com/jonbodner/proteus/logger"
)

func ptrConverter(c context.Context, isPtr bool, sType reflect.Type, out reflect.Value, err error) (interface{}, error) {
	if err != nil {
		return nil, err
	}
	if isPtr {
		out2 := reflect.New(sType)
		k := out.Type().Kind()
		logger.Log(c, logger.DEBUG, fmt.Sprintln("kind of out", k))
		if (k == reflect.Interface || k == reflect.Ptr) && out.IsNil() {
			out2 = reflect.NewAt(sType, unsafe.Pointer(nil))
		} else {
			out2.Elem().Set(out)
		}
		return out2.Interface(), nil
	}
	k := out.Type().Kind()
	if (k == reflect.Ptr || k == reflect.Interface) && out.IsNil() {
		return nil, fmt.Errorf("Attempting to return nil for non-pointer type %v", sType)
	}
	return out.Interface(), nil
}

func MakeBuilder(c context.Context, sType reflect.Type) (Builder, error) {
	if sType == nil {
		return nil, errors.New("sType cannot be nil")
	}

	isPtr := false
	if sType.Kind() == reflect.Ptr {
		sType = sType.Elem()
		isPtr = true
	}

	//only going to handle scalar types here
	if sType.Kind() == reflect.Slice {
		sType = sType.Elem()
	}

	if sType.Kind() == reflect.Map {
		if sType.Key().Kind() != reflect.String {
			return nil, errors.New("Only maps with string keys are supported")
		}
		return func(cols []string, vals []interface{}) (interface{}, error) {
			out, err := buildMap(c, sType, cols, vals)
			return ptrConverter(c, isPtr, sType, out, err)
		}, nil
	} else if sType.Kind() == reflect.Struct {
		//build map of col names to field names (makes this 2N instead of N^2)
		colFieldMap := map[string]fieldInfo{}
		buildColFieldMap(sType, fieldInfo{}, colFieldMap)
		return func(cols []string, vals []interface{}) (interface{}, error) {
			out, err := buildStruct(c, sType, cols, vals, colFieldMap)
			return ptrConverter(c, isPtr, sType, out, err)
		}, nil

	} else {
		// assume primitive
		return func(cols []string, vals []interface{}) (interface{}, error) {
			out, err := buildPrimitive(c, sType, cols, vals)
			return ptrConverter(c, isPtr, sType, out, err)
		}, nil
	}
}

func buildColFieldMap(sType reflect.Type, parentFieldInfo fieldInfo, colFieldMap map[string]fieldInfo) {
	for i := 0; i < sType.NumField(); i++ {
		sf := sType.Field(i)
		if tagVal := sf.Tag.Get("prof"); tagVal != "" {
			childFieldInfo := fieldInfo{
				name:      append([]string(nil), parentFieldInfo.name...),
				fieldType: append([]reflect.Type(nil), parentFieldInfo.fieldType...),
				pos:       append([]int(nil), parentFieldInfo.pos...),
			}
			childFieldInfo.name = append(childFieldInfo.name, sf.Name)
			childFieldInfo.fieldType = append(childFieldInfo.fieldType, sf.Type)
			childFieldInfo.pos = append(childFieldInfo.pos, i)
			//if this is a struct, recurse
			// we are going to use an prof value as a prefix
			// then we go through each of the fields in the struct
			// if any of them aren't structs and have prof tags, we
			// prepend the parent struct and store off a fieldi
			// only if this doesn't implement a scanner. If it does, then go with the scanner
			// another special case: time.Time isn't recursed into
			fmt.Println(sf.Type.Name())
			fmt.Println(sf.Type.PkgPath())
			if sf.Type.Kind() == reflect.Struct && !reflect.PtrTo(sf.Type).Implements(scannerType) && sf.Type.Name() != "Time" && sf.Type.PkgPath() != "time"  {
				buildColFieldMap(sf.Type, childFieldInfo, colFieldMap)
			} else {
				colFieldMap[strings.SplitN(tagVal, ",", 2)[0]] = childFieldInfo
			}
			continue
		}

		//last chance, if it's an anonymous struct, you can recurse into it without having a prof tag
		if sf.Type.Kind() == reflect.Struct && sf.Anonymous {
			childFieldInfo := fieldInfo{
				name:      append([]string(nil), parentFieldInfo.name...),
				fieldType: append([]reflect.Type(nil), parentFieldInfo.fieldType...),
				pos:       append([]int(nil), parentFieldInfo.pos...),
			}
			childFieldInfo.name = append(childFieldInfo.name, "")
			childFieldInfo.fieldType = append(childFieldInfo.fieldType, sf.Type)
			childFieldInfo.pos = append(childFieldInfo.pos, i)
			buildColFieldMap(sf.Type, childFieldInfo, colFieldMap)
		}
	}
}

type Builder func(cols []string, vals []interface{}) (interface{}, error)

type fieldInfo struct {
	name      []string
	fieldType []reflect.Type
	pos       []int
}

func buildMap(c context.Context, sType reflect.Type, cols []string, vals []interface{}) (reflect.Value, error) {
	out := reflect.MakeMap(sType)
	for k, v := range cols {
		curVal := vals[k]
		rv := reflect.ValueOf(curVal)
		if rv.Elem().IsNil() {
			continue
		}
		if rv.Elem().Elem().Type().ConvertibleTo(sType.Elem()) {
			out.SetMapIndex(reflect.ValueOf(v), rv.Elem().Elem().Convert(sType.Elem()))
		} else {
			return out, fmt.Errorf("Unable to assign value %v of type %v to map value of type %v with key %s", rv.Elem().Elem(), rv.Elem().Elem().Type(), sType.Elem(), v)
		}
	}
	return out, nil
}

var (
	scannerType = reflect.TypeOf((*sql.Scanner)(nil)).Elem()
)

func buildStruct(c context.Context, sType reflect.Type, cols []string, vals []interface{}, colFieldMap map[string]fieldInfo) (reflect.Value, error) {
	logger.Log(c, logger.DEBUG, fmt.Sprintf("sType: %s cols: %v vals: %+v colFieldMap: %+v", sType, cols, vals, colFieldMap))
	out := reflect.New(sType).Elem()
	for k, v := range cols {
		if sf, ok := colFieldMap[v]; ok {
			curVal := vals[k]
			rv := reflect.ValueOf(curVal)
			err := buildStructInner(c, sType, out, sf, curVal, rv, 0)
			if err != nil {
				return out, err
			}
		}
	}
	return out, nil
}

func buildStructInner(c context.Context, sType reflect.Type, out reflect.Value, sf fieldInfo, curVal interface{}, rv reflect.Value, depth int) error {
	field := out.Field(sf.pos[depth])
	curFieldType := sf.fieldType[depth]
	if curFieldType.Kind() == reflect.Ptr {
		logger.Log(c, logger.DEBUG, fmt.Sprintln("isPtr", sf, rv, rv.Type(), rv.Elem(), curVal, sType))
		if rv.Elem().IsNil() {
			logger.Log(c, logger.DEBUG, fmt.Sprintln("nil", sf, rv, curVal, sType))
			return nil
		}
		logger.Log(c, logger.DEBUG, fmt.Sprintln("isPtr Not Nil", rv.Elem().Type()))
		if curFieldType.Implements(scannerType) {
			toScan := (reflect.New(curFieldType).Elem().Interface()).(sql.Scanner)
			err := toScan.Scan(rv.Elem().Elem().Interface())
			if err != nil {
				return err
			}
			field.Set(reflect.ValueOf(toScan))
		} else if rv.Elem().Elem().Type().ConvertibleTo(curFieldType.Elem()) {
			field.Set(reflect.New(curFieldType.Elem()))
			field.Elem().Set(rv.Elem().Elem().Convert(curFieldType.Elem()))
		} else {
			logger.Log(c, logger.ERROR, fmt.Sprintln("can't find the field"))
			return fmt.Errorf("Unable to assign pointer to value %v of type %v to struct field %s of type %v", rv.Elem().Elem(), rv.Elem().Elem().Type(), sf.name[depth], curFieldType)
		}
	} else {
		if reflect.PtrTo(curFieldType).Implements(scannerType) {
			toScan := (reflect.New(curFieldType).Interface()).(sql.Scanner)
			err := toScan.Scan(rv.Elem().Interface())
			if err != nil {
				return err
			}
			field.Set(reflect.ValueOf(toScan).Elem())
		} else if rv.Elem().IsNil() {
			logger.Log(c, logger.ERROR, fmt.Sprintln("Attempting to assign a nil to a non-pointer field"))
			return fmt.Errorf("Unable to assign nil value to non-pointer struct field %s of type %v", sf.name[depth], curFieldType)
		} else if field.Kind() == reflect.Struct && field.Type().Name() != "Time" && field.Type().PkgPath() != "time" {
			err := buildStructInner(c, field.Type(), field, sf, curVal, rv, depth+1)
			if err != nil {
				return err
			}
		} else if rv.Elem().Elem().Type().ConvertibleTo(curFieldType) {
			field.Set(rv.Elem().Elem().Convert(curFieldType))
		} else {
			logger.Log(c, logger.ERROR, fmt.Sprintln("can't find the field"))
			return fmt.Errorf("Unable to assign value %v of type %v to struct field %s of type %v", rv.Elem().Elem(), rv.Elem().Elem().Type(), sf.name[depth], curFieldType)
		}
	}
	return nil
}

func buildPrimitive(c context.Context, sType reflect.Type, cols []string, vals []interface{}) (reflect.Value, error) {
	out := reflect.New(sType).Elem()
	//vals[0] is of type *interface{}, because everything in vals is of type *interface{}
	rv := reflect.ValueOf(vals[0])
	if rv.Elem().IsNil() {
		return rv.Elem(), nil
	}
	if rv.Elem().Elem().Type().ConvertibleTo(sType) {
		out.Set(rv.Elem().Elem().Convert(sType))
	} else {
		return out, fmt.Errorf("Unable to assign value %v of type %v to return type of type %v", rv.Elem().Elem(), rv.Elem().Elem().Type(), sType)
	}
	return out, nil
}
<commit_msg>Deleted leftover fmt.Println from debugging<commit_after>package mapper

import (
	"context"
	"database/sql"
	"errors"
	"fmt"
	"reflect"
	"strings"
	"unsafe"

	"github.com/jonbodner/proteus/logger"
)

func ptrConverter(c context.Context, isPtr bool, sType reflect.Type, out reflect.Value, err error) (interface{}, error) {
	if err != nil {
		return nil, err
	}
	if isPtr {
		out2 := reflect.New(sType)
		k := out.Type().Kind()
		logger.Log(c, logger.DEBUG, fmt.Sprintln("kind of out", k))
		if (k == reflect.Interface || k == reflect.Ptr) && out.IsNil() {
			out2 = reflect.NewAt(sType, unsafe.Pointer(nil))
		} else {
			out2.Elem().Set(out)
		}
		return out2.Interface(), nil
	}
	k := out.Type().Kind()
	if (k == reflect.Ptr || k == reflect.Interface) && out.IsNil() {
		return nil, fmt.Errorf("Attempting to return nil for non-pointer type %v", sType)
	}
	return out.Interface(), nil
}

func MakeBuilder(c context.Context, sType reflect.Type) (Builder, error) {
	if sType == nil {
		return nil, errors.New("sType cannot be nil")
	}

	isPtr := false
	if sType.Kind() == reflect.Ptr {
		sType = sType.Elem()
		isPtr = true
	}

	//only going to handle scalar types here
	if sType.Kind() == reflect.Slice {
		sType = sType.Elem()
	}

	if sType.Kind() == reflect.Map {
		if sType.Key().Kind() != reflect.String {
			return nil, errors.New("Only maps with string keys are supported")
		}
		return func(cols []string, vals []interface{}) (interface{}, error) {
			out, err := buildMap(c, sType, cols, vals)
			return ptrConverter(c, isPtr, sType, out, err)
		}, nil
	} else if sType.Kind() == reflect.Struct {
		//build map of col names to field names (makes this 2N instead of N^2)
		colFieldMap := map[string]fieldInfo{}
		buildColFieldMap(sType, fieldInfo{}, colFieldMap)
		return func(cols []string, vals []interface{}) (interface{}, error) {
			out, err := buildStruct(c, sType, cols, vals, colFieldMap)
			return ptrConverter(c, isPtr, sType, out, err)
		}, nil

	} else {
		// assume primitive
		return func(cols []string, vals []interface{}) (interface{}, error) {
			out, err := buildPrimitive(c, sType, cols, vals)
			return ptrConverter(c, isPtr, sType, out, err)
		}, nil
	}
}

func buildColFieldMap(sType reflect.Type, parentFieldInfo fieldInfo, colFieldMap map[string]fieldInfo) {
	for i := 0; i < sType.NumField(); i++ {
		sf := sType.Field(i)
		if tagVal := sf.Tag.Get("prof"); tagVal != "" {
			childFieldInfo := fieldInfo{
				name:      append([]string(nil), parentFieldInfo.name...),
				fieldType: append([]reflect.Type(nil), parentFieldInfo.fieldType...),
				pos:       append([]int(nil), parentFieldInfo.pos...),
			}
			childFieldInfo.name = append(childFieldInfo.name, sf.Name)
			childFieldInfo.fieldType = append(childFieldInfo.fieldType, sf.Type)
			childFieldInfo.pos = append(childFieldInfo.pos, i)
			//if this is a struct, recurse
			// we are going to use an prof value as a prefix
			// then we go through each of the fields in the struct
			// if any of them aren't structs and have prof tags, we
			// prepend the parent struct and store off a fieldi
			// only if this doesn't implement a scanner. If it does, then go with the scanner
			// another special case: time.Time isn't recursed into
			if sf.Type.Kind() == reflect.Struct && !reflect.PtrTo(sf.Type).Implements(scannerType) && sf.Type.Name() != "Time" && sf.Type.PkgPath() != "time"  {
				buildColFieldMap(sf.Type, childFieldInfo, colFieldMap)
			} else {
				colFieldMap[strings.SplitN(tagVal, ",", 2)[0]] = childFieldInfo
			}
			continue
		}

		//last chance, if it's an anonymous struct, you can recurse into it without having a prof tag
		if sf.Type.Kind() == reflect.Struct && sf.Anonymous {
			childFieldInfo := fieldInfo{
				name:      append([]string(nil), parentFieldInfo.name...),
				fieldType: append([]reflect.Type(nil), parentFieldInfo.fieldType...),
				pos:       append([]int(nil), parentFieldInfo.pos...),
			}
			childFieldInfo.name = append(childFieldInfo.name, "")
			childFieldInfo.fieldType = append(childFieldInfo.fieldType, sf.Type)
			childFieldInfo.pos = append(childFieldInfo.pos, i)
			buildColFieldMap(sf.Type, childFieldInfo, colFieldMap)
		}
	}
}

type Builder func(cols []string, vals []interface{}) (interface{}, error)

type fieldInfo struct {
	name      []string
	fieldType []reflect.Type
	pos       []int
}

func buildMap(c context.Context, sType reflect.Type, cols []string, vals []interface{}) (reflect.Value, error) {
	out := reflect.MakeMap(sType)
	for k, v := range cols {
		curVal := vals[k]
		rv := reflect.ValueOf(curVal)
		if rv.Elem().IsNil() {
			continue
		}
		if rv.Elem().Elem().Type().ConvertibleTo(sType.Elem()) {
			out.SetMapIndex(reflect.ValueOf(v), rv.Elem().Elem().Convert(sType.Elem()))
		} else {
			return out, fmt.Errorf("Unable to assign value %v of type %v to map value of type %v with key %s", rv.Elem().Elem(), rv.Elem().Elem().Type(), sType.Elem(), v)
		}
	}
	return out, nil
}

var (
	scannerType = reflect.TypeOf((*sql.Scanner)(nil)).Elem()
)

func buildStruct(c context.Context, sType reflect.Type, cols []string, vals []interface{}, colFieldMap map[string]fieldInfo) (reflect.Value, error) {
	logger.Log(c, logger.DEBUG, fmt.Sprintf("sType: %s cols: %v vals: %+v colFieldMap: %+v", sType, cols, vals, colFieldMap))
	out := reflect.New(sType).Elem()
	for k, v := range cols {
		if sf, ok := colFieldMap[v]; ok {
			curVal := vals[k]
			rv := reflect.ValueOf(curVal)
			err := buildStructInner(c, sType, out, sf, curVal, rv, 0)
			if err != nil {
				return out, err
			}
		}
	}
	return out, nil
}

func buildStructInner(c context.Context, sType reflect.Type, out reflect.Value, sf fieldInfo, curVal interface{}, rv reflect.Value, depth int) error {
	field := out.Field(sf.pos[depth])
	curFieldType := sf.fieldType[depth]
	if curFieldType.Kind() == reflect.Ptr {
		logger.Log(c, logger.DEBUG, fmt.Sprintln("isPtr", sf, rv, rv.Type(), rv.Elem(), curVal, sType))
		if rv.Elem().IsNil() {
			logger.Log(c, logger.DEBUG, fmt.Sprintln("nil", sf, rv, curVal, sType))
			return nil
		}
		logger.Log(c, logger.DEBUG, fmt.Sprintln("isPtr Not Nil", rv.Elem().Type()))
		if curFieldType.Implements(scannerType) {
			toScan := (reflect.New(curFieldType).Elem().Interface()).(sql.Scanner)
			err := toScan.Scan(rv.Elem().Elem().Interface())
			if err != nil {
				return err
			}
			field.Set(reflect.ValueOf(toScan))
		} else if rv.Elem().Elem().Type().ConvertibleTo(curFieldType.Elem()) {
			field.Set(reflect.New(curFieldType.Elem()))
			field.Elem().Set(rv.Elem().Elem().Convert(curFieldType.Elem()))
		} else {
			logger.Log(c, logger.ERROR, fmt.Sprintln("can't find the field"))
			return fmt.Errorf("Unable to assign pointer to value %v of type %v to struct field %s of type %v", rv.Elem().Elem(), rv.Elem().Elem().Type(), sf.name[depth], curFieldType)
		}
	} else {
		if reflect.PtrTo(curFieldType).Implements(scannerType) {
			toScan := (reflect.New(curFieldType).Interface()).(sql.Scanner)
			err := toScan.Scan(rv.Elem().Interface())
			if err != nil {
				return err
			}
			field.Set(reflect.ValueOf(toScan).Elem())
		} else if rv.Elem().IsNil() {
			logger.Log(c, logger.ERROR, fmt.Sprintln("Attempting to assign a nil to a non-pointer field"))
			return fmt.Errorf("Unable to assign nil value to non-pointer struct field %s of type %v", sf.name[depth], curFieldType)
		} else if field.Kind() == reflect.Struct && field.Type().Name() != "Time" && field.Type().PkgPath() != "time" {
			err := buildStructInner(c, field.Type(), field, sf, curVal, rv, depth+1)
			if err != nil {
				return err
			}
		} else if rv.Elem().Elem().Type().ConvertibleTo(curFieldType) {
			field.Set(rv.Elem().Elem().Convert(curFieldType))
		} else {
			logger.Log(c, logger.ERROR, fmt.Sprintln("can't find the field"))
			return fmt.Errorf("Unable to assign value %v of type %v to struct field %s of type %v", rv.Elem().Elem(), rv.Elem().Elem().Type(), sf.name[depth], curFieldType)
		}
	}
	return nil
}

func buildPrimitive(c context.Context, sType reflect.Type, cols []string, vals []interface{}) (reflect.Value, error) {
	out := reflect.New(sType).Elem()
	//vals[0] is of type *interface{}, because everything in vals is of type *interface{}
	rv := reflect.ValueOf(vals[0])
	if rv.Elem().IsNil() {
		return rv.Elem(), nil
	}
	if rv.Elem().Elem().Type().ConvertibleTo(sType) {
		out.Set(rv.Elem().Elem().Convert(sType))
	} else {
		return out, fmt.Errorf("Unable to assign value %v of type %v to return type of type %v", rv.Elem().Elem(), rv.Elem().Elem().Type(), sType)
	}
	return out, nil
}
<|endoftext|>


<commit_before>package market

import (
	"log"
	"time"
)

const (
	//	雅虎财经的历史分时数据没有超过90天的
	lastestDays          = 90
	companyGCCount       = 32
	retryTimes           = 50
	retryIntervalSeconds = 60
)

//	市场更新
type Market interface {
	//	名称
	Name() string
	//	时区
	Timezone() string
	//	获取上市公司列表
	LastestCompanies() ([]Company, error)

	//	抓取任务(每日)
	Crawl(market Market, company Company, day time.Time) error
}

var markets = []Market{}

//	添加市场
func Add(market Market) {
	markets = append(markets, market)

	log.Printf("市场[%s]已经加入监视列表", market.Name())
}

//	监视市场(所有操作的入口)
func Monitor() {
	log.Print("启动监视")

	for _, m := range markets {
		go func(mkt Market) {

			//	市场所处时区当前时间
			now := locationNow(mkt)

			//	启动每日定时任务
			go func(market Market){
				//	所处时区的明日0点
				tomorrowZero := now.Truncate(time.Hour * 24).Add(time.Hour * 24)
				du := tomorrowZero.Sub(now)

				log.Printf("[%s]\t定时任务已启动，将于%d时%d分%d秒后激活首次任务", market.Name(), du.Hours(), du.Minutes(), du.Seconds());
				time.AfterFunc(du, func(){

				})

			}(mkt)

			monitorMarket(mkt)
		}(m)
	}
}

//	监视市场
func monitorMarket(market Market) {

	//	获取市场所在时区的今天0点
	now := locationNow(market)
	tomorrowZero := now.Truncate(time.Hour * 24).Add(time.Hour * 24)

	//	到明天的时间间隔
	du := tomorrowZero.Sub(now)
	log.Printf("[%s]\t%s后启动首次定时任务", market.Name(), du.String())
	time.AfterFunc(du, func() {
		ticker := time.NewTicker(time.Hour * 24)
		log.Printf("[%s]\t已启动定时抓取任务", market.Name())

		//	立刻运行一次
		go func() {
			err := crawlYesterday(market)
			if err != nil {
				log.Fatalf("[%s]\t抓取%s的数据出错:%v", market.Name(), today.Format("20060102"), err)
			}
		}()

		//	每天运行一次
		for _ = range ticker.C {
			err := crawlYesterday(market)
			if err != nil {
				log.Fatalf("[%s]\t抓取%s的数据出错:%v", market.Name(), now.Format("20060102"), err)
			}
		}
	})

	marketHistory(market)
}

//	所处时区当前时间
func locationNow(market Market) time.Time {
	now := time.Now()

	//	获取市场所在时区的今天0点
	location, err := time.LoadLocation(market.Timezone())
	if err != nil {
		return now
	}

	return now.In(location)
}

//	每日定时任务
func dailyTask(market Market) {

}

//	历史数据获取任务
func historyTask(market Market, yesterday time.Time) {

}

//	抓取市场上市公司信息
func companies(market Market) ([]Company, error) {

	cl := CompanyList{}
	//	尝试更新上市公司列表
	log.Printf("[%s]\t更新上市公司列表-开始", market.Name())
	companies, err := market.LastestCompanies()
	if err != nil {

		//	如果更新失败，则尝试从上次的存档文件中读取上市公司列表
		log.Printf("[%s]\t更新上市公司列表失败，尝试从存档读取", market.Name())
		err = cl.Load(market)
		if err != nil {
			log.Printf("[%s]\t尝试从存档读取上市公司列表-失败", market.Name())
			return nil, err
		}

		companies = cl
		log.Printf("[%s]\t尝试从存档读取上市公司列表-成功,共%d家上市公司", market.Name(), len(companies))

		return companies, nil
	}

	//	存档
	cl = CompanyList(companies)
	err = cl.Save(market)
	if err != nil {
		return nil, err
	}

	log.Printf("[%s]\t更新上市公司列表-成功,共%d家上市公司", market.Name(), len(companies))

	return companies, nil
}

//	抓取市场前一天的数据
func crawlYesterday(market Market) error {

	//	获取市场内的所有上市公司
	companies, err := companies(market)
	if err != nil {
		return err
	}

	//	前一天
	yesterday := locationNow(market).Truncate(time.Hour * 24).Add(-time.Hour * 24)

	chanSend := make(chan int, companyGCCount)
	chanReceive := make(chan int)

	for _, c := range companies {
		//	并发抓取
		go func(company Company) {

			err := market.Crawl(market, company, yesterday)
			if err != nil {
				log.Fatal(err)
			}

			<-chanSend
			chanReceive <- 1
		}(c)

		chanSend <- 1
	}

	//	阻塞，直到抓取所有
	for _, _ = range companies {
		<-chanReceive
	}

	close(chanSend)
	close(chanReceive)

	return nil
}

//	市场历史
func marketHistory(market Market) error {
	//	获取市场内的所有上市公司
	companies, err := companies(market)
	if err != nil {
		return err
	}

	log.Printf("[%s]\t开始抓取%d家上市公司的历史", market.Name(), len(companies))

	//	前一天
	yesterday := locationNow(market).Truncate(time.Hour * 24).Add(-time.Hour * 24)

	chanSend := make(chan int, companyGCCount)
	chanReceive := make(chan int)

	for _, c := range companies {
		//	并发抓取
		go func(company Company) {

			marketCompanyHistory(market, company, yesterday)

			<-chanSend
			chanReceive <- 1
		}(c)

		chanSend <- 1
	}

	//	阻塞，直到抓取所有
	for _, _ = range companies {
		<-chanReceive
	}

	close(chanSend)
	close(chanReceive)

	return nil
}

//	公司历史
func marketCompanyHistory(market Market, company Company, day time.Time) {
	//	查询之前一段时间的数据
	for index := 0; index < lastestDays; index++ {
		day = day.Add(-time.Hour * 24)
		err := market.Crawl(market, company, day)
		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>重写数据抓取任务逻辑<commit_after>package market

import (
	"fmt"
	"log"
	"time"
)

const (
	//	雅虎财经的历史分时数据没有超过90天的
	lastestDays          = 90
	companyGCCount       = 32
	retryTimes           = 50
	retryIntervalSeconds = 10
)

//	市场更新
type Market interface {
	//	名称
	Name() string
	//	时区
	Timezone() string
	//	获取上市公司列表
	Companies() ([]Company, error)

	//	抓取任务(每日)
	Crawl(market Market, company Company, day time.Time) error
}

var markets = []Market{}

//	添加市场
func Add(market Market) {
	markets = append(markets, market)

	log.Printf("市场[%s]已经加入监视列表", market.Name())
}

//	监视市场(所有操作的入口)
func Monitor() {
	log.Print("启动监视")

	for _, m := range markets {

		//	启动每日定时任务
		go func(market Market) {
			//	所处时区的明日0点
			now := locationNow(market)
			tomorrowZero := now.Truncate(time.Hour * 24).Add(time.Hour * 24)
			du := tomorrowZero.Sub(now)

			log.Printf("[%s]\t定时任务已启动，将于%s后激活首次任务", market.Name(), du.String())
			time.AfterFunc(du, func() {
				//	立即运行一次
				go dailyTask(market)

				//	每天运行一次
				ticker := time.NewTicker(time.Hour * 24)
				for _ = range ticker.C {
					dailyTask(market)
				}
			})

		}(m)

		//	启动历史数据获取任务
		go func(market Market) {
			historyTask(market, yesterdayZero(market))
		}(m)
	}
}

//	所处时区当前时间
func locationNow(market Market) time.Time {
	now := time.Now()

	//	获取市场所在时区的今天0点
	location, err := time.LoadLocation(market.Timezone())
	if err != nil {
		return now
	}

	return now.In(location)
}

//	昨天0点
func yesterdayZero(market Market) time.Time {
	now := locationNow(market)
	return now.Truncate(time.Hour * 24)
}

//	每日定时任务
func dailyTask(market Market) {

	//	昨天零点
	yesterday := yesterdayZero(market)
	log.Printf("[%s]\t%s数据获取任务已启动", market.Name(), yesterday.Format("20060102"))

	//	获取市场所有上市公司
	companies, err := getCompanies(market)
	if err != nil {
		log.Fatal(err)
	}

	chanSend := make(chan int, companyGCCount)
	chanReceive := make(chan int)

	for _, c := range companies {
		//	并发抓取
		go func(company Company) {

			err := market.Crawl(market, company, yesterday)
			if err != nil {
				log.Fatal(err)
			}

			<-chanSend
			chanReceive <- 1
		}(c)

		chanSend <- 1
	}

	//	阻塞，直到抓取所有
	for _, _ = range companies {
		<-chanReceive
	}

	close(chanSend)
	close(chanReceive)

}

//	历史数据获取任务
func historyTask(market Market, yesterday time.Time) {
	//	获取市场所有上市公司
	companies, err := getCompanies(market)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("[%s]\t开始抓取%d家上市公司在%s之前的历史", market.Name(), len(companies), yesterday.Format("20060102"))

	chanSend := make(chan int, companyGCCount)
	chanReceive := make(chan int)

	for _, c := range companies {
		//	并发抓取
		go func(company Company) {

			for index := 0; index < lastestDays; index++ {
				day := yesterday.Add(-time.Hour * 24 * time.Duration(index))
				err := market.Crawl(market, company, day)
				if err != nil {
					log.Fatalf("[%s]\t抓取[%s]在%s的分时数据出错:%s", market.Name(), company.Code, day.Format("20060102"), err)
				}
			}

			<-chanSend
			chanReceive <- 1
		}(c)

		chanSend <- 1
	}

	//	阻塞，直到抓取所有
	for _, _ = range companies {
		<-chanReceive
	}

	close(chanSend)
	close(chanReceive)

	log.Printf("[%s]\t上市公司的历史分时数据已经抓取结束", market.Name())
}

//	抓取市场上市公司信息
func getCompanies(market Market) ([]Company, error) {

	cl := CompanyList{}
	//	尝试更新上市公司列表
	log.Printf("[%s]\t更新上市公司列表-开始", market.Name())
	companies, err := market.Companies()
	if err != nil {

		//	如果更新失败，则尝试从上次的存档文件中读取上市公司列表
		log.Printf("[%s]\t更新上市公司列表失败，尝试从存档读取", market.Name())
		err = cl.Load(market)
		if err != nil {
			return nil, fmt.Errorf("[%s]\t尝试从存档读取上市公司列表-失败", market.Name())
		}

		companies = cl
		log.Printf("[%s]\t尝试从存档读取上市公司列表-成功,共%d家上市公司", market.Name(), len(companies))

		return companies, nil
	}

	//	存档
	cl = CompanyList(companies)
	err = cl.Save(market)
	if err != nil {
		return nil, err
	}

	log.Printf("[%s]\t更新上市公司列表-成功,共%d家上市公司", market.Name(), len(companies))

	return companies, nil
}
<|endoftext|>