<commit_before>package s3gof3r

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"hash"
	"io"
	"math"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// defined by amazon
const (
	minPartSize = 5 * mb
	maxPartSize = 5 * gb
	maxObjSize  = 5 * tb
	maxNPart    = 10000
	md5Header   = "content-md5"
)

type part struct {
	r   io.ReadSeeker
	len int64
	b   *bytes.Buffer

	// read by xml encoder
	PartNumber int
	ETag       string

	// Used for checksum of checksums on completion
	contentMd5 string
}

type putter struct {
	url         url.URL
	client      *http.Client
	b           *Bucket
	concurrency int
	nTry        int
	c           *Config

	bufsz      int64
	buf        *bytes.Buffer
	ch         chan *part
	part       int
	closed     bool
	err        error
	wg         sync.WaitGroup
	md5OfParts hash.Hash
	md5        hash.Hash
	ETag       string

	bp *bp

	makes    int
	UploadId string
	xml      struct {
		XMLName string `xml:"CompleteMultipartUpload"`
		Part    []*part
	}
}

type completeXml struct {
	ETag string
}

// Sends an S3 multipart upload initiation request.
// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html.
// The initial request returns an UploadId that we use to identify
// subsequent PUT requests.
func newPutter(url url.URL, h http.Header, c *Config, b *Bucket) (p *putter, err error) {
	p = new(putter)
	p.url = url
	p.client = c.Client
	p.b = b
	p.concurrency = c.Concurrency
	p.nTry = c.NTry
	p.c = c

	p.bufsz = max64(minPartSize, c.PartSize)
	resp, err := p.retryRequest("POST", url.String()+"?uploads", nil, h)
	if err != nil {
		return nil, err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return nil, newRespError(resp)
	}
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return nil, err
	}
	p.ch = make(chan *part)
	for i := 0; i < p.concurrency; i++ {
		go p.worker()
	}
	p.md5OfParts = md5.New()
	p.md5 = md5.New()

	p.bp = newBufferPool(p.bufsz)
	return p, nil
}

func (p *putter) Write(b []byte) (int, error) {
	if p.closed {
		return 0, syscall.EINVAL
	}
	if p.err != nil {
		return 0, p.err
	}
	if p.buf == nil {
		p.buf = <-p.bp.get
		p.buf.Reset()
	}
	n, err := p.buf.Write(b)
	if err != nil {
		return n, err
	}
	if int64(p.buf.Len()) >= p.bufsz {
		p.flush()
	}
	return n, nil
}

func (p *putter) flush() {
	p.wg.Add(1)
	p.part++
	b := *p.buf
	part := &part{bytes.NewReader(b.Bytes()), int64(b.Len()), p.buf, p.part, "", ""}
	var err error
	part.contentMd5, part.ETag, err = p.md5Content(part.r)
	if err != nil {
		p.err = err
	}

	p.xml.Part = append(p.xml.Part, part)
	p.ch <- part
	p.buf = nil
	// double buffer size every 1000 parts to
	// avoid exceeding the 10000-part AWS limit
	// while still reaching the 5 Terabyte max object size
	if p.part%1000 == 0 {
		p.bufsz = min64(p.bufsz*2, maxPartSize)
	}

}

func (p *putter) worker() {
	for part := range p.ch {
		p.retryPutPart(part)
	}
}

// Calls putPart up to nTry times to recover from transient errors.
func (p *putter) retryPutPart(part *part) {
	defer p.wg.Done()
	var err error
	for i := 0; i < p.nTry; i++ {
		time.Sleep(time.Duration(math.Exp2(float64(i))) * 100 * time.Millisecond) // exponential back-off
		err = p.putPart(part)
		if err == nil {
			p.bp.give <- part.b
			return
		}
		logger.debugPrintf("Error on attempt %d: Retrying part: %v, Error: %s", i, part, err)
	}
	p.err = err
}

// uploads a part, checking the etag against the calculated value
func (p *putter) putPart(part *part) error {
	v := url.Values{}
	v.Set("partNumber", strconv.Itoa(part.PartNumber))
	v.Set("uploadId", p.UploadId)
	if _, err := part.r.Seek(0, 0); err != nil { // move back to beginning, if retrying
		return err
	}
	req, err := http.NewRequest("PUT", p.url.String()+"?"+v.Encode(), part.r)
	if err != nil {
		return err
	}
	req.ContentLength = part.len
	req.Header.Set(md5Header, part.contentMd5)
	p.b.Sign(req)
	resp, err := p.client.Do(req)
	if err != nil {
		return err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	s := resp.Header.Get("etag")
	s = s[1 : len(s)-1] // includes quote chars for some reason
	if part.ETag != s {
		return fmt.Errorf("Response etag does not match. Remote:%s Calculated:%s", s, p.ETag)
	}
	return nil
}

func (p *putter) Close() (err error) {
	if p.closed {
		return syscall.EINVAL
	}
	if p.buf != nil {

		buf := *p.buf
		if buf.Len() > 0 {
			p.flush()
		}
	}
	p.wg.Wait()
	close(p.ch)
	p.closed = true
	p.bp.quit <- true
	if p.err != nil {
		err := p.abort()
		if err != nil {
			return err
		}
		return p.err
	}
	logger.debugPrintln("makes:", p.bp.makes)
	// Complete Multipart upload
	body, err := xml.Marshal(p.xml)
	if err != nil {
		return
	}
	b := bytes.NewReader(body)
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	resp, err := p.retryRequest("POST", p.url.String()+"?"+v.Encode(), b, nil)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	// Check md5 hash of concatenated part md5 hashes against ETag
	// more info: https://forums.aws.amazon.com/thread.jspa?messageID=456442&#456442
	calculatedMd5ofParts := fmt.Sprintf("%x", p.md5OfParts.Sum(nil))
	// Parse etag from body of response
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return
	}
	// strip part count from end and '"' from front.
	remoteMd5ofParts := strings.Split(p.ETag, "-")[0]
	remoteMd5ofParts = remoteMd5ofParts[1:len(remoteMd5ofParts)]
	if calculatedMd5ofParts != remoteMd5ofParts {
		// TODO: Delete file from S3?
		if err != nil {
			return err
		}
		return fmt.Errorf("MD5 hash of part hashes comparison failed. Hash from multipart complete header: %s."+
			" Calculated multipart hash: %s.", remoteMd5ofParts, calculatedMd5ofParts)
	}
	//logger.debugPrintln("Hash from multipart complete header:", remoteMd5ofParts)
	//logger.debugPrintln("Calculated multipart hash:", calculatedMd5ofParts)
	if p.c.Md5Check {
		for i := 0; i < p.nTry; i++ {
			if err = p.putMd5(); err == nil {
				break
			}
		}
		return
	}
	return
}

func (p *putter) abort() (err error) {
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	s := p.url.String() + "?" + v.Encode()
	resp, err := p.retryRequest("DELETE", s, nil, nil)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 204 {
		return newRespError(resp)
	}
	return
}

// Md5 functions
func (p *putter) md5Content(r io.ReadSeeker) (string, string, error) {
	h := md5.New()
	mw := io.MultiWriter(h, p.md5)
	if _, err := io.Copy(mw, r); err != nil {
		return "", "", err
	}
	sum := h.Sum(nil)
	hexSum := fmt.Sprintf("%x", sum)
	// add to checksum of all parts for verification on upload completion
	if _, err := p.md5OfParts.Write(sum); err != nil {
		return "", "", err
	}
	return base64.StdEncoding.EncodeToString(sum), hexSum, nil
}

// Put md5 file in .md5 subdirectory of bucket  where the file is stored
// e.g. the md5 for https://mybucket.s3.amazonaws.com/gof3r will be stored in
// https://mybucket.s3.amazonaws.com/.md5/gof3r.md5
func (p *putter) putMd5() (err error) {
	calcMd5 := fmt.Sprintf("%x", p.md5.Sum(nil))
	md5Reader := strings.NewReader(calcMd5)
	md5Path := fmt.Sprint(".md5", p.url.Path, ".md5")
	md5Url := p.b.Url(md5Path, p.c)
	logger.debugPrintln("md5: ", calcMd5)
	logger.debugPrintln("md5Path: ", md5Path)
	r, err := http.NewRequest("PUT", md5Url.String(), md5Reader)
	if err != nil {
		return
	}
	p.b.Sign(r)
	resp, err := p.client.Do(r)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	return
}

func (p *putter) retryRequest(method, urlStr string, body io.ReadSeeker, h http.Header) (resp *http.Response, err error) {
	for i := 0; i < p.nTry; i++ {
		var req *http.Request
		req, err = http.NewRequest(method, urlStr, body)
		if err != nil {
			return
		}
		for k := range h {
			for _, v := range h[k] {
				req.Header.Add(k, v)
			}
		}

		p.b.Sign(req)
		resp, err = p.client.Do(req)
		if err == nil {
			return
		}
		logger.debugPrintln(err)
		if body != nil {
			if _, err = body.Seek(0, 0); err != nil {
				return
			}
		}
	}
	return
}
<commit_msg>Improve error message if no parts uploaded. #16<commit_after>package s3gof3r

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/xml"
	"fmt"
	"hash"
	"io"
	"math"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"
)

// defined by amazon
const (
	minPartSize = 5 * mb
	maxPartSize = 5 * gb
	maxObjSize  = 5 * tb
	maxNPart    = 10000
	md5Header   = "content-md5"
)

type part struct {
	r   io.ReadSeeker
	len int64
	b   *bytes.Buffer

	// read by xml encoder
	PartNumber int
	ETag       string

	// Used for checksum of checksums on completion
	contentMd5 string
}

type putter struct {
	url         url.URL
	client      *http.Client
	b           *Bucket
	concurrency int
	nTry        int
	c           *Config

	bufsz      int64
	buf        *bytes.Buffer
	ch         chan *part
	part       int
	closed     bool
	err        error
	wg         sync.WaitGroup
	md5OfParts hash.Hash
	md5        hash.Hash
	ETag       string

	bp *bp

	makes    int
	UploadId string
	xml      struct {
		XMLName string `xml:"CompleteMultipartUpload"`
		Part    []*part
	}
}

type completeXml struct {
	ETag string
}

// Sends an S3 multipart upload initiation request.
// See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html.
// The initial request returns an UploadId that we use to identify
// subsequent PUT requests.
func newPutter(url url.URL, h http.Header, c *Config, b *Bucket) (p *putter, err error) {
	p = new(putter)
	p.url = url
	p.client = c.Client
	p.b = b
	p.concurrency = c.Concurrency
	p.nTry = c.NTry
	p.c = c
	p.bufsz = max64(minPartSize, c.PartSize)
	resp, err := p.retryRequest("POST", url.String()+"?uploads", nil, h)
	if err != nil {
		return nil, err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return nil, newRespError(resp)
	}
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return nil, err
	}
	p.ch = make(chan *part)
	for i := 0; i < p.concurrency; i++ {
		go p.worker()
	}
	p.md5OfParts = md5.New()
	p.md5 = md5.New()

	p.bp = newBufferPool(p.bufsz)

	return p, nil
}

func (p *putter) Write(b []byte) (int, error) {
	if p.closed {
		return 0, syscall.EINVAL
	}
	if p.err != nil {
		return 0, p.err
	}
	if p.buf == nil {
		p.buf = <-p.bp.get
		p.buf.Reset()
	}
	n, err := p.buf.Write(b)
	if err != nil {
		return n, err
	}

	if int64(p.buf.Len()) >= p.bufsz {
		p.flush()
	}
	return n, nil
}

func (p *putter) flush() {
	p.wg.Add(1)
	p.part++
	b := *p.buf
	part := &part{bytes.NewReader(b.Bytes()), int64(b.Len()), p.buf, p.part, "", ""}
	var err error
	part.contentMd5, part.ETag, err = p.md5Content(part.r)
	if err != nil {
		p.err = err
	}

	p.xml.Part = append(p.xml.Part, part)
	p.ch <- part
	p.buf = nil
	// double buffer size every 1000 parts to
	// avoid exceeding the 10000-part AWS limit
	// while still reaching the 5 Terabyte max object size
	if p.part%1000 == 0 {
		p.bufsz = min64(p.bufsz*2, maxPartSize)
	}

}

func (p *putter) worker() {
	for part := range p.ch {
		p.retryPutPart(part)
	}
}

// Calls putPart up to nTry times to recover from transient errors.
func (p *putter) retryPutPart(part *part) {
	defer p.wg.Done()
	var err error
	for i := 0; i < p.nTry; i++ {
		time.Sleep(time.Duration(math.Exp2(float64(i))) * 100 * time.Millisecond) // exponential back-off
		err = p.putPart(part)
		if err == nil {
			p.bp.give <- part.b
			return
		}
		logger.debugPrintf("Error on attempt %d: Retrying part: %v, Error: %s", i, part, err)
	}
	p.err = err
}

// uploads a part, checking the etag against the calculated value
func (p *putter) putPart(part *part) error {
	v := url.Values{}
	v.Set("partNumber", strconv.Itoa(part.PartNumber))
	v.Set("uploadId", p.UploadId)
	if _, err := part.r.Seek(0, 0); err != nil { // move back to beginning, if retrying
		return err
	}
	req, err := http.NewRequest("PUT", p.url.String()+"?"+v.Encode(), part.r)
	if err != nil {
		return err
	}
	req.ContentLength = part.len
	req.Header.Set(md5Header, part.contentMd5)
	p.b.Sign(req)
	resp, err := p.client.Do(req)
	if err != nil {
		return err
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	s := resp.Header.Get("etag")
	s = s[1 : len(s)-1] // includes quote chars for some reason
	if part.ETag != s {
		return fmt.Errorf("Response etag does not match. Remote:%s Calculated:%s", s, p.ETag)
	}
	return nil
}

func (p *putter) Close() (err error) {
	if p.closed {
		return syscall.EINVAL
	}
	if p.buf != nil {
		buf := *p.buf
		if buf.Len() > 0 {
			p.flush()
		}
	}
	p.wg.Wait()
	close(p.ch)
	p.closed = true
	p.bp.quit <- true

	if p.part == 0 {
		return fmt.Errorf("0 bytes written")
	}
	if p.err != nil {
		err := p.abort()
		if err != nil {
			return err
		}
		return p.err
	}
	logger.debugPrintln("buffer allocations:", p.bp.makes)
	// Complete Multipart upload
	body, err := xml.Marshal(p.xml)
	if err != nil {
		return
	}
	b := bytes.NewReader(body)
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	resp, err := p.retryRequest("POST", p.url.String()+"?"+v.Encode(), b, nil)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	// Check md5 hash of concatenated part md5 hashes against ETag
	// more info: https://forums.aws.amazon.com/thread.jspa?messageID=456442&#456442
	calculatedMd5ofParts := fmt.Sprintf("%x", p.md5OfParts.Sum(nil))
	// Parse etag from body of response
	err = xml.NewDecoder(resp.Body).Decode(p)
	if err != nil {
		return
	}
	// strip part count from end and '"' from front.
	remoteMd5ofParts := strings.Split(p.ETag, "-")[0]
	remoteMd5ofParts = remoteMd5ofParts[1:len(remoteMd5ofParts)]
	if calculatedMd5ofParts != remoteMd5ofParts {
		// TODO: Delete file from S3?
		if err != nil {
			return err
		}
		return fmt.Errorf("MD5 hash of part hashes comparison failed. Hash from multipart complete header: %s."+
			" Calculated multipart hash: %s.", remoteMd5ofParts, calculatedMd5ofParts)
	}
	if p.c.Md5Check {
		for i := 0; i < p.nTry; i++ {
			if err = p.putMd5(); err == nil {
				break
			}
		}
		return
	}
	return
}

func (p *putter) abort() (err error) {
	v := url.Values{}
	v.Set("uploadId", p.UploadId)
	s := p.url.String() + "?" + v.Encode()
	resp, err := p.retryRequest("DELETE", s, nil, nil)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 204 {
		return newRespError(resp)
	}
	return
}

// Md5 functions
func (p *putter) md5Content(r io.ReadSeeker) (string, string, error) {
	h := md5.New()
	mw := io.MultiWriter(h, p.md5)
	if _, err := io.Copy(mw, r); err != nil {
		return "", "", err
	}
	sum := h.Sum(nil)
	hexSum := fmt.Sprintf("%x", sum)
	// add to checksum of all parts for verification on upload completion
	if _, err := p.md5OfParts.Write(sum); err != nil {
		return "", "", err
	}
	return base64.StdEncoding.EncodeToString(sum), hexSum, nil
}

// Put md5 file in .md5 subdirectory of bucket  where the file is stored
// e.g. the md5 for https://mybucket.s3.amazonaws.com/gof3r will be stored in
// https://mybucket.s3.amazonaws.com/.md5/gof3r.md5
func (p *putter) putMd5() (err error) {
	calcMd5 := fmt.Sprintf("%x", p.md5.Sum(nil))
	md5Reader := strings.NewReader(calcMd5)
	md5Path := fmt.Sprint(".md5", p.url.Path, ".md5")
	md5Url := p.b.Url(md5Path, p.c)
	logger.debugPrintln("md5: ", calcMd5)
	logger.debugPrintln("md5Path: ", md5Path)
	r, err := http.NewRequest("PUT", md5Url.String(), md5Reader)
	if err != nil {
		return
	}
	p.b.Sign(r)
	resp, err := p.client.Do(r)
	if err != nil {
		return
	}
	defer checkClose(resp.Body, &err)
	if resp.StatusCode != 200 {
		return newRespError(resp)
	}
	return
}

func (p *putter) retryRequest(method, urlStr string, body io.ReadSeeker, h http.Header) (resp *http.Response, err error) {
	for i := 0; i < p.nTry; i++ {
		var req *http.Request
		req, err = http.NewRequest(method, urlStr, body)
		if err != nil {
			return
		}
		for k := range h {
			for _, v := range h[k] {
				req.Header.Add(k, v)
			}
		}

		p.b.Sign(req)
		resp, err = p.client.Do(req)
		if err == nil {
			return
		}
		logger.debugPrintln(err)
		if body != nil {
			if _, err = body.Seek(0, 0); err != nil {
				return
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package xiaoice

import (
	"sync"

	"github.com/KevinGong2013/ggbot/utils"
	wx "github.com/KevinGong2013/ggbot/wechat"
)

import log "github.com/Sirupsen/logrus"

var logger = log.WithFields(log.Fields{
	"module": "xiaoice",
})

// Brain ...
type Brain struct {
	sync.Mutex
	wx             *wx.WeChat
	xiaoice        *wx.Contact
	waittingReplay []string
}

// NewBrain ...
func NewBrain() *Brain {
	return &Brain{waittingReplay: []string{}}
}

// WechatDidLogin ...
func (b *Brain) WechatDidLogin(wechat *wx.WeChat) {
	b.wx = wechat
	b.xiaoice, _ = wechat.ContactByNickName(`小冰`)
}

// WechatDidLogout ...
func (b *Brain) WechatDidLogout(wechat *wx.WeChat) {
}

// MapMsgs ...
func (b *Brain) MapMsgs(msg *wx.CountedContent) {
	for _, m := range msg.Content {
		isSendByMySelf, _ := m[`IsSendByMySelf`].(bool)
		if isSendByMySelf {
			continue
		}
		from, _ := m[`FromUserName`].(string)
		contact, err := b.wx.ContactByUserName(from)
		if err != nil {
			m[`needXiaoiceResponse`] = false
			logger.Error(err)
			continue
		}

		switch contact.Type {
		case wx.ContactTypeFriend:
			m[`needXiaoiceResponse`] = true
			m[`xiaoice_info`] = m[`Content`]
			m[`xiaoice_to`] = m[`FromUserName`]
		case wx.ContactTypeOfficial:
			if b.xiaoice.NickName == contact.NickName {
				len := len(b.waittingReplay)
				if len > 0 {
					b.Lock()
					m[`isXiaoiceReplay`] = true
					m[`ReplayUserName`] = b.waittingReplay[len-1]
					m[`localFileId`] = m[`MsgId`]
					b.waittingReplay = b.waittingReplay[:len-1]
					b.Unlock()
				} else {
					logger.Warnf(`xiaoice replay %s`, m)
				}
			} else {
				logger.Warn(`offical msg %s`, contact.NickName)
			}
			m[`needXiaoiceResponse`] = false
		case wx.ContactTypeGroup:
			m[`needXiaoiceResponse`] = true
			m[`xiaoice_info`] = m[`Content`]
			m[`xiaoice_to`] = m[`FromUserName`]
		}
	}
}

// HandleMsgs ...
func (b *Brain) HandleMsgs(msg *wx.CountedContent) {
	for _, m := range msg.Content {
		needResponse, _ := m[`needXiaoiceResponse`].(bool)
		isReplay, _ := m[`isXiaoiceReplay`].(bool)
		if needResponse {
			c, _ := m[`xiaoice_info`].(string)
			to, _ := m[`xiaoice_to`].(string)

			if b.xiaoice != nil {
				var err error
				if m[`isMediaMsg`].(bool) {
					path, e := b.wx.DownloadMedia(m[`MediaMsgDownloadUrl`].(string), m[`MsgId`].(string))
					defer utils.DeleteFile(path)
					if e == nil {
						err = b.wx.SendFile(path, b.xiaoice.To())
					} else {
						err = e
					}
				} else {
					err = b.wx.SendTextMsg(c, b.xiaoice.To())
				}
				if err == nil {
					b.Lock()
					b.waittingReplay = append(b.waittingReplay, to)
					b.Unlock()
				} else {
					logger.Error(err)
				}
			}
		}
		if isReplay {
			to, _ := m[`ReplayUserName`].(string)
			c, _ := m[`Content`].(string)
			msgType, _ := m[`MsgType`].(float64)

			if msgType == 1 {
				b.wx.SendTextMsg(c, to)
			} else if m[`isMediaMsg`].(bool) {
				path, err := b.wx.DownloadMedia(m[`MediaMsgDownloadUrl`].(string), m[`MsgId`].(string))
				defer utils.DeleteFile(path)
				if err != nil {
					logger.Error(err)
				}
				err = b.wx.SendFile(path, to)
				if err != nil {
					logger.Error(err)
				}
			}
		}
	}
}
<commit_msg>Added `protected` for `isMediaMsg`<commit_after>package xiaoice

import (
	"sync"

	"github.com/KevinGong2013/ggbot/utils"
	wx "github.com/KevinGong2013/ggbot/wechat"
)

import log "github.com/Sirupsen/logrus"

var logger = log.WithFields(log.Fields{
	"module": "xiaoice",
})

// Brain ...
type Brain struct {
	sync.Mutex
	wx             *wx.WeChat
	xiaoice        *wx.Contact
	waittingReplay []string
}

// NewBrain ...
func NewBrain() *Brain {
	return &Brain{waittingReplay: []string{}}
}

// WechatDidLogin ...
func (b *Brain) WechatDidLogin(wechat *wx.WeChat) {
	b.wx = wechat
	b.xiaoice, _ = wechat.ContactByNickName(`小冰`)
}

// WechatDidLogout ...
func (b *Brain) WechatDidLogout(wechat *wx.WeChat) {
}

// MapMsgs ...
func (b *Brain) MapMsgs(msg *wx.CountedContent) {
	for _, m := range msg.Content {
		isSendByMySelf, _ := m[`IsSendByMySelf`].(bool)
		if isSendByMySelf {
			continue
		}
		from, _ := m[`FromUserName`].(string)
		contact, err := b.wx.ContactByUserName(from)
		if err != nil {
			m[`needXiaoiceResponse`] = false
			logger.Error(err)
			continue
		}

		switch contact.Type {
		case wx.ContactTypeFriend:
			m[`needXiaoiceResponse`] = true
			m[`xiaoice_info`] = m[`Content`]
			m[`xiaoice_to`] = m[`FromUserName`]
		case wx.ContactTypeOfficial:
			if b.xiaoice.NickName == contact.NickName {
				len := len(b.waittingReplay)
				if len > 0 {
					b.Lock()
					m[`isXiaoiceReplay`] = true
					m[`ReplayUserName`] = b.waittingReplay[len-1]
					m[`localFileId`] = m[`MsgId`]
					b.waittingReplay = b.waittingReplay[:len-1]
					b.Unlock()
				} else {
					logger.Warnf(`xiaoice replay %s`, m)
				}
			} else {
				logger.Warn(`offical msg %s`, contact.NickName)
			}
			m[`needXiaoiceResponse`] = false
		case wx.ContactTypeGroup:
			m[`needXiaoiceResponse`] = true
			m[`xiaoice_info`] = m[`Content`]
			m[`xiaoice_to`] = m[`FromUserName`]
		}
	}
}

// HandleMsgs ...
func (b *Brain) HandleMsgs(msg *wx.CountedContent) {
	for _, m := range msg.Content {
		needResponse, _ := m[`needXiaoiceResponse`].(bool)
		isReplay, _ := m[`isXiaoiceReplay`].(bool)
		if needResponse {
			c, _ := m[`xiaoice_info`].(string)
			to, _ := m[`xiaoice_to`].(string)

			if b.xiaoice != nil {
				var err error
				if ok, isM := m[`isMediaMsg`].(bool); ok && isM {
					path, e := b.wx.DownloadMedia(m[`MediaMsgDownloadUrl`].(string), m[`MsgId`].(string))
					defer utils.DeleteFile(path)
					if e == nil {
						err = b.wx.SendFile(path, b.xiaoice.To())
					} else {
						err = e
					}
				} else {
					err = b.wx.SendTextMsg(c, b.xiaoice.To())
				}
				if err == nil {
					b.Lock()
					b.waittingReplay = append(b.waittingReplay, to)
					b.Unlock()
				} else {
					logger.Error(err)
				}
			}
		}
		if isReplay {
			to, _ := m[`ReplayUserName`].(string)
			c, _ := m[`Content`].(string)
			msgType, _ := m[`MsgType`].(float64)

			if msgType == 1 {
				b.wx.SendTextMsg(c, to)
			} else if m[`isMediaMsg`].(bool) {
				path, err := b.wx.DownloadMedia(m[`MediaMsgDownloadUrl`].(string), m[`MsgId`].(string))
				defer utils.DeleteFile(path)
				if err != nil {
					logger.Error(err)
				}
				err = b.wx.SendFile(path, to)
				if err != nil {
					logger.Error(err)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package kronecker

import (
    "math"
    "math/rand"
)

func YieldEdge(scale int, A, B, C float64) [2]int {
    from := 0
    to := 0

    ab := A + B
    c_norm := C/(1 - ab)
    a_norm := A/ab

    for ib := 1; ib <= scale; ib++ {
        coeff := int(math.Exp2(float64(ib - 1)))
        from2 := 0
        to2 := 0

        if rand.Float64() > ab {
            from2 = 1
        }
        if rand.Float64() > (c_norm*float64(from2) + a_norm*float64((from2+1)%2)) {
            to2 = 1
        }

        from = from + coeff*from2
        to = to + coeff*to2
    }

    return [2]int{from+1, to+1}
}
<commit_msg>I'm a fucking idiot.<commit_after>package kronecker

import (
    "math/rand"
)

func YieldEdge(scale int, A, B, C float64) [2]int {
    from := 0
    to := 0

    ab := A + B
    c_norm := C/(1 - ab)
    a_norm := A/ab

    for ib := 0; ib < scale; ib++ {
        coeff := 1 << uint(ib)

        from2 := 0
        to2 := 0

        if rand.Float64() > ab {
            from2 = 1
        }
        if rand.Float64() > (c_norm*float64(from2) + a_norm*float64((from2+1)%2)) {
            to2 = 1
        }

        from = from + coeff*from2
        to = to + coeff*to2
    }

    return [2]int{from+1, to+1}
}
<|endoftext|>


<commit_before>package kv

import (
	"encoding/json"
	"fmt"
	"strings"

	"log"

	"golang.org/x/net/context"
	elastic "gopkg.in/olivere/elastic.v5"

	"github.com/movio/kasper/util"
)

const indexSettings = `{
	"index.translog.durability": "sync",
}`

const indexMapping = `{
	"_all" : {
		"enabled" : false
	},
	"dynamic_templates": [{
		"no_index": {
			"mapping": {
				"index": "no"
			},
			"match": "*"
		}
	}]
}`

type indexAndType struct {
	indexName string
	indexType string
}

// ElasticsearchKeyValueStore is a key-value storage that uses ElasticSearch.
// In this key-value store, all keys must have the format "<index>/<type>/<_id>".
type ElasticsearchKeyValueStore struct {
	witness         *util.StructPtrWitness
	client          *elastic.Client
	context         context.Context
	existingIndexes []indexAndType
}

// NewESKeyValueStore creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
func NewESKeyValueStore(url string, structPtr interface{}) *ElasticsearchKeyValueStore {
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetSniff(false), // FIXME: workaround for issues with ES in docker
	)
	if err != nil {
		panic(fmt.Sprintf("Cannot create ElasticSearch Client to '%s': %s", url, err))
	}
	return &ElasticsearchKeyValueStore{
		witness:         util.NewStructPtrWitness(structPtr),
		client:          client,
		context:         context.Background(),
		existingIndexes: nil,
	}
}

func (s *ElasticsearchKeyValueStore) checkOrCreateIndex(indexName string, indexType string) {
	for _, existing := range s.existingIndexes {
		if existing.indexName == indexName && existing.indexType == indexType {
			return
		}
	}
	exists, err := s.client.IndexExists(indexName).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to check if index exists: %s", err))
	}
	if !exists {
		_, err = s.client.CreateIndex(indexName).BodyString(indexSettings).Do(s.context)
		if err != nil {
			panic(fmt.Sprintf("Failed to create index: %s", err))
		}
		s.putMapping(indexName, indexType)
	}

	s.existingIndexes = append(s.existingIndexes, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) putMapping(indexName string, indexType string) {
	resp, err := s.client.PutMapping().Index(indexName).Type(indexType).BodyString(indexMapping).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to put mapping for index: %s/%s: %s", indexName, indexType, err))
	}
	if resp == nil {
		panic(fmt.Sprintf("Expected put mapping response; got: %v", resp))
	}
	if !resp.Acknowledged {
		panic(fmt.Sprintf("Expected put mapping ack; got: %v", resp.Acknowledged))
	}
}

// Get gets value by key from store
func (s *ElasticsearchKeyValueStore) Get(key string) (interface{}, error) {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return nil, fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	rawValue, err := s.client.Get().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if fmt.Sprintf("%s", err) == "elastic: Error 404 (Not Found)" {
		return s.witness.Nil(), nil
	}

	if err != nil {
		return s.witness.Nil(), err
	}

	if !rawValue.Found {
		return s.witness.Nil(), nil
	}

	structPtr := s.witness.Allocate()
	err = json.Unmarshal(*rawValue.Source, structPtr)
	if err != nil {
		return s.witness.Nil(), err
	}
	return structPtr, nil
}

// TBD
func (s *ElasticsearchKeyValueStore) GetAll(keys []string) ([]*Entry, error) {
	multiGet := s.client.MultiGet()
	for _, key := range keys {
		keyParts := strings.Split(key, "/")
		if len(keyParts) != 3 {
			return nil, fmt.Errorf("invalid key: '%s'", key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.checkOrCreateIndex(indexName, indexType)

		item := elastic.NewMultiGetItem().
			Index(indexName).
			Type(indexType).
			Id(valueID)

		multiGet.Add(item)
	}
	response, err := multiGet.Do(s.context)
	if err != nil {
		return nil, err
	}
	entries := make([]*Entry, len(keys))
	for i, doc := range response.Docs {
		var structPtr interface{}
		if !doc.Found {
			structPtr = s.witness.Nil()
		} else {
			structPtr = s.witness.Allocate()
			err = json.Unmarshal(*doc.Source, structPtr)
			if err != nil {
				return nil, err
			}
		}
		entries[i] = &Entry{keys[i], structPtr}
	}
	return entries, nil
}

// Put updates key in store with serialized value
func (s *ElasticsearchKeyValueStore) Put(key string, structPtr interface{}) error {
	s.witness.Assert(structPtr)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	_, err := s.client.Index().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		BodyJson(structPtr).
		Do(s.context)

	return err
}

// PutAll bulk executes Put operation for several entries
func (s *ElasticsearchKeyValueStore) PutAll(entries []*Entry) error {
	if len(entries) == 0 {
		return nil
	}
	bulk := s.client.Bulk()
	for _, entry := range entries {
		keyParts := strings.Split(entry.Key, "/")
		if len(keyParts) != 3 {
			return fmt.Errorf("invalid key: '%s'", entry.Key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.witness.Assert(entry.Value)
		s.checkOrCreateIndex(indexName, indexType)

		bulk.Add(elastic.NewBulkIndexRequest().
			Index(indexName).
			Type(indexType).
			Id(valueID).
			Doc(entry.Value),
		)
	}
	_, err := bulk.Do(s.context)
	return err
}

// Delete removes key from store
func (s *ElasticsearchKeyValueStore) Delete(key string) error {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	response, err := s.client.Delete().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if response != nil && !response.Found {
		return nil
	}

	return err
}

// Flush the Elasticsearch translog to disk
func (s *ElasticsearchKeyValueStore) Flush() error {
	log.Println("Flusing ES indexes...")
	_, err := s.client.Flush("_all").
		WaitIfOngoing(true).
		Do(s.context)
	log.Println("Done flusing ES indexes.")
	return err
}
<commit_msg>Fix JSON parser error in index settings<commit_after>package kv

import (
	"encoding/json"
	"fmt"
	"strings"

	"log"

	"golang.org/x/net/context"
	elastic "gopkg.in/olivere/elastic.v5"

	"github.com/movio/kasper/util"
)

const indexSettings = `{
	"index.translog.durability": "sync"
}`

const indexMapping = `{
	"_all" : {
		"enabled" : false
	},
	"dynamic_templates": [{
		"no_index": {
			"mapping": {
				"index": "no"
			},
			"match": "*"
		}
	}]
}`

type indexAndType struct {
	indexName string
	indexType string
}

// ElasticsearchKeyValueStore is a key-value storage that uses ElasticSearch.
// In this key-value store, all keys must have the format "<index>/<type>/<_id>".
type ElasticsearchKeyValueStore struct {
	witness         *util.StructPtrWitness
	client          *elastic.Client
	context         context.Context
	existingIndexes []indexAndType
}

// NewESKeyValueStore creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
func NewESKeyValueStore(url string, structPtr interface{}) *ElasticsearchKeyValueStore {
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetSniff(false), // FIXME: workaround for issues with ES in docker
	)
	if err != nil {
		panic(fmt.Sprintf("Cannot create ElasticSearch Client to '%s': %s", url, err))
	}
	return &ElasticsearchKeyValueStore{
		witness:         util.NewStructPtrWitness(structPtr),
		client:          client,
		context:         context.Background(),
		existingIndexes: nil,
	}
}

func (s *ElasticsearchKeyValueStore) checkOrCreateIndex(indexName string, indexType string) {
	for _, existing := range s.existingIndexes {
		if existing.indexName == indexName && existing.indexType == indexType {
			return
		}
	}
	exists, err := s.client.IndexExists(indexName).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to check if index exists: %s", err))
	}
	if !exists {
		_, err = s.client.CreateIndex(indexName).BodyString(indexSettings).Do(s.context)
		if err != nil {
			panic(fmt.Sprintf("Failed to create index: %s", err))
		}
		s.putMapping(indexName, indexType)
	}

	s.existingIndexes = append(s.existingIndexes, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) putMapping(indexName string, indexType string) {
	resp, err := s.client.PutMapping().Index(indexName).Type(indexType).BodyString(indexMapping).Do(s.context)
	if err != nil {
		panic(fmt.Sprintf("Failed to put mapping for index: %s/%s: %s", indexName, indexType, err))
	}
	if resp == nil {
		panic(fmt.Sprintf("Expected put mapping response; got: %v", resp))
	}
	if !resp.Acknowledged {
		panic(fmt.Sprintf("Expected put mapping ack; got: %v", resp.Acknowledged))
	}
}

// Get gets value by key from store
func (s *ElasticsearchKeyValueStore) Get(key string) (interface{}, error) {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return nil, fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	rawValue, err := s.client.Get().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if fmt.Sprintf("%s", err) == "elastic: Error 404 (Not Found)" {
		return s.witness.Nil(), nil
	}

	if err != nil {
		return s.witness.Nil(), err
	}

	if !rawValue.Found {
		return s.witness.Nil(), nil
	}

	structPtr := s.witness.Allocate()
	err = json.Unmarshal(*rawValue.Source, structPtr)
	if err != nil {
		return s.witness.Nil(), err
	}
	return structPtr, nil
}

// TBD
func (s *ElasticsearchKeyValueStore) GetAll(keys []string) ([]*Entry, error) {
	multiGet := s.client.MultiGet()
	for _, key := range keys {
		keyParts := strings.Split(key, "/")
		if len(keyParts) != 3 {
			return nil, fmt.Errorf("invalid key: '%s'", key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.checkOrCreateIndex(indexName, indexType)

		item := elastic.NewMultiGetItem().
			Index(indexName).
			Type(indexType).
			Id(valueID)

		multiGet.Add(item)
	}
	response, err := multiGet.Do(s.context)
	if err != nil {
		return nil, err
	}
	entries := make([]*Entry, len(keys))
	for i, doc := range response.Docs {
		var structPtr interface{}
		if !doc.Found {
			structPtr = s.witness.Nil()
		} else {
			structPtr = s.witness.Allocate()
			err = json.Unmarshal(*doc.Source, structPtr)
			if err != nil {
				return nil, err
			}
		}
		entries[i] = &Entry{keys[i], structPtr}
	}
	return entries, nil
}

// Put updates key in store with serialized value
func (s *ElasticsearchKeyValueStore) Put(key string, structPtr interface{}) error {
	s.witness.Assert(structPtr)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	_, err := s.client.Index().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		BodyJson(structPtr).
		Do(s.context)

	return err
}

// PutAll bulk executes Put operation for several entries
func (s *ElasticsearchKeyValueStore) PutAll(entries []*Entry) error {
	if len(entries) == 0 {
		return nil
	}
	bulk := s.client.Bulk()
	for _, entry := range entries {
		keyParts := strings.Split(entry.Key, "/")
		if len(keyParts) != 3 {
			return fmt.Errorf("invalid key: '%s'", entry.Key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.witness.Assert(entry.Value)
		s.checkOrCreateIndex(indexName, indexType)

		bulk.Add(elastic.NewBulkIndexRequest().
			Index(indexName).
			Type(indexType).
			Id(valueID).
			Doc(entry.Value),
		)
	}
	_, err := bulk.Do(s.context)
	return err
}

// Delete removes key from store
func (s *ElasticsearchKeyValueStore) Delete(key string) error {
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		return fmt.Errorf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	response, err := s.client.Delete().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if response != nil && !response.Found {
		return nil
	}

	return err
}

// Flush the Elasticsearch translog to disk
func (s *ElasticsearchKeyValueStore) Flush() error {
	log.Println("Flusing ES indexes...")
	_, err := s.client.Flush("_all").
		WaitIfOngoing(true).
		Do(s.context)
	log.Println("Done flusing ES indexes.")
	return err
}
<|endoftext|>


<commit_before>// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

var alias = "cdj"

var bootstrap = `
%s() {
	CDDIR="$@"
	if [ -z "$CDDIR" ]; then
		cd $(jig root)
	else
		cd $(jig ls -n1 $CDDIR)
	fi
}
`

// bootstrapCmd represents the bootstrap command
var bootstrapCmd = &cobra.Command{
	Use:   "bootstrap",
	Short: "A brief description of your command",
	Long: `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:

Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf(bootstrap, alias)
	},
}

func init() {
	RootCmd.AddCommand(bootstrapCmd)
	bootstrapCmd.Flags().StringVarP(&alias, "cd-command", "c", "cdj", "The command name to use for changing dirs")
}
<commit_msg>Add custom jigroot for cdj<commit_after>// Copyright © 2016 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

var (
	alias   = "cdj"
	jigroot = ""
)

var bootstrap = `
%s() {
	CDDIR="$@"
	if [ -z "$CDDIR" ]; then
		cd $(%s jig root)
	else
		cd $(%s jig ls -n1 $CDDIR)
	fi
}
`

// bootstrapCmd represents the bootstrap command
var bootstrapCmd = &cobra.Command{
	Use:   "bootstrap",
	Short: "Install jig tools into your shell",
	Run: func(cmd *cobra.Command, args []string) {
		if jigroot != "" {
			jigroot = fmt.Sprintf("JIGROOT=%s", jigroot)
		}
		fmt.Printf(bootstrap, alias, jigroot, jigroot)
	},
}

func init() {
	RootCmd.AddCommand(bootstrapCmd)
	bootstrapCmd.Flags().StringVarP(&alias, "cd-command", "c", "cdj", "The command name to use for changing dirs")
	bootstrapCmd.Flags().StringVarP(&jigroot, "with-jigroot", "j", "", "Use a custom jig root for this evaluation")
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/kinesis"
	_ "github.com/lib/pq"
)

type dbInfo struct {
	Host     string
	Database string
	Username string
	Password string
}

type topicMessage struct {
	TopicID int    `json:"topic_id"`
	Message string `json:"message"`
}

type topicPageMessage struct {
	TopicID  int    `json:"topic_id"`
	Message  string `json:"message"`
	PageNum  int    `json:"page_num"`
	LastPage bool   `json:"last_page"`
}

func main() {
	pub, secret, _ := getSettings()
	auth := aws.Auth{AccessKey: pub, SecretKey: secret}
	K := kinesis.New(auth, aws.USEast)
	for k, v := range os.Args {
		log.Println(k, v)
	}
	log.Println("-----")
	if len(os.Args) > 1 {
		rawData := os.Args[1]

		var kpayload KinesisPayload
		err := json.Unmarshal([]byte(rawData), &kpayload)
		if err != nil {
			log.Println("Error Kinesisis Payload:", err)
		}
		for _, v := range kpayload.Records {
			log.Println("Record:", v)
			var tm topicMessage
			sDec, errDec := base64.StdEncoding.DecodeString(v.Kinesis.Data)
			if errDec != nil {
				log.Println("Error:", errDec)
			} else {
				log.Println(string(sDec))
				errJSON := json.Unmarshal(sDec, &tm)
				if errJSON != nil {
					log.Println("Error:", errJSON)
					return
				}
				pageCount := getDevicesByTopicIDPageCount(tm.TopicID)
				for i := 0; i < pageCount; i++ {
					tpm := topicPageMessage{}
					tpm.Message = tm.Message
					tpm.TopicID = tm.TopicID
					tpm.PageNum = i + 1
					if i+1 == pageCount {
						tpm.LastPage = true
					}
					jsonData, jsonErr := json.Marshal(&tpm)
					if jsonErr != nil {
						log.Println("Error:", jsonErr)
						break
					}
					_, err := K.PutRecord("topic-message-page", "topic-page", jsonData, "", "")
					if err != nil {
						log.Println("Error:", err)
					}
					log.Println("Sent...", tpm)
				}
				log.Println("All done!")
			}
		}
		return

	}
	log.Println("Error: os.Args was 1 length.")
}

func getDevicesByTopicIDPageCount(topicID int) int {
	var count int
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			((count(u.endpointarn)/10000)+1) as pagecoun
		from
			subscription s ,userdevices u
		where
			s.topicid= $1 and
			s.userID=u.userid

;`, topicID)
	if err != nil {
		panic(err)
	}
	rows.Next()
	errScan := rows.Scan(&count)
	if errScan != nil {
		panic(errScan)
	}
	return count
}

func getSettings() (string, string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", "", nil
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Access"], settingsMap["Secret"], nil
}

func getDBSettings() *dbInfo {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		log.Println("Error:", err)
		return nil
	}
	db := dbInfo{}
	err2 := json.Unmarshal(file, &db)
	if err2 != nil {
		log.Println("Error:", err2)
		return nil
	}
	return &db
}

type KinesisPayload struct {
	Records []struct {
		AwsRegion         string `json:"awsRegion"`
		EventID           string `json:"eventID"`
		EventName         string `json:"eventName"`
		EventSource       string `json:"eventSource"`
		EventSourceARN    string `json:"eventSourceARN"`
		EventVersion      string `json:"eventVersion"`
		InvokeIdentityArn string `json:"invokeIdentityArn"`
		Kinesis           struct {
			Data                 string `json:"data"`
			KinesisSchemaVersion string `json:"kinesisSchemaVersion"`
			PartitionKey         string `json:"partitionKey"`
			SequenceNumber       string `json:"sequenceNumber"`
		} `json:"kinesis"`
	} `json:"Records"`
}
<commit_msg>log last page<commit_after>package main

import (
	"database/sql"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/AdRoll/goamz/aws"
	"github.com/AdRoll/goamz/kinesis"
	_ "github.com/lib/pq"
)

type dbInfo struct {
	Host     string
	Database string
	Username string
	Password string
}

type topicMessage struct {
	TopicID int    `json:"topic_id"`
	Message string `json:"message"`
}

type topicPageMessage struct {
	TopicID  int    `json:"topic_id"`
	Message  string `json:"message"`
	PageNum  int    `json:"page_num"`
	LastPage bool   `json:"last_page"`
}

func main() {
	pub, secret, _ := getSettings()
	auth := aws.Auth{AccessKey: pub, SecretKey: secret}
	K := kinesis.New(auth, aws.USEast)
	for k, v := range os.Args {
		log.Println(k, v)
	}
	log.Println("-----")
	if len(os.Args) > 1 {
		rawData := os.Args[1]

		var kpayload KinesisPayload
		err := json.Unmarshal([]byte(rawData), &kpayload)
		if err != nil {
			log.Println("Error Kinesisis Payload:", err)
		}
		for _, v := range kpayload.Records {
			log.Println("Record:", v)
			var tm topicMessage
			sDec, errDec := base64.StdEncoding.DecodeString(v.Kinesis.Data)
			if errDec != nil {
				log.Println("Error:", errDec)
			} else {
				log.Println(string(sDec))
				errJSON := json.Unmarshal(sDec, &tm)
				if errJSON != nil {
					log.Println("Error:", errJSON)
					return
				}
				pageCount := getDevicesByTopicIDPageCount(tm.TopicID)
				for i := 0; i < pageCount; i++ {
					tpm := topicPageMessage{}
					tpm.Message = tm.Message
					tpm.TopicID = tm.TopicID
					tpm.PageNum = i + 1
					if i+1 == pageCount {
						tpm.LastPage = true
						log.Println("LastPage is", i+1)
					}
					jsonData, jsonErr := json.Marshal(&tpm)
					if jsonErr != nil {
						log.Println("Error:", jsonErr)
						break
					}
					_, err := K.PutRecord("topic-message-page", "topic-page", jsonData, "", "")
					if err != nil {
						log.Println("Error:", err)
					}
					log.Println("Sent...", tpm)
				}
				log.Println("All done!")
			}
		}
		return

	}
	log.Println("Error: os.Args was 1 length.")
}

func getDevicesByTopicIDPageCount(topicID int) int {
	var count int
	info := getDBSettings()
	db, errCon := sql.Open("postgres", fmt.Sprintf("host=%v user=%v password=%v dbname=%v sslmode=require", info.Host, info.Username, info.Password, info.Database))
	defer db.Close()
	if errCon != nil {
		log.Fatal(errCon)
	}
	rows, err := db.Query(`
		select
			((count(u.endpointarn)/10000)+1) as pagecoun
		from
			subscription s ,userdevices u
		where
			s.topicid= $1 and
			s.userID=u.userid

;`, topicID)
	if err != nil {
		panic(err)
	}
	rows.Next()
	errScan := rows.Scan(&count)
	if errScan != nil {
		panic(errScan)
	}
	return count
}

func getSettings() (string, string, error) {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		return "", "", nil
	}
	settingsMap := make(map[string]string)
	json.Unmarshal(file, &settingsMap)
	return settingsMap["Access"], settingsMap["Secret"], nil
}

func getDBSettings() *dbInfo {
	file, err := ioutil.ReadFile("./settings.json")
	if err != nil {
		log.Println("Error:", err)
		return nil
	}
	db := dbInfo{}
	err2 := json.Unmarshal(file, &db)
	if err2 != nil {
		log.Println("Error:", err2)
		return nil
	}
	return &db
}

type KinesisPayload struct {
	Records []struct {
		AwsRegion         string `json:"awsRegion"`
		EventID           string `json:"eventID"`
		EventName         string `json:"eventName"`
		EventSource       string `json:"eventSource"`
		EventSourceARN    string `json:"eventSourceARN"`
		EventVersion      string `json:"eventVersion"`
		InvokeIdentityArn string `json:"invokeIdentityArn"`
		Kinesis           struct {
			Data                 string `json:"data"`
			KinesisSchemaVersion string `json:"kinesisSchemaVersion"`
			PartitionKey         string `json:"partitionKey"`
			SequenceNumber       string `json:"sequenceNumber"`
		} `json:"kinesis"`
	} `json:"Records"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"

	"github.com/pda/go6502/bus"
	"github.com/pda/go6502/cpu"
	"github.com/pda/go6502/memory"
	"github.com/rjw57/burisim-golang"
)

func main() {
	// create the Buri CPU
	cpu := cpu.Cpu{}

	// create an address bus
	if bus, err := bus.CreateBus(); err == nil {
		cpu.Bus = bus
	} else {
		fmt.Errorf("error creating bus: %v", err)
	}

	// wire in the RAM at addresses [0x0000, 0x7fff]
	if err := cpu.Bus.Attach(&memory.Ram{}, "RAM", 0); err != nil {
		fmt.Errorf("error attaching RAM: %v", err)
	}

	// load the ROM from image
	rp := os.Args[1]
	if rom, err := memory.RomFromFile(rp); err != nil {
		fmt.Errorf("error loading ROM: %v", err)
	} else {
		// wire in the ROM at addresses [0xe000, 0xffff]
		if err := cpu.Bus.Attach(rom, "ROM", 0xe000); err != nil {
			fmt.Errorf("error attaching ROM: %v", err)
		}
	}

	l, err := net.Listen("tcp", ":9000")
	if err != nil {
		fmt.Errorf("error listening on socket: %v", err)
	}
	acia1 := buri.CreateACIAOnListener(l)

	// attach ACIA1 at 0xdffc
	if err := cpu.Bus.Attach(acia1, "ACIA1", 0xdffc); err != nil {
		fmt.Errorf("error attaching ACIA1: %v", err)
	}

	// go forth and execute
	cpu.Reset()
	for {
		cpu.Step()
	}
}
<commit_msg>cmd/buri: use codegangsta/cli package for CLI options<commit_after>package main

import (
	"fmt"
	"net"
	"os"

	"github.com/codegangsta/cli"
	"github.com/pda/go6502/bus"
	"github.com/pda/go6502/cpu"
	"github.com/pda/go6502/memory"
	"github.com/rjw57/burisim-golang"
)

func runSim(c *cli.Context) {
	args := c.Args()
	if len(args) == 0 {
		panic("no ROM image specified")
	}

	// create the Buri CPU
	cpu := cpu.Cpu{}

	// create an address bus
	if bus, err := bus.CreateBus(); err == nil {
		cpu.Bus = bus
	} else {
		fmt.Errorf("error creating bus: %v", err)
	}

	// wire in the RAM at addresses [0x0000, 0x7fff]
	if err := cpu.Bus.Attach(&memory.Ram{}, "RAM", 0); err != nil {
		fmt.Errorf("error attaching RAM: %v", err)
	}

	// load the ROM from image
	rp := args[0]
	if rom, err := memory.RomFromFile(rp); err != nil {
		fmt.Errorf("error loading ROM: %v", err)
	} else {
		// wire in the ROM at addresses [0xe000, 0xffff]
		if err := cpu.Bus.Attach(rom, "ROM", 0xe000); err != nil {
			fmt.Errorf("error attaching ROM: %v", err)
		}
	}

	l, err := net.Listen("tcp", ":9000")
	if err != nil {
		fmt.Errorf("error listening on socket: %v", err)
	}
	acia1 := buri.CreateACIAOnListener(l)

	// attach ACIA1 at 0xdffc
	if err := cpu.Bus.Attach(acia1, "ACIA1", 0xdffc); err != nil {
		fmt.Errorf("error attaching ACIA1: %v", err)
	}

	// go forth and execute
	cpu.Reset()
	for {
		cpu.Step()
	}
}

func main() {
	app := cli.NewApp()
	app.Name = "buri"
	app.Usage = "emulate the Buri microcomputer"
	app.Action = runSim
	app.Run(os.Args)
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"path"
	"runtime"
	"runtime/pprof"
	"sort"
	"strings"
	"time"

	"github.com/gorilla/mux"

	"github.com/blevesearch/bleve"
	bleveHttp "github.com/blevesearch/bleve/http"
	bleveRegistry "github.com/blevesearch/bleve/registry"

	log "github.com/couchbase/clog"
	"github.com/couchbase/go-couchbase"
	"github.com/couchbaselabs/cbft"
)

var VERSION = "0.0.0"

var expvars = expvar.NewMap("stats")

type Flags struct {
	BindAddr   string
	DataDir    string
	LogFlags   string
	StaticDir  string
	StaticETag string
	Server     string
	Tags       string
	Container  string
	Weight     int
	Register   string
	CfgConnect string
}

func initFlags(flags *Flags) map[string][]string {
	flagAliases := map[string][]string{} // main flag name => all aliases.

	flagStringVar := func(v *string, names []string,
		defaultVal, usage string) {
		for _, name := range names {
			flag.StringVar(v, name, defaultVal, usage)
		}
		flagAliases[names[0]] = names
	}

	flagIntVar := func(v *int, names []string,
		defaultVal int, usage string) {
		for _, name := range names {
			flag.IntVar(v, name, defaultVal, usage)
		}
		flagAliases[names[0]] = names
	}

	flagStringVar(&flags.BindAddr, []string{"bindAddr"},
		"localhost:8095",
		"http listen address:port")
	flagStringVar(&flags.DataDir, []string{"dataDir", "data"},
		"data",
		"directory path where index data and"+
			"\nlocal configuration files will be stored")
	flagStringVar(&flags.LogFlags, []string{"logFlags"},
		"",
		"comma-separated logging control flags")
	flagStringVar(&flags.StaticDir, []string{"staticDir"},
		"static", "directory for static web UI content")
	flagStringVar(&flags.StaticETag, []string{"staticETag"},
		"",
		"static etag value")
	flagStringVar(&flags.Server, []string{"server"},
		"",
		"url to datasource server;"+
			"\nexample for couchbase: http://localhost:8091")
	flagStringVar(&flags.Tags, []string{"tags"},
		"",
		"comma-separated list of tags (or roles) for this node")
	flagStringVar(&flags.Container, []string{"container"},
		"",
		"slash separated path of parent containers for this node,"+
			"\nfor shelf/rack/row/zone awareness")
	flagIntVar(&flags.Weight, []string{"weight"},
		1,
		"weight of this node (a more capable node has higher weight)")
	flagStringVar(&flags.Register, []string{"register"},
		"wanted",
		"register this node as wanted, wantedForce,"+
			"\nknown, knownForce, unwanted, unknown or unchanged")
	flagStringVar(&flags.CfgConnect, []string{"cfgConnect", "cfg"},
		"simple",
		"connection string/info to configuration provider")

	flag.Usage = func() {
		base := path.Base(os.Args[0])

		fmt.Fprintf(os.Stderr,
			"%s: couchbase full-text server\n\n", base)
		fmt.Fprintf(os.Stderr, "more information is available at:\n"+
			"  http://github.com/couchbaselabs/cbft\n\n")
		fmt.Fprintf(os.Stderr, "usage:\n  %s [flags]\n\n", base)
		fmt.Fprintf(os.Stderr, "flags:\n")

		flagsByName := map[string]*flag.Flag{}
		flag.VisitAll(func(f *flag.Flag) {
			flagsByName[f.Name] = f
		})

		flags := []string(nil)
		for name := range flagAliases {
			flags = append(flags, name)
		}
		sort.Strings(flags)

		for _, name := range flags {
			aliases := flagAliases[name]
			a := []string(nil)
			for i := len(aliases) - 1; i >= 0; i-- {
				a = append(a, aliases[i])
			}
			f := flagsByName[name]
			fmt.Fprintf(os.Stderr, "  -%s=%q\n",
				strings.Join(a, ", -"), f.Value)
			fmt.Fprintf(os.Stderr, "      %s\n",
				strings.Join(strings.Split(f.Usage, "\n"),
					"\n      "))
		}
	}

	return flagAliases
}

func main() {
	var flags Flags

	flagAliases := initFlags(&flags)

	flag.Parse()

	log.Printf("main: %s started (%s/%s)",
		os.Args[0], VERSION, cbft.VERSION)

	rand.Seed(time.Now().UTC().UnixNano())

	go dumpOnSignalForPlatform()

	mr, err := cbft.NewMsgRing(os.Stderr, 1000)
	if err != nil {
		log.Fatalf("main: could not create MsgRing, err: %v", err)
	}
	log.SetOutput(mr)

	if flags.LogFlags != "" {
		log.ParseLogFlag(flags.LogFlags)
	}

	MainWelcome(flagAliases)

	// If cfg is down, we error, leaving it to some user-supplied
	// outside watchdog to backoff and restart/retry.
	cfg, err := MainCfg(flags.CfgConnect, flags.DataDir)
	if err != nil {
		log.Fatalf("main: could not start cfg, cfgConnect: %s, err: %v\n"+
			"  Please check that your -cfg/-cfgConnect parameter (%s)\n"+
			"  is correct and/or that your configuration provider\n"+
			"  is available.",
			flags.CfgConnect, err, flags.CfgConnect)
		return
	}

	uuid, err := MainUUID(flags.DataDir)
	if err != nil {
		log.Fatalf(fmt.Sprintf("%v", err))
		return
	}

	var tagsArr []string
	if flags.Tags != "" {
		tagsArr = strings.Split(flags.Tags, ",")
	}

	expvars.Set("indexes", bleveHttp.IndexStats())

	router, err := MainStart(cfg, uuid, tagsArr,
		flags.Container, flags.Weight,
		flags.BindAddr, flags.DataDir,
		flags.StaticDir, flags.StaticETag,
		flags.Server, flags.Register, mr)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", router)
	log.Printf("main: listening on: %v", flags.BindAddr)
	log.Fatal(http.ListenAndServe(flags.BindAddr, nil))
}

func MainWelcome(flagAliases map[string][]string) {
	flag.VisitAll(func(f *flag.Flag) {
		if flagAliases[f.Name] != nil {
			log.Printf("  -%s=%q\n", f.Name, f.Value)
		}
	})
	log.Printf("  GOMAXPROCS=%d", runtime.GOMAXPROCS(-1))

	log.Printf("main: registered bleve stores")
	types, instances := bleveRegistry.KVStoreTypesAndInstances()
	for _, s := range types {
		log.Printf("  %s", s)
	}
	for _, s := range instances {
		log.Printf("  %s", s)
	}
}

func MainUUID(dataDir string) (string, error) {
	uuid := cbft.NewUUID()
	uuidPath := dataDir + string(os.PathSeparator) + "cbft.uuid"
	uuidBuf, err := ioutil.ReadFile(uuidPath)
	if err == nil {
		uuid = strings.TrimSpace(string(uuidBuf))
		if uuid == "" {
			return "", fmt.Errorf("error: could not parse uuidPath: %s",
				uuidPath)
		}
		log.Printf("main: manager uuid: %s", uuid)
		log.Printf("main: manager uuid was reloaded")
	} else {
		log.Printf("main: manager uuid: %s", uuid)
		log.Printf("main: manager uuid was generated")
	}
	err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600)
	if err != nil {
		return "", fmt.Errorf("error: could not write uuidPath: %s\n"+
			"  Please check that your -data/-dataDir parameter (%s)\n"+
			"  is to a writable directory.", uuidPath, dataDir)
	}
	return uuid, nil
}

func MainStart(cfg cbft.Cfg, uuid string, tags []string, container string,
	weight int, bindAddr, dataDir, staticDir, staticETag, server string,
	register string, mr *cbft.MsgRing) (
	*mux.Router, error) {
	if server == "" {
		return nil, fmt.Errorf("error: server URL required (-server)")
	}

	_, err := couchbase.Connect(server)
	if err != nil {
		return nil, fmt.Errorf("error: could not connect to server,"+
			" URL: %s, err: %v",
			server, err)
	}

	mgr := cbft.NewManager(cbft.VERSION, cfg, uuid, tags, container, weight,
		bindAddr, dataDir, server, &MainHandlers{})
	err = mgr.Start(register)
	if err != nil {
		return nil, err
	}

	router, _, err :=
		cbft.NewManagerRESTRouter(VERSION, mgr, staticDir, staticETag, mr)

	return router, err
}

type MainHandlers struct{}

func (meh *MainHandlers) OnRegisterPIndex(pindex *cbft.PIndex) {
	bindex, ok := pindex.Impl.(bleve.Index)
	if ok {
		bleveHttp.RegisterIndexName(pindex.Name, bindex)
	}
}

func (meh *MainHandlers) OnUnregisterPIndex(pindex *cbft.PIndex) {
	bleveHttp.UnregisterIndexByName(pindex.Name)
}

func dumpOnSignal(signals ...os.Signal) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, signals...)
	for _ = range c {
		log.Printf("dump: goroutine...")
		pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
		log.Printf("dump: heap...")
		pprof.Lookup("heap").WriteTo(os.Stderr, 1)
	}
}
<commit_msg>more friendly err message if bindAddr is unavailable<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"expvar"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"path"
	"runtime"
	"runtime/pprof"
	"sort"
	"strings"
	"time"

	"github.com/gorilla/mux"

	"github.com/blevesearch/bleve"
	bleveHttp "github.com/blevesearch/bleve/http"
	bleveRegistry "github.com/blevesearch/bleve/registry"

	log "github.com/couchbase/clog"
	"github.com/couchbase/go-couchbase"
	"github.com/couchbaselabs/cbft"
)

var VERSION = "0.0.0"

var expvars = expvar.NewMap("stats")

type Flags struct {
	BindAddr   string
	DataDir    string
	LogFlags   string
	StaticDir  string
	StaticETag string
	Server     string
	Tags       string
	Container  string
	Weight     int
	Register   string
	CfgConnect string
}

func initFlags(flags *Flags) map[string][]string {
	flagAliases := map[string][]string{} // main flag name => all aliases.

	s := func(v *string, names []string,
		defaultVal, usage string) {
		for _, name := range names {
			flag.StringVar(v, name, defaultVal, usage)
		}
		flagAliases[names[0]] = names
	}

	i := func(v *int, names []string,
		defaultVal int, usage string) {
		for _, name := range names {
			flag.IntVar(v, name, defaultVal, usage)
		}
		flagAliases[names[0]] = names
	}

	s(&flags.BindAddr, []string{"bindAddr"},
		"localhost:8095",
		"http listen address:port")
	s(&flags.DataDir, []string{"dataDir", "data"},
		"data",
		"directory path where index data and"+
			"\nlocal configuration files will be stored")
	s(&flags.LogFlags, []string{"logFlags"},
		"",
		"comma-separated logging control flags")
	s(&flags.StaticDir, []string{"staticDir"},
		"static", "directory for static web UI content")
	s(&flags.StaticETag, []string{"staticETag"},
		"",
		"static etag value")
	s(&flags.Server, []string{"server"},
		"",
		"url to datasource server;"+
			"\nexample for couchbase: http://localhost:8091")
	s(&flags.Tags, []string{"tags"},
		"",
		"comma-separated list of tags (or roles) for this node")
	s(&flags.Container, []string{"container"},
		"",
		"slash separated path of parent containers for this node,"+
			"\nfor shelf/rack/row/zone awareness")
	i(&flags.Weight, []string{"weight"},
		1,
		"weight of this node (a more capable node has higher weight)")
	s(&flags.Register, []string{"register"},
		"wanted",
		"register this node as wanted, wantedForce,"+
			"\nknown, knownForce, unwanted, unknown or unchanged")
	s(&flags.CfgConnect, []string{"cfgConnect", "cfg"},
		"simple",
		"connection string/info to configuration provider")

	flag.Usage = func() {
		base := path.Base(os.Args[0])

		fmt.Fprintf(os.Stderr,
			"%s: couchbase full-text server\n\n", base)
		fmt.Fprintf(os.Stderr, "more information is available at:\n"+
			"  http://github.com/couchbaselabs/cbft\n\n")
		fmt.Fprintf(os.Stderr, "usage:\n  %s [flags]\n\n", base)
		fmt.Fprintf(os.Stderr, "flags:\n")

		flagsByName := map[string]*flag.Flag{}
		flag.VisitAll(func(f *flag.Flag) {
			flagsByName[f.Name] = f
		})

		flags := []string(nil)
		for name := range flagAliases {
			flags = append(flags, name)
		}
		sort.Strings(flags)

		for _, name := range flags {
			aliases := flagAliases[name]
			a := []string(nil)
			for i := len(aliases) - 1; i >= 0; i-- {
				a = append(a, aliases[i])
			}
			f := flagsByName[name]
			fmt.Fprintf(os.Stderr, "  -%s=%q\n",
				strings.Join(a, ", -"), f.Value)
			fmt.Fprintf(os.Stderr, "      %s\n",
				strings.Join(strings.Split(f.Usage, "\n"),
					"\n      "))
		}
	}

	return flagAliases
}

func main() {
	var flags Flags
	flagAliases := initFlags(&flags)
	flag.Parse()

	log.Printf("main: %s started (%s/%s)",
		os.Args[0], VERSION, cbft.VERSION)

	rand.Seed(time.Now().UTC().UnixNano())

	go dumpOnSignalForPlatform()

	mr, err := cbft.NewMsgRing(os.Stderr, 1000)
	if err != nil {
		log.Fatalf("main: could not create MsgRing, err: %v", err)
	}
	log.SetOutput(mr)

	if flags.LogFlags != "" {
		log.ParseLogFlag(flags.LogFlags)
	}

	MainWelcome(flagAliases)

	// If cfg is down, we error, leaving it to some user-supplied
	// outside watchdog to backoff and restart/retry.
	cfg, err := MainCfg(flags.CfgConnect, flags.DataDir)
	if err != nil {
		log.Fatalf("main: could not start cfg, cfgConnect: %s, err: %v\n"+
			"  Please check that your -cfg/-cfgConnect parameter (%s)\n"+
			"  is correct and/or that your configuration provider\n"+
			"  is available.",
			flags.CfgConnect, err, flags.CfgConnect)
		return
	}

	uuid, err := MainUUID(flags.DataDir)
	if err != nil {
		log.Fatalf(fmt.Sprintf("%v", err))
		return
	}

	var tagsArr []string
	if flags.Tags != "" {
		tagsArr = strings.Split(flags.Tags, ",")
	}

	expvars.Set("indexes", bleveHttp.IndexStats())

	router, err := MainStart(cfg, uuid, tagsArr,
		flags.Container, flags.Weight,
		flags.BindAddr, flags.DataDir,
		flags.StaticDir, flags.StaticETag,
		flags.Server, flags.Register, mr)
	if err != nil {
		log.Fatal(err)
	}

	http.Handle("/", router)

	log.Printf("main: listening on: %v", flags.BindAddr)
	err = http.ListenAndServe(flags.BindAddr, nil)
	log.Fatalf("main: listen, err: %v\n"+
		"  Please check that your -bindAddr parameter (%s)\n"+
		"  is correct and available.", err, flags.BindAddr)
}

func MainWelcome(flagAliases map[string][]string) {
	flag.VisitAll(func(f *flag.Flag) {
		if flagAliases[f.Name] != nil {
			log.Printf("  -%s=%q\n", f.Name, f.Value)
		}
	})
	log.Printf("  GOMAXPROCS=%d", runtime.GOMAXPROCS(-1))

	log.Printf("main: registered bleve stores")
	types, instances := bleveRegistry.KVStoreTypesAndInstances()
	for _, s := range types {
		log.Printf("  %s", s)
	}
	for _, s := range instances {
		log.Printf("  %s", s)
	}
}

func MainUUID(dataDir string) (string, error) {
	uuid := cbft.NewUUID()
	uuidPath := dataDir + string(os.PathSeparator) + "cbft.uuid"
	uuidBuf, err := ioutil.ReadFile(uuidPath)
	if err == nil {
		uuid = strings.TrimSpace(string(uuidBuf))
		if uuid == "" {
			return "", fmt.Errorf("error: could not parse uuidPath: %s",
				uuidPath)
		}
		log.Printf("main: manager uuid: %s", uuid)
		log.Printf("main: manager uuid was reloaded")
	} else {
		log.Printf("main: manager uuid: %s", uuid)
		log.Printf("main: manager uuid was generated")
	}
	err = ioutil.WriteFile(uuidPath, []byte(uuid), 0600)
	if err != nil {
		return "", fmt.Errorf("error: could not write uuidPath: %s\n"+
			"  Please check that your -data/-dataDir parameter (%s)\n"+
			"  is to a writable directory.", uuidPath, dataDir)
	}
	return uuid, nil
}

func MainStart(cfg cbft.Cfg, uuid string, tags []string, container string,
	weight int, bindAddr, dataDir, staticDir, staticETag, server string,
	register string, mr *cbft.MsgRing) (
	*mux.Router, error) {
	if server == "" {
		return nil, fmt.Errorf("error: server URL required (-server)")
	}

	_, err := couchbase.Connect(server)
	if err != nil {
		return nil, fmt.Errorf("error: could not connect to server,"+
			" URL: %s, err: %v",
			server, err)
	}

	mgr := cbft.NewManager(cbft.VERSION, cfg, uuid, tags, container, weight,
		bindAddr, dataDir, server, &MainHandlers{})
	err = mgr.Start(register)
	if err != nil {
		return nil, err
	}

	router, _, err :=
		cbft.NewManagerRESTRouter(VERSION, mgr, staticDir, staticETag, mr)

	return router, err
}

type MainHandlers struct{}

func (meh *MainHandlers) OnRegisterPIndex(pindex *cbft.PIndex) {
	bindex, ok := pindex.Impl.(bleve.Index)
	if ok {
		bleveHttp.RegisterIndexName(pindex.Name, bindex)
	}
}

func (meh *MainHandlers) OnUnregisterPIndex(pindex *cbft.PIndex) {
	bleveHttp.UnregisterIndexByName(pindex.Name)
}

func dumpOnSignal(signals ...os.Signal) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, signals...)
	for _ = range c {
		log.Printf("dump: goroutine...")
		pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
		log.Printf("dump: heap...")
		pprof.Lookup("heap").WriteTo(os.Stderr, 1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/signal"
	"sort"
	"strings"

	"github.com/ianremmler/clac"
	"github.com/peterh/liner"
)

var (
	lnr     *liner.State
	cl      = clac.New()
	cmdList = []string{}
	cmdMap  = map[string]func() error{
		"neg":    cl.Neg,
		"abs":    cl.Abs,
		"inv":    cl.Inv,
		"+":      cl.Add,
		"-":      cl.Sub,
		"*":      cl.Mul,
		"/":      cl.Div,
		"mod":    cl.Mod,
		"exp":    cl.Exp,
		"pow":    cl.Pow,
		"pow2":   cl.Pow2,
		"pow10":  cl.Pow10,
		"ln":     cl.Ln,
		"log":    cl.Log,
		"lg":     cl.Lg,
		"sqrt":   cl.Sqrt,
		"hypot":  cl.Hypot,
		"sin":    cl.Sin,
		"cos":    cl.Cos,
		"tan":    cl.Tan,
		"asin":   cl.Asin,
		"acos":   cl.Acos,
		"atan":   cl.Atan,
		"sinh":   cl.Sin,
		"cosh":   cl.Cos,
		"tanh":   cl.Tan,
		"asinh":  cl.Asin,
		"acosh":  cl.Acos,
		"atanh":  cl.Atan,
		"atan2":  cl.Atan2,
		"d2r":    cl.D2R,
		"r2d":    cl.R2D,
		"floor":  cl.Floor,
		"ceil":   cl.Ceil,
		"trunc":  cl.Trunc,
		"and":    cl.And,
		"or":     cl.Or,
		"xor":    cl.Xor,
		"not":    cl.Not,
		"clear":  cl.Clear,
		"drop":   cl.Drop,
		"dropn":  cl.Dropn,
		"dropr":  cl.Dropr,
		"dup":    cl.Dup,
		"dupn":   cl.Dupn,
		"dupr":   cl.Dupr,
		"pick":   cl.Pick,
		"swap":   cl.Swap,
		"undo":   cl.Undo,
		"redo":   cl.Redo,
		"rot":    func() error { return cl.Rot(true) },
		"rotr":   func() error { return cl.Rotr(true) },
		"unrot":  func() error { return cl.Rot(false) },
		"unrotr": func() error { return cl.Rotr(false) },
		"pi":     func() error { return cl.Push(math.Pi) },
		"e":      func() error { return cl.Push(math.E) },
		"phi":    func() error { return cl.Push(math.Phi) },
		"quit":   func() error { exit(); return nil },
		"help":   func() error { help(); return nil },
	}
)

func init() {
	log.SetFlags(0)
	log.SetPrefix("Error: ")
	for cmd := range cmdMap {
		cmdList = append(cmdList, cmd)
	}
	sort.Strings(cmdList)
}

func main() {
	if processPipe() {
		fmt.Println(strings.Trim(fmt.Sprint(cl.Stack()), "[]"))
		os.Exit(0)
	}

	lnr = liner.NewLiner()
	lnr.SetWordCompleter(complete)

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)
	go func() {
		<-sigChan
		exit()
	}()

	for {
		printStack(cl.Stack())
		input, err := lnr.Prompt("> ")
		if err == io.EOF {
			exit()
		}
		if err != nil {
			continue
		}
		lnr.AppendHistory(input)
		parseInput(input)
		fmt.Println()
	}
}

func processPipe() bool {
	if stat, err := os.Stdin.Stat(); err == nil && stat.Mode()&os.ModeNamedPipe != 0 {
		if input, err := ioutil.ReadAll(os.Stdin); err == nil {
			parseInput(string(input))
			return true
		}
	}
	return false
}

func exit() {
	fmt.Println()
	lnr.Close()
	os.Exit(0)
}

func help() {
	fmt.Println()
	for i := range cmdList {
		fmt.Printf("%-8s", cmdList[i])
		if (i+1)%5 == 0 {
			fmt.Println()
		}
	}
	if len(cmdList)%5 != 0 {
		fmt.Println()
	}
	fmt.Print("\n[Press any key to continue]")
	bufio.NewReader(os.Stdin).ReadByte()
	fmt.Println()
}

func parseInput(input string) {
	cmdReader := strings.NewReader(input)
	for {
		tok := ""
		if _, err := fmt.Fscan(cmdReader, &tok); err != nil {
			if err != io.EOF {
				log.Println(err)
			}
			break
		}
		num, err := clac.ParseNum(tok)
		if err == nil {
			if err = cl.Push(num); err != nil {
				log.Println(tok+":", err)
			}
			continue
		}
		if cmd, ok := cmdMap[tok]; ok {
			if err = cmd(); err != nil {
				log.Println(tok+":", err)
			}
			continue
		}
		log.Println(tok + ": invalid input")
	}
}

func complete(in string, pos int) (string, []string, string) {
	start := strings.LastIndexAny(in[:pos], " \t") + 1
	end := len(in)
	if idx := strings.IndexAny(in[pos:], " \t"); idx >= 0 {
		end = pos + idx
	}
	head, word, tail := in[:start], in[start:end], in[end:]
	cmds := []string{}
	for i := range cmdList {
		if strings.HasPrefix(cmdList[i], word) {
			cmds = append(cmds, cmdList[i])
		}
	}
	return head, cmds, tail
}

func printStack(stack clac.Stack) {
	for i := len(stack) - 1; i >= 0; i-- {
		fmt.Printf("%2d: %16.10g", i, stack[i])
		if math.Abs(stack[i]) < math.MaxInt64 {
			fmt.Printf(" %#19x", int64(stack[i]))
		}
		fmt.Println()
	}
	fmt.Println(strings.Repeat("-", 40))
}
<commit_msg>If command line is only whitespace, don't add it to history.<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/signal"
	"sort"
	"strings"

	"github.com/ianremmler/clac"
	"github.com/peterh/liner"
)

var (
	lnr     *liner.State
	cl      = clac.New()
	cmdList = []string{}
	cmdMap  = map[string]func() error{
		"neg":    cl.Neg,
		"abs":    cl.Abs,
		"inv":    cl.Inv,
		"+":      cl.Add,
		"-":      cl.Sub,
		"*":      cl.Mul,
		"/":      cl.Div,
		"mod":    cl.Mod,
		"exp":    cl.Exp,
		"pow":    cl.Pow,
		"pow2":   cl.Pow2,
		"pow10":  cl.Pow10,
		"ln":     cl.Ln,
		"log":    cl.Log,
		"lg":     cl.Lg,
		"sqrt":   cl.Sqrt,
		"hypot":  cl.Hypot,
		"sin":    cl.Sin,
		"cos":    cl.Cos,
		"tan":    cl.Tan,
		"asin":   cl.Asin,
		"acos":   cl.Acos,
		"atan":   cl.Atan,
		"sinh":   cl.Sin,
		"cosh":   cl.Cos,
		"tanh":   cl.Tan,
		"asinh":  cl.Asin,
		"acosh":  cl.Acos,
		"atanh":  cl.Atan,
		"atan2":  cl.Atan2,
		"d2r":    cl.D2R,
		"r2d":    cl.R2D,
		"floor":  cl.Floor,
		"ceil":   cl.Ceil,
		"trunc":  cl.Trunc,
		"and":    cl.And,
		"or":     cl.Or,
		"xor":    cl.Xor,
		"not":    cl.Not,
		"clear":  cl.Clear,
		"drop":   cl.Drop,
		"dropn":  cl.Dropn,
		"dropr":  cl.Dropr,
		"dup":    cl.Dup,
		"dupn":   cl.Dupn,
		"dupr":   cl.Dupr,
		"pick":   cl.Pick,
		"swap":   cl.Swap,
		"undo":   cl.Undo,
		"redo":   cl.Redo,
		"rot":    func() error { return cl.Rot(true) },
		"rotr":   func() error { return cl.Rotr(true) },
		"unrot":  func() error { return cl.Rot(false) },
		"unrotr": func() error { return cl.Rotr(false) },
		"pi":     func() error { return cl.Push(math.Pi) },
		"e":      func() error { return cl.Push(math.E) },
		"phi":    func() error { return cl.Push(math.Phi) },
		"quit":   func() error { exit(); return nil },
		"help":   func() error { help(); return nil },
	}
)

func init() {
	log.SetFlags(0)
	log.SetPrefix("Error: ")
	for cmd := range cmdMap {
		cmdList = append(cmdList, cmd)
	}
	sort.Strings(cmdList)
}

func main() {
	if processPipe() {
		fmt.Println(strings.Trim(fmt.Sprint(cl.Stack()), "[]"))
		os.Exit(0)
	}

	lnr = liner.NewLiner()
	lnr.SetWordCompleter(complete)

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, os.Interrupt)
	go func() {
		<-sigChan
		exit()
	}()

	for {
		printStack(cl.Stack())
		input, err := lnr.Prompt("> ")
		if err == io.EOF {
			exit()
		}
		if err != nil {
			continue
		}
		if strings.TrimSpace(input) != "" {
			lnr.AppendHistory(input)
		}
		parseInput(input)
		fmt.Println()
	}
}

func processPipe() bool {
	if stat, err := os.Stdin.Stat(); err == nil && stat.Mode()&os.ModeNamedPipe != 0 {
		if input, err := ioutil.ReadAll(os.Stdin); err == nil {
			parseInput(string(input))
			return true
		}
	}
	return false
}

func exit() {
	fmt.Println()
	lnr.Close()
	os.Exit(0)
}

func help() {
	fmt.Println()
	for i := range cmdList {
		fmt.Printf("%-8s", cmdList[i])
		if (i+1)%5 == 0 {
			fmt.Println()
		}
	}
	if len(cmdList)%5 != 0 {
		fmt.Println()
	}
	fmt.Print("\n[Press any key to continue]")
	bufio.NewReader(os.Stdin).ReadByte()
	fmt.Println()
}

func parseInput(input string) {
	cmdReader := strings.NewReader(input)
	for {
		tok := ""
		if _, err := fmt.Fscan(cmdReader, &tok); err != nil {
			if err != io.EOF {
				log.Println(err)
			}
			break
		}
		num, err := clac.ParseNum(tok)
		if err == nil {
			if err = cl.Push(num); err != nil {
				log.Println(tok+":", err)
			}
			continue
		}
		if cmd, ok := cmdMap[tok]; ok {
			if err = cmd(); err != nil {
				log.Println(tok+":", err)
			}
			continue
		}
		log.Println(tok + ": invalid input")
	}
}

func complete(in string, pos int) (string, []string, string) {
	start := strings.LastIndexAny(in[:pos], " \t") + 1
	end := len(in)
	if idx := strings.IndexAny(in[pos:], " \t"); idx >= 0 {
		end = pos + idx
	}
	head, word, tail := in[:start], in[start:end], in[end:]
	cmds := []string{}
	for i := range cmdList {
		if strings.HasPrefix(cmdList[i], word) {
			cmds = append(cmds, cmdList[i])
		}
	}
	return head, cmds, tail
}

func printStack(stack clac.Stack) {
	for i := len(stack) - 1; i >= 0; i-- {
		fmt.Printf("%2d: %16.10g", i, stack[i])
		if math.Abs(stack[i]) < math.MaxInt64 {
			fmt.Printf(" %#19x", int64(stack[i]))
		}
		fmt.Println()
	}
	fmt.Println(strings.Repeat("-", 40))
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/ghodss/yaml"
	"github.com/spf13/cobra"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/coreos/dex/api"
	"github.com/coreos/dex/server"
	"github.com/coreos/dex/storage"
)

func commandServe() *cobra.Command {
	return &cobra.Command{
		Use:     "serve [ config file ]",
		Short:   "Connect to the storage and begin serving requests.",
		Long:    ``,
		Example: "dex serve config.yaml",
		Run: func(cmd *cobra.Command, args []string) {
			if err := serve(cmd, args); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(2)
			}
		},
	}
}

func serve(cmd *cobra.Command, args []string) error {
	switch len(args) {
	default:
		return errors.New("surplus arguments")
	case 0:
		// TODO(ericchiang): Consider having a default config file location.
		return errors.New("no arguments provided")
	case 1:
	}

	configFile := args[0]
	configData, err := ioutil.ReadFile(configFile)
	if err != nil {
		return fmt.Errorf("failed to read config file %s: %v", configFile, err)
	}

	var c Config
	if err := yaml.Unmarshal(configData, &c); err != nil {
		return fmt.Errorf("error parse config file %s: %v", configFile, err)
	}

	logger, err := newLogger(c.Logger.Level, c.Logger.Format)
	if err != nil {
		return fmt.Errorf("invalid config: %v", err)
	}
	if c.Logger.Level != "" {
		logger.Infof("config using log level: %s", c.Logger.Level)
	}

	// Fast checks. Perform these first for a more responsive CLI.
	checks := []struct {
		bad    bool
		errMsg string
	}{
		{c.Issuer == "", "no issuer specified in config file"},
		{!c.EnablePasswordDB && len(c.StaticPasswords) != 0, "cannot specify static passwords without enabling password db"},
		{c.Storage.Config == nil, "no storage suppied in config file"},
		{c.Web.HTTP == "" && c.Web.HTTPS == "", "must supply a HTTP/HTTPS  address to listen on"},
		{c.Web.HTTPS != "" && c.Web.TLSCert == "", "no cert specified for HTTPS"},
		{c.Web.HTTPS != "" && c.Web.TLSKey == "", "no private key specified for HTTPS"},
		{c.GRPC.TLSCert != "" && c.GRPC.Addr == "", "no address specified for gRPC"},
		{c.GRPC.TLSKey != "" && c.GRPC.Addr == "", "no address specified for gRPC"},
		{(c.GRPC.TLSCert == "") != (c.GRPC.TLSKey == ""), "must specific both a gRPC TLS cert and key"},
		{c.GRPC.TLSCert == "" && c.GRPC.TLSClientCA != "", "cannot specify gRPC TLS client CA without a gRPC TLS cert"},
	}

	for _, check := range checks {
		if check.bad {
			return fmt.Errorf("invalid config: %s", check.errMsg)
		}
	}

	logger.Infof("config issuer: %s", c.Issuer)

	var grpcOptions []grpc.ServerOption
	if c.GRPC.TLSCert != "" {
		if c.GRPC.TLSClientCA != "" {
			// Parse certificates from certificate file and key file for server.
			cert, err := tls.LoadX509KeyPair(c.GRPC.TLSCert, c.GRPC.TLSKey)
			if err != nil {
				return fmt.Errorf("invalid config: error parsing gRPC certificate file: %v", err)
			}

			// Parse certificates from client CA file to a new CertPool.
			cPool := x509.NewCertPool()
			clientCert, err := ioutil.ReadFile(c.GRPC.TLSClientCA)
			if err != nil {
				return fmt.Errorf("invalid config: reading from client CA file: %v", err)
			}
			if cPool.AppendCertsFromPEM(clientCert) != true {
				return errors.New("invalid config: failed to parse client CA")
			}

			tlsConfig := tls.Config{
				Certificates: []tls.Certificate{cert},
				ClientAuth:   tls.RequireAndVerifyClientCert,
				ClientCAs:    cPool,
			}
			grpcOptions = append(grpcOptions, grpc.Creds(credentials.NewTLS(&tlsConfig)))
		} else {
			opt, err := credentials.NewServerTLSFromFile(c.GRPC.TLSCert, c.GRPC.TLSKey)
			if err != nil {
				return fmt.Errorf("invalid config: load grpc certs: %v", err)
			}
			grpcOptions = append(grpcOptions, grpc.Creds(opt))
		}
	}

	s, err := c.Storage.Config.Open(logger)
	if err != nil {
		return fmt.Errorf("failed to initialize storage: %v", err)
	}
	logger.Infof("config storage: %s", c.Storage.Type)

	if len(c.StaticClients) > 0 {
		for _, client := range c.StaticClients {
			logger.Infof("config static client: %s", client.ID)
		}
		s = storage.WithStaticClients(s, c.StaticClients)
	}
	if len(c.StaticPasswords) > 0 {
		passwords := make([]storage.Password, len(c.StaticPasswords))
		for i, p := range c.StaticPasswords {
			passwords[i] = storage.Password(p)
		}
		s = storage.WithStaticPasswords(s, passwords)
	}

	if c.EnablePasswordDB {
		c.StaticConnectors = append(c.StaticConnectors, Connector{
			ID:   server.LocalConnector,
			Name: "Email",
			Type: server.LocalConnector,
		})
		logger.Infof("config connector: local passwords enabled")
	}

	storageConnectors := make([]storage.Connector, len(c.StaticConnectors))
	for i, c := range c.StaticConnectors {
		if c.ID == "" || c.Name == "" || c.Type == "" {
			return fmt.Errorf("invalid config: ID, Type and Name fields are required for a connector")
		}
		if c.Config == nil {
			return fmt.Errorf("invalid config: no config field for connector %q", c.ID)
		}
		logger.Infof("config connector: %s", c.ID)

		// convert to a storage connector object
		conn, err := ToStorageConnector(c)
		if err != nil {
			return fmt.Errorf("failed to initialize storage connectors: %v", err)
		}
		storageConnectors[i] = conn

	}
	s = storage.WithStaticConnectors(s, storageConnectors)

	if len(c.OAuth2.ResponseTypes) > 0 {
		logger.Infof("config response types accepted: %s", c.OAuth2.ResponseTypes)
	}
	if c.OAuth2.SkipApprovalScreen {
		logger.Infof("config skipping approval screen")
	}
	if len(c.Web.AllowedOrigins) > 0 {
		logger.Infof("config allowed origins: %s", c.Web.AllowedOrigins)
	}

	// explicitly convert to UTC.
	now := func() time.Time { return time.Now().UTC() }

	serverConfig := server.Config{
		SupportedResponseTypes: c.OAuth2.ResponseTypes,
		SkipApprovalScreen:     c.OAuth2.SkipApprovalScreen,
		AllowedOrigins:         c.Web.AllowedOrigins,
		Issuer:                 c.Issuer,
		Storage:                s,
		Web:                    c.Frontend,
		Logger:                 logger,
		Now:                    now,
	}
	if c.Expiry.SigningKeys != "" {
		signingKeys, err := time.ParseDuration(c.Expiry.SigningKeys)
		if err != nil {
			return fmt.Errorf("invalid config value %q for signing keys expiry: %v", c.Expiry.SigningKeys, err)
		}
		logger.Infof("config signing keys expire after: %v", signingKeys)
		serverConfig.RotateKeysAfter = signingKeys
	}
	if c.Expiry.IDTokens != "" {
		idTokens, err := time.ParseDuration(c.Expiry.IDTokens)
		if err != nil {
			return fmt.Errorf("invalid config value %q for id token expiry: %v", c.Expiry.IDTokens, err)
		}
		logger.Infof("config id tokens valid for: %v", idTokens)
		serverConfig.IDTokensValidFor = idTokens
	}

	serv, err := server.NewServer(context.Background(), serverConfig)
	if err != nil {
		return fmt.Errorf("failed to initialize server: %v", err)
	}

	errc := make(chan error, 3)
	if c.Web.HTTP != "" {
		logger.Infof("listening (http) on %s", c.Web.HTTP)
		go func() {
			err := http.ListenAndServe(c.Web.HTTP, serv)
			errc <- fmt.Errorf("listening on %s failed: %v", c.Web.HTTP, err)
		}()
	}
	if c.Web.HTTPS != "" {
		logger.Infof("listening (https) on %s", c.Web.HTTPS)
		go func() {
			err := http.ListenAndServeTLS(c.Web.HTTPS, c.Web.TLSCert, c.Web.TLSKey, serv)
			errc <- fmt.Errorf("listening on %s failed: %v", c.Web.HTTPS, err)
		}()
	}
	if c.GRPC.Addr != "" {
		logger.Infof("listening (grpc) on %s", c.GRPC.Addr)
		go func() {
			errc <- func() error {
				list, err := net.Listen("tcp", c.GRPC.Addr)
				if err != nil {
					return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err)
				}
				s := grpc.NewServer(grpcOptions...)
				api.RegisterDexServer(s, server.NewAPI(serverConfig.Storage, logger))
				err = s.Serve(list)
				return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err)
			}()
		}()
	}

	return <-errc
}

var (
	logLevels  = []string{"debug", "info", "error"}
	logFormats = []string{"json", "text"}
)

type utcFormatter struct {
	f logrus.Formatter
}

func (f *utcFormatter) Format(e *logrus.Entry) ([]byte, error) {
	e.Time = e.Time.UTC()
	return f.f.Format(e)
}

func newLogger(level string, format string) (logrus.FieldLogger, error) {
	var logLevel logrus.Level
	switch strings.ToLower(level) {
	case "debug":
		logLevel = logrus.DebugLevel
	case "", "info":
		logLevel = logrus.InfoLevel
	case "error":
		logLevel = logrus.ErrorLevel
	default:
		return nil, fmt.Errorf("log level is not one of the supported values (%s): %s", strings.Join(logLevels, ", "), level)
	}

	var formatter utcFormatter
	switch strings.ToLower(format) {
	case "", "text":
		formatter.f = &logrus.TextFormatter{DisableColors: true}
	case "json":
		formatter.f = &logrus.JSONFormatter{}
	default:
		return nil, fmt.Errorf("log format is not one of the supported values (%s): %s", strings.Join(logFormats, ", "), format)
	}

	return &logrus.Logger{
		Out:       os.Stderr,
		Formatter: &formatter,
		Level:     logLevel,
	}, nil
}
<commit_msg>cmd/dex/serve: add local connector directly to static connectors in storage<commit_after>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/ghodss/yaml"
	"github.com/spf13/cobra"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/coreos/dex/api"
	"github.com/coreos/dex/server"
	"github.com/coreos/dex/storage"
)

func commandServe() *cobra.Command {
	return &cobra.Command{
		Use:     "serve [ config file ]",
		Short:   "Connect to the storage and begin serving requests.",
		Long:    ``,
		Example: "dex serve config.yaml",
		Run: func(cmd *cobra.Command, args []string) {
			if err := serve(cmd, args); err != nil {
				fmt.Fprintln(os.Stderr, err)
				os.Exit(2)
			}
		},
	}
}

func serve(cmd *cobra.Command, args []string) error {
	switch len(args) {
	default:
		return errors.New("surplus arguments")
	case 0:
		// TODO(ericchiang): Consider having a default config file location.
		return errors.New("no arguments provided")
	case 1:
	}

	configFile := args[0]
	configData, err := ioutil.ReadFile(configFile)
	if err != nil {
		return fmt.Errorf("failed to read config file %s: %v", configFile, err)
	}

	var c Config
	if err := yaml.Unmarshal(configData, &c); err != nil {
		return fmt.Errorf("error parse config file %s: %v", configFile, err)
	}

	logger, err := newLogger(c.Logger.Level, c.Logger.Format)
	if err != nil {
		return fmt.Errorf("invalid config: %v", err)
	}
	if c.Logger.Level != "" {
		logger.Infof("config using log level: %s", c.Logger.Level)
	}

	// Fast checks. Perform these first for a more responsive CLI.
	checks := []struct {
		bad    bool
		errMsg string
	}{
		{c.Issuer == "", "no issuer specified in config file"},
		{!c.EnablePasswordDB && len(c.StaticPasswords) != 0, "cannot specify static passwords without enabling password db"},
		{c.Storage.Config == nil, "no storage suppied in config file"},
		{c.Web.HTTP == "" && c.Web.HTTPS == "", "must supply a HTTP/HTTPS  address to listen on"},
		{c.Web.HTTPS != "" && c.Web.TLSCert == "", "no cert specified for HTTPS"},
		{c.Web.HTTPS != "" && c.Web.TLSKey == "", "no private key specified for HTTPS"},
		{c.GRPC.TLSCert != "" && c.GRPC.Addr == "", "no address specified for gRPC"},
		{c.GRPC.TLSKey != "" && c.GRPC.Addr == "", "no address specified for gRPC"},
		{(c.GRPC.TLSCert == "") != (c.GRPC.TLSKey == ""), "must specific both a gRPC TLS cert and key"},
		{c.GRPC.TLSCert == "" && c.GRPC.TLSClientCA != "", "cannot specify gRPC TLS client CA without a gRPC TLS cert"},
	}

	for _, check := range checks {
		if check.bad {
			return fmt.Errorf("invalid config: %s", check.errMsg)
		}
	}

	logger.Infof("config issuer: %s", c.Issuer)

	var grpcOptions []grpc.ServerOption
	if c.GRPC.TLSCert != "" {
		if c.GRPC.TLSClientCA != "" {
			// Parse certificates from certificate file and key file for server.
			cert, err := tls.LoadX509KeyPair(c.GRPC.TLSCert, c.GRPC.TLSKey)
			if err != nil {
				return fmt.Errorf("invalid config: error parsing gRPC certificate file: %v", err)
			}

			// Parse certificates from client CA file to a new CertPool.
			cPool := x509.NewCertPool()
			clientCert, err := ioutil.ReadFile(c.GRPC.TLSClientCA)
			if err != nil {
				return fmt.Errorf("invalid config: reading from client CA file: %v", err)
			}
			if cPool.AppendCertsFromPEM(clientCert) != true {
				return errors.New("invalid config: failed to parse client CA")
			}

			tlsConfig := tls.Config{
				Certificates: []tls.Certificate{cert},
				ClientAuth:   tls.RequireAndVerifyClientCert,
				ClientCAs:    cPool,
			}
			grpcOptions = append(grpcOptions, grpc.Creds(credentials.NewTLS(&tlsConfig)))
		} else {
			opt, err := credentials.NewServerTLSFromFile(c.GRPC.TLSCert, c.GRPC.TLSKey)
			if err != nil {
				return fmt.Errorf("invalid config: load grpc certs: %v", err)
			}
			grpcOptions = append(grpcOptions, grpc.Creds(opt))
		}
	}

	s, err := c.Storage.Config.Open(logger)
	if err != nil {
		return fmt.Errorf("failed to initialize storage: %v", err)
	}
	logger.Infof("config storage: %s", c.Storage.Type)

	if len(c.StaticClients) > 0 {
		for _, client := range c.StaticClients {
			logger.Infof("config static client: %s", client.ID)
		}
		s = storage.WithStaticClients(s, c.StaticClients)
	}
	if len(c.StaticPasswords) > 0 {
		passwords := make([]storage.Password, len(c.StaticPasswords))
		for i, p := range c.StaticPasswords {
			passwords[i] = storage.Password(p)
		}
		s = storage.WithStaticPasswords(s, passwords)
	}

	storageConnectors := make([]storage.Connector, len(c.StaticConnectors))
	for i, c := range c.StaticConnectors {
		if c.ID == "" || c.Name == "" || c.Type == "" {
			return fmt.Errorf("invalid config: ID, Type and Name fields are required for a connector")
		}
		if c.Config == nil {
			return fmt.Errorf("invalid config: no config field for connector %q", c.ID)
		}
		logger.Infof("config connector: %s", c.ID)

		// convert to a storage connector object
		conn, err := ToStorageConnector(c)
		if err != nil {
			return fmt.Errorf("failed to initialize storage connectors: %v", err)
		}
		storageConnectors[i] = conn

	}

	if c.EnablePasswordDB {
		storageConnectors = append(storageConnectors, storage.Connector{
			ID:   server.LocalConnector,
			Name: "Email",
			Type: server.LocalConnector,
		})
		logger.Infof("config connector: local passwords enabled")
	}

	s = storage.WithStaticConnectors(s, storageConnectors)

	if len(c.OAuth2.ResponseTypes) > 0 {
		logger.Infof("config response types accepted: %s", c.OAuth2.ResponseTypes)
	}
	if c.OAuth2.SkipApprovalScreen {
		logger.Infof("config skipping approval screen")
	}
	if len(c.Web.AllowedOrigins) > 0 {
		logger.Infof("config allowed origins: %s", c.Web.AllowedOrigins)
	}

	// explicitly convert to UTC.
	now := func() time.Time { return time.Now().UTC() }

	serverConfig := server.Config{
		SupportedResponseTypes: c.OAuth2.ResponseTypes,
		SkipApprovalScreen:     c.OAuth2.SkipApprovalScreen,
		AllowedOrigins:         c.Web.AllowedOrigins,
		Issuer:                 c.Issuer,
		Storage:                s,
		Web:                    c.Frontend,
		Logger:                 logger,
		Now:                    now,
	}
	if c.Expiry.SigningKeys != "" {
		signingKeys, err := time.ParseDuration(c.Expiry.SigningKeys)
		if err != nil {
			return fmt.Errorf("invalid config value %q for signing keys expiry: %v", c.Expiry.SigningKeys, err)
		}
		logger.Infof("config signing keys expire after: %v", signingKeys)
		serverConfig.RotateKeysAfter = signingKeys
	}
	if c.Expiry.IDTokens != "" {
		idTokens, err := time.ParseDuration(c.Expiry.IDTokens)
		if err != nil {
			return fmt.Errorf("invalid config value %q for id token expiry: %v", c.Expiry.IDTokens, err)
		}
		logger.Infof("config id tokens valid for: %v", idTokens)
		serverConfig.IDTokensValidFor = idTokens
	}

	serv, err := server.NewServer(context.Background(), serverConfig)
	if err != nil {
		return fmt.Errorf("failed to initialize server: %v", err)
	}

	errc := make(chan error, 3)
	if c.Web.HTTP != "" {
		logger.Infof("listening (http) on %s", c.Web.HTTP)
		go func() {
			err := http.ListenAndServe(c.Web.HTTP, serv)
			errc <- fmt.Errorf("listening on %s failed: %v", c.Web.HTTP, err)
		}()
	}
	if c.Web.HTTPS != "" {
		logger.Infof("listening (https) on %s", c.Web.HTTPS)
		go func() {
			err := http.ListenAndServeTLS(c.Web.HTTPS, c.Web.TLSCert, c.Web.TLSKey, serv)
			errc <- fmt.Errorf("listening on %s failed: %v", c.Web.HTTPS, err)
		}()
	}
	if c.GRPC.Addr != "" {
		logger.Infof("listening (grpc) on %s", c.GRPC.Addr)
		go func() {
			errc <- func() error {
				list, err := net.Listen("tcp", c.GRPC.Addr)
				if err != nil {
					return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err)
				}
				s := grpc.NewServer(grpcOptions...)
				api.RegisterDexServer(s, server.NewAPI(serverConfig.Storage, logger))
				err = s.Serve(list)
				return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err)
			}()
		}()
	}

	return <-errc
}

var (
	logLevels  = []string{"debug", "info", "error"}
	logFormats = []string{"json", "text"}
)

type utcFormatter struct {
	f logrus.Formatter
}

func (f *utcFormatter) Format(e *logrus.Entry) ([]byte, error) {
	e.Time = e.Time.UTC()
	return f.f.Format(e)
}

func newLogger(level string, format string) (logrus.FieldLogger, error) {
	var logLevel logrus.Level
	switch strings.ToLower(level) {
	case "debug":
		logLevel = logrus.DebugLevel
	case "", "info":
		logLevel = logrus.InfoLevel
	case "error":
		logLevel = logrus.ErrorLevel
	default:
		return nil, fmt.Errorf("log level is not one of the supported values (%s): %s", strings.Join(logLevels, ", "), level)
	}

	var formatter utcFormatter
	switch strings.ToLower(format) {
	case "", "text":
		formatter.f = &logrus.TextFormatter{DisableColors: true}
	case "json":
		formatter.f = &logrus.JSONFormatter{}
	default:
		return nil, fmt.Errorf("log format is not one of the supported values (%s): %s", strings.Join(logFormats, ", "), format)
	}

	return &logrus.Logger{
		Out:       os.Stderr,
		Formatter: &formatter,
		Level:     logLevel,
	}, nil
}
<|endoftext|>