<commit_before>// Copyright (c) 2014 Soichiro Kashima
// Licensed under MIT license.

package md

type HtmlConverter struct {
}

func (c *HtmlConverter) ToH1(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH2(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH3(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH4(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH5(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH6(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToP(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) constructInlines(content []Inline) string {
	s := ""
	for _, i := range content {
		if i.NewLine {
			s += "
"
		} else if i.Href != "" {
			s += "" + i.Value + ""
		} else {
			s += i.Value
		}
	}
	return s
}
<commit_msg>Added new line after br.<commit_after>// Copyright (c) 2014 Soichiro Kashima
// Licensed under MIT license.

package md

type HtmlConverter struct {
}

func (c *HtmlConverter) ToH1(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH2(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH3(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH4(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH5(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToH6(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) ToP(content []Inline) string {
	return "
" + c.constructInlines(content) + "
"
}

func (c *HtmlConverter) constructInlines(content []Inline) string {
	s := ""
	for _, i := range content {
		if i.NewLine {
			s += "
\n"
		} else if i.Href != "" {
			s += "" + i.Value + ""
		} else {
			s += i.Value
		}
	}
	return s
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
)

const (
	HELP = `md2xml [-h] [-x] [-s] [-a] [-i dir] [-o file] file.md
Transform a given Markdown file into XML.
-h        To print this help page.
-x        Print intermediate XHTML output.
-s        Print stylesheet used for transformation.
-a        Output article (instead of blog entry).
-p        Add link to PDF version.
-i dir    To indicate image directory.
-o file   The name of the file to output.
file.md   The markdown file to convert.
Note: this program calls pandoc and xsltproc that must have been installed.`
	STYLESHEET_ARTICLE = `<?xml version="1.0" encoding="utf-8"?>

<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                version="1.0">

  <xsl:output method="xml" encoding="UTF-8"/>
  <xsl:param name="title"/>
  <xsl:param name="author"/>
  <xsl:param name="date"/>
  <xsl:param name="tags"/>
  <xsl:param name="id"/>
  <xsl:param name="email"/>
  <xsl:param name="lang">fr</xsl:param>
  <xsl:param name="toc">yes</xsl:param>
  <xsl:param name="pdf">no</xsl:param>

  <xsl:template match="/xhtml">
    <xsl:text disable-output-escaping="yes">
    <!DOCTYPE article PUBLIC "-//CAFEBABE//DTD blog 1.0//EN"
                             "../dtd/article.dtd">
    </xsl:text>

      <xsl:attribute name="id"><xsl:value-of select="$id"/></xsl:attribute>
      <xsl:attribute name="date"><xsl:value-of select="$date"/></xsl:attribute>
      <xsl:attribute name="author"><xsl:value-of select="$author"/></xsl:attribute>
      <xsl:attribute name="email"><xsl:value-of select="$email"/></xsl:attribute>
      <xsl:attribute name="keywords"><xsl:value-of select="$tags"/></xsl:attribute>
      <xsl:attribute name="lang"><xsl:value-of select="$lang"/></xsl:attribute>
      <xsl:attribute name="toc"><xsl:value-of select="$toc"/></xsl:attribute>
      <xsl:attribute name="pdf"><xsl:value-of select="$pdf"/></xsl:attribute>
      <title><xsl:value-of select="$title"/></title>
      <text>
        <xsl:apply-templates/>
      </text>

  </xsl:template>

  <xsl:template match="h1">
    <sect level="1"><title><xsl:value-of select="."/></title></sect>
  </xsl:template>

  <xsl:template match="h2">
    <sect level="2"><title><xsl:value-of select="."/></title></sect>
  </xsl:template>

  <xsl:template match="h3">
    <sect level="3"><title><xsl:value-of select="."/></title></sect>
  </xsl:template>

  <xsl:template match="h4">
    <sect level="4"><title><xsl:value-of select="."/></title></sect>
  </xsl:template>

  <xsl:template match="h5">
    <sect level="5"><title><xsl:value-of select="."/></title></sect>
  </xsl:template>

  <xsl:template match="h6">
    <sect level="6"><title><xsl:value-of select="."/></title></sect>
  </xsl:template>

  <xsl:template match="p[@class='caption']">
  </xsl:template>

  <xsl:template match="p[count(text())=0 and count(code)=1]">
    <source><xsl:apply-templates select="code"/></source>
  </xsl:template>

  <xsl:template match="p[count(text())=1 and count(img)=1]">
    <xsl:apply-templates select="img"/>
  </xsl:template>

  <xsl:template match="img">

      <xsl:if test="@title">
        <title><xsl:value-of select="@title"/></title>
      </xsl:if>

  </xsl:template>

  <xsl:template match="p">

<xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="ul">
    <list><xsl:apply-templates/></list>
  </xsl:template>

  <xsl:template match="ol">
    <enum><xsl:apply-templates/></enum>
  </xsl:template>

  <xsl:template match="li">
    <item><xsl:apply-templates/></item>
  </xsl:template>

  <xsl:template match="table">

<xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="tr[count(th)=0]">

<xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="tr[count(th) > 0]">
    <xsl:apply-templates/>
  </xsl:template>

  <xsl:template match="th">
    <co><xsl:apply-templates/></co>
  </xsl:template>

  <xsl:template match="td">
    <co><xsl:apply-templates/></co>
  </xsl:template>

  <xsl:template match="pre">
    <source><xsl:apply-templates/></source>
  </xsl:template>

  <xsl:template match="em">
    <term><xsl:apply-templates/></term>
  </xsl:template>

  <xsl:template match="strong">
    <imp><xsl:apply-templates/></imp>
  </xsl:template>

  <xsl:template match="a">
    <link url="{@href}"><xsl:apply-templates/></link>
  </xsl:template>

</xsl:stylesheet>`
	STYLESHEET_BLOG = `<?xml version="1.0" encoding="utf-8"?>

<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
                version="1.0">

  <xsl:output method="xml" encoding="UTF-8"/>
  <xsl:param name="title"/>
  <xsl:param name="author"/>
  <xsl:param name="date"/>
  <xsl:param name="tags"/>
  <xsl:param name="id"/>
  <xsl:param name="email"/>
  <xsl:param name="lang">fr</xsl:param>
  <xsl:param name="toc">yes</xsl:param>
  <xsl:param name="pdf">no</xsl:param>

  <xsl:template match="/xhtml">
    <xsl:text disable-output-escaping="yes">
    <!DOCTYPE blog PUBLIC "-//CAFEBABE//DTD blog 1.0//EN"
                          "../dtd/blog.dtd">
    </xsl:text>
    <blog>
      <xsl:attribute name="id"><xsl:value-of select="$id"/></xsl:attribute>
      <xsl:attribute name="date"><xsl:value-of select="$date"/></xsl:attribute>
      <xsl:attribute name="author"><xsl:value-of select="$author"/></xsl:attribute>
      <xsl:attribute name="email"><xsl:value-of select="$email"/></xsl:attribute>
      <xsl:attribute name="keywords"><xsl:value-of select="$tags"/></xsl:attribute>
      <xsl:attribute name="lang"><xsl:value-of select="$lang"/></xsl:attribute>
      <xsl:attribute name="toc"><xsl:value-of select="$toc"/></xsl:attribute>
      <xsl:attribute name="pdf"><xsl:value-of select="$pdf"/></xsl:attribute>
      <title><xsl:value-of select="$title"/></title>
      <xsl:apply-templates/>
    </blog>
  </xsl:template>

  <xsl:template match="h1">

<imp><xsl:value-of select="."/></imp>

  </xsl:template>

  <xsl:template match="h2">

<imp><xsl:value-of select="."/></imp>

  </xsl:template>

  <xsl:template match="h3">

<imp><xsl:value-of select="."/></imp>

  </xsl:template>

  <xsl:template match="h4">

<imp><xsl:value-of select="."/></imp>

  </xsl:template>

  <xsl:template match="h5">

<imp><xsl:value-of select="."/></imp>

  </xsl:template>

  <xsl:template match="h6">

<imp><xsl:value-of select="."/></imp>

  </xsl:template>

  <xsl:template match="p[@class='caption']">
  </xsl:template>

  <xsl:template match="p[count(text())=0 and count(code)=1]">
    <source><xsl:apply-templates select="code"/></source>
  </xsl:template>

  <xsl:template match="p[count(text())=1 and count(img)=1]">
    <xsl:apply-templates select="img"/>
  </xsl:template>

  <xsl:template match="img">

      <xsl:if test="@title">
        <title><xsl:value-of select="@title"/></title>
      </xsl:if>

  </xsl:template>

  <xsl:template match="p">

<xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="ul">
    <list><xsl:apply-templates/></list>
  </xsl:template>

  <xsl:template match="ol">
    <enum><xsl:apply-templates/></enum>
  </xsl:template>

  <xsl:template match="li">
    <item><xsl:apply-templates/></item>
  </xsl:template>

  <xsl:template match="table">

<xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="tr[count(th)=0]">

<xsl:apply-templates/>

  </xsl:template>

  <xsl:template match="tr[count(th) > 0]">
    <xsl:apply-templates/>
  </xsl:template>

  <xsl:template match="th">
    <co><xsl:apply-templates/></co>
  </xsl:template>

  <xsl:template match="td">
    <co><xsl:apply-templates/></co>
  </xsl:template>

  <xsl:template match="pre">
    <source><xsl:apply-templates/></source>
  </xsl:template>

  <xsl:template match="em">
    <term><xsl:apply-templates/></term>
  </xsl:template>

  <xsl:template match="strong">
    <imp><xsl:apply-templates/></imp>
  </xsl:template>

  <xsl:template match="a">
    <link url="{@href}"><xsl:apply-templates/></link>
  </xsl:template>

</xsl:stylesheet>`
	XHTML_HEADER = "<xhtml>\n"
	XHTML_FOOTER = "\n</xhtml>"
)

type MetaData struct {
	Title  string
	Author string
	Date   string
	Tags   []string
	Id     string
	Email  string
	Lang   string
	Toc    string
	Pdf    string
}

func (d MetaData) ToMap() map[string]string {
	data := make(map[string]string)
	if d.Title != "" {
		data["title"] = d.Title
	}
	if d.Author != "" {
		data["author"] = d.Author
	}
	if d.Date != "" {
		data["date"] = d.Date
	}
	if len(d.Tags) != 0 {
		data["tags"] = strings.Join(d.Tags, ", ")
	}
	if d.Id != "" {
		data["id"] = d.Id
	}
	if d.Email != "" {
		data["email"] = d.Email
	}
	if d.Lang != "" {
		data["lang"] = d.Lang
	}
	if d.Toc != "" {
		data["toc"] = d.Toc
	}
	if d.Pdf != "" {
		data["pdf"] = d.Pdf
	}
	return data
}

func processXsl(xmlFile string, data map[string]string, article, pdf bool) []byte {
	xslFile, err := ioutil.TempFile("/tmp", "md2xsl-")
	if err != nil {
		panic(err)
	}
	stylesheet := STYLESHEET_ARTICLE
	if !article {
		stylesheet = STYLESHEET_BLOG
	}
	err = ioutil.WriteFile(xslFile.Name(), []byte(stylesheet), 0644)
	if err != nil {
		panic(err)
	}
	defer os.Remove(xslFile.Name())
	params := make([]string, 0, 2+3*len(data))
	for name, value := range data {
		params = append(params, "--stringparam")
		params = append(params, name)
		params = append(params, value)
	}
	if pdf {
		params = append(params, "--stringparam")
		params = append(params, "pdf")
		params = append(params, "yes")
	}
	params = append(params, xslFile.Name())
	params = append(params, xmlFile)
	command := exec.Command("xsltproc", params...)
	result, err := command.CombinedOutput()
	if err != nil {
		println(result)
		panic(err)
	}
	return result
}

func markdown2xhtml(markdown string) []byte {
	mdFile, err := ioutil.TempFile("/tmp", "md2xsl-")
	if err != nil {
		panic(err)
	}
	defer os.Remove(mdFile.Name())
	ioutil.WriteFile(mdFile.Name(), []byte(markdown), 0644)
	command := exec.Command("pandoc", mdFile.Name(), "-f", "markdown", "-t", "html")
	result, err := command.CombinedOutput()
	if err != nil {
		println(result)
		panic(err)
	}
	return []byte(XHTML_HEADER + string(result) + XHTML_FOOTER)
}

func markdownData(text string) (MetaData, string) {
	var data MetaData
	r := regexp.MustCompile("(?ms)\\A---.*?(---|\\.\\.\\.)")
	yml := r.FindString(text)
	if yml == "" {
		return data, text
	}
	err := yaml.Unmarshal([]byte(yml), &data)
	if err != nil {
		panic(err)
	}
	return data, text[len(yml):]
}

func imageDir(text, imgDir string) string {
	r := regexp.MustCompile(`!\[(.*?)\]\((.*?/)*(.*?)\)`)
	if len(imgDir) > 0 {
		return r.ReplaceAllString(text, "![$1]("+imgDir+"/$3)")
	} else {
		return r.ReplaceAllString(text, "![$1]($3)")
	}
}

func processFile(filename string, printXhtml, article, pdf bool, imgDir, outFile string) {
	source, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}
	data, markdown := markdownData(string(source))
	markdown = imageDir(markdown, imgDir)
	xhtml := markdown2xhtml(markdown)
	if printXhtml {
		fmt.Println(string(xhtml))
	}
	xmlFile, err := ioutil.TempFile("/tmp", "md2xml-")
	if err != nil {
		panic(err)
	}
	defer os.Remove(xmlFile.Name())
	ioutil.WriteFile(xmlFile.Name(), xhtml, 0644)
	result := processXsl(xmlFile.Name(), data.ToMap(), article, pdf)
	if len(outFile) > 0 {
		ioutil.WriteFile(outFile, result, 0644)
	} else {
		fmt.Println(string(result))
	}
}

func main() {
	xhtml := false
	article := false
	pdf := false
	imgDir := ""
	outFile := ""
	file := ""
	if len(os.Args) < 2 {
		fmt.Println(HELP)
		os.Exit(1)
	}
	skip := false
	args := os.Args[1:]
	for i, arg := range args {
		if skip {
			skip = false
			continue
		}
		if arg == "-h" || arg == "--help" {
			fmt.Println(HELP)
			os.Exit(0)
		} else if arg == "-x" || arg == "--xhtml" {
			xhtml = true
		} else if arg == "-a" || arg == "--article" {
			article = true
		} else if arg == "-p" || arg == "--link-pdf" {
			pdf = true
		} else if arg == "-s" || arg == "--stylesheet" {
			if article {
				fmt.Println(STYLESHEET_ARTICLE)
			} else {
				fmt.Println(STYLESHEET_BLOG)
			}
		} else if arg == "-i" || arg == "--image-dir" {
			imgDir = args[i+1]
			skip = true
		} else if arg == "-o" || arg == "--output-file" {
			outFile = args[i+1]
			skip = true
		} else {
			file = arg
		}
	}
	processFile(file, xhtml, article, pdf, imgDir, outFile)
}
<commit_msg>md2xml removed from project<commit_after><|endoftext|>


<commit_before><commit_msg>vendor deps<commit_after><|endoftext|>


<commit_before>package controller

import (
	"os"

	"time"

	"fmt"

	"cloud.google.com/go/datastore"
	"github.com/enow-dev/enow/app"
	"github.com/enow-dev/enow/model"
	"github.com/enow-dev/enow/util"
	"github.com/goadesign/goa"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/search"
)

// EventsController implements the events resource.
type EventsController struct {
	*goa.Controller
}

// NewEventsController creates a events controller.
func NewEventsController(service *goa.Service) *EventsController {
	return &EventsController{Controller: service.NewController("EventsController")}
}

// List runs the list action.
// TODO: 詳しいロジックを実装したら消す
// nolint
func (c *EventsController) List(ctx *app.ListEventsContext) error {
	// EventsController_List: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	now := time.Now()
	// 未ログインでも許容する
	userKey, _ := util.GetUserKey(ctx)
	sel := model.SearchEventsLogDB{}
	indexName, err := sel.GetLatestVersion(appCtx)
	if err != nil {
		log.Errorf(appCtx, "index not found err=%v", err)
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	se := model.NewSearchEventsDB(indexName)
	se.SetLimit(appCtx, 20)
	se.Sort(appCtx, "StartAt", ctx.StartAtSort)
	// 日付指定が一切されていない時は開催終了前のものだけを出す
	if ctx.PeriodFrom == "" && ctx.PeriodTo == "" {
		se.SetPeriodDate(appCtx, "EndAt >=", now)
	} else {
		se.SetPeriodDate(appCtx, "EndAt >=", ctx.PeriodFrom)
		se.SetPeriodDate(appCtx, "EndAt <=", ctx.PeriodTo)
	}

	se.SetPref(appCtx, ctx.Pref)
	se.SetCursor(appCtx, ctx.Cursor)
	// 既読しているイベントも出す
	if !ctx.IsRed {
		se.SetNotSearchID(appCtx, userKey)
	}
	se.SetSearchKeyword(appCtx, ctx.Q)
	iterator, err := se.Run(appCtx)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	events := app.EventTinyCollection{}
	for {
		var event model.SearchEvents
		_, err := iterator.Next(&event)
		if err == search.Done {
			break
		} else if err != nil {
			return ctx.InternalServerError(goa.ErrInternal(err))
		}
		et := event.SearchEventToEventTiny()
		int64EventID, err := util.ConvertIDIntoInt64(event.ID)
		if err != nil {
			log.Errorf(appCtx, "既読済み確認時エラー(1) %v", err)
		}
		if userKey != nil {
			// 既読状態を判定する
			uerDB := model.UserEventReadsDB{}
			isRed, err := uerDB.IsRedEvent(appCtx, int64EventID, userKey.IntID())
			if err != nil {
				log.Errorf(appCtx, "既読済み確認時エラー(2) %v", err)
			}
			et.IsRed = isRed
			// お気に入り状態を判定する
			uefDB := model.UserEventFavoritesDB{}
			isFavorite, err := uefDB.IsFavoriteEvent(appCtx, int64EventID, userKey)
			if err != nil {
				log.Errorf(appCtx, "お気に入り済み確認時エラー(3) %v", err)
			}
			et.IsFavorite = isFavorite
		}
		// お気に入り状態を判定する
		events = append(events, et)

		// 既読情報をつける
		if userKey != nil {
			eventID, err := util.ConvertIDIntoInt64(event.ID)
			if err != nil {
				log.Errorf(appCtx, "既読情報の挿入時エラー(4) %v", err)
			}
			userEventReads := &model.UserEventReads{
				EventID:    eventID,
				EventEndAt: event.EndAt,
				UserID:     userKey.IntID(),
				CreatedAt:  now,
			}
			uerDB := model.UserEventReadsDB{}
			uerDB.Add(appCtx, userEventReads)
		}
	}
	l := util.CreateLinkHeader(ctx.RequestData, os.Getenv("Scheme"), fmt.Sprint(iterator.Cursor()))
	ctx.ResponseData.Header().Set("link", l.String())
	ctx.ResponseData.Header().Set("x-search-hits-count", fmt.Sprint(iterator.Count()))

	// EventsController_List: end_implement
	return ctx.OKTiny(events)
}

// Show runs the show action.
func (c *EventsController) Show(ctx *app.ShowEventsContext) error {
	// EventsController_Show: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	int64ID, err := util.ConvertIDIntoInt64(ctx.ID)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	eDB := model.EventsDB{}
	e, err := eDB.Get(appCtx, int64ID)
	if err == datastore.ErrNoSuchEntity {
		return ctx.NotFound()
	} else if err != nil {
		return ctx.BadRequest(goa.ErrBadRequest(err))
	}
	es := e.EventToEventShow()
	// ユーザーがログインしている場合既読情報やお気に入り情報を確認する
	userKey, _ := util.GetUserKey(ctx)
	if userKey != nil {
		// 既読状態を判定する
		uerDB := model.UserEventReadsDB{}
		isRed, err := uerDB.IsRedEvent(appCtx, int64ID, userKey.IntID())
		if err != nil {
			log.Errorf(appCtx, "既読済み確認時エラー(2) %v", err)
		}
		es.IsRed = isRed
		// お気に入り状態を判定する
		uefDB := model.UserEventFavoritesDB{}
		isFavorite, err := uefDB.IsFavoriteEvent(appCtx, int64ID, userKey)
		if err != nil {
			log.Errorf(appCtx, "お気に入り済み確認時エラー(3) %v", err)
		}
		es.IsFavorite = isFavorite
	}
	// EventsController_Show: end_implement
	return ctx.OKShow(e.EventToEventShow())
}

// ShowCount runs the showCount action.
func (c *EventsController) ShowCount(ctx *app.ShowCountEventsContext) error {
	// EventsController_ShowCount: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	// 未ログインでも許容する
	userKey, _ := util.GetUserKey(ctx)
	sel := model.SearchEventsLogDB{}
	indexName, err := sel.GetLatestVersion(appCtx)
	if err != nil {
		log.Errorf(appCtx, "index not found err=%v", err)
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	se := model.NewSearchEventsDB(indexName)
	se.SetPref(appCtx, ctx.Pref)
	se.SetLimit(appCtx, 1)
	se.SetNotSearchID(appCtx, userKey)
	se.SetSearchKeyword(appCtx, ctx.Q)
	iterator, err := se.Run(appCtx)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	events := app.EventTinyCollection{}
	for {
		var event model.SearchEvents
		_, err := iterator.Next(&event)
		if err == search.Done {
			break
		} else if err != nil {
			return ctx.InternalServerError(goa.ErrInternal(err))
		}
		events = append(events, event.SearchEventToEventTiny())
	}
	// EventsController_ShowCount: end_implement
	return ctx.OK(iterator.Count())
	//return ctx.OK(iterator.Count())
}

// SelfFavoriteList runs the selfFavoriteList action.
func (c *EventsController) SelfFavoriteList(ctx *app.SelfFavoriteListEventsContext) error {
	// EventsController_SelfFavoriteList: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	userKey, err := util.GetUserKey(ctx)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	ufeDB := &model.UserEventFavoritesDB{}
	events, cursor, err := ufeDB.GetListFindByUserKey(appCtx, userKey, ctx.Cursor)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	l := util.CreateLinkHeader(ctx.RequestData, os.Getenv("Scheme"), fmt.Sprint(cursor))
	ctx.ResponseData.Header().Set("link", l.String())

	// EventsController_SelfFavoriteList: end_implement
	return ctx.OKTiny(events)
}

// DeleteFavorite runs the deleteFavorite action.
func (c *EventsController) DeleteFavorite(ctx *app.DeleteFavoriteEventsContext) error {
	// EventsController_DeleteFavorite: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	userKey, err := util.GetUserKey(ctx)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	int64ID, err := util.ConvertIDIntoInt64(ctx.ID)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	ufeDB := &model.UserEventFavoritesDB{}
	err = ufeDB.Delete(appCtx, int64ID, userKey)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}

	// EventsController_DeleteFavorite: end_implement
	return nil
}

// UpdateFavorite runs the updateFavorite action.
func (c *EventsController) UpdateFavorite(ctx *app.UpdateFavoriteEventsContext) error {
	// EventsController_UpdateFavorite: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	userKey, err := util.GetUserKey(ctx)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	int64ID, err := util.ConvertIDIntoInt64(ctx.ID)
	if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}
	ufeDB := &model.UserEventFavoritesDB{}
	now := time.Now()
	err = ufeDB.Add(appCtx, int64ID, userKey, now)
	if err == fmt.Errorf("存在しないイベントIDが指定されています") {
		return ctx.BadRequest(goa.ErrBadRequest(err))
	} else if err != nil {
		return ctx.InternalServerError(goa.ErrInternal(err))
	}

	// EventsController_UpdateFavorite: end_implement
	return nil
}
<commit_msg>add: #222 events<commit_after>package controller

import (
	"os"

	"time"

	"fmt"

	"cloud.google.com/go/datastore"
	"github.com/enow-dev/enow/app"
	"github.com/enow-dev/enow/design/constant"
	"github.com/enow-dev/enow/model"
	"github.com/enow-dev/enow/util"
	"github.com/goadesign/goa"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/search"
)

// EventsController implements the events resource.
type EventsController struct {
	*goa.Controller
}

// NewEventsController creates a events controller.
func NewEventsController(service *goa.Service) *EventsController {
	return &EventsController{Controller: service.NewController("EventsController")}
}

const errTypeEvents = "events"

// List runs the list action.
// TODO: 詳しいロジックを実装したら消す
// nolint
func (c *EventsController) List(ctx *app.ListEventsContext) error {
	// EventsController_List: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	now := time.Now()
	// 未ログインでも許容する
	userKey, _ := util.GetUserKey(ctx)
	sel := model.SearchEventsLogDB{}
	indexName, err := sel.GetLatestVersion(appCtx)
	if err != nil {
		log.Errorf(appCtx, "%s index取得エラー(1): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 1)))
	}
	se := model.NewSearchEventsDB(indexName)
	se.SetLimit(appCtx, 20)
	se.Sort(appCtx, "StartAt", ctx.StartAtSort)
	// 日付指定が一切されていない時は開催終了前のものだけを出す
	if ctx.PeriodFrom == "" && ctx.PeriodTo == "" {
		se.SetPeriodDate(appCtx, "EndAt >=", now)
	} else {
		se.SetPeriodDate(appCtx, "EndAt >=", ctx.PeriodFrom)
		se.SetPeriodDate(appCtx, "EndAt <=", ctx.PeriodTo)
	}

	se.SetPref(appCtx, ctx.Pref)
	se.SetCursor(appCtx, ctx.Cursor)
	// 既読しているイベントも出す
	if !ctx.IsRed {
		se.SetNotSearchID(appCtx, userKey)
	}
	se.SetSearchKeyword(appCtx, ctx.Q)
	iterator, err := se.Run(appCtx)
	if err != nil {
		log.Errorf(appCtx, "%s index runエラー(2): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 2)))
	}
	events := app.EventTinyCollection{}
	for {
		var event model.SearchEvents
		_, err := iterator.Next(&event)
		if err == search.Done {
			break
		} else if err != nil {
			log.Errorf(appCtx, "%s iteratorエラー(3): %v", errTypeEvents, err)
			return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 3)))
		}
		et := event.SearchEventToEventTiny()
		int64EventID, err := util.ConvertIDIntoInt64(event.ID)
		if err != nil {
			log.Errorf(appCtx, "%s イベントID取得エラー(4): %v", errTypeEvents, err)
		}
		if userKey != nil {
			// 既読状態を判定する
			uerDB := model.UserEventReadsDB{}
			isRed, err := uerDB.IsRedEvent(appCtx, int64EventID, userKey.IntID())
			if err != nil {
				log.Errorf(appCtx, "%s 既読イベント取得エラー(5): %v", errTypeEvents, err)
			}
			et.IsRed = isRed
			// お気に入り状態を判定する
			uefDB := model.UserEventFavoritesDB{}
			isFavorite, err := uefDB.IsFavoriteEvent(appCtx, int64EventID, userKey)
			if err != nil {
				log.Errorf(appCtx, "%s お気に入り判定エラー(5): %v", errTypeEvents, err)
			}
			et.IsFavorite = isFavorite
		}
		// お気に入り状態を判定する
		events = append(events, et)

		// 既読情報をつける
		if userKey != nil {
			eventID, err := util.ConvertIDIntoInt64(event.ID)
			if err != nil {
				log.Errorf(appCtx, "%s 既読情報挿入エラー(5): %v", errTypeEvents, err)
			}
			userEventReads := &model.UserEventReads{
				EventID:    eventID,
				EventEndAt: event.EndAt,
				UserID:     userKey.IntID(),
				CreatedAt:  now,
			}
			uerDB := model.UserEventReadsDB{}
			uerDB.Add(appCtx, userEventReads)
		}
	}
	l := util.CreateLinkHeader(ctx.RequestData, os.Getenv("Scheme"), fmt.Sprint(iterator.Cursor()))
	ctx.ResponseData.Header().Set("link", l.String())
	ctx.ResponseData.Header().Set("x-search-hits-count", fmt.Sprint(iterator.Count()))

	// EventsController_List: end_implement
	return ctx.OKTiny(events)
}

// Show runs the show action.
func (c *EventsController) Show(ctx *app.ShowEventsContext) error {
	// EventsController_Show: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	int64ID, err := util.ConvertIDIntoInt64(ctx.ID)
	if err != nil {
		log.Errorf(appCtx, "%s ID取得エラー(6): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 6)))
	}
	eDB := model.EventsDB{}
	e, err := eDB.Get(appCtx, int64ID)
	if err == datastore.ErrNoSuchEntity {
		return ctx.NotFound()
	} else if err != nil {
		return ctx.BadRequest(goa.ErrBadRequest(err))
	}
	es := e.EventToEventShow()
	// ユーザーがログインしている場合既読情報やお気に入り情報を確認する
	userKey, _ := util.GetUserKey(ctx)
	if userKey != nil {
		// 既読状態を判定する
		uerDB := model.UserEventReadsDB{}
		isRed, err := uerDB.IsRedEvent(appCtx, int64ID, userKey.IntID())
		if err != nil {
			log.Errorf(appCtx, "%s 既読情報取得エラー(7): %v", errTypeEvents, err)
			return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 7)))
		}
		es.IsRed = isRed
		// お気に入り状態を判定する
		uefDB := model.UserEventFavoritesDB{}
		isFavorite, err := uefDB.IsFavoriteEvent(appCtx, int64ID, userKey)
		if err != nil {
			log.Errorf(appCtx, "%s お気に入り情報取得エラー(8): %v", errTypeEvents, err)
			return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 8)))
		}
		es.IsFavorite = isFavorite
	}
	// EventsController_Show: end_implement
	return ctx.OKShow(e.EventToEventShow())
}

// ShowCount runs the showCount action.
func (c *EventsController) ShowCount(ctx *app.ShowCountEventsContext) error {
	// EventsController_ShowCount: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	// 未ログインでも許容する
	userKey, _ := util.GetUserKey(ctx)
	sel := model.SearchEventsLogDB{}
	indexName, err := sel.GetLatestVersion(appCtx)
	if err != nil {
		log.Errorf(appCtx, "%s index取得エラー(9): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 9)))
	}
	se := model.NewSearchEventsDB(indexName)
	se.SetPref(appCtx, ctx.Pref)
	se.SetLimit(appCtx, 1)
	se.SetNotSearchID(appCtx, userKey)
	se.SetSearchKeyword(appCtx, ctx.Q)
	iterator, err := se.Run(appCtx)
	if err != nil {
		log.Errorf(appCtx, "%s index読み込みエラー(10): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 10)))
	}
	events := app.EventTinyCollection{}
	for {
		var event model.SearchEvents
		_, err := iterator.Next(&event)
		if err == search.Done {
			break
		} else if err != nil {
			log.Errorf(appCtx, "%s iteratorエラー(11): %v", errTypeEvents, err)
			return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 11)))
		}
		events = append(events, event.SearchEventToEventTiny())
	}
	// EventsController_ShowCount: end_implement
	return ctx.OK(iterator.Count())
	//return ctx.OK(iterator.Count())
}

// SelfFavoriteList runs the selfFavoriteList action.
func (c *EventsController) SelfFavoriteList(ctx *app.SelfFavoriteListEventsContext) error {
	// EventsController_SelfFavoriteList: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	userKey, err := util.GetUserKey(ctx)
	if err != nil {
		log.Errorf(appCtx, "%s UserKey取得エラー(12): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 12)))
	}
	ufeDB := &model.UserEventFavoritesDB{}
	events, cursor, err := ufeDB.GetListFindByUserKey(appCtx, userKey, ctx.Cursor)
	if err != nil {
		log.Errorf(appCtx, "%s ユーザーのお気に入り情報取得エラー(13): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 13)))
	}
	l := util.CreateLinkHeader(ctx.RequestData, os.Getenv("Scheme"), fmt.Sprint(cursor))
	ctx.ResponseData.Header().Set("link", l.String())

	// EventsController_SelfFavoriteList: end_implement
	return ctx.OKTiny(events)
}

// DeleteFavorite runs the deleteFavorite action.
func (c *EventsController) DeleteFavorite(ctx *app.DeleteFavoriteEventsContext) error {
	// EventsController_DeleteFavorite: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	userKey, err := util.GetUserKey(ctx)
	if err != nil {
		log.Errorf(appCtx, "%s UserKey取得エラー(14): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 14)))
	}
	int64ID, err := util.ConvertIDIntoInt64(ctx.ID)
	if err != nil {
		log.Errorf(appCtx, "%s ID変換取得エラー(15): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 15)))
	}
	ufeDB := &model.UserEventFavoritesDB{}
	err = ufeDB.Delete(appCtx, int64ID, userKey)
	if err != nil {
		log.Errorf(appCtx, "%s お気に入り削除エラー(16): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 16)))
	}

	// EventsController_DeleteFavorite: end_implement
	return nil
}

// UpdateFavorite runs the updateFavorite action.
func (c *EventsController) UpdateFavorite(ctx *app.UpdateFavoriteEventsContext) error {
	// EventsController_UpdateFavorite: start_implement

	// Put your logic here
	appCtx := appengine.NewContext(ctx.Request)
	userKey, err := util.GetUserKey(ctx)
	if err != nil {
		log.Errorf(appCtx, "%s UserKey取得エラー(15): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 15)))
	}
	int64ID, err := util.ConvertIDIntoInt64(ctx.ID)
	if err != nil {
		log.Errorf(appCtx, "%s ID変換取得エラー(16): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 16)))
	}
	ufeDB := &model.UserEventFavoritesDB{}
	now := time.Now()
	err = ufeDB.Add(appCtx, int64ID, userKey, now)
	if err == fmt.Errorf("存在しないイベントIDが指定されています") {
		log.Errorf(appCtx, "%s 存在しないイベントのお気に入り追加エラー(17): %v", errTypeEvents, err)
		return ctx.BadRequest(goa.ErrBadRequest(fmt.Errorf(constant.BadRequestErr, errTypeEvents, 17)))
	} else if err != nil {
		log.Errorf(appCtx, "%s お気に入り追加エラー(18): %v", errTypeEvents, err)
		return ctx.InternalServerError(goa.ErrInternal(fmt.Errorf(constant.InternalErr, errTypeEvents, 18)))
	}

	// EventsController_UpdateFavorite: end_implement
	return nil
}
<|endoftext|>


<commit_before>// (c) Copyright 2015-2017 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"database/sql"
	"errors"
	"fmt"
	"math"
	"strings"
)

// DocTypeID is the type of unique identifiers of document types.
type DocTypeID int64

// DocType enumerates the types of documents in the system, as defined
// by the consuming application.  Each document type has an associated
// workflow definition that drives its life cycle.
//
// Accordingly, `flow` does not assume anything about the specifics of
// the any document type.  Instead, it treats document types as plain,
// but controlled, vocabulary.  Nonetheless, it is highly recommended,
// but not necessary, that document types be defined in a system of
// hierarchical namespaces. For example:
//
//     PUR:RFQ
//
// could mean that the department is 'Purchasing', while the document
// type is 'Request For Quotation'.  As a variant,
//
//     PUR:ORD
//
// could mean that the document type is 'Purchase Order'.
//
// N.B. All document types must be defined as constant strings.
type DocType struct {
	id   DocTypeID // Unique identifier of this document type
	name string    // Unique name of this document type
}

// ID answers the unique identifier of this document type.
func (s *DocType) ID() DocTypeID {
	return s.id
}

// Name answers this type's name.
func (s *DocType) Name() string {
	return s.name
}

// Unexported type, only for convenience methods.
type _DocTypes struct{}

var _doctypes *_DocTypes

func init() {
	_doctypes = &_DocTypes{}
}

// DocTypes provides a resource-like interface to document types in
// the system.
func DocTypes() *_DocTypes {
	return _doctypes
}

// docStorName answers the appropriate table name for the given
// document type.
func (dts *_DocTypes) docStorName(dtid DocTypeID) string {
	return fmt.Sprintf("wf_documents_%03d", dtid)
}

// New creates and registers a new document type in the system.
func (dts *_DocTypes) New(otx *sql.Tx, name string) (DocTypeID, error) {
	name = strings.TrimSpace(name)
	if name == "" {
		return 0, errors.New("name cannot be empty")
	}

	var tx *sql.Tx
	if otx == nil {
		tx, err := db.Begin()
		if err != nil {
			return 0, err
		}
		defer tx.Rollback()
	} else {
		tx = otx
	}

	res, err := tx.Exec("INSERT INTO wf_doctypes_master(name) VALUES(?)", name)
	if err != nil {
		return 0, err
	}
	var id int64
	id, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	q := `
	CREATE TABLE ` + _doctypes.docStorName(DocTypeID(id)) + `(
		id INT NOT NULL AUTO_INCREMENT,
		user_id INT NOT NULL,
		docstate_id INT NOT NULL,
		ctime TIMESTAMP NOT NULL,
		title VARCHAR(250) NOT NULL,
		data BLOB NOT NULL,
		PRIMARY KEY (id),
		FOREIGN KEY (outer_doctype_id) REFERENCES wf_doctypes_master(id),
		FOREIGN KEY (docstate_id) REFERENCES wf_docstates_master(id)
	)
	`
	res, err = tx.Exec(q)
	if err != nil {
		return 0, err
	}

	if otx == nil {
		err = tx.Commit()
		if err != nil {
			return 0, err
		}
	}
	return DocTypeID(id), nil
}

// List answers a subset of the document types, based on the input
// specification.
//
// Result set begins with ID >= `offset`, and has not more than
// `limit` elements.  A value of `0` for `offset` fetches from the
// beginning, while a value of `0` for `limit` fetches until the end.
func (dts *_DocTypes) List(offset, limit int64) ([]*DocType, error) {
	if offset < 0 || limit < 0 {
		return nil, errors.New("offset and limit must be non-negative integers")
	}
	if limit == 0 {
		limit = math.MaxInt64
	}

	q := `
	SELECT name
	FROM wf_doctypes_master
	ORDER BY id
	LIMIT ? OFFSET ?
	`
	rows, err := db.Query(q, limit, offset)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	ary := make([]*DocType, 0, 10)
	for rows.Next() {
		var elem DocType
		err = rows.Scan(&elem.id, &elem.name)
		if err != nil {
			return nil, err
		}
		ary = append(ary, &elem)
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}

	return ary, nil
}

// Get retrieves the document type for the given ID.
func (dts *_DocTypes) Get(id DocTypeID) (*DocType, error) {
	if id <= 0 {
		return nil, errors.New("ID should be a positive integer")
	}

	var elem DocType
	row := db.QueryRow("SELECT id, name FROM wf_doctypes_master WHERE id = ?", id)
	err := row.Scan(&elem.id, &elem.name)
	if err != nil {
		return nil, err
	}

	return &elem, nil
}

// Rename renames the given document type.
func (dts *_DocTypes) Rename(otx *sql.Tx, elem *DocType, name string) error {
	name = strings.TrimSpace(name)
	if name == "" {
		return errors.New("name cannot be empty")
	}

	var tx *sql.Tx
	if otx == nil {
		tx, err := db.Begin()
		if err != nil {
			return err
		}
		defer tx.Rollback()
	} else {
		tx = otx
	}

	_, err := tx.Exec("UPDATE wf_doctypes_master SET name = ? WHERE id = ?", name, elem.id)
	if err != nil {
		return err
	}

	if otx == nil {
		err = tx.Commit()
		if err != nil {
			return err
		}
	}

	return nil
}

// Exists answers its unique ID, if a document type with the given
// name is registered; `0` and the error, otherwise.
func (dts *_DocTypes) Exists(name string) (DocTypeID, error) {
	name = strings.TrimSpace(name)
	if name == "" {
		return 0, errors.New("document type cannot be empty")
	}

	row := db.QueryRow("SELECT id FROM wf_doctypes_master WHERE name = ?", name)
	var n int64
	err := row.Scan(&n)
	if err != nil {
		return 0, err
	}

	return DocTypeID(n), nil
}
<commit_msg>Modify table creation for a new document type to `IF NOT EXISTS`<commit_after>// (c) Copyright 2015-2017 JONNALAGADDA Srinivas
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flow

import (
	"database/sql"
	"errors"
	"fmt"
	"math"
	"strings"
)

// DocTypeID is the type of unique identifiers of document types.
type DocTypeID int64

// DocType enumerates the types of documents in the system, as defined
// by the consuming application.  Each document type has an associated
// workflow definition that drives its life cycle.
//
// Accordingly, `flow` does not assume anything about the specifics of
// the any document type.  Instead, it treats document types as plain,
// but controlled, vocabulary.  Nonetheless, it is highly recommended,
// but not necessary, that document types be defined in a system of
// hierarchical namespaces. For example:
//
//     PUR:RFQ
//
// could mean that the department is 'Purchasing', while the document
// type is 'Request For Quotation'.  As a variant,
//
//     PUR:ORD
//
// could mean that the document type is 'Purchase Order'.
//
// N.B. All document types must be defined as constant strings.
type DocType struct {
	id   DocTypeID // Unique identifier of this document type
	name string    // Unique name of this document type
}

// ID answers the unique identifier of this document type.
func (s *DocType) ID() DocTypeID {
	return s.id
}

// Name answers this type's name.
func (s *DocType) Name() string {
	return s.name
}

// Unexported type, only for convenience methods.
type _DocTypes struct{}

var _doctypes *_DocTypes

func init() {
	_doctypes = &_DocTypes{}
}

// DocTypes provides a resource-like interface to document types in
// the system.
func DocTypes() *_DocTypes {
	return _doctypes
}

// docStorName answers the appropriate table name for the given
// document type.
func (dts *_DocTypes) docStorName(dtid DocTypeID) string {
	return fmt.Sprintf("wf_documents_%03d", dtid)
}

// New creates and registers a new document type in the system.
func (dts *_DocTypes) New(otx *sql.Tx, name string) (DocTypeID, error) {
	name = strings.TrimSpace(name)
	if name == "" {
		return 0, errors.New("name cannot be empty")
	}

	var tx *sql.Tx
	if otx == nil {
		tx, err := db.Begin()
		if err != nil {
			return 0, err
		}
		defer tx.Rollback()
	} else {
		tx = otx
	}

	res, err := tx.Exec("INSERT INTO wf_doctypes_master(name) VALUES(?)", name)
	if err != nil {
		return 0, err
	}
	var id int64
	id, err = res.LastInsertId()
	if err != nil {
		return 0, err
	}

	q := `
	CREATE TABLE IF NOT EXISTS ` + _doctypes.docStorName(DocTypeID(id)) + `(
		id INT NOT NULL AUTO_INCREMENT,
		user_id INT NOT NULL,
		docstate_id INT NOT NULL,
		ctime TIMESTAMP NOT NULL,
		title VARCHAR(250) NOT NULL,
		data BLOB NOT NULL,
		PRIMARY KEY (id),
		FOREIGN KEY (outer_doctype_id) REFERENCES wf_doctypes_master(id),
		FOREIGN KEY (docstate_id) REFERENCES wf_docstates_master(id)
	)
	`
	res, err = tx.Exec(q)
	if err != nil {
		return 0, err
	}

	if otx == nil {
		err = tx.Commit()
		if err != nil {
			return 0, err
		}
	}
	return DocTypeID(id), nil
}

// List answers a subset of the document types, based on the input
// specification.
//
// Result set begins with ID >= `offset`, and has not more than
// `limit` elements.  A value of `0` for `offset` fetches from the
// beginning, while a value of `0` for `limit` fetches until the end.
func (dts *_DocTypes) List(offset, limit int64) ([]*DocType, error) {
	if offset < 0 || limit < 0 {
		return nil, errors.New("offset and limit must be non-negative integers")
	}
	if limit == 0 {
		limit = math.MaxInt64
	}

	q := `
	SELECT name
	FROM wf_doctypes_master
	ORDER BY id
	LIMIT ? OFFSET ?
	`
	rows, err := db.Query(q, limit, offset)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	ary := make([]*DocType, 0, 10)
	for rows.Next() {
		var elem DocType
		err = rows.Scan(&elem.id, &elem.name)
		if err != nil {
			return nil, err
		}
		ary = append(ary, &elem)
	}
	if err = rows.Err(); err != nil {
		return nil, err
	}

	return ary, nil
}

// Get retrieves the document type for the given ID.
func (dts *_DocTypes) Get(id DocTypeID) (*DocType, error) {
	if id <= 0 {
		return nil, errors.New("ID should be a positive integer")
	}

	var elem DocType
	row := db.QueryRow("SELECT id, name FROM wf_doctypes_master WHERE id = ?", id)
	err := row.Scan(&elem.id, &elem.name)
	if err != nil {
		return nil, err
	}

	return &elem, nil
}

// Rename renames the given document type.
func (dts *_DocTypes) Rename(otx *sql.Tx, elem *DocType, name string) error {
	name = strings.TrimSpace(name)
	if name == "" {
		return errors.New("name cannot be empty")
	}

	var tx *sql.Tx
	if otx == nil {
		tx, err := db.Begin()
		if err != nil {
			return err
		}
		defer tx.Rollback()
	} else {
		tx = otx
	}

	_, err := tx.Exec("UPDATE wf_doctypes_master SET name = ? WHERE id = ?", name, elem.id)
	if err != nil {
		return err
	}

	if otx == nil {
		err = tx.Commit()
		if err != nil {
			return err
		}
	}

	return nil
}

// Exists answers its unique ID, if a document type with the given
// name is registered; `0` and the error, otherwise.
func (dts *_DocTypes) Exists(name string) (DocTypeID, error) {
	name = strings.TrimSpace(name)
	if name == "" {
		return 0, errors.New("document type cannot be empty")
	}

	row := db.QueryRow("SELECT id FROM wf_doctypes_master WHERE name = ?", name)
	var n int64
	err := row.Scan(&n)
	if err != nil {
		return 0, err
	}

	return DocTypeID(n), nil
}
<|endoftext|>


<commit_before>package kinesis

import (
	"time"

	"github.com/aws/aws-sdk-go/service/kinesis"
)

type Drainer struct {
	Buffer *recordBuffer
}

func newDrainer(client *kinesis.Kinesis, streamName string) (*Drainer, error) {
	buffer, err := newRecordBuffer(client, streamName)
	if err != nil {
		return nil, err
	}

	d := &Drainer{Buffer: buffer}
	// Every second, we flush the buffer
	go d.Drain()

	return d, nil
}

// Drain flushes the buffer every second.
func (d *Drainer) Drain() {
	for range time.Tick(time.Second * 1) {
		logErr(d.Buffer.Flush())
	}
}
<commit_msg>Fix range.<commit_after>package kinesis

import (
	"time"

	"github.com/aws/aws-sdk-go/service/kinesis"
)

type Drainer struct {
	Buffer *recordBuffer
}

func newDrainer(client *kinesis.Kinesis, streamName string) (*Drainer, error) {
	buffer, err := newRecordBuffer(client, streamName)
	if err != nil {
		return nil, err
	}

	d := &Drainer{Buffer: buffer}
	// Every second, we flush the buffer
	go d.Drain()

	return d, nil
}

// Drain flushes the buffer every second.
func (d *Drainer) Drain() {
	for _ = range time.Tick(time.Second * 1) {
		logErr(d.Buffer.Flush())
	}
}
<|endoftext|>


<commit_before>package mvc

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"unicode"

	"github.com/kataras/iris/core/router"
	"github.com/kataras/iris/macro"
)

const (
	tokenBy       = "By"
	tokenWildcard = "Wildcard" // "ByWildcard".
)

// word lexer, not characters.
type methodLexer struct {
	words []string
	cur   int
}

func newMethodLexer(s string) *methodLexer {
	l := new(methodLexer)
	l.reset(s)
	return l
}

func (l *methodLexer) reset(s string) {
	l.cur = -1
	var words []string
	if s != "" {
		end := len(s)
		start := -1

		for i, n := 0, end; i < n; i++ {
			c := rune(s[i])
			if unicode.IsUpper(c) {
				// it doesn't count the last uppercase
				if start != -1 {
					end = i
					words = append(words, s[start:end])
				}
				start = i
				continue
			}
			end = i + 1
		}

		if end > 0 && len(s) >= end {
			words = append(words, s[start:end])
		}
	}

	l.words = words
}

func (l *methodLexer) next() (w string) {
	cur := l.cur + 1

	if w = l.peek(cur); w != "" {
		l.cur++
	}

	return
}

func (l *methodLexer) skip() {
	if cur := l.cur + 1; cur < len(l.words) {
		l.cur = cur
	} else {
		l.cur = len(l.words) - 1
	}
}

func (l *methodLexer) peek(idx int) string {
	if idx < len(l.words) {
		return l.words[idx]
	}
	return ""
}

func (l *methodLexer) peekNext() (w string) {
	return l.peek(l.cur + 1)
}

func (l *methodLexer) peekPrev() (w string) {
	if l.cur > 0 {
		cur := l.cur - 1
		w = l.words[cur]
	}

	return w
}

func genParamKey(argIdx int) string {
	return "param" + strconv.Itoa(argIdx) // param0, param1, param2...
}

type methodParser struct {
	lexer  *methodLexer
	fn     reflect.Method
	macros macro.Macros
}

func parseMethod(macros macro.Macros, fn reflect.Method, skipper func(string) bool) (method, path string, err error) {
	if skipper(fn.Name) {
		return "", "", errSkip
	}

	p := &methodParser{
		fn:     fn,
		lexer:  newMethodLexer(fn.Name),
		macros: macros,
	}
	return p.parse()
}

func methodTitle(httpMethod string) string {
	httpMethodFuncName := strings.Title(strings.ToLower(httpMethod))
	return httpMethodFuncName
}

var errSkip = errors.New("skip")

var allMethods = append(router.AllMethods[0:], []string{"ALL", "ANY"}...)

func addPathWord(path, w string) string {
	if path[len(path)-1] != '/' {
		path += "/"
	}
	path += strings.ToLower(w)
	return path
}

func (p *methodParser) parse() (method, path string, err error) {
	funcArgPos := 0
	path = "/"
	// take the first word and check for the method.
	w := p.lexer.next()

	for _, httpMethod := range allMethods {
		possibleMethodFuncName := methodTitle(httpMethod)
		if strings.Index(w, possibleMethodFuncName) == 0 {
			method = httpMethod
			break
		}
	}

	if method == "" {
		// this is not a valid method to parse, we just skip it,
		//  it may be used for end-dev's use cases.
		return "", "", errSkip
	}

	for {
		w := p.lexer.next()
		if w == "" {
			break
		}

		if w == tokenBy {
			funcArgPos++ // starting with 1 because in typ.NumIn() the first is the struct receiver.

			// No need for these:
			// ByBy will act like /{param:type}/{param:type} as users expected
			// if func input arguments are there, else act By like normal path /by.
			//
			// if p.lexer.peekPrev() == tokenBy || typ.NumIn() == 1 { // ByBy, then act this second By like a path
			// 	a.relPath += "/" + strings.ToLower(w)
			// 	continue
			// }

			if path, funcArgPos, err = p.parsePathParam(path, w, funcArgPos); err != nil {
				return "", "", err
			}

			continue
		}
		// static path.
		path = addPathWord(path, w)
	}
	return
}

func (p *methodParser) parsePathParam(path string, w string, funcArgPos int) (string, int, error) {
	typ := p.fn.Type

	if typ.NumIn() <= funcArgPos {

		// By found but input arguments are not there, so act like /by path without restricts.
		path = addPathWord(path, w)
		return path, funcArgPos, nil
	}

	var (
		paramKey  = genParamKey(funcArgPos) // argfirst, argsecond...
		m         = p.macros.GetMaster()    // default (String by-default)
		trailings = p.macros.GetTrailings()
	)

	// string, int...
	goType := typ.In(funcArgPos).Kind()
	nextWord := p.lexer.peekNext()

	if nextWord == tokenWildcard {
		p.lexer.skip() // skip the Wildcard word.
		if len(trailings) == 0 {
			return "", 0, errors.New("no trailing path parameter found")
		}
		m = trailings[0]
	} else {
		// validMacros := p.macros.LookupForGoType(goType)

		// instead of mapping with a reflect.Kind which has its limitation,
		// we map the param types with a go type as a string,
		// so custom structs such as "user" can be mapped to a macro with indent || alias == "user".
		m = p.macros.Get(strings.ToLower(goType.String()))

		if m == nil {
			if typ.NumIn() > funcArgPos {
				// has more input arguments but we are not in the correct
				// index now, maybe the first argument was an `iris/context.Context`
				// so retry with the "funcArgPos" incremented.
				//
				// the "funcArgPos" will be updated to the caller as well
				// because we return it among the path and the error.
				return p.parsePathParam(path, w, funcArgPos+1)
			}

			return "", 0, fmt.Errorf("invalid syntax: the standard go type: %s found in controller's function: %s at position: %d does not match any valid macro", goType, p.fn.Name, funcArgPos)
		}
	}

	// /{argfirst:path}, /{argfirst:long}...
	if path[len(path)-1] != '/' {
		path += "/"
	}
	path += fmt.Sprintf("{%s:%s}", paramKey, m.Indent())

	if nextWord == "" && typ.NumIn() > funcArgPos+1 {
		// By is the latest word but func is expected
		// more path parameters values, i.e:
		// GetBy(name string, age int)
		// The caller (parse) doesn't need to know
		// about the incremental funcArgPos because
		// it will not need it.
		return p.parsePathParam(path, nextWord, funcArgPos+1)
	}

	return path, funcArgPos, nil
}
<commit_msg>Fix bug:about the MVC package route binding<commit_after>package mvc

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"unicode"

	"github.com/kataras/iris/core/router"
	"github.com/kataras/iris/macro"
)

const (
	tokenBy       = "By"
	tokenWildcard = "Wildcard" // "ByWildcard".
)

// word lexer, not characters.
type methodLexer struct {
	words []string
	cur   int
}

func newMethodLexer(s string) *methodLexer {
	l := new(methodLexer)
	l.reset(s)
	return l
}

func (l *methodLexer) reset(s string) {
	l.cur = -1
	var words []string
	if s != "" {
		end := len(s)
		start := -1

		for i, n := 0, end; i < n; i++ {
			c := rune(s[i])
			if unicode.IsUpper(c) {
				// it doesn't count the last uppercase
				if start != -1 {
					end = i
					words = append(words, s[start:end])
				}
				start = i
				continue
			}
			end = i + 1
		}

		if end > 0 && len(s) >= end {
			words = append(words, s[start:])
		}
	}

	l.words = words
}

func (l *methodLexer) next() (w string) {
	cur := l.cur + 1

	if w = l.peek(cur); w != "" {
		l.cur++
	}

	return
}

func (l *methodLexer) skip() {
	if cur := l.cur + 1; cur < len(l.words) {
		l.cur = cur
	} else {
		l.cur = len(l.words) - 1
	}
}

func (l *methodLexer) peek(idx int) string {
	if idx < len(l.words) {
		return l.words[idx]
	}
	return ""
}

func (l *methodLexer) peekNext() (w string) {
	return l.peek(l.cur + 1)
}

func (l *methodLexer) peekPrev() (w string) {
	if l.cur > 0 {
		cur := l.cur - 1
		w = l.words[cur]
	}

	return w
}

func genParamKey(argIdx int) string {
	return "param" + strconv.Itoa(argIdx) // param0, param1, param2...
}

type methodParser struct {
	lexer  *methodLexer
	fn     reflect.Method
	macros macro.Macros
}

func parseMethod(macros macro.Macros, fn reflect.Method, skipper func(string) bool) (method, path string, err error) {
	if skipper(fn.Name) {
		return "", "", errSkip
	}

	p := &methodParser{
		fn:     fn,
		lexer:  newMethodLexer(fn.Name),
		macros: macros,
	}
	return p.parse()
}

func methodTitle(httpMethod string) string {
	httpMethodFuncName := strings.Title(strings.ToLower(httpMethod))
	return httpMethodFuncName
}

var errSkip = errors.New("skip")

var allMethods = append(router.AllMethods[0:], []string{"ALL", "ANY"}...)

func addPathWord(path, w string) string {
	if path[len(path)-1] != '/' {
		path += "/"
	}
	path += strings.ToLower(w)
	return path
}

func (p *methodParser) parse() (method, path string, err error) {
	funcArgPos := 0
	path = "/"
	// take the first word and check for the method.
	w := p.lexer.next()

	for _, httpMethod := range allMethods {
		possibleMethodFuncName := methodTitle(httpMethod)
		if strings.Index(w, possibleMethodFuncName) == 0 {
			method = httpMethod
			break
		}
	}

	if method == "" {
		// this is not a valid method to parse, we just skip it,
		//  it may be used for end-dev's use cases.
		return "", "", errSkip
	}

	for {
		w := p.lexer.next()
		if w == "" {
			break
		}

		if w == tokenBy {
			funcArgPos++ // starting with 1 because in typ.NumIn() the first is the struct receiver.

			// No need for these:
			// ByBy will act like /{param:type}/{param:type} as users expected
			// if func input arguments are there, else act By like normal path /by.
			//
			// if p.lexer.peekPrev() == tokenBy || typ.NumIn() == 1 { // ByBy, then act this second By like a path
			// 	a.relPath += "/" + strings.ToLower(w)
			// 	continue
			// }

			if path, funcArgPos, err = p.parsePathParam(path, w, funcArgPos); err != nil {
				return "", "", err
			}

			continue
		}
		// static path.
		path = addPathWord(path, w)
	}
	return
}

func (p *methodParser) parsePathParam(path string, w string, funcArgPos int) (string, int, error) {
	typ := p.fn.Type

	if typ.NumIn() <= funcArgPos {

		// By found but input arguments are not there, so act like /by path without restricts.
		path = addPathWord(path, w)
		return path, funcArgPos, nil
	}

	var (
		paramKey  = genParamKey(funcArgPos) // argfirst, argsecond...
		m         = p.macros.GetMaster()    // default (String by-default)
		trailings = p.macros.GetTrailings()
	)

	// string, int...
	goType := typ.In(funcArgPos).Kind()
	nextWord := p.lexer.peekNext()

	if nextWord == tokenWildcard {
		p.lexer.skip() // skip the Wildcard word.
		if len(trailings) == 0 {
			return "", 0, errors.New("no trailing path parameter found")
		}
		m = trailings[0]
	} else {
		// validMacros := p.macros.LookupForGoType(goType)

		// instead of mapping with a reflect.Kind which has its limitation,
		// we map the param types with a go type as a string,
		// so custom structs such as "user" can be mapped to a macro with indent || alias == "user".
		m = p.macros.Get(strings.ToLower(goType.String()))

		if m == nil {
			if typ.NumIn() > funcArgPos {
				// has more input arguments but we are not in the correct
				// index now, maybe the first argument was an `iris/context.Context`
				// so retry with the "funcArgPos" incremented.
				//
				// the "funcArgPos" will be updated to the caller as well
				// because we return it among the path and the error.
				return p.parsePathParam(path, w, funcArgPos+1)
			}

			return "", 0, fmt.Errorf("invalid syntax: the standard go type: %s found in controller's function: %s at position: %d does not match any valid macro", goType, p.fn.Name, funcArgPos)
		}
	}

	// /{argfirst:path}, /{argfirst:long}...
	if path[len(path)-1] != '/' {
		path += "/"
	}
	path += fmt.Sprintf("{%s:%s}", paramKey, m.Indent())

	if nextWord == "" && typ.NumIn() > funcArgPos+1 {
		// By is the latest word but func is expected
		// more path parameters values, i.e:
		// GetBy(name string, age int)
		// The caller (parse) doesn't need to know
		// about the incremental funcArgPos because
		// it will not need it.
		return p.parsePathParam(path, nextWord, funcArgPos+1)
	}

	return path, funcArgPos, nil
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"github.com/HouzuoGuo/laitos/frontend/common"
	"github.com/HouzuoGuo/laitos/global"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

// Render an HTML page that has client IP and current time injected into the content.
type HandleHTMLDocument struct {
	HTMLFilePath string `json:"HTMLFilePath"`
}

func (index *HandleHTMLDocument) MakeHandler(logger global.Logger, cmdProc *common.CommandProcessor) (http.HandlerFunc, error) {
	var err error
	var contentBytes []byte
	if contentBytes, err = ioutil.ReadFile(index.HTMLFilePath); err != nil {
		return nil, fmt.Errorf("HandleHTMLDocument.MakeHandler: failed to open HTML file at %s - %v", index.HTMLFilePath, err)
	}
	contentStr := string(contentBytes)
	// Inject browser client IP and current time into index document and return.
	fun := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		NoCache(w)
		page := strings.Replace(contentStr, "#LAITOS_3339TIME", time.Now().Format(time.RFC3339), -1)
		page = strings.Replace(page, "#LAITOS_CLIENTADDR", GetRealClientIP(r), -1)
		w.Write([]byte(page))
	}
	return fun, nil
}

func (index *HandleHTMLDocument) GetRateLimitFactor() int {
	return 20
}
<commit_msg>further increase rate limit on HTML documents<commit_after>package api

import (
	"fmt"
	"github.com/HouzuoGuo/laitos/frontend/common"
	"github.com/HouzuoGuo/laitos/global"
	"io/ioutil"
	"net/http"
	"strings"
	"time"
)

// Render an HTML page that has client IP and current time injected into the content.
type HandleHTMLDocument struct {
	HTMLFilePath string `json:"HTMLFilePath"`
}

func (index *HandleHTMLDocument) MakeHandler(logger global.Logger, cmdProc *common.CommandProcessor) (http.HandlerFunc, error) {
	var err error
	var contentBytes []byte
	if contentBytes, err = ioutil.ReadFile(index.HTMLFilePath); err != nil {
		return nil, fmt.Errorf("HandleHTMLDocument.MakeHandler: failed to open HTML file at %s - %v", index.HTMLFilePath, err)
	}
	contentStr := string(contentBytes)
	// Inject browser client IP and current time into index document and return.
	fun := func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		NoCache(w)
		page := strings.Replace(contentStr, "#LAITOS_3339TIME", time.Now().Format(time.RFC3339), -1)
		page = strings.Replace(page, "#LAITOS_CLIENTADDR", GetRealClientIP(r), -1)
		w.Write([]byte(page))
	}
	return fun, nil
}

func (index *HandleHTMLDocument) GetRateLimitFactor() int {
	return 25
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package certs implements a CertSource which speaks to the public CLoud SQL API endpoint.
package certs

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"fmt"
	"log"
	"math"
	mrand "math/rand"
	"net/http"
	"strings"
	"time"

	"google.golang.org/api/googleapi"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// NewCertSource returns a CertSource which can be used to authenticate using
// the provided oauth token. The provided client must not be nil.
func NewCertSource(host string, c *http.Client, checkRegion bool) *RemoteCertSource {
	pkey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		panic(err) // very unexpected.
	}
	serv, err := sqladmin.New(c)
	if err != nil {
		panic(err) // only possible if 'c' is nil.
	}
	return &RemoteCertSource{pkey, host + "projects/", serv, checkRegion}
}

// RemoteCertSource implements a CertSource, using Cloud SQL APIs to
// return Local certificates for identifying oneself as a specific user
// to the remote instance and Remote certificates for confirming the
// remote database's identity.
type RemoteCertSource struct {
	// key is the private key used for certificates returned by Local.
	key *rsa.PrivateKey
	// basepath is the URL prefix for Cloud SQL API calls.
	basepath string
	// serv is used to make authenticated API calls to Cloud SQL.
	serv *sqladmin.Service
	// If set, providing an incorrect region in their connection string will be
	// treated as an error. This is to provide the same functionality that will
	// occur when API calls require the region.
	checkRegion bool
}

// splitName splits a fully qualified instance into its project, region, and
// instance name components. While we make the transition to regionalized
// metadata, the region is optional.
//
// Examples:
//    "proj:region:my-db" -> ("proj", "region", "my-db")
//		"google.com:project:region:instance" -> ("google.com:project", "region", "instance")
//		"google.com:missing:part" -> ("google.com:missing", "", "part")
func splitName(instance string) (project, region, name string) {
	spl := strings.Split(instance, ":")
	if len(spl) < 2 {
		return "", "", instance
	}
	if dot := strings.Index(spl[0], "."); dot != -1 {
		spl[1] = spl[0] + ":" + spl[1]
		spl = spl[1:]
	}
	switch {
	case len(spl) < 2:
		return "", "", instance
	case len(spl) == 2:
		return spl[0], "", spl[1]
	default:
		return spl[0], spl[1], spl[2]
	}
}

const (
	baseBackoff = float64(time.Second / 5)
	backoffMult = 1.618
)

func backoffAPIRetry(iters int, desc string, do func() error) error {
	var err error
	for i := 0; i < iters; i++ {
		err = do()
		// Only Server-level HTTP errors are immediately retryable.
		// 'ok' will also be false if err is nil.
		gerr, ok := err.(*googleapi.Error)
		if !ok || gerr.Code < 500 {
			return err
		}

		exp := float64(i) * (1 - mrand.Float64()/10)
		sleep := time.Duration(baseBackoff * math.Pow(backoffMult, exp))
		log.Printf("Error in %s: %v; retrying in %v", desc, err, sleep)
		time.Sleep(sleep)
	}
	return err
}

// Local returns a certificate that may be used to establish a TLS
// connection to the specified instance.
func (s *RemoteCertSource) Local(instance string) (ret tls.Certificate, err error) {
	pkix, err := x509.MarshalPKIXPublicKey(&s.key.PublicKey)
	if err != nil {
		return ret, err
	}

	p, _, n := splitName(instance)
	req := s.serv.SslCerts.CreateEphemeral(p, n,
		&sqladmin.SslCertsCreateEphemeralRequest{
			PublicKey: string(pem.EncodeToMemory(&pem.Block{Bytes: pkix, Type: "RSA PUBLIC KEY"})),
		},
	)

	var data *sqladmin.SslCert
	err = backoffAPIRetry(5, "createEphemeral for "+instance, func() error {
		data, err = req.Do()
		return err
	})
	if err != nil {
		return ret, err
	}

	c, err := parseCert(data.Cert)
	if err != nil {
		return ret, fmt.Errorf("couldn't parse ephemeral certificate for instance %q: %v", instance, err)
	}
	return tls.Certificate{
		Certificate: [][]byte{c.Raw},
		PrivateKey:  s.key,
		Leaf:        c,
	}, nil
}

func parseCert(pemCert string) (*x509.Certificate, error) {
	bl, _ := pem.Decode([]byte(pemCert))
	if bl == nil {
		return nil, errors.New("invalid PEM: " + pemCert)
	}
	return x509.ParseCertificate(bl.Bytes)
}

// Remote returns the specified instance's CA certificate, address, and name.
func (s *RemoteCertSource) Remote(instance string) (cert *x509.Certificate, addr, name string, err error) {
	p, region, n := splitName(instance)
	req := s.serv.Instances.Get(p, n)

	var data *sqladmin.DatabaseInstance
	err = backoffAPIRetry(5, "get instance "+instance, func() error {
		data, err = req.Do()
		return err
	})
	if err != nil {
		return nil, "", "", err
	}

	// TODO(chowski): remove this when us-central is removed.
	if data.Region == "us-central" {
		data.Region = "us-central1"
	}
	if data.Region != region {
		if region == "" {
			err = fmt.Errorf("instance %v doesn't provide region", instance)
		} else {
			err = fmt.Errorf(`for connection string "%s": got region %q, want %q`, instance, region, data.Region)
		}
		if s.checkRegion {
			return nil, "", "", err
		}
		log.Print(err)
	}
	if len(data.IpAddresses) == 0 || data.IpAddresses[0].IpAddress == "" {
		return nil, "", "", fmt.Errorf("no IP address found for %v", instance)
	}
	c, err := parseCert(data.ServerCaCert.Cert)
	return c, data.IpAddresses[0].IpAddress, p + ":" + n, err
}
<commit_msg>Address some comments in the previous commit.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package certs implements a CertSource which speaks to the public CLoud SQL API endpoint.
package certs

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"errors"
	"fmt"
	"log"
	"math"
	mrand "math/rand"
	"net/http"
	"strings"
	"time"

	"google.golang.org/api/googleapi"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// NewCertSource returns a CertSource which can be used to authenticate using
// the provided oauth token. The provided client must not be nil.
func NewCertSource(host string, c *http.Client, checkRegion bool) *RemoteCertSource {
	pkey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		panic(err) // very unexpected.
	}
	serv, err := sqladmin.New(c)
	if err != nil {
		panic(err) // only possible if 'c' is nil.
	}
	return &RemoteCertSource{pkey, host + "projects/", serv, checkRegion}
}

// RemoteCertSource implements a CertSource, using Cloud SQL APIs to
// return Local certificates for identifying oneself as a specific user
// to the remote instance and Remote certificates for confirming the
// remote database's identity.
type RemoteCertSource struct {
	// key is the private key used for certificates returned by Local.
	key *rsa.PrivateKey
	// basepath is the URL prefix for Cloud SQL API calls.
	basepath string
	// serv is used to make authenticated API calls to Cloud SQL.
	serv *sqladmin.Service
	// If set, providing an incorrect region in their connection string will be
	// treated as an error. This is to provide the same functionality that will
	// occur when API calls require the region.
	checkRegion bool
}

// splitName splits a fully qualified instance into its project, region, and
// instance name components. While we make the transition to regionalized
// metadata, the region is optional.
//
// Examples:
//    "proj:region:my-db" -> ("proj", "region", "my-db")
//		"google.com:project:region:instance" -> ("google.com:project", "region", "instance")
//		"google.com:missing:part" -> ("google.com:missing", "", "part")
func splitName(instance string) (project, region, name string) {
	spl := strings.Split(instance, ":")
	if len(spl) < 2 {
		return "", "", instance
	}
	if dot := strings.Index(spl[0], "."); dot != -1 {
		spl[1] = spl[0] + ":" + spl[1]
		spl = spl[1:]
	}
	switch {
	case len(spl) < 2:
		return "", "", instance
	case len(spl) == 2:
		return spl[0], "", spl[1]
	default:
		return spl[0], spl[1], spl[2]
	}
}

// Constants for backoffAPIRetry. These cause the retry logic to scale the
// backoff delay from 200ms to around 3.5s.
const (
	baseBackoff    = float64(200 * time.Millisecond)
	backoffMult    = 1.618
	backoffRetries = 5
)
 asdfasdf

func backoffAPIRetry(desc string, do func() error) error {
	var err error
	for i := 0; i < backoffRetries; i++ {
		err = do()
		// Only Server-level HTTP errors are immediately retryable.
		// 'ok' will also be false if err is nil.
		gErr, ok := err.(*googleapi.Error)
		if !ok || gErr.Code < 500 {
			return err
		}

		// sleep = baseBackoff * backoffMult^(retries + randomFactor)
		exp := float64(i+1) + mrand.Float64()
		sleep := time.Duration(baseBackoff * math.Pow(backoffMult, exp))
		log.Printf("Error in %s: %v; retrying in %v", desc, err, sleep)
		time.Sleep(sleep)
	}
	return err
}

// Local returns a certificate that may be used to establish a TLS
// connection to the specified instance.
func (s *RemoteCertSource) Local(instance string) (ret tls.Certificate, err error) {
	pkix, err := x509.MarshalPKIXPublicKey(&s.key.PublicKey)
	if err != nil {
		return ret, err
	}

	p, _, n := splitName(instance)
	req := s.serv.SslCerts.CreateEphemeral(p, n,
		&sqladmin.SslCertsCreateEphemeralRequest{
			PublicKey: string(pem.EncodeToMemory(&pem.Block{Bytes: pkix, Type: "RSA PUBLIC KEY"})),
		},
	)

	var data *sqladmin.SslCert
	err = backoffAPIRetry("createEphemeral for "+instance, func() error {
		data, err = req.Do()
		return err
	})
	if err != nil {
		return ret, err
	}

	c, err := parseCert(data.Cert)
	if err != nil {
		return ret, fmt.Errorf("couldn't parse ephemeral certificate for instance %q: %v", instance, err)
	}
	return tls.Certificate{
		Certificate: [][]byte{c.Raw},
		PrivateKey:  s.key,
		Leaf:        c,
	}, nil
}

func parseCert(pemCert string) (*x509.Certificate, error) {
	bl, _ := pem.Decode([]byte(pemCert))
	if bl == nil {
		return nil, errors.New("invalid PEM: " + pemCert)
	}
	return x509.ParseCertificate(bl.Bytes)
}

// Remote returns the specified instance's CA certificate, address, and name.
func (s *RemoteCertSource) Remote(instance string) (cert *x509.Certificate, addr, name string, err error) {
	p, region, n := splitName(instance)
	req := s.serv.Instances.Get(p, n)

	var data *sqladmin.DatabaseInstance
	err = backoffAPIRetry("get instance "+instance, func() error {
		data, err = req.Do()
		return err
	})
	if err != nil {
		return nil, "", "", err
	}

	// TODO(chowski): remove this when us-central is removed.
	if data.Region == "us-central" {
		data.Region = "us-central1"
	}
	if data.Region != region {
		if region == "" {
			err = fmt.Errorf("instance %v doesn't provide region", instance)
		} else {
			err = fmt.Errorf(`for connection string "%s": got region %q, want %q`, instance, region, data.Region)
		}
		if s.checkRegion {
			return nil, "", "", err
		}
		log.Print(err)
	}
	if len(data.IpAddresses) == 0 || data.IpAddresses[0].IpAddress == "" {
		return nil, "", "", fmt.Errorf("no IP address found for %v", instance)
	}
	c, err := parseCert(data.ServerCaCert.Cert)
	return c, data.IpAddresses[0].IpAddress, p + ":" + n, err
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/fzzy/sockjs-go/sockjs"
	"log"
	"warcluster/db_manager"
	"warcluster/entities"
)

// The information for each person is stored in two seperate structures. Player and Client.
// This is one of them. The purpouse of the Client struct is to hold the server(connection) information.
// 1.Session holds the curent player session socket for comunication.
// 2.Nickname is used as a key because its the common data between the two structures and the database.
// 3.This is a pointer to the player struct for easy acsess.
type Client struct {
	Session  sockjs.Session
	Nickname string
	Player   *entities.Player
}

// Authenticate is a function called for every client's new session.
// It manages several important tasks at the start of the session.
// 1.Ask the user for Username and twitter ID.
// 2.Search the DB to find the player if it's not a new one.
// 3.If the player is new there is a subsequence initiated:
// 3.1.Create a new sun with GenerateSun
// 3.2.Choose home planet from the newly created solar sysitem.
// 3.3.Create a reccord of the new player and start comunication.
func authenticate(session sockjs.Session) (string, *entities.Player, error) {
	var player *entities.Player
	var nickname string
	var twitter_id string
	request := new(Request)

	for {
		if message := session.Receive(); message == nil {
			return "", nil, errors.New("No credentials provided")
		} else {
			if err := json.Unmarshal(message, request); err == nil {
				if len(request.Username) > 0 && len(request.TwitterID) > 0 {
					nickname = request.Username
					twitter_id = request.TwitterID
					break
				}
			} else {
				log.Print("Error in server.client.authenticate: ", err.Error())
			}
		}
	}

	entity, _ := db_manager.GetEntity(fmt.Sprintf("player.%s", []byte(nickname)))
	if entity == nil {
		all_suns_entities := db_manager.GetEntities("sun.*")
		all_suns := []entities.Sun{}
		for _, entity := range all_suns_entities {
			all_suns = append(all_suns, *entity.(*entities.Sun))
		}
		sun := entities.GenerateSun(nickname, all_suns, []entities.Sun{})
		hash := entities.GenerateHash(nickname)
		planets, home_planet := entities.GeneratePlanets(hash, sun.GetPosition())

		//TODO: Remove the bottom three lines when the client is smart enough to invoke scope of view on all clients in order to osee the generated system
		for planet := range planets {
			key, serialized_planet, _ := planets[planet].Serialize()
			sessions.Broadcast([]byte(fmt.Sprintf("{\"Command\": \"state_change\", \"Planets\": {\"%s\": %s}}", key, serialized_planet)))
		}

		player = entities.CreatePlayer(nickname, twitter_id, home_planet)
		log.Print("Player", player)
		db_manager.SetEntity(player)
		db_manager.SetEntity(sun)
		for i := 0; i < len(planets); i++ {
			db_manager.SetEntity(planets[i])
		}
	} else {
		player = entity.(*entities.Player)
	}
	return nickname, player, nil
}
<commit_msg>Add sun to new player status update<commit_after>package server

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/fzzy/sockjs-go/sockjs"
	"log"
	"warcluster/db_manager"
	"warcluster/entities"
)

// The information for each person is stored in two seperate structures. Player and Client.
// This is one of them. The purpouse of the Client struct is to hold the server(connection) information.
// 1.Session holds the curent player session socket for comunication.
// 2.Nickname is used as a key because its the common data between the two structures and the database.
// 3.This is a pointer to the player struct for easy acsess.
type Client struct {
	Session  sockjs.Session
	Nickname string
	Player   *entities.Player
}

// Authenticate is a function called for every client's new session.
// It manages several important tasks at the start of the session.
// 1.Ask the user for Username and twitter ID.
// 2.Search the DB to find the player if it's not a new one.
// 3.If the player is new there is a subsequence initiated:
// 3.1.Create a new sun with GenerateSun
// 3.2.Choose home planet from the newly created solar sysitem.
// 3.3.Create a reccord of the new player and start comunication.
func authenticate(session sockjs.Session) (string, *entities.Player, error) {
	var player *entities.Player
	var nickname string
	var twitter_id string
	request := new(Request)

	for {
		if message := session.Receive(); message == nil {
			return "", nil, errors.New("No credentials provided")
		} else {
			if err := json.Unmarshal(message, request); err == nil {
				if len(request.Username) > 0 && len(request.TwitterID) > 0 {
					nickname = request.Username
					twitter_id = request.TwitterID
					break
				}
			} else {
				log.Print("Error in server.client.authenticate: ", err.Error())
			}
		}
	}

	entity, _ := db_manager.GetEntity(fmt.Sprintf("player.%s", []byte(nickname)))
	if entity == nil {
		all_suns_entities := db_manager.GetEntities("sun.*")
		all_suns := []entities.Sun{}
		for _, entity := range all_suns_entities {
			all_suns = append(all_suns, *entity.(*entities.Sun))
		}
		sun := entities.GenerateSun(nickname, all_suns, []entities.Sun{})
		hash := entities.GenerateHash(nickname)
		planets, home_planet := entities.GeneratePlanets(hash, sun.GetPosition())

		//TODO: Remove the bottom three lines when the client is smart enough to invoke scope of view on all clients in order to osee the generated system
		for planet := range planets {
			key, serialized_planet, _ := planets[planet].Serialize()
			sessions.Broadcast([]byte(fmt.Sprintf("{\"Command\": \"state_change\", \"Planets\": {\"%s\": %s}}", key, serialized_planet)))
		}
		sun_key, serialized_sun, _ := sun.Serialize()
		sessions.Broadcast([]byte(fmt.Sprintf("{\"Command\": \"state_change\", \"Suns\": {\"%s\": %s}}", sun_key, serialized_sun)))

		player = entities.CreatePlayer(nickname, twitter_id, home_planet)
		log.Print("Player", player)
		db_manager.SetEntity(player)
		db_manager.SetEntity(sun)
		for i := 0; i < len(planets); i++ {
			db_manager.SetEntity(planets[i])
		}
	} else {
		player = entity.(*entities.Player)
	}
	return nickname, player, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/grafana/grafana/pkg/log"
	met "github.com/grafana/grafana/pkg/metric"
	"github.com/grafana/grafana/pkg/metric/helper"
	"github.com/nsqio/go-nsq"
	"github.com/raintank/raintank-metric/app"
	"github.com/raintank/raintank-metric/instrumented_nsq"

	"github.com/codeskyblue/go-uuid"
	"github.com/raintank/raintank-metric/eventdef"
	"github.com/raintank/raintank-metric/schema"
	"github.com/rakyll/globalconf"
)

var (
	showVersion = flag.Bool("version", false, "print version string")

	topic       = flag.String("topic", "probe_events", "NSQ topic")
	channel     = flag.String("channel", "elasticsearch", "NSQ channel")
	maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")

	concurrency = flag.Int("concurrency", 10, "number of workers parsing messages")

	esAddr = flag.String("elastic-addr", "localhost:9200", "elasticsearch address (default: localhost:9200)")

	statsdAddr = flag.String("statsd-addr", "localhost:8125", "statsd address (default: localhost:8125)")
	statsdType = flag.String("statsd-type", "standard", "statsd type: standard or datadog (default: standard)")
	confFile   = flag.String("config", "/etc/raintank/nsq_probe_events_to_elasticsearch.ini", "configuration file (default /etc/raintank/nsq_probe_events_to_elasticsearch.ini")

	consumerOpts     = flag.String("consumer-opt", "", "option to passthrough to nsq.Consumer (may be given multiple times as comma-separated list, http://godoc.org/github.com/nsqio/go-nsq#Config)")
	nsqdTCPAddrs     = flag.String("nsqd-tcp-address", "", "nsqd TCP address (may be given multiple times as comma-separated list)")
	lookupdHTTPAddrs = flag.String("lookupd-http-address", "", "lookupd HTTP address (may be given multiple times as comma-separated list)")
	logLevel         = flag.Int("log-level", 2, "log level. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL")
	listenAddr       = flag.String("listen", ":6060", "http listener address.")

	eventsToEsOK   met.Count
	eventsToEsFail met.Count
	esPutDuration  met.Timer
	messagesSize   met.Meter
	msgsAge        met.Meter // in ms
	msgsHandleOK   met.Count
	msgsHandleFail met.Count

	writeQueue *InProgressMessageQueue
)

type ESHandler struct {
}

func NewESHandler() (*ESHandler, error) {
	return &ESHandler{}, nil
}

func (k *ESHandler) HandleMessage(m *nsq.Message) error {
	log.Debug("received message.")
	format := "unknown"
	if m.Body[0] == '\x00' {
		format = "msgFormatJson"
	}

	var id int64
	buf := bytes.NewReader(m.Body[1:9])
	binary.Read(buf, binary.BigEndian, &id)
	produced := time.Unix(0, id)

	msgsAge.Value(time.Now().Sub(produced).Nanoseconds() / 1000)
	messagesSize.Value(int64(len(m.Body)))

	event := new(schema.ProbeEvent)
	if err := json.Unmarshal(m.Body[9:], &event); err != nil {
		log.Error(3, "ERROR: failure to unmarshal message body via format %s: %s. skipping message", format, err)
		return nil
	}

	// Since these messages are being batched, we'll need to hold onto this
	// and ack or requeue it on our own
	m.DisableAutoResponse()
	if event.Id == "" {
		// per http://blog.mikemccandless.com/2014/05/choosing-fast-unique-identifier-uuid.html,
		// using V1 UUIDs is much faster than v4 like we were using
		u := uuid.NewUUID()
		event.Id = u.String()
	}
	writeQueue.EnQueue(event.Id, m)

	if err := eventdef.Save(event); err != nil {
		log.Error(3, "couldn't process %s: %s", event.Id, err)
		msgsHandleFail.Inc(1)
		m.Requeue(-1)
		return err
	}

	return nil
}

type inProgressMessage struct {
	timestamp time.Time
	message   *nsq.Message
}

type InProgressMessageQueue struct {
	sync.RWMutex
	inProgress map[string]*inProgressMessage
	status     chan *eventdef.BulkSaveStatus
}

func (q *InProgressMessageQueue) EnQueue(id string, m *nsq.Message) {
	q.Lock()
	q.inProgress[id] = &inProgressMessage{
		timestamp: time.Now(),
		message:   m,
	}
	q.Unlock()
}

func (q *InProgressMessageQueue) loop() {
	for {
		select {
		case s := <-q.status:
			q.Lock()
			if m, ok := q.inProgress[s.Id]; ok {
				if s.Ok {
					m.message.Finish()
					eventsToEsOK.Inc(1)
					msgsHandleOK.Inc(1)
					log.Debug("event %s commited to ES", s.Id)
				} else {
					m.message.Requeue(-1)
					eventsToEsFail.Inc(1)
					msgsHandleFail.Inc(1)
					log.Error(3, "event %s failed to save, requeueing", s.Id)
				}
				esPutDuration.Value(time.Now().Sub(m.timestamp))
			} else {
				log.Error(3, "got processing response for unknown message. event %s", s.Id)
			}
			delete(q.inProgress, s.Id)
			q.Unlock()
		}
	}
}

func NewInProgressMessageQueue() *InProgressMessageQueue {
	q := &InProgressMessageQueue{
		inProgress: make(map[string]*inProgressMessage),
		status:     make(chan *eventdef.BulkSaveStatus, *maxInFlight),
	}
	for i := 0; i < *concurrency; i++ {
		go q.loop()
	}
	return q
}

func main() {
	flag.Parse()

	// Only try and parse the conf file if it exists
	if _, err := os.Stat(*confFile); err == nil {
		conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
		if err != nil {
			log.Fatal(4, err.Error())
		}
		conf.ParseAll()
	}

	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))

	if *showVersion {
		fmt.Println("nsq_probe_events_to_elasticsearch")
		return
	}

	if *channel == "" {
		rand.Seed(time.Now().UnixNano())
		*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
	}

	if *topic == "" {
		log.Fatal(4, "--topic is required")
	}

	if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
		log.Fatal(4, "--nsqd-tcp-address or --lookupd-http-address required")
	}
	if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
		log.Fatal(4, "use --nsqd-tcp-address or --lookupd-http-address not both")
	}

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(4, err.Error())
	}
	metrics, err := helper.New(true, *statsdAddr, *statsdType, "nsq_probe_events_to_elasticsearch", strings.Replace(hostname, ".", "_", -1))
	if err != nil {
		log.Fatal(4, err.Error())
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	messagesSize = metrics.NewMeter("message_size", 0)
	msgsAge = metrics.NewMeter("message_age", 0)
	eventsToEsOK = metrics.NewCount("events_to_es.ok")
	eventsToEsFail = metrics.NewCount("events_to_es.fail")
	esPutDuration = metrics.NewTimer("es_put_duration", 0)
	msgsHandleOK = metrics.NewCount("handle.ok")
	msgsHandleFail = metrics.NewCount("handle.fail")

	writeQueue = NewInProgressMessageQueue()

	err = eventdef.InitElasticsearch(*esAddr, "", "", writeQueue.status, *maxInFlight)
	if err != nil {
		log.Fatal(4, err.Error())
	}

	cfg := nsq.NewConfig()
	cfg.UserAgent = "nsq_probe_events_to_elasticsearch"
	err = app.ParseOpts(cfg, *consumerOpts)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	cfg.MaxInFlight = *maxInFlight

	consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", metrics)

	if err != nil {
		log.Fatal(4, err.Error())
	}

	handler, err := NewESHandler()
	if err != nil {
		log.Fatal(4, err.Error())
	}

	consumer.AddConcurrentHandlers(handler, *concurrency)

	nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
	if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
		nsqdAdds = []string{}
	}
	err = consumer.ConnectToNSQDs(nsqdAdds)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	log.Info("connected to nsqd")

	lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
	if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
		lookupdAdds = []string{}
	}
	err = consumer.ConnectToNSQLookupds(lookupdAdds)
	if err != nil {
		log.Fatal(4, err.Error())
	}

	go func() {
		log.Info("INFO starting listener for http/debug on %s", *listenAddr)
		httperr := http.ListenAndServe(*listenAddr, nil)
		if httperr != nil {
			log.Info(httperr.Error())
		}
	}()

	for {
		select {
		case <-consumer.StopChan:
			return
		case <-sigChan:
			consumer.Stop()
			eventdef.StopBulkIndexer()
		}
	}
}
<commit_msg>dont try and manage nil nsq messages<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/grafana/grafana/pkg/log"
	met "github.com/grafana/grafana/pkg/metric"
	"github.com/grafana/grafana/pkg/metric/helper"
	"github.com/nsqio/go-nsq"
	"github.com/raintank/raintank-metric/app"
	"github.com/raintank/raintank-metric/instrumented_nsq"

	"github.com/codeskyblue/go-uuid"
	"github.com/raintank/raintank-metric/eventdef"
	"github.com/raintank/raintank-metric/schema"
	"github.com/rakyll/globalconf"
)

var (
	showVersion = flag.Bool("version", false, "print version string")

	topic       = flag.String("topic", "probe_events", "NSQ topic")
	channel     = flag.String("channel", "elasticsearch", "NSQ channel")
	maxInFlight = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")

	concurrency = flag.Int("concurrency", 10, "number of workers parsing messages")

	esAddr = flag.String("elastic-addr", "localhost:9200", "elasticsearch address (default: localhost:9200)")

	statsdAddr = flag.String("statsd-addr", "localhost:8125", "statsd address (default: localhost:8125)")
	statsdType = flag.String("statsd-type", "standard", "statsd type: standard or datadog (default: standard)")
	confFile   = flag.String("config", "/etc/raintank/nsq_probe_events_to_elasticsearch.ini", "configuration file (default /etc/raintank/nsq_probe_events_to_elasticsearch.ini")

	consumerOpts     = flag.String("consumer-opt", "", "option to passthrough to nsq.Consumer (may be given multiple times as comma-separated list, http://godoc.org/github.com/nsqio/go-nsq#Config)")
	nsqdTCPAddrs     = flag.String("nsqd-tcp-address", "", "nsqd TCP address (may be given multiple times as comma-separated list)")
	lookupdHTTPAddrs = flag.String("lookupd-http-address", "", "lookupd HTTP address (may be given multiple times as comma-separated list)")
	logLevel         = flag.Int("log-level", 2, "log level. 0=TRACE|1=DEBUG|2=INFO|3=WARN|4=ERROR|5=CRITICAL|6=FATAL")
	listenAddr       = flag.String("listen", ":6060", "http listener address.")

	eventsToEsOK   met.Count
	eventsToEsFail met.Count
	esPutDuration  met.Timer
	messagesSize   met.Meter
	msgsAge        met.Meter // in ms
	msgsHandleOK   met.Count
	msgsHandleFail met.Count

	writeQueue *InProgressMessageQueue
)

type ESHandler struct {
}

func NewESHandler() (*ESHandler, error) {
	return &ESHandler{}, nil
}

func (k *ESHandler) HandleMessage(m *nsq.Message) error {
	log.Debug("received message.")
	format := "unknown"
	if m.Body[0] == '\x00' {
		format = "msgFormatJson"
	}

	var id int64
	buf := bytes.NewReader(m.Body[1:9])
	binary.Read(buf, binary.BigEndian, &id)
	produced := time.Unix(0, id)

	msgsAge.Value(time.Now().Sub(produced).Nanoseconds() / 1000)
	messagesSize.Value(int64(len(m.Body)))

	event := new(schema.ProbeEvent)
	if err := json.Unmarshal(m.Body[9:], &event); err != nil {
		log.Error(3, "ERROR: failure to unmarshal message body via format %s: %s. skipping message", format, err)
		return nil
	}

	// Since these messages are being batched, we'll need to hold onto this
	// and ack or requeue it on our own
	m.DisableAutoResponse()
	if event.Id == "" {
		// per http://blog.mikemccandless.com/2014/05/choosing-fast-unique-identifier-uuid.html,
		// using V1 UUIDs is much faster than v4 like we were using
		u := uuid.NewUUID()
		event.Id = u.String()
	}
	writeQueue.EnQueue(event.Id, m)

	if err := eventdef.Save(event); err != nil {
		log.Error(3, "couldn't process %s: %s", event.Id, err)
		msgsHandleFail.Inc(1)
		m.Requeue(-1)
		return err
	}

	return nil
}

type inProgressMessage struct {
	timestamp time.Time
	message   *nsq.Message
}

type InProgressMessageQueue struct {
	sync.RWMutex
	inProgress map[string]*inProgressMessage
	status     chan *eventdef.BulkSaveStatus
}

func (q *InProgressMessageQueue) EnQueue(id string, m *nsq.Message) {
	q.Lock()
	q.inProgress[id] = &inProgressMessage{
		timestamp: time.Now(),
		message:   m,
	}
	q.Unlock()
}

func (q *InProgressMessageQueue) loop() {
	for {
		select {
		case s := <-q.status:
			q.Lock()
			if m, ok := q.inProgress[s.Id]; ok {
				if s.Ok {
					if m.message != nil {
						m.message.Finish()
					}
					eventsToEsOK.Inc(1)
					msgsHandleOK.Inc(1)
					log.Debug("event %s commited to ES", s.Id)
				} else {
					if m.message != nil {
						m.message.Requeue(-1)
					}
					eventsToEsFail.Inc(1)
					msgsHandleFail.Inc(1)
					log.Error(3, "event %s failed to save, requeueing", s.Id)
				}
				esPutDuration.Value(time.Now().Sub(m.timestamp))
			} else {
				log.Error(3, "got processing response for unknown message. event %s", s.Id)
			}
			delete(q.inProgress, s.Id)
			q.Unlock()
		}
	}
}

func NewInProgressMessageQueue() *InProgressMessageQueue {
	q := &InProgressMessageQueue{
		inProgress: make(map[string]*inProgressMessage),
		status:     make(chan *eventdef.BulkSaveStatus, *maxInFlight),
	}
	for i := 0; i < *concurrency; i++ {
		go q.loop()
	}
	return q
}

func main() {
	flag.Parse()

	// Only try and parse the conf file if it exists
	if _, err := os.Stat(*confFile); err == nil {
		conf, err := globalconf.NewWithOptions(&globalconf.Options{Filename: *confFile})
		if err != nil {
			log.Fatal(4, err.Error())
		}
		conf.ParseAll()
	}

	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":true}`, *logLevel))

	if *showVersion {
		fmt.Println("nsq_probe_events_to_elasticsearch")
		return
	}

	if *channel == "" {
		rand.Seed(time.Now().UnixNano())
		*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
	}

	if *topic == "" {
		log.Fatal(4, "--topic is required")
	}

	if *nsqdTCPAddrs == "" && *lookupdHTTPAddrs == "" {
		log.Fatal(4, "--nsqd-tcp-address or --lookupd-http-address required")
	}
	if *nsqdTCPAddrs != "" && *lookupdHTTPAddrs != "" {
		log.Fatal(4, "use --nsqd-tcp-address or --lookupd-http-address not both")
	}

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(4, err.Error())
	}
	metrics, err := helper.New(true, *statsdAddr, *statsdType, "nsq_probe_events_to_elasticsearch", strings.Replace(hostname, ".", "_", -1))
	if err != nil {
		log.Fatal(4, err.Error())
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	messagesSize = metrics.NewMeter("message_size", 0)
	msgsAge = metrics.NewMeter("message_age", 0)
	eventsToEsOK = metrics.NewCount("events_to_es.ok")
	eventsToEsFail = metrics.NewCount("events_to_es.fail")
	esPutDuration = metrics.NewTimer("es_put_duration", 0)
	msgsHandleOK = metrics.NewCount("handle.ok")
	msgsHandleFail = metrics.NewCount("handle.fail")

	writeQueue = NewInProgressMessageQueue()

	err = eventdef.InitElasticsearch(*esAddr, "", "", writeQueue.status, *maxInFlight)
	if err != nil {
		log.Fatal(4, err.Error())
	}

	cfg := nsq.NewConfig()
	cfg.UserAgent = "nsq_probe_events_to_elasticsearch"
	err = app.ParseOpts(cfg, *consumerOpts)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	cfg.MaxInFlight = *maxInFlight

	consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", metrics)

	if err != nil {
		log.Fatal(4, err.Error())
	}

	handler, err := NewESHandler()
	if err != nil {
		log.Fatal(4, err.Error())
	}

	consumer.AddConcurrentHandlers(handler, *concurrency)

	nsqdAdds := strings.Split(*nsqdTCPAddrs, ",")
	if len(nsqdAdds) == 1 && nsqdAdds[0] == "" {
		nsqdAdds = []string{}
	}
	err = consumer.ConnectToNSQDs(nsqdAdds)
	if err != nil {
		log.Fatal(4, err.Error())
	}
	log.Info("connected to nsqd")

	lookupdAdds := strings.Split(*lookupdHTTPAddrs, ",")
	if len(lookupdAdds) == 1 && lookupdAdds[0] == "" {
		lookupdAdds = []string{}
	}
	err = consumer.ConnectToNSQLookupds(lookupdAdds)
	if err != nil {
		log.Fatal(4, err.Error())
	}

	go func() {
		log.Info("INFO starting listener for http/debug on %s", *listenAddr)
		httperr := http.ListenAndServe(*listenAddr, nil)
		if httperr != nil {
			log.Info(httperr.Error())
		}
	}()

	for {
		select {
		case <-consumer.StopChan:
			return
		case <-sigChan:
			consumer.Stop()
			eventdef.StopBulkIndexer()
		}
	}
}
<|endoftext|>