<commit_before>package dashboard_screen

import (
	"fmt"
	"math"
	"strconv"
	"strings"

	h "github.com/Cepave/open-falcon-backend/modules/f2e-api/app/helper"
	m "github.com/Cepave/open-falcon-backend/modules/f2e-api/app/model/dashboard"
	"github.com/gin-gonic/gin"
)

type APIScreenCreateInput struct {
	Pid  int64  `json:"pid" form:"pid"`
	Name string `json:"name" form:"name" binding:"required"`
}

func ScreenCreate(c *gin.Context) {
	inputs := APIScreenCreateInput{Pid: 0}
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	user, err := h.GetUser(c)
	if err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	newDS := m.DashboardScreen{
		PID:     inputs.Pid,
		Name:    inputs.Name,
		Creator: user.Name,
	}
	rcount := 0
	dt := db.Dashboard.Begin()
	dt = dt.Table(newDS.TableName()).Where("name = ?", newDS.Name).Count(&rcount)
	if rcount != 0 {
		h.JSONR(c, badstatus, fmt.Errorf("screen name '%s' alreay exist", newDS.Name))
		dt.Rollback()
		return
	}
	if inputs.Pid != 0 {
		dt = dt.Table(newDS.TableName()).Where("pid = ?", newDS.PID).Count(&rcount)
		if rcount == 0 {
			dt.Rollback()
			h.JSONR(c, badstatus, fmt.Errorf("parent id: %v, record not found", newDS.PID))
			return
		}
	}
	dt = dt.Save(&newDS)
	if dt.Error != nil {
		dt.Rollback()
		h.JSONR(c, badstatus, dt.Error)
		return
	}
	dt.Commit()
	h.JSONR(c, newDS)
}

func ScreenGet(c *gin.Context) {
	id := c.Param("screen_id")

	sid, err := strconv.Atoi(id)
	if err != nil {
		h.JSONR(c, badstatus, "invalid screen id")
		return
	}

	screen := m.DashboardScreen{}
	dt := db.Dashboard.Table("dashboard_screen").Where("id = ?", sid).First(&screen)
	if dt.Error != nil {
		h.JSONR(c, badstatus, dt.Error)
		return
	}

	graphsTmp := []m.DashboardGraph{}
	db.Dashboard.Model(&graphsTmp).Where("screen_id = ?", screen.ID).Scan(&graphsTmp)
	graphs := []map[string]interface{}{}
	for _, graph := range graphsTmp {
		es := strings.Split(graph.Hosts, TMP_GRAPH_FILED_DELIMITER)
		cs := strings.Split(graph.Counters, TMP_GRAPH_FILED_DELIMITER)
		graphs = append(graphs, map[string]interface{}{
			"graph_id":    graph.ID,
			"title":       graph.Title,
			"endpoints":   es,
			"counters":    cs,
			"screen_id":   graph.ScreenId,
			"graph_type":  graph.GraphType,
			"timespan":    graph.TimeSpan,
			"method":      graph.Method,
			"position":    graph.Position,
			"falcon_tags": graph.FalconTags,
		})
	}
	h.JSONR(c, map[string]interface{}{
		"scren":  screen,
		"graphs": graphs,
	})
}

func ScreenGetsByPid(c *gin.Context) {
	id := c.Param("pid")

	pid, err := strconv.Atoi(id)
	if err != nil {
		h.JSONR(c, badstatus, "invalid screen pid")
		return
	}

	screens := []m.DashboardScreen{}
	dt := db.Dashboard.Table("dashboard_screen").Where("pid = ?", pid).Find(&screens)
	if dt.Error != nil {
		h.JSONR(c, badstatus, dt.Error)
		return
	}

	h.JSONR(c, screens)
}

type APIScreenGetsAllInputs struct {
	Limit   int    `json:"limit" form:"limit"`
	Page    int    `json:"page" form:"page"`
	KeyWord string `json:"key_word" form:"key_word"`
	Order   bool   `json:"desc" form:"desc"`
}

type APIScreenCreateOutput struct {
	GraphNames []string `json:"graph_names"`
	m.DashboardScreen
}

func ScreenGetsAll(c *gin.Context) {
	inputs := APIScreenGetsAllInputs{Limit: 500, Page: -1, Order: false}
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	screens := []m.DashboardScreen{}
	totallCount := 0
	dt := db.Dashboard.Model(&screens)
	if inputs.KeyWord != "" {
		dt = dt.Where("name like ?", "%"+inputs.KeyWord+"%")
	}
	if inputs.Page <= 0 {
		if inputs.Order {
			dt = dt.Order("id desc")
		} else {
			dt = dt.Order("id asc")
		}
		dt = dt.Limit(inputs.Limit).Find(&screens)
		if dt.Error != nil {
			h.JSONR(c, badstatus, dt.Error)
			return
		} else {
			outputs := builtAllScreenOuput(screens)
			h.JSONR(c, outputs)
			return
		}
	} else {
		dt.Count(&totallCount)
		if inputs.Order {
			dt = dt.Order("id desc")
		} else {
			dt = dt.Order("id asc")
		}
		dt.Offset(inputs.Limit * (inputs.Page - 1)).Limit(inputs.Limit).Find(&screens)
		if dt.Error != nil {
			h.JSONR(c, badstatus, dt.Error)
			return
		} else {
			outputs := builtAllScreenOuput(screens)
			h.JSONR(c,
				map[string]interface{}{
					"current_page": inputs.Page,
					"totall_count": totallCount,
					"totall_page":  math.Ceil(float64(totallCount) / float64(inputs.Limit)),
					"data":         outputs,
					"order_by":     "id",
					"desc_order":   inputs.Order,
					"key_word":     inputs.KeyWord,
				})
			return
		}
	}
}

func ScreenDelete(c *gin.Context) {
	id := c.Param("screen_id")

	sid, err := strconv.Atoi(id)
	if err != nil {
		h.JSONR(c, badstatus, "invalid screen id")
		return
	}

	tx := db.Dashboard.Begin()
	screen := m.DashboardScreen{ID: int64(sid)}
	if dt := tx.Model(&screen).Where(&screen).Delete(&screen); dt.Error != nil {
		tx.Rollback()
		h.JSONR(c, badstatus, dt.Error)
		return
	}
	graph := m.DashboardGraph{ScreenId: int64(sid)}
	graphsID := []int64{}
	tx.Model(&graph).Where(&graph).Pluck("id", &graphsID)
	dt := tx.Model(&graph).Where(&graph).Delete(&graph)
	if dt := tx.Model(&graph).Where(&graph).Delete(&graph); dt.Error != nil {
		tx.Rollback()
		h.JSONR(c, badstatus, dt.Error)
		return
	}
	tx.Commit()
	h.JSONR(c, map[string]interface{}{
		"message":           "ok",
		"deleted_rows":      dt.RowsAffected,
		"deleted_graph_ids": graphsID,
	})
}

type APIScreenUpdateInputs struct {
	ID   int64  `json:"id" form:"id" binding:"required"`
	Name string `json:"name" form:"name" binding:"required"`
}

// For rename usage
func ScreenUpdate(c *gin.Context) {
	inputs := APIScreenUpdateInputs{}
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	if inputs.Name == "" {
		h.JSONR(c, badstatus, "name can not be empty")
		return
	}

	newData := m.DashboardScreen{
		ID:   inputs.ID,
		Name: inputs.Name,
	}

	if newData.ExistName() {
		h.JSONR(c, badstatus, fmt.Errorf("screen name '%s' alreay exist", newData.Name))
		return
	}
	dt := db.Dashboard.Table(newData.TableName()).Where("id = ?", inputs.ID).Update(newData)
	if dt.Error != nil {
		h.JSONR(c, badstatus, dt.Error)
		return
	}

	h.JSONR(c, "ok")
}

// For clone screen by id
type APIScreenCloneInputs struct {
	ID int64 `json:"id" form:"id" binding:"required"`
}

func ScreenClone(c *gin.Context) {
	inputs := APIScreenCloneInputs{}
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	user, err := h.GetUser(c)
	if err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	originalScreen := m.DashboardScreen{ID: inputs.ID}
	if dt := db.Dashboard.Model(&originalScreen).Where(&originalScreen).Scan(&originalScreen); dt.Error != nil {
		h.JSONR(c, badstatus, fmt.Errorf("find screen by id:%d, got error:%s", inputs.ID, dt.Error.Error()))
		return
	}
	tx := db.Dashboard.Begin()
	newScreen := m.DashboardScreen{
		Name:    fmt.Sprintf("%s_copy", originalScreen.Name),
		Creator: user.Name,
	}
	if dt := tx.Model(&newScreen).Save(&newScreen); dt.Error != nil {
		tx.Rollback()
		h.JSONR(c, badstatus, fmt.Errorf("create new screen got error:%s", dt.Error.Error()))
		return
	}
	originalGList := originalScreen.Graphs()
	graphNames := make([]string, len(originalGList))
	for indx, gh := range originalGList {
		newGraph := m.DashboardGraph{
			Title:      gh.Title,
			Hosts:      gh.Hosts,
			Counters:   gh.Counters,
			ScreenId:   newScreen.ID,
			TimeSpan:   gh.TimeSpan,
			GraphType:  gh.GraphType,
			Method:     gh.Method,
			Position:   gh.Position,
			FalconTags: gh.FalconTags,
			Creator:    user.Name,
		}
		if dt := tx.Model(&newGraph).Save(&newGraph); dt.Error != nil {
			tx.Rollback()
			h.JSONR(c, badstatus, fmt.Errorf("create new graph with graph id: %d ,got error:%s", gh.ID, dt.Error.Error()))
			return
		}
		graphNames[indx] = gh.Title
	}
	tx.Commit()
	h.JSONR(c, APIScreenCreateOutput{
		DashboardScreen: newScreen,
		GraphNames:      graphNames,
	})
}

func builtAllScreenOuput(screens []m.DashboardScreen) []APIScreenCreateOutput {
	outputs := make([]APIScreenCreateOutput, len(screens))
	for indx, s := range screens {
		outputs[indx].DashboardScreen = s
		graphs := s.Graphs()
		if len(graphs) != 0 {
			gnames := make([]string, len(graphs))
			for indx, graphs := range graphs {
				gnames[indx] = graphs.Title
			}
			outputs[indx].GraphNames = gnames
		} else {
			outputs[indx].GraphNames = []string{}
		}
	}
	return outputs
}
<commit_msg>increase reliability of screen clone api<commit_after>package dashboard_screen

import (
	"fmt"
	"math"
	"strconv"
	"strings"
	"time"

	h "github.com/Cepave/open-falcon-backend/modules/f2e-api/app/helper"
	m "github.com/Cepave/open-falcon-backend/modules/f2e-api/app/model/dashboard"
	"github.com/gin-gonic/gin"
)

type APIScreenCreateInput struct {
	Pid  int64  `json:"pid" form:"pid"`
	Name string `json:"name" form:"name" binding:"required"`
}

func ScreenCreate(c *gin.Context) {
	inputs := APIScreenCreateInput{Pid: 0}
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	user, err := h.GetUser(c)
	if err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	newDS := m.DashboardScreen{
		PID:     inputs.Pid,
		Name:    inputs.Name,
		Creator: user.Name,
	}
	rcount := 0
	dt := db.Dashboard.Begin()
	dt = dt.Table(newDS.TableName()).Where("name = ?", newDS.Name).Count(&rcount)
	if rcount != 0 {
		h.JSONR(c, badstatus, fmt.Errorf("screen name '%s' alreay exist", newDS.Name))
		dt.Rollback()
		return
	}
	if inputs.Pid != 0 {
		dt = dt.Table(newDS.TableName()).Where("pid = ?", newDS.PID).Count(&rcount)
		if rcount == 0 {
			dt.Rollback()
			h.JSONR(c, badstatus, fmt.Errorf("parent id: %v, record not found", newDS.PID))
			return
		}
	}
	dt = dt.Save(&newDS)
	if dt.Error != nil {
		dt.Rollback()
		h.JSONR(c, badstatus, dt.Error)
		return
	}
	dt.Commit()
	h.JSONR(c, newDS)
}

func ScreenGet(c *gin.Context) {
	id := c.Param("screen_id")

	sid, err := strconv.Atoi(id)
	if err != nil {
		h.JSONR(c, badstatus, "invalid screen id")
		return
	}

	screen := m.DashboardScreen{}
	dt := db.Dashboard.Table("dashboard_screen").Where("id = ?", sid).First(&screen)
	if dt.Error != nil {
		h.JSONR(c, badstatus, dt.Error)
		return
	}

	graphsTmp := []m.DashboardGraph{}
	db.Dashboard.Model(&graphsTmp).Where("screen_id = ?", screen.ID).Scan(&graphsTmp)
	graphs := []map[string]interface{}{}
	for _, graph := range graphsTmp {
		es := strings.Split(graph.Hosts, TMP_GRAPH_FILED_DELIMITER)
		cs := strings.Split(graph.Counters, TMP_GRAPH_FILED_DELIMITER)
		graphs = append(graphs, map[string]interface{}{
			"graph_id":    graph.ID,
			"title":       graph.Title,
			"endpoints":   es,
			"counters":    cs,
			"screen_id":   graph.ScreenId,
			"graph_type":  graph.GraphType,
			"timespan":    graph.TimeSpan,
			"method":      graph.Method,
			"position":    graph.Position,
			"falcon_tags": graph.FalconTags,
		})
	}
	h.JSONR(c, map[string]interface{}{
		"scren":  screen,
		"graphs": graphs,
	})
}

func ScreenGetsByPid(c *gin.Context) {
	id := c.Param("pid")

	pid, err := strconv.Atoi(id)
	if err != nil {
		h.JSONR(c, badstatus, "invalid screen pid")
		return
	}

	screens := []m.DashboardScreen{}
	dt := db.Dashboard.Table("dashboard_screen").Where("pid = ?", pid).Find(&screens)
	if dt.Error != nil {
		h.JSONR(c, badstatus, dt.Error)
		return
	}

	h.JSONR(c, screens)
}

type APIScreenGetsAllInputs struct {
	Limit   int    `json:"limit" form:"limit"`
	Page    int    `json:"page" form:"page"`
	KeyWord string `json:"key_word" form:"key_word"`
	Order   bool   `json:"desc" form:"desc"`
}

type APIScreenCreateOutput struct {
	GraphNames []string `json:"graph_names"`
	m.DashboardScreen
}

func ScreenGetsAll(c *gin.Context) {
	inputs := APIScreenGetsAllInputs{Limit: 500, Page: -1, Order: false}
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	screens := []m.DashboardScreen{}
	totallCount := 0
	dt := db.Dashboard.Model(&screens)
	if inputs.KeyWord != "" {
		dt = dt.Where("name like ?", "%"+inputs.KeyWord+"%")
	}
	if inputs.Page <= 0 {
		if inputs.Order {
			dt = dt.Order("id desc")
		} else {
			dt = dt.Order("id asc")
		}
		dt = dt.Limit(inputs.Limit).Find(&screens)
		if dt.Error != nil {
			h.JSONR(c, badstatus, dt.Error)
			return
		} else {
			outputs := builtAllScreenOuput(screens)
			h.JSONR(c, outputs)
			return
		}
	} else {
		dt.Count(&totallCount)
		if inputs.Order {
			dt = dt.Order("id desc")
		} else {
			dt = dt.Order("id asc")
		}
		dt.Offset(inputs.Limit * (inputs.Page - 1)).Limit(inputs.Limit).Find(&screens)
		if dt.Error != nil {
			h.JSONR(c, badstatus, dt.Error)
			return
		} else {
			outputs := builtAllScreenOuput(screens)
			h.JSONR(c,
				map[string]interface{}{
					"current_page": inputs.Page,
					"totall_count": totallCount,
					"totall_page":  math.Ceil(float64(totallCount) / float64(inputs.Limit)),
					"data":         outputs,
					"order_by":     "id",
					"desc_order":   inputs.Order,
					"key_word":     inputs.KeyWord,
				})
			return
		}
	}
}

func ScreenDelete(c *gin.Context) {
	id := c.Param("screen_id")

	sid, err := strconv.Atoi(id)
	if err != nil {
		h.JSONR(c, badstatus, "invalid screen id")
		return
	}

	tx := db.Dashboard.Begin()
	screen := m.DashboardScreen{ID: int64(sid)}
	if dt := tx.Model(&screen).Where(&screen).Delete(&screen); dt.Error != nil {
		tx.Rollback()
		h.JSONR(c, badstatus, dt.Error)
		return
	}
	graph := m.DashboardGraph{ScreenId: int64(sid)}
	graphsID := []int64{}
	tx.Model(&graph).Where(&graph).Pluck("id", &graphsID)
	dt := tx.Model(&graph).Where(&graph).Delete(&graph)
	if dt := tx.Model(&graph).Where(&graph).Delete(&graph); dt.Error != nil {
		tx.Rollback()
		h.JSONR(c, badstatus, dt.Error)
		return
	}
	tx.Commit()
	h.JSONR(c, map[string]interface{}{
		"message":           "ok",
		"deleted_rows":      dt.RowsAffected,
		"deleted_graph_ids": graphsID,
	})
}

type APIScreenUpdateInputs struct {
	ID   int64  `json:"id" form:"id" binding:"required"`
	Name string `json:"name" form:"name" binding:"required"`
}

// For rename usage
func ScreenUpdate(c *gin.Context) {
	inputs := APIScreenUpdateInputs{}
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	if inputs.Name == "" {
		h.JSONR(c, badstatus, "name can not be empty")
		return
	}

	newData := m.DashboardScreen{
		ID:   inputs.ID,
		Name: inputs.Name,
	}

	if newData.ExistName() {
		h.JSONR(c, badstatus, fmt.Errorf("screen name '%s' alreay exist", newData.Name))
		return
	}
	dt := db.Dashboard.Table(newData.TableName()).Where("id = ?", inputs.ID).Update(newData)
	if dt.Error != nil {
		h.JSONR(c, badstatus, dt.Error)
		return
	}

	h.JSONR(c, "ok")
}

// For clone screen by id
type APIScreenCloneInputs struct {
	ID int64 `json:"id" form:"id" binding:"required"`
}

func ScreenClone(c *gin.Context) {
	inputs := APIScreenCloneInputs{}
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	user, err := h.GetUser(c)
	if err != nil {
		h.JSONR(c, badstatus, err)
		return
	}
	originalScreen := m.DashboardScreen{ID: inputs.ID}
	if dt := db.Dashboard.Model(&originalScreen).Where(&originalScreen).Scan(&originalScreen); dt.Error != nil {
		h.JSONR(c, badstatus, fmt.Errorf("find screen by id:%d, got error:%s", inputs.ID, dt.Error.Error()))
		return
	}
	tx := db.Dashboard.Begin()
	newScreen := m.DashboardScreen{
		Name:    fmt.Sprintf("%s_copy", originalScreen.Name),
		Creator: user.Name,
	}
	if newScreen.ExistName() {
		newScreen.Name = fmt.Sprintf("%s_%s", newScreen.Name, time.Now().Unix())
	}
	if dt := tx.Model(&newScreen).Save(&newScreen); dt.Error != nil {
		tx.Rollback()
		h.JSONR(c, badstatus, fmt.Errorf("create new screen got error:%s", dt.Error.Error()))
		return
	}
	originalGList := originalScreen.Graphs()
	graphNames := make([]string, len(originalGList))
	for indx, gh := range originalGList {
		newGraph := m.DashboardGraph{
			Title:      gh.Title,
			Hosts:      gh.Hosts,
			Counters:   gh.Counters,
			ScreenId:   newScreen.ID,
			TimeSpan:   gh.TimeSpan,
			GraphType:  gh.GraphType,
			Method:     gh.Method,
			Position:   gh.Position,
			FalconTags: gh.FalconTags,
			Creator:    user.Name,
		}
		if dt := tx.Model(&newGraph).Save(&newGraph); dt.Error != nil {
			tx.Rollback()
			h.JSONR(c, badstatus, fmt.Errorf("create new graph with graph id: %d ,got error:%s", gh.ID, dt.Error.Error()))
			return
		}
		graphNames[indx] = gh.Title
	}
	tx.Commit()
	h.JSONR(c, APIScreenCreateOutput{
		DashboardScreen: newScreen,
		GraphNames:      graphNames,
	})
}

func builtAllScreenOuput(screens []m.DashboardScreen) []APIScreenCreateOutput {
	outputs := make([]APIScreenCreateOutput, len(screens))
	for indx, s := range screens {
		outputs[indx].DashboardScreen = s
		graphs := s.Graphs()
		if len(graphs) != 0 {
			gnames := make([]string, len(graphs))
			for indx, graphs := range graphs {
				gnames[indx] = graphs.Title
			}
			outputs[indx].GraphNames = gnames
		} else {
			outputs[indx].GraphNames = []string{}
		}
	}
	return outputs
}
<|endoftext|>


<commit_before>package render

import (
	"strings"
	"bufio"
	"io"
	"github.com/aymerick/raymond"
	"gopkg.in/russross/blackfriday.v2"
	"github.com/PuerkitoBio/goquery"
	"errors"
	"fmt"
	"regexp"
	"github.com/gogank/papillon/mapper"
	"net/url"
)

type renderer struct {
}

func New() *renderer {
	return &renderer{}
}

// DoRender 进行渲染，将markdown 原始内容 和模板传入，并进行渲染
// 如果 raw = nil, 则仅进行 tpl + user_ctx 渲染
// 如果 raw != nil 则先对raw 进行markdown解析，然后把解析结果 user_ctx["content"] = parsedHtml
func (render *renderer) DoRender(raw []byte, tpl []byte, user_ctx map[string]interface{}) (map[string]interface{}, []byte, error) {
	inner_ctx := make(map[string]interface{})
	if raw != nil {
		post_ctx, raw_content, err := readPostConfig(raw)
		if err != nil {
			return nil, nil, err
		}
		html_content := blackfriday.Run(raw_content)
		//TODO 临时方案
		inner_ctx["content"] = string(html_content)
		for key, value := range post_ctx {
			inner_ctx[key] = value
		}
	}

	if user_ctx != nil {
		for key, value := range user_ctx {
			inner_ctx[key] = value
		}
	}

	result_s, err := raymond.Render(string(tpl), inner_ctx)
	if err != nil {
		return nil, nil, err
	}
	result := []byte(result_s)
	return inner_ctx, result, nil
}

//readPostConfig 读取文章，过滤文章元信息，并返回需要进行markdown parse的内容
func readPostConfig(raw []byte) (map[string]string, []byte, error) {
	sr := strings.NewReader(string(raw))
	buf := bufio.NewReaderSize(sr, 4096)

	content := make([]byte, 0)
	postConf := make(map[string]string)

	str_flag := false
	end_flag := false

	for line, isPrefix, err := []byte{0}, false, error(nil); len(line) >= 0 && err == nil; {
		line, isPrefix, err = buf.ReadLine()
		// 如果 isPrefix 是true的话，则需要再读一行，然后继续处理
		for isPrefix {
			var newLine []byte
			newLine, isPrefix,err = buf.ReadLine()
			if err != nil && err != io.EOF{
				return nil,nil,err
			}
			line = append(line,newLine...)
		}
		if err != io.EOF && err != nil {
			return nil, nil, err
		}
		// 开始读取第一行
		if !str_flag && ! end_flag {
			line_str := strings.TrimSpace(string(line))
			// 防止没有 `---`
			if !strings.HasPrefix(line_str, "-") {
				continue
			}
			dash_line := string(line_str[0:3])
			if dash_line == "---" {
				str_flag = true
			}
			// 判断是否是配置区域
		} else if str_flag && !end_flag {
			line_str := strings.TrimSpace(string(line))
			dash_line := string(line_str[0:3])
			if dash_line != "---" && strings.Contains(line_str, ":") {
				tmp := strings.Split(string(line), ":")
				key := strings.TrimSpace(tmp[0])
				value := strings.TrimSpace(tmp[1])
				postConf[key] = value
			} else if dash_line == "---" {
				end_flag = true
			}
		} else if str_flag && end_flag {
			content = append(content, line...)
			// 需要加上换行符，否则markdown会解析错误
			content = append(content, byte('\n'))
		}
	}

	if len(content) == 0 {
		return nil, nil, errors.New("post has no configuration part")
	}

	return postConf, content, nil
}

//GetMeta 取得文章元信息
//返回文章元信息
func GetMeta(raw []byte) (map[string]string, error) {
	meta, _, err := readPostConfig(raw)
	return meta, err
}

// 将Html内容中的所有内部链接进行转换
// 0. 如果是内部链接则继续
// 1. 如果能够在ipfs网络中找到，则替换
// 2. 如果不能在ipfs网络中找到，则不处理
func (render *renderer)ConvertLink(raw []byte, parent_dir string) ([]byte, error) {
	// upload files
	_,err := mapper.WalkDir(parent_dir)
	if err != nil{
		return nil,err
	}

	sr := strings.NewReader(string(raw))
	doc, err := goquery.NewDocumentFromReader(sr)
	if err != nil {
		return nil, err
	}
	tagList := map[string]func(int, *goquery.Selection){}
	tagList["link"] = changeHref
	tagList["a"] = changeHref
	tagList["script"] = changeSrc
	tagList["img"] = changeSrc
	for k,v := range tagList{
		doc.Find(k).Each(v)
	}
	html, err := doc.Html()
	if err != nil {
		return nil, err
	}

	return []byte(html), nil
}

// changeSrc 将img/script中的 src属性 进行替换
func changeSrc(i int, s *goquery.Selection) {
	if src, ok := s.Attr("src"); ok && isInternal(src) {
		fmt.Println("%%%%%%%%%%%%%%%%%%%%%", parseLink(src))
		if ipfs_link, ok := mapper.Get(parseLink(src)); ok {
				fmt.Println("convert ", parseLink(src), " to ", ipfs_link)
				s.SetAttr("src", addIPFSPrefix(ipfs_link))
		}
	}
}

// changeHref 将 link/a 中的 link href 属性进行替换
func changeHref(i int, s *goquery.Selection) {
	if src, ok := s.Attr("link"); ok && isInternal(src) {
		// 如果是内部链接，进行处理
		if ipfs_link, ok := mapper.Get(parseLink(src)); ok {
				s.SetAttr("link", addIPFSPrefix(ipfs_link))
		}
	}
}

//addIPFSPrefix 添加 IPFS 网络前缀
func addIPFSPrefix(hash string) string{
	return "https://ipfs.io/ipfs/" + hash
}

//解析link 返回能够查询的静态资源key
func parseLink(link string) string{
	if isInternal(link) {
			if isSlashEnd(link) {
				link = link + "index.html"
			}
			if isRelative(link){
				fmt.Println("relative " + link)
				if link[0] == '.' {
					link = link[1:]
				}
				link = string(append([]byte("/"),link...))
			}
	}
	// 排除 /verndors/script/main.js?v=2.1.3
	if strings.Contains(link,"?"){
		link = strings.Split(link,"?")[0]
	}
	return link
}

//isInternal 判断是否是内部链接
func isInternal(link string) bool {
	reg, err := regexp.Compile("(?i:^http).*")
	if err != nil {
		panic(err)
	}
	if reg.MatchString(link) {
		return false
	}
	return true
}

//isSlashEnd 判断是否由/结尾
func isSlashEnd(link string) bool {
	return strings.HasSuffix(link, "/")
}

//isRelative 判断是否是相对url
func isRelative(link string) bool{
	url,err := url.Parse(link)
	if err != nil{
		return false
	}
	return !url.IsAbs() && !isSlashStart(link)
}

//isSlashStart 是否由`/`开始
func isSlashStart(link string) bool {
	return strings.HasPrefix(link, "/")
}
<commit_msg>render: fix link into href<commit_after>package render

import (
	"strings"
	"bufio"
	"io"
	"github.com/aymerick/raymond"
	"gopkg.in/russross/blackfriday.v2"
	"github.com/PuerkitoBio/goquery"
	"errors"
	"fmt"
	"regexp"
	"github.com/gogank/papillon/mapper"
	"net/url"
)

type renderer struct {
}

func New() *renderer {
	return &renderer{}
}

// DoRender 进行渲染，将markdown 原始内容 和模板传入，并进行渲染
// 如果 raw = nil, 则仅进行 tpl + user_ctx 渲染
// 如果 raw != nil 则先对raw 进行markdown解析，然后把解析结果 user_ctx["content"] = parsedHtml
func (render *renderer) DoRender(raw []byte, tpl []byte, user_ctx map[string]interface{}) (map[string]interface{}, []byte, error) {
	inner_ctx := make(map[string]interface{})
	if raw != nil {
		post_ctx, raw_content, err := readPostConfig(raw)
		if err != nil {
			return nil, nil, err
		}
		html_content := blackfriday.Run(raw_content)
		//TODO 临时方案
		inner_ctx["content"] = string(html_content)
		for key, value := range post_ctx {
			inner_ctx[key] = value
		}
	}

	if user_ctx != nil {
		for key, value := range user_ctx {
			inner_ctx[key] = value
		}
	}

	result_s, err := raymond.Render(string(tpl), inner_ctx)
	if err != nil {
		return nil, nil, err
	}
	result := []byte(result_s)
	return inner_ctx, result, nil
}

//readPostConfig 读取文章，过滤文章元信息，并返回需要进行markdown parse的内容
func readPostConfig(raw []byte) (map[string]string, []byte, error) {
	sr := strings.NewReader(string(raw))
	buf := bufio.NewReaderSize(sr, 4096)

	content := make([]byte, 0)
	postConf := make(map[string]string)

	str_flag := false
	end_flag := false

	for line, isPrefix, err := []byte{0}, false, error(nil); len(line) >= 0 && err == nil; {
		line, isPrefix, err = buf.ReadLine()
		// 如果 isPrefix 是true的话，则需要再读一行，然后继续处理
		for isPrefix {
			var newLine []byte
			newLine, isPrefix,err = buf.ReadLine()
			if err != nil && err != io.EOF{
				return nil,nil,err
			}
			line = append(line,newLine...)
		}
		if err != io.EOF && err != nil {
			return nil, nil, err
		}
		// 开始读取第一行
		if !str_flag && ! end_flag {
			line_str := strings.TrimSpace(string(line))
			// 防止没有 `---`
			if !strings.HasPrefix(line_str, "-") {
				continue
			}
			dash_line := string(line_str[0:3])
			if dash_line == "---" {
				str_flag = true
			}
			// 判断是否是配置区域
		} else if str_flag && !end_flag {
			line_str := strings.TrimSpace(string(line))
			dash_line := string(line_str[0:3])
			if dash_line != "---" && strings.Contains(line_str, ":") {
				tmp := strings.Split(string(line), ":")
				key := strings.TrimSpace(tmp[0])
				value := strings.TrimSpace(tmp[1])
				postConf[key] = value
			} else if dash_line == "---" {
				end_flag = true
			}
		} else if str_flag && end_flag {
			content = append(content, line...)
			// 需要加上换行符，否则markdown会解析错误
			content = append(content, byte('\n'))
		}
	}

	if len(content) == 0 {
		return nil, nil, errors.New("post has no configuration part")
	}

	return postConf, content, nil
}

//GetMeta 取得文章元信息
//返回文章元信息
func GetMeta(raw []byte) (map[string]string, error) {
	meta, _, err := readPostConfig(raw)
	return meta, err
}

// 将Html内容中的所有内部链接进行转换
// 0. 如果是内部链接则继续
// 1. 如果能够在ipfs网络中找到，则替换
// 2. 如果不能在ipfs网络中找到，则不处理
func (render *renderer)ConvertLink(raw []byte, parent_dir string) ([]byte, error) {
	// upload files
	_,err := mapper.WalkDir(parent_dir)
	if err != nil{
		return nil,err
	}

	sr := strings.NewReader(string(raw))
	doc, err := goquery.NewDocumentFromReader(sr)
	if err != nil {
		return nil, err
	}
	tagList := map[string]func(int, *goquery.Selection){}
	tagList["link"] = changeHref
	tagList["a"] = changeHref
	tagList["script"] = changeSrc
	tagList["img"] = changeSrc
	for k,v := range tagList{
		doc.Find(k).Each(v)
	}
	html, err := doc.Html()
	if err != nil {
		return nil, err
	}

	return []byte(html), nil
}

// changeSrc 将img/script中的 src属性 进行替换
func changeSrc(i int, s *goquery.Selection) {
	if src, ok := s.Attr("src"); ok && isInternal(src) {
		fmt.Println("--->", addIPFSPrefix(parseLink(src)))
		if ipfs_link, ok := mapper.Get(parseLink(src)); ok {
				fmt.Println("convert ", parseLink(src), " to ", ipfs_link)
				s.SetAttr("src", addIPFSPrefix(ipfs_link))
		}
	}
}

// changeHref 将 link/a 中的 link href 属性进行替换
func changeHref(i int, s *goquery.Selection) {
	if src, ok := s.Attr("href"); ok && isInternal(src) {
		// 如果是内部链接，进行处理
		if ipfs_link, ok := mapper.Get(parseLink(src)); ok {
				s.SetAttr("href", addIPFSPrefix(ipfs_link))
		}
	}
}

//addIPFSPrefix 添加 IPFS 网络前缀
func addIPFSPrefix(hash string) string{
	return "https://ipfs.io/ipfs/" + hash
}

//解析link 返回能够查询的静态资源key
func parseLink(link string) string{
	if isInternal(link) {
			if isSlashEnd(link) {
				link = link + "index.html"
			}
			if isRelative(link){
				fmt.Println("relative " + link)
				if link[0] == '.' {
					link = link[1:]
				}
				link = string(append([]byte("/"),link...))
			}
	}
	// 排除 /verndors/script/main.js?v=2.1.3
	if strings.Contains(link,"?"){
		link = strings.Split(link,"?")[0]
	}
	return link
}

//isInternal 判断是否是内部链接
func isInternal(link string) bool {
	reg, err := regexp.Compile("(?i:^http).*")
	if err != nil {
		panic(err)
	}
	if reg.MatchString(link) {
		return false
	}
	return true
}

//isSlashEnd 判断是否由/结尾
func isSlashEnd(link string) bool {
	return strings.HasSuffix(link, "/")
}

//isRelative 判断是否是相对url
func isRelative(link string) bool{
	url,err := url.Parse(link)
	if err != nil{
		return false
	}
	return !url.IsAbs() && !isSlashStart(link)
}

//isSlashStart 是否由`/`开始
func isSlashStart(link string) bool {
	return strings.HasPrefix(link, "/")
}
<|endoftext|>


<commit_before>// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.

package main

import (
	"flag"
	"os"
	"strings"

	"github.com/couchbase/cbauth"
	"github.com/couchbase/indexing/secondary/common"
	"github.com/couchbase/indexing/secondary/fdb"
	"github.com/couchbase/indexing/secondary/indexer"
	"github.com/couchbase/indexing/secondary/logging"
	"github.com/couchbase/indexing/secondary/platform"
)

var (
	logLevel          = flag.String("loglevel", "Info", "Log Level - Silent, Fatal, Error, Info, Debug, Trace")
	numVbuckets       = flag.Int("vbuckets", indexer.MAX_NUM_VBUCKETS, "Number of vbuckets configured in Couchbase")
	cluster           = flag.String("cluster", indexer.DEFAULT_CLUSTER_ENDPOINT, "Couchbase cluster address")
	adminPort         = flag.String("adminPort", "9100", "Index ddl and status port")
	scanPort          = flag.String("scanPort", "9101", "Index scanner port")
	httpPort          = flag.String("httpPort", "9102", "Index http mgmt port")
	streamInitPort    = flag.String("streamInitPort", "9103", "Index initial stream port")
	streamCatchupPort = flag.String("streamCatchupPort", "9104", "Index catchup stream port")
	streamMaintPort   = flag.String("streamMaintPort", "9105", "Index maintenance stream port")
	storageDir        = flag.String("storageDir", "./", "Index file storage directory path")
	enableManager     = flag.Bool("enable_manager", true, "Enable Index Manager")
	auth              = flag.String("auth", "", "Auth user and password")

	// so we don't need to sync with ns_server for merge. remove this soon
	unused = flag.String("log", "", "Ignored")
)

func main() {
	platform.HideConsole(true)
	defer platform.HideConsole(false)
	common.SeedProcess()

	logging.Infof("Indexer started with command line: %v\n", os.Args)
	flag.Parse()

	logging.SetLogLevel(logging.Level(*logLevel))
	forestdb.Log = &logging.SystemLogger

	// setup cbauth
	if *auth != "" {
		up := strings.Split(*auth, ":")
		logging.Tracef("Initializing cbauth with user %v for cluster %v\n", up[0], *cluster)
		if _, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1]); err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s", err)
		}
	}

	go platform.DumpOnSignal()
	go common.ExitOnStdinClose()

	config := common.SystemConfig
	config.SetValue("indexer.clusterAddr", *cluster)
	config.SetValue("indexer.numVbuckets", *numVbuckets)
	config.SetValue("indexer.enableManager", *enableManager)
	config.SetValue("indexer.adminPort", *adminPort)
	config.SetValue("indexer.scanPort", *scanPort)
	config.SetValue("indexer.httpPort", *httpPort)
	config.SetValue("indexer.streamInitPort", *streamInitPort)
	config.SetValue("indexer.streamCatchupPort", *streamCatchupPort)
	config.SetValue("indexer.streamMaintPort", *streamMaintPort)
	config.SetValue("indexer.storage_dir", *storageDir)

	_, msg := indexer.NewIndexer(config)

	if msg.GetMsgType() != indexer.MSG_SUCCESS {
		logging.Warnf("Indexer Failure to Init %v", msg)
	}

	logging.Infof("Indexer exiting normally\n")
}
<commit_msg>MB-15130: create index-path if not already created.<commit_after>// Copyright (c) 2014 Couchbase, Inc.
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
// except in compliance with the License. You may obtain a copy of the License at
//   http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software distributed under the
// License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
// either express or implied. See the License for the specific language governing permissions
// and limitations under the License.

package main

import (
	"flag"
	"os"
	"strings"

	"github.com/couchbase/cbauth"
	"github.com/couchbase/indexing/secondary/common"
	"github.com/couchbase/indexing/secondary/fdb"
	"github.com/couchbase/indexing/secondary/indexer"
	"github.com/couchbase/indexing/secondary/logging"
	"github.com/couchbase/indexing/secondary/platform"
)

var (
	logLevel          = flag.String("loglevel", "Info", "Log Level - Silent, Fatal, Error, Info, Debug, Trace")
	numVbuckets       = flag.Int("vbuckets", indexer.MAX_NUM_VBUCKETS, "Number of vbuckets configured in Couchbase")
	cluster           = flag.String("cluster", indexer.DEFAULT_CLUSTER_ENDPOINT, "Couchbase cluster address")
	adminPort         = flag.String("adminPort", "9100", "Index ddl and status port")
	scanPort          = flag.String("scanPort", "9101", "Index scanner port")
	httpPort          = flag.String("httpPort", "9102", "Index http mgmt port")
	streamInitPort    = flag.String("streamInitPort", "9103", "Index initial stream port")
	streamCatchupPort = flag.String("streamCatchupPort", "9104", "Index catchup stream port")
	streamMaintPort   = flag.String("streamMaintPort", "9105", "Index maintenance stream port")
	storageDir        = flag.String("storageDir", "./", "Index file storage directory path")
	enableManager     = flag.Bool("enable_manager", true, "Enable Index Manager")
	auth              = flag.String("auth", "", "Auth user and password")

	// so we don't need to sync with ns_server for merge. remove this soon
	unused = flag.String("log", "", "Ignored")
)

func main() {
	platform.HideConsole(true)
	defer platform.HideConsole(false)
	common.SeedProcess()

	logging.Infof("Indexer started with command line: %v\n", os.Args)
	flag.Parse()

	logging.SetLogLevel(logging.Level(*logLevel))
	forestdb.Log = &logging.SystemLogger

	// setup cbauth
	if *auth != "" {
		up := strings.Split(*auth, ":")
		logging.Tracef("Initializing cbauth with user %v for cluster %v\n", up[0], *cluster)
		if _, err := cbauth.InternalRetryDefaultInit(*cluster, up[0], up[1]); err != nil {
			logging.Fatalf("Failed to initialize cbauth: %s", err)
		}
	}

	go platform.DumpOnSignal()
	go common.ExitOnStdinClose()

	config := common.SystemConfig
	config.SetValue("indexer.clusterAddr", *cluster)
	config.SetValue("indexer.numVbuckets", *numVbuckets)
	config.SetValue("indexer.enableManager", *enableManager)
	config.SetValue("indexer.adminPort", *adminPort)
	config.SetValue("indexer.scanPort", *scanPort)
	config.SetValue("indexer.httpPort", *httpPort)
	config.SetValue("indexer.streamInitPort", *streamInitPort)
	config.SetValue("indexer.streamCatchupPort", *streamCatchupPort)
	config.SetValue("indexer.streamMaintPort", *streamMaintPort)
	config.SetValue("indexer.storage_dir", *storageDir)

	storage_dir := config["indexer.storage_dir"].String()
	if err := os.MkdirAll(storage_dir, 0755); err != nil {
		common.CrashOnError(err)
	}

	_, msg := indexer.NewIndexer(config)

	if msg.GetMsgType() != indexer.MSG_SUCCESS {
		logging.Warnf("Indexer Failure to Init %v", msg)
	}

	logging.Infof("Indexer exiting normally\n")
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"reflect"
	"syscall"
	"time"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/security/access"
	"v.io/v23/services/groups"
	"v.io/v23/verror"
	"v.io/x/lib/set"
	"v.io/x/ref/lib/xrpc"
	"v.io/x/ref/services/groups/groupsd/testdata/kvstore"
	"v.io/x/ref/test/modules"
	"v.io/x/ref/test/v23tests"
)

//go:generate v23 test generate

type relateResult struct {
	Remainder      map[string]struct{}
	Approximations []groups.Approximation
	Version        string
}

// V23TestGroupServerIntegration tests the integration between the
// "groups" command-line client and the "groupsd" server.
func V23TestGroupServerIntegration(t *v23tests.T) {
	v23tests.RunRootMT(t, "--v23.tcp.address=127.0.0.1:0")

	// Build binaries for the client and server.
	var (
		clientBin  = t.BuildV23Pkg("v.io/x/ref/services/groups/groups")
		serverBin  = t.BuildV23Pkg("v.io/x/ref/services/groups/groupsd")
		serverName = "groups-server"
		groupA     = naming.Join(serverName, "groupA")
		groupB     = naming.Join(serverName, "groupB")
	)

	// Start the groups server.
	serverBin.Start("-name="+serverName, "-v23.tcp.address=127.0.0.1:0")

	// Create a couple of groups.
	clientBin.Start("create", groupA).WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("create", groupB, "a", "a/b").WaitOrDie(os.Stdout, os.Stderr)

	// Add a couple of blessing patterns.
	clientBin.Start("add", groupA, "<grp:groups-server/groupB>").WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("add", groupA, "a").WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("add", groupB, "a/b/c").WaitOrDie(os.Stdout, os.Stderr)

	// Remove a blessing pattern.
	clientBin.Start("remove", groupB, "a").WaitOrDie(os.Stdout, os.Stderr)

	// Test simple group resolution.
	{
		var buffer bytes.Buffer
		clientBin.Start("relate", groupB, "a/b/c/d").WaitOrDie(&buffer, &buffer)
		var got relateResult
		if err := json.Unmarshal(buffer.Bytes(), &got); err != nil {
			t.Fatalf("Unmarshal(%v) failed: %v", buffer.Bytes(), err)
		}
		want := relateResult{
			Remainder:      set.String.FromSlice([]string{"c/d", "d"}),
			Approximations: nil,
			Version:        "2",
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}

	// Test recursive group resolution.
	{
		var buffer bytes.Buffer
		clientBin.Start("relate", groupA, "a/b/c/d").WaitOrDie(&buffer, &buffer)
		var got relateResult
		if err := json.Unmarshal(buffer.Bytes(), &got); err != nil {
			t.Fatalf("Unmarshal(%v) failed: %v", buffer.Bytes(), err)
		}
		want := relateResult{
			Remainder:      set.String.FromSlice([]string{"b/c/d", "c/d", "d"}),
			Approximations: nil,
			Version:        "2",
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}

	// Test group resolution failure. Note that under-approximation is
	// used as the default to handle resolution failures.
	{
		clientBin.Start("add", groupB, "<grp:groups-server/groupC>").WaitOrDie(os.Stdout, os.Stderr)
		var buffer bytes.Buffer
		clientBin.Start("relate", groupB, "a/b/c/d").WaitOrDie(&buffer, &buffer)
		var got relateResult
		if err := json.Unmarshal(buffer.Bytes(), &got); err != nil {
			t.Fatalf("Unmarshal(%v) failed: %v", buffer.Bytes(), err)
		}
		want := relateResult{
			Remainder: set.String.FromSlice([]string{"c/d", "d"}),
			Approximations: []groups.Approximation{
				groups.Approximation{
					Reason:  "v.io/v23/verror.NoExist",
					Details: `groupsd:"groupC".Relate: Does not exist: groupC`,
				},
			},
			Version: "3",
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}

	// Delete the groups.
	clientBin.Start("delete", groupA).WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("delete", groupB).WaitOrDie(os.Stdout, os.Stderr)
}

// store implements the kvstore.Store interface.
type store map[string]string

func (s store) Get(ctx *context.T, call rpc.ServerCall, key string) (string, error) {
	return s[key], nil
}

func (s store) Set(ctx *context.T, call rpc.ServerCall, key string, value string) error {
	s[key] = value
	return nil
}

const (
	kvServerName = "key-value-store"
	getFailed    = "GET FAILED"
	getOK        = "GET OK"
	setFailed    = "SET FAILED"
	setOK        = "SET OK"
)

var runServer = modules.Register(func(env *modules.Env, args ...string) error {
	ctx, shutdown := v23.Init()
	defer shutdown()
	// Use a shorter timeout to reduce the test overall runtime as the
	// permission authorizer will attempt to connect to a non-existing
	// groups server at some point in the test.
	ctx, _ = context.WithTimeout(ctx, 2*time.Second)
	authorizer, err := groups.PermissionsAuthorizer(access.Permissions{
		"Read":  access.AccessList{In: []security.BlessingPattern{"<grp:groups-server/readers>"}},
		"Write": access.AccessList{In: []security.BlessingPattern{"<grp:groups-server/writers>"}},
	}, access.TypicalTagType())
	if err != nil {
		return err
	}
	if _, err := xrpc.NewServer(ctx, kvServerName, kvstore.StoreServer(&store{}), authorizer); err != nil {
		return err
	}
	modules.WaitForEOF(env.Stdin)
	return nil
}, "server")

var runClient = modules.Register(func(env *modules.Env, args ...string) error {
	ctx, shutdown := v23.Init()
	defer shutdown()

	if got, want := len(args), 1; got < want {
		return fmt.Errorf("unexpected number of arguments: got %v, want at least %v", got, want)
	}
	command := args[0]
	client := kvstore.StoreClient("key-value-store")
	switch command {
	case "get":
		if got, want := len(args), 2; got != want {
			return fmt.Errorf("unexpected number of arguments: got %v, want %v", got, want)
		}
		key := args[1]
		value, err := client.Get(ctx, key)
		if err != nil {
			fmt.Fprintf(env.Stdout, "%v %v\n", getFailed, verror.ErrorID(err))
		} else {
			fmt.Fprintf(env.Stdout, "%v %v\n", getOK, value)
		}
	case "set":
		if got, want := len(args), 3; got != want {
			return fmt.Errorf("unexpected number of arguments: got %v, want %v", got, want)
		}
		key, value := args[1], args[2]
		if err := client.Set(ctx, key, value); err != nil {
			fmt.Fprintf(env.Stdout, "%v %v\n", setFailed, verror.ErrorID(err))
		} else {
			fmt.Fprintf(env.Stdout, "%v\n", setOK)
		}
	}
	return nil
}, "client")

func startClient(t *v23tests.T, name string, args ...string) modules.Handle {
	creds, err := t.Shell().NewChildCredentials(name)
	if err != nil {
		t.Fatalf("NewChildCredentials(%v) failed: %v", name, err)
	}
	handle, err := t.Shell().StartWithOpts(
		t.Shell().DefaultStartOpts().WithCustomCredentials(creds).WithSessions(t, time.Minute),
		nil,
		runClient,
		args...,
	)
	if err != nil {
		t.Fatalf("StartWithOpts() failed: %v", err)
	}
	return handle
}

// V23TestGroupServerAuthorization uses an instance of the
// KeyValueStore server with an groups-based authorizer to test the
// group server implementation.
func V23TestGroupServerAuthorization(t *v23tests.T) {
	v23tests.RunRootMT(t, "--v23.tcp.address=127.0.0.1:0")

	// Build binaries for the groups client and server.
	var (
		clientBin  = t.BuildV23Pkg("v.io/x/ref/services/groups/groups")
		serverBin  = t.BuildV23Pkg("v.io/x/ref/services/groups/groupsd")
		serverName = "groups-server"
		readers    = naming.Join(serverName, "readers")
		writers    = naming.Join(serverName, "writers")
	)

	// Start the groups server.
	server := serverBin.Start("-name="+serverName, "-v23.tcp.address=127.0.0.1:0")

	// Create a couple of groups. The <readers> and <writers> groups
	// identify blessings that can be used to read from and write to the
	// key value store server respectively.
	clientBin.Start("create", readers, "root/alice", "root/bob").WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("create", writers, "root/alice").WaitOrDie(os.Stdout, os.Stderr)

	// Start an instance of the key value store server.
	if _, err := t.Shell().Start(nil, runServer); err != nil {
		t.Fatalf("Start() failed: %v", err)
	}

	// Test that alice can write.
	startClient(t, "alice", "set", "foo", "bar").Expect(setOK)
	// Test that alice can read.
	startClient(t, "alice", "get", "foo").Expectf("%v %v", getOK, "bar")
	// Test that bob can read.
	startClient(t, "bob", "get", "foo").Expectf("%v %v", getOK, "bar")
	// Test that bob cannot write.
	startClient(t, "bob", "set", "foo", "bar").Expectf("%v %v", setFailed, verror.ErrNoAccess.ID)
	// Stop the groups server and check that as a consequence "alice"
	// cannot read from the key value store server anymore.
	if err := server.Kill(syscall.SIGTERM); err != nil {
		t.Fatalf("Kill() failed: %v", err)
	}
	if err := server.Wait(os.Stdout, os.Stderr); err != nil {
		t.Fatalf("Wait() failed: %v", err)
	}
	startClient(t, "alice", "get", "foo").Expectf("%v %v", getFailed, verror.ErrNoAccess.ID)
}
<commit_msg>ref/services/groups: log error output using a string instead of bytes<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"reflect"
	"syscall"
	"time"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/naming"
	"v.io/v23/rpc"
	"v.io/v23/security"
	"v.io/v23/security/access"
	"v.io/v23/services/groups"
	"v.io/v23/verror"
	"v.io/x/lib/set"
	"v.io/x/ref/lib/xrpc"
	"v.io/x/ref/services/groups/groupsd/testdata/kvstore"
	"v.io/x/ref/test/modules"
	"v.io/x/ref/test/v23tests"
)

//go:generate v23 test generate

type relateResult struct {
	Remainder      map[string]struct{}
	Approximations []groups.Approximation
	Version        string
}

// V23TestGroupServerIntegration tests the integration between the
// "groups" command-line client and the "groupsd" server.
func V23TestGroupServerIntegration(t *v23tests.T) {
	v23tests.RunRootMT(t, "--v23.tcp.address=127.0.0.1:0")

	// Build binaries for the client and server.
	var (
		clientBin  = t.BuildV23Pkg("v.io/x/ref/services/groups/groups")
		serverBin  = t.BuildV23Pkg("v.io/x/ref/services/groups/groupsd")
		serverName = "groups-server"
		groupA     = naming.Join(serverName, "groupA")
		groupB     = naming.Join(serverName, "groupB")
	)

	// Start the groups server.
	serverBin.Start("-name="+serverName, "-v23.tcp.address=127.0.0.1:0")

	// Create a couple of groups.
	clientBin.Start("create", groupA).WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("create", groupB, "a", "a/b").WaitOrDie(os.Stdout, os.Stderr)

	// Add a couple of blessing patterns.
	clientBin.Start("add", groupA, "<grp:groups-server/groupB>").WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("add", groupA, "a").WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("add", groupB, "a/b/c").WaitOrDie(os.Stdout, os.Stderr)

	// Remove a blessing pattern.
	clientBin.Start("remove", groupB, "a").WaitOrDie(os.Stdout, os.Stderr)

	// Test simple group resolution.
	{
		var buffer bytes.Buffer
		clientBin.Start("relate", groupB, "a/b/c/d").WaitOrDie(&buffer, &buffer)
		var got relateResult
		if err := json.Unmarshal(buffer.Bytes(), &got); err != nil {
			t.Fatalf("Unmarshal(%v) failed: %v", buffer.String(), err)
		}
		want := relateResult{
			Remainder:      set.String.FromSlice([]string{"c/d", "d"}),
			Approximations: nil,
			Version:        "2",
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}

	// Test recursive group resolution.
	{
		var buffer bytes.Buffer
		clientBin.Start("relate", groupA, "a/b/c/d").WaitOrDie(&buffer, &buffer)
		var got relateResult
		if err := json.Unmarshal(buffer.Bytes(), &got); err != nil {
			t.Fatalf("Unmarshal(%v) failed: %v", buffer.String(), err)
		}
		want := relateResult{
			Remainder:      set.String.FromSlice([]string{"b/c/d", "c/d", "d"}),
			Approximations: nil,
			Version:        "2",
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}

	// Test group resolution failure. Note that under-approximation is
	// used as the default to handle resolution failures.
	{
		clientBin.Start("add", groupB, "<grp:groups-server/groupC>").WaitOrDie(os.Stdout, os.Stderr)
		var buffer bytes.Buffer
		clientBin.Start("relate", groupB, "a/b/c/d").WaitOrDie(&buffer, &buffer)
		var got relateResult
		if err := json.Unmarshal(buffer.Bytes(), &got); err != nil {
			t.Fatalf("Unmarshal(%v) failed: %v", buffer.String(), err)
		}
		want := relateResult{
			Remainder: set.String.FromSlice([]string{"c/d", "d"}),
			Approximations: []groups.Approximation{
				groups.Approximation{
					Reason:  "v.io/v23/verror.NoExist",
					Details: `groupsd:"groupC".Relate: Does not exist: groupC`,
				},
			},
			Version: "3",
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("got %v, want %v", got, want)
		}
	}

	// Delete the groups.
	clientBin.Start("delete", groupA).WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("delete", groupB).WaitOrDie(os.Stdout, os.Stderr)
}

// store implements the kvstore.Store interface.
type store map[string]string

func (s store) Get(ctx *context.T, call rpc.ServerCall, key string) (string, error) {
	return s[key], nil
}

func (s store) Set(ctx *context.T, call rpc.ServerCall, key string, value string) error {
	s[key] = value
	return nil
}

const (
	kvServerName = "key-value-store"
	getFailed    = "GET FAILED"
	getOK        = "GET OK"
	setFailed    = "SET FAILED"
	setOK        = "SET OK"
)

var runServer = modules.Register(func(env *modules.Env, args ...string) error {
	ctx, shutdown := v23.Init()
	defer shutdown()
	// Use a shorter timeout to reduce the test overall runtime as the
	// permission authorizer will attempt to connect to a non-existing
	// groups server at some point in the test.
	ctx, _ = context.WithTimeout(ctx, 2*time.Second)
	authorizer, err := groups.PermissionsAuthorizer(access.Permissions{
		"Read":  access.AccessList{In: []security.BlessingPattern{"<grp:groups-server/readers>"}},
		"Write": access.AccessList{In: []security.BlessingPattern{"<grp:groups-server/writers>"}},
	}, access.TypicalTagType())
	if err != nil {
		return err
	}
	if _, err := xrpc.NewServer(ctx, kvServerName, kvstore.StoreServer(&store{}), authorizer); err != nil {
		return err
	}
	modules.WaitForEOF(env.Stdin)
	return nil
}, "server")

var runClient = modules.Register(func(env *modules.Env, args ...string) error {
	ctx, shutdown := v23.Init()
	defer shutdown()

	if got, want := len(args), 1; got < want {
		return fmt.Errorf("unexpected number of arguments: got %v, want at least %v", got, want)
	}
	command := args[0]
	client := kvstore.StoreClient("key-value-store")
	switch command {
	case "get":
		if got, want := len(args), 2; got != want {
			return fmt.Errorf("unexpected number of arguments: got %v, want %v", got, want)
		}
		key := args[1]
		value, err := client.Get(ctx, key)
		if err != nil {
			fmt.Fprintf(env.Stdout, "%v %v\n", getFailed, verror.ErrorID(err))
		} else {
			fmt.Fprintf(env.Stdout, "%v %v\n", getOK, value)
		}
	case "set":
		if got, want := len(args), 3; got != want {
			return fmt.Errorf("unexpected number of arguments: got %v, want %v", got, want)
		}
		key, value := args[1], args[2]
		if err := client.Set(ctx, key, value); err != nil {
			fmt.Fprintf(env.Stdout, "%v %v\n", setFailed, verror.ErrorID(err))
		} else {
			fmt.Fprintf(env.Stdout, "%v\n", setOK)
		}
	}
	return nil
}, "client")

func startClient(t *v23tests.T, name string, args ...string) modules.Handle {
	creds, err := t.Shell().NewChildCredentials(name)
	if err != nil {
		t.Fatalf("NewChildCredentials(%v) failed: %v", name, err)
	}
	handle, err := t.Shell().StartWithOpts(
		t.Shell().DefaultStartOpts().WithCustomCredentials(creds).WithSessions(t, time.Minute),
		nil,
		runClient,
		args...,
	)
	if err != nil {
		t.Fatalf("StartWithOpts() failed: %v", err)
	}
	return handle
}

// V23TestGroupServerAuthorization uses an instance of the
// KeyValueStore server with an groups-based authorizer to test the
// group server implementation.
func V23TestGroupServerAuthorization(t *v23tests.T) {
	v23tests.RunRootMT(t, "--v23.tcp.address=127.0.0.1:0")

	// Build binaries for the groups client and server.
	var (
		clientBin  = t.BuildV23Pkg("v.io/x/ref/services/groups/groups")
		serverBin  = t.BuildV23Pkg("v.io/x/ref/services/groups/groupsd")
		serverName = "groups-server"
		readers    = naming.Join(serverName, "readers")
		writers    = naming.Join(serverName, "writers")
	)

	// Start the groups server.
	server := serverBin.Start("-name="+serverName, "-v23.tcp.address=127.0.0.1:0")

	// Create a couple of groups. The <readers> and <writers> groups
	// identify blessings that can be used to read from and write to the
	// key value store server respectively.
	clientBin.Start("create", readers, "root/alice", "root/bob").WaitOrDie(os.Stdout, os.Stderr)
	clientBin.Start("create", writers, "root/alice").WaitOrDie(os.Stdout, os.Stderr)

	// Start an instance of the key value store server.
	if _, err := t.Shell().Start(nil, runServer); err != nil {
		t.Fatalf("Start() failed: %v", err)
	}

	// Test that alice can write.
	startClient(t, "alice", "set", "foo", "bar").Expect(setOK)
	// Test that alice can read.
	startClient(t, "alice", "get", "foo").Expectf("%v %v", getOK, "bar")
	// Test that bob can read.
	startClient(t, "bob", "get", "foo").Expectf("%v %v", getOK, "bar")
	// Test that bob cannot write.
	startClient(t, "bob", "set", "foo", "bar").Expectf("%v %v", setFailed, verror.ErrNoAccess.ID)
	// Stop the groups server and check that as a consequence "alice"
	// cannot read from the key value store server anymore.
	if err := server.Kill(syscall.SIGTERM); err != nil {
		t.Fatalf("Kill() failed: %v", err)
	}
	if err := server.Wait(os.Stdout, os.Stderr); err != nil {
		t.Fatalf("Wait() failed: %v", err)
	}
	startClient(t, "alice", "get", "foo").Expectf("%v %v", getFailed, verror.ErrNoAccess.ID)
}
<|endoftext|>


<commit_before>package models

import (
	"sort"
	"strings"
)

type ScopeGroup struct {
	Scope string
	Group string
}

var characterScopes = []ScopeGroup{
	{"esi-assets.read_assets.v1", "assets"},

	{"esi-characters.read_contacts.v1", "contacts"},
	{"esi-characters.write_contacts.v1", "contacts"},

	{"esi-characters.read_notifications.v1", "notifications"},

	{"esi-universe.read_structures.v1", "market"},
	{"esi-search.search_structures.v1", "market"},
	{"esi-markets.structure_markets.v1", "market"},

	{"esi-ui.open_window.v1", "ui-control"},
	{"esi-ui.write_waypoint.v1", "ui-control"},

	{"esi-wallet.read_character_wallet.v1", "wallet"},

	{"esi-characters.read_corporation_roles.v1", "roles"},
	{"esi-characters.read_titles.v1", "roles"},
	{"esi-alliances.read_contacts.v1", "roles"},
	{"esi-corporations.read_contacts.v1", "roles"},
}

var groupReasons = map[string]string{
	"market":        "Reporting and Market tools",
	"contacts":      "War Contact Synchronization",
	"wallet":        "Profit and Loss Tools",
	"assets":        "Asset Value Tools",
	"ui-control":    "Control of in-game UI",
	"notifications": "Notification tools (locators, structures, wars)",
	"roles":         "Corporation services based on corp roles",
}

// shareReasons for data shares between characters and entities
var shareReasons = map[string]string{
	"application": "Corporation Applications",
	"locator":     "Locator Responses",
	"structure":   "Corporation structures under attack",
	"war":         "War Declared on Corporation",
}

func GetCharacterShareGroups() map[string]string {
	return shareReasons
}

func GetCharacterScopes() []string {
	var s []string
	for _, scope := range characterScopes {
		s = append(s, scope.Scope)
	}
	return s
}

func GetCharacterScopeGroups() map[string]string {
	return groupReasons
}

func GetCharacterScopesByGroups(groups []string) []string {
	var s []string
	for _, group := range groups {
		for _, scope := range characterScopes {
			if scope.Group == group {
				s = append(s, scope.Scope)
			}
		}
	}
	return s
}

// GetCharacterGroupsByScopesString takes a space seperated string of scopes and returns the groups
func GetCharacterGroupsByScopesString(scopes string) string {
	groups := make(map[string]bool)
	for _, scope := range strings.Split(scopes, " ") {
		for _, charScope := range characterScopes {
			if charScope.Scope == scope {
				groups[charScope.Group] = true
			}
		}
	}

	m := make([]string, len(groups))
	i := 0
	for k := range groups {
		m[i] = k
		i++
	}
	sort.Strings(m)
	return strings.Join(m, ", ")
}
<commit_msg>add mail proxy scopes<commit_after>package models

import (
	"sort"
	"strings"
)

type ScopeGroup struct {
	Scope string
	Group string
}

var characterScopes = []ScopeGroup{
	{"esi-assets.read_assets.v1", "assets"},

	{"esi-characters.read_contacts.v1", "contacts"},
	{"esi-characters.write_contacts.v1", "contacts"},

	{"esi-characters.read_notifications.v1", "notifications"},

	{"esi-universe.read_structures.v1", "market"},
	{"esi-search.search_structures.v1", "market"},
	{"esi-markets.structure_markets.v1", "market"},

	{"esi-ui.open_window.v1", "ui-control"},
	{"esi-ui.write_waypoint.v1", "ui-control"},

	{"esi-wallet.read_character_wallet.v1", "wallet"},

	{"esi-characters.read_corporation_roles.v1", "roles"},
	{"esi-characters.read_titles.v1", "roles"},
	{"esi-alliances.read_contacts.v1", "roles"},
	{"esi-corporations.read_contacts.v1", "roles"},

	{"esi-mail.send_mail.v1", "evemail"},
	{"esi-mail.read_mail.v1", "evemail"},
	{"esi-mail.organize_mail.v1", "evemail"},
}

var groupReasons = map[string]string{
	"market":        "Reporting and Market tools",
	"contacts":      "War Contact Synchronization",
	"wallet":        "Profit and Loss Tools",
	"assets":        "Asset Value Tools",
	"ui-control":    "Control of in-game UI",
	"notifications": "Notification tools (locators, structures, wars)",
	"roles":         "Corporation services based on corp roles",
	"evemail":       "EVE Mail Proxy Service",
}

// shareReasons for data shares between characters and entities
var shareReasons = map[string]string{
	"application": "Corporation Applications",
	"locator":     "Locator Responses",
	"structure":   "Corporation structures under attack",
	"war":         "War Declared on Corporation",
}

func GetCharacterShareGroups() map[string]string {
	return shareReasons
}

func GetCharacterScopes() []string {
	var s []string
	for _, scope := range characterScopes {
		s = append(s, scope.Scope)
	}
	return s
}

func GetCharacterScopeGroups() map[string]string {
	return groupReasons
}

func GetCharacterScopesByGroups(groups []string) []string {
	var s []string
	for _, group := range groups {
		for _, scope := range characterScopes {
			if scope.Group == group {
				s = append(s, scope.Scope)
			}
		}
	}
	return s
}

// GetCharacterGroupsByScopesString takes a space seperated string of scopes and returns the groups
func GetCharacterGroupsByScopesString(scopes string) string {
	groups := make(map[string]bool)
	for _, scope := range strings.Split(scopes, " ") {
		for _, charScope := range characterScopes {
			if charScope.Scope == scope {
				groups[charScope.Group] = true
			}
		}
	}

	m := make([]string, len(groups))
	i := 0
	for k := range groups {
		m[i] = k
		i++
	}
	sort.Strings(m)
	return strings.Join(m, ", ")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"math/rand"
	"sync/atomic"
	"time"

	"github.com/vbauerster/mpb/v7"
	"github.com/vbauerster/mpb/v7/decor"
)

var curTask uint32
var doneTasks uint32

type task struct {
	id    int
	total int
	bar   *mpb.Bar
}

func main() {
	numTasks := 4

	var total int
	var filler mpb.BarFiller
	tasks := make([]*task, numTasks)

	for i := 0; i < numTasks; i++ {
		task := &task{
			id:    i,
			total: rand.Intn(101) + 100,
		}
		total += task.total
		filler = middleware(filler, task.id)
		tasks[i] = task
	}

	filler = newLineMiddleware(filler)

	p := mpb.New()

	for i := 0; i < numTasks; i++ {
		var waitBar *mpb.Bar
		if i != 0 {
			waitBar = tasks[i-1].bar
		}
		total := tasks[i].total
		bar := p.AddBar(int64(total),
			mpb.BarExtenderRev(filler),
			mpb.BarQueueAfter(waitBar, false),
			mpb.PrependDecorators(
				decor.Name("current:", decor.WCSyncWidthR),
			),
			mpb.AppendDecorators(
				decor.Percentage(decor.WCSyncWidth),
			),
		)
		tasks[i].bar = bar
	}

	tb := p.AddBar(int64(total),
		mpb.PrependDecorators(
			decor.Any(func(st decor.Statistics) string {
				var done uint32
				if st.Completed {
					done = uint32(len(tasks))
				} else {
					done = atomic.LoadUint32(&doneTasks)
				}
				return fmt.Sprintf("TOTAL(%d/%d)", done, len(tasks))
			}, decor.WCSyncWidthR),
		),
		mpb.AppendDecorators(
			decor.Percentage(decor.WCSyncWidth),
		),
	)

	for _, t := range tasks {
		atomic.StoreUint32(&curTask, uint32(t.id))
		complete(tb, t)
		atomic.AddUint32(&doneTasks, 1)
	}

	p.Wait()
}

func middleware(base mpb.BarFiller, id int) mpb.BarFiller {
	var done bool
	fn := func(w io.Writer, _ int, st decor.Statistics) {
		if !done {
			cur := atomic.LoadUint32(&curTask) == uint32(id)
			if !st.Completed && cur {
				fmt.Fprintf(w, "=> Taksk %02d\n", id)
				return
			} else if !cur {
				fmt.Fprintf(w, "   Taksk %02d\n", id)
				return
			} else {
				done = cur
			}
		}
		fmt.Fprintf(w, "   Taksk %02d: Done!\n", id)
	}
	if id == 0 {
		return mpb.BarFillerFunc(fn)
	}
	return mpb.BarFillerFunc(func(w io.Writer, reqWidth int, st decor.Statistics) {
		fn(w, reqWidth, st)
		base.Fill(w, reqWidth, st)
	})
}

func newLineMiddleware(base mpb.BarFiller) mpb.BarFiller {
	return mpb.BarFillerFunc(func(w io.Writer, reqWidth int, st decor.Statistics) {
		fmt.Fprintln(w)
		base.Fill(w, reqWidth, st)
	})
}

func complete(tb *mpb.Bar, t *task) {
	bar := t.bar
	max := 100 * time.Millisecond
	for !bar.Completed() {
		n := rand.Int63n(10) + 1
		bar.IncrInt64(n)
		tb.IncrInt64(n)
		time.Sleep(time.Duration(rand.Intn(10)+1) * max / 10)
	}
	bar.Wait()
}
<commit_msg>refactoring: else isn't necessary<commit_after>package main

import (
	"fmt"
	"io"
	"math/rand"
	"sync/atomic"
	"time"

	"github.com/vbauerster/mpb/v7"
	"github.com/vbauerster/mpb/v7/decor"
)

var curTask uint32
var doneTasks uint32

type task struct {
	id    int
	total int
	bar   *mpb.Bar
}

func main() {
	numTasks := 4

	var total int
	var filler mpb.BarFiller
	tasks := make([]*task, numTasks)

	for i := 0; i < numTasks; i++ {
		task := &task{
			id:    i,
			total: rand.Intn(101) + 100,
		}
		total += task.total
		filler = middleware(filler, task.id)
		tasks[i] = task
	}

	filler = newLineMiddleware(filler)

	p := mpb.New()

	for i := 0; i < numTasks; i++ {
		var waitBar *mpb.Bar
		if i != 0 {
			waitBar = tasks[i-1].bar
		}
		total := tasks[i].total
		bar := p.AddBar(int64(total),
			mpb.BarExtenderRev(filler),
			mpb.BarQueueAfter(waitBar, false),
			mpb.PrependDecorators(
				decor.Name("current:", decor.WCSyncWidthR),
			),
			mpb.AppendDecorators(
				decor.Percentage(decor.WCSyncWidth),
			),
		)
		tasks[i].bar = bar
	}

	tb := p.AddBar(int64(total),
		mpb.PrependDecorators(
			decor.Any(func(st decor.Statistics) string {
				var done uint32
				if st.Completed {
					done = uint32(len(tasks))
				} else {
					done = atomic.LoadUint32(&doneTasks)
				}
				return fmt.Sprintf("TOTAL(%d/%d)", done, len(tasks))
			}, decor.WCSyncWidthR),
		),
		mpb.AppendDecorators(
			decor.Percentage(decor.WCSyncWidth),
		),
	)

	for _, t := range tasks {
		atomic.StoreUint32(&curTask, uint32(t.id))
		complete(tb, t)
		atomic.AddUint32(&doneTasks, 1)
	}

	p.Wait()
}

func middleware(base mpb.BarFiller, id int) mpb.BarFiller {
	var done bool
	fn := func(w io.Writer, _ int, st decor.Statistics) {
		if !done {
			cur := atomic.LoadUint32(&curTask) == uint32(id)
			if !cur {
				fmt.Fprintf(w, "   Taksk %02d\n", id)
				return
			}
			if !st.Completed {
				fmt.Fprintf(w, "=> Taksk %02d\n", id)
				return
			}
			done = cur
		}
		fmt.Fprintf(w, "   Taksk %02d: Done!\n", id)
	}
	if id == 0 {
		return mpb.BarFillerFunc(fn)
	}
	return mpb.BarFillerFunc(func(w io.Writer, reqWidth int, st decor.Statistics) {
		fn(w, reqWidth, st)
		base.Fill(w, reqWidth, st)
	})
}

func newLineMiddleware(base mpb.BarFiller) mpb.BarFiller {
	return mpb.BarFillerFunc(func(w io.Writer, reqWidth int, st decor.Statistics) {
		fmt.Fprintln(w)
		base.Fill(w, reqWidth, st)
	})
}

func complete(tb *mpb.Bar, t *task) {
	bar := t.bar
	max := 100 * time.Millisecond
	for !bar.Completed() {
		n := rand.Int63n(10) + 1
		bar.IncrInt64(n)
		tb.IncrInt64(n)
		time.Sleep(time.Duration(rand.Intn(10)+1) * max / 10)
	}
	bar.Wait()
}
<|endoftext|>


<commit_before>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transform

import (
	"fmt"
	"testing"

	"github.com/gohugoio/hugo/helpers"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/require"
)

func TestRemarshal(t *testing.T) {
	t.Parallel()

	ns := New(newDeps(viper.New()))
	assert := require.New(t)

	tomlExample := `title = "Test Metadata"

[[resources]]
  src = "**image-4.png"
  title = "The Fourth Image!"
  [resources.params]
    byline = "picasso"

[[resources]]
  name = "my-cool-image-:counter"
  src = "**.png"
  title = "TOML: The Image #:counter"
  [resources.params]
    byline = "bep"
`

	yamlExample := `resources:
- params:
    byline: picasso
  src: '**image-4.png'
  title: The Fourth Image!
- name: my-cool-image-:counter
  params:
    byline: bep
  src: '**.png'
  title: 'TOML: The Image #:counter'
title: Test Metadata
`

	jsonExample := `{
   "resources": [
      {
         "params": {
            "byline": "picasso"
         },
         "src": "**image-4.png",
         "title": "The Fourth Image!"
      },
      {
         "name": "my-cool-image-:counter",
         "params": {
            "byline": "bep"
         },
         "src": "**.png",
         "title": "TOML: The Image #:counter"
      }
   ],
   "title": "Test Metadata"
}
`

	variants := []struct {
		format string
		data   string
	}{
		{"yaml", yamlExample},
		{"json", jsonExample},
		{"toml", tomlExample},
		{"TOML", tomlExample},
		{"Toml", tomlExample},
		{" TOML ", tomlExample},
	}

	for _, v1 := range variants {
		for _, v2 := range variants {
			// Both from and to may be the same here, but that is fine.
			fromTo := fmt.Sprintf("%s => %s", v2.format, v1.format)

			converted, err := ns.Remarshal(v1.format, v2.data)
			assert.NoError(err, fromTo)
			diff := helpers.DiffStrings(v1.data, converted)
			if len(diff) > 0 {
				t.Errorf("[%s] Expected \n%v\ngot\n%v\ndiff:\n%v", fromTo, v1.data, converted, diff)
			}

		}
	}

}

func TestTestRemarshalError(t *testing.T) {
	t.Parallel()

	ns := New(newDeps(viper.New()))
	assert := require.New(t)

	_, err := ns.Remarshal("asdf", "asdf")
	assert.Error(err)

	_, err = ns.Remarshal("json", "asdf")
	assert.Error(err)

}

func TestRemarshalDetectFormat(t *testing.T) {
	t.Parallel()
	assert := require.New(t)

	for i, test := range []struct {
		data   string
		expect interface{}
	}{
		{`foo = "bar"`, "toml"},
		{`   foo = "bar"`, "toml"},
		{`foo="bar"`, "toml"},
		{`foo: "bar"`, "yaml"},
		{`foo:"bar"`, "yaml"},
		{`{ "foo": "bar"`, "json"},
		{`asdfasdf`, false},
		{``, false},
	} {
		errMsg := fmt.Sprintf("[%d] %s", i, test.data)

		result, err := detectFormat(test.data)

		if b, ok := test.expect.(bool); ok && !b {
			assert.Error(err, errMsg)
			continue
		}

		assert.NoError(err, errMsg)
		assert.Equal(test.expect, result, errMsg)
	}
}
<commit_msg>tpl/transform: Add a comments test for Remarshal<commit_after>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transform

import (
	"fmt"
	"testing"

	"github.com/gohugoio/hugo/helpers"
	"github.com/spf13/viper"
	"github.com/stretchr/testify/require"
)

func TestRemarshal(t *testing.T) {
	t.Parallel()

	ns := New(newDeps(viper.New()))
	assert := require.New(t)

	tomlExample := `title = "Test Metadata"

[[resources]]
  src = "**image-4.png"
  title = "The Fourth Image!"
  [resources.params]
    byline = "picasso"

[[resources]]
  name = "my-cool-image-:counter"
  src = "**.png"
  title = "TOML: The Image #:counter"
  [resources.params]
    byline = "bep"
`

	yamlExample := `resources:
- params:
    byline: picasso
  src: '**image-4.png'
  title: The Fourth Image!
- name: my-cool-image-:counter
  params:
    byline: bep
  src: '**.png'
  title: 'TOML: The Image #:counter'
title: Test Metadata
`

	jsonExample := `{
   "resources": [
      {
         "params": {
            "byline": "picasso"
         },
         "src": "**image-4.png",
         "title": "The Fourth Image!"
      },
      {
         "name": "my-cool-image-:counter",
         "params": {
            "byline": "bep"
         },
         "src": "**.png",
         "title": "TOML: The Image #:counter"
      }
   ],
   "title": "Test Metadata"
}
`

	variants := []struct {
		format string
		data   string
	}{
		{"yaml", yamlExample},
		{"json", jsonExample},
		{"toml", tomlExample},
		{"TOML", tomlExample},
		{"Toml", tomlExample},
		{" TOML ", tomlExample},
	}

	for _, v1 := range variants {
		for _, v2 := range variants {
			// Both from and to may be the same here, but that is fine.
			fromTo := fmt.Sprintf("%s => %s", v2.format, v1.format)

			converted, err := ns.Remarshal(v1.format, v2.data)
			assert.NoError(err, fromTo)
			diff := helpers.DiffStrings(v1.data, converted)
			if len(diff) > 0 {
				t.Errorf("[%s] Expected \n%v\ngot\n%v\ndiff:\n%v", fromTo, v1.data, converted, diff)
			}

		}
	}

}

func TestRemarshalComments(t *testing.T) {
	t.Parallel()

	ns := New(newDeps(viper.New()))
	assert := require.New(t)

	input := `
Hugo = "Rules"

# It really does!

[m]
# A comment
a = "b"

`

	expected := `
Hugo = "Rules"

[m]
  a = "b"
`

	for _, format := range []string{"json", "yaml", "toml"} {
		fromTo := fmt.Sprintf("%s => %s", "toml", format)

		converted := input
		var err error
		// Do a round-trip conversion
		for _, toFormat := range []string{format, "toml"} {
			converted, err = ns.Remarshal(toFormat, converted)
			assert.NoError(err, fromTo)
		}

		diff := helpers.DiffStrings(expected, converted)
		if len(diff) > 0 {
			t.Fatalf("[%s] Expected \n%v\ngot\n%v\ndiff:\n%v\n", fromTo, expected, converted, diff)
		}
	}
}

func TestTestRemarshalError(t *testing.T) {
	t.Parallel()

	ns := New(newDeps(viper.New()))
	assert := require.New(t)

	_, err := ns.Remarshal("asdf", "asdf")
	assert.Error(err)

	_, err = ns.Remarshal("json", "asdf")
	assert.Error(err)

}

func TestRemarshalDetectFormat(t *testing.T) {
	t.Parallel()
	assert := require.New(t)

	for i, test := range []struct {
		data   string
		expect interface{}
	}{
		{`foo = "bar"`, "toml"},
		{`   foo = "bar"`, "toml"},
		{`foo="bar"`, "toml"},
		{`foo: "bar"`, "yaml"},
		{`foo:"bar"`, "yaml"},
		{`{ "foo": "bar"`, "json"},
		{`asdfasdf`, false},
		{``, false},
	} {
		errMsg := fmt.Sprintf("[%d] %s", i, test.data)

		result, err := detectFormat(test.data)

		if b, ok := test.expect.(bool); ok && !b {
			assert.Error(err, errMsg)
			continue
		}

		assert.NoError(err, errMsg)
		assert.Equal(test.expect, result, errMsg)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"image"
	"image/color"
	"image/draw"
	"os"
	"path"

	"github.com/MJKWoolnough/byteio"
	"github.com/MJKWoolnough/minecraft"
	"github.com/MJKWoolnough/minecraft/nbt"
	"github.com/MJKWoolnough/ora"
)

func (t Transfer) generate(name string, _ *byteio.StickyReader, w *byteio.StickyWriter, f *os.File, size int64) error {
	o, err := ora.Open(f, size)
	if err != nil {
		return err
	}
	terrain := o.Layer("terrain")
	if terrain == nil {
		return layerError{"terrain"}
	}
	height := o.Layer("height")
	if height == nil {
		return layerError{"height"}
	}
	mp := t.c.NewMap()
	if mp == nil {
		return errors.New("failed to create map")
	}

	done := false
	defer func() {
		if !done {
			t.c.RemoveMap(mp.ID)
		}
		go t.c.Save()
	}()

	mp.Lock()
	mp.Name = name
	mapPath := mp.Path
	mp.Server = -2
	mp.Unlock()

	ms := DefaultMapSettings()
	ms["level-type"] = minecraft.FlatGenerator
	ms["generator-settings"] = "0"
	ms["motd"] = name

	pf, err := os.Create(path.Join(mapPath, "properties.map"))
	if err != nil {
		return err
	}

	if err = ms.WriteTo(pf); err != nil {
		return err
	}
	pf.Close()

	b := o.Bounds()
	w.WriteUint8(2)
	w.WriteInt32(int32(b.Max.X) >> 4)
	w.WriteInt32(int32(b.Max.Y) >> 4)
	c := make(chan paint, 1024)
	m := make(chan string, 4)
	e := make(chan struct{}, 0)
	defer close(e)
	go func() {
		defer close(c)
		defer close(m)
		for {
			select {
			case message := <-m:
				w.WriteUint8(3)
				writeString(w, message)
			case p := <-c:
				w.WriteUint8(4)
				w.WriteInt32(p.X)
				w.WriteInt32(p.Y)
				r, g, b, a := p.RGBA()
				w.WriteUint8(uint8(r >> 8))
				w.WriteUint8(uint8(g >> 8))
				w.WriteUint8(uint8(b >> 8))
				w.WriteUint8(uint8(a >> 8))
			case <-e:
				return
			}
		}
	}()

	sTerrain := image.NewPaletted(o.Bounds(), terrainColours)
	terrainI, err := terrain.Image()
	if err != nil {
		return err
	}
	draw.Draw(sTerrain, image.Rect(terrain.X, terrain.Y, sTerrain.Bounds().Max.X, sTerrain.Bounds().Max.Y), terrainI, image.Point{}, draw.Src)
	terrainI = nil
	sHeight := image.NewGray(o.Bounds())
	heightI, err := height.Image()
	if err != nil {
		return err
	}
	draw.Draw(sHeight, image.Rect(height.X, height.Y, sTerrain.Bounds().Max.X, sTerrain.Bounds().Max.Y), heightI, image.Point{}, draw.Src)
	heightI = nil

	var sBiomes *image.Paletted
	if biomes := o.Layer("biomes"); biomes != nil {
		sBiomes = image.NewPaletted(o.Bounds(), biomePalette)
		biomesI, err := biomes.Image()
		if err != nil {
			return err
		}
		draw.Draw(sBiomes, image.Rect(height.X, height.Y, sBiomes.Bounds().Max.X, sBiomes.Bounds().Max.Y), biomesI, image.Point{}, draw.Src)
	}

	var sWater *image.Gray
	if water := o.Layer("biomes"); water != nil {
		sWater = image.NewGray(o.Bounds())
		waterI, err := water.Image()
		if err != nil {
			return err
		}
		draw.Draw(sWater, image.Rect(height.X, height.Y, sWater.Bounds().Max.X, sWater.Bounds().Max.Y), waterI, image.Point{}, draw.Src)
	}

	p, err := minecraft.NewFilePath(mapPath)
	if err != nil {
		return err
	}

	level, err := minecraft.NewLevel(p)
	if err != nil {
		return err
	}

	level.LevelName(name)

	m <- "Building Terrain"
	if err := buildTerrain(p, level, sTerrain, sBiomes, sHeight, sWater, c); err != nil {
		return err
	}

	level.LevelName(name)
	level.MobSpawning(false)
	level.KeepInventory(true)
	level.FireTick(false)
	level.DayLightCycle(false)
	level.MobGriefing(false)
	level.Spawn(10, 250, 10)
	level.Generator(minecraft.FlatGenerator)
	level.GeneratorOptions("0")
	level.GameMode(minecraft.Creative)
	level.AllowCommands(true)

	m <- "Exporting"
	level.Save()
	level.Close()
	done = true
	mp.Lock()
	mp.Server = -1
	mp.Unlock()

	return nil
}

type paint struct {
	color.Color
	X, Y int32
}

type layerError struct {
	name string
}

func (l layerError) Error() string {
	return "missing layer: " + l.name
}

type terrain struct {
	Base, Top minecraft.Block
	TopLevel  uint8
}

var (
	terrainColours = color.Palette{
		color.RGBA{},
		color.RGBA{255, 255, 0, 255},   // Yellow - Sand
		color.RGBA{0, 255, 0, 255},     // Green - Grass
		color.RGBA{87, 59, 12, 255},    // Brown - Dirt
		color.RGBA{255, 128, 0, 255},   // Orange - Farm
		color.RGBA{128, 128, 128, 255}, // Grey - Stone
		color.RGBA{255, 255, 255, 255}, // White - Snow
	}
	terrainBlocks = []terrain{
		{},
		{minecraft.Block{ID: 24, Data: 2}, minecraft.Block{ID: 12}, 5}, // Sandstone - Sand
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 2}, 1},            // Dirt - Grass
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 3}, 0},            // Dirt - Dirt
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 60, Data: 7}, 1},  // Dirt - Farmland
		{minecraft.Block{ID: 1}, minecraft.Block{ID: 1}, 0},            // Stone - Stone
		{minecraft.Block{ID: 1}, minecraft.Block{ID: 80}, 3},           // Stone - Snow
		{minecraft.Block{ID: 9}, minecraft.Block{ID: 9}, 0},
	}
	biomePalette = color.Palette{}
	biomeList    = []minecraft.Biome{}
)

func modeTerrain(p *image.Paletted) uint8 {
	b := p.Bounds()
	modeMap := make([]uint16, len(terrainColours))
	most := uint16(0)
	mode := uint8(0)
	for i := b.Min.X; i < b.Max.X; i++ {
		for j := b.Min.Y; j < b.Max.Y; j++ {
			pos := p.ColorIndexAt(i, j)
			modeMap[pos]++
			if m := modeMap[pos]; m > most {
				most = m
				mode = pos
			}
		}
	}
	return mode
}

func meanHeight(g *image.Gray) uint8 {
	b := g.Bounds()
	var total int64
	for i := b.Min.X; i < b.Max.X; i++ {
		for j := b.Min.Y; j < b.Max.Y; j++ {
			total += int64(g.GrayAt(i, j).Y)
		}
	}
	return uint8(total / int64((b.Dx() * b.Dy())))
}

type chunkCache struct {
	mem   *minecraft.MemPath
	level *minecraft.Level
	clear nbt.Tag
	cache map[uint16]nbt.Tag
}

func newCache() *chunkCache {
	mem := minecraft.NewMemPath()
	l, _ := minecraft.NewLevel(mem)

	bedrock := minecraft.Block{ID: 7}

	l.SetBlock(0, 0, 0, minecraft.Block{})
	l.Save()
	l.Close()
	clearChunk, _ := mem.GetChunk(0, 0)

	for i := int32(-16); i < 16; i++ {
		for j := int32(0); j < 255; j++ {
			for k := int32(-16); j < 16; j++ {
				l.SetBlock(i, j, k, bedrock)
			}
		}
	}
	l.Save()
	l.Close()
	mem.SetChunk(clearChunk)
	return &chunkCache{
		mem,
		l,
		clearChunk,
		make(map[uint16]nbt.Tag),
	}
}

func (c *chunkCache) getFromCache(x, z int32, terrain uint8, height int32) nbt.Tag {
	cacheID := uint16(terrain)<<8 | uint16(height)
	chunk, ok := c.cache[cacheID]
	if !ok {
		b := terrainBlocks[terrain].Base
		for j := int32(0); j < height; j++ {
			for i := int32(0); i < 16; i++ {
				for k := int32(0); k < 16; k++ {
					c.level.SetBlock(i, j, k, b)
				}
			}
		}
		c.level.Save()
		c.level.Close()
		chunk, _ = c.mem.GetChunk(0, 0)
		c.mem.SetChunk(c.clear)
		c.cache[cacheID] = chunk
	}
	ld := chunk.Data().(nbt.Compound).Get("Level").Data().(nbt.Compound)
	ld.Set(nbt.NewTag("xPos", nbt.Int(x)))
	ld.Set(nbt.NewTag("zPos", nbt.Int(z)))
	return chunk
}

func buildTerrain(mpath minecraft.Path, level *minecraft.Level, terrain, biomes *image.Paletted, height, water *image.Gray, c chan paint) error {
	b := terrain.Bounds()
	proceed := make(chan uint8, 10)
	errChan := make(chan error, 1)
	go func() {
		defer close(proceed)
		cc := newCache()
		for j := 0; j < b.Max.Y; j += 16 {
			chunkZ := int32(j >> 4)
			for i := 0; i < b.Max.X; i += 16 {
				chunkX := int32(i >> 4)
				p := terrain.SubImage(image.Rect(i, j, i+16, j+16)).(*image.Paletted)
				g := height.SubImage(image.Rect(i, j, i+16, j+16)).(*image.Gray)
				w := water.SubImage(image.Rect(i, j, i+16, j+16)).(*image.Gray)
				h := int32(meanHeight(g))
				wh := int32(meanHeight(w))
				var t uint8
				if wh >= h<<1 { // more water than land...
					c <- paint{
						color.RGBA{0, 0, 255, 255},
						chunkX, chunkZ,
					}
					t = uint8(len(terrainBlocks) - 1)
				} else {
					t = modeTerrain(p)
					c <- paint{
						terrainColours[t],
						chunkX, chunkZ,
					}
				}
				if err := mpath.SetChunk(cc.getFromCache(chunkX, chunkZ, t, h)); err != nil {
					errChan <- err
					return
				}
				proceed <- t
			}
		}
	}()
	ts := make([]uint8, 0, 1024)
	for i := 0; i < (b.Max.X>>4)+2; i++ {
		ts = append(ts, <-proceed) // get far enough ahead so all chunks are surrounded before shaping, to get correct lighting
	}
	select {
	case err := <-errChan:
		return err
	default:
	}
	for j := int32(0); j < int32(b.Max.Y); j += 16 {
		chunkZ := j >> 4
		for i := int32(0); i < int32(b.Max.X); i += 16 {
			chunkX := i >> 4
			var totalHeight int32
			ot := ts[0]
			ts = ts[1:]
			for x := i; x < i+16; x++ {
				for z := j; z < j+16; z++ {
					if biomes != nil {
						level.SetBiome(x, z, biomeList[biomes.ColorIndexAt(int(x), int(z))])
					}
					h := int32(height.GrayAt(int(x), int(z)).Y)
					totalHeight += h
					y, _ := level.GetHeight(x, z)
					wl := int32(water.GrayAt(int(x), int(z)).Y)
					for ; y > h && y > wl; y-- {
						level.SetBlock(x, y, z, minecraft.Block{})
					}
					for ; y > h; y-- {
						level.SetBlock(x, y, z, minecraft.Block{ID: 9})
					}
					t := terrain.ColorIndexAt(int(x), int(z))
					tb := terrainBlocks[t]
					for ; y > h-int32(tb.TopLevel); y-- {
						level.SetBlock(x, y, z, tb.Top)
					}
					if t != ot {
						for ; y >= 0; y-- {
							level.SetBlock(x, y, z, tb.Base)
						}
					}
				}
			}
			c <- paint{
				color.Alpha{uint8(totalHeight >> 8)},
				chunkX, chunkZ,
			}
			select {
			case p, ok := <-proceed:
				if ok {
					ts = append(ts, p)
				}
			case err := <-errChan:
				return err
			}
		}
	}
	return nil
}
<commit_msg>Added some biome colours and corrected water layer name<commit_after>package main

import (
	"errors"
	"image"
	"image/color"
	"image/draw"
	"os"
	"path"

	"github.com/MJKWoolnough/byteio"
	"github.com/MJKWoolnough/minecraft"
	"github.com/MJKWoolnough/minecraft/nbt"
	"github.com/MJKWoolnough/ora"
)

func (t Transfer) generate(name string, _ *byteio.StickyReader, w *byteio.StickyWriter, f *os.File, size int64) error {
	o, err := ora.Open(f, size)
	if err != nil {
		return err
	}
	terrain := o.Layer("terrain")
	if terrain == nil {
		return layerError{"terrain"}
	}
	height := o.Layer("height")
	if height == nil {
		return layerError{"height"}
	}
	mp := t.c.NewMap()
	if mp == nil {
		return errors.New("failed to create map")
	}

	done := false
	defer func() {
		if !done {
			t.c.RemoveMap(mp.ID)
		}
		go t.c.Save()
	}()

	mp.Lock()
	mp.Name = name
	mapPath := mp.Path
	mp.Server = -2
	mp.Unlock()

	ms := DefaultMapSettings()
	ms["level-type"] = minecraft.FlatGenerator
	ms["generator-settings"] = "0"
	ms["motd"] = name

	pf, err := os.Create(path.Join(mapPath, "properties.map"))
	if err != nil {
		return err
	}

	if err = ms.WriteTo(pf); err != nil {
		return err
	}
	pf.Close()

	b := o.Bounds()
	w.WriteUint8(2)
	w.WriteInt32(int32(b.Max.X) >> 4)
	w.WriteInt32(int32(b.Max.Y) >> 4)
	c := make(chan paint, 1024)
	m := make(chan string, 4)
	e := make(chan struct{}, 0)
	defer close(e)
	go func() {
		defer close(c)
		defer close(m)
		for {
			select {
			case message := <-m:
				w.WriteUint8(3)
				writeString(w, message)
			case p := <-c:
				w.WriteUint8(4)
				w.WriteInt32(p.X)
				w.WriteInt32(p.Y)
				r, g, b, a := p.RGBA()
				w.WriteUint8(uint8(r >> 8))
				w.WriteUint8(uint8(g >> 8))
				w.WriteUint8(uint8(b >> 8))
				w.WriteUint8(uint8(a >> 8))
			case <-e:
				return
			}
		}
	}()

	sTerrain := image.NewPaletted(o.Bounds(), terrainColours)
	terrainI, err := terrain.Image()
	if err != nil {
		return err
	}
	draw.Draw(sTerrain, image.Rect(terrain.X, terrain.Y, sTerrain.Bounds().Max.X, sTerrain.Bounds().Max.Y), terrainI, image.Point{}, draw.Src)
	terrainI = nil
	sHeight := image.NewGray(o.Bounds())
	heightI, err := height.Image()
	if err != nil {
		return err
	}
	draw.Draw(sHeight, image.Rect(height.X, height.Y, sTerrain.Bounds().Max.X, sTerrain.Bounds().Max.Y), heightI, image.Point{}, draw.Src)
	heightI = nil

	var sBiomes *image.Paletted
	if biomes := o.Layer("biomes"); biomes != nil {
		sBiomes = image.NewPaletted(o.Bounds(), biomePalette)
		biomesI, err := biomes.Image()
		if err != nil {
			return err
		}
		draw.Draw(sBiomes, image.Rect(height.X, height.Y, sBiomes.Bounds().Max.X, sBiomes.Bounds().Max.Y), biomesI, image.Point{}, draw.Src)
	}

	var sWater *image.Gray
	if water := o.Layer("water"); water != nil {
		sWater = image.NewGray(o.Bounds())
		waterI, err := water.Image()
		if err != nil {
			return err
		}
		draw.Draw(sWater, image.Rect(height.X, height.Y, sWater.Bounds().Max.X, sWater.Bounds().Max.Y), waterI, image.Point{}, draw.Src)
	}

	p, err := minecraft.NewFilePath(mapPath)
	if err != nil {
		return err
	}

	level, err := minecraft.NewLevel(p)
	if err != nil {
		return err
	}

	level.LevelName(name)

	m <- "Building Terrain"
	if err := buildTerrain(p, level, sTerrain, sBiomes, sHeight, sWater, c); err != nil {
		return err
	}

	level.LevelName(name)
	level.MobSpawning(false)
	level.KeepInventory(true)
	level.FireTick(false)
	level.DayLightCycle(false)
	level.MobGriefing(false)
	level.Spawn(10, 250, 10)
	level.Generator(minecraft.FlatGenerator)
	level.GeneratorOptions("0")
	level.GameMode(minecraft.Creative)
	level.AllowCommands(true)

	m <- "Exporting"
	level.Save()
	level.Close()
	done = true
	mp.Lock()
	mp.Server = -1
	mp.Unlock()

	return nil
}

type paint struct {
	color.Color
	X, Y int32
}

type layerError struct {
	name string
}

func (l layerError) Error() string {
	return "missing layer: " + l.name
}

type terrain struct {
	Base, Top minecraft.Block
	TopLevel  uint8
}

var (
	terrainColours = color.Palette{
		color.RGBA{},
		color.RGBA{255, 255, 0, 255},   // Yellow - Sand
		color.RGBA{0, 255, 0, 255},     // Green - Grass
		color.RGBA{87, 59, 12, 255},    // Brown - Dirt
		color.RGBA{255, 128, 0, 255},   // Orange - Farm
		color.RGBA{128, 128, 128, 255}, // Grey - Stone
		color.RGBA{255, 255, 255, 255}, // White - Snow
	}
	terrainBlocks = []terrain{
		{},
		{minecraft.Block{ID: 24, Data: 2}, minecraft.Block{ID: 12}, 5}, // Sandstone - Sand
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 2}, 1},            // Dirt - Grass
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 3}, 0},            // Dirt - Dirt
		{minecraft.Block{ID: 3}, minecraft.Block{ID: 60, Data: 7}, 1},  // Dirt - Farmland
		{minecraft.Block{ID: 1}, minecraft.Block{ID: 1}, 0},            // Stone - Stone
		{minecraft.Block{ID: 1}, minecraft.Block{ID: 80}, 3},           // Stone - Snow
		{minecraft.Block{ID: 9}, minecraft.Block{ID: 9}, 0},
	}
	biomePalette = color.Palette{
		color.RGBA{},
		color.RGBA{60, 60, 255, 255},
		color.RGBA{20, 100, 20, 255},
		color.RGBA{255, 255, 0, 255},
		color.RGBA{0, 255, 255, 255},
	}
	biomeList = []minecraft.Biome{
		minecraft.Plains,
		minecraft.Ocean,
		minecraft.Forest,
		minecraft.Desert,
		minecraft.River,
	}
)

func modeTerrain(p *image.Paletted) uint8 {
	b := p.Bounds()
	modeMap := make([]uint16, len(terrainColours))
	most := uint16(0)
	mode := uint8(0)
	for i := b.Min.X; i < b.Max.X; i++ {
		for j := b.Min.Y; j < b.Max.Y; j++ {
			pos := p.ColorIndexAt(i, j)
			modeMap[pos]++
			if m := modeMap[pos]; m > most {
				most = m
				mode = pos
			}
		}
	}
	return mode
}

func meanHeight(g *image.Gray) uint8 {
	b := g.Bounds()
	var total int64
	for i := b.Min.X; i < b.Max.X; i++ {
		for j := b.Min.Y; j < b.Max.Y; j++ {
			total += int64(g.GrayAt(i, j).Y)
		}
	}
	return uint8(total / int64((b.Dx() * b.Dy())))
}

type chunkCache struct {
	mem   *minecraft.MemPath
	level *minecraft.Level
	clear nbt.Tag
	cache map[uint16]nbt.Tag
}

func newCache() *chunkCache {
	mem := minecraft.NewMemPath()
	l, _ := minecraft.NewLevel(mem)

	bedrock := minecraft.Block{ID: 7}

	l.SetBlock(0, 0, 0, minecraft.Block{})
	l.Save()
	l.Close()
	clearChunk, _ := mem.GetChunk(0, 0)

	for i := int32(-16); i < 16; i++ {
		for j := int32(0); j < 255; j++ {
			for k := int32(-16); j < 16; j++ {
				l.SetBlock(i, j, k, bedrock)
			}
		}
	}
	l.Save()
	l.Close()
	mem.SetChunk(clearChunk)
	return &chunkCache{
		mem,
		l,
		clearChunk,
		make(map[uint16]nbt.Tag),
	}
}

func (c *chunkCache) getFromCache(x, z int32, terrain uint8, height int32) nbt.Tag {
	cacheID := uint16(terrain)<<8 | uint16(height)
	chunk, ok := c.cache[cacheID]
	if !ok {
		b := terrainBlocks[terrain].Base
		for j := int32(0); j < height; j++ {
			for i := int32(0); i < 16; i++ {
				for k := int32(0); k < 16; k++ {
					c.level.SetBlock(i, j, k, b)
				}
			}
		}
		c.level.Save()
		c.level.Close()
		chunk, _ = c.mem.GetChunk(0, 0)
		c.mem.SetChunk(c.clear)
		c.cache[cacheID] = chunk
	}
	ld := chunk.Data().(nbt.Compound).Get("Level").Data().(nbt.Compound)
	ld.Set(nbt.NewTag("xPos", nbt.Int(x)))
	ld.Set(nbt.NewTag("zPos", nbt.Int(z)))
	return chunk
}

func buildTerrain(mpath minecraft.Path, level *minecraft.Level, terrain, biomes *image.Paletted, height, water *image.Gray, c chan paint) error {
	b := terrain.Bounds()
	proceed := make(chan uint8, 10)
	errChan := make(chan error, 1)
	go func() {
		defer close(proceed)
		cc := newCache()
		for j := 0; j < b.Max.Y; j += 16 {
			chunkZ := int32(j >> 4)
			for i := 0; i < b.Max.X; i += 16 {
				chunkX := int32(i >> 4)
				p := terrain.SubImage(image.Rect(i, j, i+16, j+16)).(*image.Paletted)
				g := height.SubImage(image.Rect(i, j, i+16, j+16)).(*image.Gray)
				w := water.SubImage(image.Rect(i, j, i+16, j+16)).(*image.Gray)
				h := int32(meanHeight(g))
				wh := int32(meanHeight(w))
				var t uint8
				if wh >= h<<1 { // more water than land...
					c <- paint{
						color.RGBA{0, 0, 255, 255},
						chunkX, chunkZ,
					}
					t = uint8(len(terrainBlocks) - 1)
				} else {
					t = modeTerrain(p)
					c <- paint{
						terrainColours[t],
						chunkX, chunkZ,
					}
				}
				if err := mpath.SetChunk(cc.getFromCache(chunkX, chunkZ, t, h)); err != nil {
					errChan <- err
					return
				}
				proceed <- t
			}
		}
	}()
	ts := make([]uint8, 0, 1024)
	for i := 0; i < (b.Max.X>>4)+2; i++ {
		ts = append(ts, <-proceed) // get far enough ahead so all chunks are surrounded before shaping, to get correct lighting
	}
	select {
	case err := <-errChan:
		return err
	default:
	}
	for j := int32(0); j < int32(b.Max.Y); j += 16 {
		chunkZ := j >> 4
		for i := int32(0); i < int32(b.Max.X); i += 16 {
			chunkX := i >> 4
			var totalHeight int32
			ot := ts[0]
			ts = ts[1:]
			for x := i; x < i+16; x++ {
				for z := j; z < j+16; z++ {
					if biomes != nil {
						level.SetBiome(x, z, biomeList[biomes.ColorIndexAt(int(x), int(z))])
					}
					h := int32(height.GrayAt(int(x), int(z)).Y)
					totalHeight += h
					y, _ := level.GetHeight(x, z)
					wl := int32(water.GrayAt(int(x), int(z)).Y)
					for ; y > h && y > wl; y-- {
						level.SetBlock(x, y, z, minecraft.Block{})
					}
					for ; y > h; y-- {
						level.SetBlock(x, y, z, minecraft.Block{ID: 9})
					}
					t := terrain.ColorIndexAt(int(x), int(z))
					tb := terrainBlocks[t]
					for ; y > h-int32(tb.TopLevel); y-- {
						level.SetBlock(x, y, z, tb.Top)
					}
					if t != ot {
						for ; y >= 0; y-- {
							level.SetBlock(x, y, z, tb.Base)
						}
					}
				}
			}
			c <- paint{
				color.Alpha{uint8(totalHeight >> 8)},
				chunkX, chunkZ,
			}
			select {
			case p, ok := <-proceed:
				if ok {
					ts = append(ts, p)
				}
			case err := <-errChan:
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package translit

import (
	"bytes"
	"log"
	"regexp"
	"strings"
)

//EncodeTags transliteration of an array of tags
func EncodeTags(tag []string) []string {
	var arrEncTag []string
	for _, val := range tag {
		str, count := encode(val)
		if count > 0 {
			str = "ru--" + str
		}
		arrEncTag = append(arrEncTag, str)
	}
	return arrEncTag
}

func DecodeTags(tags []string) []string {
	arrDecTag := make([]string, 0)
	for _, tag := range tags {
		dec := DecodeTag(tag)
		arrDecTag = append(arrDecTag, dec)
	}
	return arrDecTag
}

//EncodeTag transliteration of a tag
func EncodeTag(tag string) string {
	str, count := encode(tag)
	if count > 0 {
		str = "ru--" + str
	}
	return str
}

func DecodeTag(tag string) string {
	if tag[0:4] == "ru--" {
		str, _ := decode(tag[4:])
		return str
	}
	return tag
}

//EncodeTitle transliteration of the title
func EncodeTitle(title string) string {
	var str string
	reg, err := regexp.Compile("[^a-zA-Z0-9а-яА-Я.,]+")
	if err != nil {
		log.Fatal(err)
	}
	processedString := reg.ReplaceAllString(title, "-")
	s1, _ := encode(processedString)
	s2 := strings.Replace(s1, ".", "", -1)
	s3 := strings.Split(s2, "")
	if s3[0] == "-" {
		str = strings.Join(s3[1:], "")
	} else {
		str = strings.Join(s3, "")
	}
	return str
}

func DecodeTitle(title string) string {
	str, _ := decode(title)
	return str
}

func encode(text string) (string, int) {
	if text == "" {
		return "", 0
	}
	text = strings.ToLower(text)
	var input = bytes.NewBufferString(text)
	var output = bytes.NewBuffer(nil)

	// Previous, next letter for special processor
	var rr string
	var ok bool

	i := 0

	for {
		r, _, err := input.ReadRune()
		if err != nil {
			break
		}

		rr, ok = encMap[string(r)]
		if ok {
			output.WriteString(rr)
			i++
			continue
		} else {
			output.WriteString(string(r))
			continue
		}
		rr, ok = encMap[string(r)]
		if ok {
			output.WriteString(rr)
			i++
		} else {
			output.WriteString(string(r))
			continue
		}
	}

	return output.String(), i
}

func decode(text string) (string, int) {
	if text == "" {
		return "", 0
	}

	output := text

	for _, trans := range decOrder {
		output = strings.Replace(output, trans, decMap[trans], -1)
	}

	return output, len(output)
}
<commit_msg>HOTFIX: Check decode tag for length.<commit_after>package translit

import (
	"bytes"
	"log"
	"regexp"
	"strings"
)

//EncodeTags transliteration of an array of tags
func EncodeTags(tag []string) []string {
	var arrEncTag []string
	for _, val := range tag {
		str, count := encode(val)
		if count > 0 {
			str = "ru--" + str
		}
		arrEncTag = append(arrEncTag, str)
	}
	return arrEncTag
}

func DecodeTags(tags []string) []string {
	arrDecTag := make([]string, 0)
	for _, tag := range tags {
		dec := DecodeTag(tag)
		arrDecTag = append(arrDecTag, dec)
	}
	return arrDecTag
}

//EncodeTag transliteration of a tag
func EncodeTag(tag string) string {
	str, count := encode(tag)
	if count > 0 {
		str = "ru--" + str
	}
	return str
}

func DecodeTag(tag string) string {
	if len(tag) < 5 {
		return tag
	}
	if tag[0:4] == "ru--" {
		str, _ := decode(tag[4:])
		return str
	}
	return tag
}

//EncodeTitle transliteration of the title
func EncodeTitle(title string) string {
	var str string
	reg, err := regexp.Compile("[^a-zA-Z0-9а-яА-Я.,]+")
	if err != nil {
		log.Fatal(err)
	}
	processedString := reg.ReplaceAllString(title, "-")
	s1, _ := encode(processedString)
	s2 := strings.Replace(s1, ".", "", -1)
	s3 := strings.Split(s2, "")
	if s3[0] == "-" {
		str = strings.Join(s3[1:], "")
	} else {
		str = strings.Join(s3, "")
	}
	return str
}

func DecodeTitle(title string) string {
	str, _ := decode(title)
	return str
}

func encode(text string) (string, int) {
	if text == "" {
		return "", 0
	}
	text = strings.ToLower(text)
	var input = bytes.NewBufferString(text)
	var output = bytes.NewBuffer(nil)

	// Previous, next letter for special processor
	var rr string
	var ok bool

	i := 0

	for {
		r, _, err := input.ReadRune()
		if err != nil {
			break
		}

		rr, ok = encMap[string(r)]
		if ok {
			output.WriteString(rr)
			i++
			continue
		} else {
			output.WriteString(string(r))
			continue
		}
		rr, ok = encMap[string(r)]
		if ok {
			output.WriteString(rr)
			i++
		} else {
			output.WriteString(string(r))
			continue
		}
	}

	return output.String(), i
}

func decode(text string) (string, int) {
	if text == "" {
		return "", 0
	}

	output := text

	for _, trans := range decOrder {
		output = strings.Replace(output, trans, decMap[trans], -1)
	}

	return output, len(output)
}
<|endoftext|>