<commit_before>package main

import (
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"

	"github.com/dfordsoft/golib/fsutil"
	"github.com/gin-gonic/gin"
)

var (
	fontFiles   []string
	mutexMaking sync.Mutex
	books       = Books{}
	sha1ver     string // sha1 revision used to build the program
	buildTime   string // when the executable was built
)

// Books book collect
type Books struct {
	sync.Mutex
	items []*HistoryItem
}

func (books *Books) append(item *HistoryItem) {
	books.Lock()
	books.items = append(books.items, item)
	books.Unlock()
}

func (books *Books) clear() {
	books.Lock()
	books.items = []*HistoryItem{}
	books.Unlock()
}

func (books *Books) delete(name string) {
	books.Lock()
	for i, book := range books.items {
		if book.BookName == name {
			books.items = append(books.items[:i], books.items[i+1:]...)
			break
		}
	}
	books.Unlock()
}

// HistoryItem - 书籍记录，有4种状态，分别是有效，失败，进行，等待
type HistoryItem struct {
	TOCURL       string `template:"tocurl"`
	BookName     string `template:"name"`
	Status       string `template:"status"`
	DownloadLink string `template:"downloadLink"`
	DeleteLink   string `template:"deleteLink"`
}

func homePage(c *gin.Context) {
	c.HTML(http.StatusOK, "home.tmpl", gin.H{
		"title":     "GetNovel",
		"fontFiles": fontFiles,
		"items":     books.items,
	})
}

type GetNovelAuguments struct {
	TOCURL          string  `form:"tocurl"`
	Format          string  `form:"format"`
	PageType        string  `form:"pageType"`
	LeftMargin      int     `form:"leftMargin"`
	TopMargin       int     `form:"topMargin"`
	TitleFontSize   int     `form:"titleFontSize"`
	ContentFontSize int     `form:"contentFontSize"`
	LineSpacing     float32 `form:"lineSpacing"`
	PagesPerFile    int     `form:"pagesPerFile"`
	ChaptersPerFile int     `form:"chaptersPerFile"`
	FontFile        string  `form:"fontFile"`
	FromTitle       string  `form:"fromTitle"`
	ToTitle         string  `form:"toTitle"`
	FromChapter     int     `form:"fromChapter"`
	ToChapter       int     `form:"toChapter"`
}

func makeEbook(c *gin.Context) {
	var gnargs GetNovelAuguments
	if err := c.ShouldBindJSON(&gnargs); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": fmt.Sprintf("%v", err),
		})
		return
	}
	getnovel, _ := exec.LookPath(`getnovel`)

	if b, e := fsutil.FileExists(getnovel); e != nil || !b {
		if dir, err := filepath.Abs(filepath.Dir(os.Args[0])); err == nil {
			getnovel = filepath.Join(dir, `getnovel`)
		}
	}

	if b, e := fsutil.FileExists(getnovel); e != nil || !b {
		c.JSON(http.StatusInternalServerError, gin.H{
			"error": "getnovel not found",
		})
		return
	}

	args := []string{
		"-f",
		gnargs.Format,
		"-p",
		gnargs.PageType,
		"--fontFile",
		gnargs.FontFile,
	}

	if gnargs.Format == "pdf" {
		args = append(args,
			fmt.Sprintf("--leftMargin=%d", gnargs.LeftMargin),
			fmt.Sprintf("--topMargin=%d", gnargs.TopMargin),
			fmt.Sprintf("--titleFontSize=%d", gnargs.TitleFontSize),
			fmt.Sprintf("--contentFontSize=%d", gnargs.ContentFontSize),
			fmt.Sprintf("--lineSpacing=%f", gnargs.LineSpacing),
			fmt.Sprintf("--pagesPerFile=%d", gnargs.PagesPerFile),
			fmt.Sprintf("--chaptersPerFile=%d", gnargs.ChaptersPerFile),
		)
	}
	if gnargs.FromTitle != "" {
		args = append(args, "--fromTitl", gnargs.FromTitle)
	}
	if gnargs.ToTitle != "" {
		args = append(args, "--toTitle", gnargs.ToTitle)
	}
	if gnargs.FromChapter != 0 {
		args = append(args, "--fromChapter", strconv.Itoa(gnargs.FromChapter))
	}
	if gnargs.ToChapter != 0 {
		args = append(args, "--toChapter", strconv.Itoa(gnargs.ToChapter))
	}
	args = append(args, gnargs.TOCURL)

	cmd := exec.Command(getnovel, args...)

	kindlegenName := `kindlegen`
	if runtime.GOOS == "windows" {
		kindlegenName = `kindlegen.exe`
	}
	kindlegenPath, _ := exec.LookPath(kindlegenName)
	if b, e := fsutil.FileExists(kindlegenPath); e != nil || !b {
		if dir, err := filepath.Abs(filepath.Dir(os.Args[0])); err == nil {
			kindlegenPath = filepath.Join(dir, kindlegenName)
		}
	}
	if !filepath.IsAbs(kindlegenPath) {
		kindlegenPath, _ = filepath.Abs(kindlegenPath)
	}
	cmd.Env = append(os.Environ(),
		"KINDLEGEN_PATH="+kindlegenPath, // ignored
	)
	go func() {
		item := &HistoryItem{
			TOCURL:   gnargs.TOCURL,
			BookName: gnargs.TOCURL,
			Status:   "等待制作",
		}
		books.append(item)

		mutexMaking.Lock()

		item.Status = "制作中"
		if err := cmd.Run(); err != nil {
			item.Status = "制作失败"
		} else {
			item.Status = "有效"

			bookPath := fmt.Sprintf("%s/%s.mobi", item.BookName, item.BookName)
			if b, e := fsutil.FileExists(bookPath); e != nil || !b {
				bookPath = fmt.Sprintf("%s/%s.epub", item.BookName, item.BookName)
			}
			if b, e := fsutil.FileExists(bookPath); e == nil && b {
				baseName := filepath.Base(bookPath)
				item.BookName = baseName
				item.DownloadLink = "/download/" + baseName[:len(baseName)-5] + "/" + baseName
				item.DeleteLink = "/delete/" + baseName[:len(baseName)-5] + "/" + baseName
			}
		}
		mutexMaking.Unlock()

		books.clear()
		scanEbooks()
	}()

	c.JSON(http.StatusOK, gin.H{})
}

func downloadBook(c *gin.Context) {
	path := c.Param("path")
	if path == "pdf" {
		path = ""
	}
	name := c.Param("name")
	c.File(filepath.Join(path, name))
}

func deleteBook(c *gin.Context) {
	path := c.Param("path")
	if path == "pdf" {
		path = ""
	}
	name := c.Param("name")
	os.Remove(filepath.Join(path, name))
	books.delete(name)
	c.Redirect(http.StatusFound, "/")
}

func scanEbooks() {
	matches, err := filepath.Glob("*.pdf")
	if err == nil {
		for _, v := range matches {
			books.append(&HistoryItem{
				BookName:     v,
				Status:       "有效",
				DownloadLink: "/download/pdf/" + v,
				DeleteLink:   "/delete/pdf/" + v,
			})
		}
	}

	matches, err = filepath.Glob("**/*.mobi")
	if err == nil {
		for _, v := range matches {
			books.append(&HistoryItem{
				BookName:     filepath.Base(v),
				Status:       "有效",
				DownloadLink: "/download/" + strings.Replace(v, "\\", "/", -1),
				DeleteLink:   "/delete/" + strings.Replace(v, "\\", "/", -1),
			})
		}
	}
	matches, err = filepath.Glob("**/*.epub")
	if err == nil {
		for _, v := range matches {
			books.append(&HistoryItem{
				BookName:     filepath.Base(v),
				Status:       "有效",
				DownloadLink: "/download/" + strings.Replace(v, "\\", "/", -1),
				DeleteLink:   "/delete/" + strings.Replace(v, "\\", "/", -1),
			})
		}
	}
}

func main() {
	addr := ":8089"
	if bind, ok := os.LookupEnv("BIND"); ok {
		addr = bind
	}
	r := gin.Default()
	r.LoadHTMLGlob("templates/*")
	r.GET("/", homePage)
	r.GET("/download/:path/:name", downloadBook)
	r.GET("/delete/:path/:name", deleteBook)
	r.POST("/makeebook", makeEbook)
	// glob font files
	matches, err := filepath.Glob("fonts/*.ttf")
	if err != nil {
		panic(err)
	}
	for _, v := range matches {
		fontFiles = append(fontFiles, filepath.Base(v))
	}
	matches, err = filepath.Glob("fonts/**/*.ttf")
	for _, v := range matches {
		fontFiles = append(fontFiles, filepath.Base(v))
	}
	scanEbooks()
	r.Run(addr)
}
<commit_msg>(*)fix font path<commit_after>package main

import (
	"fmt"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"

	"github.com/dfordsoft/golib/fsutil"
	"github.com/gin-gonic/gin"
)

var (
	fontFiles   []string
	mutexMaking sync.Mutex
	books       = Books{}
	sha1ver     string // sha1 revision used to build the program
	buildTime   string // when the executable was built
)

// Books book collect
type Books struct {
	sync.Mutex
	items []*HistoryItem
}

func (books *Books) append(item *HistoryItem) {
	books.Lock()
	books.items = append(books.items, item)
	books.Unlock()
}

func (books *Books) clear() {
	books.Lock()
	books.items = []*HistoryItem{}
	books.Unlock()
}

func (books *Books) delete(name string) {
	books.Lock()
	for i, book := range books.items {
		if book.BookName == name {
			books.items = append(books.items[:i], books.items[i+1:]...)
			break
		}
	}
	books.Unlock()
}

// HistoryItem - 书籍记录，有4种状态，分别是有效，失败，进行，等待
type HistoryItem struct {
	TOCURL       string `template:"tocurl"`
	BookName     string `template:"name"`
	Status       string `template:"status"`
	DownloadLink string `template:"downloadLink"`
	DeleteLink   string `template:"deleteLink"`
}

func homePage(c *gin.Context) {
	c.HTML(http.StatusOK, "home.tmpl", gin.H{
		"title":     "GetNovel",
		"fontFiles": fontFiles,
		"items":     books.items,
	})
}

type GetNovelAuguments struct {
	TOCURL          string  `form:"tocurl"`
	Format          string  `form:"format"`
	PageType        string  `form:"pageType"`
	LeftMargin      int     `form:"leftMargin"`
	TopMargin       int     `form:"topMargin"`
	TitleFontSize   int     `form:"titleFontSize"`
	ContentFontSize int     `form:"contentFontSize"`
	LineSpacing     float32 `form:"lineSpacing"`
	PagesPerFile    int     `form:"pagesPerFile"`
	ChaptersPerFile int     `form:"chaptersPerFile"`
	FontFile        string  `form:"fontFile"`
	FromTitle       string  `form:"fromTitle"`
	ToTitle         string  `form:"toTitle"`
	FromChapter     int     `form:"fromChapter"`
	ToChapter       int     `form:"toChapter"`
}

func makeEbook(c *gin.Context) {
	var gnargs GetNovelAuguments
	if err := c.ShouldBindJSON(&gnargs); err != nil {
		c.JSON(http.StatusBadRequest, gin.H{
			"error": fmt.Sprintf("%v", err),
		})
		return
	}
	getnovel, _ := exec.LookPath(`getnovel`)

	if b, e := fsutil.FileExists(getnovel); e != nil || !b {
		if dir, err := filepath.Abs(filepath.Dir(os.Args[0])); err == nil {
			getnovel = filepath.Join(dir, `getnovel`)
		}
	}

	if b, e := fsutil.FileExists(getnovel); e != nil || !b {
		c.JSON(http.StatusInternalServerError, gin.H{
			"error": "getnovel not found",
		})
		return
	}

	args := []string{
		"-f",
		gnargs.Format,
		"-p",
		gnargs.PageType,
		"--fontFile",
		gnargs.FontFile,
	}

	if gnargs.Format == "pdf" {
		args = append(args,
			fmt.Sprintf("--leftMargin=%d", gnargs.LeftMargin),
			fmt.Sprintf("--topMargin=%d", gnargs.TopMargin),
			fmt.Sprintf("--titleFontSize=%d", gnargs.TitleFontSize),
			fmt.Sprintf("--contentFontSize=%d", gnargs.ContentFontSize),
			fmt.Sprintf("--lineSpacing=%f", gnargs.LineSpacing),
			fmt.Sprintf("--pagesPerFile=%d", gnargs.PagesPerFile),
			fmt.Sprintf("--chaptersPerFile=%d", gnargs.ChaptersPerFile),
		)
	}
	if gnargs.FromTitle != "" {
		args = append(args, "--fromTitl", gnargs.FromTitle)
	}
	if gnargs.ToTitle != "" {
		args = append(args, "--toTitle", gnargs.ToTitle)
	}
	if gnargs.FromChapter != 0 {
		args = append(args, "--fromChapter", strconv.Itoa(gnargs.FromChapter))
	}
	if gnargs.ToChapter != 0 {
		args = append(args, "--toChapter", strconv.Itoa(gnargs.ToChapter))
	}
	args = append(args, gnargs.TOCURL)

	cmd := exec.Command(getnovel, args...)

	kindlegenName := `kindlegen`
	if runtime.GOOS == "windows" {
		kindlegenName = `kindlegen.exe`
	}
	kindlegenPath, _ := exec.LookPath(kindlegenName)
	if b, e := fsutil.FileExists(kindlegenPath); e != nil || !b {
		if dir, err := filepath.Abs(filepath.Dir(os.Args[0])); err == nil {
			kindlegenPath = filepath.Join(dir, kindlegenName)
		}
	}
	if !filepath.IsAbs(kindlegenPath) {
		kindlegenPath, _ = filepath.Abs(kindlegenPath)
	}
	cmd.Env = append(os.Environ(),
		"KINDLEGEN_PATH="+kindlegenPath, // ignored
	)
	go func() {
		item := &HistoryItem{
			TOCURL:   gnargs.TOCURL,
			BookName: gnargs.TOCURL,
			Status:   "等待制作",
		}
		books.append(item)

		mutexMaking.Lock()

		item.Status = "制作中"
		if err := cmd.Run(); err != nil {
			item.Status = "制作失败"
		} else {
			item.Status = "有效"

			bookPath := fmt.Sprintf("%s/%s.mobi", item.BookName, item.BookName)
			if b, e := fsutil.FileExists(bookPath); e != nil || !b {
				bookPath = fmt.Sprintf("%s/%s.epub", item.BookName, item.BookName)
			}
			if b, e := fsutil.FileExists(bookPath); e == nil && b {
				baseName := filepath.Base(bookPath)
				item.BookName = baseName
				item.DownloadLink = "/download/" + baseName[:len(baseName)-5] + "/" + baseName
				item.DeleteLink = "/delete/" + baseName[:len(baseName)-5] + "/" + baseName
			}
		}
		mutexMaking.Unlock()

		books.clear()
		scanEbooks()
	}()

	c.JSON(http.StatusOK, gin.H{})
}

func downloadBook(c *gin.Context) {
	path := c.Param("path")
	if path == "pdf" {
		path = ""
	}
	name := c.Param("name")
	c.File(filepath.Join(path, name))
}

func deleteBook(c *gin.Context) {
	path := c.Param("path")
	if path == "pdf" {
		path = ""
	}
	name := c.Param("name")
	os.Remove(filepath.Join(path, name))
	books.delete(name)
	c.Redirect(http.StatusFound, "/")
}

func scanEbooks() {
	matches, err := filepath.Glob("*.pdf")
	if err == nil {
		for _, v := range matches {
			books.append(&HistoryItem{
				BookName:     v,
				Status:       "有效",
				DownloadLink: "/download/pdf/" + v,
				DeleteLink:   "/delete/pdf/" + v,
			})
		}
	}

	matches, err = filepath.Glob("**/*.mobi")
	if err == nil {
		for _, v := range matches {
			books.append(&HistoryItem{
				BookName:     filepath.Base(v),
				Status:       "有效",
				DownloadLink: "/download/" + strings.Replace(v, "\\", "/", -1),
				DeleteLink:   "/delete/" + strings.Replace(v, "\\", "/", -1),
			})
		}
	}
	matches, err = filepath.Glob("**/*.epub")
	if err == nil {
		for _, v := range matches {
			books.append(&HistoryItem{
				BookName:     filepath.Base(v),
				Status:       "有效",
				DownloadLink: "/download/" + strings.Replace(v, "\\", "/", -1),
				DeleteLink:   "/delete/" + strings.Replace(v, "\\", "/", -1),
			})
		}
	}
}

func main() {
	addr := ":8089"
	if bind, ok := os.LookupEnv("BIND"); ok {
		addr = bind
	}
	r := gin.Default()
	r.LoadHTMLGlob("templates/*")
	r.GET("/", homePage)
	r.GET("/download/:path/:name", downloadBook)
	r.GET("/delete/:path/:name", deleteBook)
	r.POST("/makeebook", makeEbook)
	// glob font files
	matches, err := filepath.Glob("fonts/*.ttf")
	if err != nil {
		panic(err)
	}
	for _, v := range matches {
		fontFiles = append(fontFiles, filepath.Base(v))
	}
	matches, err = filepath.Glob("fonts/**/*.ttf")
	for _, v := range matches {
		fontFiles = append(fontFiles, v[6:])
	}
	scanEbooks()
	r.Run(addr)
}
<|endoftext|>


<commit_before>package clock

import "testing"

func TestStringEmpty(t *testing.T) {
	var v vector
	if g, e := v.String(), `{}`; g != e {
		t.Errorf("bad stringer: %#v -> %s != %s", v, g, e)
	}
}

func TestStringSimple(t *testing.T) {
	var v vector
	v.update(10, 1)
	v.update(11, 1)
	v.update(10, 2)
	if g, e := v.String(), `{10:2 11:1}`; g != e {
		t.Errorf("bad stringer: %#v -> %s != %s", v, g, e)
	}
}

func TestMergeSimple(t *testing.T) {
	var a vector
	a.update(10, 1)
	a.update(11, 1)
	a.update(10, 2)
	var b vector
	b.merge(a)
	// trigger bugs if they accidentally share state
	a.update(10, 3)

	if g, e := b.String(), `{10:2 11:1}`; g != e {
		t.Errorf("bad merge: %s != %s", g, e)
	}
}

func TestRebaseSimple(t *testing.T) {
	var a vector
	a.update(10, 1)
	a.update(11, 1)
	var b vector
	b.merge(a)
	a.update(10, 2)
	b.update(12, 3)

	a.rebase(b)

	if g, e := b.String(), `{10:1 11:1 12:3}`; g != e {
		t.Errorf("bad rebase: %s != %s", g, e)
	}
	if g, e := a.String(), `{10:2}`; g != e {
		t.Errorf("bad rebase: %s != %s", g, e)
	}
}

func TestCompareLEEmpty(t *testing.T) {
	var a vector
	var b vector
	if g, e := compareLE(a, b), true; g != e {
		t.Errorf("bad comparison: %s is to %s -> %v != %v", a, b, g, e)
	}
}

func TestCompareLESimple(t *testing.T) {
	var a vector
	a.update(10, 1)
	var b vector
	b.merge(a)
	if g, e := compareLE(a, b), true; g != e {
		t.Errorf("bad comparison: %s is to %s -> %v != %v", a, b, g, e)
	}
}

func TestCompareLEConcurrent(t *testing.T) {
	var a vector
	a.update(10, 1)
	var b vector
	b.update(11, 1)
	if g, e := compareLE(a, b), false; g != e {
		t.Errorf("bad comparison: %s is to %s -> %v != %v", a, b, g, e)
	}
}
<commit_msg>fs/clock: More test coverage<commit_after>package clock

import (
	"path/filepath"
	"runtime"
	"testing"
)

func TestStringEmpty(t *testing.T) {
	var v vector
	if g, e := v.String(), `{}`; g != e {
		t.Errorf("bad stringer: %#v -> %s != %s", v, g, e)
	}
}

func TestStringSimple(t *testing.T) {
	var v vector
	v.update(10, 1)
	v.update(11, 1)
	v.update(10, 2)
	if g, e := v.String(), `{10:2 11:1}`; g != e {
		t.Errorf("bad stringer: %#v -> %s != %s", v, g, e)
	}
}

func TestMergeSimple(t *testing.T) {
	var a vector
	a.update(10, 1)
	a.update(11, 1)
	a.update(10, 2)
	var b vector
	b.merge(a)
	// trigger bugs if they accidentally share state
	a.update(10, 3)

	if g, e := b.String(), `{10:2 11:1}`; g != e {
		t.Errorf("bad merge: %s != %s", g, e)
	}
}

func TestRebaseSimple(t *testing.T) {
	var a vector
	a.update(10, 1)
	a.update(11, 1)
	var b vector
	b.merge(a)
	a.update(10, 2)
	b.update(12, 3)

	a.rebase(b)

	if g, e := b.String(), `{10:1 11:1 12:3}`; g != e {
		t.Errorf("bad rebase: %s != %s", g, e)
	}
	if g, e := a.String(), `{10:2}`; g != e {
		t.Errorf("bad rebase: %s != %s", g, e)
	}
}

func TestCompareLEEmpty(t *testing.T) {
	var a vector
	var b vector
	if g, e := compareLE(a, b), true; g != e {
		t.Errorf("bad comparison: %s is to %s -> %v != %v", a, b, g, e)
	}
}

func TestCompareLESimple(t *testing.T) {
	var a vector
	a.update(10, 1)
	var b vector
	b.merge(a)
	if g, e := compareLE(a, b), true; g != e {
		t.Errorf("bad comparison: %s is to %s -> %v != %v", a, b, g, e)
	}
}

func TestCompareLEConcurrent(t *testing.T) {
	var a vector
	a.update(10, 1)
	var b vector
	b.update(11, 1)
	if g, e := compareLE(a, b), false; g != e {
		t.Errorf("bad comparison: %s is to %s -> %v != %v", a, b, g, e)
	}
}

func le(t testing.TB, a, b vector) {
	if !compareLE(a, b) {
		_, file, line, _ := runtime.Caller(1)
		file = filepath.Base(file)
		t.Errorf("%s:%d: expected a<=b: %s </= %s", file, line, a, b)
	}
}

func nle(t testing.TB, a, b vector) {
	if compareLE(a, b) {
		_, file, line, _ := runtime.Caller(1)
		file = filepath.Base(file)
		t.Errorf("%s:%d: expected a<=b: %s </= %s", file, line, a, b)
	}
}

func TestCompareScenarios(t *testing.T) {
	var a vector
	var b vector
	le(t, a, b)
	b.update(10, 1)
	le(t, a, b)
	nle(t, b, a)
	a.update(10, 1)
	le(t, a, b)
	le(t, b, a)
	a.update(11, 1)
	nle(t, a, b)
	le(t, b, a)
	b.update(10, 2)
	nle(t, a, b)
	nle(t, b, a)
	b.update(11, 1)
	le(t, a, b)
	nle(t, b, a)
	b.update(11, 2)
	le(t, a, b)
	nle(t, b, a)
}

func TestCompareConcurrent(t *testing.T) {
	var a vector
	var b vector
	a.update(10, 1)
	b.merge(a)
	b.update(11, 1)
	a.update(12, 1)
	nle(t, a, b)
	nle(t, b, a)
	a.update(13, 1)
	nle(t, a, b)
	nle(t, b, a)
}

func TestMergeDiverged(t *testing.T) {
	var a vector
	var b vector
	a.update(10, 2)
	a.update(11, 1)
	a.update(12, 1)
	b.update(10, 1)
	b.update(11, 2)
	b.update(13, 1)

	b.merge(a)
	le(t, a, b)
	// make them equal for easy validation
	a.update(11, 2)
	a.update(13, 1)
	le(t, b, a)
}
<|endoftext|>


<commit_before>package jiralert

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"reflect"
	"strings"

	"github.com/andygrunwald/go-jira"
	"github.com/free/jiralert/alertmanager"
	log "github.com/golang/glog"
	"github.com/trivago/tgo/tcontainer"
)

// Receiver wraps a JIRA client corresponding to a specific Alertmanager receiver, with its configuration and templates.
type Receiver struct {
	conf   *ReceiverConfig
	tmpl   *Template
	client *jira.Client
}

// NewReceiver creates a Receiver using the provided configuration and template.
func NewReceiver(c *ReceiverConfig, t *Template) (*Receiver, error) {
	client, err := jira.NewClient(http.DefaultClient, c.APIURL)
	if err != nil {
		return nil, err
	}
	client.Authentication.SetBasicAuth(c.User, string(c.Password))

	return &Receiver{conf: c, tmpl: t, client: client}, nil
}

// Notify implements the Notifier interface.
func (r *Receiver) Notify(data *alertmanager.Data) (bool, error) {
	project := r.tmpl.Execute(r.conf.Project, data)
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	// Looks like an ALERT metric name, with spaces removed.
	issueLabel := toIssueLabel(data.GroupLabels)

	issue, retry, err := r.search(project, issueLabel)
	if err != nil {
		return retry, err
	}

	if issue != nil {
		// The set of JIRA status categories is fixed, this is a safe check to make.
		if issue.Fields.Status.StatusCategory.Key != "done" {
			// Issue is in a "to do" or "in progress" state, all done here.
			log.V(1).Infof("Issue %s for %s is unresolved, nothing to do", issue.Key, issueLabel)
			return false, nil
		}
		if r.conf.WontFixResolution != "" && issue.Fields.Resolution.Name == r.conf.WontFixResolution {
			// Issue is resolved as "Won't Fix" or equivalent, log a message just in case.
			log.Infof("Issue %s for %s is resolved as %q, not reopening", issue.Key, issueLabel, issue.Fields.Resolution.Name)
			return false, nil
		}
		log.Infof("Issue %s for %s was resolved, reopening", issue.Key, issueLabel)
		return r.reopen(issue.Key)
	}

	log.Infof("No issue matching %s found, creating new issue", issueLabel)
	issue = &jira.Issue{
		Fields: &jira.IssueFields{
			Project:     jira.Project{Key: project},
			Type:        jira.IssueType{Name: r.tmpl.Execute(r.conf.IssueType, data)},
			Description: r.tmpl.Execute(r.conf.Description, data),
			Summary:     r.tmpl.Execute(r.conf.Summary, data),
			Labels: []string{
				issueLabel,
			},
			Unknowns: tcontainer.NewMarshalMap(),
		},
	}
	if r.conf.Priority != "" {
		issue.Fields.Priority = &jira.Priority{Name: r.tmpl.Execute(r.conf.Priority, data)}
	}

	// Add Components
	if len(r.conf.Components) > 0 {
		issue.Fields.Components = make([]*jira.Component, 0, len(r.conf.Components))
		for _, component := range r.conf.Components {
			issue.Fields.Components = append(issue.Fields.Components, &jira.Component{Name: component})
		}
	}

	// Add Labels
	if r.conf.AddGroupLabels {
		for k, v := range data.GroupLabels {
			issue.Fields.Labels = append(issue.Fields.Labels, fmt.Sprintf("%s=%q", k, v))
		}
	}

	for key, value := range r.conf.Fields {
		issue.Fields.Unknowns[key] = deepCopyWithTemplate(value, r.tmpl, data)
	}
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	retry, err = r.create(issue)
	if err != nil {
		log.Infof("Issue created: key=%s ID=%s", issue.Key, issue.ID)
	}
	return retry, err
}

// deepCopyWithTemplate returns a deep copy of a map/slice/array/string/int/bool or combination thereof, executing the
// provided template (with the provided data) on all string keys or values. All maps are connverted to
// map[string]interface{}, with all non-string keys discarded.
func deepCopyWithTemplate(value interface{}, tmpl *Template, data interface{}) interface{} {
	if value == nil {
		return value
	}

	valueMeta := reflect.ValueOf(value)
	switch valueMeta.Kind() {

	case reflect.String:
		return tmpl.Execute(value.(string), data)

	case reflect.Array, reflect.Slice:
		arrayLen := valueMeta.Len()
		converted := make([]interface{}, arrayLen)
		for i := 0; i < arrayLen; i++ {
			converted[i] = deepCopyWithTemplate(valueMeta.Index(i).Interface(), tmpl, data)
		}
		return converted

	case reflect.Map:
		keys := valueMeta.MapKeys()
		converted := make(map[string]interface{}, len(keys))

		for _, keyMeta := range keys {
			strKey, isString := keyMeta.Interface().(string)
			if !isString {
				continue
			}
			strKey = tmpl.Execute(strKey, data)
			converted[strKey] = deepCopyWithTemplate(valueMeta.MapIndex(keyMeta).Interface(), tmpl, data)
		}
		return converted

	default:
		return value
	}
}

// toIssueLabel returns the group labels in the form of an ALERT metric name, with all spaces removed.
func toIssueLabel(groupLabels alertmanager.KV) string {
	buf := bytes.NewBufferString("ALERT{")
	for _, p := range groupLabels.SortedPairs() {
		buf.WriteString(p.Name)
		buf.WriteString(fmt.Sprintf("=%q,", p.Value))
	}
	buf.Truncate(buf.Len() - 1)
	buf.WriteString("}")
	return strings.Replace(buf.String(), " ", "", -1)
}

func (r *Receiver) search(project, issueLabel string) (*jira.Issue, bool, error) {
	query := fmt.Sprintf("project=%s and labels=%q order by key", project, issueLabel)
	options := &jira.SearchOptions{
		Fields:     []string{"summary", "status", "resolution"},
		MaxResults: 50,
	}
	log.V(1).Infof("search: query=%v options=%+v", query, options)
	issues, resp, err := r.client.Issue.Search(query, options)
	if err != nil {
		retry, err := handleJiraError("Issue.Search", resp, err)
		return nil, retry, err
	}
	if len(issues) > 0 {
		if len(issues) > 1 {
			// Swallow it, but log an error.
			log.Errorf("More than one issue matched %s, will only update first: %+v", query, issues)
		}
		log.V(1).Infof("  found: %+v", issues[0])
		return &issues[0], false, nil
	}
	log.V(1).Infof("  no results")
	return nil, false, nil
}

func (r *Receiver) reopen(issueKey string) (bool, error) {
	transitions, resp, err := r.client.Issue.GetTransitions(issueKey)
	if err != nil {
		return handleJiraError("Issue.GetTransitions", resp, err)
	}
	for _, t := range transitions {
		if t.Name == r.conf.ReopenState {
			log.V(1).Infof("reopen: issueKey=%v transitionID=%v", issueKey, t.ID)
			resp, err = r.client.Issue.DoTransition(issueKey, t.ID)
			if err != nil {
				return handleJiraError("Issue.DoTransition", resp, err)
			}
			log.V(1).Infof("  done")
			return false, nil
		}
	}
	return false, fmt.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
}

func (r *Receiver) create(issue *jira.Issue) (bool, error) {
	log.V(1).Infof("create: issue=%v", *issue)
	issue, resp, err := r.client.Issue.Create(issue)
	if err != nil {
		return handleJiraError("Issue.Create", resp, err)
	}

	log.V(1).Infof("  done: key=%s ID=%s", issue.Key, issue.ID)
	return false, nil
}

func handleJiraError(api string, resp *jira.Response, err error) (bool, error) {
	if resp == nil || resp.Request == nil {
		log.V(1).Infof("handleJiraError: api=%s, err=%s", api, err)
	} else {
		log.V(1).Infof("handleJiraError: api=%s, url=%s, err=%s", api, resp.Request.URL, err)
	}

	if resp != nil && resp.StatusCode/100 != 2 {
		retry := resp.StatusCode == 500 || resp.StatusCode == 503
		body, _ := ioutil.ReadAll(resp.Body)
		// go-jira error message is not particularly helpful, replace it
		return retry, fmt.Errorf("JIRA request %s returned status %s, body %q", resp.Request.URL, resp.Status, string(body))
	}
	return false, fmt.Errorf("JIRA request %s failed: %s", api, err)
}
<commit_msg>Attempt to fix #6 by checking for nil resolution in the returned JIRA issue.<commit_after>package jiralert

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"reflect"
	"strings"

	"github.com/andygrunwald/go-jira"
	"github.com/free/jiralert/alertmanager"
	log "github.com/golang/glog"
	"github.com/trivago/tgo/tcontainer"
)

// Receiver wraps a JIRA client corresponding to a specific Alertmanager receiver, with its configuration and templates.
type Receiver struct {
	conf   *ReceiverConfig
	tmpl   *Template
	client *jira.Client
}

// NewReceiver creates a Receiver using the provided configuration and template.
func NewReceiver(c *ReceiverConfig, t *Template) (*Receiver, error) {
	client, err := jira.NewClient(http.DefaultClient, c.APIURL)
	if err != nil {
		return nil, err
	}
	client.Authentication.SetBasicAuth(c.User, string(c.Password))

	return &Receiver{conf: c, tmpl: t, client: client}, nil
}

// Notify implements the Notifier interface.
func (r *Receiver) Notify(data *alertmanager.Data) (bool, error) {
	project := r.tmpl.Execute(r.conf.Project, data)
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	// Looks like an ALERT metric name, with spaces removed.
	issueLabel := toIssueLabel(data.GroupLabels)

	issue, retry, err := r.search(project, issueLabel)
	if err != nil {
		return retry, err
	}

	if issue != nil {
		// The set of JIRA status categories is fixed, this is a safe check to make.
		if issue.Fields.Status.StatusCategory.Key != "done" {
			// Issue is in a "to do" or "in progress" state, all done here.
			log.V(1).Infof("Issue %s for %s is unresolved, nothing to do", issue.Key, issueLabel)
			return false, nil
		}
		if r.conf.WontFixResolution != "" && issue.Fields.Resolution != nil &&
			issue.Fields.Resolution.Name == r.conf.WontFixResolution {
			// Issue is resolved as "Won't Fix" or equivalent, log a message just in case.
			log.Infof("Issue %s for %s is resolved as %q, not reopening", issue.Key, issueLabel, issue.Fields.Resolution.Name)
			return false, nil
		}
		log.Infof("Issue %s for %s was resolved, reopening", issue.Key, issueLabel)
		return r.reopen(issue.Key)
	}

	log.Infof("No issue matching %s found, creating new issue", issueLabel)
	issue = &jira.Issue{
		Fields: &jira.IssueFields{
			Project:     jira.Project{Key: project},
			Type:        jira.IssueType{Name: r.tmpl.Execute(r.conf.IssueType, data)},
			Description: r.tmpl.Execute(r.conf.Description, data),
			Summary:     r.tmpl.Execute(r.conf.Summary, data),
			Labels: []string{
				issueLabel,
			},
			Unknowns: tcontainer.NewMarshalMap(),
		},
	}
	if r.conf.Priority != "" {
		issue.Fields.Priority = &jira.Priority{Name: r.tmpl.Execute(r.conf.Priority, data)}
	}

	// Add Components
	if len(r.conf.Components) > 0 {
		issue.Fields.Components = make([]*jira.Component, 0, len(r.conf.Components))
		for _, component := range r.conf.Components {
			issue.Fields.Components = append(issue.Fields.Components, &jira.Component{Name: component})
		}
	}

	// Add Labels
	if r.conf.AddGroupLabels {
		for k, v := range data.GroupLabels {
			issue.Fields.Labels = append(issue.Fields.Labels, fmt.Sprintf("%s=%q", k, v))
		}
	}

	for key, value := range r.conf.Fields {
		issue.Fields.Unknowns[key] = deepCopyWithTemplate(value, r.tmpl, data)
	}
	// check errors from r.tmpl.Execute()
	if r.tmpl.err != nil {
		return false, r.tmpl.err
	}
	retry, err = r.create(issue)
	if err != nil {
		log.Infof("Issue created: key=%s ID=%s", issue.Key, issue.ID)
	}
	return retry, err
}

// deepCopyWithTemplate returns a deep copy of a map/slice/array/string/int/bool or combination thereof, executing the
// provided template (with the provided data) on all string keys or values. All maps are connverted to
// map[string]interface{}, with all non-string keys discarded.
func deepCopyWithTemplate(value interface{}, tmpl *Template, data interface{}) interface{} {
	if value == nil {
		return value
	}

	valueMeta := reflect.ValueOf(value)
	switch valueMeta.Kind() {

	case reflect.String:
		return tmpl.Execute(value.(string), data)

	case reflect.Array, reflect.Slice:
		arrayLen := valueMeta.Len()
		converted := make([]interface{}, arrayLen)
		for i := 0; i < arrayLen; i++ {
			converted[i] = deepCopyWithTemplate(valueMeta.Index(i).Interface(), tmpl, data)
		}
		return converted

	case reflect.Map:
		keys := valueMeta.MapKeys()
		converted := make(map[string]interface{}, len(keys))

		for _, keyMeta := range keys {
			strKey, isString := keyMeta.Interface().(string)
			if !isString {
				continue
			}
			strKey = tmpl.Execute(strKey, data)
			converted[strKey] = deepCopyWithTemplate(valueMeta.MapIndex(keyMeta).Interface(), tmpl, data)
		}
		return converted

	default:
		return value
	}
}

// toIssueLabel returns the group labels in the form of an ALERT metric name, with all spaces removed.
func toIssueLabel(groupLabels alertmanager.KV) string {
	buf := bytes.NewBufferString("ALERT{")
	for _, p := range groupLabels.SortedPairs() {
		buf.WriteString(p.Name)
		buf.WriteString(fmt.Sprintf("=%q,", p.Value))
	}
	buf.Truncate(buf.Len() - 1)
	buf.WriteString("}")
	return strings.Replace(buf.String(), " ", "", -1)
}

func (r *Receiver) search(project, issueLabel string) (*jira.Issue, bool, error) {
	query := fmt.Sprintf("project=%s and labels=%q order by key", project, issueLabel)
	options := &jira.SearchOptions{
		Fields:     []string{"summary", "status", "resolution"},
		MaxResults: 50,
	}
	log.V(1).Infof("search: query=%v options=%+v", query, options)
	issues, resp, err := r.client.Issue.Search(query, options)
	if err != nil {
		retry, err := handleJiraError("Issue.Search", resp, err)
		return nil, retry, err
	}
	if len(issues) > 0 {
		if len(issues) > 1 {
			// Swallow it, but log an error.
			log.Errorf("More than one issue matched %s, will only update first: %+v", query, issues)
		}
		log.V(1).Infof("  found: %+v", issues[0])
		return &issues[0], false, nil
	}
	log.V(1).Infof("  no results")
	return nil, false, nil
}

func (r *Receiver) reopen(issueKey string) (bool, error) {
	transitions, resp, err := r.client.Issue.GetTransitions(issueKey)
	if err != nil {
		return handleJiraError("Issue.GetTransitions", resp, err)
	}
	for _, t := range transitions {
		if t.Name == r.conf.ReopenState {
			log.V(1).Infof("reopen: issueKey=%v transitionID=%v", issueKey, t.ID)
			resp, err = r.client.Issue.DoTransition(issueKey, t.ID)
			if err != nil {
				return handleJiraError("Issue.DoTransition", resp, err)
			}
			log.V(1).Infof("  done")
			return false, nil
		}
	}
	return false, fmt.Errorf("JIRA state %q does not exist or no transition possible for %s", r.conf.ReopenState, issueKey)
}

func (r *Receiver) create(issue *jira.Issue) (bool, error) {
	log.V(1).Infof("create: issue=%v", *issue)
	issue, resp, err := r.client.Issue.Create(issue)
	if err != nil {
		return handleJiraError("Issue.Create", resp, err)
	}

	log.V(1).Infof("  done: key=%s ID=%s", issue.Key, issue.ID)
	return false, nil
}

func handleJiraError(api string, resp *jira.Response, err error) (bool, error) {
	if resp == nil || resp.Request == nil {
		log.V(1).Infof("handleJiraError: api=%s, err=%s", api, err)
	} else {
		log.V(1).Infof("handleJiraError: api=%s, url=%s, err=%s", api, resp.Request.URL, err)
	}

	if resp != nil && resp.StatusCode/100 != 2 {
		retry := resp.StatusCode == 500 || resp.StatusCode == 503
		body, _ := ioutil.ReadAll(resp.Body)
		// go-jira error message is not particularly helpful, replace it
		return retry, fmt.Errorf("JIRA request %s returned status %s, body %q", resp.Request.URL, resp.Status, string(body))
	}
	return false, fmt.Errorf("JIRA request %s failed: %s", api, err)
}
<|endoftext|>


<commit_before>package table

import (
	"bytes"

	gram "github.com/ghigt/gocyk/grammar"
)

type Item []gram.Token

func (item *Item) Add(t ...gram.Token) *Item {
	if item == nil {
		item = new(Item)
	}
	*item = append(*item, t...)
	return item
}

func (item *Item) GetTokens() []gram.Token {
	return *item
}

func (item *Item) Empty() bool {
	if len(*item) == 0 {
		return true
	}
	return false
}

func (item *Item) String() string {
	var buf bytes.Buffer

	if item == nil {
		return "nil"
	}
	buf.WriteString("{ ")
	for _, i := range *item {
		buf.WriteString(string(i) + " ")
	}
	buf.WriteString("}")

	return buf.String()
}
<commit_msg>add comments to table/item.go<commit_after>package table

import (
	"bytes"

	gram "github.com/ghigt/gocyk/grammar"
)

// Item type represents a slice of tokens for the recognition table.
type Item []gram.Token

// Add adds multiple token at the end of the item.
func (item *Item) Add(t ...gram.Token) *Item {
	if item == nil {
		item = new(Item)
	}
	*item = append(*item, t...)
	return item
}

// GetTokens returns all the tokens of the item.
func (item *Item) GetTokens() []gram.Token {
	return *item
}

// Empty checks if the item is empty or not.
func (item *Item) Empty() bool {
	if len(*item) == 0 {
		return true
	}
	return false
}

func (item *Item) String() string {
	var buf bytes.Buffer

	if item == nil {
		return "nil"
	}
	buf.WriteString("{ ")
	for _, i := range *item {
		buf.WriteString(string(i) + " ")
	}
	buf.WriteString("}")

	return buf.String()
}
<|endoftext|>


<commit_before>package git

import (
	"bytes"
	"crypto/sha1"
	"strconv"
)

type Type int
const (
	BlobType Type = iota
	TreeType
	CommitType
	TagType
)

type Object interface {
	Type() Type
	Raw() []byte
	Id() Id
}

// Returns an object ID given an object header an the object's content.
func idHelper(header string, content []byte) Id {
	h := sha1.New()
	h.Write([]byte(header + " "))
	h.Write([]byte(strconv.Itoa(len(content))))
	h.Write([]byte{0})
	h.Write(content)
	return IdFromBytes(h.Sum())
}

type Blob struct {
	raw []byte
}

func NewBlob(raw []byte) *Blob {
	return &Blob{raw}
}

func (b *Blob) Type() Type { return BlobType }

func (b *Blob) Id() Id { return idHelper("blob", b.Raw()) }

func (b *Blob) Raw() []byte {
	return b.raw
}

type Tree struct {
	names []string
	children []Id
}

func NewTree(cap int) *Tree {
	t := &Tree{}
	if cap > 0 {
		t.names = make([]string, 0, cap)
		t.children = make([]Id, 0, cap)
	}
	return t
}

func (c *Tree) Type() Type { return TreeType }

func (t *Tree) Add(name string, child Id) {
	t.names = append(t.names, name)
	t.children = append(t.children, child)
}

func (t *Tree) Id() Id { return idHelper("tree", t.Raw()) }

func (t *Tree) Raw() []byte {
	content := bytes.NewBuffer(nil)
	// TODO: Print out the children in sorted order
	for i := range t.names {
		// TODO: Fix permissions
		content.WriteString("100644 ")
		content.WriteString(t.names[i])
		content.WriteByte('\x00')
		content.WriteString(t.children[i].String())
	}
	return content.Bytes()
}

type time struct {
	seconds int64
	offset int // time zone offset in minutes
}

func (t *time) String() string {
	if t == nil {
		return ""
	}
	pre := " "
	offset := t.offset
	if t.offset < 0 {
		pre += "-"
		offset = -t.offset
	}
	tz := strconv.Itoa(offset)
	for i := 0; len(tz) + i < 4; i++ {
		pre += "0"
	}
	return strconv.Itoa64(t.seconds) + pre + tz
}

type Commit struct {
	authorName string
	authorEmail string
	authorTime *time
	committerName string
	committerEmail string
	committerTime *time
	tree Id
	parents []Id
	msg string
}

func NewCommit(authorName, authorEmail string, authorTime *time, committerName, committerEmail string, commitTime *time, tree Id, parents []Id, msg string) *Commit {
	// TODO: Set unset things
	return &Commit{authorName, authorEmail, authorTime, committerName, committerEmail, commitTime, tree, parents, msg}
}

func NewCommitSimple(name, email string, time *time, tree Id, parent Id) *Commit {
	return &Commit{name, email, time, name, email, time, tree, []Id{parent}, "empty message"}
}

func (c *Commit) Type() Type { return CommitType }

func (c *Commit) Id() Id { return idHelper("commit", c.Raw()) }

func (c *Commit) Raw() []byte {
	content := "tree " + c.tree.String()
	for i := range c.parents {
		content += "\nparent " + c.parents[i].String()
	}
	content += "\nauthor " + c.authorName + " <" + c.authorEmail + "> " + c.authorTime.String()
	content += "\ncommitter " + c.committerName + " <" + c.committerEmail + "> " + c.committerTime.String() + "\n\n"
	content += c.msg
	return []byte(content)
}
<commit_msg>Remove Type type and Type method from objects<commit_after>package git

import (
	"bytes"
	"crypto/sha1"
	"strconv"
)

type Object interface {
	Raw() []byte
	Id() Id
}

// Returns an object ID given an object header an the object's content.
func idHelper(header string, content []byte) Id {
	h := sha1.New()
	h.Write([]byte(header + " "))
	h.Write([]byte(strconv.Itoa(len(content))))
	h.Write([]byte{0})
	h.Write(content)
	return IdFromBytes(h.Sum())
}

type Blob struct {
	raw []byte
}

func NewBlob(raw []byte) *Blob {
	return &Blob{raw}
}

func (b *Blob) Id() Id { return idHelper("blob", b.Raw()) }

func (b *Blob) Raw() []byte {
	return b.raw
}

type Tree struct {
	names []string
	children []Id
}

func NewTree(cap int) *Tree {
	t := &Tree{}
	if cap > 0 {
		t.names = make([]string, 0, cap)
		t.children = make([]Id, 0, cap)
	}
	return t
}

func (t *Tree) Add(name string, child Id) {
	t.names = append(t.names, name)
	t.children = append(t.children, child)
}

func (t *Tree) Id() Id { return idHelper("tree", t.Raw()) }

func (t *Tree) Raw() []byte {
	content := bytes.NewBuffer(nil)
	// TODO: Print out the children in sorted order
	for i := range t.names {
		// TODO: Fix permissions
		content.WriteString("100644 ")
		content.WriteString(t.names[i])
		content.WriteByte('\x00')
		content.WriteString(t.children[i].String())
	}
	return content.Bytes()
}

type time struct {
	seconds int64
	offset int // time zone offset in minutes
}

func (t *time) String() string {
	if t == nil {
		return ""
	}
	pre := " "
	offset := t.offset
	if t.offset < 0 {
		pre += "-"
		offset = -t.offset
	}
	tz := strconv.Itoa(offset)
	for i := 0; len(tz) + i < 4; i++ {
		pre += "0"
	}
	return strconv.Itoa64(t.seconds) + pre + tz
}

type Commit struct {
	authorName string
	authorEmail string
	authorTime *time
	committerName string
	committerEmail string
	committerTime *time
	tree Id
	parents []Id
	msg string
}

func NewCommit(authorName, authorEmail string, authorTime *time, committerName, committerEmail string, commitTime *time, tree Id, parents []Id, msg string) *Commit {
	// TODO: Set unset things
	return &Commit{authorName, authorEmail, authorTime, committerName, committerEmail, commitTime, tree, parents, msg}
}

func NewCommitSimple(name, email string, time *time, tree Id, parent Id) *Commit {
	return &Commit{name, email, time, name, email, time, tree, []Id{parent}, "empty message"}
}

func (c *Commit) Id() Id { return idHelper("commit", c.Raw()) }

func (c *Commit) Raw() []byte {
	content := "tree " + c.tree.String()
	for i := range c.parents {
		content += "\nparent " + c.parents[i].String()
	}
	content += "\nauthor " + c.authorName + " <" + c.authorEmail + "> " + c.authorTime.String()
	content += "\ncommitter " + c.committerName + " <" + c.committerEmail + "> " + c.committerTime.String() + "\n\n"
	content += c.msg
	return []byte(content)
}
<|endoftext|>


<commit_before>package ParquetReader

import (
	"fmt"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
)

type ColumnBufferType struct {
	PFile        ParquetFile.ParquetFile
	ThriftReader *thrift.TBufferedTransport

	Footer        *parquet.FileMetaData
	SchemaHandler *SchemaHandler.SchemaHandler

	PathStr       string
	RowGroupIndex int64
	ChunkHeader   *parquet.ColumnChunk

	ChunkReadValues int64

	DictPage *Layout.Page

	DataTable        *Layout.Table
	DataTableNumRows int64
}

func NewColumnBuffer(pFile ParquetFile.ParquetFile, footer *parquet.FileMetaData, schemaHandler *SchemaHandler.SchemaHandler, pathStr string) (*ColumnBufferType, error) {
	newPFile, err := pFile.Open("")
	if err != nil {
		return nil, err
	}
	res := &ColumnBufferType{
		PFile:            newPFile,
		Footer:           footer,
		SchemaHandler:    schemaHandler,
		PathStr:          pathStr,
		DataTableNumRows: -1,
	}
	err = res.NextRowGroup()
	return res, err
}

func (self *ColumnBufferType) NextRowGroup() error {
	var err error
	rowGroups := self.Footer.GetRowGroups()
	ln := int64(len(rowGroups))
	if self.RowGroupIndex >= ln {
		self.DataTableNumRows++ //very important, because DataTableNumRows is one smaller than real rows number
		return fmt.Errorf("End of row groups")
	}
	self.RowGroupIndex++

	columnChunks := rowGroups[self.RowGroupIndex-1].GetColumns()
	i := int64(0)
	ln = int64(len(columnChunks))
	for i = 0; i < ln; i++ {
		path := make([]string, 0)
		path = append(path, self.SchemaHandler.GetRootName())
		path = append(path, columnChunks[i].MetaData.GetPathInSchema()...)
		if self.PathStr == Common.PathToStr(path) {
			break
		}
	}
	if i >= ln {
		return fmt.Errorf("Column not found: %v", self.PathStr)
	}

	self.ChunkHeader = columnChunks[i]
	if columnChunks[i].FilePath != nil {
		self.PFile.Close()
		if self.PFile, err = self.PFile.Open(*columnChunks[i].FilePath); err != nil {
			return err
		}
	}
	//offset := columnChunks[i].FileOffset
	offset := columnChunks[i].MetaData.DataPageOffset
	if columnChunks[i].MetaData.DictionaryPageOffset != nil {
		offset = *columnChunks[i].MetaData.DictionaryPageOffset
	}
	size := columnChunks[i].MetaData.GetTotalCompressedSize()
	if self.ThriftReader != nil {
		self.ThriftReader.Close()
	}
	self.ThriftReader = ParquetFile.ConvertToThriftReader(self.PFile, offset, size)
	self.ChunkReadValues = 0
	self.DictPage = nil
	return nil
}

func (self *ColumnBufferType) ReadPage() error {
	if self.ChunkReadValues < self.ChunkHeader.MetaData.NumValues {
		page, numValues, numRows, err := Layout.ReadPage(self.ThriftReader, self.SchemaHandler, self.ChunkHeader.MetaData)
		if err != nil {
			return err
		}
		if page.Header.GetType() == parquet.PageType_DICTIONARY_PAGE {
			self.DictPage = page
			return nil
		}

		page.Decode(self.DictPage)

		if self.DataTable == nil {
			self.DataTable = Layout.NewTableFromTable(page.DataTable)
		}
		self.DataTable.Merge(page.DataTable)
		self.ChunkReadValues += numValues

		self.DataTableNumRows += numRows
	} else {
		if err := self.NextRowGroup(); err != nil {
			return err
		}
		return self.ReadPage()
	}

	return nil
}

func (self *ColumnBufferType) ReadPageForSkip() (*Layout.Page, error) {
	if self.ChunkReadValues < self.ChunkHeader.MetaData.NumValues {
		page, err := Layout.ReadPageRawData(self.ThriftReader, self.SchemaHandler, self.ChunkHeader.MetaData)
		if err != nil {
			return nil, err
		}
		numValues, numRows, err := page.GetRLDLFromRawData(self.SchemaHandler)
		if err != nil {
			return nil, err
		}
		if page.Header.GetType() == parquet.PageType_DICTIONARY_PAGE {
			page.GetValueFromRawData(self.SchemaHandler)
			self.DictPage = page
			return page, nil
		}
		if self.DataTable == nil {
			self.DataTable = Layout.NewTableFromTable(page.DataTable)
		}
		self.DataTable.Merge(page.DataTable)
		self.ChunkReadValues += numValues
		self.DataTableNumRows += numRows
		return page, nil

	} else {
		if err := self.NextRowGroup(); err != nil {
			return nil, err
		}
		return self.ReadPageForSkip()
	}
}

func (self *ColumnBufferType) SkipRows(num int64) int64 {
	var (
		err  error
		page *Layout.Page
	)

	for self.DataTableNumRows < num && err == nil {
		page, err = self.ReadPageForSkip()
	}
	if num > self.DataTableNumRows {
		num = self.DataTableNumRows
	}
	if page != nil {
		if err = page.GetValueFromRawData(self.SchemaHandler); err != nil {
			return 0
		}
		page.Decode(self.DictPage)
		i, j := len(self.DataTable.Values)-1, len(page.DataTable.Values)-1
		for i >= 0 && j >= 0 {
			self.DataTable.Values[i] = page.DataTable.Values[j]
			i, j = i-1, j-1
		}
	}

	self.DataTable.Pop(num)
	self.DataTableNumRows -= num
	if self.DataTableNumRows <= 0 {
		tmp := self.DataTable
		self.DataTable = Layout.NewTableFromTable(tmp)
		self.DataTable.Merge(tmp)
	}
	return num
}

func (self *ColumnBufferType) ReadRows(num int64) (*Layout.Table, int64) {
	var err error

	for self.DataTableNumRows < num && err == nil {
		err = self.ReadPage()
	}

	if num > self.DataTableNumRows {
		num = self.DataTableNumRows
	}
	res := self.DataTable.Pop(num)
	self.DataTableNumRows -= num

	if self.DataTableNumRows <= 0 { //release previous slice memory
		tmp := self.DataTable
		self.DataTable = Layout.NewTableFromTable(tmp)
		self.DataTable.Merge(tmp)
	}
	return res, num

}
<commit_msg>fix nil datapage bug<commit_after>package ParquetReader

import (
	"fmt"

	"github.com/apache/thrift/lib/go/thrift"
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/ParquetFile"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
)

type ColumnBufferType struct {
	PFile        ParquetFile.ParquetFile
	ThriftReader *thrift.TBufferedTransport

	Footer        *parquet.FileMetaData
	SchemaHandler *SchemaHandler.SchemaHandler

	PathStr       string
	RowGroupIndex int64
	ChunkHeader   *parquet.ColumnChunk

	ChunkReadValues int64

	DictPage *Layout.Page

	DataTable        *Layout.Table
	DataTableNumRows int64
}

func NewColumnBuffer(pFile ParquetFile.ParquetFile, footer *parquet.FileMetaData, schemaHandler *SchemaHandler.SchemaHandler, pathStr string) (*ColumnBufferType, error) {
	newPFile, err := pFile.Open("")
	if err != nil {
		return nil, err
	}
	res := &ColumnBufferType{
		PFile:            newPFile,
		Footer:           footer,
		SchemaHandler:    schemaHandler,
		PathStr:          pathStr,
		DataTableNumRows: -1,
	}
	err = res.NextRowGroup()
	return res, err
}

func (self *ColumnBufferType) NextRowGroup() error {
	var err error
	rowGroups := self.Footer.GetRowGroups()
	ln := int64(len(rowGroups))
	if self.RowGroupIndex >= ln {
		self.DataTableNumRows++ //very important, because DataTableNumRows is one smaller than real rows number
		return fmt.Errorf("End of row groups")
	}
	self.RowGroupIndex++

	columnChunks := rowGroups[self.RowGroupIndex-1].GetColumns()
	i := int64(0)
	ln = int64(len(columnChunks))
	for i = 0; i < ln; i++ {
		path := make([]string, 0)
		path = append(path, self.SchemaHandler.GetRootName())
		path = append(path, columnChunks[i].MetaData.GetPathInSchema()...)
		if self.PathStr == Common.PathToStr(path) {
			break
		}
	}
	if i >= ln {
		return fmt.Errorf("Column not found: %v", self.PathStr)
	}

	self.ChunkHeader = columnChunks[i]
	if columnChunks[i].FilePath != nil {
		self.PFile.Close()
		if self.PFile, err = self.PFile.Open(*columnChunks[i].FilePath); err != nil {
			return err
		}
	}
	//offset := columnChunks[i].FileOffset
	offset := columnChunks[i].MetaData.DataPageOffset
	if columnChunks[i].MetaData.DictionaryPageOffset != nil {
		offset = *columnChunks[i].MetaData.DictionaryPageOffset
	}
	size := columnChunks[i].MetaData.GetTotalCompressedSize()
	if self.ThriftReader != nil {
		self.ThriftReader.Close()
	}
	self.ThriftReader = ParquetFile.ConvertToThriftReader(self.PFile, offset, size)
	self.ChunkReadValues = 0
	self.DictPage = nil
	return nil
}

func (self *ColumnBufferType) ReadPage() error {
	if self.ChunkReadValues < self.ChunkHeader.MetaData.NumValues {
		page, numValues, numRows, err := Layout.ReadPage(self.ThriftReader, self.SchemaHandler, self.ChunkHeader.MetaData)
		if err != nil {
			return err
		}
		if page.Header.GetType() == parquet.PageType_DICTIONARY_PAGE {
			self.DictPage = page
			return nil
		}

		page.Decode(self.DictPage)

		if self.DataTable == nil {
			self.DataTable = Layout.NewTableFromTable(page.DataTable)
		}
		self.DataTable.Merge(page.DataTable)
		self.ChunkReadValues += numValues

		self.DataTableNumRows += numRows
	} else {
		if err := self.NextRowGroup(); err != nil {
			return err
		}
		return self.ReadPage()
	}

	return nil
}

func (self *ColumnBufferType) ReadPageForSkip() (*Layout.Page, error) {
	if self.ChunkReadValues < self.ChunkHeader.MetaData.NumValues {
		page, err := Layout.ReadPageRawData(self.ThriftReader, self.SchemaHandler, self.ChunkHeader.MetaData)
		if err != nil {
			return nil, err
		}
		numValues, numRows, err := page.GetRLDLFromRawData(self.SchemaHandler)
		if err != nil {
			return nil, err
		}
		if page.Header.GetType() == parquet.PageType_DICTIONARY_PAGE {
			page.GetValueFromRawData(self.SchemaHandler)
			self.DictPage = page
			return page, nil
		}
		if self.DataTable == nil {
			self.DataTable = Layout.NewTableFromTable(page.DataTable)
		}
		self.DataTable.Merge(page.DataTable)
		self.ChunkReadValues += numValues
		self.DataTableNumRows += numRows
		return page, nil

	} else {
		if err := self.NextRowGroup(); err != nil {
			return nil, err
		}
		return self.ReadPageForSkip()
	}
}

func (self *ColumnBufferType) SkipRows(num int64) int64 {
	var (
		err  error
		page *Layout.Page
	)

	for self.DataTableNumRows < num && err == nil {
		page, err = self.ReadPageForSkip()
	}
	if num > self.DataTableNumRows {
		num = self.DataTableNumRows
	}
	if page != nil {
		if err = page.GetValueFromRawData(self.SchemaHandler); err != nil {
			return 0
		}
		page.Decode(self.DictPage)
		i, j := len(self.DataTable.Values)-1, len(page.DataTable.Values)-1
		for i >= 0 && j >= 0 {
			self.DataTable.Values[i] = page.DataTable.Values[j]
			i, j = i-1, j-1
		}
	}

	self.DataTable.Pop(num)
	self.DataTableNumRows -= num
	if self.DataTableNumRows <= 0 {
		tmp := self.DataTable
		self.DataTable = Layout.NewTableFromTable(tmp)
		self.DataTable.Merge(tmp)
	}
	return num
}

func (self *ColumnBufferType) ReadRows(num int64) (*Layout.Table, int64) {
	var err error

	for self.DataTableNumRows < num && err == nil {
		err = self.ReadPage()
	}

	if self.DataTableNumRows < 0 {
		self.DataTableNumRows = 0
		self.DataTable = Layout.NewEmptyTable()
	}

	if num > self.DataTableNumRows {
		num = self.DataTableNumRows
	}

	res := self.DataTable.Pop(num)
	self.DataTableNumRows -= num

	if self.DataTableNumRows <= 0 { //release previous slice memory
		tmp := self.DataTable
		self.DataTable = Layout.NewTableFromTable(tmp)
		self.DataTable.Merge(tmp)
	}
	return res, num

}
<|endoftext|>


<commit_before>package shippers

import "bytes"
import "encoding/json"
import "fmt"
import "net/http"
import "github.com/Sirupsen/logrus"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/josegonzalez/metricsd/utils"
import "github.com/vaughan0/go-ini"

type actionMap map[string]indexMap
type indexMap map[string]string

// LogstashElasticsearchShipper is an exported type that
// allows shipping metrics to elasticsearch in logstash format
type LogstashElasticsearchShipper struct {
	enabled    bool
	index      string
	metricType string
	url        string
}

// Enabled allows checking whether the shipper is enabled or not
func (s *LogstashElasticsearchShipper) Enabled() bool {
	return s.enabled
}

// State allows setting the enabled state of the shipper
func (s *LogstashElasticsearchShipper) State(state bool) {
	s.enabled = state
}

// Setup configures the shipper
func (s *LogstashElasticsearchShipper) Setup(conf ini.File) {
	s.State(true)

	if url, ok := conf.Get("LogstashElasticsearchShipper", "url"); ok {
		s.url = url
	} else {
		s.url = "http://127.0.0.1:9200"
	}

	if index, ok := conf.Get("LogstashElasticsearchShipper", "enabled"); ok {
		s.index = index
	} else {
		s.index = "metricsd-data"
	}

	if metricType, ok := conf.Get("LogstashElasticsearchShipper", "type"); ok {
		s.metricType = metricType
	} else {
		s.metricType = "metricsd"
	}

	s.setupTemplate()
}

// Ship sends a list of MetricSlices to elasticsearch
func (s *LogstashElasticsearchShipper) Ship(logs structs.MetricSlice) error {
	action := actionMap{
		"index": indexMap{
			"_index": s.index,
			"_type":  s.metricType,
		},
	}
	serializedAction, err := json.Marshal(action)
	if err != nil {
		return fmt.Errorf("Failed to marshal action to JSON, %v", err)
	}

	var slice []byte
	newline := []byte("\n")

	for _, item := range logs {
		serialized := item.ToJSON()
		slice = utils.Extend(slice, serializedAction)
		slice = utils.Extend(slice, newline)
		slice = utils.Extend(slice, serialized)
		slice = utils.Extend(slice, newline)
	}

	status, err := s.elasticsearchPost("/_bulk", slice)
	if err != nil {
		logrus.Warning("indexing serialized data failed with err: ", err)
	}

	if status != http.StatusOK {
		logrus.Warning("indexing serialized data failed with status: ", status)
	}
	return nil
}

func (s *LogstashElasticsearchShipper) elasticsearchPost(url string, data []byte) (int, error) {
	req, err := http.NewRequest("POST", fmt.Sprintf("%s%s", s.url, url), bytes.NewBuffer(data))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		err = fmt.Errorf("Failed to make request, %v", err)
		return 0, err
	}
	defer resp.Body.Close()

	return resp.StatusCode, nil
}

func (s *LogstashElasticsearchShipper) setupTemplate() {
	template := `
{
	"order": 0,
	"template": "metricsd-*",
	"settings": {
		"index.refresh_interval": "5s"
	},
	"mappings": {
		"_default_": {
			"dynamic_templates": [
				{
					"string_fields": {
						"mapping": {
							"index": "analyzed",
							"omit_norms": true,
							"type": "string",
							"fields": {
								"raw": {
									"index": "not_analyzed",
									"ignore_above": 256,
									"type": "string"
								}
							}
						},
						"match_mapping_type": "string",
						"match": "*"
					}
				}
			],
			"properties": {
				"@version": {
					"index": "not_analyzed",
					"type": "string"
				}
			},
			"_all": {
				"enabled": true
			}
		}
	},
	"aliases": {}
}
`
	var data = []byte(template)

	status, err := s.elasticsearchPost("/_template/metricsd", data)
	if err != nil {
		logrus.Fatal("creating index failed: ", err)
	}

	if status != http.StatusOK {
		logrus.Fatal("creating index failed")
	}
}
<commit_msg>get the correct value for index from the ini<commit_after>package shippers

import "bytes"
import "encoding/json"
import "fmt"
import "net/http"
import "github.com/Sirupsen/logrus"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/josegonzalez/metricsd/utils"
import "github.com/vaughan0/go-ini"

type actionMap map[string]indexMap
type indexMap map[string]string

// LogstashElasticsearchShipper is an exported type that
// allows shipping metrics to elasticsearch in logstash format
type LogstashElasticsearchShipper struct {
	enabled    bool
	index      string
	metricType string
	url        string
}

// Enabled allows checking whether the shipper is enabled or not
func (s *LogstashElasticsearchShipper) Enabled() bool {
	return s.enabled
}

// State allows setting the enabled state of the shipper
func (s *LogstashElasticsearchShipper) State(state bool) {
	s.enabled = state
}

// Setup configures the shipper
func (s *LogstashElasticsearchShipper) Setup(conf ini.File) {
	s.State(true)

	if url, ok := conf.Get("LogstashElasticsearchShipper", "url"); ok {
		s.url = url
	} else {
		s.url = "http://127.0.0.1:9200"
	}

	if index, ok := conf.Get("LogstashElasticsearchShipper", "index"); ok {
		s.index = index
	} else {
		s.index = "metricsd-data"
	}

	if metricType, ok := conf.Get("LogstashElasticsearchShipper", "type"); ok {
		s.metricType = metricType
	} else {
		s.metricType = "metricsd"
	}

	s.setupTemplate()
}

// Ship sends a list of MetricSlices to elasticsearch
func (s *LogstashElasticsearchShipper) Ship(logs structs.MetricSlice) error {
	action := actionMap{
		"index": indexMap{
			"_index": s.index,
			"_type":  s.metricType,
		},
	}
	serializedAction, err := json.Marshal(action)
	if err != nil {
		return fmt.Errorf("Failed to marshal action to JSON, %v", err)
	}

	var slice []byte
	newline := []byte("\n")

	for _, item := range logs {
		serialized := item.ToJSON()
		slice = utils.Extend(slice, serializedAction)
		slice = utils.Extend(slice, newline)
		slice = utils.Extend(slice, serialized)
		slice = utils.Extend(slice, newline)
	}

	status, err := s.elasticsearchPost("/_bulk", slice)
	if err != nil {
		logrus.Warning("indexing serialized data failed with err: ", err)
	}

	if status != http.StatusOK {
		logrus.Warning("indexing serialized data failed with status: ", status)
	}
	return nil
}

func (s *LogstashElasticsearchShipper) elasticsearchPost(url string, data []byte) (int, error) {
	req, err := http.NewRequest("POST", fmt.Sprintf("%s%s", s.url, url), bytes.NewBuffer(data))
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		err = fmt.Errorf("Failed to make request, %v", err)
		return 0, err
	}
	defer resp.Body.Close()

	return resp.StatusCode, nil
}

func (s *LogstashElasticsearchShipper) setupTemplate() {
	template := `
{
	"order": 0,
	"template": "metricsd-*",
	"settings": {
		"index.refresh_interval": "5s"
	},
	"mappings": {
		"_default_": {
			"dynamic_templates": [
				{
					"string_fields": {
						"mapping": {
							"index": "analyzed",
							"omit_norms": true,
							"type": "string",
							"fields": {
								"raw": {
									"index": "not_analyzed",
									"ignore_above": 256,
									"type": "string"
								}
							}
						},
						"match_mapping_type": "string",
						"match": "*"
					}
				}
			],
			"properties": {
				"@version": {
					"index": "not_analyzed",
					"type": "string"
				}
			},
			"_all": {
				"enabled": true
			}
		}
	},
	"aliases": {}
}
`
	var data = []byte(template)

	status, err := s.elasticsearchPost("/_template/metricsd", data)
	if err != nil {
		logrus.Fatal("creating index failed: ", err)
	}

	if status != http.StatusOK {
		logrus.Fatal("creating index failed")
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"net/http"
	"reflect"
	"runtime"
	"testing"
)

type simpleGetTestFn func(Client) (interface{}, error)

func simpleGetTest(t *testing.T, url string, testObject interface{}, runTest simpleGetTestFn) {
	callerPC, _, _, _ := runtime.Caller(1)
	testName := runtime.FuncForPC(callerPC).Name()

	client, servers, err := mkTestClientAndServers(t, MuxHandlers{
		brain: Mux{
			url: func(wr http.ResponseWriter, r *http.Request) {
				assertMethod(t, r, "GET")
				writeJSON(t, wr, testObject)
			},
		},
	})
	defer servers.Close()
	if err != nil {
		t.Fatal(err)
	}
	err = client.AuthWithCredentials(map[string]string{})
	if err != nil {
		t.Fatal(err)
	}
	object, err := runTest(client)
	if err != nil {
		t.Errorf("%s errored: %s", testName, err.Error())
	}

	if !reflect.DeepEqual(testObject, object) {
		t.Errorf("%s didn't get expected object.\r\nExpected: %#v\r\nActual:   %#v", testName, testObject, object)
	}

}

func TestGetVLANS(t *testing.T) {
	testVLANs := []*brain.VLAN{
		{
			ID:        90210,
			Num:       123,
			UsageType: "recipes",
			IPRanges: []*brain.IPRange{
				{
					ID:      1234,
					Spec:    "192.168.13.0/24",
					VLANNum: 123,
					Zones: []string{
						"test-zone",
					},
					Available: 200.0,
				},
			},
		},
	}
	simpleGetTest(t, "/admin/vlans", testVLANs, func(client Client) (interface{}, error) {
		return client.GetVLANs()
	})
}

func TestGetIPRanges(t *testing.T) {
	testIPRanges := []*brain.IPRange{
		{
			ID:      1234,
			Spec:    "192.168.13.0/24",
			VLANNum: 123,
			Zones: []string{
				"test-zone",
			},
			Available: 200.0,
		},
	}
	simpleGetTest(t, "/admin/ip_ranges", testIPRanges, func(client Client) (interface{}, error) {
		return client.GetIPRanges()
	})

}

func TestGetIPRange(t *testing.T) {
	testIPRange := brain.IPRange{
		ID:      1234,
		Spec:    "192.168.13.0/24",
		VLANNum: 123,
		Zones: []string{
			"test-zone",
		},
		Available: 200.0,
	}
	simpleGetTest(t, "/admin/ip_ranges/1234", &testIPRange, func(client Client) (interface{}, error) {
		return client.GetIPRange(1234)
	})
}
func TestGetHeads(t *testing.T) {
	testHeads := []*brain.Head{
		{
			ID:       315,
			UUID:     "234833-2493-3423-324235",
			Label:    "test-head315",
			ZoneName: "awesomecoolguyzone",

			Architecture: "x86_64",
			// because of the way json Unmarshals net.IPs different to specifying them in this way this line is commented out
			// CCAddress:     &net.IP{214, 233, 32, 31},
			Note:          "melons",
			Memory:        241000,
			UsageStrategy: "",
			Models:        []string{"generic", "intel"},

			MemoryFree:          123400,
			IsOnline:            true,
			UsedCores:           9,
			VirtualMachineCount: 3,
		}, {
			ID:       239,
			UUID:     "235670-2493-3423-324235",
			Label:    "test-head239",
			ZoneName: "awesomecoolguyzone",

			Architecture: "x86_64",
			// because of the way json Unmarshals net.IPs different to specifying them in this way this line is commented out
			// CCAddress:     &net.IP{24, 43, 32, 49},
			Note:          "more than a hundred years old",
			Memory:        241000,
			UsageStrategy: "",
			Models:        []string{"generic", "intel"},

			MemoryFree:          234000,
			IsOnline:            true,
			UsedCores:           1,
			VirtualMachineCount: 1,
		},
	}
	simpleGetTest(t, "/admin/heads", testHeads, func(client Client) (interface{}, error) {
		return client.GetHeads()
	})

}

func TestGetHead(t *testing.T) {
	testHead := brain.Head{
		ID:       239,
		UUID:     "235670-2493-3423-324235",
		Label:    "test-head239",
		ZoneName: "awesomecoolguyzone",

		Architecture: "x86_64",
		// because of the way json Unmarshals net.IPs different to specifying them in this way this line is commented out
		// CCAddress:     &net.IP{24, 43, 32, 49},
		Note:          "more than a hundred years old",
		Memory:        241000,
		UsageStrategy: "",
		Models:        []string{"generic", "intel"},

		MemoryFree:          234000,
		IsOnline:            true,
		UsedCores:           1,
		VirtualMachineCount: 1,
	}
	simpleGetTest(t, "/admin/heads/239", &testHead, func(client Client) (interface{}, error) {
		return client.GetHead("239")
	})
}

}
<commit_msg>Add TestGetTails<commit_after>package lib

import (
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"net/http"
	"reflect"
	"runtime"
	"testing"
)

type simpleGetTestFn func(Client) (interface{}, error)

func simpleGetTest(t *testing.T, url string, testObject interface{}, runTest simpleGetTestFn) {
	callerPC, _, _, _ := runtime.Caller(1)
	testName := runtime.FuncForPC(callerPC).Name()

	client, servers, err := mkTestClientAndServers(t, MuxHandlers{
		brain: Mux{
			url: func(wr http.ResponseWriter, r *http.Request) {
				assertMethod(t, r, "GET")
				writeJSON(t, wr, testObject)
			},
		},
	})
	defer servers.Close()
	if err != nil {
		t.Fatal(err)
	}
	err = client.AuthWithCredentials(map[string]string{})
	if err != nil {
		t.Fatal(err)
	}
	object, err := runTest(client)
	if err != nil {
		t.Errorf("%s errored: %s", testName, err.Error())
	}

	if !reflect.DeepEqual(testObject, object) {
		t.Errorf("%s didn't get expected object.\r\nExpected: %#v\r\nActual:   %#v", testName, testObject, object)
	}

}

func TestGetVLANS(t *testing.T) {
	testVLANs := []*brain.VLAN{
		{
			ID:        90210,
			Num:       123,
			UsageType: "recipes",
			IPRanges: []*brain.IPRange{
				{
					ID:      1234,
					Spec:    "192.168.13.0/24",
					VLANNum: 123,
					Zones: []string{
						"test-zone",
					},
					Available: 200.0,
				},
			},
		},
	}
	simpleGetTest(t, "/admin/vlans", testVLANs, func(client Client) (interface{}, error) {
		return client.GetVLANs()
	})
}

func TestGetIPRanges(t *testing.T) {
	testIPRanges := []*brain.IPRange{
		{
			ID:      1234,
			Spec:    "192.168.13.0/24",
			VLANNum: 123,
			Zones: []string{
				"test-zone",
			},
			Available: 200.0,
		},
	}
	simpleGetTest(t, "/admin/ip_ranges", testIPRanges, func(client Client) (interface{}, error) {
		return client.GetIPRanges()
	})

}

func TestGetIPRange(t *testing.T) {
	testIPRange := brain.IPRange{
		ID:      1234,
		Spec:    "192.168.13.0/24",
		VLANNum: 123,
		Zones: []string{
			"test-zone",
		},
		Available: 200.0,
	}
	simpleGetTest(t, "/admin/ip_ranges/1234", &testIPRange, func(client Client) (interface{}, error) {
		return client.GetIPRange(1234)
	})
}
func TestGetHeads(t *testing.T) {
	testHeads := []*brain.Head{
		{
			ID:       315,
			UUID:     "234833-2493-3423-324235",
			Label:    "test-head315",
			ZoneName: "awesomecoolguyzone",

			Architecture: "x86_64",
			// because of the way json Unmarshals net.IPs different to specifying them in this way this line is commented out
			// CCAddress:     &net.IP{214, 233, 32, 31},
			Note:          "melons",
			Memory:        241000,
			UsageStrategy: "",
			Models:        []string{"generic", "intel"},

			MemoryFree:          123400,
			IsOnline:            true,
			UsedCores:           9,
			VirtualMachineCount: 3,
		}, {
			ID:       239,
			UUID:     "235670-2493-3423-324235",
			Label:    "test-head239",
			ZoneName: "awesomecoolguyzone",

			Architecture: "x86_64",
			// because of the way json Unmarshals net.IPs different to specifying them in this way this line is commented out
			// CCAddress:     &net.IP{24, 43, 32, 49},
			Note:          "more than a hundred years old",
			Memory:        241000,
			UsageStrategy: "",
			Models:        []string{"generic", "intel"},

			MemoryFree:          234000,
			IsOnline:            true,
			UsedCores:           1,
			VirtualMachineCount: 1,
		},
	}
	simpleGetTest(t, "/admin/heads", testHeads, func(client Client) (interface{}, error) {
		return client.GetHeads()
	})

}

func TestGetHead(t *testing.T) {
	testHead := brain.Head{
		ID:       239,
		UUID:     "235670-2493-3423-324235",
		Label:    "test-head239",
		ZoneName: "awesomecoolguyzone",

		Architecture: "x86_64",
		// because of the way json Unmarshals net.IPs different to specifying them in this way this line is commented out
		// CCAddress:     &net.IP{24, 43, 32, 49},
		Note:          "more than a hundred years old",
		Memory:        241000,
		UsageStrategy: "",
		Models:        []string{"generic", "intel"},

		MemoryFree:          234000,
		IsOnline:            true,
		UsedCores:           1,
		VirtualMachineCount: 1,
	}
	simpleGetTest(t, "/admin/heads/239", &testHead, func(client Client) (interface{}, error) {
		return client.GetHead("239")
	})
}

func TestGetTails(t *testing.T) {
	testTails := []*brain.Tail{
		{
			ID:           1345,
			UUID:         "idont-reallyknowwhat-uuids-looklike",
			Label:        "coolTailForCoolDiscs",
			ZoneName:     "frozone",
			IsOnline:     false,
			StoragePools: []string{"swimming", "paddling"},
		}, {
			ID:           1235,
			UUID:         "888888-8888-8888-888888",
			Label:        "eight",
			ZoneName:     "eighth zone",
			IsOnline:     true,
			StoragePools: []string{"pool-eight"},
		},
	}
	simpleGetTest(t, "/admin/tails", testTails, func(client Client) (interface{}, error) {
		return client.GetTails()
	})
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ipam

import (
	"errors"
	"fmt"
	"github.com/romana/core/common"
	"github.com/romana/core/tenant"
	"log"
	"net"
)

// IPAM provides ipam service.
type IPAM struct {
	config common.ServiceConfig
	store  ipamStore
	dc     common.Datacenter
}

const (
	infoListPath = "/info"
)

// Routes provided by ipam.
func (ipam *IPAM) Routes() common.Routes {
	routes := common.Routes{
		common.Route{
			Method:          "POST",
			Pattern:         "/endpoints",
			Handler:         ipam.addEndpoint,
			MakeMessage:     func() interface{} { return &Endpoint{} },
			UseRequestToken: true,
		},
		common.Route{
			Method:          "DELETE",
			Pattern:         "/endpoints/{ip}",
			Handler:         ipam.deleteEndpoint,
			MakeMessage:     nil,
			UseRequestToken: false,
		},
		common.Route{
			Method:          "GET",
			Pattern:         "/allocateIP",
			Handler:         ipam.allocateIP,
			MakeMessage:     nil,
			UseRequestToken: false,
		},
	}
	return routes
}

// allocateIP finds internal Romana information based on tenantID/tenantName and other provided parameters, then adds
// that endpoint to IPAM, and passes through the allocated IP
func (ipam *IPAM) allocateIP(input interface{}, ctx common.RestContext) (interface{}, error) {
	tenantParam := ""
	tenantLookupField := ""

	if tenantID := ctx.QueryVariables.Get("tenantID"); tenantID != "" {
		tenantParam = tenantID
		tenantLookupField = "ExternalID"
	} else if tenantName := ctx.QueryVariables.Get("tenantName"); tenantName != "" {
		tenantParam = tenantName
		tenantLookupField = "Name"
	}

	// check for missing/empty required parameters
	if tenantParam == "" {
		err := errors.New("Missing or empty tenantName/tenantID parameter")
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	segmentName := ctx.QueryVariables.Get("segmentName")
	if segmentName == "" {
		err := errors.New("Missing or empty segmentName parameter")
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	hostName := ctx.QueryVariables.Get("hostName")
	if hostName == "" {
		err := errors.New("Missing or empty hostName parameter")
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	endpoint := Endpoint{}
	instanceName := ctx.QueryVariables.Get("instanceName")
	if instanceName != "" {
		endpoint.Name = instanceName
	}

	client, err := common.NewRestClient(common.GetRestClientConfig(ipam.config))
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	// Get host info from topology service
	topoUrl, err := client.GetServiceUrl("topology")
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	index := common.IndexResponse{}
	err = client.Get(topoUrl, &index)
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	hostsURL := index.Links.FindByRel("host-list")
	var hosts []common.HostMessage

	err = client.Get(hostsURL, &hosts)
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	found := false
	for _, h := range hosts {
		if h.Name == hostName {
			found = true
			endpoint.HostId = h.Id
			break
		}
	}

	if !found {
		msg := fmt.Sprintf("Host with name %s not found", hostName)
		err := errors.New(msg)
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	log.Printf("Host name %s has ID %s", hostName, endpoint.HostId)

	tenantSvcUrl, err := client.GetServiceUrl("tenant")
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	// TODO follow links once tenant service supports it. For now...

	tenantsUrl := fmt.Sprintf("%s/tenants", tenantSvcUrl)
	var tenants []tenant.Tenant
	err = client.Get(tenantsUrl, &tenants)
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	found = false
	for _, t := range tenants {
		switch tenantLookupField {
		case "Name":
			if t.Name == tenantParam {
				found = true
			}
		case "ExternalID":
			if t.ExternalID == tenantParam {
				found = true
			}
		}

		if found {
			endpoint.TenantID = fmt.Sprintf("%d", t.ID)
			log.Printf("IPAM: Tenant '%s' has ID %s, original %d", tenantParam, endpoint.TenantID, t.ID)
			break
		}
	}
	if !found {
		err := fmt.Errorf("Tenant with name '%s' not found", tenantParam)
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	log.Printf("IPAM: Tenant name %s has ID %s", tenantParam, endpoint.TenantID)

	segmentsUrl := fmt.Sprintf("/tenants/%s/segments", endpoint.TenantID)
	var segments []tenant.Segment
	err = client.Get(segmentsUrl, &segments)
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	found = false
	fmt.Printf("IPAM found %d segments for tenant %s\n", len(segments), endpoint.TenantID)
	for _, s := range segments {
//		log.Printf("IPAM checking %s (not %s) against %s", s.Name, s.ExternalID, segmentName)
		if s.Name == segmentName {
			found = true
			endpoint.SegmentID = fmt.Sprintf("%d", s.ID)
			break
		}
	}
	if !found {
		err := fmt.Errorf("Segment with name '%s' not found in %s", segmentName, segments)
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	log.Printf("Segment name %s has ID %s", segmentName, endpoint.SegmentID)
	return ipam.addEndpoint(&endpoint, ctx)
}

// addEndpoint handles request to add an endpoint and
// allocate an IP address.
func (ipam *IPAM) addEndpoint(input interface{}, ctx common.RestContext) (interface{}, error) {
	endpoint := input.(*Endpoint)
	client, err := common.NewRestClient(common.GetRestClientConfig(ipam.config))
	if err != nil {
		log.Printf("IPAM encountered an error getting a REST client instance: %v", err)
		return nil, err
	}
	// Get host info from topology service
	topoUrl, err := client.GetServiceUrl("topology")
	if err != nil {
		log.Printf("IPAM encountered an error getting a topology service URL %v", err)
		return nil, err
	}

	index := common.IndexResponse{}
	err = client.Get(topoUrl, &index)
	if err != nil {
		log.Printf("IPAM encountered an error querying topology: %v", err)
		return nil, err
	}

	hostsURL := index.Links.FindByRel("host-list")
	host := common.HostMessage{}

	hostInfoURL := fmt.Sprintf("%s/%s", hostsURL, endpoint.HostId)
	err = client.Get(hostInfoURL, &host)

	if err != nil {
		log.Printf("IPAM encountered an error querying topology for hosts: %v", err)
		return nil, err
	}

	tenantUrl, err := client.GetServiceUrl("tenant")
	if err != nil {
		log.Printf("IPAM encountered an error getting tenant srevice URL: %v", err)
		return nil, err
	}

	// TODO follow links once tenant service supports it. For now...

	t := &tenant.Tenant{}
	tenantsUrl := fmt.Sprintf("%s/tenants/%s", tenantUrl, endpoint.TenantID)
	log.Printf("IPAM calling %s\n", tenantsUrl)
	err = client.Get(tenantsUrl, t)
	if err != nil {
		log.Printf("IPAM encountered an error querying tenant service for tenant %d: %v", endpoint.TenantID, err)
		return nil, err
	}
	log.Printf("IPAM: received tenant %s ID %d, sequence %d\n", t.Name, t.ID, t.Seq)

	segmentUrl := fmt.Sprintf("/tenants/%s/segments/%s", endpoint.TenantID, endpoint.SegmentID)
	log.Printf("IPAM: calling %s\n", segmentUrl)
	segment := &tenant.Segment{}
	err = client.Get(segmentUrl, segment)
	if err != nil {
		log.Printf("IPAM encountered an error querying tenant service for tenant %d and segment %d: %v", endpoint.TenantID, endpoint.SegmentID, err)
		return nil, err
	}

	log.Printf("Constructing IP from Host IP %s, Tenant %d, Segment %d", host.RomanaIp, t.Seq, segment.Seq)

	endpointBits := 32 - ipam.dc.PrefixBits - ipam.dc.PortBits - ipam.dc.TenantBits - ipam.dc.SegmentBits - ipam.dc.EndpointSpaceBits
	segmentBitShift := endpointBits
	//	prefixBitShift := 32 - ipam.dc.PrefixBits
	tenantBitShift := segmentBitShift + ipam.dc.SegmentBits
	log.Printf("Parsing Romana IP address of host %s: %s\n", host.Name, host.RomanaIp)
	_, network, err := net.ParseCIDR(host.RomanaIp)
	if err != nil {
		log.Printf("IPAM encountered an error parsing %s: %v", host.RomanaIp, err)
		return nil, err
	}
	hostIpInt := common.IPv4ToInt(network.IP)
	tSeq := t.Seq
	sSeq := segment.Seq
	upToEndpointIpInt := hostIpInt | (tSeq << tenantBitShift) | (sSeq << segmentBitShift)
	log.Printf("IPAM: before calling addEndpoint:  %v | (%v << %v) | (%v << %v): %v ", network.IP.String(), tSeq, tenantBitShift, sSeq, segmentBitShift, common.IntToIPv4(upToEndpointIpInt))
	err = ipam.store.addEndpoint(endpoint, upToEndpointIpInt, ipam.dc.EndpointSpaceBits)
	if err != nil {
		log.Printf("IPAM encountered an error adding endpoint to db: %v", err)
		return nil, err
	}
	return endpoint, nil

}

// deleteEndpoint releases the IP(s) owned by the endpoint into assignable
// pool.
func (ipam *IPAM) deleteEndpoint(input interface{}, ctx common.RestContext) (interface{}, error) {
	return ipam.store.deleteEndpoint(ctx.PathVariables["ip"])
}

// Name provides name of this service.
func (ipam *IPAM) Name() string {
	return "ipam"
}

// SetConfig implements SetConfig function of the Service interface.
// Returns an error if cannot connect to the data store
func (ipam *IPAM) SetConfig(config common.ServiceConfig) error {
	// TODO this is a copy-paste of topology service, to refactor
	log.Println(config)
	ipam.config = config
	storeConfig := config.ServiceSpecific["store"].(map[string]interface{})
	log.Printf("IPAM port: %d", config.Common.Api.Port)
	ipam.store = ipamStore{}
	ipam.store.ServiceStore = &ipam.store
	return ipam.store.SetConfig(storeConfig)

}

func (ipam *IPAM) createSchema(overwrite bool) error {
	return ipam.store.CreateSchema(overwrite)
}

// Run mainly runs IPAM service.
func Run(rootServiceUrl string, cred *common.Credential) (*common.RestServiceInfo, error) {
	clientConfig := common.GetDefaultRestClientConfig(rootServiceUrl)
	clientConfig.Credential = cred
	client, err := common.NewRestClient(clientConfig)
	if err != nil {
		return nil, err
	}
	ipam := &IPAM{}
	config, err := client.GetServiceConfig(ipam.Name())
	if err != nil {
		return nil, err
	}
	return common.InitializeService(ipam, *config)

}

// Initialize implements Initialize method of Service interface
func (ipam *IPAM) Initialize() error {
	log.Println("Entering ipam.Initialize()")
	err := ipam.store.Connect()
	if err != nil {
		return err
	}
	client, err := common.NewRestClient(common.GetRestClientConfig(ipam.config))
	if err != nil {
		return err
	}

	topologyURL, err := client.GetServiceUrl("topology")
	if err != nil {
		return err
	}

	index := common.IndexResponse{}
	err = client.Get(topologyURL, &index)
	if err != nil {
		return err
	}

	dcURL := index.Links.FindByRel("datacenter")
	dc := common.Datacenter{}
	log.Printf("IPAM received datacenter information from topology service: %#v\n", dc)
	err = client.Get(dcURL, &dc)
	if err != nil {
		return err
	}
	// TODO should this always be queried?
	ipam.dc = dc
	return nil
}

// CreateSchema creates schema for IPAM service.
func CreateSchema(rootServiceUrl string, overwrite bool) error {
	log.Println("In CreateSchema(", rootServiceUrl, ",", overwrite, ")")
	ipam := &IPAM{}

	client, err := common.NewRestClient(common.GetDefaultRestClientConfig(rootServiceUrl))
	if err != nil {
		return err
	}
	config, err := client.GetServiceConfig(ipam.Name())
	if err != nil {
		return err
	}

	err = ipam.SetConfig(*config)
	if err != nil {
		return err
	}
	return ipam.store.CreateSchema(overwrite)
}
<commit_msg>Go vet fixes<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package ipam

import (
	"errors"
	"fmt"
	"github.com/romana/core/common"
	"github.com/romana/core/tenant"
	"log"
	"net"
)

// IPAM provides ipam service.
type IPAM struct {
	config common.ServiceConfig
	store  ipamStore
	dc     common.Datacenter
}

const (
	infoListPath = "/info"
)

// Routes provided by ipam.
func (ipam *IPAM) Routes() common.Routes {
	routes := common.Routes{
		common.Route{
			Method:          "POST",
			Pattern:         "/endpoints",
			Handler:         ipam.addEndpoint,
			MakeMessage:     func() interface{} { return &Endpoint{} },
			UseRequestToken: true,
		},
		common.Route{
			Method:          "DELETE",
			Pattern:         "/endpoints/{ip}",
			Handler:         ipam.deleteEndpoint,
			MakeMessage:     nil,
			UseRequestToken: false,
		},
		common.Route{
			Method:          "GET",
			Pattern:         "/allocateIP",
			Handler:         ipam.allocateIP,
			MakeMessage:     nil,
			UseRequestToken: false,
		},
	}
	return routes
}

// allocateIP finds internal Romana information based on tenantID/tenantName and other provided parameters, then adds
// that endpoint to IPAM, and passes through the allocated IP
func (ipam *IPAM) allocateIP(input interface{}, ctx common.RestContext) (interface{}, error) {
	tenantParam := ""
	tenantLookupField := ""

	if tenantID := ctx.QueryVariables.Get("tenantID"); tenantID != "" {
		tenantParam = tenantID
		tenantLookupField = "ExternalID"
	} else if tenantName := ctx.QueryVariables.Get("tenantName"); tenantName != "" {
		tenantParam = tenantName
		tenantLookupField = "Name"
	}

	// check for missing/empty required parameters
	if tenantParam == "" {
		err := errors.New("Missing or empty tenantName/tenantID parameter")
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	segmentName := ctx.QueryVariables.Get("segmentName")
	if segmentName == "" {
		err := errors.New("Missing or empty segmentName parameter")
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	hostName := ctx.QueryVariables.Get("hostName")
	if hostName == "" {
		err := errors.New("Missing or empty hostName parameter")
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	endpoint := Endpoint{}
	instanceName := ctx.QueryVariables.Get("instanceName")
	if instanceName != "" {
		endpoint.Name = instanceName
	}

	client, err := common.NewRestClient(common.GetRestClientConfig(ipam.config))
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	// Get host info from topology service
	topoUrl, err := client.GetServiceUrl("topology")
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	index := common.IndexResponse{}
	err = client.Get(topoUrl, &index)
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	hostsURL := index.Links.FindByRel("host-list")
	var hosts []common.HostMessage

	err = client.Get(hostsURL, &hosts)
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	found := false
	for _, h := range hosts {
		if h.Name == hostName {
			found = true
			endpoint.HostId = h.Id
			break
		}
	}

	if !found {
		msg := fmt.Sprintf("Host with name %s not found", hostName)
		err := errors.New(msg)
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	log.Printf("Host name %s has ID %s", hostName, endpoint.HostId)

	tenantSvcUrl, err := client.GetServiceUrl("tenant")
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	// TODO follow links once tenant service supports it. For now...

	tenantsUrl := fmt.Sprintf("%s/tenants", tenantSvcUrl)
	var tenants []tenant.Tenant
	err = client.Get(tenantsUrl, &tenants)
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	found = false
	for _, t := range tenants {
		switch tenantLookupField {
		case "Name":
			if t.Name == tenantParam {
				found = true
			}
		case "ExternalID":
			if t.ExternalID == tenantParam {
				found = true
			}
		}

		if found {
			endpoint.TenantID = fmt.Sprintf("%d", t.ID)
			log.Printf("IPAM: Tenant '%s' has ID %s, original %d", tenantParam, endpoint.TenantID, t.ID)
			break
		}
	}
	if !found {
		err := fmt.Errorf("Tenant with name '%s' not found", tenantParam)
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	log.Printf("IPAM: Tenant name %s has ID %s", tenantParam, endpoint.TenantID)

	segmentsUrl := fmt.Sprintf("/tenants/%s/segments", endpoint.TenantID)
	var segments []tenant.Segment
	err = client.Get(segmentsUrl, &segments)
	if err != nil {
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}
	found = false
	fmt.Printf("IPAM found %d segments for tenant %s\n", len(segments), endpoint.TenantID)
	for _, s := range segments {
//		log.Printf("IPAM checking %s (not %s) against %s", s.Name, s.ExternalID, segmentName)
		if s.Name == segmentName {
			found = true
			endpoint.SegmentID = fmt.Sprintf("%d", s.ID)
			break
		}
	}
	if !found {
		err := fmt.Errorf("Segment with name '%s' not found in %v", segmentName, segments)
		log.Printf("IPAM encountered an error: %v", err)
		return nil, err
	}

	log.Printf("Segment name %s has ID %s", segmentName, endpoint.SegmentID)
	return ipam.addEndpoint(&endpoint, ctx)
}

// addEndpoint handles request to add an endpoint and
// allocate an IP address.
func (ipam *IPAM) addEndpoint(input interface{}, ctx common.RestContext) (interface{}, error) {
	endpoint := input.(*Endpoint)
	client, err := common.NewRestClient(common.GetRestClientConfig(ipam.config))
	if err != nil {
		log.Printf("IPAM encountered an error getting a REST client instance: %v", err)
		return nil, err
	}
	// Get host info from topology service
	topoUrl, err := client.GetServiceUrl("topology")
	if err != nil {
		log.Printf("IPAM encountered an error getting a topology service URL %v", err)
		return nil, err
	}

	index := common.IndexResponse{}
	err = client.Get(topoUrl, &index)
	if err != nil {
		log.Printf("IPAM encountered an error querying topology: %v", err)
		return nil, err
	}

	hostsURL := index.Links.FindByRel("host-list")
	host := common.HostMessage{}

	hostInfoURL := fmt.Sprintf("%s/%s", hostsURL, endpoint.HostId)
	err = client.Get(hostInfoURL, &host)

	if err != nil {
		log.Printf("IPAM encountered an error querying topology for hosts: %v", err)
		return nil, err
	}

	tenantUrl, err := client.GetServiceUrl("tenant")
	if err != nil {
		log.Printf("IPAM encountered an error getting tenant srevice URL: %v", err)
		return nil, err
	}

	// TODO follow links once tenant service supports it. For now...

	t := &tenant.Tenant{}
	tenantsUrl := fmt.Sprintf("%s/tenants/%s", tenantUrl, endpoint.TenantID)
	log.Printf("IPAM calling %s\n", tenantsUrl)
	err = client.Get(tenantsUrl, t)
	if err != nil {
		log.Printf("IPAM encountered an error querying tenant service for tenant %s: %v", endpoint.TenantID, err)
		return nil, err
	}
	log.Printf("IPAM: received tenant %s ID %d, sequence %d\n", t.Name, t.ID, t.Seq)

	segmentUrl := fmt.Sprintf("/tenants/%s/segments/%s", endpoint.TenantID, endpoint.SegmentID)
	log.Printf("IPAM: calling %s\n", segmentUrl)
	segment := &tenant.Segment{}
	err = client.Get(segmentUrl, segment)
	if err != nil {
		log.Printf("IPAM encountered an error querying tenant service for tenant %s and segment %s: %v", endpoint.TenantID, endpoint.SegmentID, err)
		return nil, err
	}

	log.Printf("Constructing IP from Host IP %s, Tenant %d, Segment %d", host.RomanaIp, t.Seq, segment.Seq)

	endpointBits := 32 - ipam.dc.PrefixBits - ipam.dc.PortBits - ipam.dc.TenantBits - ipam.dc.SegmentBits - ipam.dc.EndpointSpaceBits
	segmentBitShift := endpointBits
	//	prefixBitShift := 32 - ipam.dc.PrefixBits
	tenantBitShift := segmentBitShift + ipam.dc.SegmentBits
	log.Printf("Parsing Romana IP address of host %s: %s\n", host.Name, host.RomanaIp)
	_, network, err := net.ParseCIDR(host.RomanaIp)
	if err != nil {
		log.Printf("IPAM encountered an error parsing %s: %v", host.RomanaIp, err)
		return nil, err
	}
	hostIpInt := common.IPv4ToInt(network.IP)
	tSeq := t.Seq
	sSeq := segment.Seq
	upToEndpointIpInt := hostIpInt | (tSeq << tenantBitShift) | (sSeq << segmentBitShift)
	log.Printf("IPAM: before calling addEndpoint:  %v | (%v << %v) | (%v << %v): %v ", network.IP.String(), tSeq, tenantBitShift, sSeq, segmentBitShift, common.IntToIPv4(upToEndpointIpInt))
	err = ipam.store.addEndpoint(endpoint, upToEndpointIpInt, ipam.dc.EndpointSpaceBits)
	if err != nil {
		log.Printf("IPAM encountered an error adding endpoint to db: %v", err)
		return nil, err
	}
	return endpoint, nil

}

// deleteEndpoint releases the IP(s) owned by the endpoint into assignable
// pool.
func (ipam *IPAM) deleteEndpoint(input interface{}, ctx common.RestContext) (interface{}, error) {
	return ipam.store.deleteEndpoint(ctx.PathVariables["ip"])
}

// Name provides name of this service.
func (ipam *IPAM) Name() string {
	return "ipam"
}

// SetConfig implements SetConfig function of the Service interface.
// Returns an error if cannot connect to the data store
func (ipam *IPAM) SetConfig(config common.ServiceConfig) error {
	// TODO this is a copy-paste of topology service, to refactor
	log.Println(config)
	ipam.config = config
	storeConfig := config.ServiceSpecific["store"].(map[string]interface{})
	log.Printf("IPAM port: %d", config.Common.Api.Port)
	ipam.store = ipamStore{}
	ipam.store.ServiceStore = &ipam.store
	return ipam.store.SetConfig(storeConfig)

}

func (ipam *IPAM) createSchema(overwrite bool) error {
	return ipam.store.CreateSchema(overwrite)
}

// Run mainly runs IPAM service.
func Run(rootServiceUrl string, cred *common.Credential) (*common.RestServiceInfo, error) {
	clientConfig := common.GetDefaultRestClientConfig(rootServiceUrl)
	clientConfig.Credential = cred
	client, err := common.NewRestClient(clientConfig)
	if err != nil {
		return nil, err
	}
	ipam := &IPAM{}
	config, err := client.GetServiceConfig(ipam.Name())
	if err != nil {
		return nil, err
	}
	return common.InitializeService(ipam, *config)

}

// Initialize implements Initialize method of Service interface
func (ipam *IPAM) Initialize() error {
	log.Println("Entering ipam.Initialize()")
	err := ipam.store.Connect()
	if err != nil {
		return err
	}
	client, err := common.NewRestClient(common.GetRestClientConfig(ipam.config))
	if err != nil {
		return err
	}

	topologyURL, err := client.GetServiceUrl("topology")
	if err != nil {
		return err
	}

	index := common.IndexResponse{}
	err = client.Get(topologyURL, &index)
	if err != nil {
		return err
	}

	dcURL := index.Links.FindByRel("datacenter")
	dc := common.Datacenter{}
	log.Printf("IPAM received datacenter information from topology service: %#v\n", dc)
	err = client.Get(dcURL, &dc)
	if err != nil {
		return err
	}
	// TODO should this always be queried?
	ipam.dc = dc
	return nil
}

// CreateSchema creates schema for IPAM service.
func CreateSchema(rootServiceUrl string, overwrite bool) error {
	log.Println("In CreateSchema(", rootServiceUrl, ",", overwrite, ")")
	ipam := &IPAM{}

	client, err := common.NewRestClient(common.GetDefaultRestClientConfig(rootServiceUrl))
	if err != nil {
		return err
	}
	config, err := client.GetServiceConfig(ipam.Name())
	if err != nil {
		return err
	}

	err = ipam.SetConfig(*config)
	if err != nil {
		return err
	}
	return ipam.store.CreateSchema(overwrite)
}
<|endoftext|>