<commit_before>package model

type Locale struct {
	Country  string
	Language string
	Code     string
}

var Locales = []Locale{
	Locale{Country: "Albanian", Language: "Albania", Code: "sq_AL"},
	Locale{Country: "Arabic", Language: "Algeria", Code: "ar_DZ"},
	Locale{Country: "Arabic", Language: "Bahrain", Code: "ar_BH"},
	Locale{Country: "Arabic", Language: "Egypt", Code: "ar_EG"},
	Locale{Country: "Arabic", Language: "Iraq", Code: "ar_IQ"},
	Locale{Country: "Arabic", Language: "Jordan", Code: "ar_JO"},
	Locale{Country: "Arabic", Language: "Kuwait", Code: "ar_KW"},
	Locale{Country: "Arabic", Language: "Lebanon", Code: "ar_LB"},
	Locale{Country: "Arabic", Language: "Libya", Code: "ar_LY"},
	Locale{Country: "Arabic", Language: "Morocco", Code: "ar_MA"},
	Locale{Country: "Arabic", Language: "Oman", Code: "ar_OM"},
	Locale{Country: "Arabic", Language: "Qatar", Code: "ar_QA"},
	Locale{Country: "Arabic", Language: "Saudi Arabia", Code: "ar_SA"},
	Locale{Country: "Arabic", Language: "Sudan", Code: "ar_SD"},
	Locale{Country: "Arabic", Language: "Syria", Code: "ar_SY"},
	Locale{Country: "Arabic", Language: "Tunisia", Code: "ar_TN"},
	Locale{Country: "Arabic", Language: "United Arab Emirates", Code: "ar_AE"},
	Locale{Country: "Arabic", Language: "Yemen", Code: "ar_YE"},
	Locale{Country: "Belarusian", Language: "Belarus", Code: "be_BY"},
	Locale{Country: "Bulgarian", Language: "Bulgaria", Code: "bg_BG"},
	Locale{Country: "Catalan", Language: "Spain", Code: "ca_ES"},
	Locale{Country: "Chinese (Simplified)", Language: "China", Code: "zh_CN"},
	Locale{Country: "Chinese (Simplified)", Language: "Singapore", Code: "zh_SG"},
	Locale{Country: "Chinese (Traditional)", Language: "Hong Kong", Code: "zh_HK"},
	Locale{Country: "Chinese (Traditional)", Language: "Taiwan", Code: "zh_TW"},
	Locale{Country: "Croatian", Language: "Croatia", Code: "hr_HR"},
	Locale{Country: "Czech", Language: "Czech Republic", Code: "cs_CZ"},
	Locale{Country: "Danish", Language: "Denmark", Code: "da_DK"},
	Locale{Country: "Dutch", Language: "Belgium", Code: "nl_BE"},
	Locale{Country: "Dutch", Language: "Netherlands", Code: "nl_NL"},
	Locale{Country: "English", Language: "Australia", Code: "en_AU"},
	Locale{Country: "English", Language: "Canada", Code: "en_CA"},
	Locale{Country: "English", Language: "India", Code: "en_IN"},
	Locale{Country: "English", Language: "Ireland", Code: "en_IE"},
	Locale{Country: "English", Language: "Malta", Code: "en_MT"},
	Locale{Country: "English", Language: "New Zealand", Code: "en_NZ"},
	Locale{Country: "English", Language: "Philippines", Code: "en_PH"},
	Locale{Country: "English", Language: "Singapore", Code: "en_SG"},
	Locale{Country: "English", Language: "South Africa", Code: "en_ZA"},
	Locale{Country: "English", Language: "United Kingdom", Code: "en_GB"},
	Locale{Country: "English", Language: "United States", Code: "en_US"},
	Locale{Country: "Estonian", Language: "Estonia", Code: "et_EE"},
	Locale{Country: "Finnish", Language: "Finland", Code: "fi_FI"},
	Locale{Country: "French", Language: "Belgium", Code: "fr_BE"},
	Locale{Country: "French", Language: "Canada", Code: "fr_CA"},
	Locale{Country: "French", Language: "France", Code: "fr_FR"},
	Locale{Country: "French", Language: "Luxembourg", Code: "fr_LU"},
	Locale{Country: "French", Language: "Switzerland", Code: "fr_CH"},
	Locale{Country: "German", Language: "Austria", Code: "de_AT"},
	Locale{Country: "German", Language: "Germany", Code: "de_DE"},
	Locale{Country: "German", Language: "Luxembourg", Code: "de_LU"},
	Locale{Country: "German", Language: "Switzerland", Code: "de_CH"},
	Locale{Country: "Greek", Language: "Cyprus", Code: "el_CY"},
	Locale{Country: "Greek", Language: "Greece", Code: "el_GR"},
	Locale{Country: "Hebrew", Language: "Israel", Code: "iw_IL"},
	Locale{Country: "Hindi", Language: "India", Code: "hi_IN"},
	Locale{Country: "Hungarian", Language: "Hungary", Code: "hu_HU"},
	Locale{Country: "Icelandic", Language: "Iceland", Code: "is_IS"},
	Locale{Country: "Indonesian", Language: "Indonesia", Code: "in_ID"},
	Locale{Country: "Irish", Language: "Ireland", Code: "ga_IE"},
	Locale{Country: "Italian", Language: "Italy", Code: "it_IT"},
	Locale{Country: "Italian", Language: "Switzerland", Code: "it_CH"},
	Locale{Country: "Japanese", Language: "Japan", Code: "ja_JP"},
	Locale{Country: "Korean", Language: "South Korea", Code: "ko_KR"},
	Locale{Country: "Latvian", Language: "Latvia", Code: "lv_LV"},
	Locale{Country: "Lithuanian", Language: "Lithuania", Code: "lt_LT"},
	Locale{Country: "Macedonian", Language: "Macedonia", Code: "mk_MK"},
	Locale{Country: "Malay", Language: "Malaysia", Code: "ms_MY"},
	Locale{Country: "Maltese", Language: "Malta", Code: "mt_MT"},
	Locale{Country: "Norwegian (Bokmål)", Language: "Norway", Code: "no_NO"},
	Locale{Country: "Norwegian (Nynorsk)", Language: "Norway", Code: "no_NO_NY"},
	Locale{Country: "Polish", Language: "Poland", Code: "pl_PL"},
	Locale{Country: "Portuguese", Language: "Brazil", Code: "pt_BR"},
	Locale{Country: "Portuguese", Language: "Portugal", Code: "pt_PT"},
	Locale{Country: "Romanian", Language: "Romania", Code: "ro_RO"},
	Locale{Country: "Russian", Language: "Russia", Code: "ru_RU"},
	Locale{Country: "Serbian (Cyrillic)", Language: "Bosnia and Herzegovina", Code: "sr_BA"},
	Locale{Country: "Serbian (Cyrillic)", Language: "Montenegro", Code: "sr_ME"},
	Locale{Country: "Serbian (Cyrillic)", Language: "Serbia", Code: "sr_RS"},
	Locale{Country: "Serbian (Latin)", Language: "Bosnia and Herzegovina", Code: "sr_La_tn_BA"},
	Locale{Country: "Serbian (Latin)", Language: "Montenegro", Code: "sr_La_tn_ME"},
	Locale{Country: "Serbian (Latin)", Language: "Serbia", Code: "sr_La_tn_RS"},
	Locale{Country: "Slovak", Language: "Slovakia", Code: "sk_SK"},
	Locale{Country: "Slovenian", Language: "Slovenia", Code: "sl_SI"},
	Locale{Country: "Spanish", Language: "Argentina", Code: "es_AR"},
	Locale{Country: "Spanish", Language: "Bolivia", Code: "es_BO"},
	Locale{Country: "Spanish", Language: "Chile", Code: "es_CL"},
	Locale{Country: "Spanish", Language: "Colombia", Code: "es_CO"},
	Locale{Country: "Spanish", Language: "Costa Rica", Code: "es_CR"},
	Locale{Country: "Spanish", Language: "Dominican Republic", Code: "es_DO"},
	Locale{Country: "Spanish", Language: "Ecuador", Code: "es_EC"},
	Locale{Country: "Spanish", Language: "El Salvador", Code: "es_SV"},
	Locale{Country: "Spanish", Language: "Guatemala", Code: "es_GT"},
	Locale{Country: "Spanish", Language: "Honduras", Code: "es_HN"},
	Locale{Country: "Spanish", Language: "Mexico", Code: "es_MX"},
	Locale{Country: "Spanish", Language: "Nicaragua", Code: "es_NI"},
	Locale{Country: "Spanish", Language: "Panama", Code: "es_PA"},
	Locale{Country: "Spanish", Language: "Paraguay", Code: "es_PY"},
	Locale{Country: "Spanish", Language: "Peru", Code: "es_PE"},
	Locale{Country: "Spanish", Language: "Puerto Rico", Code: "es_PR"},
	Locale{Country: "Spanish", Language: "Spain", Code: "es_ES"},
	Locale{Country: "Spanish", Language: "United States", Code: "es_US"},
	Locale{Country: "Spanish", Language: "Uruguay", Code: "es_UY"},
	Locale{Country: "Spanish", Language: "Venezuela", Code: "es_VE"},
	Locale{Country: "Swedish", Language: "Sweden", Code: "sv_SE"},
	Locale{Country: "Thai (Western digits)", Language: "Thailand", Code: "th_TH"},
	Locale{Country: "Thai (Thai digits)", Language: "Thailand", Code: "th_TH_TH"},
	Locale{Country: "Turkish", Language: "Turkey", Code: "tr_TR"},
	Locale{Country: "Ukrainian", Language: "Ukraine", Code: "uk_UA"},
	Locale{Country: "Vietnamese", Language: "Vietnam", Code: "vi_VN"},
}
<commit_msg>Add locale list<commit_after>package model

type LocaleInfo struct {
	Country  string
	Language string
}

var Locales = map[string]LocaleInfo{
	"sq_AL":       LocaleInfo{Country: "Albanian", Language: "Albania"},
	"ar_DZ":       LocaleInfo{Country: "Arabic", Language: "Algeria"},
	"ar_BH":       LocaleInfo{Country: "Arabic", Language: "Bahrain"},
	"ar_EG":       LocaleInfo{Country: "Arabic", Language: "Egypt"},
	"ar_IQ":       LocaleInfo{Country: "Arabic", Language: "Iraq"},
	"ar_JO":       LocaleInfo{Country: "Arabic", Language: "Jordan"},
	"ar_KW":       LocaleInfo{Country: "Arabic", Language: "Kuwait"},
	"ar_LB":       LocaleInfo{Country: "Arabic", Language: "Lebanon"},
	"ar_LY":       LocaleInfo{Country: "Arabic", Language: "Libya"},
	"ar_MA":       LocaleInfo{Country: "Arabic", Language: "Morocco"},
	"ar_OM":       LocaleInfo{Country: "Arabic", Language: "Oman"},
	"ar_QA":       LocaleInfo{Country: "Arabic", Language: "Qatar"},
	"ar_SA":       LocaleInfo{Country: "Arabic", Language: "Saudi Arabia"},
	"ar_SD":       LocaleInfo{Country: "Arabic", Language: "Sudan"},
	"ar_SY":       LocaleInfo{Country: "Arabic", Language: "Syria"},
	"ar_TN":       LocaleInfo{Country: "Arabic", Language: "Tunisia"},
	"ar_AE":       LocaleInfo{Country: "Arabic", Language: "United Arab Emirates"},
	"ar_YE":       LocaleInfo{Country: "Arabic", Language: "Yemen"},
	"be_BY":       LocaleInfo{Country: "Belarusian", Language: "Belarus"},
	"bg_BG":       LocaleInfo{Country: "Bulgarian", Language: "Bulgaria"},
	"ca_ES":       LocaleInfo{Country: "Catalan", Language: "Spain"},
	"zh_CN":       LocaleInfo{Country: "Chinese (Simplified)", Language: "China"},
	"zh_SG":       LocaleInfo{Country: "Chinese (Simplified)", Language: "Singapore"},
	"zh_HK":       LocaleInfo{Country: "Chinese (Traditional)", Language: "Hong Kong"},
	"zh_TW":       LocaleInfo{Country: "Chinese (Traditional)", Language: "Taiwan"},
	"hr_HR":       LocaleInfo{Country: "Croatian", Language: "Croatia"},
	"cs_CZ":       LocaleInfo{Country: "Czech", Language: "Czech Republic"},
	"da_DK":       LocaleInfo{Country: "Danish", Language: "Denmark"},
	"nl_BE":       LocaleInfo{Country: "Dutch", Language: "Belgium"},
	"nl_NL":       LocaleInfo{Country: "Dutch", Language: "Netherlands"},
	"en_AU":       LocaleInfo{Country: "English", Language: "Australia"},
	"en_CA":       LocaleInfo{Country: "English", Language: "Canada"},
	"en_IN":       LocaleInfo{Country: "English", Language: "India"},
	"en_IE":       LocaleInfo{Country: "English", Language: "Ireland"},
	"en_MT":       LocaleInfo{Country: "English", Language: "Malta"},
	"en_NZ":       LocaleInfo{Country: "English", Language: "New Zealand"},
	"en_PH":       LocaleInfo{Country: "English", Language: "Philippines"},
	"en_SG":       LocaleInfo{Country: "English", Language: "Singapore"},
	"en_ZA":       LocaleInfo{Country: "English", Language: "South Africa"},
	"en_GB":       LocaleInfo{Country: "English", Language: "United Kingdom"},
	"en_US":       LocaleInfo{Country: "English", Language: "United States"},
	"et_EE":       LocaleInfo{Country: "Estonian", Language: "Estonia"},
	"fi_FI":       LocaleInfo{Country: "Finnish", Language: "Finland"},
	"fr_BE":       LocaleInfo{Country: "French", Language: "Belgium"},
	"fr_CA":       LocaleInfo{Country: "French", Language: "Canada"},
	"fr_FR":       LocaleInfo{Country: "French", Language: "France"},
	"fr_LU":       LocaleInfo{Country: "French", Language: "Luxembourg"},
	"fr_CH":       LocaleInfo{Country: "French", Language: "Switzerland"},
	"de_AT":       LocaleInfo{Country: "German", Language: "Austria"},
	"de_DE":       LocaleInfo{Country: "German", Language: "Germany"},
	"de_LU":       LocaleInfo{Country: "German", Language: "Luxembourg"},
	"de_CH":       LocaleInfo{Country: "German", Language: "Switzerland"},
	"el_CY":       LocaleInfo{Country: "Greek", Language: "Cyprus"},
	"el_GR":       LocaleInfo{Country: "Greek", Language: "Greece"},
	"iw_IL":       LocaleInfo{Country: "Hebrew", Language: "Israel"},
	"hi_IN":       LocaleInfo{Country: "Hindi", Language: "India"},
	"hu_HU":       LocaleInfo{Country: "Hungarian", Language: "Hungary"},
	"is_IS":       LocaleInfo{Country: "Icelandic", Language: "Iceland"},
	"in_ID":       LocaleInfo{Country: "Indonesian", Language: "Indonesia"},
	"ga_IE":       LocaleInfo{Country: "Irish", Language: "Ireland"},
	"it_IT":       LocaleInfo{Country: "Italian", Language: "Italy"},
	"it_CH":       LocaleInfo{Country: "Italian", Language: "Switzerland"},
	"ja_JP":       LocaleInfo{Country: "Japanese", Language: "Japan"},
	"ko_KR":       LocaleInfo{Country: "Korean", Language: "South Korea"},
	"lv_LV":       LocaleInfo{Country: "Latvian", Language: "Latvia"},
	"lt_LT":       LocaleInfo{Country: "Lithuanian", Language: "Lithuania"},
	"mk_MK":       LocaleInfo{Country: "Macedonian", Language: "Macedonia"},
	"ms_MY":       LocaleInfo{Country: "Malay", Language: "Malaysia"},
	"mt_MT":       LocaleInfo{Country: "Maltese", Language: "Malta"},
	"no_NO":       LocaleInfo{Country: "Norwegian (Bokmål)", Language: "Norway"},
	"no_NO_NY":    LocaleInfo{Country: "Norwegian (Nynorsk)", Language: "Norway"},
	"pl_PL":       LocaleInfo{Country: "Polish", Language: "Poland"},
	"pt_BR":       LocaleInfo{Country: "Portuguese", Language: "Brazil"},
	"pt_PT":       LocaleInfo{Country: "Portuguese", Language: "Portugal"},
	"ro_RO":       LocaleInfo{Country: "Romanian", Language: "Romania"},
	"ru_RU":       LocaleInfo{Country: "Russian", Language: "Russia"},
	"sr_BA":       LocaleInfo{Country: "Serbian (Cyrillic)", Language: "Bosnia and Herzegovina"},
	"sr_ME":       LocaleInfo{Country: "Serbian (Cyrillic)", Language: "Montenegro"},
	"sr_RS":       LocaleInfo{Country: "Serbian (Cyrillic)", Language: "Serbia"},
	"sr_La_tn_BA": LocaleInfo{Country: "Serbian (Latin)", Language: "Bosnia and Herzegovina"},
	"sr_La_tn_ME": LocaleInfo{Country: "Serbian (Latin)", Language: "Montenegro"},
	"sr_La_tn_RS": LocaleInfo{Country: "Serbian (Latin)", Language: "Serbia"},
	"sk_SK":       LocaleInfo{Country: "Slovak", Language: "Slovakia"},
	"sl_SI":       LocaleInfo{Country: "Slovenian", Language: "Slovenia"},
	"es_AR":       LocaleInfo{Country: "Spanish", Language: "Argentina"},
	"es_BO":       LocaleInfo{Country: "Spanish", Language: "Bolivia"},
	"es_CL":       LocaleInfo{Country: "Spanish", Language: "Chile"},
	"es_CO":       LocaleInfo{Country: "Spanish", Language: "Colombia"},
	"es_CR":       LocaleInfo{Country: "Spanish", Language: "Costa Rica"},
	"es_DO":       LocaleInfo{Country: "Spanish", Language: "Dominican Republic"},
	"es_EC":       LocaleInfo{Country: "Spanish", Language: "Ecuador"},
	"es_SV":       LocaleInfo{Country: "Spanish", Language: "El Salvador"},
	"es_GT":       LocaleInfo{Country: "Spanish", Language: "Guatemala"},
	"es_HN":       LocaleInfo{Country: "Spanish", Language: "Honduras"},
	"es_MX":       LocaleInfo{Country: "Spanish", Language: "Mexico"},
	"es_NI":       LocaleInfo{Country: "Spanish", Language: "Nicaragua"},
	"es_PA":       LocaleInfo{Country: "Spanish", Language: "Panama"},
	"es_PY":       LocaleInfo{Country: "Spanish", Language: "Paraguay"},
	"es_PE":       LocaleInfo{Country: "Spanish", Language: "Peru"},
	"es_PR":       LocaleInfo{Country: "Spanish", Language: "Puerto Rico"},
	"es_ES":       LocaleInfo{Country: "Spanish", Language: "Spain"},
	"es_US":       LocaleInfo{Country: "Spanish", Language: "United States"},
	"es_UY":       LocaleInfo{Country: "Spanish", Language: "Uruguay"},
	"es_VE":       LocaleInfo{Country: "Spanish", Language: "Venezuela"},
	"sv_SE":       LocaleInfo{Country: "Swedish", Language: "Sweden"},
	"th_TH":       LocaleInfo{Country: "Thai (Western digits)", Language: "Thailand"},
	"th_TH_TH":    LocaleInfo{Country: "Thai (Thai digits)", Language: "Thailand"},
	"tr_TR":       LocaleInfo{Country: "Turkish", Language: "Turkey"},
	"uk_UA":       LocaleInfo{Country: "Ukrainian", Language: "Ukraine"},
	"vi_VN":       LocaleInfo{Country: "Vietnamese", Language: "Vietnam"},
}
<|endoftext|>


<commit_before>package models

import (
	"appengine"
	"appengine/datastore"
	"fmt"
	"github.com/russross/blackfriday"
	"html/template"
	"regexp"
	"time"
)

type Entry struct {
	Id       int64
	Title    string // optional
	Content  string `datastore:",noindex"` // Markdown
	Datetime time.Time
	Created  time.Time
	Modified time.Time
	Tags     []string
	Public   bool
	// TODO(icco): Define a meta field that is a json hash of extra data
}

var HashtagRegex *regexp.Regexp = regexp.MustCompile(`(\s)#(\w+)`)
var TwitterHandleRegex *regexp.Regexp = regexp.MustCompile(`(\s)@([_A-Za-z0-9]+)`)

func NewEntry(title string, content string, datetime time.Time, public bool, tags []string) *Entry {
	e := new(Entry)

	// User supplied content
	e.Title = title
	e.Content = content
	e.Datetime = datetime
	e.Tags = tags
	e.Public = public

	// Computer generated content
	e.Created = time.Now()
	e.Modified = time.Now()

	return e
}

func ParseTags(text string) ([]string, error) {
	// http://golang.org/pkg/regexp/#Regexp.FindAllStringSubmatch
	finds := HashtagRegex.FindAllStringSubmatch(text, -1)
	ret := make([]string, 0)
	for _, v := range finds {
		if len(v) > 2 {
			ret = append(ret, v[2])
		}
	}

	return ret, nil
}

func GetEntry(c appengine.Context, id int64) (*Entry, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Filter("Id =", id).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Warningf("Error getting entry %d", id)
		return nil, err
	}

	return &entry, nil
}

func MaxId(c appengine.Context) (int64, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Id").Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		return 0, err
	}
	return entry.Id, nil
}

func AllPosts(c appengine.Context) (*[]Entry, error) {
	q := datastore.NewQuery("Entry").Order("-Datetime")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	return entries, err
}

func (e *Entry) HasId() bool {
	return (e.Id > 0)
}

func (e *Entry) Save(c appengine.Context) error {
	var k *datastore.Key
	if !e.HasId() {
		id, _ := MaxId(c)
		e.Id = id + 1
		k = datastore.NewIncompleteKey(c, "Entry", nil)
	} else {
		// Find the key
		var err error
		q := datastore.NewQuery("Entry").Filter("Id =", e.Id).Limit(1).KeysOnly()
		k, err = q.Run(c).Next(nil)
		if err != nil {
			return err
		}
	}

	// Figure out Tags
	tags, err := ParseTags(e.Content)
	if err != nil {
		return err
	}
	e.Tags = tags

	k2, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
		c.Infof("Old key: %+v; New Key: %+v", k, k2)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}

func (e *Entry) Url() string {
	return fmt.Sprintf("/post/%d", e.Id)
}

func (e *Entry) EditUrl() string {
	return fmt.Sprintf("/edit/%d", e.Id)
}

func (e *Entry) Html() template.HTML {
	return Markdown(e.Content)
}

func (e *Entry) PrevPost(c appengine.Context) string {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Datetime").Filter("Datetime <", e.Datetime).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Infof("Error getting previous post for %d.", e.Id)
		return ""
	}

	return entry.Url()
}

func (e *Entry) NextPost(c appengine.Context) string {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("Datetime").Filter("Datetime >", e.Datetime).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Infof("Error getting next post for %d.", e.Id)
		return ""
	}

	return entry.Url()
}

// Markdown.
func Markdown(args ...interface{}) template.HTML {
	inc := []byte(fmt.Sprintf("%s", args...))
	inc = twitterHandleToMarkdown(inc)
	inc = hashTagsToMarkdown(inc)
	s := blackfriday.MarkdownCommon(inc)
	return template.HTML(s)
}

func twitterHandleToMarkdown(in []byte) []byte {
	return TwitterHandleRegex.ReplaceAll(in, []byte("$1[@$2](http://twitter.com/$2)"))
}

func hashTagsToMarkdown(in []byte) []byte {
	return HashtagRegex.ReplaceAll(in, []byte("$1[#$2](/tags/$2)"))
}
<commit_msg>attempt at link finding<commit_after>package models

import (
	"appengine"
	"appengine/datastore"
	"fmt"
	"github.com/russross/blackfriday"
	"html/template"
	"regexp"
	"time"
)

type Entry struct {
	Id       int64
	Title    string // optional
	Content  string `datastore:",noindex"` // Markdown
	Datetime time.Time
	Created  time.Time
	Modified time.Time
	Tags     []string
	Public   bool
	// TODO(icco): Define a meta field that is a json hash of extra data
}

var HashtagRegex *regexp.Regexp = regexp.MustCompile(`(\s)#(\w+)`)
var TwitterHandleRegex *regexp.Regexp = regexp.MustCompile(`(\s)@([_A-Za-z0-9]+)`)

func NewEntry(title string, content string, datetime time.Time, public bool, tags []string) *Entry {
	e := new(Entry)

	// User supplied content
	e.Title = title
	e.Content = content
	e.Datetime = datetime
	e.Tags = tags
	e.Public = public

	// Computer generated content
	e.Created = time.Now()
	e.Modified = time.Now()

	return e
}

func ParseTags(text string) ([]string, error) {
	// http://golang.org/pkg/regexp/#Regexp.FindAllStringSubmatch
	finds := HashtagRegex.FindAllStringSubmatch(text, -1)
	ret := make([]string, 0)
	for _, v := range finds {
		if len(v) > 2 {
			ret = append(ret, v[2])
		}
	}

	return ret, nil
}

func GetEntry(c appengine.Context, id int64) (*Entry, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Filter("Id =", id).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Warningf("Error getting entry %d", id)
		return nil, err
	}

	return &entry, nil
}

func MaxId(c appengine.Context) (int64, error) {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Id").Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		return 0, err
	}
	return entry.Id, nil
}

func AllPosts(c appengine.Context) (*[]Entry, error) {
	q := datastore.NewQuery("Entry").Order("-Datetime")
	entries := new([]Entry)
	_, err := q.GetAll(c, entries)
	return entries, err
}

func (e *Entry) HasId() bool {
	return (e.Id > 0)
}

func (e *Entry) Save(c appengine.Context) error {
	var k *datastore.Key
	if !e.HasId() {
		id, _ := MaxId(c)
		e.Id = id + 1
		k = datastore.NewIncompleteKey(c, "Entry", nil)
	} else {
		// Find the key
		var err error
		q := datastore.NewQuery("Entry").Filter("Id =", e.Id).Limit(1).KeysOnly()
		k, err = q.Run(c).Next(nil)
		if err != nil {
			return err
		}
	}

	// Pull out links
	GetLinksFromContent(c, e.Content)

	// Figure out Tags
	tags, err := ParseTags(e.Content)
	if err != nil {
		return err
	}
	e.Tags = tags

	k2, err := datastore.Put(c, k, e)
	if err == nil {
		c.Infof("Wrote %+v", e)
		c.Infof("Old key: %+v; New Key: %+v", k, k2)
	} else {
		c.Warningf("Error writing entry: %v", e)
	}
	return err
}

func (e *Entry) Url() string {
	return fmt.Sprintf("/post/%d", e.Id)
}

func (e *Entry) EditUrl() string {
	return fmt.Sprintf("/edit/%d", e.Id)
}

func (e *Entry) Html() template.HTML {
	return Markdown(e.Content)
}

func (e *Entry) PrevPost(c appengine.Context) string {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("-Datetime").Filter("Datetime <", e.Datetime).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Infof("Error getting previous post for %d.", e.Id)
		return ""
	}

	return entry.Url()
}

func (e *Entry) NextPost(c appengine.Context) string {
	var entry Entry
	q := datastore.NewQuery("Entry").Order("Datetime").Filter("Datetime >", e.Datetime).Limit(1)
	_, err := q.Run(c).Next(&entry)
	if err != nil {
		c.Infof("Error getting next post for %d.", e.Id)
		return ""
	}

	return entry.Url()
}

func GetLinksFromContent(c appengine.Context, content string) ([]string, error) {
	httpRegex := regexp.MustCompile(`http:\/\/`)
	matches := httpRegex.FindAllString(content, -1)
	if matches == nil {
		return []string{}, nil
	}

	for _, match := range matches {
		c.Infof("%+v", match)
	}

	return []string{}, nil
}

// Markdown.
func Markdown(args ...interface{}) template.HTML {
	inc := []byte(fmt.Sprintf("%s", args...))
	inc = twitterHandleToMarkdown(inc)
	inc = hashTagsToMarkdown(inc)
	s := blackfriday.MarkdownCommon(inc)
	return template.HTML(s)
}

func twitterHandleToMarkdown(in []byte) []byte {
	return TwitterHandleRegex.ReplaceAll(in, []byte("$1[@$2](http://twitter.com/$2)"))
}

func hashTagsToMarkdown(in []byte) []byte {
	return HashtagRegex.ReplaceAll(in, []byte("$1[#$2](/tags/$2)"))
}
<|endoftext|>


<commit_before>package models

import (
	"sync"
)

type ChangeDirection struct {
	Gopher    *Gopher
	Direction Direction
	Wait      *sync.WaitGroup
}

type Field struct {
	Height, Width int
	needed        int //number of players needed to start the game

	Gophers []*Gopher
	Board   [][]bool

	Change chan ChangeDirection
	Remove chan *Gopher

	mu         *sync.RWMutex
	inProgress bool

	cycles int
}

func New(height, width int) *Field {
	field := &Field{
		Height: height,
		Width:  width,
		Board:  make([][]bool, height),
		Change: make(chan ChangeDirection),
		Remove: make(chan *Gopher),
		mu:     new(sync.RWMutex),
	}

	for i := range field.Board {
		field.Board[i] = make([]bool, width)
	}

	return field
}

func (f *Field) Add(g *Gopher) error {
	f.mu.RLock()

	if f.inProgress {
		f.mu.RUnlock()
		return ErrInProgress
	}
	f.mu.RUnlock()

	f.Gophers = append(f.Gophers, g)
	if len(f.Gophers) == f.needed {
		f.mu.Lock()
		f.inProgress = true
		f.mu.Unlock()
		go f.start()
	}

	return nil
}

func (f *Field) remove(gopher *Gopher) {
	for i, currGopher := range f.Gophers {
		if currGopher == gopher {
			f.Gophers, f.Gophers[len(f.Gophers)-1] = append(f.Gophers[:i], f.Gophers[i+1:]...), nil
		}
	}
}

func (f *Field) increment(g *Gopher) bool {
	g.Path = append(g.Path, Coordinate{g.X, g.Y})
	switch g.Direction {
	case North:
		g.Y++
	case South:
		g.Y--
	case East:
		g.X++
	case West:
		g.X--
	}

	if g.X == 100 || g.Y == 100 {
		return true
	}
	if f.Board[g.X][g.Y] {
		return true
	}

	f.Board[g.X][g.Y] = true
	return false
}

func (f *Field) clearPath(g *Gopher) {
	for _, c := range g.Path {
		f.Board[c.X][c.Y] = false
	}
}

func (f *Field) Start() {
	for {
		select {
		case dir := <-f.Change:
			dir.Gopher.Direction = dir.Direction
			dir.Wait.Done()
		default:
			for i, gopher := range f.Gophers {
			f.remove(gopher)
			if len(f.Gophers) == 1 {
				f.end()
				return
			}
				if f.increment(gopher) {
					//gopher collided, clear it's path and remove it
					//from the field
					f.clearPath(gopher)
					f.Gophers, f.Gophers[len(f.Gophers)-1] = append(f.Gophers[:i], f.Gophers[i+1:]...), nil
					f.remove(gopher)

					if len(f.Gophers) == 1 {
						f.End()
						f.end()
						return
					}
				}
				f.Broadcast()
			}
		}
	}
}

func (f *Field) End() {

}

func (f *Field) Broadcast() {

}
<commit_msg>Limit board cycles to 1 per 100 milliseconds<commit_after>package models

import (
	"errors"
	"sync"
	"time"
)

type ChangeDirection struct {
	Gopher    *Gopher
	Direction Direction
	Wait      *sync.WaitGroup
}

type Field struct {
	Height, Width int
	needed        int //number of players needed to start the game

	Gophers []*Gopher
	Board   [][]bool

	Change chan ChangeDirection
	Remove chan *Gopher

	mu         *sync.RWMutex
	inProgress bool

	cycles int
}

var ErrInProgress = errors.New("Game is already in progress.")

func New(height, width int) *Field {
	field := &Field{
		Height: height,
		Width:  width,
		Board:  make([][]bool, height),
		Change: make(chan ChangeDirection),
		Remove: make(chan *Gopher),
		mu:     new(sync.RWMutex),
	}

	for i := range field.Board {
		field.Board[i] = make([]bool, width)
	}

	return field
}

func (f *Field) Add(g *Gopher) error {
	f.mu.RLock()

	if f.inProgress {
		f.mu.RUnlock()
		return ErrInProgress
	}
	f.mu.RUnlock()

	f.Gophers = append(f.Gophers, g)
	if len(f.Gophers) == f.needed {
		f.mu.Lock()
		f.inProgress = true
		f.mu.Unlock()
		go f.start()
	}

	return nil
}

func (f *Field) remove(gopher *Gopher) {
	for i, currGopher := range f.Gophers {
		if currGopher == gopher {
			f.Gophers, f.Gophers[len(f.Gophers)-1] = append(f.Gophers[:i], f.Gophers[i+1:]...), nil
		}
	}
}

func (f *Field) increment(g *Gopher) bool {
	g.Path = append(g.Path, Coordinate{g.X, g.Y})
	switch g.Direction {
	case North:
		g.Y++
	case South:
		g.Y--
	case East:
		g.X++
	case West:
		g.X--
	}

	if g.X == f.Height || g.Y == f.Width {
		return true
	}
	if f.Board[g.X][g.Y] {
		return true
	}

	f.Board[g.X][g.Y] = true
	g.Score++

	if f.cycles > 10 {
		f.Board[g.Path[0].X][g.Path[0].Y] = false
	} else {
		f.cycles++
	}

	return false
}

func (f *Field) clearPath(g *Gopher) {
	for _, c := range g.Path {
		f.Board[c.X][c.Y] = false
	}
}

func (f *Field) start() {
	tick := time.NewTicker(100 * time.Millisecond)
	for {
		select {
		case dir := <-f.Change:
			dir.Gopher.Direction = dir.Direction
			dir.Wait.Done()
		case gopher := <-f.Remove:
			f.remove(gopher)
			if len(f.Gophers) == 1 {
				f.end()
				return
			}
		case <-tick.C:
			for _, gopher := range f.Gophers {
				if f.increment(gopher) {
					//gopher collided, clear it's path and remove it
					//from the field
					f.clearPath(gopher)
					f.remove(gopher)

					if len(f.Gophers) == 1 {
						f.end()
						return
					}
				}

				f.broadcast()
			}
		}
	}
}

func (f *Field) end() {

}

func (f *Field) broadcast() {

}
<|endoftext|>


<commit_before>package models

import (
	"sync"
)

type ChangeDirection struct {
	Gopher    *Gopher
	Direction Direction
	Wait      *sync.WaitGroup
}

type Field struct {
	Gophers []*Gopher
	Board   [100][100]bool
	Change  chan ChangeDirection
func New(height, width int) *Field {
	field := &Field{
		Height: height,
		Width:  width,
		Board:  make([][]bool, height),
		Change: make(chan ChangeDirection),
		Remove: make(chan *Gopher),
		mu:     new(sync.RWMutex),
	}

	for i := range field.Board {
		field.Board[i] = make([]bool, width)
	}

	return field
}

func (f *Field) Add(g *Gopher) error {
	f.mu.RLock()

	if f.inProgress {
		f.mu.RUnlock()
		return ErrInProgress
	}
	f.mu.RUnlock()

	f.Gophers = append(f.Gophers, g)
	if len(f.Gophers) == f.needed {
		f.mu.Lock()
		f.inProgress = true
		f.mu.Unlock()
		go f.start()
	}

	return nil
}

func (f *Field) increment(g *Gopher) bool {
	g.Path = append(g.Path, Coordinate{g.X, g.Y})
	switch g.Direction {
	case North:
		g.Y++
	case South:
		g.Y--
	case East:
		g.X++
	case West:
		g.X--
	}

	if g.X == 100 || g.Y == 100 {
		return true
	}
	if f.Board[g.X][g.Y] {
		return true
	}

	f.Board[g.X][g.Y] = true
	return false
}

func (f *Field) clearPath(g *Gopher) {
	for _, c := range g.Path {
		f.Board[c.X][c.Y] = false
	}
}

func (f *Field) Start() {
	for {
		select {
		case dir := <-f.Change:
			dir.Gopher.Direction = dir.Direction
			dir.Wait.Done()
		default:
			for i, gopher := range f.Gophers {
				if f.increment(gopher) {
					//gopher collided, clear it's path and remove it
					//from the field
					f.clearPath(gopher)
					f.Gophers, f.Gophers[len(f.Gophers)-1] = append(f.Gophers[:i], f.Gophers[i+1:]...), nil
					if len(f.Gophers) == 1 {
						f.End()
						return
					}
				}
				f.Broadcast()
			}
		}
	}
}

func (f *Field) End() {

}

func (f *Field) Broadcast() {

}
<commit_msg>Add (*Field).remove(), use it.<commit_after>package models

import (
	"sync"
)

type ChangeDirection struct {
	Gopher    *Gopher
	Direction Direction
	Wait      *sync.WaitGroup
}

type Field struct {
	Gophers []*Gopher
	Board   [100][100]bool
	Change  chan ChangeDirection
func New(height, width int) *Field {
	field := &Field{
		Height: height,
		Width:  width,
		Board:  make([][]bool, height),
		Change: make(chan ChangeDirection),
		Remove: make(chan *Gopher),
		mu:     new(sync.RWMutex),
	}

	for i := range field.Board {
		field.Board[i] = make([]bool, width)
	}

	return field
}

func (f *Field) Add(g *Gopher) error {
	f.mu.RLock()

	if f.inProgress {
		f.mu.RUnlock()
		return ErrInProgress
	}
	f.mu.RUnlock()

	f.Gophers = append(f.Gophers, g)
	if len(f.Gophers) == f.needed {
		f.mu.Lock()
		f.inProgress = true
		f.mu.Unlock()
		go f.start()
	}

	return nil
}

func (f *Field) remove(gopher *Gopher) {
	for i, currGopher := range f.Gophers {
		if currGopher == gopher {
			f.Gophers, f.Gophers[len(f.Gophers)-1] = append(f.Gophers[:i], f.Gophers[i+1:]...), nil
		}
	}
}

func (f *Field) increment(g *Gopher) bool {
	g.Path = append(g.Path, Coordinate{g.X, g.Y})
	switch g.Direction {
	case North:
		g.Y++
	case South:
		g.Y--
	case East:
		g.X++
	case West:
		g.X--
	}

	if g.X == 100 || g.Y == 100 {
		return true
	}
	if f.Board[g.X][g.Y] {
		return true
	}

	f.Board[g.X][g.Y] = true
	return false
}

func (f *Field) clearPath(g *Gopher) {
	for _, c := range g.Path {
		f.Board[c.X][c.Y] = false
	}
}

func (f *Field) Start() {
	for {
		select {
		case dir := <-f.Change:
			dir.Gopher.Direction = dir.Direction
			dir.Wait.Done()
		default:
			for i, gopher := range f.Gophers {
			f.remove(gopher)
			if len(f.Gophers) == 1 {
				f.end()
				return
			}
				if f.increment(gopher) {
					//gopher collided, clear it's path and remove it
					//from the field
					f.clearPath(gopher)
					f.Gophers, f.Gophers[len(f.Gophers)-1] = append(f.Gophers[:i], f.Gophers[i+1:]...), nil
					f.remove(gopher)

					if len(f.Gophers) == 1 {
						f.End()
						f.end()
						return
					}
				}
				f.Broadcast()
			}
		}
	}
}

func (f *Field) End() {

}

func (f *Field) Broadcast() {

}
<|endoftext|>


<commit_before>package lazycache

import "fmt"
import "net/http"
import "strings"
import "strconv"
import "path/filepath"

//import "strings"
import "io"
import "encoding/json"
import "github.com/amarburg/go-fast-png"
import "image/jpeg"
import "bytes"
import "regexp"
import "time"

import "sync"

import "github.com/amarburg/go-lazyquicktime"

var leadingNumbers, _ = regexp.Compile("^\\d+")

//go:generate easyjson -all $GOFILE
// I've isolated these structs so I can use ffjson

type MoovHandlerTiming struct {
	Handler, Metadata, Extraction, Encode time.Duration
}

type moovOutputMetadata struct {
	URL       string
	NumFrames int
	Duration  float32
	FileSize  int64
}

type QTEntry struct {
	lqt      *lazyquicktime.LazyQuicktime
	metadata moovOutputMetadata
}

type QTStore struct {
	Cache map[string](*QTEntry)
	Mutex sync.Mutex

	Stats struct {
		Requests, Misses int64
	}
}

var qtCache QTStore

func init() {
	qtCache = QTStore{
		Cache: make(map[string](*QTEntry)),
	}
}

func (cache *QTStore) getLQT(node *Node) (*QTEntry, error) {

	cache.Mutex.Lock()
	defer cache.Mutex.Unlock()

	cache.Stats.Requests++

	// Initialize or update as necessary
	Logger.Log("debug", fmt.Sprintf("Querying metadata store for %s", node.Path))
	qte, has := cache.Cache[node.trimPath]

	if !has {
		cache.Stats.Misses++

		//Logger.Log("msg", fmt.Errorf("Initializing LazyFile to %s", node.Path))
		fs, err := node.Fs.LazyFile(node.Path)

		if err != nil {
			return nil, fmt.Errorf("Something went boom while opening the HTTP Source!")
		}

		//Logger.Log("msg", fmt.Sprintf("Need to pull quicktime information for %s", fs.Path()))
		lqt, err := lazyquicktime.LoadMovMetadata(fs)
		if err != nil {
			return nil, fmt.Errorf("Something went boom while storing the quicktime file: %s", err.Error())
		}

		//Logger.Log("msg", fmt.Sprintf("Updating metadata store for %s", fs.Path()))
		qte = &QTEntry{
			lqt: lqt,
			metadata: moovOutputMetadata{
				FileSize:  lqt.FileSize,
				URL:       node.Path,
				NumFrames: lqt.NumFrames(),
				Duration:  lqt.Duration(),
			},
		}

		cache.Cache[node.trimPath] = qte

	} else {
		Logger.Log("msg", fmt.Sprintf("lqt cache has entry for %s", node.Path))
	}

	return qte, nil
}

func MoovHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node {
	Logger.Log("msg", fmt.Sprintf("Quicktime handler: %s with residual path (%d): (%s)", node.Path, len(path), strings.Join(path, ":")))

	timing := MoovHandlerTiming{}
	movStart := time.Now()

	// uri := node.Fs.Uri
	// uri.Path += node.Path

	metadataStart := time.Now()
	qte, err := qtCache.getLQT(node)
	timeTrack(metadataStart, &timing.Metadata)

	if err != nil {
		Logger.Log("msg", err.Error())

		b, _ := json.MarshalIndent(struct {
			URL, Error string
		}{
			URL:   node.Path,
			Error: err.Error(),
		}, "", "  ")

		// http.Error(w, err.Error(), 500)
		w.Write(b)
		return nil
	}

	if len(path) == 0 {
		// Leaf node
		startEncode := time.Now()

		Logger.Log("msg", fmt.Sprintf("Returning movie information for %s", node.Path))

		b, err := qte.metadata.MarshalJSON()
		if err != nil {
			fmt.Fprintln(w, "JSON error:", err)
		}

		timeTrack(startEncode, &timing.Encode)

		//Logger.Log("msg", fmt.Sprintf("..... done"))

		w.Write(b)
	} else {

		// Handle any residual path elements (frames, etc) here
		switch strings.ToLower(path[0]) {
		case "frame":
			extractFrame(node, qte, path[1:], w, req, &timing)
		default:
			http.Error(w, fmt.Sprintf("Didn't understand request \"%s\"", path[0]), 500)
		}
	}

	timeTrack(movStart, &timing.Handler)

	t, _ := timing.MarshalJSON()
	Logger.Log("timing", t)

	return nil
}

func extractFrame(node *Node, qte *QTEntry, path []string, w http.ResponseWriter, req *http.Request, timing *MoovHandlerTiming) {

	if qte == nil || qte.lqt == nil {
		http.Error(w, "Error in extractFrame", 500)
		return
	}

	if len(path) == 0 {
		http.Error(w, fmt.Sprintf("Need to specify frame number"), 500)
		return

	}

	frameNum, err := strconv.Atoi(leadingNumbers.FindString(path[0]))
	if err != nil {
		http.Error(w, fmt.Sprintf("Error parsing frame number \"%s\"", path[0]), 500)
		return
	}

	if frameNum > qte.metadata.NumFrames {
		http.Error(w, fmt.Sprintf("Requested frame %d in movie of length %d frames", frameNum, qte.metadata.NumFrames), 400)
		return
	}

	if frameNum < 1 {
		http.Error(w, "Requested frame 0, Quicktime movies start with frame 1", 400)
		return
	}

	// Looks for extension
	extension := filepath.Ext(path[0])

	var contentType string

	switch extension {
	case ".jpg", ".jpeg":
		contentType = "image/jpeg"
		extension = ".jpg"
	case "", ".png":
		extension = ".png"
		contentType = "image/png"
	default:
		http.Error(w, fmt.Sprintf("Unknown image extension \"%s\"", extension), 500)
		return
	}

	UUID := req.URL.Path + extension
	url, ok := ImageCache.Url(UUID)

	if ok {
		Logger.Log("msg", fmt.Sprintf("Image %s exists in the Image store at %s", UUID, url))
		// Set Content-Type or response
		w.Header().Set("Content-Type", contentType)
		// w.Header().Set("Location", url)
		Logger.Log("msg", fmt.Sprintf("Redirecting to %s", url))
		http.Redirect(w, req, url, http.StatusTemporaryRedirect)

	} else {

		startExt := time.Now()
		img, err := qte.lqt.ExtractFrame(frameNum)
		if err != nil {
			http.Error(w, fmt.Sprintf("Error generating image for frame %d: %s", frameNum, err.Error()), 500)
			return
		}
		timeTrack(startExt, &timing.Extraction)

		buffer := new(bytes.Buffer)

		startEncode := time.Now()

		switch contentType {
		case "image/png":
			encoder := fastpng.Encoder{
				CompressionLevel: fastpng.DefaultCompression,
			}
			err = encoder.Encode(buffer, img)
		case "image/jpeg":
			err = jpeg.Encode(buffer, img, &jpeg.Options{Quality: jpeg.DefaultQuality})
		}

		timeTrack(startEncode, &timing.Encode)

		Logger.Log("debug", fmt.Sprintf("%s size %d MB\n", contentType, buffer.Len()/(1024*1024)))

		imgReader := bytes.NewReader(buffer.Bytes())

		// write image to Image store
		ImageCache.Store(UUID, imgReader)

		imgReader.Seek(0, io.SeekStart)
		_, err = imgReader.WriteTo(w)
		if err != nil {
			fmt.Printf("Error writing to HTTP buffer: %s\n", err.Error())
		}

	}

}
<commit_msg>Added support for BMP output.<commit_after>package lazycache

import "fmt"
import "net/http"
import "strings"
import "strconv"
import "path/filepath"

//import "strings"
import "io"
import "encoding/json"
import "github.com/amarburg/go-fast-png"
import "image/jpeg"
import "golang.org/x/image/bmp"
import "bytes"
import "regexp"
import "time"

import "sync"

import "github.com/amarburg/go-lazyquicktime"

var leadingNumbers, _ = regexp.Compile("^\\d+")

//go:generate easyjson -all $GOFILE
// I've isolated these structs so I can use ffjson

type MoovHandlerTiming struct {
	Handler, Metadata, Extraction, Encode time.Duration
}

type moovOutputMetadata struct {
	URL       string
	NumFrames int
	Duration  float32
	FileSize  int64
}

type QTEntry struct {
	lqt      *lazyquicktime.LazyQuicktime
	metadata moovOutputMetadata
}

type QTStore struct {
	Cache map[string](*QTEntry)
	Mutex sync.Mutex

	Stats struct {
		Requests, Misses int64
	}
}

var qtCache QTStore

func init() {
	qtCache = QTStore{
		Cache: make(map[string](*QTEntry)),
	}
}

func (cache *QTStore) getLQT(node *Node) (*QTEntry, error) {

	cache.Mutex.Lock()
	defer cache.Mutex.Unlock()

	cache.Stats.Requests++

	// Initialize or update as necessary
	Logger.Log("debug", fmt.Sprintf("Querying metadata store for %s", node.Path))
	qte, has := cache.Cache[node.trimPath]

	if !has {
		cache.Stats.Misses++

		//Logger.Log("msg", fmt.Errorf("Initializing LazyFile to %s", node.Path))
		fs, err := node.Fs.LazyFile(node.Path)

		if err != nil {
			return nil, fmt.Errorf("Something went boom while opening the HTTP Source!")
		}

		//Logger.Log("msg", fmt.Sprintf("Need to pull quicktime information for %s", fs.Path()))
		lqt, err := lazyquicktime.LoadMovMetadata(fs)
		if err != nil {
			return nil, fmt.Errorf("Something went boom while storing the quicktime file: %s", err.Error())
		}

		//Logger.Log("msg", fmt.Sprintf("Updating metadata store for %s", fs.Path()))
		qte = &QTEntry{
			lqt: lqt,
			metadata: moovOutputMetadata{
				FileSize:  lqt.FileSize,
				URL:       node.Path,
				NumFrames: lqt.NumFrames(),
				Duration:  lqt.Duration(),
			},
		}

		cache.Cache[node.trimPath] = qte

	} else {
		Logger.Log("msg", fmt.Sprintf("lqt cache has entry for %s", node.Path))
	}

	return qte, nil
}

func MoovHandler(node *Node, path []string, w http.ResponseWriter, req *http.Request) *Node {
	Logger.Log("msg", fmt.Sprintf("Quicktime handler: %s with residual path (%d): (%s)", node.Path, len(path), strings.Join(path, ":")))

	timing := MoovHandlerTiming{}
	movStart := time.Now()

	// uri := node.Fs.Uri
	// uri.Path += node.Path

	metadataStart := time.Now()
	qte, err := qtCache.getLQT(node)
	timeTrack(metadataStart, &timing.Metadata)

	if err != nil {
		Logger.Log("msg", err.Error())

		b, _ := json.MarshalIndent(struct {
			URL, Error string
		}{
			URL:   node.Path,
			Error: err.Error(),
		}, "", "  ")

		// http.Error(w, err.Error(), 500)
		w.Write(b)
		return nil
	}

	if len(path) == 0 {
		// Leaf node
		startEncode := time.Now()

		Logger.Log("msg", fmt.Sprintf("Returning movie information for %s", node.Path))

		b, err := qte.metadata.MarshalJSON()
		if err != nil {
			fmt.Fprintln(w, "JSON error:", err)
		}

		timeTrack(startEncode, &timing.Encode)

		//Logger.Log("msg", fmt.Sprintf("..... done"))

		w.Write(b)
	} else {

		// Handle any residual path elements (frames, etc) here
		switch strings.ToLower(path[0]) {
		case "frame":
			extractFrame(node, qte, path[1:], w, req, &timing)
		default:
			http.Error(w, fmt.Sprintf("Didn't understand request \"%s\"", path[0]), 500)
		}
	}

	timeTrack(movStart, &timing.Handler)

	t, _ := timing.MarshalJSON()
	Logger.Log("timing", t)

	return nil
}

func extractFrame(node *Node, qte *QTEntry, path []string, w http.ResponseWriter, req *http.Request, timing *MoovHandlerTiming) {

	if qte == nil || qte.lqt == nil {
		http.Error(w, "Error in extractFrame", 500)
		return
	}

	if len(path) == 0 {
		http.Error(w, fmt.Sprintf("Need to specify frame number"), 500)
		return

	}

	frameNum, err := strconv.Atoi(leadingNumbers.FindString(path[0]))
	if err != nil {
		http.Error(w, fmt.Sprintf("Error parsing frame number \"%s\"", path[0]), 500)
		return
	}

	if frameNum > qte.metadata.NumFrames {
		http.Error(w, fmt.Sprintf("Requested frame %d in movie of length %d frames", frameNum, qte.metadata.NumFrames), 400)
		return
	}

	if frameNum < 1 {
		http.Error(w, "Requested frame 0, Quicktime movies start with frame 1", 400)
		return
	}

	// Looks for extension
	extension := filepath.Ext(path[0])

	var contentType string

	switch extension {
	case ".bmp":
		contentType = "image/bmp"
		extension = ".bmp"
	case ".jpg", ".jpeg":
		contentType = "image/jpeg"
		extension = ".jpg"
	case "", ".png":
		extension = ".png"
		contentType = "image/png"
	default:
		http.Error(w, fmt.Sprintf("Unknown image extension \"%s\"", extension), 500)
		return
	}

	UUID := req.URL.Path + extension
	url, ok := ImageCache.Url(UUID)

	if ok {
		Logger.Log("msg", fmt.Sprintf("Image %s exists in the Image store at %s", UUID, url))
		// Set Content-Type or response
		w.Header().Set("Content-Type", contentType)
		Logger.Log("msg", fmt.Sprintf("Redirecting to %s", url))
		http.Redirect(w, req, url, http.StatusTemporaryRedirect)

	} else {

		startExt := time.Now()
		img, err := qte.lqt.ExtractFrame(frameNum)
		if err != nil {
			http.Error(w, fmt.Sprintf("Error generating image for frame %d: %s", frameNum, err.Error()), 500)
			return
		}
		timeTrack(startExt, &timing.Extraction)

		buffer := new(bytes.Buffer)

		startEncode := time.Now()

		switch contentType {
		case "image/png":
			encoder := fastpng.Encoder{
				CompressionLevel: fastpng.BestSpeed,
			}
			err = encoder.Encode(buffer, img)
		case "image/jpeg":
			err = jpeg.Encode(buffer, img, &jpeg.Options{Quality: jpeg.DefaultQuality})
		case "image/bmp":
			err = bmp.Encode(buffer, img)
		}

		timeTrack(startEncode, &timing.Encode)

		Logger.Log("debug", fmt.Sprintf("%s size %d MB\n", contentType, buffer.Len()/(1024*1024)))

		imgReader := bytes.NewReader(buffer.Bytes())

		// write image to Image store
		ImageCache.Store(UUID, imgReader)

		imgReader.Seek(0, io.SeekStart)
		_, err = imgReader.WriteTo(w)
		if err != nil {
			fmt.Printf("Error writing to HTTP buffer: %s\n", err.Error())
		}

	}

}
<|endoftext|>


<commit_before>package chart

import (
	"testing"
	"time"

	"github.com/blendlabs/go-assert"
)

func TestTimeValueFormatterWithFormat(t *testing.T) {
	assert := assert.New(t)

	d := time.Now()
	di := Time.ToFloat64(d)
	df := float64(di)

	s := TimeValueFormatterWithFormat(d, DefaultDateFormat)
	si := TimeValueFormatterWithFormat(di, DefaultDateFormat)
	sf := TimeValueFormatterWithFormat(df, DefaultDateFormat)
	assert.Equal(s, si)
	assert.Equal(s, sf)

	sd := TimeValueFormatter(d)
	sdi := TimeValueFormatter(di)
	sdf := TimeValueFormatter(df)
	assert.Equal(s, sd)
	assert.Equal(s, sdi)
	assert.Equal(s, sdf)
}

func TestFloatValueFormatter(t *testing.T) {
	assert := assert.New(t)
	assert.Equal("1234.00", FloatValueFormatter(1234.00))
}

func TestFloatValueFormatterWithFloat32Input(t *testing.T) {
	assert := assert.New(t)
	assert.Equal("1234.00", FloatValueFormatter(float32(1234.00)))
}

func TestFloatValueFormatterWithIntegerInput(t *testing.T) {
	assert := assert.New(t)
	assert.Equal("1234.00", FloatValueFormatter(1234))
}

func TestFloatValueFormatterWithInt64Input(t *testing.T) {
	assert := assert.New(t)
	assert.Equal("1234.00", FloatValueFormatter(int64(1234)))
}

func TestFloatValueFormatterWithFormat(t *testing.T) {
	assert := assert.New(t)

	v := 123.456
	sv := FloatValueFormatterWithFormat(v, "%.3f")
	assert.Equal("123.456", sv)
	assert.Equal("123.000", FloatValueFormatterWithFormat(123, "%.3f"))
}
<commit_msg>fixing tests<commit_after>package chart

import (
	"testing"
	"time"

	"github.com/blendlabs/go-assert"
)

func TestTimeValueFormatterWithFormat(t *testing.T) {
	assert := assert.New(t)

	d := time.Now()
	di := Time.ToFloat64(d)
	df := float64(di)

	s := formatTime(d, DefaultDateFormat)
	si := formatTime(di, DefaultDateFormat)
	sf := formatTime(df, DefaultDateFormat)
	assert.Equal(s, si)
	assert.Equal(s, sf)

	sd := TimeValueFormatter(d)
	sdi := TimeValueFormatter(di)
	sdf := TimeValueFormatter(df)
	assert.Equal(s, sd)
	assert.Equal(s, sdi)
	assert.Equal(s, sdf)
}

func TestFloatValueFormatter(t *testing.T) {
	assert := assert.New(t)
	assert.Equal("1234.00", FloatValueFormatter(1234.00))
}

func TestFloatValueFormatterWithFloat32Input(t *testing.T) {
	assert := assert.New(t)
	assert.Equal("1234.00", FloatValueFormatter(float32(1234.00)))
}

func TestFloatValueFormatterWithIntegerInput(t *testing.T) {
	assert := assert.New(t)
	assert.Equal("1234.00", FloatValueFormatter(1234))
}

func TestFloatValueFormatterWithInt64Input(t *testing.T) {
	assert := assert.New(t)
	assert.Equal("1234.00", FloatValueFormatter(int64(1234)))
}

func TestFloatValueFormatterWithFormat(t *testing.T) {
	assert := assert.New(t)

	v := 123.456
	sv := FloatValueFormatterWithFormat(v, "%.3f")
	assert.Equal("123.456", sv)
	assert.Equal("123.000", FloatValueFormatterWithFormat(123, "%.3f"))
}
<|endoftext|>


<commit_before>package test_helpers

import (
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"runtime"
	"syscall"
	"testing"
	"time"
)

// Indexed by mountpoint. Initialized in doInit().
var MountInfo map[string]mountInfo

type mountInfo struct {
	// PID of the running gocryptfs process. Set by Mount().
	pid int
	// List of open FDs of the running gocrypts process. Set by Mount().
	fds []string
}

// Mount CIPHERDIR "c" on PLAINDIR "p"
// Creates "p" if it does not exist.
func Mount(c string, p string, showOutput bool, extraArgs ...string) error {
	args := []string{"-q", "-wpanic", "-nosyslog", "-fg", fmt.Sprintf("-notifypid=%d", os.Getpid())}
	args = append(args, extraArgs...)
	//args = append(args, "-fusedebug")
	//args = append(args, "-d")
	args = append(args, c, p)

	if _, err := os.Stat(p); err != nil {
		err = os.Mkdir(p, 0777)
		if err != nil {
			return err
		}
	}

	cmd := exec.Command(GocryptfsBinary, args...)
	if showOutput {
		// The Go test logic waits for our stdout to close, and when we share
		// it with the subprocess, it will wait for it to close it as well.
		// Use an intermediate pipe so the tests do not hang when unmouting
		// fails.
		pr, pw, err := os.Pipe()
		if err != nil {
			return err
		}
		// We can close the fd after cmd.Run() has executed
		defer pw.Close()
		cmd.Stderr = pw
		cmd.Stdout = pw
		go func() {
			io.Copy(os.Stdout, pr)
			pr.Close()
		}()
	}

	// Two things can happen:
	// 1) The mount fails and the process exits
	// 2) The mount succeeds and the process sends us USR1
	chanExit := make(chan error, 1)
	chanUsr1 := make(chan os.Signal, 1)
	signal.Notify(chanUsr1, syscall.SIGUSR1)

	// Start the process and save the PID
	err := cmd.Start()
	if err != nil {
		return err
	}
	pid := cmd.Process.Pid

	// Wait for exit or usr1
	go func() {
		chanExit <- cmd.Wait()
	}()
	select {
	case err := <-chanExit:
		return err
	case <-chanUsr1:
		// noop
	case <-time.After(1 * time.Second):
		log.Panicf("Timeout waiting for process %d", pid)
	}

	// Save PID and open FDs
	MountInfo[p] = mountInfo{pid, ListFds(pid)}
	return nil
}

// MountOrExit calls Mount() and exits on failure.
func MountOrExit(c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		fmt.Printf("mount failed: %v\n", err)
		os.Exit(1)
	}
}

// MountOrFatal calls Mount() and calls t.Fatal() on failure.
func MountOrFatal(t *testing.T, c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		t.Fatal(fmt.Errorf("mount failed: %v", err))
	}
}

// UnmountPanic tries to umount "dir" and panics on error.
func UnmountPanic(dir string) {
	err := UnmountErr(dir)
	if err != nil {
		fmt.Printf("UnmountPanic: %v. Running lsof %s\n", err, dir)
		cmd := exec.Command("lsof", dir)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
		panic("UnmountPanic: unmount failed: " + err.Error())
	}
}

// UnmountErr tries to unmount "dir", retrying 10 times, and returns the
// resulting error.
func UnmountErr(dir string) (err error) {
	var fdsNow []string
	pid := MountInfo[dir].pid
	fds := MountInfo[dir].fds
	if pid <= 0 {
		fmt.Printf("UnmountErr: %q was not found in MountInfo, cannot check for FD leaks\n", dir)
	}

	max := 10
	// When a new filesystem is mounted, Gnome tries to read files like
	// .xdg-volume-info, autorun.inf, .Trash.
	// If we try to unmount before Gnome is done, the unmount fails with
	// "Device or resource busy", causing spurious test failures.
	// Retry a few times to hide that problem.
	for i := 1; i <= max; i++ {
		if pid > 0 {
			fdsNow = ListFds(pid)
		}
		cmd := exec.Command(UnmountScript, "-u", dir)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		err = cmd.Run()
		if err == nil {
			if pid > 0 && len(fdsNow) > len(fds) {
				fmt.Printf("FD leak? Details:\nold=%v \nnew=%v\n", fds, fdsNow)
			}
			return nil
		}
		code := ExtractCmdExitCode(err)
		fmt.Printf("UnmountErr: got exit code %d, retrying (%d/%d)\n", code, i, max)
		time.Sleep(100 * time.Millisecond)
	}
	return err
}

// ListFds lists the open file descriptors for process "pid". Pass pid=0 for
// ourselves.
func ListFds(pid int) []string {
	// We need /proc to get the list of fds for other processes. Only exists
	// on Linux.
	if runtime.GOOS != "linux" && pid > 0 {
		return nil
	}
	// Both Linux and MacOS have /dev/fd
	dir := "/dev/fd"
	if pid > 0 {
		dir = fmt.Sprintf("/proc/%d/fd", pid)
	}
	f, err := os.Open(dir)
	if err != nil {
		log.Panic(err)
	}
	defer f.Close()
	names, err := f.Readdirnames(0)
	if err != nil {
		log.Panic(err)
	}
	for i, n := range names {
		// Note: Readdirnames filters "." and ".."
		target, _ := os.Readlink(dir + "/" + n)
		names[i] = n + "=" + target
	}
	return names
}
<commit_msg>tests: ListFds: show r/w status<commit_after>package test_helpers

import (
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"runtime"
	"syscall"
	"testing"
	"time"
)

// Indexed by mountpoint. Initialized in doInit().
var MountInfo map[string]mountInfo

type mountInfo struct {
	// PID of the running gocryptfs process. Set by Mount().
	pid int
	// List of open FDs of the running gocrypts process. Set by Mount().
	fds []string
}

// Mount CIPHERDIR "c" on PLAINDIR "p"
// Creates "p" if it does not exist.
func Mount(c string, p string, showOutput bool, extraArgs ...string) error {
	args := []string{"-q", "-wpanic", "-nosyslog", "-fg", fmt.Sprintf("-notifypid=%d", os.Getpid())}
	args = append(args, extraArgs...)
	//args = append(args, "-fusedebug")
	//args = append(args, "-d")
	args = append(args, c, p)

	if _, err := os.Stat(p); err != nil {
		err = os.Mkdir(p, 0777)
		if err != nil {
			return err
		}
	}

	cmd := exec.Command(GocryptfsBinary, args...)
	if showOutput {
		// The Go test logic waits for our stdout to close, and when we share
		// it with the subprocess, it will wait for it to close it as well.
		// Use an intermediate pipe so the tests do not hang when unmouting
		// fails.
		pr, pw, err := os.Pipe()
		if err != nil {
			return err
		}
		// We can close the fd after cmd.Run() has executed
		defer pw.Close()
		cmd.Stderr = pw
		cmd.Stdout = pw
		go func() {
			io.Copy(os.Stdout, pr)
			pr.Close()
		}()
	}

	// Two things can happen:
	// 1) The mount fails and the process exits
	// 2) The mount succeeds and the process sends us USR1
	chanExit := make(chan error, 1)
	chanUsr1 := make(chan os.Signal, 1)
	signal.Notify(chanUsr1, syscall.SIGUSR1)

	// Start the process and save the PID
	err := cmd.Start()
	if err != nil {
		return err
	}
	pid := cmd.Process.Pid

	// Wait for exit or usr1
	go func() {
		chanExit <- cmd.Wait()
	}()
	select {
	case err := <-chanExit:
		return err
	case <-chanUsr1:
		// noop
	case <-time.After(1 * time.Second):
		log.Panicf("Timeout waiting for process %d", pid)
	}

	// Save PID and open FDs
	MountInfo[p] = mountInfo{pid, ListFds(pid)}
	return nil
}

// MountOrExit calls Mount() and exits on failure.
func MountOrExit(c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		fmt.Printf("mount failed: %v\n", err)
		os.Exit(1)
	}
}

// MountOrFatal calls Mount() and calls t.Fatal() on failure.
func MountOrFatal(t *testing.T, c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		t.Fatal(fmt.Errorf("mount failed: %v", err))
	}
}

// UnmountPanic tries to umount "dir" and panics on error.
func UnmountPanic(dir string) {
	err := UnmountErr(dir)
	if err != nil {
		fmt.Printf("UnmountPanic: %v. Running lsof %s\n", err, dir)
		cmd := exec.Command("lsof", dir)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmd.Run()
		panic("UnmountPanic: unmount failed: " + err.Error())
	}
}

// UnmountErr tries to unmount "dir", retrying 10 times, and returns the
// resulting error.
func UnmountErr(dir string) (err error) {
	var fdsNow []string
	pid := MountInfo[dir].pid
	fds := MountInfo[dir].fds
	if pid <= 0 {
		fmt.Printf("UnmountErr: %q was not found in MountInfo, cannot check for FD leaks\n", dir)
	}

	max := 10
	// When a new filesystem is mounted, Gnome tries to read files like
	// .xdg-volume-info, autorun.inf, .Trash.
	// If we try to unmount before Gnome is done, the unmount fails with
	// "Device or resource busy", causing spurious test failures.
	// Retry a few times to hide that problem.
	for i := 1; i <= max; i++ {
		if pid > 0 {
			fdsNow = ListFds(pid)
		}
		cmd := exec.Command(UnmountScript, "-u", dir)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		err = cmd.Run()
		if err == nil {
			if pid > 0 && len(fdsNow) > len(fds) {
				fmt.Printf("FD leak? Details:\nold=%v \nnew=%v\n", fds, fdsNow)
			}
			return nil
		}
		code := ExtractCmdExitCode(err)
		fmt.Printf("UnmountErr: got exit code %d, retrying (%d/%d)\n", code, i, max)
		time.Sleep(100 * time.Millisecond)
	}
	return err
}

// ListFds lists the open file descriptors for process "pid". Pass pid=0 for
// ourselves.
func ListFds(pid int) []string {
	// We need /proc to get the list of fds for other processes. Only exists
	// on Linux.
	if runtime.GOOS != "linux" && pid > 0 {
		return nil
	}
	// Both Linux and MacOS have /dev/fd
	dir := "/dev/fd"
	if pid > 0 {
		dir = fmt.Sprintf("/proc/%d/fd", pid)
	}
	f, err := os.Open(dir)
	if err != nil {
		log.Panic(err)
	}
	defer f.Close()
	// Note: Readdirnames filters "." and ".."
	names, err := f.Readdirnames(0)
	if err != nil {
		log.Panic(err)
	}
	var out []string
	for _, n := range names {
		fdPath := dir + "/" + n
		fi, err := os.Lstat(fdPath)
		if err != nil {
			// fd was closed in the meantime
			continue
		}
		if fi.Mode()&0400 > 0 {
			n += "r"
		}
		if fi.Mode()&0200 > 0 {
			n += "w"
		}
		target, err := os.Readlink(fdPath)
		if err != nil {
			// fd was closed in the meantime
			continue
		}
		out = append(out, n+"="+target)
	}
	return out
}
<|endoftext|>


<commit_before>// Copyright (C) 2015  The GoHBase Authors.  All rights reserved.
// This file is part of GoHBase.
// Use of this source code is governed by the Apache License 2.0
// that can be found in the COPYING file.

package region

import (
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/tsuna/gohbase/hrpc"
	"github.com/tsuna/gohbase/pb"
)

var (
	// ErrShortWrite is used when the writer thread only succeeds in writing
	// part of its buffer to the socket, and not all of the buffer was sent
	ErrShortWrite = errors.New("short write occurred while writing to socket")
)

// Client manages a connection to a RegionServer.
type Client struct {
	id uint32

	conn net.Conn

	// Hostname or IP address of the RegionServer.
	host string

	// Port of the RegionServer.
	port uint16

	// writeMutex is used to prevent multiple threads from writing to the
	// socket at the same time.
	writeMutex *sync.Mutex

	// sendErr is set once a write fails.
	sendErr error

	rpcs []hrpc.Call

	// Once the rpcs list has grown to a large enough size, this channel is
	// written to to notify the writer thread that it should stop sleeping and
	// process the list
	process chan struct{}
}

// NewClient creates a new RegionClient.
func NewClient(host string, port uint16) (*Client, error) {
	addr := fmt.Sprintf("%s:%d", host, port)
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil,
			fmt.Errorf("failed to connect to the RegionServer at %s: %s", addr, err)
	}
	c := &Client{
		conn:       conn,
		host:       host,
		port:       port,
		writeMutex: &sync.Mutex{},
		process:    make(chan struct{}),
	}
	err = c.sendHello()
	if err != nil {
		return nil, err
	}
	go c.processRpcs()
	return c, nil
}

func (c *Client) processRpcs() {
	for {
		// TODO: make this value configurable
		select {
		case <-time.After(time.Millisecond * 100):
			c.writeMutex.Lock()
		case <-c.process:
			// We don't acquire the lock here, because the thread that sent
			// something on the process channel will have locked the mutex,
			// and will not release it so as to transfer ownership
		}

		// If c.sendErr is set, this writer has encountered an unrecoverable
		// error. It will repeat the error it encountered to any outstanding
		// rpcs, and not attempt to send them.
		if c.sendErr != nil {
			for _, rpc := range c.rpcs {
				resch := rpc.GetResultChan()
				resch <- hrpc.RPCResult{nil, c.sendErr}
			}
			c.rpcs = nil
			c.writeMutex.Unlock()
			break
		}
		rpcs := make([]hrpc.Call, len(c.rpcs))
		for i, rpc := range c.rpcs {
			rpcs[i] = rpc
		}
		c.rpcs = nil
		c.writeMutex.Unlock()

		var newrpcs []hrpc.Call

		for _, rpc := range rpcs {
			// If the deadline has been exceeded, don't bother sending the
			// request. The function that placed the RPC in our queue should
			// stop waiting for a result and return an error.
			select {
			case _, ok := <-rpc.Context().Done():
				if !ok {
					continue
				}
			default:
			}

			resch := rpc.GetResultChan()
			msg, err := c.SendRPC(rpc)
			if isRecoverable(err) {
				newrpcs = append(newrpcs, rpc)
			} else {
				if err != nil {
					c.sendErr = err
				}
				resch <- hrpc.RPCResult{msg, err}
			}
		}

		c.writeMutex.Lock()
		c.rpcs = append(c.rpcs, newrpcs...)
		c.writeMutex.Unlock()
	}
}

func isRecoverable(err error) bool {
	// TODO: identify which errors we can treat as recoverable
	if err == ErrShortWrite {
		return true
	}
	return false
}

// Sends the given buffer to the RegionServer.
func (c *Client) write(buf []byte) error {
	n, err := c.conn.Write(buf)

	if err != nil {
		// There was an error while writing
		return err
	}
	if n != len(buf) {
		// We failed to write the entire buffer
		// TODO: Perhaps handle this in another way than closing down
		return ErrShortWrite
	}
	return nil
}

// Tries to read enough data to fully fill up the given buffer.
func (c *Client) readFully(buf []byte) error {
	// TODO: Handle short reads.
	n, err := c.conn.Read(buf)
	if err != nil {
		return fmt.Errorf("Failed to read from the RS: %s", err)
	} else if n != len(buf) {
		return fmt.Errorf("Failed to read everything from the RS: %s", err)
	}
	return nil
}

// Sends the "hello" message needed when opening a new connection.
func (c *Client) sendHello() error {
	connHeader := &pb.ConnectionHeader{
		UserInfo: &pb.UserInformation{
			EffectiveUser: proto.String("gopher"),
		},
		ServiceName: proto.String("ClientService"),
		//CellBlockCodecClass: "org.apache.hadoop.hbase.codec.KeyValueCodec",
	}
	data, err := proto.Marshal(connHeader)
	if err != nil {
		return fmt.Errorf("failed to marshal connection header: %s", err)
	}

	const header = "HBas\x00\x50" // \x50 = Simple Auth.
	buf := make([]byte, 0, len(header)+4+len(data))
	buf = append(buf, header...)
	buf = buf[:len(header)+4]
	binary.BigEndian.PutUint32(buf[6:], uint32(len(data)))
	buf = append(buf, data...)

	return c.write(buf)
}

// QueueRPC will add an rpc call to the queue for processing by the writer
// goroutine
func (c *Client) QueueRPC(rpc hrpc.Call) {
	c.writeMutex.Lock()
	c.rpcs = append(c.rpcs, rpc)
	if len(c.rpcs) > 100 { //TODO: make configurable or pick a good number
		c.process <- struct{}{}
		// We don't release the lock here, because we want to transfer ownership
		// of the lock to the goroutine that processes the RPCs
	} else {
		c.writeMutex.Unlock()
	}
}

// SendRPC sends an RPC out to the wire.
// Returns the response (for now, as the call is synchronous).
func (c *Client) SendRPC(rpc hrpc.Call) (proto.Message, error) {
	// Header.
	c.id++
	reqheader := &pb.RequestHeader{
		CallId:       &c.id,
		MethodName:   proto.String(rpc.Name()),
		RequestParam: proto.Bool(true),
	}

	payload, err := rpc.Serialize()
	if err != nil {
		return nil, fmt.Errorf("Failed to serialize RPC: %s", err)
	}
	payloadLen := proto.EncodeVarint(uint64(len(payload)))

	headerData, err := proto.Marshal(reqheader)
	if err != nil {
		return nil, fmt.Errorf("Failed to marshal Get request: %s", err)
	}

	buf := make([]byte, 5, 4+1+len(headerData)+len(payloadLen)+len(payload))
	binary.BigEndian.PutUint32(buf, uint32(cap(buf)-4))
	buf[4] = byte(len(headerData))
	buf = append(buf, headerData...)
	buf = append(buf, payloadLen...)
	buf = append(buf, payload...)

	err = c.write(buf)
	if err != nil {
		return nil, err
	}

	var sz [4]byte
	err = c.readFully(sz[:])
	if err != nil {
		return nil, err
	}

	buf = make([]byte, binary.BigEndian.Uint32(sz[:]))
	err = c.readFully(buf)
	if err != nil {
		return nil, err
	}

	resp := &pb.ResponseHeader{}
	respLen, nb := proto.DecodeVarint(buf)
	buf = buf[nb:]
	err = proto.UnmarshalMerge(buf[:respLen], resp)
	buf = buf[respLen:]
	if err != nil {
		return nil, fmt.Errorf("Failed to deserialize the response header: %s", err)
	}
	if resp.CallId == nil {
		return nil, fmt.Errorf("Response doesn't have a call ID!")
	} else if *resp.CallId != c.id {
		return nil, fmt.Errorf("Not the callId we expected: %d", *resp.CallId)
	}

	if resp.Exception != nil {
		return nil, fmt.Errorf("remote exception %s:\n%s",
			*resp.Exception.ExceptionClassName, *resp.Exception.StackTrace)
	}

	respLen, nb = proto.DecodeVarint(buf)
	buf = buf[nb:]
	rpcResp := rpc.NewResponse()
	err = proto.UnmarshalMerge(buf, rpcResp)
	buf = buf[respLen:]

	return rpcResp, err
}
<commit_msg>Separate reader thread<commit_after>// Copyright (C) 2015  The GoHBase Authors.  All rights reserved.
// This file is part of GoHBase.
// Use of this source code is governed by the Apache License 2.0
// that can be found in the COPYING file.

package region

import (
	"encoding/binary"
	"errors"
	"fmt"
	"log"
	"net"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/tsuna/gohbase/hrpc"
	"github.com/tsuna/gohbase/pb"
)

var (
	// ErrShortWrite is used when the writer thread only succeeds in writing
	// part of its buffer to the socket, and not all of the buffer was sent
	ErrShortWrite = errors.New("short write occurred while writing to socket")
)

// Client manages a connection to a RegionServer.
type Client struct {
	id uint32

	conn net.Conn

	// Hostname or IP address of the RegionServer.
	host string

	// Port of the RegionServer.
	port uint16

	// writeMutex is used to prevent multiple threads from writing to the
	// socket at the same time.
	writeMutex *sync.Mutex

	// sendErr is set once a write fails.
	sendErr error

	rpcs []hrpc.Call

	// Once the rpcs list has grown to a large enough size, this channel is
	// written to to notify the writer thread that it should stop sleeping and
	// process the list
	process chan struct{}

	// sentRPCs contains the mapping of sent call IDs to RPC calls, so that when
	// a response is received it can be tied to the correct RPC
	sentRPCs      map[uint32]hrpc.Call
	sentRPCsMutex *sync.Mutex
}

// NewClient creates a new RegionClient.
func NewClient(host string, port uint16) (*Client, error) {
	addr := fmt.Sprintf("%s:%d", host, port)
	conn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil,
			fmt.Errorf("failed to connect to the RegionServer at %s: %s", addr, err)
	}
	c := &Client{
		conn:          conn,
		host:          host,
		port:          port,
		writeMutex:    &sync.Mutex{},
		process:       make(chan struct{}),
		sentRPCsMutex: &sync.Mutex{},
		sentRPCs:      make(map[uint32]hrpc.Call),
	}
	err = c.sendHello()
	if err != nil {
		return nil, err
	}
	go c.processRpcs() // Writer goroutine
	go c.receiveRpcs() // Reader goroutine
	return c, nil
}

func (c *Client) processRpcs() {
	for {
		select {
		// TODO: make this value configurable
		case <-time.After(time.Millisecond * 100):
			c.writeMutex.Lock()
		case <-c.process:
			// We don't acquire the lock here, because the thread that sent
			// something on the process channel will have locked the mutex,
			// and will not release it so as to transfer ownership
		}

		// If c.sendErr is set, this writer has encountered an unrecoverable
		// error. It will repeat the error it encountered to any outstanding
		// rpcs, and not attempt to send them.
		if c.sendErr != nil {
			for _, rpc := range c.rpcs {
				resch := rpc.GetResultChan()
				resch <- hrpc.RPCResult{nil, c.sendErr}
			}
			c.rpcs = nil
			c.writeMutex.Unlock()
			continue
		}
		rpcs := make([]hrpc.Call, len(c.rpcs))
		for i, rpc := range c.rpcs {
			rpcs[i] = rpc
		}
		c.rpcs = nil
		c.writeMutex.Unlock()

		var newrpcs []hrpc.Call

		for _, rpc := range rpcs {
			// If the deadline has been exceeded, don't bother sending the
			// request. The function that placed the RPC in our queue should
			// stop waiting for a result and return an error.
			select {
			case _, ok := <-rpc.Context().Done():
				if !ok {
					continue
				}
			default:
			}

			err := c.sendRPC(rpc)
			if isRecoverable(err) {
				newrpcs = append(newrpcs, rpc)
			} else if err != nil {
				c.sendErr = err
				resch := rpc.GetResultChan()
				resch <- hrpc.RPCResult{nil, err}
			}
		}

		c.writeMutex.Lock()
		c.rpcs = append(c.rpcs, newrpcs...)
		c.writeMutex.Unlock()
	}
}

func (c *Client) receiveRpcs() {
	for {
		var sz [4]byte
		err := c.readFully(sz[:])
		if err != nil {
			log.Printf("Encountered an error while reading: %v\n", err)
			continue
		}

		buf := make([]byte, binary.BigEndian.Uint32(sz[:]))
		err = c.readFully(buf)
		if err != nil {
			log.Printf("Encountered an error while reading: %v\n", err)
			continue
		}

		resp := &pb.ResponseHeader{}
		respLen, nb := proto.DecodeVarint(buf)
		buf = buf[nb:]
		err = proto.UnmarshalMerge(buf[:respLen], resp)
		buf = buf[respLen:]
		if err != nil {
			log.Printf("Failed to deserialize the response header: %s\n", err)
			continue
		}
		if resp.CallId == nil {
			log.Printf("Response doesn't have a call ID!\n")
			continue
		}

		c.sentRPCsMutex.Lock()
		rpc, ok := c.sentRPCs[*resp.CallId]
		c.sentRPCsMutex.Unlock()

		if !ok {
			log.Printf("Received a response with an unexpected call ID!\n")
			continue
		}

		if resp.Exception != nil {
			// TODO: Properly handle this error
			log.Printf("remote exception %s:\n%s",
				*resp.Exception.ExceptionClassName, *resp.Exception.StackTrace)
		}

		respLen, nb = proto.DecodeVarint(buf)
		buf = buf[nb:]
		rpcResp := rpc.NewResponse()
		err = proto.UnmarshalMerge(buf, rpcResp)
		buf = buf[respLen:]

		resch := rpc.GetResultChan()
		resch <- hrpc.RPCResult{rpcResp, err}

		c.sentRPCsMutex.Lock()
		delete(c.sentRPCs, *resp.CallId)

		c.sentRPCsMutex.Unlock()
	}
}

func isRecoverable(err error) bool {
	// TODO: identify which errors we can treat as recoverable
	if err == ErrShortWrite {
		return true
	}
	return false
}

// Sends the given buffer to the RegionServer.
func (c *Client) write(buf []byte) error {
	n, err := c.conn.Write(buf)

	if err != nil {
		// There was an error while writing
		return err
	}
	if n != len(buf) {
		// We failed to write the entire buffer
		// TODO: Perhaps handle this in another way than closing down
		return ErrShortWrite
	}
	return nil
}

// Tries to read enough data to fully fill up the given buffer.
func (c *Client) readFully(buf []byte) error {
	// TODO: Handle short reads.
	n, err := c.conn.Read(buf)
	if err != nil {
		return fmt.Errorf("Failed to read from the RS: %s", err)
	} else if n != len(buf) {
		return fmt.Errorf("Failed to read everything from the RS: %s", err)
	}
	return nil
}

// Sends the "hello" message needed when opening a new connection.
func (c *Client) sendHello() error {
	connHeader := &pb.ConnectionHeader{
		UserInfo: &pb.UserInformation{
			EffectiveUser: proto.String("gopher"),
		},
		ServiceName: proto.String("ClientService"),
		//CellBlockCodecClass: "org.apache.hadoop.hbase.codec.KeyValueCodec",
	}
	data, err := proto.Marshal(connHeader)
	if err != nil {
		return fmt.Errorf("failed to marshal connection header: %s", err)
	}

	const header = "HBas\x00\x50" // \x50 = Simple Auth.
	buf := make([]byte, 0, len(header)+4+len(data))
	buf = append(buf, header...)
	buf = buf[:len(header)+4]
	binary.BigEndian.PutUint32(buf[6:], uint32(len(data)))
	buf = append(buf, data...)

	return c.write(buf)
}

// QueueRPC will add an rpc call to the queue for processing by the writer
// goroutine
func (c *Client) QueueRPC(rpc hrpc.Call) {
	c.writeMutex.Lock()
	c.rpcs = append(c.rpcs, rpc)
	if len(c.rpcs) > 100 { //TODO: make configurable or pick a good number
		c.process <- struct{}{}
		// We don't release the lock here, because we want to transfer ownership
		// of the lock to the goroutine that processes the RPCs
	} else {
		c.writeMutex.Unlock()
	}
}

// sendRPC sends an RPC out to the wire.
// Returns the response (for now, as the call is synchronous).
func (c *Client) sendRPC(rpc hrpc.Call) error {
	// Header.
	c.id++
	reqheader := &pb.RequestHeader{
		CallId:       &c.id,
		MethodName:   proto.String(rpc.Name()),
		RequestParam: proto.Bool(true),
	}

	payload, err := rpc.Serialize()
	if err != nil {
		return fmt.Errorf("Failed to serialize RPC: %s", err)
	}
	payloadLen := proto.EncodeVarint(uint64(len(payload)))

	headerData, err := proto.Marshal(reqheader)
	if err != nil {
		return fmt.Errorf("Failed to marshal Get request: %s", err)
	}

	buf := make([]byte, 5, 4+1+len(headerData)+len(payloadLen)+len(payload))
	binary.BigEndian.PutUint32(buf, uint32(cap(buf)-4))
	buf[4] = byte(len(headerData))
	buf = append(buf, headerData...)
	buf = append(buf, payloadLen...)
	buf = append(buf, payload...)

	err = c.write(buf)
	if err != nil {
		return err
	}

	c.sentRPCsMutex.Lock()
	c.sentRPCs[c.id] = rpc
	c.sentRPCsMutex.Unlock()

	return nil
}
<|endoftext|>


<commit_before>package version

import (
	"testing"
)

func TestVersioning(t *testing.T) {
	if !NewVersionAvailable("v0.1.0") {
		t.Error("should be a version newer than v0.1.0")
	}
}
<commit_msg>Fix version check test too<commit_after>package version

import (
	"testing"
)

func TestVersioning(t *testing.T) {
	if !NewVersionAvailable("v1.0.0") {
		t.Error("should be a version newer than v1.0.0")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire

import (
	"fmt"
	"io"
)

// MaxBlockLocatorsPerMsg is the maximum number of block locator hashes allowed
// per message.
const MaxBlockLocatorsPerMsg = 500

// MsgGetBlocks implements the Message interface and represents a bitcoin
// getblocks message.  It is used to request a list of blocks starting after the
// last known hash in the slice of block locator hashes.  The list is returned
// via an inv message (MsgInv) and is limited by a specific hash to stop at or
// the maximum number of blocks per message, which is currently 500.
//
// Set the HashStop field to the hash at which to stop and use
// AddBlockLocatorHash to build up the list of block locator hashes.
//
// The algorithm for building the block locator hashes should be to add the
// hashes in reverse order until you reach the genesis block.  In order to keep
// the list of locator hashes to a reasonable number of entries, first add the
// most recent 10 block hashes, then double the step each loop iteration to
// exponentially decrease the number of hashes the further away from head and
// closer to the genesis block you get.
type MsgGetBlocks struct {
	ProtocolVersion    uint32
	BlockLocatorHashes []*ShaHash
	HashStop           ShaHash
}

// AddBlockLocatorHash adds a new block locator hash to the message.
func (msg *MsgGetBlocks) AddBlockLocatorHash(hash *ShaHash) error {
	if len(msg.BlockLocatorHashes)+1 > MaxBlockLocatorsPerMsg {
		str := "MsgGetBlocks.AddBlockLocatorHash: too many block " +
			"locator hashes for message [max %v]"
		return fmt.Errorf(str, MaxBlockLocatorsPerMsg)
	}

	msg.BlockLocatorHashes = append(msg.BlockLocatorHashes, hash)
	return nil
}

// BtcDecode decodes r using the bitcoin protocol encoding into the receiver.
// This is part of the Message interface implementation.
func (msg *MsgGetBlocks) BtcDecode(r io.Reader, pver uint32) error {
	err := readElement(r, &msg.ProtocolVersion)
	if err != nil {
		return err
	}

	// Read num block locator hashes and limit to max.
	count, err := readVarInt(r, pver)
	if err != nil {
		return err
	}
	if count > MaxBlockLocatorsPerMsg {
		str := "%v: too many block locator hashes in message [%v]"
		return fmt.Errorf(str, "MsgGetBlocks.BtcDecode", count)
	}

	for i := uint64(0); i < count; i++ {
		sha := ShaHash{}
		err := readElement(r, &sha)
		if err != nil {
			return err
		}
		msg.AddBlockLocatorHash(&sha)
	}

	err = readElement(r, &msg.HashStop)
	if err != nil {
		return err
	}

	return nil
}

// BtcEncode encodes the receiver to w using the bitcoin protocol encoding.
// This is part of the Message interface implementation.
func (msg *MsgGetBlocks) BtcEncode(w io.Writer, pver uint32) error {
	count := len(msg.BlockLocatorHashes)
	if count > MaxBlockLocatorsPerMsg {
		str := "%v: too many block locator hashes in message [%v]"
		return fmt.Errorf(str, "MsgGetBlocks.BtcEncode", count)
	}

	err := writeElement(w, msg.ProtocolVersion)
	if err != nil {
		return err
	}

	err = writeVarInt(w, pver, uint64(count))
	if err != nil {
		return err
	}

	for _, hash := range msg.BlockLocatorHashes {
		err = writeElement(w, hash)
		if err != nil {
			return err
		}
	}

	err = writeElement(w, msg.HashStop)
	if err != nil {
		return err
	}

	return nil
}

// Command returns the protocol command string for the message.  This is part
// of the Message interface implementation.
func (msg *MsgGetBlocks) Command() string {
	return cmdGetBlocks
}

// MaxPayloadLength returns the maximum length the payload can be for the
// receiver.  This is part of the Message interface implementation.
func (msg *MsgGetBlocks) MaxPayloadLength(pver uint32) uint32 {
	// Protocol version 4 bytes + num hashes (varInt) + max block locator
	// hashes + hash stop.
	return 4 + maxVarIntPayload + (MaxBlockLocatorsPerMsg * HashSize) + HashSize
}

// NewMsgGetBlocks returns a new bitcoin getblocks message that conforms to the
// Message interface using the passed parameters and defaults for the remaining
// fields.
func NewMsgGetBlocks(hashStop *ShaHash) *MsgGetBlocks {
	return &MsgGetBlocks{
		ProtocolVersion: ProtocolVersion,
		HashStop:        *hashStop,
	}
}
<commit_msg>Convert MsgGetBlocks errors to MessageError type.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcwire

import (
	"fmt"
	"io"
)

// MaxBlockLocatorsPerMsg is the maximum number of block locator hashes allowed
// per message.
const MaxBlockLocatorsPerMsg = 500

// MsgGetBlocks implements the Message interface and represents a bitcoin
// getblocks message.  It is used to request a list of blocks starting after the
// last known hash in the slice of block locator hashes.  The list is returned
// via an inv message (MsgInv) and is limited by a specific hash to stop at or
// the maximum number of blocks per message, which is currently 500.
//
// Set the HashStop field to the hash at which to stop and use
// AddBlockLocatorHash to build up the list of block locator hashes.
//
// The algorithm for building the block locator hashes should be to add the
// hashes in reverse order until you reach the genesis block.  In order to keep
// the list of locator hashes to a reasonable number of entries, first add the
// most recent 10 block hashes, then double the step each loop iteration to
// exponentially decrease the number of hashes the further away from head and
// closer to the genesis block you get.
type MsgGetBlocks struct {
	ProtocolVersion    uint32
	BlockLocatorHashes []*ShaHash
	HashStop           ShaHash
}

// AddBlockLocatorHash adds a new block locator hash to the message.
func (msg *MsgGetBlocks) AddBlockLocatorHash(hash *ShaHash) error {
	if len(msg.BlockLocatorHashes)+1 > MaxBlockLocatorsPerMsg {
		str := fmt.Sprintf("too many block locator hashes for message [max %v]",
			MaxBlockLocatorsPerMsg)
		return messageError("MsgGetBlocks.AddBlockLocatorHash", str)
	}

	msg.BlockLocatorHashes = append(msg.BlockLocatorHashes, hash)
	return nil
}

// BtcDecode decodes r using the bitcoin protocol encoding into the receiver.
// This is part of the Message interface implementation.
func (msg *MsgGetBlocks) BtcDecode(r io.Reader, pver uint32) error {
	err := readElement(r, &msg.ProtocolVersion)
	if err != nil {
		return err
	}

	// Read num block locator hashes and limit to max.
	count, err := readVarInt(r, pver)
	if err != nil {
		return err
	}
	if count > MaxBlockLocatorsPerMsg {
		str := fmt.Sprintf("too many block locator hashes for message "+
			"[count %v, max %v]", count, MaxBlockLocatorsPerMsg)
		return messageError("MsgGetBlocks.BtcDecode", str)
	}

	for i := uint64(0); i < count; i++ {
		sha := ShaHash{}
		err := readElement(r, &sha)
		if err != nil {
			return err
		}
		msg.AddBlockLocatorHash(&sha)
	}

	err = readElement(r, &msg.HashStop)
	if err != nil {
		return err
	}

	return nil
}

// BtcEncode encodes the receiver to w using the bitcoin protocol encoding.
// This is part of the Message interface implementation.
func (msg *MsgGetBlocks) BtcEncode(w io.Writer, pver uint32) error {
	count := len(msg.BlockLocatorHashes)
	if count > MaxBlockLocatorsPerMsg {
		str := fmt.Sprintf("too many block locator hashes for message "+
			"[count %v, max %v]", count, MaxBlockLocatorsPerMsg)
		return messageError("MsgGetBlocks.BtcEncode", str)
	}

	err := writeElement(w, msg.ProtocolVersion)
	if err != nil {
		return err
	}

	err = writeVarInt(w, pver, uint64(count))
	if err != nil {
		return err
	}

	for _, hash := range msg.BlockLocatorHashes {
		err = writeElement(w, hash)
		if err != nil {
			return err
		}
	}

	err = writeElement(w, msg.HashStop)
	if err != nil {
		return err
	}

	return nil
}

// Command returns the protocol command string for the message.  This is part
// of the Message interface implementation.
func (msg *MsgGetBlocks) Command() string {
	return cmdGetBlocks
}

// MaxPayloadLength returns the maximum length the payload can be for the
// receiver.  This is part of the Message interface implementation.
func (msg *MsgGetBlocks) MaxPayloadLength(pver uint32) uint32 {
	// Protocol version 4 bytes + num hashes (varInt) + max block locator
	// hashes + hash stop.
	return 4 + maxVarIntPayload + (MaxBlockLocatorsPerMsg * HashSize) + HashSize
}

// NewMsgGetBlocks returns a new bitcoin getblocks message that conforms to the
// Message interface using the passed parameters and defaults for the remaining
// fields.
func NewMsgGetBlocks(hashStop *ShaHash) *MsgGetBlocks {
	return &MsgGetBlocks{
		ProtocolVersion: ProtocolVersion,
		HashStop:        *hashStop,
	}
}
<|endoftext|>


<commit_before>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package costs

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/trackit/jsonlog"
	"github.com/trackit/trackit2/db"
	"github.com/trackit/trackit2/es"
	"github.com/trackit/trackit2/routes"
	"github.com/trackit/trackit2/users"

	"gopkg.in/olivere/elastic.v5"
)

// simpleCriterionMap will map simple criterion to the boolean true.
// This will be used in parseCriterionQueryParams to validate the queryParam.
// It does not take into account the 'tag:*' criterion as it is not fixed.
var simpleCriterionMap = map[string]bool{
	"year":    true,
	"month":   true,
	"week":    true,
	"day":     true,
	"account": true,
	"product": true,
	"region":  true,
}

// esQueryParams will store the parsed query params
type esQueryParams struct {
	dateBegin         time.Time
	dateEnd           time.Time
	accountList       []string
	aggregationParams []string
}

// costQueryArgs allows to get required queryArgs params
var costsQueryArgs = []routes.QueryArg{
	routes.QueryArg{
		Name:        "accounts",
		Description: "List of comma separated AWS accounts ids",
		Type:        routes.QueryArgStringSlice{},
		Optional:    true,
	},
	routes.QueryArg{
		Name:        "begin",
		Description: "Begining of date interval. Format is ISO8601",
		Type:        routes.QueryArgDate{},
		Optional:    false,
	},
	routes.QueryArg{
		Name:        "end",
		Description: "End of date interval. Format is ISO8601",
		Type:        routes.QueryArgDate{},
		Optional:    false,
	},
	routes.QueryArg{
		Name:        "by",
		Description: "Criteria for the ES aggregation, comma separated. Possible values are year, month, week, day, account, product, region, tag(soon)",
		Type:        routes.QueryArgStringSlice{},
		Optional:    false,
	},
}

func init() {
	routes.MethodMuxer{
		http.MethodGet: routes.H(getCostData).With(
			db.RequestTransaction{Db: db.Db},
			users.RequireAuthenticatedUser{},
			routes.QueryArgs(costsQueryArgs),
			routes.Documentation{
				Summary:     "get the costs data",
				Description: "Responds with cost data based on the query args passed to it",
			},
		),
	}.H().Register("/costs")
}

// validateCriteraParam will validate the different criterions.
// It validate the criterion by checking its presence in the simpleCriterionMap
// or, in the case of the special criterion tag, will check if it is in the
// correct format : 'tag:*' (with no more than one ':')
// Right now the tags are not enabled and will generate an error if they are
// used because they are not yet implemented in the new ElasticSearch mapping
func validateCriteriaParam(parsedParams esQueryParams) error {
	for _, criterion := range parsedParams.aggregationParams {
		if !simpleCriterionMap[criterion] {
			if len(criterion) >= 5 && criterion[:4] == "tag:" && strings.Count(criterion, ":") == 1 {
				return fmt.Errorf("tags not yet implemented")
			}
			return fmt.Errorf("Error parsing criterion : %s", criterion)
		}
	}
	return nil
}

// validateAwsAccounts will validate awsAccounts passed to it.
// It checks that they are numbers that are 12 character long
func validateAwsAccounts(parsedParams esQueryParams) error {
	for _, account := range parsedParams.accountList {
		if _, err := strconv.ParseInt(account, 10, 0); err != nil || len(account) != 12 {
			return fmt.Errorf("Error invalid account format : %s", account)
		}
	}
	return nil
}

// makeElasticSearchRequestAndParseIt will make the actual request to the ElasticSearch parse the results and return them
// It will return the data, an http status code (as int) and an error.
// Because an error can be generated, but is not critical and is not needed to be known by
// the user (e.g if the index does not exists because it was not yet indexed ) the error will
// be returned, but instead of having a 500 status code, it will return the provided status code
// with empy data
func makeElasticSearchRequestAndParseIt(ctx context.Context, parsedParams esQueryParams, user users.User) (es.SimplifiedCostsDocument, int, error) {
	l := jsonlog.LoggerFromContextOrDefault(ctx)
	index := es.IndexNameForUser(user, "lineitems")
	searchService := GetElasticSearchParams(
		parsedParams.accountList,
		parsedParams.dateBegin,
		parsedParams.dateEnd,
		parsedParams.aggregationParams,
		es.Client,
		index,
	)
	res, err := searchService.Do(ctx)
	if err != nil {
		if elastic.IsNotFound(err) {
			l.Warning("Query execution failed, ES index does not exists : "+index, err)
			return es.SimplifiedCostsDocument{}, http.StatusOK, err
		}
		l.Error("Query execution failed : "+err.Error(), nil)
		return es.SimplifiedCostsDocument{}, http.StatusInternalServerError, fmt.Errorf("could not execute the ElasticSearch query")
	}
	simplifiedCostDocument, err := es.SimplifyCostsDocument(ctx, res)
	if err != nil {
		l.Error("Error parsing cost response : "+err.Error(), nil)
		return simplifiedCostDocument, http.StatusInternalServerError, fmt.Errorf("could not parse ElasticSearch response")
	}
	return simplifiedCostDocument, http.StatusOK, nil
}

// getCostsData returns the cost data based on the query params, in JSON format.
func getCostData(request *http.Request, a routes.Arguments) (int, interface{}) {
	user := a[users.AuthenticatedUser].(users.User)
	parsedParams := esQueryParams{
		accountList:       []string{},
		dateBegin:         a[costsQueryArgs[1]].(time.Time),
		dateEnd:           a[costsQueryArgs[2]].(time.Time),
		aggregationParams: a[costsQueryArgs[3]].([]string),
	}
	if a[costsQueryArgs[0]] != nil {
		parsedParams.accountList = a[costsQueryArgs[0]].([]string)
	}
	if err := validateCriteriaParam(parsedParams); err != nil {
		return http.StatusBadRequest, err
	}
	if err := validateAwsAccounts(parsedParams); err != nil {
		return http.StatusBadRequest, err
	}
	simplifiedCostDocument, returnCode, err := makeElasticSearchRequestAndParseIt(request.Context(), parsedParams, user)
	if err != nil {
		if returnCode == http.StatusOK {
			return returnCode, es.SimplifiedCostsDocument{}.ToJsonable()
		} else {
			return returnCode, err
		}
	}
	return http.StatusOK, simplifiedCostDocument.ToJsonable()
}
<commit_msg>costs: fixed capitalization in error message<commit_after>//   Copyright 2017 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package costs

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/trackit/jsonlog"
	"github.com/trackit/trackit2/db"
	"github.com/trackit/trackit2/es"
	"github.com/trackit/trackit2/routes"
	"github.com/trackit/trackit2/users"

	"gopkg.in/olivere/elastic.v5"
)

// simpleCriterionMap will map simple criterion to the boolean true.
// This will be used in parseCriterionQueryParams to validate the queryParam.
// It does not take into account the 'tag:*' criterion as it is not fixed.
var simpleCriterionMap = map[string]bool{
	"year":    true,
	"month":   true,
	"week":    true,
	"day":     true,
	"account": true,
	"product": true,
	"region":  true,
}

// esQueryParams will store the parsed query params
type esQueryParams struct {
	dateBegin         time.Time
	dateEnd           time.Time
	accountList       []string
	aggregationParams []string
}

// costQueryArgs allows to get required queryArgs params
var costsQueryArgs = []routes.QueryArg{
	routes.QueryArg{
		Name:        "accounts",
		Description: "List of comma separated AWS accounts ids",
		Type:        routes.QueryArgStringSlice{},
		Optional:    true,
	},
	routes.QueryArg{
		Name:        "begin",
		Description: "Begining of date interval. Format is ISO8601",
		Type:        routes.QueryArgDate{},
		Optional:    false,
	},
	routes.QueryArg{
		Name:        "end",
		Description: "End of date interval. Format is ISO8601",
		Type:        routes.QueryArgDate{},
		Optional:    false,
	},
	routes.QueryArg{
		Name:        "by",
		Description: "Criteria for the ES aggregation, comma separated. Possible values are year, month, week, day, account, product, region, tag(soon)",
		Type:        routes.QueryArgStringSlice{},
		Optional:    false,
	},
}

func init() {
	routes.MethodMuxer{
		http.MethodGet: routes.H(getCostData).With(
			db.RequestTransaction{Db: db.Db},
			users.RequireAuthenticatedUser{},
			routes.QueryArgs(costsQueryArgs),
			routes.Documentation{
				Summary:     "get the costs data",
				Description: "Responds with cost data based on the query args passed to it",
			},
		),
	}.H().Register("/costs")
}

// validateCriteraParam will validate the different criterions.
// It validate the criterion by checking its presence in the simpleCriterionMap
// or, in the case of the special criterion tag, will check if it is in the
// correct format : 'tag:*' (with no more than one ':')
// Right now the tags are not enabled and will generate an error if they are
// used because they are not yet implemented in the new ElasticSearch mapping
func validateCriteriaParam(parsedParams esQueryParams) error {
	for _, criterion := range parsedParams.aggregationParams {
		if !simpleCriterionMap[criterion] {
			if len(criterion) >= 5 && criterion[:4] == "tag:" && strings.Count(criterion, ":") == 1 {
				return fmt.Errorf("tags not yet implemented")
			}
			return fmt.Errorf("Error parsing criterion : %s", criterion)
		}
	}
	return nil
}

// validateAwsAccounts will validate awsAccounts passed to it.
// It checks that they are numbers that are 12 character long
func validateAwsAccounts(parsedParams esQueryParams) error {
	for _, account := range parsedParams.accountList {
		if _, err := strconv.ParseInt(account, 10, 0); err != nil || len(account) != 12 {
			return fmt.Errorf("invalid account format : %s", account)
		}
	}
	return nil
}

// makeElasticSearchRequestAndParseIt will make the actual request to the ElasticSearch parse the results and return them
// It will return the data, an http status code (as int) and an error.
// Because an error can be generated, but is not critical and is not needed to be known by
// the user (e.g if the index does not exists because it was not yet indexed ) the error will
// be returned, but instead of having a 500 status code, it will return the provided status code
// with empy data
func makeElasticSearchRequestAndParseIt(ctx context.Context, parsedParams esQueryParams, user users.User) (es.SimplifiedCostsDocument, int, error) {
	l := jsonlog.LoggerFromContextOrDefault(ctx)
	index := es.IndexNameForUser(user, "lineitems")
	searchService := GetElasticSearchParams(
		parsedParams.accountList,
		parsedParams.dateBegin,
		parsedParams.dateEnd,
		parsedParams.aggregationParams,
		es.Client,
		index,
	)
	res, err := searchService.Do(ctx)
	if err != nil {
		if elastic.IsNotFound(err) {
			l.Warning("Query execution failed, ES index does not exists : "+index, err)
			return es.SimplifiedCostsDocument{}, http.StatusOK, err
		}
		l.Error("Query execution failed : "+err.Error(), nil)
		return es.SimplifiedCostsDocument{}, http.StatusInternalServerError, fmt.Errorf("could not execute the ElasticSearch query")
	}
	simplifiedCostDocument, err := es.SimplifyCostsDocument(ctx, res)
	if err != nil {
		l.Error("Error parsing cost response : "+err.Error(), nil)
		return simplifiedCostDocument, http.StatusInternalServerError, fmt.Errorf("could not parse ElasticSearch response")
	}
	return simplifiedCostDocument, http.StatusOK, nil
}

// getCostsData returns the cost data based on the query params, in JSON format.
func getCostData(request *http.Request, a routes.Arguments) (int, interface{}) {
	user := a[users.AuthenticatedUser].(users.User)
	parsedParams := esQueryParams{
		accountList:       []string{},
		dateBegin:         a[costsQueryArgs[1]].(time.Time),
		dateEnd:           a[costsQueryArgs[2]].(time.Time),
		aggregationParams: a[costsQueryArgs[3]].([]string),
	}
	if a[costsQueryArgs[0]] != nil {
		parsedParams.accountList = a[costsQueryArgs[0]].([]string)
	}
	if err := validateCriteriaParam(parsedParams); err != nil {
		return http.StatusBadRequest, err
	}
	if err := validateAwsAccounts(parsedParams); err != nil {
		return http.StatusBadRequest, err
	}
	simplifiedCostDocument, returnCode, err := makeElasticSearchRequestAndParseIt(request.Context(), parsedParams, user)
	if err != nil {
		if returnCode == http.StatusOK {
			return returnCode, es.SimplifiedCostsDocument{}.ToJsonable()
		} else {
			return returnCode, err
		}
	}
	return http.StatusOK, simplifiedCostDocument.ToJsonable()
}
<|endoftext|>