<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"sort"
	"time"
)

var httpCmd = &Command{
	UsageLine: "http [port]",
	Short:     "launch a web server for test",
	Long: `
	launch a web server for test.

	default port is :80.
	`,
}

type Comment struct {
	Id       string `json:"id"`
	Nickname string `json:"nickname"`
	Url      string `json:"url"`
	Content  string `json:"content"`
	Time     string `json:"time"`
}

type AllComment []Comment

func (p AllComment) Len() int {
	return len(p)
}

func (p AllComment) Less(i, j int) bool {
	p1 := p[i].Time
	p2 := p[j].Time
	pt1, err := time.Parse("2006-01-02 15:04:05", p1)
	if err != nil {
		log.Fatal(err)
	}
	pt2, err := time.Parse("2006-01-02 15:04:05", p2)
	if err != nil {
		log.Fatal(err)
	}
	return pt1.After(pt2)
}

func (p AllComment) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

var comments map[string]AllComment

func init() {
	comments = make(map[string]AllComment, 0)
	httpCmd.Run = httpApp
	AddCommand(httpCmd)
}

func httpApp(cmd *Command, args []string) {
	port := ":80"
	if len(args) > 0 {
		port = args[0]
	}
	http.HandleFunc("/comment.list", func(w http.ResponseWriter, r *http.Request) {
		var data AllComment
		id := r.FormValue("id")
		data = comments[id]
		sort.Sort(data)
		w.Header().Add("Content-Type", "application/json;charset=utf-8")
		w.Header().Add("Access-Control-Allow-Origin", "*")
		if err := json.NewEncoder(w).Encode(data); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	})
	http.HandleFunc("/comment.new", func(w http.ResponseWriter, r *http.Request) {
		var data Comment
		data.Id = r.FormValue("id")
		data.Nickname = r.FormValue("nickname")
		data.Url = r.FormValue("url")
		data.Content = r.FormValue("content")
		if len(data.Nickname) == 0 || len(data.Content) == 0 {
			return
		}
		data.Time = time.Now().Format("2006-01-02 15:04:05")
		data.Content = MarkdownToHtml(data.Content)
		comments[data.Id] = append(comments[data.Id], data)
		w.Header().Add("Access-Control-Allow-Origin", "*")
	})
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.FileServer(http.Dir(".")).ServeHTTP(w, r)
	})
	http.ListenAndServe(port, nil)
}
<commit_msg>update<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
	"sort"
	"time"
)

var httpCmd = &Command{
	UsageLine: "http [port]",
	Short:     "launch a web server for test",
	Long: `
	launch a web server for test.

	default port is :80.
	`,
}

type Comment struct {
	Id       string `json:"id"`
	Nickname string `json:"nickname"`
	Url      string `json:"url"`
	Content  string `json:"content"`
	Time     string `json:"time"`
}

type AllComment []Comment

func (p AllComment) Len() int {
	return len(p)
}

func (p AllComment) Less(i, j int) bool {
	p1 := p[i].Time
	p2 := p[j].Time
	pt1, err := time.Parse("2006-01-02 15:04:05", p1)
	if err != nil {
		log.Fatal(err)
	}
	pt2, err := time.Parse("2006-01-02 15:04:05", p2)
	if err != nil {
		log.Fatal(err)
	}
	return pt1.After(pt2)
}

func (p AllComment) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

var comments map[string]AllComment

func init() {
	comments = make(map[string]AllComment, 0)
	httpCmd.Run = httpApp
	AddCommand(httpCmd)
}

func httpApp(cmd *Command, args []string) {
	port := ":80"
	if len(args) > 0 {
		port = args[0]
	}
	http.HandleFunc("/comment.list", func(w http.ResponseWriter, r *http.Request) {
		var data AllComment
		id := r.FormValue("id")
		data = comments[id]
		sort.Sort(data)
		w.Header().Add("Content-Type", "application/json;charset=utf-8")
		w.Header().Add("Access-Control-Allow-Origin", "*")
		if err := json.NewEncoder(w).Encode(data); err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}
	})
	http.HandleFunc("/comment.new", func(w http.ResponseWriter, r *http.Request) {
		var data Comment
		data.Id = r.FormValue("id")
		data.Nickname = r.FormValue("nickname")
		data.Url = r.FormValue("url")
		data.Content = r.FormValue("content")
		if len(data.Nickname) == 0 || len(data.Content) == 0 {
			return
		}
		data.Time = time.Now().Format("2006-01-02 15:04:05")
		data.Content = MarkdownToHtml(data.Content)
		comments[data.Id] = append(comments[data.Id], data)
		w.Header().Add("Access-Control-Allow-Origin", "*")
	})
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		http.FileServer(http.Dir(".")).ServeHTTP(w, r)
	})
	log.Fatal(http.ListenAndServe(port, nil))
}
<|endoftext|>


<commit_before>// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package i18n is a middleware that provides app Internationalization and Localization of Macaron.
package i18n

import (
	"fmt"
	"path"
	"strings"

	"github.com/Unknwon/i18n"
	"github.com/Unknwon/macaron"
)

// Initialized language type list.
func initLocales(opt Options) {
	for i, lang := range opt.Langs {
		if err := i18n.SetMessageWithDesc(lang, opt.Names[i],
			path.Join(opt.Directory, fmt.Sprintf(opt.Format, lang))); err != nil {
			panic(fmt.Errorf("fail to set message file(%s): %v", lang, err))
		}
	}
}

// A Locale describles the information of localization.
type Locale struct {
	*i18n.Locale
}

// Options represents a struct for specifying configuration options for the i18n middleware.
type Options struct {
	// Suburl of path. Default is empty.
	SubURL string
	// Directory to load locale files. Default is "conf/locale"
	Directory string
	// Langauges that will be supported, order is meaningful.
	Langs []string
	// Human friendly names corresponding to Langs list.
	Names []string
	// Locale file naming style. Default is "locale_%s.ini".
	Format string
	// Name of language parameter name in URL. Default is "lang".
	Parameter string
	// Redirect when user uses get parameter to specify language.
	Redirect bool
	// Name that maps into template variable. Default is "i18n".
	TmplName string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}

	opt.SubURL = strings.TrimSuffix(opt.SubURL, "/")

	// Defaults
	if len(opt.Langs) == 0 {
		panic("no language is specified")
	} else if len(opt.Langs) != len(opt.Names) {
		panic("length of langs is not same as length of names")
	}

	if len(opt.Directory) == 0 {
		opt.Directory = "conf/locale"
	}
	if len(opt.Format) == 0 {
		opt.Format = "locale_%s.ini"
	}
	if len(opt.Parameter) == 0 {
		opt.Parameter = "lang"
	}
	if len(opt.TmplName) == 0 {
		opt.TmplName = "i18n"
	}

	return opt
}

type LangType struct {
	Lang, Name string
}

// I18n is a middleware provides localization layer for your application.
// Paramenter langs must be in the form of "en-US", "zh-CN", etc.
// Otherwise it may not recognize browser input.
func I18n(options ...Options) macaron.Handler {
	opt := prepareOptions(options)
	initLocales(opt)
	return func(ctx *macaron.Context) {
		if ctx.Data == nil {
			panic("Render middleware is not registered")
		}

		isNeedRedir := false
		hasCookie := false

		// 1. Check URL arguments.
		lang := ctx.Query(opt.Parameter)

		// 2. Get language information from cookies.
		if len(lang) == 0 {
			lang = ctx.GetCookie("lang")
			hasCookie = true
		} else {
			isNeedRedir = true
		}

		// Check again in case someone modify by purpose.
		if !i18n.IsExist(lang) {
			lang = ""
			isNeedRedir = false
			hasCookie = false
		}

		// 3. Get language information from 'Accept-Language'.
		if len(lang) == 0 {
			al := ctx.Req.Header.Get("Accept-Language")
			if len(al) > 4 {
				al = al[:5] // Only compare first 5 letters.
				if i18n.IsExist(al) {
					lang = al
				}
			}
		}

		// 4. Default language is the first element in the list.
		if len(lang) == 0 {
			lang = i18n.GetLangByIndex(0)
			isNeedRedir = false
		}

		curLang := LangType{
			Lang: lang,
		}

		// Save language information in cookies.
		if !hasCookie {
			ctx.SetCookie("lang", curLang.Lang, 1<<31-1, "/")
		}

		restLangs := make([]LangType, 0, i18n.Count()-1)
		langs := i18n.ListLangs()
		names := i18n.ListLangDescs()
		for i, v := range langs {
			if lang != v {
				restLangs = append(restLangs, LangType{v, names[i]})
			} else {
				curLang.Name = names[i]
			}
		}

		// Set language properties.
		locale := Locale{&i18n.Locale{lang}}
		ctx.Map(locale)
		ctx.Data[opt.TmplName] = locale
		ctx.Data["Tr"] = i18n.Tr
		ctx.Data["Lang"] = locale.Lang
		ctx.Data["AllLangs"] = append([]LangType{curLang}, restLangs...)
		ctx.Data["RestLangs"] = restLangs

		if opt.Redirect && isNeedRedir {
			ctx.Redirect(opt.SubURL + ctx.Req.RequestURI[:strings.Index(ctx.Req.RequestURI, "?")])
		}
	}
}
<commit_msg>Set cookiepath to suburl<commit_after>// Copyright 2014 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package i18n is a middleware that provides app Internationalization and Localization of Macaron.
package i18n

import (
	"fmt"
	"path"
	"strings"

	"github.com/Unknwon/i18n"
	"github.com/Unknwon/macaron"
)

// Initialized language type list.
func initLocales(opt Options) {
	for i, lang := range opt.Langs {
		if err := i18n.SetMessageWithDesc(lang, opt.Names[i],
			path.Join(opt.Directory, fmt.Sprintf(opt.Format, lang))); err != nil {
			panic(fmt.Errorf("fail to set message file(%s): %v", lang, err))
		}
	}
}

// A Locale describles the information of localization.
type Locale struct {
	*i18n.Locale
}

// Options represents a struct for specifying configuration options for the i18n middleware.
type Options struct {
	// Suburl of path. Default is empty.
	SubURL string
	// Directory to load locale files. Default is "conf/locale"
	Directory string
	// Langauges that will be supported, order is meaningful.
	Langs []string
	// Human friendly names corresponding to Langs list.
	Names []string
	// Locale file naming style. Default is "locale_%s.ini".
	Format string
	// Name of language parameter name in URL. Default is "lang".
	Parameter string
	// Redirect when user uses get parameter to specify language.
	Redirect bool
	// Name that maps into template variable. Default is "i18n".
	TmplName string
}

func prepareOptions(options []Options) Options {
	var opt Options
	if len(options) > 0 {
		opt = options[0]
	}

	opt.SubURL = strings.TrimSuffix(opt.SubURL, "/")

	// Defaults
	if len(opt.Langs) == 0 {
		panic("no language is specified")
	} else if len(opt.Langs) != len(opt.Names) {
		panic("length of langs is not same as length of names")
	}

	if len(opt.Directory) == 0 {
		opt.Directory = "conf/locale"
	}
	if len(opt.Format) == 0 {
		opt.Format = "locale_%s.ini"
	}
	if len(opt.Parameter) == 0 {
		opt.Parameter = "lang"
	}
	if len(opt.TmplName) == 0 {
		opt.TmplName = "i18n"
	}

	return opt
}

type LangType struct {
	Lang, Name string
}

// I18n is a middleware provides localization layer for your application.
// Paramenter langs must be in the form of "en-US", "zh-CN", etc.
// Otherwise it may not recognize browser input.
func I18n(options ...Options) macaron.Handler {
	opt := prepareOptions(options)
	initLocales(opt)
	return func(ctx *macaron.Context) {
		if ctx.Data == nil {
			panic("Render middleware is not registered")
		}

		isNeedRedir := false
		hasCookie := false

		// 1. Check URL arguments.
		lang := ctx.Query(opt.Parameter)

		// 2. Get language information from cookies.
		if len(lang) == 0 {
			lang = ctx.GetCookie("lang")
			hasCookie = true
		} else {
			isNeedRedir = true
		}

		// Check again in case someone modify by purpose.
		if !i18n.IsExist(lang) {
			lang = ""
			isNeedRedir = false
			hasCookie = false
		}

		// 3. Get language information from 'Accept-Language'.
		if len(lang) == 0 {
			al := ctx.Req.Header.Get("Accept-Language")
			if len(al) > 4 {
				al = al[:5] // Only compare first 5 letters.
				if i18n.IsExist(al) {
					lang = al
				}
			}
		}

		// 4. Default language is the first element in the list.
		if len(lang) == 0 {
			lang = i18n.GetLangByIndex(0)
			isNeedRedir = false
		}

		curLang := LangType{
			Lang: lang,
		}

		// Save language information in cookies.
		if !hasCookie {
			ctx.SetCookie("lang", curLang.Lang, 1<<31-1, "/" + strings.TrimPrefix(opt.SubURL, "/"))
		}

		restLangs := make([]LangType, 0, i18n.Count()-1)
		langs := i18n.ListLangs()
		names := i18n.ListLangDescs()
		for i, v := range langs {
			if lang != v {
				restLangs = append(restLangs, LangType{v, names[i]})
			} else {
				curLang.Name = names[i]
			}
		}

		// Set language properties.
		locale := Locale{&i18n.Locale{lang}}
		ctx.Map(locale)
		ctx.Data[opt.TmplName] = locale
		ctx.Data["Tr"] = i18n.Tr
		ctx.Data["Lang"] = locale.Lang
		ctx.Data["AllLangs"] = append([]LangType{curLang}, restLangs...)
		ctx.Data["RestLangs"] = restLangs

		if opt.Redirect && isNeedRedir {
			ctx.Redirect(opt.SubURL + ctx.Req.RequestURI[:strings.Index(ctx.Req.RequestURI, "?")])
		}
	}
}
<|endoftext|>


<commit_before>// Package iban contains utility functions for working with IBAN account numbers.
package iban

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
)

// IBAN represents an IBAN number, split up into its different parts.
type IBAN struct {
	Number      string
	CountryCode string
	Checksum    string
	BBAN        string
	BankCode    string
}

// NewIBAN creates a new instance of IBAN and checks before if the IBAN
// number is actually valid or not.
func NewIBAN(ibanNumber string) (IBAN, error) {
	var newIBAN IBAN

	_, formatedIBANNumber, err := IsCorrectIban(ibanNumber, false)
	if err != nil {
		return IBAN{}, err
	}

	newIBAN.Number = formatedIBANNumber
	newIBAN.CountryCode, newIBAN.Checksum, newIBAN.BBAN = splitIbanUp(formatedIBANNumber)

	newIBAN.BankCode, err = getBankCode(newIBAN)
	if err != nil {
		newIBAN.BankCode = ""
	}

	return newIBAN, nil
}

func getBankCode(iban IBAN) (string, error) {
	for _, code := range countryList {
		if code.code == iban.CountryCode {
			firstIndex := strings.Index(code.ibanFields, "b")
			lastIndex := strings.LastIndex(code.ibanFields, "b")
			return iban.Number[firstIndex : lastIndex+1], nil
		}
	}

	return "", errors.New("No bank code found")
}

type ibanCountry struct {
	country           string
	chars             int
	bbanFormat        string
	code              string
	ibanFields        string
	comment           string
	standardTreatment bool
}

// IsCorrectIban checks if the given iban number corresponds to the rules of a valid iban number and also
// returns a properly formated iban number string.
func IsCorrectIban(iban string, debug bool) (isValid bool, wellFormated string, err error) {
	var ibanConfig ibanCountry
	ibanValid := false
	wellFormated = ""
	if len(iban) > 15 { // Minimum length for IBAN

		// Clean up string
		iban = strings.ToUpper(strings.Replace(iban, " ", "", -1))

		// Split string up
		passedChars := len(iban)
		passedCode, passedChecksum, passedBban := splitIbanUp(iban)

		//fmt.Println(country_list)

		ibanConfig = countryList[passedCode]

		if ibanConfig.chars > 0 {
			if ibanConfig.chars == passedChars {
				convertedIban := rearrangeIBAN(passedCode, passedChecksum, passedBban)
				convertedIban = convertCharToNumber(convertedIban)

				if calculateModulo(convertedIban) == 1 {
					ibanValid = true
					wellFormated = splitTo4(iban)
				}
			} else {
				return false, wellFormated, fmt.Errorf("IBAN: lenght (%s) does not match configuration length (%s)", strconv.Itoa(passedChars), strconv.Itoa(ibanConfig.chars))
			}

		} else {
			return false, wellFormated, fmt.Errorf("IBAN: country <%s> is not in the list", passedCode)
		}

	} else {
		return false, wellFormated, fmt.Errorf("IBAN: incorect IBAN string passed <%s>", iban)
	}
	return ibanValid, wellFormated, nil
}

func splitIbanUp(iban string) (countryCode string, checksum string, bban string) {
	countryCode = iban[0:2]
	checksum = iban[2:4]
	bban = iban[4:len(iban)]
	return countryCode, checksum, bban
}

func splitTo4(value string) (returnValue string) {
	n := 0
	for n < len(value) {

		if n < len(value)-3 {
			returnValue += value[n:n+4] + " "
		} else {
			returnValue += value[n:len(value)]
		}
		n += 4
	}
	return returnValue
}

// GetIbanChecksum returns the checksum of the given iban number.
func GetIbanChecksum(iban string) (int, error) {
	ibanChecksum := 0

	if len(iban) > 15 { // Minimum length for IBAN

		iban = strings.ToUpper(strings.Replace(iban, " ", "", -1))

		// Split string up
		passedCode, passedChecksum, passedBban := splitIbanUp(iban)
		passedChecksum = "00"
		convertedIban := rearrangeIBAN(passedCode, passedChecksum, passedBban)
		convertedIban = convertCharToNumber(convertedIban)

		ibanChecksum = 98 - calculateModulo(convertedIban)

	} else {
		return -1, fmt.Errorf("IBAN: incorect IBAN string passed <%s>", iban)
	}
	return ibanChecksum, nil
}

func convertCharToNumber(value string) string {
	var temp = []byte(value)
	var result string

	for _, item := range temp {
		if !((item < "A"[0]) || (item > "Z"[0])) {
			item = byte(item - 55)
			result += strconv.Itoa(int(item))
		} else {
			result += string(item)
		}
	}
	return result
}

func rearrangeIBAN(countryCode string, checksum string, bban string) string {
	return bban + countryCode + checksum
}

func calculateModulo(iban string) int {
	exit := false
	tempIBAN := ""
	rest := 0
	for !exit {
		if len(iban) > 9 {
			tempIBAN = iban[0:9]

			value, err := strconv.Atoi(tempIBAN)
			if err != nil {
				exit = true
				return -1
			}

			rest = (value % 97)
			iban = strconv.Itoa(rest) + iban[9:len(iban)]
		} else {
			tempIBAN = iban

			value, err := strconv.Atoi(tempIBAN)
			if err != nil {
				exit = true
				return -1
			}

			rest = (value % 97)
			exit = true
		}
	}
	return rest
}
<commit_msg>Fixed a bug where entering an invalid IBAN number would cause a panic<commit_after>// Package iban contains utility functions for working with IBAN account numbers.
package iban

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
)

// ErrInvalidIBAN is returned when an invalid IBAN number was received
var ErrInvalidIBAN = errors.New("Invalid IBAN number received")

// IBAN represents an IBAN number, split up into its different parts.
type IBAN struct {
	Number      string
	CountryCode string
	Checksum    string
	BBAN        string
	BankCode    string
}

// NewIBAN creates a new instance of IBAN and checks before if the IBAN
// number is actually valid or not.
func NewIBAN(ibanNumber string) (IBAN, error) {
	var newIBAN IBAN

	isIBANValid, formatedIBANNumber, err := IsCorrectIban(ibanNumber, false)
	if !isIBANValid {
		return IBAN{}, ErrInvalidIBAN
	}

	if err != nil {
		return IBAN{}, err
	}

	newIBAN.Number = formatedIBANNumber
	newIBAN.CountryCode, newIBAN.Checksum, newIBAN.BBAN = splitIbanUp(formatedIBANNumber)

	newIBAN.BankCode, err = getBankCode(newIBAN)
	if err != nil {
		newIBAN.BankCode = ""
	}

	return newIBAN, nil
}

func getBankCode(iban IBAN) (string, error) {
	for _, code := range countryList {
		if code.code == iban.CountryCode {
			firstIndex := strings.Index(code.ibanFields, "b")
			lastIndex := strings.LastIndex(code.ibanFields, "b")
			return iban.Number[firstIndex : lastIndex+1], nil
		}
	}

	return "", errors.New("No bank code found")
}

type ibanCountry struct {
	country           string
	chars             int
	bbanFormat        string
	code              string
	ibanFields        string
	comment           string
	standardTreatment bool
}

// IsCorrectIban checks if the given iban number corresponds to the rules of a valid iban number and also
// returns a properly formated iban number string.
func IsCorrectIban(iban string, debug bool) (isValid bool, wellFormated string, err error) {
	var ibanConfig ibanCountry
	ibanValid := false
	wellFormated = ""
	if len(iban) > 15 { // Minimum length for IBAN

		// Clean up string
		iban = strings.ToUpper(strings.Replace(iban, " ", "", -1))

		// Split string up
		passedChars := len(iban)
		passedCode, passedChecksum, passedBban := splitIbanUp(iban)

		//fmt.Println(country_list)

		ibanConfig = countryList[passedCode]

		if ibanConfig.chars > 0 {
			if ibanConfig.chars == passedChars {
				convertedIban := rearrangeIBAN(passedCode, passedChecksum, passedBban)
				convertedIban = convertCharToNumber(convertedIban)

				if calculateModulo(convertedIban) == 1 {
					ibanValid = true
					wellFormated = splitTo4(iban)
				}
			} else {
				return false, wellFormated, fmt.Errorf("IBAN: lenght (%s) does not match configuration length (%s)", strconv.Itoa(passedChars), strconv.Itoa(ibanConfig.chars))
			}

		} else {
			return false, wellFormated, fmt.Errorf("IBAN: country <%s> is not in the list", passedCode)
		}

	} else {
		return false, wellFormated, fmt.Errorf("IBAN: incorect IBAN string passed <%s>", iban)
	}
	return ibanValid, wellFormated, nil
}

func splitIbanUp(iban string) (countryCode string, checksum string, bban string) {
	countryCode = iban[0:2]
	checksum = iban[2:4]
	bban = iban[4:len(iban)]
	return countryCode, checksum, bban
}

func splitTo4(value string) (returnValue string) {
	n := 0
	for n < len(value) {

		if n < len(value)-3 {
			returnValue += value[n:n+4] + " "
		} else {
			returnValue += value[n:len(value)]
		}
		n += 4
	}
	return returnValue
}

// GetIbanChecksum returns the checksum of the given iban number.
func GetIbanChecksum(iban string) (int, error) {
	ibanChecksum := 0

	if len(iban) > 15 { // Minimum length for IBAN

		iban = strings.ToUpper(strings.Replace(iban, " ", "", -1))

		// Split string up
		passedCode, passedChecksum, passedBban := splitIbanUp(iban)
		passedChecksum = "00"
		convertedIban := rearrangeIBAN(passedCode, passedChecksum, passedBban)
		convertedIban = convertCharToNumber(convertedIban)

		ibanChecksum = 98 - calculateModulo(convertedIban)

	} else {
		return -1, fmt.Errorf("IBAN: incorect IBAN string passed <%s>", iban)
	}
	return ibanChecksum, nil
}

func convertCharToNumber(value string) string {
	var temp = []byte(value)
	var result string

	for _, item := range temp {
		if !((item < "A"[0]) || (item > "Z"[0])) {
			item = byte(item - 55)
			result += strconv.Itoa(int(item))
		} else {
			result += string(item)
		}
	}
	return result
}

func rearrangeIBAN(countryCode string, checksum string, bban string) string {
	return bban + countryCode + checksum
}

func calculateModulo(iban string) int {
	exit := false
	tempIBAN := ""
	rest := 0
	for !exit {
		if len(iban) > 9 {
			tempIBAN = iban[0:9]

			value, err := strconv.Atoi(tempIBAN)
			if err != nil {
				exit = true
				return -1
			}

			rest = (value % 97)
			iban = strconv.Itoa(rest) + iban[9:len(iban)]
		} else {
			tempIBAN = iban

			value, err := strconv.Atoi(tempIBAN)
			if err != nil {
				exit = true
				return -1
			}

			rest = (value % 97)
			exit = true
		}
	}
	return rest
}
<|endoftext|>


<commit_before>package ssh

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/signal"
	"os/user"
	"regexp"
	"strings"
	"syscall"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/net/proxy"

	"github.com/blacknon/lssh/conf"
)

type Connect struct {
	Server           string
	Conf             conf.Config
	IsTerm           bool
	IsParallel       bool
	IsLocalRc        bool
	LocalRcData      string
	LocalRcDecodeCmd string
}

type Proxy struct {
	Name string
	Type string
}

// @brief: create ssh session
func (c *Connect) CreateSession() (session *ssh.Session, err error) {
	// New connect
	conn, err := c.createSshClient()
	if err != nil {
		return session, err
	}

	// New session
	session, err = conn.NewSession()
	if err != nil {
		return session, err
	}

	return
}

// @brief: create ssh client
// @note:
//     support multiple proxy connect
func (c *Connect) createSshClient() (client *ssh.Client, err error) {
	// New ClientConfig
	serverConf := c.Conf.Server[c.Server]
	sshConf, err := c.createSshClientConfig(c.Server)
	if err != nil {
		return client, err
	}

	// not use proxy
	if serverConf.Proxy == "" {
		client, err = ssh.Dial("tcp", net.JoinHostPort(serverConf.Addr, serverConf.Port), sshConf)
		if err != nil {
			return client, err
		}
	} else {
		client, err = c.createSshClientOverProxy(serverConf, sshConf)
		if err != nil {
			return client, err
		}
	}

	return client, err
}

// @brief:
//     Create ssh client via proxy
func (c *Connect) createSshClientOverProxy(serverConf conf.ServerConfig, sshConf *ssh.ClientConfig) (client *ssh.Client, err error) {
	// get proxy slice
	proxyList, proxyType, err := GetProxyList(c.Server, c.Conf)
	if err != nil {
		return client, err
	}

	// var
	var proxyClient *ssh.Client
	var proxyDialer proxy.Dialer

	for _, proxy := range proxyList {
		switch proxyType[proxy] {
		case "http", "https":
			proxyConf := c.Conf.Proxy[proxy]
			proxyDialer, err = createProxyDialerHttp(proxyConf)

		case "socks5":
			proxyConf := c.Conf.Proxy[proxy]
			proxyDialer, err = createProxyDialerSocks5(proxyConf)

		default:
			proxyConf := c.Conf.Server[proxy]
			proxySshConf, err := c.createSshClientConfig(proxy)
			if err != nil {
				return client, err
			}
			proxyClient, err = createSshClientViaProxy(proxyConf, proxySshConf, proxyClient, proxyDialer)

		}

		if err != nil {
			return client, err
		}
	}

	client, err = createSshClientViaProxy(serverConf, sshConf, proxyClient, proxyDialer)
	if err != nil {
		return client, err
	}

	return
}

// @brief:
//     Create ssh Client
func (c *Connect) createSshClientConfig(server string) (clientConfig *ssh.ClientConfig, err error) {
	conf := c.Conf.Server[server]

	auth, err := c.createSshAuth(server)
	if err != nil {
		return clientConfig, err
	}

	// create ssh ClientConfig
	clientConfig = &ssh.ClientConfig{
		User:            conf.User,
		Auth:            auth,
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Timeout:         3600 * time.Hour,
	}
	return clientConfig, err
}

// @brief: Create ssh session auth
func (c *Connect) createSshAuth(server string) (auth []ssh.AuthMethod, err error) {
	usr, _ := user.Current()
	conf := c.Conf.Server[server]

	if conf.Key != "" {
		conf.Key = strings.Replace(conf.Key, "~", usr.HomeDir, 1)

		// Read PrivateKey file
		key, err := ioutil.ReadFile(conf.Key)
		if err != nil {
			return auth, err
		}

		// Read signer from PrivateKey
		var signer ssh.Signer
		if conf.KeyPass != "" {
			signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(conf.KeyPass))
		} else {
			signer, err = ssh.ParsePrivateKey(key)
		}

		// check err
		if err != nil {
			return auth, err
		}

		auth = []ssh.AuthMethod{ssh.PublicKeys(signer)}
	} else {
		auth = []ssh.AuthMethod{ssh.Password(conf.Pass)}
	}

	return auth, err
}

// @brief:
//    run command over ssh
func (c *Connect) RunCmd(session *ssh.Session, command []string) (err error) {
	defer session.Close()

	// set timeout
	go func() {
		time.Sleep(2419200 * time.Second)
		session.Close()
	}()

	// set TerminalModes
	if session, err = c.setIsTerm(session); err != nil {
		return
	}

	// join command
	execCmd := strings.Join(command, " ")

	// run command
	isExit := make(chan bool)
	go func() {
		err = session.Run(execCmd)
		isExit <- true
	}()

	// check command exit
CheckCommandExit:
	for {
		time.Sleep(100 * time.Millisecond)
		select {
		case <-isExit:
			break CheckCommandExit
		case <-time.After(100 * time.Millisecond):
			continue CheckCommandExit
		}
	}

	return
}

// @brief:
//     Run command over ssh, output to gochannel
func (c *Connect) RunCmdWithOutput(session *ssh.Session, command []string, outputChan chan string) {
	var outputBuf bytes.Buffer
	session.Stdout = &outputBuf
	session.Stderr = &outputBuf

	// run command
	isExit := make(chan bool)
	go func() {
		c.RunCmd(session, command)
		isExit <- true
	}()

	readedLineBytes := 0

GetOutputLoop:
	for {
		time.Sleep(100 * time.Millisecond)

		outputBufStr := outputBuf.String()
		if len(outputBufStr) == 0 {
			continue
		}

		outputBufByte := []byte(outputBufStr)
		outputBufSlice := regexp.MustCompile("\r\n|\n\r|\n|\r").Split(string(outputBufByte[readedLineBytes:]), -1)

		readedLineBytes = len(outputBufByte)

		for i, outputLine := range outputBufSlice {
			if i == len(outputBufSlice)-1 {
				break
			}
			outputChan <- outputLine
		}

		select {
		case <-isExit:
			break GetOutputLoop
		case <-time.After(100 * time.Millisecond):
			continue GetOutputLoop
		}
	}

	// last check
	outputBufByte := []byte(outputBuf.String())
	if len(outputBufByte) > readedLineBytes {
		outputBufSlice := regexp.MustCompile("\r\n|\n\r|\n|\r").Split(string(outputBufByte[readedLineBytes:]), -1)
		for i, outputLine := range outputBufSlice {
			if i == len(outputBufSlice)-1 {
				break
			}
			outputChan <- outputLine
		}
	}
}

// @brief:
//     connect ssh terminal
func (c *Connect) ConTerm(session *ssh.Session) (err error) {
	// defer session.Close()
	fd := int(os.Stdin.Fd())
	state, err := terminal.MakeRaw(fd)
	if err != nil {
		return
	}
	defer terminal.Restore(fd, state)

	// get terminal size
	width, height, err := terminal.GetSize(fd)
	if err != nil {
		return
	}

	modes := ssh.TerminalModes{
		ssh.ECHO:          1,
		ssh.TTY_OP_ISPEED: 14400,
		ssh.TTY_OP_OSPEED: 14400,
	}

	err = session.RequestPty("xterm", height, width, modes)
	if err != nil {
		return
	}

	// start shell
	if c.IsLocalRc {
		session, err = c.runLocalRcShell(session)
		if err != nil {
			return
		}
	} else {
		err = session.Shell()
		if err != nil {
			return
		}
	}

	// Terminal resize
	signal_chan := make(chan os.Signal, 1)
	signal.Notify(signal_chan, syscall.SIGWINCH)
	go func() {
		for {
			s := <-signal_chan
			switch s {
			case syscall.SIGWINCH:
				fd := int(os.Stdout.Fd())
				width, height, _ = terminal.GetSize(fd)
				session.WindowChange(height, width)
			}
		}
	}()

	// keep alive packet
	go func() {
		for {
			_, _ = session.SendRequest("keepalive@golang.org", true, nil)
			time.Sleep(15 * time.Second)
		}
	}()

	err = session.Wait()
	if err != nil {
		return
	}

	return
}

// @brief:
//     set pesudo (run command only)
func (c *Connect) setIsTerm(preSession *ssh.Session) (session *ssh.Session, err error) {
	if c.IsTerm {
		modes := ssh.TerminalModes{
			ssh.ECHO:          0,
			ssh.TTY_OP_ISPEED: 14400,
			ssh.TTY_OP_OSPEED: 14400,
		}

		// Get terminal window size
		fd := int(os.Stdin.Fd())
		width, hight, err := terminal.GetSize(fd)
		if err != nil {
			preSession.Close()
			return session, err
		}

		if err = preSession.RequestPty("xterm", hight, width, modes); err != nil {
			preSession.Close()
			return session, err
		}
	}
	session = preSession
	return
}

// @brief:
//     get ssh proxy server slice
func GetProxyList(server string, config conf.Config) (proxyList []string, proxyType map[string]string, err error) {
	var targetType string
	var preProxy, preProxyType string

	targetServer := server
	proxyType = map[string]string{}

	for {
		isOk := false

		switch targetType {
		case "http", "https", "socks5":
			_, isOk = config.Proxy[targetServer]
			preProxy = ""
			preProxyType = ""

		default:
			var preProxyConf conf.ServerConfig
			preProxyConf, isOk = config.Server[targetServer]
			preProxy = preProxyConf.Proxy
			preProxyType = preProxyConf.ProxyType
		}

		// not use pre proxy
		if preProxy == "" {
			break
		}

		if !isOk {
			err = fmt.Errorf("Not Found proxy : %s", targetServer)
			return nil, nil, err
		}

		// set proxy info
		proxy := new(Proxy)
		proxy.Name = preProxy

		switch preProxyType {
		case "http", "https", "socks5":
			proxy.Type = preProxyType
		default:
			proxy.Type = "ssh"
		}

		proxyList = append(proxyList, proxy.Name)
		proxyType[proxy.Name] = proxy.Type

		targetServer = proxy.Name
		targetType = proxy.Type
	}

	// reverse proxyServers slice
	for i, j := 0, len(proxyList)-1; i < j; i, j = i+1, j-1 {
		proxyList[i], proxyList[j] = proxyList[j], proxyList[i]
	}

	return
}

// @brief:
//    run shell use local rc file.
func (c *Connect) runLocalRcShell(preSession *ssh.Session) (session *ssh.Session, err error) {
	session = preSession

	// command
	cmd := fmt.Sprintf("bash --rcfile <(echo %s|((base64 --help | grep -q coreutils) && base64 -d <(cat) || base64 -D <(cat) ))", c.LocalRcData)
	if len(c.LocalRcDecodeCmd) > 0 {

		cmd = fmt.Sprintf("bash --rcfile <(echo %s | %s)", c.LocalRcData, c.LocalRcDecodeCmd)
		fmt.Println(cmd)
	}

	err = session.Start(cmd)

	return session, err
}
<commit_msg>Unnecessary processing<commit_after>package ssh

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/signal"
	"os/user"
	"regexp"
	"strings"
	"syscall"
	"time"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/net/proxy"

	"github.com/blacknon/lssh/conf"
)

type Connect struct {
	Server           string
	Conf             conf.Config
	IsTerm           bool
	IsParallel       bool
	IsLocalRc        bool
	LocalRcData      string
	LocalRcDecodeCmd string
}

type Proxy struct {
	Name string
	Type string
}

// @brief: create ssh session
func (c *Connect) CreateSession() (session *ssh.Session, err error) {
	// New connect
	conn, err := c.createSshClient()
	if err != nil {
		return session, err
	}

	// New session
	session, err = conn.NewSession()
	if err != nil {
		return session, err
	}

	return
}

// @brief: create ssh client
// @note:
//     support multiple proxy connect
func (c *Connect) createSshClient() (client *ssh.Client, err error) {
	// New ClientConfig
	serverConf := c.Conf.Server[c.Server]
	sshConf, err := c.createSshClientConfig(c.Server)
	if err != nil {
		return client, err
	}

	// not use proxy
	if serverConf.Proxy == "" {
		client, err = ssh.Dial("tcp", net.JoinHostPort(serverConf.Addr, serverConf.Port), sshConf)
		if err != nil {
			return client, err
		}
	} else {
		client, err = c.createSshClientOverProxy(serverConf, sshConf)
		if err != nil {
			return client, err
		}
	}

	return client, err
}

// @brief:
//     Create ssh client via proxy
func (c *Connect) createSshClientOverProxy(serverConf conf.ServerConfig, sshConf *ssh.ClientConfig) (client *ssh.Client, err error) {
	// get proxy slice
	proxyList, proxyType, err := GetProxyList(c.Server, c.Conf)
	if err != nil {
		return client, err
	}

	// var
	var proxyClient *ssh.Client
	var proxyDialer proxy.Dialer

	for _, proxy := range proxyList {
		switch proxyType[proxy] {
		case "http", "https":
			proxyConf := c.Conf.Proxy[proxy]
			proxyDialer, err = createProxyDialerHttp(proxyConf)

		case "socks5":
			proxyConf := c.Conf.Proxy[proxy]
			proxyDialer, err = createProxyDialerSocks5(proxyConf)

		default:
			proxyConf := c.Conf.Server[proxy]
			proxySshConf, err := c.createSshClientConfig(proxy)
			if err != nil {
				return client, err
			}
			proxyClient, err = createSshClientViaProxy(proxyConf, proxySshConf, proxyClient, proxyDialer)

		}

		if err != nil {
			return client, err
		}
	}

	client, err = createSshClientViaProxy(serverConf, sshConf, proxyClient, proxyDialer)
	if err != nil {
		return client, err
	}

	return
}

// @brief:
//     Create ssh Client
func (c *Connect) createSshClientConfig(server string) (clientConfig *ssh.ClientConfig, err error) {
	conf := c.Conf.Server[server]

	auth, err := c.createSshAuth(server)
	if err != nil {
		return clientConfig, err
	}

	// create ssh ClientConfig
	clientConfig = &ssh.ClientConfig{
		User:            conf.User,
		Auth:            auth,
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Timeout:         3600 * time.Hour,
	}
	return clientConfig, err
}

// @brief: Create ssh session auth
func (c *Connect) createSshAuth(server string) (auth []ssh.AuthMethod, err error) {
	usr, _ := user.Current()
	conf := c.Conf.Server[server]

	if conf.Key != "" {
		conf.Key = strings.Replace(conf.Key, "~", usr.HomeDir, 1)

		// Read PrivateKey file
		key, err := ioutil.ReadFile(conf.Key)
		if err != nil {
			return auth, err
		}

		// Read signer from PrivateKey
		var signer ssh.Signer
		if conf.KeyPass != "" {
			signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(conf.KeyPass))
		} else {
			signer, err = ssh.ParsePrivateKey(key)
		}

		// check err
		if err != nil {
			return auth, err
		}

		auth = []ssh.AuthMethod{ssh.PublicKeys(signer)}
	} else {
		auth = []ssh.AuthMethod{ssh.Password(conf.Pass)}
	}

	return auth, err
}

// @brief:
//    run command over ssh
func (c *Connect) RunCmd(session *ssh.Session, command []string) (err error) {
	defer session.Close()

	// set timeout
	go func() {
		time.Sleep(2419200 * time.Second)
		session.Close()
	}()

	// set TerminalModes
	if session, err = c.setIsTerm(session); err != nil {
		return
	}

	// join command
	execCmd := strings.Join(command, " ")

	// run command
	isExit := make(chan bool)
	go func() {
		err = session.Run(execCmd)
		isExit <- true
	}()

	// check command exit
CheckCommandExit:
	for {
		time.Sleep(100 * time.Millisecond)
		select {
		case <-isExit:
			break CheckCommandExit
		case <-time.After(100 * time.Millisecond):
			continue CheckCommandExit
		}
	}

	return
}

// @brief:
//     Run command over ssh, output to gochannel
func (c *Connect) RunCmdWithOutput(session *ssh.Session, command []string, outputChan chan string) {
	var outputBuf bytes.Buffer
	session.Stdout = &outputBuf
	session.Stderr = &outputBuf

	// run command
	isExit := make(chan bool)
	go func() {
		c.RunCmd(session, command)
		isExit <- true
	}()

	readedLineBytes := 0

GetOutputLoop:
	for {
		time.Sleep(100 * time.Millisecond)

		outputBufStr := outputBuf.String()
		if len(outputBufStr) == 0 {
			continue
		}

		outputBufByte := []byte(outputBufStr)
		outputBufSlice := regexp.MustCompile("\r\n|\n\r|\n|\r").Split(string(outputBufByte[readedLineBytes:]), -1)

		readedLineBytes = len(outputBufByte)

		for i, outputLine := range outputBufSlice {
			if i == len(outputBufSlice)-1 {
				break
			}
			outputChan <- outputLine
		}

		select {
		case <-isExit:
			break GetOutputLoop
		case <-time.After(100 * time.Millisecond):
			continue GetOutputLoop
		}
	}

	// last check
	outputBufByte := []byte(outputBuf.String())
	if len(outputBufByte) > readedLineBytes {
		outputBufSlice := regexp.MustCompile("\r\n|\n\r|\n|\r").Split(string(outputBufByte[readedLineBytes:]), -1)
		for i, outputLine := range outputBufSlice {
			if i == len(outputBufSlice)-1 {
				break
			}
			outputChan <- outputLine
		}
	}
}

// @brief:
//     connect ssh terminal
func (c *Connect) ConTerm(session *ssh.Session) (err error) {
	// defer session.Close()
	fd := int(os.Stdin.Fd())
	state, err := terminal.MakeRaw(fd)
	if err != nil {
		return
	}
	defer terminal.Restore(fd, state)

	// get terminal size
	width, height, err := terminal.GetSize(fd)
	if err != nil {
		return
	}

	modes := ssh.TerminalModes{
		ssh.ECHO:          1,
		ssh.TTY_OP_ISPEED: 14400,
		ssh.TTY_OP_OSPEED: 14400,
	}

	err = session.RequestPty("xterm", height, width, modes)
	if err != nil {
		return
	}

	// start shell
	if c.IsLocalRc {
		session, err = c.runLocalRcShell(session)
		if err != nil {
			return
		}
	} else {
		err = session.Shell()
		if err != nil {
			return
		}
	}

	// Terminal resize
	signal_chan := make(chan os.Signal, 1)
	signal.Notify(signal_chan, syscall.SIGWINCH)
	go func() {
		for {
			s := <-signal_chan
			switch s {
			case syscall.SIGWINCH:
				fd := int(os.Stdout.Fd())
				width, height, _ = terminal.GetSize(fd)
				session.WindowChange(height, width)
			}
		}
	}()

	// keep alive packet
	go func() {
		for {
			_, _ = session.SendRequest("keepalive@golang.org", true, nil)
			time.Sleep(15 * time.Second)
		}
	}()

	err = session.Wait()
	if err != nil {
		return
	}

	return
}

// @brief:
//     set pesudo (run command only)
func (c *Connect) setIsTerm(preSession *ssh.Session) (session *ssh.Session, err error) {
	if c.IsTerm {
		modes := ssh.TerminalModes{
			ssh.ECHO:          0,
			ssh.TTY_OP_ISPEED: 14400,
			ssh.TTY_OP_OSPEED: 14400,
		}

		// Get terminal window size
		fd := int(os.Stdin.Fd())
		width, hight, err := terminal.GetSize(fd)
		if err != nil {
			preSession.Close()
			return session, err
		}

		if err = preSession.RequestPty("xterm", hight, width, modes); err != nil {
			preSession.Close()
			return session, err
		}
	}
	session = preSession
	return
}

// @brief:
//     get ssh proxy server slice
func GetProxyList(server string, config conf.Config) (proxyList []string, proxyType map[string]string, err error) {
	var targetType string
	var preProxy, preProxyType string

	targetServer := server
	proxyType = map[string]string{}

	for {
		isOk := false

		switch targetType {
		case "http", "https", "socks5":
			_, isOk = config.Proxy[targetServer]
			preProxy = ""
			preProxyType = ""

		default:
			var preProxyConf conf.ServerConfig
			preProxyConf, isOk = config.Server[targetServer]
			preProxy = preProxyConf.Proxy
			preProxyType = preProxyConf.ProxyType
		}

		// not use pre proxy
		if preProxy == "" {
			break
		}

		if !isOk {
			err = fmt.Errorf("Not Found proxy : %s", targetServer)
			return nil, nil, err
		}

		// set proxy info
		proxy := new(Proxy)
		proxy.Name = preProxy

		switch preProxyType {
		case "http", "https", "socks5":
			proxy.Type = preProxyType
		default:
			proxy.Type = "ssh"
		}

		proxyList = append(proxyList, proxy.Name)
		proxyType[proxy.Name] = proxy.Type

		targetServer = proxy.Name
		targetType = proxy.Type
	}

	// reverse proxyServers slice
	for i, j := 0, len(proxyList)-1; i < j; i, j = i+1, j-1 {
		proxyList[i], proxyList[j] = proxyList[j], proxyList[i]
	}

	return
}

// @brief:
//    run shell use local rc file.
func (c *Connect) runLocalRcShell(preSession *ssh.Session) (session *ssh.Session, err error) {
	session = preSession

	// command
	cmd := fmt.Sprintf("bash --rcfile <(echo %s|((base64 --help | grep -q coreutils) && base64 -d <(cat) || base64 -D <(cat) ))", c.LocalRcData)
	if len(c.LocalRcDecodeCmd) > 0 {

		cmd = fmt.Sprintf("bash --rcfile <(echo %s | %s)", c.LocalRcData, c.LocalRcDecodeCmd)
	}

	err = session.Start(cmd)

	return session, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/cloudfoundry/go_cfmessagebus"
	"github.com/cloudfoundry/sshark"
	"io/ioutil"
	"log"
	"os"
)

func main() {
	agent := sshark.NewAgent("/tmp/warden.sock")

	pubkeyPath := fmt.Sprintf("%s/.ssh/id_rsa.pub", os.Getenv("HOME"))

	pubkey, err := ioutil.ReadFile(pubkeyPath)
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	sess, err := agent.StartSession("foo")
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	log.Println(sess.Port)

	err = sess.LoadPublicKey(string(pubkey))
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	err = sess.StartSSHServer()
	if err != nil {
		log.Fatal(err.Error())
		return
	}
}
<commit_msg>remove unused go_cfmessagebus import<commit_after>package main

import (
	"fmt"
	"github.com/cloudfoundry/sshark"
	"io/ioutil"
	"log"
	"os"
)

func main() {
	agent := sshark.NewAgent("/tmp/warden.sock")

	pubkeyPath := fmt.Sprintf("%s/.ssh/id_rsa.pub", os.Getenv("HOME"))

	pubkey, err := ioutil.ReadFile(pubkeyPath)
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	sess, err := agent.StartSession("foo")
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	log.Println(sess.Port)

	err = sess.LoadPublicKey(string(pubkey))
	if err != nil {
		log.Fatal(err.Error())
		return
	}

	err = sess.StartSSHServer()
	if err != nil {
		log.Fatal(err.Error())
		return
	}
}
<|endoftext|>


<commit_before>package stack

import "log"
import "io/ioutil"

import "crypto/x509"
import "encoding/pem"
import "encoding/json"

import tlvCodec "github.com/chris-wood/spud/codec"
import "github.com/chris-wood/spud/tables/lpm"
import "github.com/chris-wood/spud/messages"
import "github.com/chris-wood/spud/messages/name"
import "github.com/chris-wood/spud/stack/config"
import "github.com/chris-wood/spud/stack/cache"
import "github.com/chris-wood/spud/stack/pit"
import "github.com/chris-wood/spud/stack/component/tunnel"
import "github.com/chris-wood/spud/stack/component/connector"
import "github.com/chris-wood/spud/stack/component/codec"
import "github.com/chris-wood/spud/stack/component/crypto"
import "github.com/chris-wood/spud/stack/component/crypto/validator"
import "github.com/chris-wood/spud/stack/component/crypto/context"

type MessageCallback func(msg messages.MessageWrapper)

type Component interface {
    // XXX: rename to Push and Pop, respectively
    Enqueue(messages.MessageWrapper)
    Dequeue() messages.MessageWrapper
    ProcessEgressMessages()
    ProcessIngressMessages()
}

type Stack struct {
    cryptoComponent Component
    codecComponent Component
    head Component
    bottom Component

    pendingMap map[string]MessageCallback
    pendingQueue chan messages.MessageWrapper
    prefixTable lpm.LPM
}

func (s *Stack) Enqueue(msg messages.MessageWrapper) {
    s.head.Enqueue(msg)
}

func (s *Stack) Dequeue() messages.MessageWrapper {
    return <- s.pendingQueue
}

func (s *Stack) Get(msg messages.MessageWrapper, callback MessageCallback) {
    if (msg.GetPacketType() != tlvCodec.T_INTEREST) {
        return
    }

    // Register the callback
    s.pendingMap[msg.Identifier()] = callback

    // Enqueue the message into the top of the stack
    s.head.Enqueue(msg)
}

func (s *Stack) Service(nameString string, callback MessageCallback) {
    prefixName, err := name.Parse(nameString)
    if err == nil {
        nameComponents := prefixName.SegmentStrings()
        s.prefixTable.Insert(nameComponents, callback)
    }
}

func (s *Stack) processInputQueue() {
    for ;; {
        // Dequeue messages as they arrive
        msg := s.head.Dequeue()

        switch (msg.GetPacketType()) {
        case tlvCodec.T_INTEREST:
            requestName := msg.Name()
            numSsegments := len(requestName.Segments)

            handled := false
            for index := 1; index <= numSsegments; index++ {
                nameComponents := requestName.SegmentStrings()
                callbackInterface, ok := s.prefixTable.Lookup(nameComponents)

                if ok {
                    callback := callbackInterface.(MessageCallback)
                    callback(msg)
                    s.prefixTable.Drop(nameComponents)
                    handled = true
                }
            }

            if !handled {
                s.pendingQueue <- msg
            }
        default:
            callback, ok := s.pendingMap[msg.Identifier()]
            if ok {
                callback(msg)
            } else {
                s.pendingQueue <- msg
            }
        }
    }
}

func (s *Stack) AddTunnel(tun tunnel.TunnelComponent) {
}

func Create(config config.StackConfig) (*Stack, error) {
  var err error

  // Create the link
  var fc connector.ForwarderConnector
  switch config.Link {
  case "tcp":
      locator := config.FwdAddress
      fc, err = connector.NewAthenaTCPForwarderConnector(locator)
      if err != nil {
          panic(err)
      }
      break
  case "loopback":
      fallthrough
  default:
      fc, _ = connector.NewLoopbackForwarderConnector()
      break
  }

  if fc == nil {
      log.Panic("Catastrophic failure: a connector was not created.")
  }

  // Create the shared data structures
  stackCache := cache.NewCache()
  stackPit := pit.NewPIT()

  // Create codec
  codecComponent := codec.NewCodecComponent(fc, stackCache, stackPit)
  go codecComponent.ProcessEgressMessages()
  go codecComponent.ProcessIngressMessages()

  // Create crypto component
  // XXX: defer validator and encryptor creation to the crypto component
  trustStore := context.NewTrustStore()
  cryptoComponent := crypto.NewCryptoComponent(trustStore, codecComponent)
  go cryptoComponent.ProcessEgressMessages()
  go cryptoComponent.ProcessIngressMessages()

  // Create the right crypto processors
  if len(config.Keys) > 0 {
      for _, keyFileName := range(config.Keys) {
          keyData, err := ioutil.ReadFile(keyFileName)
          block, _ := pem.Decode(keyData)
          privateKey, parseError := x509.ParsePKCS1PrivateKey(block.Bytes)
          if parseError != nil {
              log.Printf("Failed to parse private key: %s", err)
          } else {
              rsaProcessor, _ := validator.NewRSAProcessorWithKey(privateKey)
              cryptoComponent.AddCryptoProcessor("/", rsaProcessor)
          }
      }
  } else {
      rsaProcessor, _ := validator.NewRSAProcessor(2048)
      cryptoComponent.AddCryptoProcessor("/", rsaProcessor)
  }

  // Assemble the stack
  stack := &Stack{
      cryptoComponent: cryptoComponent,
      codecComponent: codecComponent,
      head: cryptoComponent,
      pendingQueue: make(chan messages.MessageWrapper, config.PendingBufferSize), // random constant -- make this configurable
      pendingMap: make(map[string]MessageCallback),
      prefixTable: lpm.LPM{},
  }

  // Start the stack processing loop
  go stack.processInputQueue()

  return stack, nil
}

func CreateRaw(configString string) (*Stack, error) {
    var configStruct config.StackConfig
    err := json.Unmarshal([]byte(configString), &configStruct)
    if err == nil {
        return Create(configStruct)
    }
    return nil, err
}

func CreateTest() (*Stack, error) {
    return CreateRaw(`{"link": "loopback"}`)
}
<commit_msg>Defer component initialization to the very end<commit_after>package stack

import "log"
import "io/ioutil"

import "crypto/x509"
import "encoding/pem"
import "encoding/json"

import tlvCodec "github.com/chris-wood/spud/codec"
import "github.com/chris-wood/spud/tables/lpm"
import "github.com/chris-wood/spud/messages"
import "github.com/chris-wood/spud/messages/name"
import "github.com/chris-wood/spud/stack/config"
import "github.com/chris-wood/spud/stack/cache"
import "github.com/chris-wood/spud/stack/pit"
import "github.com/chris-wood/spud/stack/component/tunnel"
import "github.com/chris-wood/spud/stack/component/connector"
import "github.com/chris-wood/spud/stack/component/codec"
import "github.com/chris-wood/spud/stack/component/crypto"
import "github.com/chris-wood/spud/stack/component/crypto/validator"
import "github.com/chris-wood/spud/stack/component/crypto/context"

type MessageCallback func(msg messages.MessageWrapper)

type Component interface {
    // XXX: rename to Push and Pop, respectively
    Enqueue(messages.MessageWrapper)
    Dequeue() messages.MessageWrapper
    ProcessEgressMessages()
    ProcessIngressMessages()
}

type Stack struct {
    cryptoComponent Component
    codecComponent Component
    head Component
    bottom Component

    pendingMap map[string]MessageCallback
    pendingQueue chan messages.MessageWrapper
    prefixTable lpm.LPM
}

func (s *Stack) Enqueue(msg messages.MessageWrapper) {
    s.head.Enqueue(msg)
}

func (s *Stack) Dequeue() messages.MessageWrapper {
    return <- s.pendingQueue
}

func (s *Stack) Get(msg messages.MessageWrapper, callback MessageCallback) {
    if (msg.GetPacketType() != tlvCodec.T_INTEREST) {
        return
    }

    // Register the callback
    s.pendingMap[msg.Identifier()] = callback

    // Enqueue the message into the top of the stack
    s.head.Enqueue(msg)
}

func (s *Stack) Service(nameString string, callback MessageCallback) {
    prefixName, err := name.Parse(nameString)
    if err == nil {
        nameComponents := prefixName.SegmentStrings()
        s.prefixTable.Insert(nameComponents, callback)
    }
}

func (s *Stack) processInputQueue() {
    for ;; {
        // Dequeue messages as they arrive
        msg := s.head.Dequeue()

        switch (msg.GetPacketType()) {
        case tlvCodec.T_INTEREST:
            requestName := msg.Name()
            numSsegments := len(requestName.Segments)

            handled := false
            for index := 1; index <= numSsegments; index++ {
                nameComponents := requestName.SegmentStrings()
                callbackInterface, ok := s.prefixTable.Lookup(nameComponents)

                if ok {
                    callback := callbackInterface.(MessageCallback)
                    callback(msg)
                    s.prefixTable.Drop(nameComponents)
                    handled = true
                }
            }

            if !handled {
                s.pendingQueue <- msg
            }
        default:
            callback, ok := s.pendingMap[msg.Identifier()]
            if ok {
                callback(msg)
            } else {
                s.pendingQueue <- msg
            }
        }
    }
}

func (s *Stack) AddTunnel(tun tunnel.TunnelComponent) {
}

func Create(config config.StackConfig) (*Stack, error) {
  var err error

  // Create the link
  var fc connector.ForwarderConnector
  switch config.Link {
  case "tcp":
      locator := config.FwdAddress
      fc, err = connector.NewAthenaTCPForwarderConnector(locator)
      if err != nil {
          panic(err)
      }
      break
  case "loopback":
      fallthrough
  default:
      fc, _ = connector.NewLoopbackForwarderConnector()
      break
  }

  if fc == nil {
      log.Panic("Catastrophic failure: a connector was not created.")
  }

  // Create the shared data structures
  stackCache := cache.NewCache()
  stackPit := pit.NewPIT()

  // Create codec
  codecComponent := codec.NewCodecComponent(fc, stackCache, stackPit)

  // Create crypto component
  // XXX: defer validator and encryptor creation to the crypto component
  trustStore := context.NewTrustStore()
  cryptoComponent := crypto.NewCryptoComponent(trustStore, codecComponent)

  // Create the right crypto processors
  if len(config.Keys) > 0 {
      for _, keyFileName := range(config.Keys) {
          keyData, err := ioutil.ReadFile(keyFileName)
          block, _ := pem.Decode(keyData)
          privateKey, parseError := x509.ParsePKCS1PrivateKey(block.Bytes)
          if parseError != nil {
              log.Printf("Failed to parse private key: %s", err)
          } else {
              rsaProcessor, _ := validator.NewRSAProcessorWithKey(privateKey)
              cryptoComponent.AddCryptoProcessor("/", rsaProcessor)
          }
      }
  } else {
      rsaProcessor, _ := validator.NewRSAProcessor(2048)
      cryptoComponent.AddCryptoProcessor("/", rsaProcessor)
  }

  // Assemble the stack
  stack := &Stack{
      cryptoComponent: cryptoComponent,
      codecComponent: codecComponent,
      head: cryptoComponent,
      pendingQueue: make(chan messages.MessageWrapper, config.PendingBufferSize), // random constant -- make this configurable
      pendingMap: make(map[string]MessageCallback),
      prefixTable: lpm.LPM{},
  }

  // Start the stack processing loop
  go codecComponent.ProcessEgressMessages()
  go codecComponent.ProcessIngressMessages()
  go cryptoComponent.ProcessEgressMessages()
  go cryptoComponent.ProcessIngressMessages()
  go stack.processInputQueue()

  return stack, nil
}

func CreateRaw(configString string) (*Stack, error) {
    var configStruct config.StackConfig
    err := json.Unmarshal([]byte(configString), &configStruct)
    if err == nil {
        return Create(configStruct)
    }
    return nil, err
}

func CreateTest() (*Stack, error) {
    return CreateRaw(`{"link": "loopback"}`)
}
<|endoftext|>


<commit_before>package hsup

import (
	"bufio"
	"compress/gzip"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"syscall"

	"github.com/htcat/htcat"
	"gopkg.in/yaml.v2"
)

const HEROKU_STACKS_MANIFEST_URL = "https://s3.amazonaws.com/heroku_stacks_production/manifest.yml"

type HerokuStackImage struct {
	Name    string
	Version string
	URL     string `yaml:"url"`
	Md5     string
	Primary bool

	basedir string
}

func HerokuStacksFromManifest(stacksDir string) ([]HerokuStackImage, error) {
	// TODO: cache the manifest
	resp, err := http.Get(HEROKU_STACKS_MANIFEST_URL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(
			"invalid stacks manifest at %q",
			HEROKU_STACKS_MANIFEST_URL,
		)
	}
	manifest, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var stacks []HerokuStackImage
	if err := yaml.Unmarshal(manifest, &stacks); err != nil {
		return nil, err
	}
	for i, _ := range stacks {
		stacks[i].basedir = stacksDir
	}
	return stacks, nil
}

func CurrentStackImagePath(stacksDir, name string) (string, error) {
	// TODO: check if it is really mounted
	// TODO: order by version numbers and pick the last
	names, err := filepath.Glob(filepath.Join(stacksDir, name+"-*"))
	if err != nil {
		return "", err
	}
	last := len(names) - 1
	for i := range names {
		n := names[last-i]
		if !strings.HasSuffix(n, ".img") {
			return n, nil
		}
	}

	return "", errors.New("no matching stack image found")
}

func (img *HerokuStackImage) Dir() string {
	return filepath.Join(img.basedir, img.Name+"-"+img.Version)
}

func (img *HerokuStackImage) Filename() string {
	return img.Dir() + ".img"
}

func (img *HerokuStackImage) mount() error {
	var (
		imgFile = img.Filename()
		imgDir  = img.Dir()
	)
	if _, err := os.Stat(imgFile); err != nil {
		if err := img.fetch(); err != nil {
			return err
		}
	}
	if contents, err := ioutil.ReadDir(imgDir); err != nil {
		return err
	} else if len(contents) != 0 {
		return nil // already mounted
	}
	var flags uintptr = syscall.MS_NOATIME | syscall.MS_NODIRATIME |
		syscall.MS_NOSUID | syscall.MS_NODEV | syscall.MS_RDONLY
	return syscall.Mount(imgFile, imgDir, "", flags, "")
}

func (img *HerokuStackImage) fetch() error {
	log.Printf("Downloading stach image %q. This may take a while...", img.Name)
	// TODO check md5
	pr, pw := io.Pipe()
	defer pr.Close()
	defer pw.Close()

	u, err := url.Parse(img.URL)
	if err != nil {
		return err
	}

	dlResult := make(chan error)
	go func() {
		client := *http.DefaultClient
		if u.Scheme == "https" {
			client.Transport = &http.Transport{
				TLSClientConfig: &tls.Config{},
			}
		}
		// buffer downloads in case decompression can't keep up
		w := bufio.NewWriter(pw)
		_, err := htcat.New(&client, u, 5).WriteTo(w)
		dlResult <- err
	}()

	r, err := gzip.NewReader(pr)
	if err != nil {
		return err
	}
	defer r.Close()

	imageFile, err := os.Create(img.Filename())
	if err != nil {
		return err
	}
	defer imageFile.Close()
	if err := imageFile.Chmod(0644); err != nil {
		return err
	}

	if _, err := io.Copy(imageFile, r); err != nil {
		return err
	}
	log.Println("Stack image download finished")
	return <-dlResult
}
<commit_msg>just shell out to /bin/mount<commit_after>package hsup

import (
	"bufio"
	"compress/gzip"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/htcat/htcat"
	"gopkg.in/yaml.v2"
)

const HEROKU_STACKS_MANIFEST_URL = "https://s3.amazonaws.com/heroku_stacks_production/manifest.yml"

type HerokuStackImage struct {
	Name    string
	Version string
	URL     string `yaml:"url"`
	Md5     string
	Primary bool

	basedir string
}

func HerokuStacksFromManifest(stacksDir string) ([]HerokuStackImage, error) {
	// TODO: cache the manifest
	resp, err := http.Get(HEROKU_STACKS_MANIFEST_URL)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf(
			"invalid stacks manifest at %q",
			HEROKU_STACKS_MANIFEST_URL,
		)
	}
	manifest, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var stacks []HerokuStackImage
	if err := yaml.Unmarshal(manifest, &stacks); err != nil {
		return nil, err
	}
	for i, _ := range stacks {
		stacks[i].basedir = stacksDir
	}
	return stacks, nil
}

func CurrentStackImagePath(stacksDir, name string) (string, error) {
	// TODO: check if it is really mounted
	// TODO: order by version numbers and pick the last
	names, err := filepath.Glob(filepath.Join(stacksDir, name+"-*"))
	if err != nil {
		return "", err
	}
	last := len(names) - 1
	for i := range names {
		n := names[last-i]
		if !strings.HasSuffix(n, ".img") {
			return n, nil
		}
	}

	return "", errors.New("no matching stack image found")
}

func (img *HerokuStackImage) Dir() string {
	return filepath.Join(img.basedir, img.Name+"-"+img.Version)
}

func (img *HerokuStackImage) Filename() string {
	return img.Dir() + ".img"
}

func (img *HerokuStackImage) mount() error {
	var (
		imgFile = img.Filename()
		imgDir  = img.Dir()
	)
	if _, err := os.Stat(imgFile); err != nil {
		if err := img.fetch(); err != nil {
			return err
		}
	}
	if err := os.MkdirAll(imgDir, 0755); err != nil {
		return err
	}
	if contents, err := ioutil.ReadDir(imgDir); err != nil {
		return err
	} else if len(contents) != 0 {
		return nil // already mounted
	}
	log.Printf("Mounting stach image %q onto %q", imgFile, imgDir)
	out, err := exec.Command(
		"mount", "-o", "loop,nosuid,nodev,noatime,nodiratime,ro",
		imgFile, imgDir,
	).CombinedOutput()
	if err != nil {
		log.Println(string(out))
	}
	return err
}

func (img *HerokuStackImage) fetch() error {
	log.Printf("Downloading stach image %q. This may take a while...", img.Name)
	// TODO check md5
	pr, pw := io.Pipe()
	defer pr.Close()
	defer pw.Close()

	u, err := url.Parse(img.URL)
	if err != nil {
		return err
	}

	dlResult := make(chan error)
	go func() {
		client := *http.DefaultClient
		if u.Scheme == "https" {
			client.Transport = &http.Transport{
				TLSClientConfig: &tls.Config{},
			}
		}
		// buffer downloads in case decompression can't keep up
		w := bufio.NewWriter(pw)
		_, err := htcat.New(&client, u, 5).WriteTo(w)
		dlResult <- err
	}()

	r, err := gzip.NewReader(pr)
	if err != nil {
		return err
	}
	defer r.Close()

	imageFile, err := os.Create(img.Filename())
	if err != nil {
		return err
	}
	defer imageFile.Close()
	if err := imageFile.Chmod(0644); err != nil {
		return err
	}

	if _, err := io.Copy(imageFile, r); err != nil {
		return err
	}
	log.Println("Stack image download finished")
	return <-dlResult
}
<|endoftext|>


<commit_before>package stash

//go:generate easyjson $GOFILE

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strconv"

	"strings"

	"github.com/mailru/easyjson"
)

// PropertyValue holds a string value alongside an
// associated PrintKey
type PropertyValue struct {
	Value    string
	PrintKey int
}

// MarshalJSON implements custom serialization for this type
func (v *PropertyValue) MarshalJSON() ([]byte, error) {
	raw := make([]interface{}, 2)
	raw[0] = v.Value
	raw[1] = v.PrintKey

	return json.Marshal(raw)
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return [string, int] which is very unhelpful,
// so we take care of that right here
func (v *PropertyValue) UnmarshalJSON(b []byte) error {
	var raw []interface{}

	err := json.Unmarshal(b, &raw)
	if err != nil {
		return fmt.Errorf("invalid property pair, unparseable, err=%s", err)
	}

	if len(raw) != 2 {
		return fmt.Errorf("invalid property pair, (len()==%d)!=2", len(raw))
	}

	var ok bool
	v.Value, ok = raw[0].(string)
	if !ok {
		return fmt.Errorf("invalid property pair, first element not string")
	}

	// We have to use the widest possible type here and narrow...
	var broad float64
	broad, ok = raw[1].(float64)
	if !ok {
		return fmt.Errorf("invalid property pair, second element not float64")
	}
	v.PrintKey = int(broad)

	return nil
}

// Regexes we use for ItemMod parsing
var numberRegex = regexp.MustCompile(`\d{1,}`) // Grabbing digits
var hashRegex = regexp.MustCompile("#")        // Filling templates

// ItemMod is a modifier an item can have
type ItemMod struct {
	Template []byte
	Values   []uint16
}

// MarshalJSON implements custom serialization for this type
func (m *ItemMod) MarshalJSON() ([]byte, error) {
	result := []byte(m.Template)

	for _, value := range m.Values {

		loc := hashRegex.FindIndex(result)
		if loc == nil {
			return nil, fmt.Errorf("failed to match template string slot with expected value")
		}

		insert := []byte(strconv.FormatUint(uint64(value), 10))

		portion := hashRegex.ReplaceAll(result[:loc[1]], insert)
		result = append(portion, result[loc[1]:]...)
	}

	return result, nil
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return string which we run through
// regexp for ease
func (m *ItemMod) UnmarshalJSON(b []byte) error {
	// Find magnitudes in the mod
	matches := numberRegex.FindAll(b, -1)

	// Grab the template string representing the mod
	template := string(numberRegex.ReplaceAll(b, []byte("#")))

	// Convert the matches to numbers
	m.Values = make([]uint16, len(matches))
	for i, match := range matches {
		parsed, err := strconv.ParseUint(string(match), 10, 16)
		if err != nil {
			return fmt.Errorf("failed to convert match to int, err=%s", err)
		}
		m.Values[i] = uint16(parsed)
	}

	// If enclosed by quotes, get rid of them.
	//
	// This theoretically makes boltdb searches faster but
	// it just improves my quality of life...
	if len(template) > 2 &&
		strings.HasPrefix(template, "\"") &&
		strings.HasSuffix(template, "\"") {

		template = strings.TrimPrefix(template, "\"")
		template = strings.TrimSuffix(template, "\"")
	}

	m.Template = []byte(template)

	return nil
}

// Item represents a single item found from the stash api
//easyjson:json
type Item struct {
	Verified     bool      `json:"verified"`
	League       string    `json:"league"`
	ID           string    `json:"id"`
	Name         string    `json:"name"`
	TypeLine     string    `json:"typeLine"`
	Identified   bool      `json:"identified"`
	Corrupted    bool      `json:"corrupted"`
	ImplicitMods []ItemMod `json:"implicitMods,omitempty"`
	ExplicitMods []ItemMod `json:"explicitMods,omitempty"`
	Note         string    `json:"note,omitempty"`
	UtilityMods  []string  `json:"utilityMods,omitempty"`
	DescrText    string    `json:"descrText,omitempty"`
	// Additional data not present in response
	StashID    string `json:"-"`
	RootType   string `json:"-"`
	RootFlavor string `json:"-"`
}

// GetMods concats both ExplicitMods and ImplicitMods, returning the result
func (item *Item) GetMods() []ItemMod {
	mods := make([]ItemMod, 0)
	mods = append(mods, item.ImplicitMods...)
	mods = append(mods, item.ExplicitMods...)
	return mods
}

// Stash represents a stash tab with items and associated metadata
//easyjson:json
type Stash struct {
	AccountName       string `json:"accountName"`
	LastCharacterName string `json:"lastCharacterName"`
	ID                string `json:"id"`
	Stash             string `json:"stash"`
	StashType         string `json:"stashType"`
	Items             []Item `json:"items"`
	Public            bool   `json:"public"`
}

// Response represents expected structure of a stash api call
//easyjson:json
type Response struct {
	NextChangeID string  `json:"next_change_id"`
	Stashes      []Stash `json:"stashes"`
}

// StashAPIBase is the URL the stash api is located at
const StashAPIBase string = "http://www.pathofexile.com/api/public-stash-tabs"

// TestResponseLoc is where testing data is kept
const TestResponseLoc string = "StashResponse.json"

// GetStored gets the stored testing data and returns it
func GetStored() (*Response, error) {
	f, err := os.Open(TestResponseLoc)
	if err != nil {
		return nil, fmt.Errorf("failed to open TestResponseLoc, err=%s", err)
	}
	defer f.Close()

	var response Response
	err = easyjson.UnmarshalFromReader(f, &response)
	// err = decoder.Decode(&response)
	if err != nil {
		return nil, fmt.Errorf("failed to decode TestResponseLoc, err=%s", err)
	}

	// Add additioanl data to items and clean them up
	for _, stash := range response.Stashes {
		for i, item := range stash.Items {
			item.StashID = stash.ID
			// Handle cases where the item is identified purely by its typeline
			if len(item.Name) == 0 {
				item.Name = item.TypeLine
			}

			if len(item.Note) == 0 {
				item.Note = "unknown"
			}

			// Resolve the typeLine on an item to its flavor and root
			flavor, root, ok := MatchTypeline(item.TypeLine)
			if !ok {
				return nil, fmt.Errorf("failed to discover flavor and root for Item, name=%s, id=%s",
					item.Name, item.ID)
			}
			item.RootType = root
			item.RootFlavor = flavor

			stash.Items[i] = item
		}
	}

	return &response, nil
}

// FetchAndSetStore grabs the latest stash tab api update
// and stores it in TestResponseLoc
func FetchAndSetStore() error {
	resp, err := http.Get(StashAPIBase)
	if err != nil {
		return fmt.Errorf("failed to call stash api, err=%s", err)
	}
	defer resp.Body.Close()

	var response Response
	err = easyjson.UnmarshalFromReader(resp.Body, &response)
	if err != nil {
		return fmt.Errorf("failed to decode stash tab response, err=%s", err)
	}

	serial, err := json.Marshal(response)
	if err != nil {
		return fmt.Errorf("failed to marshal stash tab response, err=%s", err)
	}

	if err = ioutil.WriteFile(TestResponseLoc, serial, 0777); err != nil {
		return fmt.Errorf("failed to save stashResult.json, err=%s", err)
	}

	return nil
}
<commit_msg>stash switch to easyjson serialization in FetchAndSetStore<commit_after>package stash

//go:generate easyjson $GOFILE

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"regexp"
	"strconv"

	"strings"

	"github.com/mailru/easyjson"
)

// PropertyValue holds a string value alongside an
// associated PrintKey
type PropertyValue struct {
	Value    string
	PrintKey int
}

// MarshalJSON implements custom serialization for this type
func (v *PropertyValue) MarshalJSON() ([]byte, error) {
	raw := make([]interface{}, 2)
	raw[0] = v.Value
	raw[1] = v.PrintKey

	return json.Marshal(raw)
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return [string, int] which is very unhelpful,
// so we take care of that right here
func (v *PropertyValue) UnmarshalJSON(b []byte) error {
	var raw []interface{}

	err := json.Unmarshal(b, &raw)
	if err != nil {
		return fmt.Errorf("invalid property pair, unparseable, err=%s", err)
	}

	if len(raw) != 2 {
		return fmt.Errorf("invalid property pair, (len()==%d)!=2", len(raw))
	}

	var ok bool
	v.Value, ok = raw[0].(string)
	if !ok {
		return fmt.Errorf("invalid property pair, first element not string")
	}

	// We have to use the widest possible type here and narrow...
	var broad float64
	broad, ok = raw[1].(float64)
	if !ok {
		return fmt.Errorf("invalid property pair, second element not float64")
	}
	v.PrintKey = int(broad)

	return nil
}

// Regexes we use for ItemMod parsing
var numberRegex = regexp.MustCompile(`\d{1,}`) // Grabbing digits
var hashRegex = regexp.MustCompile("#")        // Filling templates

// ItemMod is a modifier an item can have
type ItemMod struct {
	Template []byte
	Values   []uint16
}

// MarshalJSON implements custom serialization for this type
func (m *ItemMod) MarshalJSON() ([]byte, error) {
	result := []byte(m.Template)

	for _, value := range m.Values {

		loc := hashRegex.FindIndex(result)
		if loc == nil {
			return nil, fmt.Errorf("failed to match template string slot with expected value")
		}

		insert := []byte(strconv.FormatUint(uint64(value), 10))

		portion := hashRegex.ReplaceAll(result[:loc[1]], insert)
		result = append(portion, result[loc[1]:]...)
	}

	return result, nil
}

// UnmarshalJSON implements custom deserialization for this type
//
// Typically, the GGG api will return string which we run through
// regexp for ease
func (m *ItemMod) UnmarshalJSON(b []byte) error {
	// Find magnitudes in the mod
	matches := numberRegex.FindAll(b, -1)

	// Grab the template string representing the mod
	template := string(numberRegex.ReplaceAll(b, []byte("#")))

	// Convert the matches to numbers
	m.Values = make([]uint16, len(matches))
	for i, match := range matches {
		parsed, err := strconv.ParseUint(string(match), 10, 16)
		if err != nil {
			return fmt.Errorf("failed to convert match to int, err=%s", err)
		}
		m.Values[i] = uint16(parsed)
	}

	// If enclosed by quotes, get rid of them.
	//
	// This theoretically makes boltdb searches faster but
	// it just improves my quality of life...
	if len(template) > 2 &&
		strings.HasPrefix(template, "\"") &&
		strings.HasSuffix(template, "\"") {

		template = strings.TrimPrefix(template, "\"")
		template = strings.TrimSuffix(template, "\"")
	}

	m.Template = []byte(template)

	return nil
}

// Item represents a single item found from the stash api
//easyjson:json
type Item struct {
	Verified     bool      `json:"verified"`
	League       string    `json:"league"`
	ID           string    `json:"id"`
	Name         string    `json:"name"`
	TypeLine     string    `json:"typeLine"`
	Identified   bool      `json:"identified"`
	Corrupted    bool      `json:"corrupted"`
	ImplicitMods []ItemMod `json:"implicitMods,omitempty"`
	ExplicitMods []ItemMod `json:"explicitMods,omitempty"`
	Note         string    `json:"note,omitempty"`
	UtilityMods  []string  `json:"utilityMods,omitempty"`
	DescrText    string    `json:"descrText,omitempty"`
	// Additional data not present in response
	StashID    string `json:"-"`
	RootType   string `json:"-"`
	RootFlavor string `json:"-"`
}

// GetMods concats both ExplicitMods and ImplicitMods, returning the result
func (item *Item) GetMods() []ItemMod {
	mods := make([]ItemMod, 0)
	mods = append(mods, item.ImplicitMods...)
	mods = append(mods, item.ExplicitMods...)
	return mods
}

// Stash represents a stash tab with items and associated metadata
//easyjson:json
type Stash struct {
	AccountName       string `json:"accountName"`
	LastCharacterName string `json:"lastCharacterName"`
	ID                string `json:"id"`
	Stash             string `json:"stash"`
	StashType         string `json:"stashType"`
	Items             []Item `json:"items"`
	Public            bool   `json:"public"`
}

// Response represents expected structure of a stash api call
//easyjson:json
type Response struct {
	NextChangeID string  `json:"next_change_id"`
	Stashes      []Stash `json:"stashes"`
}

// StashAPIBase is the URL the stash api is located at
const StashAPIBase string = "http://www.pathofexile.com/api/public-stash-tabs"

// TestResponseLoc is where testing data is kept
const TestResponseLoc string = "StashResponse.json"

// GetStored gets the stored testing data and returns it
func GetStored() (*Response, error) {
	f, err := os.Open(TestResponseLoc)
	if err != nil {
		return nil, fmt.Errorf("failed to open TestResponseLoc, err=%s", err)
	}
	defer f.Close()

	var response Response
	err = easyjson.UnmarshalFromReader(f, &response)
	// err = decoder.Decode(&response)
	if err != nil {
		return nil, fmt.Errorf("failed to decode TestResponseLoc, err=%s", err)
	}

	// Add additioanl data to items and clean them up
	for _, stash := range response.Stashes {
		for i, item := range stash.Items {
			item.StashID = stash.ID
			// Handle cases where the item is identified purely by its typeline
			if len(item.Name) == 0 {
				item.Name = item.TypeLine
			}

			if len(item.Note) == 0 {
				item.Note = "unknown"
			}

			// Resolve the typeLine on an item to its flavor and root
			flavor, root, ok := MatchTypeline(item.TypeLine)
			if !ok {
				return nil, fmt.Errorf("failed to discover flavor and root for Item, name=%s, id=%s",
					item.Name, item.ID)
			}
			item.RootType = root
			item.RootFlavor = flavor

			stash.Items[i] = item
		}
	}

	return &response, nil
}

// FetchAndSetStore grabs the latest stash tab api update
// and stores it in TestResponseLoc
func FetchAndSetStore() error {
	resp, err := http.Get(StashAPIBase)
	if err != nil {
		return fmt.Errorf("failed to call stash api, err=%s", err)
	}
	defer resp.Body.Close()

	var response Response
	err = easyjson.UnmarshalFromReader(resp.Body, &response)
	if err != nil {
		return fmt.Errorf("failed to decode stash tab response, err=%s", err)
	}

	serial, err := response.MarshalJSON()
	if err != nil {
		return fmt.Errorf("failed to marshal stash tab response, err=%s", err)
	}

	if err = ioutil.WriteFile(TestResponseLoc, serial, 0777); err != nil {
		return fmt.Errorf("failed to save stashResult.json, err=%s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package stats

import (
	"fmt"
	"math"
	"sort"
)

type Stats struct {
	Values []int
	sorted bool
	sum    int
}

func (stats *Stats) Add(i int) {
	stats.sum += i
	stats.Values = append(stats.Values, i)
}

func (stats *Stats) Max() int {
	stats.Sort()
	if len(stats.Values) == 0 {
		return 0
	}
	return stats.Values[len(stats.Values)-1]
}

func (stats *Stats) Min() int {
	stats.Sort()
	if len(stats.Values) == 0 {
		return 0
	}
	return stats.Values[0]
}

func (stats *Stats) Sum() int {
	return stats.sum
}

func (stats *Stats) Sort() {
	if !stats.sorted {
		sort.Ints(stats.Values)
		stats.sorted = true
	}
}

func (stats *Stats) Avg() int {
	return int(math.Floor(float64(stats.Sum()) / float64(len(stats.Values))))
}

func (stats *Stats) Median() int {
	return stats.Perc(50)
}

func Percentile(values []int, perc int) (o int) {
	middle := float64(len(values)) * float64(perc) / 100.0
	floor := int(math.Floor(middle))
	return values[floor]
}

func (stats *Stats) Perc(perc int) int {
	stats.Sort()
	return Percentile(stats.Values, perc)
}

func (stats *Stats) String() string {
	return fmt.Sprintf("len: %d, avg: %d, med: %d, perc_95: %d, perc_99: %d, max: %d, min: %d",
		len(stats.Values), stats.Avg(), stats.Median(), stats.Perc(95), stats.Perc(99),
		stats.Max(),
		stats.Min(),
	)
}

func (stats *Stats) Len() int {
	return len(stats.Values)
}

func (stats *Stats) Reset() {
	stats.Values = stats.Values[:0]
	stats.sorted = false
}
<commit_msg>add comment<commit_after>package stats

import (
	"fmt"
	"math"
	"sort"
)

type Stats struct {
	Values []int
	sorted bool
	sum    int
}

func (stats *Stats) Add(i int) {
	stats.sum += i
	stats.Values = append(stats.Values, i)
}

func (stats *Stats) Max() int {
	stats.Sort()
	if len(stats.Values) == 0 {
		return 0
	}
	return stats.Values[len(stats.Values)-1]
}

func (stats *Stats) Min() int {
	stats.Sort()
	if len(stats.Values) == 0 {
		return 0
	}
	return stats.Values[0]
}

func (stats *Stats) Sum() int {
	return stats.sum
}

func (stats *Stats) Sort() {
	if !stats.sorted {
		sort.Ints(stats.Values)
		stats.sorted = true
	}
}

func (stats *Stats) Avg() int {
	return int(math.Floor(float64(stats.Sum()) / float64(len(stats.Values))))
}

func (stats *Stats) Median() int {
	return stats.Perc(50)
}

func Percentile(values []int, perc int) (o int) {
	middle := float64(len(values)) * float64(perc) / 100.0
	floor := int(math.Floor(middle))
	return values[floor]
}

// Perc calculates the percentile, use 50 for median
func (stats *Stats) Perc(perc int) int {
	stats.Sort()
	return Percentile(stats.Values, perc)
}

func (stats *Stats) String() string {
	return fmt.Sprintf("len: %d, avg: %d, med: %d, perc_95: %d, perc_99: %d, max: %d, min: %d",
		len(stats.Values), stats.Avg(), stats.Median(), stats.Perc(95), stats.Perc(99),
		stats.Max(),
		stats.Min(),
	)
}

func (stats *Stats) Len() int {
	return len(stats.Values)
}

func (stats *Stats) Reset() {
	stats.Values = stats.Values[:0]
	stats.sorted = false
}
<|endoftext|>


<commit_before>package statsd

import (
	"bytes"
	"net"
	"strconv"
	"testing"
	"time"
)

func TestTimerSend(t *testing.T) {
	conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: 8125})
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()

	conn.SetDeadline(time.Now().Add(time.Second))

	timer := Timer()

	time.Sleep(time.Millisecond)

	took := timer.Send("foo")

	if took == 0 {
		t.Error("Send() took no time")
	}

	got := make([]byte, 8)
	if _, _, err := conn.ReadFromUDP(got); err != nil {
		t.Fatal(err)
	}

	exp := []byte("foo:" +
		strconv.FormatUint(uint64(took.Nanoseconds()/1e6), 10) + "|ms")

	if !bytes.Equal(got, exp) {
		t.Errorf("got: %s; want: %s", got, exp)
	}
}
<commit_msg>Expand test coverage<commit_after>package statsd

import (
	"bytes"
	"net"
	"strconv"
	"testing"
	"time"
)

func TestTime(t *testing.T) {
	test(t, func(conn *net.UDPConn) {
		dur, err := time.ParseDuration("1h30m45s")
		if err != nil {
			t.Fatal(err)
		}

		Time("foo", dur)

		exp := []byte("foo:" +
			strconv.FormatUint(uint64(dur.Nanoseconds()/1e6), 10) + "|ms")

		got := make([]byte, len(exp))
		if _, _, err := conn.ReadFromUDP(got); err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(got, exp) {
			t.Errorf("got: %s; want: %s", got, exp)
		}
	})
}

func TestTimeSampled(t *testing.T) {
	test(t, func(conn *net.UDPConn) {
		dur, err := time.ParseDuration("1h30m45s")
		if err != nil {
			t.Fatal(err)
		}

		TimeSampled(0.5, "foo", dur)

		exp := []byte("foo:" +
			strconv.FormatUint(uint64(dur.Nanoseconds()/1e6), 10) + "|ms|@0.5")

		got := make([]byte, len(exp))
		if _, _, err := conn.ReadFromUDP(got); err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(got, exp) {
			t.Errorf("got: %s; want: %s", got, exp)
		}
	})
}

func TestTimerReset(t *testing.T) {
	timer := Timer()
	time.Sleep(time.Millisecond)

	then := timer.start
	timer.Reset()

	if then == timer.start {
		t.Errorf("timer did not reset: was %s; is %s", then, timer.start)
	}
}

func TestTimerSend(t *testing.T) {
	test(t, func(conn *net.UDPConn) {
		timer := Timer()
		time.Sleep(time.Millisecond)

		took := timer.Send("foo")
		if took == 0 {
			t.Error("Send() took no time")
		}

		exp := []byte("foo:" +
			strconv.FormatUint(uint64(took.Nanoseconds()/1e6), 10) + "|ms")

		got := make([]byte, len(exp))
		if _, _, err := conn.ReadFromUDP(got); err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(got, exp) {
			t.Errorf("got: %s; want: %s", got, exp)
		}
	})
}

func TestTimerSendSampled(t *testing.T) {
	test(t, func(conn *net.UDPConn) {
		timer := Timer()
		time.Sleep(time.Millisecond)

		took := timer.SendSampled(0.5, "foo")
		if took == 0 {
			t.Error("Send() took no time")
		}

		exp := []byte("foo:" +
			strconv.FormatUint(uint64(took.Nanoseconds()/1e6), 10) + "|ms|@0.5")

		got := make([]byte, len(exp))
		if _, _, err := conn.ReadFromUDP(got); err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(got, exp) {
			t.Errorf("got: %s; want: %s", got, exp)
		}
	})
}

func TestInc(t *testing.T) {
	test(t, func(conn *net.UDPConn) {
		Inc("foo")

		exp := []byte("foo:1|c")
		got := make([]byte, len(exp))
		if _, _, err := conn.ReadFromUDP(got); err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(got, exp) {
			t.Errorf("got: %s; want: %s", got, exp)
		}
	})
}

func TestIncSampled(t *testing.T) {
	test(t, func(conn *net.UDPConn) {
		IncSampled(0.5, "foo")

		exp := []byte("foo:1|c|@0.5")
		got := make([]byte, len(exp))
		if _, _, err := conn.ReadFromUDP(got); err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(got, exp) {
			t.Errorf("got: %s; want: %s", got, exp)
		}
	})
}

func TestGauge(t *testing.T) {
	test(t, func(conn *net.UDPConn) {
		Gauge("foo", 42)

		exp := []byte("foo:42|g")
		got := make([]byte, len(exp))
		if _, _, err := conn.ReadFromUDP(got); err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(got, exp) {
			t.Errorf("got: %s; want: %s", got, exp)
		}
	})
}

func TestGaugeSampled(t *testing.T) {
	test(t, func(conn *net.UDPConn) {
		GaugeSampled(0.5, "foo", 42)

		exp := []byte("foo:42|g|@0.5")
		got := make([]byte, len(exp))
		if _, _, err := conn.ReadFromUDP(got); err != nil {
			t.Fatal(err)
		}

		if !bytes.Equal(got, exp) {
			t.Errorf("got: %s; want: %s", got, exp)
		}
	})
}

func test(t *testing.T, check func(*net.UDPConn)) {
	conn, err := net.ListenUDP("udp", &net.UDPAddr{Port: 8125})
	if err != nil {
		t.Fatal(err)
	}
	defer conn.Close()

	conn.SetDeadline(time.Now().Add(time.Second))

	check(conn)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const (
	VERSION                 = "0.5.2-alpha"
	MAX_UNPROCESSED_PACKETS = 1000
	MAX_UDP_PACKET_SIZE     = 512
)

var signalchan chan os.Signal

type Packet struct {
	Bucket   string
	Value    interface{}
	Modifier string
	Sampling float32
}

type Uint64Slice []uint64

func (s Uint64Slice) Len() int           { return len(s) }
func (s Uint64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] }

type Percentiles []*Percentile
type Percentile struct {
	float float64
	str   string
}

func (a *Percentiles) Set(s string) error {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return err
	}
	*a = append(*a, &Percentile{f, strings.Replace(s, ".", "_", -1)})
	return nil
}
func (p *Percentile) String() string {
	return p.str
}
func (a *Percentiles) String() string {
	return fmt.Sprintf("%v", *a)
}

var (
	serviceAddress   = flag.String("address", ":8125", "UDP service address")
	graphiteAddress  = flag.String("graphite", "127.0.0.1:2003", "Graphite service address (or - to disable)")
	flushInterval    = flag.Int64("flush-interval", 10, "Flush interval (seconds)")
	debug            = flag.Bool("debug", false, "print statistics sent to graphite")
	showVersion      = flag.Bool("version", false, "print version string")
	persistCountKeys = flag.Int64("persist-count-keys", 60, "number of flush-interval's to persist count keys")
	percentThreshold = Percentiles{}
)

func init() {
	flag.Var(&percentThreshold, "percent-threshold", "Threshold percent (may be given multiple times)")
}

var (
	In       = make(chan *Packet, MAX_UNPROCESSED_PACKETS)
	counters = make(map[string]int64)
	gauges   = make(map[string]uint64)
	timers   = make(map[string]Uint64Slice)
)

func monitor() {
	ticker := time.NewTicker(time.Duration(*flushInterval) * time.Second)
	for {
		select {
		case sig := <-signalchan:
			fmt.Printf("!! Caught signal %d... shutting down\n", sig)
			submit()
			return
		case <-ticker.C:
			submit()
		case s := <-In:
			if s.Modifier == "ms" {
				_, ok := timers[s.Bucket]
				if !ok {
					var t Uint64Slice
					timers[s.Bucket] = t
				}
				timers[s.Bucket] = append(timers[s.Bucket], s.Value.(uint64))
			} else if s.Modifier == "g" {
				gauges[s.Bucket] = s.Value.(uint64)
			} else {
				v, ok := counters[s.Bucket]
				if !ok || v < 0 {
					counters[s.Bucket] = 0
				}
				counters[s.Bucket] += int64(float64(s.Value.(int64)) * float64(1/s.Sampling))
			}
		}
	}
}

func submit() {
	var buffer bytes.Buffer
	var num int64

	now := time.Now().Unix()

	client, err := net.Dial("tcp", *graphiteAddress)
	if err != nil {
		log.Printf("ERROR: dialing %s - %s", *graphiteAddress, err)
		if *debug {
			log.Printf("WARNING: resetting counters when in debug mode")
			processCounters(&buffer, now)
			processGauges(&buffer, now)
			processTimers(&buffer, now)
		}
		return
	}
	defer client.Close()

	num += processCounters(&buffer, now)
	num += processGauges(&buffer, now)
	num += processTimers(&buffer, now)
	if num == 0 {
		return
	}

	if *debug {
		for _, line := range bytes.Split(buffer.Bytes(), []byte("\n")) {
			if len(line) == 0 {
				continue
			}
			log.Printf("DEBUG: %s", line)
		}
	}

	_, err = client.Write(buffer.Bytes())
	if err != nil {
		log.Printf("ERROR: failed to write stats - %s", err)
		return
	}

	log.Printf("sent %d stats to %s", num, *graphiteAddress)
}

func processCounters(buffer *bytes.Buffer, now int64) int64 {
	var num int64
	// continue sending zeros for counters for a short period of time
	// even if we have no new data. for more context see https://github.com/bitly/statsdaemon/pull/8
	for s, c := range counters {
		switch {
		case c <= *persistCountKeys:
			continue
		case c < 0:
			counters[s] -= 1
			fmt.Fprintf(buffer, "%s %d %d\n", s, 0, now)
		case c >= 0:
			counters[s] = -1
			fmt.Fprintf(buffer, "%s %d %d\n", s, c, now)
		}
		num++
	}
	return num
}

func processGauges(buffer *bytes.Buffer, now int64) int64 {
	var num int64
	for g, c := range gauges {
		if c == math.MaxUint64 {
			continue
		}
		fmt.Fprintf(buffer, "%s %d %d\n", g, c, now)
		gauges[g] = math.MaxUint64
		num++
	}
	return num
}

func processTimers(buffer *bytes.Buffer, now int64) int64 {
	var num int64
	for u, t := range timers {
		if len(t) > 0 {
			num++

			sort.Sort(t)
			min := t[0]
			max := t[len(t)-1]
			maxAtThreshold := max
			count := len(t)

			sum := uint64(0)
			for _, value := range t {
				sum += value
			}
			mean := float64(sum) / float64(len(t))

			for _, pct := range percentThreshold {

				if len(t) > 1 {
					indexOfPerc := int(math.Ceil(((pct.float / 100.0) * float64(count)) + 0.5))
					if indexOfPerc >= count {
						indexOfPerc = count - 1
					}
					maxAtThreshold = t[indexOfPerc]
				}

				fmt.Fprintf(buffer, "%s.upper_%s %d %d\n", u, pct.str, maxAtThreshold, now)
			}

			var z Uint64Slice
			timers[u] = z

			fmt.Fprintf(buffer, "%s.mean %f %d\n", u, mean, now)
			fmt.Fprintf(buffer, "%s.upper %d %d\n", u, max, now)
			fmt.Fprintf(buffer, "%s.lower %d %d\n", u, min, now)
			fmt.Fprintf(buffer, "%s.count %d %d\n", u, count, now)
		}
	}
	return num
}

var packetRegexp = regexp.MustCompile("^([^:]+):(-?[0-9]+)\\|(g|c|ms)(\\|@([0-9\\.]+))?\n?$")

func parseMessage(data []byte) []*Packet {
	var output []*Packet
	for _, line := range bytes.Split(data, []byte("\n")) {
		if len(line) == 0 {
			continue
		}

		item := packetRegexp.FindSubmatch(line)
		if len(item) == 0 {
			continue
		}

		var err error
		var value interface{}
		modifier := string(item[3])
		switch modifier {
		case "c":
			value, err = strconv.ParseInt(string(item[2]), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseInt %s - %s", item[2], err)
				continue
			}
		default:
			value, err = strconv.ParseUint(string(item[2]), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseUint %s - %s", item[2], err)
				continue
			}
		}

		sampleRate, err := strconv.ParseFloat(string(item[5]), 32)
		if err != nil {
			sampleRate = 1
		}

		packet := &Packet{
			Bucket:   string(item[1]),
			Value:    value,
			Modifier: modifier,
			Sampling: float32(sampleRate),
		}
		output = append(output, packet)
	}
	return output
}

func udpListener() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	log.Printf("listening on %s", address)
	listener, err := net.ListenUDP("udp", address)
	if err != nil {
		log.Fatalf("ERROR: ListenUDP - %s", err)
	}
	defer listener.Close()

	message := make([]byte, MAX_UDP_PACKET_SIZE)
	for {
		n, remaddr, err := listener.ReadFromUDP(message)
		if err != nil {
			log.Printf("ERROR: reading UDP packet from %+v - %s", remaddr, err)
			continue
		}

		for _, p := range parseMessage(message[:n]) {
			In <- p
		}
	}
}

func main() {
	flag.Parse()

	if *showVersion {
		fmt.Printf("statsdaemon v%s (built w/%s)\n", VERSION, runtime.Version())
		return
	}

	signalchan = make(chan os.Signal, 1)
	signal.Notify(signalchan, syscall.SIGTERM)
	*persistCountKeys = -1 * (*persistCountKeys)

	go udpListener()
	monitor()
}
<commit_msg>Clarifying that percentile threshold is 0-100<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"regexp"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"
)

const (
	VERSION                 = "0.5.2-alpha"
	MAX_UNPROCESSED_PACKETS = 1000
	MAX_UDP_PACKET_SIZE     = 512
)

var signalchan chan os.Signal

type Packet struct {
	Bucket   string
	Value    interface{}
	Modifier string
	Sampling float32
}

type Uint64Slice []uint64

func (s Uint64Slice) Len() int           { return len(s) }
func (s Uint64Slice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s Uint64Slice) Less(i, j int) bool { return s[i] < s[j] }

type Percentiles []*Percentile
type Percentile struct {
	float float64
	str   string
}

func (a *Percentiles) Set(s string) error {
	f, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return err
	}
	*a = append(*a, &Percentile{f, strings.Replace(s, ".", "_", -1)})
	return nil
}
func (p *Percentile) String() string {
	return p.str
}
func (a *Percentiles) String() string {
	return fmt.Sprintf("%v", *a)
}

var (
	serviceAddress   = flag.String("address", ":8125", "UDP service address")
	graphiteAddress  = flag.String("graphite", "127.0.0.1:2003", "Graphite service address (or - to disable)")
	flushInterval    = flag.Int64("flush-interval", 10, "Flush interval (seconds)")
	debug            = flag.Bool("debug", false, "print statistics sent to graphite")
	showVersion      = flag.Bool("version", false, "print version string")
	persistCountKeys = flag.Int64("persist-count-keys", 60, "number of flush-interval's to persist count keys")
	percentThreshold = Percentiles{}
)

func init() {
	flag.Var(&percentThreshold, "percent-threshold", "Threshold percent (0-100, may be given multiple times)")
}

var (
	In       = make(chan *Packet, MAX_UNPROCESSED_PACKETS)
	counters = make(map[string]int64)
	gauges   = make(map[string]uint64)
	timers   = make(map[string]Uint64Slice)
)

func monitor() {
	ticker := time.NewTicker(time.Duration(*flushInterval) * time.Second)
	for {
		select {
		case sig := <-signalchan:
			fmt.Printf("!! Caught signal %d... shutting down\n", sig)
			submit()
			return
		case <-ticker.C:
			submit()
		case s := <-In:
			if s.Modifier == "ms" {
				_, ok := timers[s.Bucket]
				if !ok {
					var t Uint64Slice
					timers[s.Bucket] = t
				}
				timers[s.Bucket] = append(timers[s.Bucket], s.Value.(uint64))
			} else if s.Modifier == "g" {
				gauges[s.Bucket] = s.Value.(uint64)
			} else {
				v, ok := counters[s.Bucket]
				if !ok || v < 0 {
					counters[s.Bucket] = 0
				}
				counters[s.Bucket] += int64(float64(s.Value.(int64)) * float64(1/s.Sampling))
			}
		}
	}
}

func submit() {
	var buffer bytes.Buffer
	var num int64

	now := time.Now().Unix()

	client, err := net.Dial("tcp", *graphiteAddress)
	if err != nil {
		log.Printf("ERROR: dialing %s - %s", *graphiteAddress, err)
		if *debug {
			log.Printf("WARNING: resetting counters when in debug mode")
			processCounters(&buffer, now)
			processGauges(&buffer, now)
			processTimers(&buffer, now)
		}
		return
	}
	defer client.Close()

	num += processCounters(&buffer, now)
	num += processGauges(&buffer, now)
	num += processTimers(&buffer, now)
	if num == 0 {
		return
	}

	if *debug {
		for _, line := range bytes.Split(buffer.Bytes(), []byte("\n")) {
			if len(line) == 0 {
				continue
			}
			log.Printf("DEBUG: %s", line)
		}
	}

	_, err = client.Write(buffer.Bytes())
	if err != nil {
		log.Printf("ERROR: failed to write stats - %s", err)
		return
	}

	log.Printf("sent %d stats to %s", num, *graphiteAddress)
}

func processCounters(buffer *bytes.Buffer, now int64) int64 {
	var num int64
	// continue sending zeros for counters for a short period of time
	// even if we have no new data. for more context see https://github.com/bitly/statsdaemon/pull/8
	for s, c := range counters {
		switch {
		case c <= *persistCountKeys:
			continue
		case c < 0:
			counters[s] -= 1
			fmt.Fprintf(buffer, "%s %d %d\n", s, 0, now)
		case c >= 0:
			counters[s] = -1
			fmt.Fprintf(buffer, "%s %d %d\n", s, c, now)
		}
		num++
	}
	return num
}

func processGauges(buffer *bytes.Buffer, now int64) int64 {
	var num int64
	for g, c := range gauges {
		if c == math.MaxUint64 {
			continue
		}
		fmt.Fprintf(buffer, "%s %d %d\n", g, c, now)
		gauges[g] = math.MaxUint64
		num++
	}
	return num
}

func processTimers(buffer *bytes.Buffer, now int64) int64 {
	var num int64
	for u, t := range timers {
		if len(t) > 0 {
			num++

			sort.Sort(t)
			min := t[0]
			max := t[len(t)-1]
			maxAtThreshold := max
			count := len(t)

			sum := uint64(0)
			for _, value := range t {
				sum += value
			}
			mean := float64(sum) / float64(len(t))

			for _, pct := range percentThreshold {

				if len(t) > 1 {
					indexOfPerc := int(math.Ceil(((pct.float / 100.0) * float64(count)) + 0.5))
					if indexOfPerc >= count {
						indexOfPerc = count - 1
					}
					maxAtThreshold = t[indexOfPerc]
				}

				fmt.Fprintf(buffer, "%s.upper_%s %d %d\n", u, pct.str, maxAtThreshold, now)
			}

			var z Uint64Slice
			timers[u] = z

			fmt.Fprintf(buffer, "%s.mean %f %d\n", u, mean, now)
			fmt.Fprintf(buffer, "%s.upper %d %d\n", u, max, now)
			fmt.Fprintf(buffer, "%s.lower %d %d\n", u, min, now)
			fmt.Fprintf(buffer, "%s.count %d %d\n", u, count, now)
		}
	}
	return num
}

var packetRegexp = regexp.MustCompile("^([^:]+):(-?[0-9]+)\\|(g|c|ms)(\\|@([0-9\\.]+))?\n?$")

func parseMessage(data []byte) []*Packet {
	var output []*Packet
	for _, line := range bytes.Split(data, []byte("\n")) {
		if len(line) == 0 {
			continue
		}

		item := packetRegexp.FindSubmatch(line)
		if len(item) == 0 {
			continue
		}

		var err error
		var value interface{}
		modifier := string(item[3])
		switch modifier {
		case "c":
			value, err = strconv.ParseInt(string(item[2]), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseInt %s - %s", item[2], err)
				continue
			}
		default:
			value, err = strconv.ParseUint(string(item[2]), 10, 64)
			if err != nil {
				log.Printf("ERROR: failed to ParseUint %s - %s", item[2], err)
				continue
			}
		}

		sampleRate, err := strconv.ParseFloat(string(item[5]), 32)
		if err != nil {
			sampleRate = 1
		}

		packet := &Packet{
			Bucket:   string(item[1]),
			Value:    value,
			Modifier: modifier,
			Sampling: float32(sampleRate),
		}
		output = append(output, packet)
	}
	return output
}

func udpListener() {
	address, _ := net.ResolveUDPAddr("udp", *serviceAddress)
	log.Printf("listening on %s", address)
	listener, err := net.ListenUDP("udp", address)
	if err != nil {
		log.Fatalf("ERROR: ListenUDP - %s", err)
	}
	defer listener.Close()

	message := make([]byte, MAX_UDP_PACKET_SIZE)
	for {
		n, remaddr, err := listener.ReadFromUDP(message)
		if err != nil {
			log.Printf("ERROR: reading UDP packet from %+v - %s", remaddr, err)
			continue
		}

		for _, p := range parseMessage(message[:n]) {
			In <- p
		}
	}
}

func main() {
	flag.Parse()

	if *showVersion {
		fmt.Printf("statsdaemon v%s (built w/%s)\n", VERSION, runtime.Version())
		return
	}

	signalchan = make(chan os.Signal, 1)
	signal.Notify(signalchan, syscall.SIGTERM)
	*persistCountKeys = -1 * (*persistCountKeys)

	go udpListener()
	monitor()
}
<|endoftext|>