<commit_before>package xdg

import (
	"os"
	"path/filepath"
	"strings"
)

// App is an aggregation of XDG information for a named application. Useful to
// propagate app configuration.
type App struct {
	Name string
}

// DataPath determines the full path of a data file.
func (a App) DataPath(file string) string {
	base := os.Getenv("XDG_DATA_HOME")
	if base == "" {
		base = DataHome()
	}
	return filepath.Join(base, a.Name, file)
}

// ConfigPath determines the full path of a data file.
func (a App) ConfigPath(file string) string {
	base := os.Getenv("XDG_CONFIG_HOME")
	if base == "" {
		base = ConfigHome()
	}
	return filepath.Join(base, a.Name, file)
}

// CachePath determines the full path of a cached file.
func (a App) CachePath(file string) string {
	base := os.Getenv("XDG_CACHE_HOME")
	if base == "" {
		base = CacheHome()
	}
	return filepath.Join(base, a.Name, file)
}

// SystemDataPaths determines system-wide possible paths for a data file.
func (a App) SystemDataPaths(file string) []string {
	var bases []string
	env := os.Getenv("XDG_DATA_DIRS")
	if env != "" {
		bases = strings.Split(env, ":")
	} else {
		bases = DataDirs()
	}
	var dirs []string
	for _, it := range bases {
		dirs = append(dirs, filepath.Join(it, a.Name, file))
	}
	return dirs
}

// SystemConfigPaths determines system-wide possible paths for a config file.
func (a App) SystemConfigPaths(file string) []string {
	var bases []string
	env := os.Getenv("XDG_CONFIG_DIRS")
	if env != "" {
		bases = strings.Split(env, ":")
	} else {
		bases = ConfigDirs()
	}
	var dirs []string
	for _, it := range bases {
		dirs = append(dirs, filepath.Join(it, a.Name, file))
	}
	return dirs
}

var defaultApp App

// SetName for the default application. Used for package-wide functions.
func SetName(name string) {
	defaultApp.Name = name
}

// DataPath determines the full path of a data file.
func DataPath(file string) string {
	return defaultApp.DataPath(file)
}

// ConfigPath determines the full path of a data file.
func ConfigPath(file string) string {
	return defaultApp.ConfigPath(file)
}

// CachePath determines the full path of a cached file.
func CachePath(file string) string {
	return defaultApp.CachePath(file)
}

// SystemDataPaths determines system-wide possible paths for a data file.
func SystemDataPaths(file string) []string {
	return defaultApp.SystemDataPaths(file)
}

// SystemConfigPaths determines system-wide possible paths for a config file.
func SystemConfigPaths(file string) []string {
	return defaultApp.SystemConfigPaths(file)
}
<commit_msg>refactor internals<commit_after>package xdg

import (
	"os"
	"path/filepath"
	"strings"
)

// App is an aggregation of XDG information for a named application. Useful to
// propagate app configuration.
type App struct {
	Name string
}

func (a App) path(envVar string, defaultFn func() string, file string) string {
	base := os.Getenv(envVar)
	if base == "" {
		base = defaultFn()
	}
	return filepath.Join(base, a.Name, file)
}

// DataPath determines the full path of a data file.
func (a App) DataPath(file string) string {
	return a.path("XDG_DATA_HOME", DataHome, file)
}

// ConfigPath determines the full path of a data file.
func (a App) ConfigPath(file string) string {
	return a.path("XDG_CONFIG_HOME", ConfigHome, file)
}

// CachePath determines the full path of a cached file.
func (a App) CachePath(file string) string {
	return a.path("XDG_CACHE_HOME", CacheHome, file)
}

func (a App) multiPaths(envVar string, defaultFn func() []string, file string) []string {
	var bases []string
	env := os.Getenv(envVar)
	if env != "" {
		bases = strings.Split(env, ":")
	} else {
		bases = defaultFn()
	}
	var dirs []string
	for _, it := range bases {
		dirs = append(dirs, filepath.Join(it, a.Name, file))
	}
	return dirs
}

// SystemDataPaths determines system-wide possible paths for a data file.
func (a App) SystemDataPaths(file string) []string {
	return a.multiPaths("XDG_DATA_DIRS", DataDirs, file)
}

// SystemConfigPaths determines system-wide possible paths for a config file.
func (a App) SystemConfigPaths(file string) []string {
	return a.multiPaths("XDG_CONFIG_DIRS", ConfigDirs, file)
}

var defaultApp App

// SetName for the default application. Used for package-wide functions.
func SetName(name string) {
	defaultApp.Name = name
}

// DataPath determines the full path of a data file.
func DataPath(file string) string {
	return defaultApp.DataPath(file)
}

// ConfigPath determines the full path of a data file.
func ConfigPath(file string) string {
	return defaultApp.ConfigPath(file)
}

// CachePath determines the full path of a cached file.
func CachePath(file string) string {
	return defaultApp.CachePath(file)
}

// SystemDataPaths determines system-wide possible paths for a data file.
func SystemDataPaths(file string) []string {
	return defaultApp.SystemDataPaths(file)
}

// SystemConfigPaths determines system-wide possible paths for a config file.
func SystemConfigPaths(file string) []string {
	return defaultApp.SystemConfigPaths(file)
}
<|endoftext|>


<commit_before>package libvirt

import (
	"bytes"
	"text/template"

	"github.com/mistifyio/mistify-agent/client"
)

var tmpl *template.Template

func init() {
	const xml = `
<domain type="{{.Type}}">
  <name>{{.Id}}</name>
  <memory unit="MiB">{{.Memory}}</memory>
  <vcpu>{{.Cpu}}</vcpu>

  {{if .Metadata}}
  <metadata>
    {{range .Metadata}}
    {{end}}
  </metadata>
  {{end}}

  <os>
    <type>hvm</type>
  </os>
  <devices>
    {{range .Nics}}
    <interface type="bridge">
      <guest dev="{{.Name}}" />
      {{if .Mac}}<mac address="{{.Mac}}" />{{end}}
      <source bridge='{{.Network}}'/>
      <target dev="{{.Device}}" />
      {{if .Model}}<model type="{{.Model}}" />{{end}}
    </interface>
    {{end}}

    {{range .Disks}}
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" />
      <source dev="/dev/zvol/%s/images/{{.Source}}" />
      <target dev="{{.Device}}" bus="{{.Bus}}" />
    </disk>
    {{end}}
  </devices>
</domain>
`
	tmpl = template.Must(template.New("xml").Parse(xml))
}

// DomainXML populates a libvirt domain xml template with guest properties
func (lv *Libvirt) DomainXML(guest *client.Guest) (string, error) {

	buf := new(bytes.Buffer)
	err := tmpl.Execute(buf, guest)
	if err != nil {
		return "", err
	}

	return buf.String(), nil
}
<commit_msg>xml: .Source already has the full path to the guest zvol<commit_after>package libvirt

import (
	"bytes"
	"text/template"

	"github.com/mistifyio/mistify-agent/client"
)

var tmpl *template.Template

func init() {
	const xml = `
<domain type="{{.Type}}">
  <name>{{.Id}}</name>
  <memory unit="MiB">{{.Memory}}</memory>
  <vcpu>{{.Cpu}}</vcpu>

  {{if .Metadata}}
  <metadata>
    {{range .Metadata}}
    {{end}}
  </metadata>
  {{end}}

  <os>
    <type>hvm</type>
  </os>
  <devices>
    {{range .Nics}}
    <interface type="bridge">
      <guest dev="{{.Name}}" />
      {{if .Mac}}<mac address="{{.Mac}}" />{{end}}
      <source bridge='{{.Network}}'/>
      <target dev="{{.Device}}" />
      {{if .Model}}<model type="{{.Model}}" />{{end}}
    </interface>
    {{end}}

    {{range .Disks}}
    <disk type="block" device="disk">
      <driver name="qemu" type="raw" />
      <source dev="{{.Source}}" />
      <target dev="{{.Device}}" bus="{{.Bus}}" />
    </disk>
    {{end}}
  </devices>
</domain>
`
	tmpl = template.Must(template.New("xml").Parse(xml))
}

// DomainXML populates a libvirt domain xml template with guest properties
func (lv *Libvirt) DomainXML(guest *client.Guest) (string, error) {

	buf := new(bytes.Buffer)
	err := tmpl.Execute(buf, guest)
	if err != nil {
		return "", err
	}

	return buf.String(), nil
}
<|endoftext|>


<commit_before>// Yandex YML generator: https://yandex.ru/support/partnermarket/yml/about-yml.xml
package goyml

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

const Header = `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE yml_catalog SYSTEM "shops.dtd">
`

func NewYML(name, company, url string) Catalog {
	yml := Catalog{}
	yml.SetDate(time.Now())
	yml.Shop.Name = name
	yml.Shop.Company = company
	yml.Shop.Url = url
	return yml
}

type Catalog struct {
	XMLName struct{} `xml:"yml_catalog"`
	Shop    Shop     `xml:"shop"`
	Date    string   `xml:"date,attr"`
}

func (c *Catalog) SetDate(t time.Time) {
	c.Date = t.Format("2006-01-02 15:04")
}

func (c *Catalog) AddCurrency(id string, rate, plus float64) {
	c.Shop.Currencies.add(id, rate, plus)
}

func (c *Catalog) AddCategory(id, parentId int, name string) {
	c.Shop.Categories.add(id, parentId, name)
}

func (c *Catalog) AddDeliveryOption(cost int, daysFrom, daysTo int, orderBefore int) {
	if c.Shop.DeliveryOptions == nil {
		c.Shop.DeliveryOptions = &DeliveryOptions{}
	}
	c.Shop.DeliveryOptions.add(cost, daysFrom, daysTo, orderBefore)
}

func (c *Catalog) AddOffer(offer Offer) {
	c.Shop.Offers.add(offer)
}

type Shop struct {
	Name            string           `xml:"name"`
	Company         string           `xml:"company"`
	Url             string           `xml:"url"`
	Platform        string           `xml:"platform,omitempty"`
	Version         string           `xml:"version,omitempty"`
	Agency          string           `xml:"agency,omitempty"`
	Email           string           `xml:"email,omitempty"`
	Currencies      Currencies       `xml:"currencies"`
	Categories      Categories       `xml:"categories"`
	DeliveryOptions *DeliveryOptions `xml:"delivery-options"`
	Cpa             Offers           `xml:"cpa,omitempty"`
	Offers          Offers           `xml:"offers"`
}

type Currencies struct {
	Currency []Currency `xml:"currency"`
}

func (cur *Currencies) add(id string, rate, plus float64) {
	cur.Currency = append(cur.Currency, Currency{Id: id, Rate: rate, Plus: plus})
}

type Currency struct {
	Id   string  `xml:"id,attr"`
	Rate float64 `xml:"rate,attr"`
	Plus float64 `xml:"plus,attr"`
}

type Categories struct {
	Category []Category `xml:"category"`
}

func (cat *Categories) add(id, parentId int, name string) {
	cat.Category = append(cat.Category, Category{Id: id,
		ParentId: parentId, Name: name})
}

type Category struct {
	Id       int    `xml:"id,attr"`
	ParentId int    `xml:"parentId,attr,omitempty"`
	Name     string `xml:",innerxml"`
}

type DeliveryOptions struct {
	Options []DeliveryOption `xml:"option"`
}

//// cost 0 - free of charge delivery
func (do *DeliveryOptions) add(cost int, daysFrom, daysTo int, orderBefore int) {
	daysStr := ""
	if daysFrom > 255 {
		daysFrom = 255
	}
	if daysTo > 255 {
		daysTo = 255
	}
	if daysTo < daysFrom {
		daysTo = daysFrom
	}
	if daysFrom == daysTo || daysTo == 0 {
		daysStr = strconv.Itoa(daysFrom)
	} else {
		daysStr = fmt.Sprintf("%d-%d", daysFrom, daysTo)
	}
	option := DeliveryOption{
		Cost:        cost,
		Days:        daysStr,
		OrderBefore: orderBefore,
	}
	do.Options = append(do.Options, option)
}

type DeliveryOption struct {
	Cost        int    `xml:"cost,attr"`
	Days        string `xml:"days,attr"`
	OrderBefore int    `xml:"order-before,attr,omitempty"`
}

type Offers struct {
	Offers []Offer `xml:"offer"`
}

func (o *Offers) add(offer Offer) {
	o.Offers = append(o.Offers, offer)
}

type OfferType string

const (
	TypeVendorModel OfferType = "vendor.model"
)

// Available = true - delivery in 2 days
// Available = false - delivery from 3 days to 2 months
type Offer struct {
	Id                   string           `xml:"id,attr"`
	Bid                  uint             `xml:"bid,attr,omitempty"`
	CBid                 uint             `xml:"cid,attr,omitempty"`
	Type                 OfferType        `xml:"type,attr,omitempty"`
	Available            bool             `xml:"available,attr"`
	Url                  string           `xml:"url,omitempty"`
	Price                float64          `xml:"price"`
	OldPrice             float64          `xml:"oldprice,omitempty"`
	CurrencyId           string           `xml:"currencyId"`
	CategoryId           int              `xml:"categoryId"`
	MarketCategory       string           `xml:"market_category,omitempty"`
	Picture              string           `xml:"picture,omitempty"`
	Store                bool             `xml:"store,omitempty"`
	Pickup               bool             `xml:"pickup,omitempty"`
	Delivery             bool             `xml:"delivery,omitempty"`
	DeliveryOptions      *DeliveryOptions `xml:"delivery-options,omitempty"`
	Name                 string           `xml:"name,omitempty"`
	TypePrefix           string           `xml:"typePrefix,omitempty"`
	Vendor               string           `xml:"vendor,omitempty"`
	VendorCode           string           `xml:"vendorCode,omitempty"`
	Model                string           `xml:"model,omitempty"`
	Description          string           `xml:"description,omitempty"`
	SalesNotes           string           `xml:"sales_notes,omitempty"`
	ManufacturerWarranty bool             `xml:"manufacturer_warranty,omitempty"`
	CountryOfOrigin      string           `xml:"country_of_origin,omitempty"`
	Downloadable         bool             `xml:"downloadable,omitempty"`
	Adult                bool             `xml:"adult,omitempty"`
	Age                  *Age             `xml:"age,omitempty"`
	Barcode              []string         `xml:"barcode,omitempty"`
	Cpa                  int              `xml:"cpa,omitempty"`
	Rec                  string           `xml:"rec,omitempty"`
	Expiry               string           `xml:"expiry,omitempty"`
	Weight               float64          `xml:"weight,omitempty"`
	Dimensions           string           `xml:"dimensions,omitempty"`
	Params               []Param          `xml:"param,omitempty"`
}

func (o *Offer) AddParam(name, unit, value string) {
	o.Params = append(o.Params, Param{Name: name, Unit: unit, Value: value})
}

func (o *Offer) AddBarcode(barcode string) {
	o.Barcode = append(o.Barcode, barcode)
}

func (o *Offer) AddDeliveryOption(cost int, daysFrom, daysTo int, orderBefore int) {
	if o.DeliveryOptions == nil {
		o.DeliveryOptions = &DeliveryOptions{}
	}
	o.DeliveryOptions.add(cost, daysFrom, daysTo, orderBefore)
}

func (o *Offer) AddAge(unit string, value string) {
	if o.Age == nil {
		o.Age = &Age{Unit: unit, Value: value}
	}
}

type Param struct {
	Name  string `xml:"name,attr"`
	Unit  string `xml:"unit,attr,omitempty"`
	Value string `xml:",innerxml"`
}

// Validate offer
func (o Offer) Validate() error {
	if utf8.RuneCountInString(o.Id) > 20 {
		return errors.New("Id more than 20 cahrs")
	}
	if o.Type == TypeVendorModel && (o.Vendor == "" || o.Model == "") {
		return errors.New("Vendor or Model is empty")
	}
	if o.Price == 0 {
		return errors.New("Price is zero")
	}
	if o.OldPrice > 0 && o.OldPrice <= o.Price {
		return errors.New("OldPrice less than Price")
	}
	if utf8.RuneCountInString(o.CurrencyId) != 3 {
		return errors.New("CurrencyId less than 3 chars")
	}
	if o.CategoryId > 999999999999999999 {
		return errors.New("CategoryId more than 18 cahrs")
	}
	if utf8.RuneCountInString(o.Picture) > 512 {
		return errors.New("Picture more than 512 cahrs")
	}

	descrTrim := strings.Replace(o.Description, ",", "", -1)
	descrTrim = strings.Replace(descrTrim, ".", "", -1)
	if utf8.RuneCountInString(descrTrim) > 175 {
		return errors.New("Description more than 175 cahrs")
	}

	if utf8.RuneCountInString(o.SalesNotes) > 50 {
		log.Println(o.SalesNotes)
		return errors.New("SalesNotes more than 50 cahrs")
	}

	if o.CountryOfOrigin != "" {
		if _, ok := Countries[o.CountryOfOrigin]; !ok {
			return errors.New("CountryOfOrigin not valid")
		}
	}
	if o.Age != nil && o.Age.Unit != "" {
		switch o.Age.Unit {
		case "year":
			if !isInSlice([]string{"0", "6", "12", "16", "18"}, o.Age.Value) {
				return errors.New("Age.Value is incorrect")
			}
		case "month":
			if !isInSlice([]string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"}, o.Age.Value) {
				return errors.New("Age.Value is incorrect")
			}
		default:
			return errors.New("Age.Unit is incorrect")
		}
	}
	return nil
}

type Age struct {
	Unit  string `xml:"unit,attr"`
	Value string `xml:",innerxml"`
}

func isInSlice(slice []string, val string) bool {
	for _, item := range slice {
		if item == val {
			return true
		}
	}
	return false
}
<commit_msg>bugfix<commit_after>// Yandex YML generator: https://yandex.ru/support/partnermarket/yml/about-yml.xml
package goyml

import (
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

const Header = `<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE yml_catalog SYSTEM "shops.dtd">
`

func NewYML(name, company, url string) Catalog {
	yml := Catalog{}
	yml.SetDate(time.Now())
	yml.Shop.Name = name
	yml.Shop.Company = company
	yml.Shop.Url = url
	return yml
}

type Catalog struct {
	XMLName struct{} `xml:"yml_catalog"`
	Shop    Shop     `xml:"shop"`
	Date    string   `xml:"date,attr"`
}

func (c *Catalog) SetDate(t time.Time) {
	c.Date = t.Format("2006-01-02 15:04")
}

func (c *Catalog) AddCurrency(id string, rate, plus float64) {
	c.Shop.Currencies.add(id, rate, plus)
}

func (c *Catalog) AddCategory(id, parentId int, name string) {
	c.Shop.Categories.add(id, parentId, name)
}

func (c *Catalog) AddDeliveryOption(cost int, daysFrom, daysTo int, orderBefore int) {
	if c.Shop.DeliveryOptions == nil {
		c.Shop.DeliveryOptions = &DeliveryOptions{}
	}
	c.Shop.DeliveryOptions.add(cost, daysFrom, daysTo, orderBefore)
}

func (c *Catalog) AddOffer(offer Offer) {
	c.Shop.Offers.add(offer)
}

type Shop struct {
	Name            string           `xml:"name"`
	Company         string           `xml:"company"`
	Url             string           `xml:"url"`
	Platform        string           `xml:"platform,omitempty"`
	Version         string           `xml:"version,omitempty"`
	Agency          string           `xml:"agency,omitempty"`
	Email           string           `xml:"email,omitempty"`
	Currencies      Currencies       `xml:"currencies"`
	Categories      Categories       `xml:"categories"`
	DeliveryOptions *DeliveryOptions `xml:"delivery-options"`
	Cpa             int              `xml:"cpa,omitempty"`
	Offers          Offers           `xml:"offers"`
}

type Currencies struct {
	Currency []Currency `xml:"currency"`
}

func (cur *Currencies) add(id string, rate, plus float64) {
	cur.Currency = append(cur.Currency, Currency{Id: id, Rate: rate, Plus: plus})
}

type Currency struct {
	Id   string  `xml:"id,attr"`
	Rate float64 `xml:"rate,attr"`
	Plus float64 `xml:"plus,attr"`
}

type Categories struct {
	Category []Category `xml:"category"`
}

func (cat *Categories) add(id, parentId int, name string) {
	cat.Category = append(cat.Category, Category{Id: id,
		ParentId: parentId, Name: name})
}

type Category struct {
	Id       int    `xml:"id,attr"`
	ParentId int    `xml:"parentId,attr,omitempty"`
	Name     string `xml:",innerxml"`
}

type DeliveryOptions struct {
	Options []DeliveryOption `xml:"option"`
}

//// cost 0 - free of charge delivery
func (do *DeliveryOptions) add(cost int, daysFrom, daysTo int, orderBefore int) {
	daysStr := ""
	if daysFrom > 255 {
		daysFrom = 255
	}
	if daysTo > 255 {
		daysTo = 255
	}
	if daysTo < daysFrom {
		daysTo = daysFrom
	}
	if daysFrom == daysTo || daysTo == 0 {
		daysStr = strconv.Itoa(daysFrom)
	} else {
		daysStr = fmt.Sprintf("%d-%d", daysFrom, daysTo)
	}
	option := DeliveryOption{
		Cost:        cost,
		Days:        daysStr,
		OrderBefore: orderBefore,
	}
	do.Options = append(do.Options, option)
}

type DeliveryOption struct {
	Cost        int    `xml:"cost,attr"`
	Days        string `xml:"days,attr"`
	OrderBefore int    `xml:"order-before,attr,omitempty"`
}

type Offers struct {
	Offers []Offer `xml:"offer"`
}

func (o *Offers) add(offer Offer) {
	o.Offers = append(o.Offers, offer)
}

type OfferType string

const (
	TypeVendorModel OfferType = "vendor.model"
)

// Available = true - delivery in 2 days
// Available = false - delivery from 3 days to 2 months
type Offer struct {
	Id                   string           `xml:"id,attr"`
	Bid                  uint             `xml:"bid,attr,omitempty"`
	CBid                 uint             `xml:"cid,attr,omitempty"`
	Type                 OfferType        `xml:"type,attr,omitempty"`
	Available            bool             `xml:"available,attr"`
	Url                  string           `xml:"url,omitempty"`
	Price                float64          `xml:"price"`
	OldPrice             float64          `xml:"oldprice,omitempty"`
	CurrencyId           string           `xml:"currencyId"`
	CategoryId           int              `xml:"categoryId"`
	MarketCategory       string           `xml:"market_category,omitempty"`
	Picture              string           `xml:"picture,omitempty"`
	Store                bool             `xml:"store,omitempty"`
	Pickup               bool             `xml:"pickup,omitempty"`
	Delivery             bool             `xml:"delivery,omitempty"`
	DeliveryOptions      *DeliveryOptions `xml:"delivery-options,omitempty"`
	Name                 string           `xml:"name,omitempty"`
	TypePrefix           string           `xml:"typePrefix,omitempty"`
	Vendor               string           `xml:"vendor,omitempty"`
	VendorCode           string           `xml:"vendorCode,omitempty"`
	Model                string           `xml:"model,omitempty"`
	Description          string           `xml:"description,omitempty"`
	SalesNotes           string           `xml:"sales_notes,omitempty"`
	ManufacturerWarranty bool             `xml:"manufacturer_warranty,omitempty"`
	CountryOfOrigin      string           `xml:"country_of_origin,omitempty"`
	Downloadable         bool             `xml:"downloadable,omitempty"`
	Adult                bool             `xml:"adult,omitempty"`
	Age                  *Age             `xml:"age,omitempty"`
	Barcode              []string         `xml:"barcode,omitempty"`
	Cpa                  int              `xml:"cpa,omitempty"`
	Rec                  string           `xml:"rec,omitempty"`
	Expiry               string           `xml:"expiry,omitempty"`
	Weight               float64          `xml:"weight,omitempty"`
	Dimensions           string           `xml:"dimensions,omitempty"`
	Params               []Param          `xml:"param,omitempty"`
}

func (o *Offer) AddParam(name, unit, value string) {
	o.Params = append(o.Params, Param{Name: name, Unit: unit, Value: value})
}

func (o *Offer) AddBarcode(barcode string) {
	o.Barcode = append(o.Barcode, barcode)
}

func (o *Offer) AddDeliveryOption(cost int, daysFrom, daysTo int, orderBefore int) {
	if o.DeliveryOptions == nil {
		o.DeliveryOptions = &DeliveryOptions{}
	}
	o.DeliveryOptions.add(cost, daysFrom, daysTo, orderBefore)
}

func (o *Offer) AddAge(unit string, value string) {
	if o.Age == nil {
		o.Age = &Age{Unit: unit, Value: value}
	}
}

type Param struct {
	Name  string `xml:"name,attr"`
	Unit  string `xml:"unit,attr,omitempty"`
	Value string `xml:",innerxml"`
}

// Validate offer
func (o Offer) Validate() error {
	if utf8.RuneCountInString(o.Id) > 20 {
		return errors.New("Id more than 20 cahrs")
	}
	if o.Type == TypeVendorModel && (o.Vendor == "" || o.Model == "") {
		return errors.New("Vendor or Model is empty")
	}
	if o.Price == 0 {
		return errors.New("Price is zero")
	}
	if o.OldPrice > 0 && o.OldPrice <= o.Price {
		return errors.New("OldPrice less than Price")
	}
	if utf8.RuneCountInString(o.CurrencyId) != 3 {
		return errors.New("CurrencyId less than 3 chars")
	}
	if o.CategoryId > 999999999999999999 {
		return errors.New("CategoryId more than 18 cahrs")
	}
	if utf8.RuneCountInString(o.Picture) > 512 {
		return errors.New("Picture more than 512 cahrs")
	}

	descrTrim := strings.Replace(o.Description, ",", "", -1)
	descrTrim = strings.Replace(descrTrim, ".", "", -1)
	if utf8.RuneCountInString(descrTrim) > 175 {
		return errors.New("Description more than 175 cahrs")
	}

	if utf8.RuneCountInString(o.SalesNotes) > 50 {
		log.Println(o.SalesNotes)
		return errors.New("SalesNotes more than 50 cahrs")
	}

	if o.CountryOfOrigin != "" {
		if _, ok := Countries[o.CountryOfOrigin]; !ok {
			return errors.New("CountryOfOrigin not valid")
		}
	}
	if o.Age != nil && o.Age.Unit != "" {
		switch o.Age.Unit {
		case "year":
			if !isInSlice([]string{"0", "6", "12", "16", "18"}, o.Age.Value) {
				return errors.New("Age.Value is incorrect")
			}
		case "month":
			if !isInSlice([]string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12"}, o.Age.Value) {
				return errors.New("Age.Value is incorrect")
			}
		default:
			return errors.New("Age.Unit is incorrect")
		}
	}
	return nil
}

type Age struct {
	Unit  string `xml:"unit,attr"`
	Value string `xml:",innerxml"`
}

func isInSlice(slice []string, val string) bool {
	for _, item := range slice {
		if item == val {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package lzma

import "io"

// buffer implements a ring buffer with features to support an LZMA reader and
// writer dictionary.
//
// The ring buffer supports writing at the end offset and writing at the cursor
// offset. The field start stores the start offset of the buffer, but could be
// computed by b.end - b.Cap().
//
// The reader dictionary will have the head at the end offset the writer
// dictionary will have the head at the cursor offset.
type buffer struct {
	// plain constant length field to store the data
	data []byte
	// start offset of the buffer; b.end - len(b.data)
	start int64
	// offset for reading
	cursor int64
	// end offset of the current buffer window; use setEnd method for
	// updates
	end int64
	// writeLimit gives the maximum distance between end and cursor
	writeLimit int
	// marks the buffer as closed for writing
	closed bool
}

// errors generated by the ring buffer functions
var (
	errOffset       = newError("offset outside buffer range")
	errAgain        = newError("buffer exceeded; repeat")
	errNegLen       = newError("length is negative")
	errNrOverflow   = newError("number overflow")
	errCapacity     = newError("capacity must be larger than zero")
	errClosedBuffer = newError("buffer is closed for writing")
)

// Cap returns the capacity of the ring buffer.
func (b *buffer) Cap() int {
	return len(b.data)
}

// Len returns the data amount currently stored in the buffer. It will grow
// from zero to the capacity of the buffer.
func (b *buffer) Len() int {
	return int(b.end - b.start)
}

// Readable returns the number of bytes that are currently available for
// reading.
func (b *buffer) Readable() int {
	return int(b.end - b.cursor)
}

// Writable returns the number of byte that are curently available for writing.
func (b *buffer) Writable() int {
	return int(b.cursor + int64(b.writeLimit) - b.end)
}

// setEnd updates the start and end fields. Use it always to update the end
// field.
func (b *buffer) setEnd(x int64) {
	if x < 0 {
		panic("b.end overflow?")
	}
	b.end = x
	b.start = x - int64(len(b.data))
	if b.start < 0 {
		b.start = 0
	}
}

// index computes the index into the data slice for a particular offset.
func (b *buffer) index(off int64) int {
	if off < 0 {
		panic("negative offsets are not supported")
	}
	return int(off % int64(len(b.data)))
}

// initBuffer initializes an already allocated buffer.
func initBuffer(b *buffer, capacity int) error {
	if capacity <= 0 {
		return errCapacity
	}
	*b = buffer{data: make([]byte, capacity), writeLimit: capacity}
	return nil
}

// newBuffer creates a new buffer instance.
func newBuffer(capacity int) (b *buffer, err error) {
	b = new(buffer)
	err = initBuffer(b, capacity)
	return
}

// readOff is a helper method for the ReadAt and Read methods.
func (b *buffer) readOff(p []byte, off int64) {
	for len(p) > 0 {
		s := b.index(off)
		m := copy(p, b.data[s:])
		off += int64(m)
		p = p[m:]
	}
}

// ReadAt reads data into p at the specified offset. The offset must be inside
// the buffer. It the slice cannot be completely filed errAgain is reported
// unless the buffer is not closed. In that case the function returns io.EOF.
func (b *buffer) ReadAt(p []byte, off int64) (n int, err error) {
	if off < b.start {
		return 0, errOffset
	}
	k := b.end - off
	n = len(p)
	if k < int64(n) {
		if k < 0 {
			return 0, errOffset
		}
		if b.closed {
			err = io.EOF
		} else {
			err = errAgain
		}
		n = int(k)
	}
	b.readOff(p[:n], off)
	return
}

// Read tries to read data into the buffer. No error is returned if n < len(p).
func (b *buffer) Read(p []byte) (n int, err error) {
	k := b.end - b.cursor
	n = len(p)
	if k < int64(n) {
		if k < 0 {
			panic("wrong b.cursor")
		}
		if k == 0 && b.closed {
			return 0, io.EOF
		}
		n = int(k)
	}
	b.readOff(p[:n], b.cursor)
	b.cursor += int64(n)
	return
}

// Discard skips n bytes of reading bytes. If less than n bytes are skipped
// errAgain or io.EOF if closed is returned.
func (b *buffer) Discard(n int) (discarded int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	k := b.end - b.cursor
	if k < int64(n) {
		if k < 0 {
			panic("wrong b.cursor")
		}
		if b.closed {
			err = io.EOF
		} else {
			err = errAgain
		}
		n = int(k)
	}
	discarded = n
	b.cursor += int64(n)
	return
}

// copyNOff is a helper method for CopyAt and Copy.
func (b *buffer) copyNOff(w io.Writer, n int, off int64) (copied int, err error) {
	start, end := off, off+int64(n)
	e := b.index(end)
	for off < end {
		s := b.index(off)
		var q []byte
		if s < e {
			q = b.data[s:e]
		} else {
			q = b.data[s:]
		}
		var m int
		m, err = w.Write(q)
		off += int64(m)
		if err != nil {
			break
		}
	}
	return int(off - start), err
}

// CopyAt behaves like ReadAt but writes the data into the writer.
func (b *buffer) CopyAt(w io.Writer, n int, off int64) (copied int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	if off < b.start {
		return 0, errOffset
	}
	k := b.end - off
	if k < int64(n) {
		if k < 0 {
			return 0, errOffset
		}
		if b.closed {
			err = io.EOF
		} else {
			err = errAgain
		}
		n = int(k)
	}
	var cerr error
	copied, cerr = b.copyNOff(w, n, off)
	if cerr != nil {
		err = cerr
	}
	return
}

// Copy behaves like read but the data is written into the writer.
func (b *buffer) Copy(w io.Writer, n int) (copied int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	k := b.end - b.cursor
	if k < int64(n) {
		if k < 0 {
			panic("wrong b.cursor")
		}
		if k == 0 && b.closed {
			return 0, io.EOF
		}
		n = int(k)
	}
	copied, err = b.copyNOff(w, n, b.cursor)
	b.cursor += int64(copied)
	return
}

// ReadByteAt reads a single byte at the given offset.
func (b *buffer) ReadByteAt(off int64) (c byte, err error) {
	if !(b.start <= off && off <= b.end) {
		return 0, errOffset
	}
	if off == b.end {
		if b.closed {
			return 0, io.EOF
		}
		return 0, errAgain
	}
	i := b.index(off)
	return b.data[i], nil
}

// ReadByte reads a single byte.
func (b *buffer) ReadByte() (c byte, err error) {
	if b.cursor == b.end {
		if b.closed {
			return 0, io.EOF
		}
		return 0, errAgain
	}
	i := b.index(b.cursor)
	b.cursor++
	return b.data[i], nil
}

// writeSlice writes the slice p unconditionally into the buffer.
func (b *buffer) writeSlice(p []byte) {
	off := b.end
	for len(p) > 0 {
		i := b.index(off)
		m := copy(b.data[i:], p)
		off += int64(m)
		if off < 0 {
			panic("overflow b.end")
		}
		p = p[m:]
	}
	b.setEnd(off)
}

// Writes data into the buffer. If n < len(p) errAgain is returned. If the
// buffer is closed errClosedBuffer is returned.
func (b *buffer) Write(p []byte) (n int, err error) {
	if b.closed {
		return 0, errClosedBuffer
	}
	n = b.Writable()
	if n < len(p) {
		err = errAgain
		p = p[:n]
	}
	n = len(p)
	b.writeSlice(p)
	return
}

// Writes a single byte into the buffer.
func (b *buffer) WriteByte(c byte) error {
	if b.closed {
		return errClosedBuffer
	}
	if b.Writable() < 1 {
		return errAgain
	}
	i := b.index(b.end)
	b.data[i] = c
	b.setEnd(b.end + 1)
	return nil
}

// WriteRepOff writes a match at the given offset into the buffer.
func (b *buffer) WriteRepOff(n int, off int64) (written int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	if b.closed {
		return 0, errClosedBuffer
	}
	if !(b.start <= off && off < b.end) {
		return 0, errOffset
	}
	if b.Writable() < n {
		return 0, errAgain
	}
	end := off + int64(n)
	e := b.index(end)
	for off < end {
		s := b.index(off)
		var t int
		if end > b.end {
			t = b.index(b.end)
		} else {
			t = e
		}
		var q []byte
		if s < t {
			q = b.data[s:t]
		} else {
			q = b.data[s:]
		}
		b.writeSlice(q)
		off += int64(len(q))
	}
	return n, nil
}

// Close closes the buffer.
func (b *buffer) Close() error {
	b.closed = true
	return nil
}

// EqualBytes count the equal bytes at off1 and off2 until max is reached.
func (b *buffer) EqualBytes(off1, off2 int64, max int) int {
	if off1 < b.start || off2 < b.start {
		return 0
	}
	if max < 0 {
		return 0
	}
	n := b.end - off1
	if n < int64(max) {
		if n < 1 {
			return 0
		}
		max = int(n)
	}
	n = b.end - off2
	if n < int64(max) {
		if n < 1 {
			return 0
		}
		max = int(n)
	}
	for k := 0; k < max; k++ {
		i, j := b.index(off1+int64(k)), b.index(off2+int64(k))
		if b.data[i] != b.data[j] {
			return k
		}
	}
	return max
}
<commit_msg>lzma: reviewed buffer.go<commit_after>package lzma

import "io"

// buffer implements a ring buffer with features to support an LZMA reader and
// writer dictionary.
//
// The ring buffer supports writing at the end offset and writing at the cursor
// offset. The field start stores the start offset of the buffer, but could be
// computed by b.end - b.Cap().
//
// The reader dictionary will have the head at the end offset the writer
// dictionary will have the head at the cursor offset.
type buffer struct {
	// plain constant length field to store the data
	data []byte
	// start offset of the buffer; b.end - len(b.data)
	start int64
	// offset for reading
	cursor int64
	// end offset of the current buffer window; use setEnd method for
	// updates
	end int64
	// writeLimit gives the maximum distance between end and cursor
	writeLimit int
	// marks the buffer as closed for writing
	closed bool
}

// errors generated by the ring buffer functions
var (
	errOffset       = newError("offset outside buffer range")
	errAgain        = newError("buffer exceeded; repeat")
	errNegLen       = newError("length is negative")
	errNrOverflow   = newError("number overflow")
	errCapacity     = newError("capacity must be larger than zero")
	errClosedBuffer = newError("buffer is closed for writing")
)

// Cap returns the capacity of the ring buffer.
func (b *buffer) Cap() int {
	return len(b.data)
}

// Len returns the data amount currently stored in the buffer. It will grow
// from zero to the capacity of the buffer.
func (b *buffer) Len() int {
	return int(b.end - b.start)
}

// Readable returns the number of bytes that are currently available for
// reading.
func (b *buffer) Readable() int {
	return int(b.end - b.cursor)
}

// Writable returns the number of byte that are curently available for writing.
func (b *buffer) Writable() int {
	return int(b.cursor + int64(b.writeLimit) - b.end)
}

// setEnd updates the start and end fields. Use it always to update the end
// field.
func (b *buffer) setEnd(x int64) {
	if x < 0 {
		panic("b.end overflow?")
	}
	b.end = x
	b.start = x - int64(len(b.data))
	if b.start < 0 {
		b.start = 0
	}
}

// index computes the index into the data slice for a particular offset.
func (b *buffer) index(off int64) int {
	if off < 0 {
		panic("negative offsets are not supported")
	}
	return int(off % int64(len(b.data)))
}

// initBuffer initializes an already allocated buffer.
func initBuffer(b *buffer, capacity int) error {
	if capacity <= 0 {
		return errCapacity
	}
	*b = buffer{data: make([]byte, capacity), writeLimit: capacity}
	return nil
}

// newBuffer creates a new buffer instance.
func newBuffer(capacity int) (b *buffer, err error) {
	b = new(buffer)
	err = initBuffer(b, capacity)
	return
}

// readOff is a helper method for the ReadAt and Read methods.
func (b *buffer) readOff(p []byte, off int64) {
	for len(p) > 0 {
		s := b.index(off)
		m := copy(p, b.data[s:])
		off += int64(m)
		p = p[m:]
	}
}

// ReadAt reads data into p at the specified offset. The offset must be inside
// the buffer. It the slice cannot be completely filed errAgain is reported
// unless the buffer is not closed. In that case the function returns io.EOF.
func (b *buffer) ReadAt(p []byte, off int64) (n int, err error) {
	if off < b.start {
		return 0, errOffset
	}
	k := b.end - off
	n = len(p)
	if k < int64(n) {
		if k < 0 {
			return 0, errOffset
		}
		if b.closed {
			err = io.EOF
		} else {
			err = errAgain
		}
		n = int(k)
	}
	b.readOff(p[:n], off)
	return
}

// Read tries to read data into the buffer. No error is returned if n < len(p).
func (b *buffer) Read(p []byte) (n int, err error) {
	k := b.end - b.cursor
	n = len(p)
	if k < int64(n) {
		if k < 0 {
			panic("wrong b.cursor")
		}
		if k == 0 && b.closed {
			return 0, io.EOF
		}
		n = int(k)
	}
	b.readOff(p[:n], b.cursor)
	b.cursor += int64(n)
	return
}

// Discard skips n bytes of reading bytes. If less than n bytes are skipped
// errAgain or io.EOF if closed is returned.
func (b *buffer) Discard(n int) (discarded int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	k := b.end - b.cursor
	if k < int64(n) {
		if k < 0 {
			panic("wrong b.cursor")
		}
		if b.closed {
			err = io.EOF
		} else {
			err = errAgain
		}
		n = int(k)
	}
	discarded = n
	b.cursor += int64(n)
	return
}

// copyAt is a helper for Copy.
func (b *buffer) copyAt(w io.Writer, n int, off int64) (copied int, err error) {
	start, end := off, off+int64(n)
	e := b.index(end)
	for off < end {
		s := b.index(off)
		var q []byte
		if s < e {
			q = b.data[s:e]
		} else {
			q = b.data[s:]
		}
		var m int
		m, err = w.Write(q)
		off += int64(m)
		if err != nil {
			break
		}
	}
	return int(off - start), err
}

// Copy behaves like read but the data is written into the writer.
func (b *buffer) Copy(w io.Writer, n int) (copied int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	k := b.end - b.cursor
	if k < int64(n) {
		if k < 0 {
			panic("wrong b.cursor")
		}
		if k == 0 && b.closed {
			return 0, io.EOF
		}
		n = int(k)
	}
	copied, err = b.copyAt(w, n, b.cursor)
	b.cursor += int64(copied)
	return
}

// ReadByteAt reads a single byte at the given offset.
func (b *buffer) ReadByteAt(off int64) (c byte, err error) {
	if !(b.start <= off && off <= b.end) {
		return 0, errOffset
	}
	if off == b.end {
		if b.closed {
			return 0, io.EOF
		}
		return 0, errAgain
	}
	i := b.index(off)
	return b.data[i], nil
}

// ReadByte reads a single byte.
func (b *buffer) ReadByte() (c byte, err error) {
	if b.cursor == b.end {
		if b.closed {
			return 0, io.EOF
		}
		return 0, errAgain
	}
	i := b.index(b.cursor)
	b.cursor++
	return b.data[i], nil
}

// writeSlice writes the slice p unconditionally into the buffer.
func (b *buffer) writeSlice(p []byte) {
	off := b.end
	for len(p) > 0 {
		i := b.index(off)
		m := copy(b.data[i:], p)
		off += int64(m)
		if off < 0 {
			panic("overflow b.end")
		}
		p = p[m:]
	}
	b.setEnd(off)
}

// Writes data into the buffer. If n < len(p) errAgain is returned. If the
// buffer is closed errClosedBuffer is returned.
func (b *buffer) Write(p []byte) (n int, err error) {
	if b.closed {
		return 0, errClosedBuffer
	}
	n = b.Writable()
	if n < len(p) {
		err = errAgain
		p = p[:n]
	}
	n = len(p)
	b.writeSlice(p)
	return
}

// Writes a single byte into the buffer.
func (b *buffer) WriteByte(c byte) error {
	if b.closed {
		return errClosedBuffer
	}
	if b.Writable() < 1 {
		return errAgain
	}
	i := b.index(b.end)
	b.data[i] = c
	b.setEnd(b.end + 1)
	return nil
}

// WriteRepOff writes a match at the given offset into the buffer.
func (b *buffer) WriteRepOff(n int, off int64) (written int, err error) {
	if n < 0 {
		return 0, errNegLen
	}
	if b.closed {
		return 0, errClosedBuffer
	}
	if !(b.start <= off && off < b.end) {
		return 0, errOffset
	}
	if b.Writable() < n {
		return 0, errAgain
	}
	end := off + int64(n)
	e := b.index(end)
	for off < end {
		s := b.index(off)
		var t int
		if end > b.end {
			t = b.index(b.end)
		} else {
			t = e
		}
		var q []byte
		if s < t {
			q = b.data[s:t]
		} else {
			q = b.data[s:]
		}
		b.writeSlice(q)
		off += int64(len(q))
	}
	return n, nil
}

// Close closes the buffer.
func (b *buffer) Close() error {
	b.closed = true
	return nil
}

// EqualBytes count the equal bytes at off1 and off2 until max is reached.
func (b *buffer) EqualBytes(off1, off2 int64, max int) int {
	if off1 < b.start || off2 < b.start {
		return 0
	}
	if max < 0 {
		return 0
	}
	n := b.end - off1
	if n < int64(max) {
		if n < 1 {
			return 0
		}
		max = int(n)
	}
	n = b.end - off2
	if n < int64(max) {
		if n < 1 {
			return 0
		}
		max = int(n)
	}
	for k := 0; k < max; k++ {
		i, j := b.index(off1+int64(k)), b.index(off2+int64(k))
		if b.data[i] != b.data[j] {
			return k
		}
	}
	return max
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Waiting for FDs via epoll(7).

package net

import (
	"net";
	"os";
	"syscall";
)

const (
	Read = syscall.EPOLLIN | syscall.EPOLLRDHUP;
	Write = syscall.EPOLLOUT
)

export type Pollster struct {
	epfd int64;

	// Events we're already waiting for
	events *map[int64] uint;
}

export func NewPollster() (p *Pollster, err *os.Error) {
	p = new(Pollster);
	var e int64;

	// The arg to epoll_create is a hint to the kernel
	// about the number of FDs we will care about.
	// We don't know.
	if p.epfd, e = syscall.epoll_create(16); e != 0 {
		return nil, os.ErrnoToError(e)
	}
	p.events = new(map[int64] uint);
	return p, nil
}

func (p *Pollster) AddFD(fd int64, mode int, repeat bool) *os.Error {
	var ev syscall.EpollEvent
	var already bool;
	ev.fd = int32(fd);
	ev.events, already = p.events[fd];
	if !repeat {
		ev.events |= syscall.EPOLLONESHOT
	}
	if mode == 'r' {
		ev.events |= Read
	} else {
		ev.events |= Write
	}

	var op int64;
	if already {
		op = syscall.EPOLL_CTL_MOD
	} else {
		op = syscall.EPOLL_CTL_ADD
	}
	if e := syscall.epoll_ctl(p.epfd, op, fd, &ev); e != 0 {
		return os.ErrnoToError(e)
	}
	p.events[fd] = ev.events;
	return nil
}

func (p *Pollster) StopWaiting(fd int64, bits uint) {
	events, already := p.events[fd];
	if !already {
		print("Epoll unexpected fd=", fd, "\n");
		return
	}

	// If syscall.EPOLLONESHOT is not set, the wait
	// is a repeating wait, so don't change it.
	if events & syscall.EPOLLONESHOT == 0 {
		return
	}

	// Disable the given bits.
	// If we're still waiting for other events, modify the fd
	// event in the kernel.  Otherwise, delete it.
	events &= ^bits;
	if int32(events) & ^syscall.EPOLLONESHOT != 0 {
		var ev syscall.EpollEvent;
		ev.fd = int32(fd);
		ev.events = events;
		if e := syscall.epoll_ctl(p.epfd, syscall.EPOLL_CTL_MOD, fd, &ev); e != 0 {
			print("Epoll modify fd=", fd, ": ", os.ErrnoToError(e).String(), "\n")
		}
		p.events[fd] = events
	} else {
		if e := syscall.epoll_ctl(p.epfd, syscall.EPOLL_CTL_DEL, fd, nil); e != 0 {
			print("Epoll delete fd=", fd, ": ", os.ErrnoToError(e).String(), "\n")
		}
		p.events[fd] = 0, false
	}
}

func (p *Pollster) WaitFD() (fd int64, mode int, err *os.Error) {
	// Get an event.
	var evarray [1]syscall.EpollEvent;
	ev := &evarray[0];
	n, e := syscall.epoll_wait(p.epfd, &evarray, -1);
	for e == syscall.EAGAIN || e == syscall.EINTR {
		n, e = syscall.epoll_wait(p.epfd, &evarray, -1)
	}
	if e != 0 {
		return -1, 0, os.ErrnoToError(e)
	}
	fd = int64(ev.fd);

	if ev.events & Write != 0 {
		p.StopWaiting(fd, Write);
		return fd, 'w', nil
	}
	if ev.events & Read != 0 {
		p.StopWaiting(fd, Read);
		return fd, 'r', nil
	}

	// Other events are error conditions - wake whoever is waiting.
	events, already := p.events[fd];
	if events & Write != 0 {
		p.StopWaiting(fd, Write);
		return fd, 'w', nil
	}
	p.StopWaiting(fd, Read);
	return fd, 'r', nil
}

func (p *Pollster) Close() *os.Error {
	r, e := syscall.close(p.epfd);
	return os.ErrnoToError(e)
}
<commit_msg>Unterminated declaration breaks build.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Waiting for FDs via epoll(7).

package net

import (
	"net";
	"os";
	"syscall";
)

const (
	Read = syscall.EPOLLIN | syscall.EPOLLRDHUP;
	Write = syscall.EPOLLOUT
)

export type Pollster struct {
	epfd int64;

	// Events we're already waiting for
	events *map[int64] uint;
}

export func NewPollster() (p *Pollster, err *os.Error) {
	p = new(Pollster);
	var e int64;

	// The arg to epoll_create is a hint to the kernel
	// about the number of FDs we will care about.
	// We don't know.
	if p.epfd, e = syscall.epoll_create(16); e != 0 {
		return nil, os.ErrnoToError(e)
	}
	p.events = new(map[int64] uint);
	return p, nil
}

func (p *Pollster) AddFD(fd int64, mode int, repeat bool) *os.Error {
	var ev syscall.EpollEvent;
	var already bool;
	ev.fd = int32(fd);
	ev.events, already = p.events[fd];
	if !repeat {
		ev.events |= syscall.EPOLLONESHOT
	}
	if mode == 'r' {
		ev.events |= Read
	} else {
		ev.events |= Write
	}

	var op int64;
	if already {
		op = syscall.EPOLL_CTL_MOD
	} else {
		op = syscall.EPOLL_CTL_ADD
	}
	if e := syscall.epoll_ctl(p.epfd, op, fd, &ev); e != 0 {
		return os.ErrnoToError(e)
	}
	p.events[fd] = ev.events;
	return nil
}

func (p *Pollster) StopWaiting(fd int64, bits uint) {
	events, already := p.events[fd];
	if !already {
		print("Epoll unexpected fd=", fd, "\n");
		return
	}

	// If syscall.EPOLLONESHOT is not set, the wait
	// is a repeating wait, so don't change it.
	if events & syscall.EPOLLONESHOT == 0 {
		return
	}

	// Disable the given bits.
	// If we're still waiting for other events, modify the fd
	// event in the kernel.  Otherwise, delete it.
	events &= ^bits;
	if int32(events) & ^syscall.EPOLLONESHOT != 0 {
		var ev syscall.EpollEvent;
		ev.fd = int32(fd);
		ev.events = events;
		if e := syscall.epoll_ctl(p.epfd, syscall.EPOLL_CTL_MOD, fd, &ev); e != 0 {
			print("Epoll modify fd=", fd, ": ", os.ErrnoToError(e).String(), "\n")
		}
		p.events[fd] = events
	} else {
		if e := syscall.epoll_ctl(p.epfd, syscall.EPOLL_CTL_DEL, fd, nil); e != 0 {
			print("Epoll delete fd=", fd, ": ", os.ErrnoToError(e).String(), "\n")
		}
		p.events[fd] = 0, false
	}
}

func (p *Pollster) WaitFD() (fd int64, mode int, err *os.Error) {
	// Get an event.
	var evarray [1]syscall.EpollEvent;
	ev := &evarray[0];
	n, e := syscall.epoll_wait(p.epfd, &evarray, -1);
	for e == syscall.EAGAIN || e == syscall.EINTR {
		n, e = syscall.epoll_wait(p.epfd, &evarray, -1)
	}
	if e != 0 {
		return -1, 0, os.ErrnoToError(e)
	}
	fd = int64(ev.fd);

	if ev.events & Write != 0 {
		p.StopWaiting(fd, Write);
		return fd, 'w', nil
	}
	if ev.events & Read != 0 {
		p.StopWaiting(fd, Read);
		return fd, 'r', nil
	}

	// Other events are error conditions - wake whoever is waiting.
	events, already := p.events[fd];
	if events & Write != 0 {
		p.StopWaiting(fd, Write);
		return fd, 'w', nil
	}
	p.StopWaiting(fd, Read);
	return fd, 'r', nil
}

func (p *Pollster) Close() *os.Error {
	r, e := syscall.close(p.epfd);
	return os.ErrnoToError(e)
}
<|endoftext|>


<commit_before>package models

import (
	"context"
	"fmt"
	"net"
	"os"
)

// TCP_BUFFER_SIZE is the maximum packet size
const TCP_BUFFER_SIZE = 1024 * 64

// DEFAULT_RELAY is the default relay used (can be set using --relay)
var (
	DEFAULT_RELAY      = "croc.schollz.com"
	DEFAULT_RELAY6     = "croc6.schollz.com"
	DEFAULT_PORT       = "9009"
	DEFAULT_PASSPHRASE = "pass123"
	INTERNAL_DNS       = false
)

// publicDns are servers to be queried if a local lookup fails
var publicDns = []string{
	"1.0.0.1",                // Cloudflare
	"1.1.1.1",                // Cloudflare
	"8.8.4.4",                // Google
	"8.8.8.8",                // Google
	"8.26.56.26",             // Comodo
	"208.67.220.220",         // Cisco OpenDNS
	"208.67.222.222",         // Cisco OpenDNS
	"[2001:4860:4860::8844]", // Google
	"[2001:4860:4860::8888]", // Google
}

func init() {
	for _, flag := range os.Args {
		if flag == "--internal-dns" {
			INTERNAL_DNS = true
			break
		}
	}
	var err error
	DEFAULT_RELAY, err = lookup(DEFAULT_RELAY)
	if err == nil {
		DEFAULT_RELAY += ":" + DEFAULT_PORT
	} else {
		DEFAULT_RELAY = ""
	}
	DEFAULT_RELAY6, err = lookup(DEFAULT_RELAY6)
	if err == nil {
		DEFAULT_RELAY6 = "[" + DEFAULT_RELAY6 + "]:" + DEFAULT_PORT
	} else {
		DEFAULT_RELAY6 = ""
	}
}

// Resolve a hostname to an IP address using DNS.
func lookup(address string) (ipaddress string, err error) {
	if !INTERNAL_DNS {
		return localLookupIP(address)
	}
	result := make(chan string, len(publicDns))
	for _, dns := range publicDns {
		go func(dns string) {
			s, err := remoteLookupIP(address, dns)
			if err == nil {
				result <- s
			}
		}(dns)
	}
	for i := 0; i < len(publicDns); i++ {
		ipaddress = <-result
		if ipaddress != "" {
			return
		}
	}
	err = fmt.Errorf("failed to resolve %s: all DNS servers exhausted", address)
	return
}

// localLookupIP returns a host's IP address based on the local resolver.
func localLookupIP(address string) (ipaddress string, err error) {
	ip, err := net.LookupHost(address)
	if err != nil {
		return
	}
	ipaddress = ip[0]
	return
}

// remoteLookupIP returns a host's IP address based on a remote DNS server.
func remoteLookupIP(address, dns string) (ipaddress string, err error) {
	r := &net.Resolver{
		PreferGo: true,
		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
			d := new(net.Dialer)
			return d.DialContext(ctx, network, dns+":53")
		},
	}
	ip, err := r.LookupHost(context.Background(), address)
	if err != nil {
		return
	}
	ipaddress = ip[0]
	return
}
<commit_msg>Improve built-in list of public DNS recursors<commit_after>package models

import (
	"context"
	"fmt"
	"net"
	"os"
)

// TCP_BUFFER_SIZE is the maximum packet size
const TCP_BUFFER_SIZE = 1024 * 64

// DEFAULT_RELAY is the default relay used (can be set using --relay)
var (
	DEFAULT_RELAY      = "croc.schollz.com"
	DEFAULT_RELAY6     = "croc6.schollz.com"
	DEFAULT_PORT       = "9009"
	DEFAULT_PASSPHRASE = "pass123"
	INTERNAL_DNS       = false
)

// publicDns are servers to be queried if a local lookup fails
var publicDns = []string{
	"1.0.0.1",                // Cloudflare
	"1.1.1.1",                // Cloudflare
	"[2606:4700:4700::1111]", // Cloudflare
	"[2606:4700:4700::1001]", // Cloudflare
	"8.8.4.4",                // Google
	"8.8.8.8",                // Google
	"[2001:4860:4860::8844]", // Google
	"[2001:4860:4860::8888]", // Google
	"9.9.9.9",                // Quad9
	"149.112.112.112",        // Quad9
	"[2620:fe::fe]",          // Quad9
	"[2620:fe::fe:9]",        // Quad9
	"8.26.56.26",             // Comodo
	"8.20.247.20",            // Comodo
	"208.67.220.220",         // Cisco OpenDNS
	"208.67.222.222",         // Cisco OpenDNS
	"[2620:119:35::35]",      // Cisco OpenDNS
	"[2620:119:53::53]",      // Cisco OpenDNS
}

func init() {
	for _, flag := range os.Args {
		if flag == "--internal-dns" {
			INTERNAL_DNS = true
			break
		}
	}
	var err error
	DEFAULT_RELAY, err = lookup(DEFAULT_RELAY)
	if err == nil {
		DEFAULT_RELAY += ":" + DEFAULT_PORT
	} else {
		DEFAULT_RELAY = ""
	}
	DEFAULT_RELAY6, err = lookup(DEFAULT_RELAY6)
	if err == nil {
		DEFAULT_RELAY6 = "[" + DEFAULT_RELAY6 + "]:" + DEFAULT_PORT
	} else {
		DEFAULT_RELAY6 = ""
	}
}

// Resolve a hostname to an IP address using DNS.
func lookup(address string) (ipaddress string, err error) {
	if !INTERNAL_DNS {
		return localLookupIP(address)
	}
	result := make(chan string, len(publicDns))
	for _, dns := range publicDns {
		go func(dns string) {
			s, err := remoteLookupIP(address, dns)
			if err == nil {
				result <- s
			}
		}(dns)
	}
	for i := 0; i < len(publicDns); i++ {
		ipaddress = <-result
		if ipaddress != "" {
			return
		}
	}
	err = fmt.Errorf("failed to resolve %s: all DNS servers exhausted", address)
	return
}

// localLookupIP returns a host's IP address based on the local resolver.
func localLookupIP(address string) (ipaddress string, err error) {
	ip, err := net.LookupHost(address)
	if err != nil {
		return
	}
	ipaddress = ip[0]
	return
}

// remoteLookupIP returns a host's IP address based on a remote DNS server.
func remoteLookupIP(address, dns string) (ipaddress string, err error) {
	r := &net.Resolver{
		PreferGo: true,
		Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
			d := new(net.Dialer)
			return d.DialContext(ctx, network, dns+":53")
		},
	}
	ip, err := r.LookupHost(context.Background(), address)
	if err != nil {
		return
	}
	ipaddress = ip[0]
	return
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package cmd

import (
	"crypto/md5"
	"encoding/binary"
	"fmt"
	"hash"
	"hash/crc32"
	"io"
	"log"
	"os"

	"github.com/ikravets/errs"
	"github.com/jessevdk/go-flags"

	"my/ev/anal"
	"my/ev/efhsim"
	"my/ev/rec"
)

type cmdEfhsim struct {
	InputFileName           string `long:"input" short:"i" required:"y" value-name:"PCAP_FILE" description:"input pcap file to read"`
	TobBook                 bool   `long:"tob" short:"t" description:"use 1-level-deep book (for exchange disseminating ToB only)"`
	SubscriptionFileName    string `long:"subscribe" short:"s" value-name:"SUBSCRIPTION_FILE" description:"read subscriptions from file"`
	OutputFileNameSimOrders string `long:"output-sim-orders" value-name:"FILE" description:"output file for hw simulator"`
	OutputFileNameSimQuotes string `long:"output-sim-quotes" value-name:"FILE" description:"output file for hw simulator"`
	OutputFileNameEfhOrders string `long:"output-efh-orders" value-name:"FILE" description:"output file for EFH order messages"`
	OutputFileNameEfhQuotes string `long:"output-efh-quotes" value-name:"FILE" description:"output file for EFH quote messages"`
	OutputFileNameAvt       string `long:"output-avt" value-name:"FILE" description:"output file for AVT CSV"`
	InputFileNameAvtDict    string `long:"avt-dict" value-name:"DICT" description:"read dictionary for AVT CSV output"`
	OutputDirStats          string `long:"output-stats" value-name:"DIR" description:"output dir for stats"`
	PacketNumLimit          int    `long:"count" short:"c" value-name:"NUM" description:"limit number of input packets"`
	NoHwLim                 bool   `long:"no-hw-lim" description:"do not enforce HW limits"`
	Md5sum                  bool   `long:"md5sum" description:"compute md5sum on output file(s)"`
	shouldExecute           bool
	closers                 []io.Closer
}

func (c *cmdEfhsim) Execute(args []string) error {
	c.shouldExecute = true
	return nil
}

func (c *cmdEfhsim) ConfigParser(parser *flags.Parser) {
	_, err := parser.AddCommand("efhsim", "simulate efh", "", c)
	errs.CheckE(err)
}

func (c *cmdEfhsim) ParsingFinished() (err error) {
	if !c.shouldExecute {
		return
	}
	efh := efhsim.NewEfhSim(c.TobBook)
	efh.SetInput(c.InputFileName, c.PacketNumLimit)
	if c.SubscriptionFileName != "" {
		file, err := os.Open(c.SubscriptionFileName)
		errs.CheckE(err)
		errs.CheckE(efh.SubscribeFromReader(file))
		errs.CheckE(file.Close())
	}
	if !c.NoHwLim {
		if efh.SubscriptionsNum() == 0 {
			log.Println("running in auto-subscription mode, not enforcing hw limit")
		} else if efh.SubscriptionsNum() > rec.HwMaxSubscriptions {
			log.Println("too many subscriptions, not enforcing hw limit")
		} else {
			efh.AddLogger(rec.NewHwLimChecker())
		}
	}
	supernodeLevels := rec.SimLoggerDefaultSupernodeLevels
	if c.TobBook {
		supernodeLevels = 1
	}
	c.addOut(c.OutputFileNameSimOrders, func(w io.Writer) error {
		logger := rec.NewSimLogger(w)
		logger.SetSupernodeLevels(supernodeLevels)
		logger.SetOutputMode(rec.EfhLoggerOutputOrders)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameSimQuotes, func(w io.Writer) error {
		logger := rec.NewSimLogger(w)
		logger.SetSupernodeLevels(supernodeLevels)
		logger.SetOutputMode(rec.EfhLoggerOutputQuotes)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameEfhOrders, func(w io.Writer) error {
		logger := rec.NewEfhLogger(rec.NewTestefhPrinter(w))
		logger.SetOutputMode(rec.EfhLoggerOutputOrders)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameEfhQuotes, func(w io.Writer) error {
		logger := rec.NewEfhLogger(rec.NewTestefhPrinter(w))
		logger.SetOutputMode(rec.EfhLoggerOutputQuotes)
		return efh.AddLogger(logger)
	})
	c.addOut(c.OutputFileNameAvt, func(w io.Writer) (err error) {
		defer errs.PassE(&err)
		var dict io.ReadCloser
		if c.InputFileNameAvtDict != "" {
			dict, err := os.Open(c.InputFileNameAvtDict)
			errs.CheckE(err)
			c.closers = append(c.closers, dict)
		}
		return efh.AddLogger(rec.NewAvtLogger(w, dict))
	})
	reporter := c.addAnalyzer(efh)

	// run efhsim
	errs.CheckE(efh.AnalyzeInput())

	for _, cl := range c.closers {
		errs.CheckE(cl.Close())
	}
	if reporter != nil {
		reporter.SaveAll()
	}
	return
}

func (c *cmdEfhsim) addOut(fileName string, setOut func(io.Writer) error) {
	if fileName == "" {
		return
	}
	var err error
	var o io.WriteCloser
	if c.Md5sum {
		o, err = NewHashedOut(fileName)
	} else {
		o, err = os.Create(fileName)
	}
	errs.CheckE(err)
	errs.CheckE(setOut(o))
	c.closers = append(c.closers, o)
}
func (c *cmdEfhsim) addAnalyzer(efh *efhsim.EfhSim) *anal.Reporter {
	if c.OutputDirStats == "" {
		return nil
	}
	hashFn := func(v uint64) uint64 {
		data := make([]byte, 8)
		binary.BigEndian.PutUint64(data, v)
		h := crc32.ChecksumIEEE(data)
		return uint64(h & (1<<24 - 1))
	}
	moduloFn := func(v uint64) uint64 {
		return v & (1<<24 - 1)
	}
	analyzer := anal.NewAnalyzer()
	analyzer.AddOrderHashFunction(hashFn)
	analyzer.AddOrderHashFunction(moduloFn)
	efh.AddLogger(analyzer.Observer())
	reporter := anal.NewReporter()
	reporter.SetAnalyzer(analyzer)
	reporter.SetOutputDir(c.OutputDirStats)
	return reporter
}

func init() {
	var c cmdEfhsim
	Registry.Register(&c)
}

type hashedOut struct {
	file    *os.File
	md5file *os.File
	md5sum  hash.Hash
	mw      io.Writer
}

func NewHashedOut(baseName string) (o *hashedOut, err error) {
	defer errs.PassE(&err)
	o = &hashedOut{}
	o.file, err = os.Create(baseName)
	errs.CheckE(err)
	o.md5file, err = os.Create(baseName + ".md5sum")
	errs.CheckE(err)
	o.md5sum = md5.New()
	o.mw = io.MultiWriter(o.file, o.md5sum)
	return
}
func (o *hashedOut) writer() io.Writer {
	return o.mw
}
func (o *hashedOut) Write(b []byte) (int, error) {
	return o.mw.Write(b)
}
func (o *hashedOut) Close() (err error) {
	name := "-"
	defer errs.PassE(&err)
	if o.file != nil {
		errs.CheckE(o.file.Close())
		name = o.file.Name()
	}
	if o.md5file != nil {
		_, err := fmt.Fprintf(o.md5file, "%x  %s\n", o.md5sum.Sum(nil), name)
		errs.CheckE(err)
		errs.CheckE(o.md5file.Close())
	}
	return
}
<commit_msg>cmd:efhsim: use logger config<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package cmd

import (
	"crypto/md5"
	"encoding/binary"
	"fmt"
	"hash"
	"hash/crc32"
	"io"
	"log"
	"os"

	"github.com/ikravets/errs"
	"github.com/jessevdk/go-flags"

	"my/ev/anal"
	"my/ev/efhsim"
	"my/ev/rec"
)

type cmdEfhsim struct {
	InputFileName           string `long:"input" short:"i" required:"y" value-name:"PCAP_FILE" description:"input pcap file to read"`
	TobBook                 bool   `long:"tob" short:"t" description:"use 1-level-deep book (for exchange disseminating ToB only)"`
	SubscriptionFileName    string `long:"subscribe" short:"s" value-name:"SUBSCRIPTION_FILE" description:"read subscriptions from file"`
	OutputFileNameSimOrders string `long:"output-sim-orders" value-name:"FILE" description:"output file for hw simulator"`
	OutputFileNameSimQuotes string `long:"output-sim-quotes" value-name:"FILE" description:"output file for hw simulator"`
	OutputFileNameEfhOrders string `long:"output-efh-orders" value-name:"FILE" description:"output file for EFH order messages"`
	OutputFileNameEfhQuotes string `long:"output-efh-quotes" value-name:"FILE" description:"output file for EFH quote messages"`
	OutputFileNameAvt       string `long:"output-avt" value-name:"FILE" description:"output file for AVT CSV"`
	InputFileNameAvtDict    string `long:"avt-dict" value-name:"DICT" description:"read dictionary for AVT CSV output"`
	OutputDirStats          string `long:"output-stats" value-name:"DIR" description:"output dir for stats"`
	PacketNumLimit          int    `long:"count" short:"c" value-name:"NUM" description:"limit number of input packets"`
	NoHwLim                 bool   `long:"no-hw-lim" description:"do not enforce HW limits"`
	Md5sum                  bool   `long:"md5sum" description:"compute md5sum on output file(s)"`
	shouldExecute           bool
	closers                 []io.Closer
}

func (c *cmdEfhsim) Execute(args []string) error {
	c.shouldExecute = true
	return nil
}

func (c *cmdEfhsim) ConfigParser(parser *flags.Parser) {
	_, err := parser.AddCommand("efhsim", "simulate efh", "", c)
	errs.CheckE(err)
}

func (c *cmdEfhsim) ParsingFinished() (err error) {
	if !c.shouldExecute {
		return
	}
	efh := efhsim.NewEfhSim(c.TobBook)
	efh.SetInput(c.InputFileName, c.PacketNumLimit)
	if c.SubscriptionFileName != "" {
		file, err := os.Open(c.SubscriptionFileName)
		errs.CheckE(err)
		errs.CheckE(efh.SubscribeFromReader(file))
		errs.CheckE(file.Close())
	}
	if !c.NoHwLim {
		if efh.SubscriptionsNum() == 0 {
			log.Println("running in auto-subscription mode, not enforcing hw limit")
		} else if efh.SubscriptionsNum() > rec.HwMaxSubscriptions {
			log.Println("too many subscriptions, not enforcing hw limit")
		} else {
			efh.AddLogger(rec.NewHwLimChecker())
		}
	}
	efhLoggerConfig := rec.EfhLoggerConfig{}
	simLoggerConfig := rec.SimLoggerConfig{}
	if c.TobBook {
		simLoggerConfig.SupernodeLevels = 1
	}
	c.addOut(c.OutputFileNameSimOrders, func(w io.Writer) error {
		lc := simLoggerConfig
		lc.Writer = w
		lc.Mode = rec.EfhLoggerOutputOrders
		return efh.AddLogger(rec.NewSimLogger(lc))
	})
	c.addOut(c.OutputFileNameSimQuotes, func(w io.Writer) error {
		lc := simLoggerConfig
		lc.Writer = w
		lc.Mode = rec.EfhLoggerOutputQuotes
		return efh.AddLogger(rec.NewSimLogger(lc))
	})
	c.addOut(c.OutputFileNameEfhOrders, func(w io.Writer) error {
		lc := efhLoggerConfig
		lc.Writer = w
		lc.Mode = rec.EfhLoggerOutputOrders
		return efh.AddLogger(rec.NewEfhLogger(lc))
	})
	c.addOut(c.OutputFileNameEfhQuotes, func(w io.Writer) error {
		lc := efhLoggerConfig
		lc.Writer = w
		lc.Mode = rec.EfhLoggerOutputQuotes
		return efh.AddLogger(rec.NewEfhLogger(lc))
	})
	c.addOut(c.OutputFileNameAvt, func(w io.Writer) (err error) {
		defer errs.PassE(&err)
		var dict io.ReadCloser
		if c.InputFileNameAvtDict != "" {
			dict, err := os.Open(c.InputFileNameAvtDict)
			errs.CheckE(err)
			c.closers = append(c.closers, dict)
		}
		return efh.AddLogger(rec.NewAvtLogger(w, dict))
	})
	reporter := c.addAnalyzer(efh)

	// run efhsim
	errs.CheckE(efh.AnalyzeInput())

	for _, cl := range c.closers {
		errs.CheckE(cl.Close())
	}
	if reporter != nil {
		reporter.SaveAll()
	}
	return
}

func (c *cmdEfhsim) addOut(fileName string, setOut func(io.Writer) error) {
	if fileName == "" {
		return
	}
	var err error
	var o io.WriteCloser
	if c.Md5sum {
		o, err = NewHashedOut(fileName)
	} else {
		o, err = os.Create(fileName)
	}
	errs.CheckE(err)
	errs.CheckE(setOut(o))
	c.closers = append(c.closers, o)
}
func (c *cmdEfhsim) addAnalyzer(efh *efhsim.EfhSim) *anal.Reporter {
	if c.OutputDirStats == "" {
		return nil
	}
	hashFn := func(v uint64) uint64 {
		data := make([]byte, 8)
		binary.BigEndian.PutUint64(data, v)
		h := crc32.ChecksumIEEE(data)
		return uint64(h & (1<<24 - 1))
	}
	moduloFn := func(v uint64) uint64 {
		return v & (1<<24 - 1)
	}
	analyzer := anal.NewAnalyzer()
	analyzer.AddOrderHashFunction(hashFn)
	analyzer.AddOrderHashFunction(moduloFn)
	efh.AddLogger(analyzer.Observer())
	reporter := anal.NewReporter()
	reporter.SetAnalyzer(analyzer)
	reporter.SetOutputDir(c.OutputDirStats)
	return reporter
}

func init() {
	var c cmdEfhsim
	Registry.Register(&c)
}

type hashedOut struct {
	file    *os.File
	md5file *os.File
	md5sum  hash.Hash
	mw      io.Writer
}

func NewHashedOut(baseName string) (o *hashedOut, err error) {
	defer errs.PassE(&err)
	o = &hashedOut{}
	o.file, err = os.Create(baseName)
	errs.CheckE(err)
	o.md5file, err = os.Create(baseName + ".md5sum")
	errs.CheckE(err)
	o.md5sum = md5.New()
	o.mw = io.MultiWriter(o.file, o.md5sum)
	return
}
func (o *hashedOut) writer() io.Writer {
	return o.mw
}
func (o *hashedOut) Write(b []byte) (int, error) {
	return o.mw.Write(b)
}
func (o *hashedOut) Close() (err error) {
	name := "-"
	defer errs.PassE(&err)
	if o.file != nil {
		errs.CheckE(o.file.Close())
		name = o.file.Name()
	}
	if o.md5file != nil {
		_, err := fmt.Fprintf(o.md5file, "%x  %s\n", o.md5sum.Sum(nil), name)
		errs.CheckE(err)
		errs.CheckE(o.md5file.Close())
	}
	return
}
<|endoftext|>


<commit_before>package xtimeout

import (
	"context"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/Comcast/webpa-common/xhttp"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func testTimeoutHandlerPanic(t *testing.T) {
	var (
		assert  = assert.New(t)
		require = require.New(t)

		panicDelegate = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
			panic("expected")
		})

		handler = NewConstructor(Options{Timeout: time.Minute})(panicDelegate)

		request  = httptest.NewRequest("GET", "/", nil)
		response = httptest.NewRecorder()
	)

	require.NotNil(handler)
	assert.Panics(func() {
		handler.ServeHTTP(response, request)
	})
}

func testTimeoutHandlerSuccess(t *testing.T) {
	const body = "success!"

	var (
		assert = assert.New(t)

		delegate = xhttp.Constant{
			Code: 299,
			Header: http.Header{
				"X-Custom": {"value"},
				"X-Multi":  {"1", "2"},
			},
			Body: []byte(body),
		}

		handler = NewConstructor(Options{Timeout: time.Minute})(delegate)

		request  = httptest.NewRequest("GET", "/", nil)
		response = httptest.NewRecorder()
	)

	handler.ServeHTTP(response, request)
	assert.Equal(299, response.Code)
	assert.Equal([]string{"value"}, response.HeaderMap["X-Custom"])
	assert.Equal([]string{"1", "2"}, response.HeaderMap["X-Multi"])
	assert.Equal(body, response.Body.String())
}

func testTimeoutHandlerTimeout(t *testing.T) {
	const body = "timeout!"

	var (
		assert = assert.New(t)

		delegateReady = make(chan struct{})
		delegateBlock = make(chan struct{})
		delegate      = http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
			response.WriteHeader(599)
			response.Write([]byte("the delegate's write should have been discarded"))
			close(delegateReady)
			<-delegateBlock
		})

		timedOut = xhttp.Constant{
			Code: 499,
			Header: http.Header{
				"X-Custom": {"value"},
				"X-Multi":  {"1", "2"},
			},
			Body: []byte(body),
		}

		handlerDone = make(chan struct{})
		handler     = NewConstructor(Options{Timeout: time.Hour, TimedOut: timedOut})(delegate)

		parentCtx, parentCancel = context.WithCancel(context.Background())
		request                 = httptest.NewRequest("GET", "/", nil).WithContext(parentCtx)
		response                = httptest.NewRecorder()
	)

	go func() {
		defer close(handlerDone)
		handler.ServeHTTP(response, request)
	}()

	select {
	case <-delegateReady:
		// passing
	case <-time.After(10 * time.Second):
		assert.Fail("Delegate was not called")
	}

	parentCancel() // simulate a timeout

	select {
	case <-handlerDone:
		// passing
	case <-time.After(10 * time.Second):
		assert.Fail("Handler did not complete")
	}

	assert.Equal(499, response.Code)
	assert.Equal([]string{"value"}, response.HeaderMap["X-Custom"])
	assert.Equal([]string{"1", "2"}, response.HeaderMap["X-Multi"])
	assert.Equal(body, response.Body.String())
}

func TestTimeoutHandler(t *testing.T) {
	t.Run("Panic", testTimeoutHandlerPanic)
	t.Run("Success", testTimeoutHandlerSuccess)
	t.Run("Timeout", testTimeoutHandlerTimeout)
}

func testNewConstructorNoTimeout(t *testing.T) {
	var (
		assert  = assert.New(t)
		require = require.New(t)

		delegate    = xhttp.Constant{Code: 388}
		constructor = NewConstructor(Options{})
	)

	require.NotNil(constructor)
	assert.Equal(delegate, constructor(delegate))
}

func testNewConstructorDefaultTimedOut(t *testing.T) {
	var (
		assert  = assert.New(t)
		require = require.New(t)

		delegate = xhttp.Constant{Code: 123}
		handler  = NewConstructor(Options{Timeout: time.Hour})(delegate)
	)

	th, ok := handler.(*timeoutHandler)
	require.True(ok)

	assert.Equal(time.Hour, th.timeout)
	assert.Equal(defaultTimedOut, th.timedOut)
	assert.Equal(delegate, th.next)
}

func TestNewConstructor(t *testing.T) {
	t.Run("NoTimeout", testNewConstructorNoTimeout)
	t.Run("DefaultTimedOut", testNewConstructorDefaultTimedOut)
}
<commit_msg>Fixed test to verify the panic value<commit_after>package xtimeout

import (
	"context"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	"github.com/Comcast/webpa-common/xhttp"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func testTimeoutHandlerPanic(t *testing.T) {
	var (
		assert  = assert.New(t)
		require = require.New(t)

		panicDelegate = http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
			panic("expected")
		})

		handler = NewConstructor(Options{Timeout: time.Minute})(panicDelegate)

		request  = httptest.NewRequest("GET", "/", nil)
		response = httptest.NewRecorder()
	)

	require.NotNil(handler)

	defer func() {
		assert.Equal("expected", recover())
	}()

	handler.ServeHTTP(response, request)
	assert.Fail("ServeHTTP should have panicked")
}

func testTimeoutHandlerSuccess(t *testing.T) {
	const body = "success!"

	var (
		assert = assert.New(t)

		delegate = xhttp.Constant{
			Code: 299,
			Header: http.Header{
				"X-Custom": {"value"},
				"X-Multi":  {"1", "2"},
			},
			Body: []byte(body),
		}

		handler = NewConstructor(Options{Timeout: time.Minute})(delegate)

		request  = httptest.NewRequest("GET", "/", nil)
		response = httptest.NewRecorder()
	)

	handler.ServeHTTP(response, request)
	assert.Equal(299, response.Code)
	assert.Equal([]string{"value"}, response.HeaderMap["X-Custom"])
	assert.Equal([]string{"1", "2"}, response.HeaderMap["X-Multi"])
	assert.Equal(body, response.Body.String())
}

func testTimeoutHandlerTimeout(t *testing.T) {
	const body = "timeout!"

	var (
		assert = assert.New(t)

		delegateReady = make(chan struct{})
		delegateBlock = make(chan struct{})
		delegate      = http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
			response.WriteHeader(599)
			response.Write([]byte("the delegate's write should have been discarded"))
			close(delegateReady)
			<-delegateBlock
		})

		timedOut = xhttp.Constant{
			Code: 499,
			Header: http.Header{
				"X-Custom": {"value"},
				"X-Multi":  {"1", "2"},
			},
			Body: []byte(body),
		}

		handlerDone = make(chan struct{})
		handler     = NewConstructor(Options{Timeout: time.Hour, TimedOut: timedOut})(delegate)

		parentCtx, parentCancel = context.WithCancel(context.Background())
		request                 = httptest.NewRequest("GET", "/", nil).WithContext(parentCtx)
		response                = httptest.NewRecorder()
	)

	go func() {
		defer close(handlerDone)
		handler.ServeHTTP(response, request)
	}()

	select {
	case <-delegateReady:
		// passing
	case <-time.After(10 * time.Second):
		assert.Fail("Delegate was not called")
	}

	parentCancel() // simulate a timeout

	select {
	case <-handlerDone:
		// passing
	case <-time.After(10 * time.Second):
		assert.Fail("Handler did not complete")
	}

	assert.Equal(499, response.Code)
	assert.Equal([]string{"value"}, response.HeaderMap["X-Custom"])
	assert.Equal([]string{"1", "2"}, response.HeaderMap["X-Multi"])
	assert.Equal(body, response.Body.String())
}

func TestTimeoutHandler(t *testing.T) {
	t.Run("Panic", testTimeoutHandlerPanic)
	t.Run("Success", testTimeoutHandlerSuccess)
	t.Run("Timeout", testTimeoutHandlerTimeout)
}

func testNewConstructorNoTimeout(t *testing.T) {
	var (
		assert  = assert.New(t)
		require = require.New(t)

		delegate    = xhttp.Constant{Code: 388}
		constructor = NewConstructor(Options{})
	)

	require.NotNil(constructor)
	assert.Equal(delegate, constructor(delegate))
}

func testNewConstructorDefaultTimedOut(t *testing.T) {
	var (
		assert  = assert.New(t)
		require = require.New(t)

		delegate = xhttp.Constant{Code: 123}
		handler  = NewConstructor(Options{Timeout: time.Hour})(delegate)
	)

	th, ok := handler.(*timeoutHandler)
	require.True(ok)

	assert.Equal(time.Hour, th.timeout)
	assert.Equal(defaultTimedOut, th.timedOut)
	assert.Equal(delegate, th.next)
}

func TestNewConstructor(t *testing.T) {
	t.Run("NoTimeout", testNewConstructorNoTimeout)
	t.Run("DefaultTimedOut", testNewConstructorDefaultTimedOut)
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2018 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package traversal

import (
	"encoding/json"

	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/filters"
	"github.com/skydive-project/skydive/topology/graph"
	"github.com/skydive-project/skydive/topology/graph/traversal"
	"github.com/skydive-project/skydive/topology/probes/socketinfo"
)

// SocketsTraversalExtension describes a new extension to enhance the topology
type SocketsTraversalExtension struct {
	SocketsToken traversal.Token
}

// SocketsGremlinTraversalStep describes the Sockets gremlin traversal step
type SocketsGremlinTraversalStep struct {
	context traversal.GremlinTraversalContext
}

// NewMetricsTraversalExtension returns a new graph traversal extension
func NewSocketsTraversalExtension() *SocketsTraversalExtension {
	return &SocketsTraversalExtension{
		SocketsToken: traversalSocketsToken,
	}
}

// ScanIdent returns an associated graph token
func (e *SocketsTraversalExtension) ScanIdent(s string) (traversal.Token, bool) {
	switch s {
	case "SOCKETS":
		return e.SocketsToken, true
	}
	return traversal.IDENT, false
}

// ParseStep parse connections step
func (e *SocketsTraversalExtension) ParseStep(t traversal.Token, p traversal.GremlinTraversalContext) (traversal.GremlinTraversalStep, error) {
	switch t {
	case e.SocketsToken:
		return &SocketsGremlinTraversalStep{context: p}, nil
	}
	return nil, nil
}

// Exec executes the metrics step
func (s *SocketsGremlinTraversalStep) Exec(last traversal.GraphTraversalStep) (traversal.GraphTraversalStep, error) {
	switch tv := last.(type) {
	case *traversal.GraphTraversalV:
		return Sockets(tv), nil
	case *FlowTraversalStep:
		return tv.Sockets(), nil
	}
	return nil, traversal.ErrExecutionError
}

// Reduce flow step
func (s *SocketsGremlinTraversalStep) Reduce(next traversal.GremlinTraversalStep) traversal.GremlinTraversalStep {
	return next
}

// Context sockets step
func (s *SocketsGremlinTraversalStep) Context() *traversal.GremlinTraversalContext {
	return &s.context
}

// SocketsTraversalStep connections step
type SocketsTraversalStep struct {
	GraphTraversal *traversal.GraphTraversal
	sockets        map[string][]*socketinfo.ConnectionInfo
	error          error
}

// PropertyValues returns a flow field value
func (s *SocketsTraversalStep) PropertyValues(keys ...interface{}) *traversal.GraphTraversalValue {
	if s.error != nil {
		return traversal.NewGraphTraversalValue(s.GraphTraversal, nil, s.error)
	}

	key := keys[0].(string)
	var values []interface{}
	for _, sockets := range s.sockets {
		for _, socket := range sockets {
			v, err := socket.GetField(key)
			if err != nil {
				return traversal.NewGraphTraversalValue(s.GraphTraversal, nil, common.ErrFieldNotFound)
			}
			values = append(values, v)
		}
	}

	return traversal.NewGraphTraversalValue(s.GraphTraversal, values, nil)
}

// Values returns list of socket informations
func (s *SocketsTraversalStep) Values() []interface{} {
	if len(s.sockets) == 0 {
		return []interface{}{}
	}
	return []interface{}{s.sockets}
}

// MarshalJSON serialize in JSON
func (s *SocketsTraversalStep) MarshalJSON() ([]byte, error) {
	values := s.Values()
	s.GraphTraversal.RLock()
	defer s.GraphTraversal.RUnlock()
	return json.Marshal(values)
}

// Error returns traversal error
func (s *SocketsTraversalStep) Error() error {
	return s.error
}

func getSockets(n *graph.Node) (sockets []*socketinfo.ConnectionInfo) {
	if socks, err := n.GetField("Sockets"); err == nil {
		for _, socket := range socks.([]interface{}) {
			var conn socketinfo.ConnectionInfo
			if err = conn.Decode(socket); err == nil {
				sockets = append(sockets, &conn)
			}
		}
	}
	return
}

// NewSocketIndexer returns a new socket graph indexer
func NewSocketIndexer(g *graph.Graph) *graph.GraphIndexer {
	hashNode := func(n *graph.Node) map[string]interface{} {
		sockets := getSockets(n)
		kv := make(map[string]interface{}, len(sockets))
		for _, socket := range sockets {
			kv[socket.Hash()] = socket
		}
		return kv
	}

	graphIndexer := graph.NewGraphIndexer(g, hashNode, true)
	socketFilter := graph.NewGraphElementFilter(filters.NewNotNullFilter("Sockets"))
	for _, node := range g.GetNodes(socketFilter) {
		graphIndexer.OnNodeAdded(node)
	}
	return graphIndexer
}
<commit_msg>gremlin: support Has on Sockets step<commit_after>/*
 * Copyright (C) 2018 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package traversal

import (
	"encoding/json"

	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/filters"
	"github.com/skydive-project/skydive/topology/graph"
	"github.com/skydive-project/skydive/topology/graph/traversal"
	"github.com/skydive-project/skydive/topology/probes/socketinfo"
)

// SocketsTraversalExtension describes a new extension to enhance the topology
type SocketsTraversalExtension struct {
	SocketsToken traversal.Token
}

// SocketsGremlinTraversalStep describes the Sockets gremlin traversal step
type SocketsGremlinTraversalStep struct {
	context traversal.GremlinTraversalContext
}

// NewMetricsTraversalExtension returns a new graph traversal extension
func NewSocketsTraversalExtension() *SocketsTraversalExtension {
	return &SocketsTraversalExtension{
		SocketsToken: traversalSocketsToken,
	}
}

// ScanIdent returns an associated graph token
func (e *SocketsTraversalExtension) ScanIdent(s string) (traversal.Token, bool) {
	switch s {
	case "SOCKETS":
		return e.SocketsToken, true
	}
	return traversal.IDENT, false
}

// ParseStep parse connections step
func (e *SocketsTraversalExtension) ParseStep(t traversal.Token, p traversal.GremlinTraversalContext) (traversal.GremlinTraversalStep, error) {
	switch t {
	case e.SocketsToken:
		return &SocketsGremlinTraversalStep{context: p}, nil
	}
	return nil, nil
}

// Exec executes the metrics step
func (s *SocketsGremlinTraversalStep) Exec(last traversal.GraphTraversalStep) (traversal.GraphTraversalStep, error) {
	switch tv := last.(type) {
	case *traversal.GraphTraversalV:
		return Sockets(tv), nil
	case *FlowTraversalStep:
		return tv.Sockets(), nil
	}
	return nil, traversal.ErrExecutionError
}

// Reduce flow step
func (s *SocketsGremlinTraversalStep) Reduce(next traversal.GremlinTraversalStep) traversal.GremlinTraversalStep {
	return next
}

// Context sockets step
func (s *SocketsGremlinTraversalStep) Context() *traversal.GremlinTraversalContext {
	return &s.context
}

// SocketsTraversalStep connections step
type SocketsTraversalStep struct {
	GraphTraversal *traversal.GraphTraversal
	sockets        map[string][]*socketinfo.ConnectionInfo
	error          error
}

// PropertyValues returns a flow field value
func (s *SocketsTraversalStep) PropertyValues(keys ...interface{}) *traversal.GraphTraversalValue {
	if s.error != nil {
		return traversal.NewGraphTraversalValue(s.GraphTraversal, nil, s.error)
	}

	key := keys[0].(string)
	var values []interface{}
	for _, sockets := range s.sockets {
		for _, socket := range sockets {
			v, err := socket.GetField(key)
			if err != nil {
				return traversal.NewGraphTraversalValue(s.GraphTraversal, nil, common.ErrFieldNotFound)
			}
			values = append(values, v)
		}
	}

	return traversal.NewGraphTraversalValue(s.GraphTraversal, values, nil)
}

// Has step
func (s *SocketsTraversalStep) Has(params ...interface{}) *SocketsTraversalStep {
	if s.error != nil {
		return s
	}

	filter, err := paramsToFilter(params...)
	if err != nil {
		return &SocketsTraversalStep{error: err}
	}

	s.GraphTraversal.RLock()
	defer s.GraphTraversal.RUnlock()

	flowSockets := make(map[string][]*socketinfo.ConnectionInfo)
	for id, sockets := range s.sockets {
		var socks []*socketinfo.ConnectionInfo
		for _, socket := range sockets {
			if filter == nil || filter.Eval(socket) {
				socks = append(socks, socket)
			}
		}
		if len(socks) > 0 {
			flowSockets[id] = socks
		}
	}

	return &SocketsTraversalStep{GraphTraversal: s.GraphTraversal, sockets: flowSockets}
}

// Values returns list of socket informations
func (s *SocketsTraversalStep) Values() []interface{} {
	if len(s.sockets) == 0 {
		return []interface{}{}
	}
	return []interface{}{s.sockets}
}

// MarshalJSON serialize in JSON
func (s *SocketsTraversalStep) MarshalJSON() ([]byte, error) {
	values := s.Values()
	s.GraphTraversal.RLock()
	defer s.GraphTraversal.RUnlock()
	return json.Marshal(values)
}

// Error returns traversal error
func (s *SocketsTraversalStep) Error() error {
	return s.error
}

func getSockets(n *graph.Node) (sockets []*socketinfo.ConnectionInfo) {
	if socks, err := n.GetField("Sockets"); err == nil {
		for _, socket := range socks.([]interface{}) {
			var conn socketinfo.ConnectionInfo
			if err = conn.Decode(socket); err == nil {
				sockets = append(sockets, &conn)
			}
		}
	}
	return
}

// NewSocketIndexer returns a new socket graph indexer
func NewSocketIndexer(g *graph.Graph) *graph.GraphIndexer {
	hashNode := func(n *graph.Node) map[string]interface{} {
		sockets := getSockets(n)
		kv := make(map[string]interface{}, len(sockets))
		for _, socket := range sockets {
			kv[socket.Hash()] = socket
		}
		return kv
	}

	graphIndexer := graph.NewGraphIndexer(g, hashNode, true)
	socketFilter := graph.NewGraphElementFilter(filters.NewNotNullFilter("Sockets"))
	for _, node := range g.GetNodes(socketFilter) {
		graphIndexer.OnNodeAdded(node)
	}
	return graphIndexer
}
<|endoftext|>


<commit_before>package tools

import (
	"fmt"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"testing"
)

func Test_findBaseDir(t *testing.T) {
	cases := []struct {
		dirList []string
		want    string
	}{
		{[]string{"bin/bossac"}, "bin/"},
		{[]string{"bin/", "bin/bossac"}, "bin/"},
		{[]string{"bin/", "bin/bossac", "example"}, ""},
		{[]string{"avrdude/bin/avrdude", "avrdude/etc/avrdude.conf"}, "avrdude/"},
		{[]string{"pax_global_header", "bin/", "bin/bossac"}, "bin/"},
	}
	for _, tt := range cases {
		t.Run(fmt.Sprintln(tt.dirList), func(t *testing.T) {
			if got := findBaseDir(tt.dirList); got != tt.want {
				t.Errorf("findBaseDir() = got %v, want %v", got, tt.want)
			}
		})
	}
}

func TestTools_DownloadAndUnpackBehaviour(t *testing.T) {
	urls := []string{
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-armhf-pc-linux-gnu.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-aarch64-pc-linux-gnu.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i386-apple-darwin11.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-x86_64-pc-linux-gnu.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i686-pc-linux-gnu.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i686-w64-mingw32.zip",
	}
	expectedDirList := []string{"bin", "etc"}

	for _, url := range urls {
		t.Log("Downloading tool from " + url)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("%v", err)
		}
		defer resp.Body.Close()

		// Read the body
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Errorf("%v", err)
		}

		location := path.Join("/tmp", dir(), "arduino", "avrdude", "6.3.0-arduino14")
		os.MkdirAll(location, os.ModePerm)
		err = os.RemoveAll(location)

		if err != nil {
			t.Errorf("%v", err)
		}

		srcType, err := mimeType(body)
		if err != nil {
			t.Errorf("%v", err)
		}

		switch srcType {
		case "application/zip":
			location, err = extractZip(func(msg string) { t.Log(msg) }, body, location)
		case "application/x-bz2":
		case "application/octet-stream":
			location, err = extractBz2(func(msg string) { t.Log(msg) }, body, location)
		case "application/x-gzip":
			location, err = extractTarGz(func(msg string) { t.Log(msg) }, body, location)
		default:
			t.Errorf("no suitable type found")
		}
		files, err := ioutil.ReadDir(location)
		if err != nil {
			t.Errorf("%v", err)
		}
		dirList := []string{}
		for _, f := range files {
			dirList = append(dirList, f.Name())
		}

		assert.ElementsMatchf(t, dirList, expectedDirList, "error message %s", "formatted")

	}

}
<commit_msg>updated testcase for findBaseDir with winavrdude zip content<commit_after>package tools

import (
	"fmt"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"testing"
)

func Test_findBaseDir(t *testing.T) {
	cases := []struct {
		dirList []string
		want    string
	}{
		{[]string{"bin/bossac"}, "bin/"},
		{[]string{"bin/", "bin/bossac"}, "bin/"},
		{[]string{"bin/", "bin/bossac", "example"}, ""},
		{[]string{"avrdude/bin/",
			"avrdude/bin/avrdude.exe",
			"avrdude/bin/remove_giveio.bat",
			"avrdude/bin/status_giveio.bat",
			"avrdude/bin/giveio.sys",
			"avrdude/bin/loaddrv.exe",
			"avrdude/bin/libusb0.dll",
			"avrdude/bin/install_giveio.bat",
			"avrdude/etc/avrdude.conf"}, "avrdude/"},
		{[]string{"pax_global_header", "bin/", "bin/bossac"}, "bin/"},
	}
	for _, tt := range cases {
		t.Run(fmt.Sprintln(tt.dirList), func(t *testing.T) {
			if got := findBaseDir(tt.dirList); got != tt.want {
				t.Errorf("findBaseDir() = got %v, want %v", got, tt.want)
			}
		})
	}
}

func TestTools_DownloadAndUnpackBehaviour(t *testing.T) {
	urls := []string{
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-armhf-pc-linux-gnu.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-aarch64-pc-linux-gnu.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i386-apple-darwin11.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-x86_64-pc-linux-gnu.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i686-pc-linux-gnu.tar.bz2",
		"http://downloads.arduino.cc/tools/avrdude-6.3.0-arduino14-i686-w64-mingw32.zip",
	}
	expectedDirList := []string{"bin", "etc"}

	for _, url := range urls {
		t.Log("Downloading tool from " + url)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("%v", err)
		}
		defer resp.Body.Close()

		// Read the body
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			t.Errorf("%v", err)
		}

		location := path.Join("/tmp", dir(), "arduino", "avrdude", "6.3.0-arduino14")
		os.MkdirAll(location, os.ModePerm)
		err = os.RemoveAll(location)

		if err != nil {
			t.Errorf("%v", err)
		}

		srcType, err := mimeType(body)
		if err != nil {
			t.Errorf("%v", err)
		}

		switch srcType {
		case "application/zip":
			location, err = extractZip(func(msg string) { t.Log(msg) }, body, location)
		case "application/x-bz2":
		case "application/octet-stream":
			location, err = extractBz2(func(msg string) { t.Log(msg) }, body, location)
		case "application/x-gzip":
			location, err = extractTarGz(func(msg string) { t.Log(msg) }, body, location)
		default:
			t.Errorf("no suitable type found")
		}
		files, err := ioutil.ReadDir(location)
		if err != nil {
			t.Errorf("%v", err)
		}
		dirList := []string{}
		for _, f := range files {
			dirList = append(dirList, f.Name())
		}

		assert.ElementsMatchf(t, dirList, expectedDirList, "error message %s", "formatted")

	}

}
<|endoftext|>


<commit_before>package filepathfilter

import (
	"fmt"
	"path/filepath"
	"regexp"
	"strings"
)

type Pattern interface {
	Match(filename string) bool
}

type Filter struct {
	include []Pattern
	exclude []Pattern
}

func NewFromPatterns(include, exclude []Pattern) *Filter {
	return &Filter{include: include, exclude: exclude}
}

func New(include, exclude []string) *Filter {
	return NewFromPatterns(convertToPatterns(include), convertToPatterns(exclude))
}

func (f *Filter) Allows(filename string) bool {
	if f == nil {
		return true
	}

	if len(f.include)+len(f.exclude) == 0 {
		return true
	}

	cleanedName := filepath.Clean(filename)

	if len(f.include) > 0 {
		matched := false
		for _, inc := range f.include {
			matched = inc.Match(cleanedName)
			if matched {
				break
			}
		}
		if !matched {
			return false
		}
	}

	if len(f.exclude) > 0 {
		for _, ex := range f.exclude {
			if ex.Match(cleanedName) {
				return false
			}
		}
	}

	return true
}

func NewPattern(rawpattern string) Pattern {
	cleanpattern := filepath.Clean(rawpattern)

	// Special case local dir, matches all (inc subpaths)
	if _, local := localDirSet[cleanpattern]; local {
		return noOpMatcher{}
	}

	hasPathSep := strings.Contains(cleanpattern, string(filepath.Separator))

	// special case * when there are no path separators
	// filepath.Match never allows * to match a path separator, which is correct
	// for gitignore IF the pattern includes a path separator, but not otherwise
	// So *.txt should match in any subdir, as should test*, but sub/*.txt would
	// only match directly in the sub dir
	// Don't need to test cross-platform separators as both cleaned above
	if !hasPathSep && strings.Contains(cleanpattern, "*") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*", ".*", -1))
		return &pathlessWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
		// Also support ** with path separators
	} else if hasPathSep && strings.Contains(cleanpattern, "**") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*\\*", ".*", -1))
		return &doubleWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	} else {
		s := string(filepath.Separator)
		return &basicPattern{
			rawPattern: cleanpattern,
			prefix:     cleanpattern + s,
			suffix:     s + cleanpattern,
			inner:      s + cleanpattern + s,
		}
	}
}

func convertToPatterns(rawpatterns []string) []Pattern {
	patterns := make([]Pattern, len(rawpatterns))
	for i, raw := range rawpatterns {
		patterns[i] = NewPattern(raw)
	}
	return patterns
}

type basicPattern struct {
	rawPattern string
	prefix     string
	suffix     string
	inner      string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *basicPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched ||
		strings.HasPrefix(name, p.prefix) ||
		strings.HasSuffix(name, p.suffix) ||
		strings.Contains(name, p.inner)
}

type pathlessWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathlessWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(filepath.Base(name))
}

type doubleWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *doubleWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(name)
}

type noOpMatcher struct {
}

func (n noOpMatcher) Match(name string) bool {
	return true
}

var localDirSet = map[string]struct{}{
	".":   struct{}{},
	"./":  struct{}{},
	".\\": struct{}{},
}
<commit_msg>filepathfilter: optimize simple extension patterns<commit_after>package filepathfilter

import (
	"fmt"
	"path/filepath"
	"regexp"
	"strings"
)

type Pattern interface {
	Match(filename string) bool
}

type Filter struct {
	include []Pattern
	exclude []Pattern
}

func NewFromPatterns(include, exclude []Pattern) *Filter {
	return &Filter{include: include, exclude: exclude}
}

func New(include, exclude []string) *Filter {
	return NewFromPatterns(convertToPatterns(include), convertToPatterns(exclude))
}

func (f *Filter) Allows(filename string) bool {
	if f == nil {
		return true
	}

	if len(f.include)+len(f.exclude) == 0 {
		return true
	}

	cleanedName := filepath.Clean(filename)

	if len(f.include) > 0 {
		matched := false
		for _, inc := range f.include {
			matched = inc.Match(cleanedName)
			if matched {
				break
			}
		}
		if !matched {
			return false
		}
	}

	if len(f.exclude) > 0 {
		for _, ex := range f.exclude {
			if ex.Match(cleanedName) {
				return false
			}
		}
	}

	return true
}

func NewPattern(rawpattern string) Pattern {
	cleanpattern := filepath.Clean(rawpattern)

	// Special case local dir, matches all (inc subpaths)
	if _, local := localDirSet[cleanpattern]; local {
		return noOpMatcher{}
	}

	sep := string(filepath.Separator)
	hasPathSep := strings.Contains(cleanpattern, sep)
	ext := filepath.Ext(cleanpattern)
	plen := len(cleanpattern)
	if plen > 1 && !hasPathSep && strings.HasPrefix(cleanpattern, "*") && cleanpattern[1:plen] == ext {
		return &simpleExtPattern{ext: ext}
	}

	// special case * when there are no path separators
	// filepath.Match never allows * to match a path separator, which is correct
	// for gitignore IF the pattern includes a path separator, but not otherwise
	// So *.txt should match in any subdir, as should test*, but sub/*.txt would
	// only match directly in the sub dir
	// Don't need to test cross-platform separators as both cleaned above
	if !hasPathSep && strings.Contains(cleanpattern, "*") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*", ".*", -1))
		return &pathlessWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	// Also support ** with path separators
	if hasPathSep && strings.Contains(cleanpattern, "**") {
		pattern := regexp.QuoteMeta(cleanpattern)
		regpattern := fmt.Sprintf("^%s$", strings.Replace(pattern, "\\*\\*", ".*", -1))
		return &doubleWildcardPattern{
			rawPattern: cleanpattern,
			wildcardRE: regexp.MustCompile(regpattern),
		}
	}

	return &pathPattern{
		rawPattern: cleanpattern,
		prefix:     cleanpattern + sep,
		suffix:     sep + cleanpattern,
		inner:      sep + cleanpattern + sep,
	}
}

func convertToPatterns(rawpatterns []string) []Pattern {
	patterns := make([]Pattern, len(rawpatterns))
	for i, raw := range rawpatterns {
		patterns[i] = NewPattern(raw)
	}
	return patterns
}

type pathPattern struct {
	rawPattern string
	prefix     string
	suffix     string
	inner      string
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	return matched ||
		strings.HasPrefix(name, p.prefix) ||
		strings.HasSuffix(name, p.suffix) ||
		strings.Contains(name, p.inner)
}

type simpleExtPattern struct {
	ext string
}

func (p *simpleExtPattern) Match(name string) bool {
	return strings.HasSuffix(name, p.ext)
}

type pathlessWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *pathlessWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(filepath.Base(name))
}

type doubleWildcardPattern struct {
	rawPattern string
	wildcardRE *regexp.Regexp
}

// Match is a revised version of filepath.Match which makes it behave more
// like gitignore
func (p *doubleWildcardPattern) Match(name string) bool {
	matched, _ := filepath.Match(p.rawPattern, name)
	// Match the whole of the base name but allow matching in folders if no path
	return matched || p.wildcardRE.MatchString(name)
}

type noOpMatcher struct {
}

func (n noOpMatcher) Match(name string) bool {
	return true
}

var localDirSet = map[string]struct{}{
	".":   struct{}{},
	"./":  struct{}{},
	".\\": struct{}{},
}
<|endoftext|>