<commit_before>/*
Package html2data - extract data from HTML via CSS selectors

Install package and command line utility:

	go get -u github.com/msoap/html2data/cmd/html2data

Install package only:

	go get -u github.com/msoap/html2data

Allowed pseudo-selectors:

:attr(attr_name) - for getting attributes instead text

:html - for getting HTML instead text

:get(N) - get n-th element from list

Command line utility:

	html2data URL "css selector"
	html2data file.html "css selector"
	cat file.html | html2data "css selector"

*/
package html2data

import (
	"fmt"
	"io"
	"net/http"
	"net/http/cookiejar"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html/charset"
)

// docOrSelection - for exec .Find
type docOrSelection interface {
	Find(string) *goquery.Selection
}

// Doc - html document for parse
type Doc struct {
	doc docOrSelection
	Err error
}

// CSSSelector - selector with settings
type CSSSelector struct {
	selector string
	attrName string
	getHTML  bool
	getNth   int
}

// Cfg - config for GetData* methods
type Cfg struct {
	DontTrimSpaces bool // get text as is, by default trim spaces
}

// getDataFromDocOrSelection - extract data by CSS-selectors from goquery.Selection or goquery.Doc
func (doc Doc) getDataFromDocOrSelection(docOrSelection docOrSelection, selectors map[string]string, config Cfg) (result map[string][]string, err error) {
	if doc.Err != nil {
		return result, fmt.Errorf("parse document error: %s", doc.Err)
	}
	defer func() {
		if errRecoverRaw := recover(); errRecoverRaw != nil {
			result, err = map[string][]string{}, fmt.Errorf("%s", errRecoverRaw)
		}
	}()

	result = map[string][]string{}
	for name, selectorRaw := range selectors {
		selector := parseSelector(selectorRaw)

		texts := []string{}
		docOrSelection.Find(selector.selector).Each(func(i int, selection *goquery.Selection) {
			if selector.getNth > 0 && selector.getNth != i+1 {
				return
			}

			foundText := ""
			switch {
			case selector.attrName != "":
				foundText = selection.AttrOr(selector.attrName, "")
			case selector.getHTML:
				foundText, err = selection.Html()
				if err != nil {
					return
				}
			default:
				foundText = selection.Text()
			}

			if !config.DontTrimSpaces {
				foundText = strings.TrimSpace(foundText)
			}
			texts = append(texts, foundText)
		})
		result[name] = texts
	}

	return result, err
}

// parseSelector - parse pseudo-selectors:
// :attr(href) - for getting attribute instead text node
func parseSelector(inputSelector string) (outSelector CSSSelector) {
	htmlAttrRe := regexp.MustCompile(`^\s*(\w+)\s*(?:\(\s*(\w+)\s*\))?\s*$`)

	parts := strings.Split(inputSelector, ":")
	outSelector.selector, parts = parts[0], parts[1:]
	for _, part := range parts {
		reParts := htmlAttrRe.FindStringSubmatch(part)
		switch {
		case len(reParts) == 3 && reParts[1] == "attr":
			outSelector.attrName = reParts[2]
		case len(reParts) == 3 && reParts[1] == "html":
			outSelector.getHTML = true
		case len(reParts) == 3 && reParts[1] == "get":
			outSelector.getNth, _ = strconv.Atoi(reParts[2])
		default:
			outSelector.selector += ":" + part
		}
	}

	return outSelector
}

// getConfig - get first config element from list
func getConfig(configs []Cfg) Cfg {
	switch {
	case len(configs) == 0:
		return Cfg{}
	case len(configs) == 1:
		return configs[0]
	default:
		panic("[]Cfg length must be equal 0 or 1")
	}
}

// GetData - extract data by CSS-selectors
//  texts, err := doc.GetData(map[string]string{"h1": "h1"})
func (doc Doc) GetData(selectors map[string]string, configs ...Cfg) (result map[string][]string, err error) {
	result, err = doc.getDataFromDocOrSelection(doc.doc, selectors, getConfig(configs))
	return result, err
}

// GetDataFirst - extract data by CSS-selectors, get first entry for each selector or ""
//  texts, err := doc.GetDataFirst(map[string]string{"h1": "h1"})
func (doc Doc) GetDataFirst(selectors map[string]string, configs ...Cfg) (result map[string]string, err error) {
	resultRaw, err := doc.getDataFromDocOrSelection(doc.doc, selectors, getConfig(configs))
	if err != nil {
		return result, err
	}

	result = map[string]string{}
	for key := range resultRaw {
		if len(resultRaw[key]) > 0 {
			result[key] = resultRaw[key][0]
		} else {
			result[key] = ""
		}
	}

	return result, err
}

// GetDataNested - extract nested data by CSS-selectors from another CSS-selector
//  texts, err := doc.GetDataNested("CSS.selector", map[string]string{"h1": "h1"}) - get h1 from CSS.selector
func (doc Doc) GetDataNested(selectorRaw string, nestedSelectors map[string]string, configs ...Cfg) (result []map[string][]string, err error) {
	selector := parseSelector(selectorRaw)
	defer func() {
		if errRecoverRaw := recover(); errRecoverRaw != nil {
			result, err = []map[string][]string{}, fmt.Errorf("%s", errRecoverRaw)
		}
	}()
	result = []map[string][]string{}

	doc.doc.Find(selector.selector).Each(func(i int, selection *goquery.Selection) {
		if selector.getNth > 0 && selector.getNth != i+1 {
			return
		}

		nestedResult, nestedErr := doc.getDataFromDocOrSelection(selection, nestedSelectors, getConfig(configs))
		if nestedErr != nil {
			err = nestedErr
			return
		}

		result = append(result, nestedResult)
	})

	return result, err
}

// GetDataNestedFirst - extract nested data by CSS-selectors from another CSS-selector
// get first entry for each selector or ""
//  texts, err := doc.GetDataNestedFirst("CSS.selector", map[string]string{"h1": "h1"}) - get h1 from CSS.selector
func (doc Doc) GetDataNestedFirst(selectorRaw string, nestedSelectors map[string]string, configs ...Cfg) (result []map[string]string, err error) {
	resultRaw, err := doc.GetDataNested(selectorRaw, nestedSelectors, configs...)
	if err != nil {
		return result, err
	}

	result = []map[string]string{}
	for i, resultRawPart := range resultRaw {
		result = append(result, map[string]string{})
		for key := range resultRawPart {
			if len(resultRawPart[key]) > 0 {
				result[i][key] = resultRawPart[key][0]
			} else {
				result[i][key] = ""
			}
		}
	}

	return result, err
}

// GetDataSingle - extract data by one CSS-selector
//  title, err := doc.GetDataSingle("title")
func (doc Doc) GetDataSingle(selector string, configs ...Cfg) (result string, err error) {
	texts, err := doc.GetData(map[string]string{"single": selector}, getConfig(configs))
	if err != nil {
		return result, err
	}

	if textOne, ok := texts["single"]; ok && len(textOne) > 0 {
		result = textOne[0]
	}

	return result, err
}

// FromReader - get doc from io.Reader
func FromReader(reader io.Reader) Doc {
	doc, err := goquery.NewDocumentFromReader(reader)
	return Doc{doc, err}
}

// FromFile - get doc from file
func FromFile(fileName string) Doc {
	fileReader, err := os.Open(fileName)
	if err != nil {
		return Doc{Err: err}
	}

	doc := FromReader(fileReader)
	err = fileReader.Close()
	if err != nil {
		return Doc{Err: err}
	}

	return doc
}

// URLCfg - config for FromURL()
type URLCfg struct {
	UA                string // custom user-agent
	TimeOut           int    // timeout in seconds
	DontDetectCharset bool   // dont autoconvert to UTF8
}

// FromURL - get doc from URL
//
//  FromURL("https://url")
//  FromURL("https://url", URLCfg{UA: "Custom UA 1.0", TimeOut: 10})
func FromURL(URL string, config ...URLCfg) Doc {
	ua, timeout, dontDetectCharset := "", 0, false
	if len(config) == 1 {
		ua = config[0].UA
		timeout = config[0].TimeOut
		dontDetectCharset = config[0].DontDetectCharset
	} else if len(config) > 1 {
		panic("FromURL(): only one config argument allowed")
	}

	htmlReader, err := getHTMLPage(URL, ua, timeout, dontDetectCharset)
	if err != nil {
		return Doc{Err: err}
	}

	return FromReader(htmlReader)
}

// getHTMLPage - get html by http(s) as http.Response
func getHTMLPage(url string, ua string, timeout int, dontDetectCharset bool) (htmlReader io.Reader, err error) {
	cookie, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookie,
		Timeout: time.Duration(time.Duration(timeout) * time.Second),
	}

	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return htmlReader, err
	}

	if ua != "" {
		request.Header.Set("User-Agent", ua)
	}

	response, err := client.Do(request)
	if err != nil {
		return htmlReader, err
	}

	if contentType := response.Header.Get("Content-Type"); contentType != "" && !dontDetectCharset {
		htmlReader, err = charset.NewReader(response.Body, contentType)
		if err != nil {
			return htmlReader, err
		}
	} else {
		return response.Body, nil
	}

	return htmlReader, nil
}
<commit_msg>Removed unnecessary conversion again :)<commit_after>/*
Package html2data - extract data from HTML via CSS selectors

Install package and command line utility:

	go get -u github.com/msoap/html2data/cmd/html2data

Install package only:

	go get -u github.com/msoap/html2data

Allowed pseudo-selectors:

:attr(attr_name) - for getting attributes instead text

:html - for getting HTML instead text

:get(N) - get n-th element from list

Command line utility:

	html2data URL "css selector"
	html2data file.html "css selector"
	cat file.html | html2data "css selector"

*/
package html2data

import (
	"fmt"
	"io"
	"net/http"
	"net/http/cookiejar"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html/charset"
)

// docOrSelection - for exec .Find
type docOrSelection interface {
	Find(string) *goquery.Selection
}

// Doc - html document for parse
type Doc struct {
	doc docOrSelection
	Err error
}

// CSSSelector - selector with settings
type CSSSelector struct {
	selector string
	attrName string
	getHTML  bool
	getNth   int
}

// Cfg - config for GetData* methods
type Cfg struct {
	DontTrimSpaces bool // get text as is, by default trim spaces
}

// getDataFromDocOrSelection - extract data by CSS-selectors from goquery.Selection or goquery.Doc
func (doc Doc) getDataFromDocOrSelection(docOrSelection docOrSelection, selectors map[string]string, config Cfg) (result map[string][]string, err error) {
	if doc.Err != nil {
		return result, fmt.Errorf("parse document error: %s", doc.Err)
	}
	defer func() {
		if errRecoverRaw := recover(); errRecoverRaw != nil {
			result, err = map[string][]string{}, fmt.Errorf("%s", errRecoverRaw)
		}
	}()

	result = map[string][]string{}
	for name, selectorRaw := range selectors {
		selector := parseSelector(selectorRaw)

		texts := []string{}
		docOrSelection.Find(selector.selector).Each(func(i int, selection *goquery.Selection) {
			if selector.getNth > 0 && selector.getNth != i+1 {
				return
			}

			foundText := ""
			switch {
			case selector.attrName != "":
				foundText = selection.AttrOr(selector.attrName, "")
			case selector.getHTML:
				foundText, err = selection.Html()
				if err != nil {
					return
				}
			default:
				foundText = selection.Text()
			}

			if !config.DontTrimSpaces {
				foundText = strings.TrimSpace(foundText)
			}
			texts = append(texts, foundText)
		})
		result[name] = texts
	}

	return result, err
}

// parseSelector - parse pseudo-selectors:
// :attr(href) - for getting attribute instead text node
func parseSelector(inputSelector string) (outSelector CSSSelector) {
	htmlAttrRe := regexp.MustCompile(`^\s*(\w+)\s*(?:\(\s*(\w+)\s*\))?\s*$`)

	parts := strings.Split(inputSelector, ":")
	outSelector.selector, parts = parts[0], parts[1:]
	for _, part := range parts {
		reParts := htmlAttrRe.FindStringSubmatch(part)
		switch {
		case len(reParts) == 3 && reParts[1] == "attr":
			outSelector.attrName = reParts[2]
		case len(reParts) == 3 && reParts[1] == "html":
			outSelector.getHTML = true
		case len(reParts) == 3 && reParts[1] == "get":
			outSelector.getNth, _ = strconv.Atoi(reParts[2])
		default:
			outSelector.selector += ":" + part
		}
	}

	return outSelector
}

// getConfig - get first config element from list
func getConfig(configs []Cfg) Cfg {
	switch {
	case len(configs) == 0:
		return Cfg{}
	case len(configs) == 1:
		return configs[0]
	default:
		panic("[]Cfg length must be equal 0 or 1")
	}
}

// GetData - extract data by CSS-selectors
//  texts, err := doc.GetData(map[string]string{"h1": "h1"})
func (doc Doc) GetData(selectors map[string]string, configs ...Cfg) (result map[string][]string, err error) {
	result, err = doc.getDataFromDocOrSelection(doc.doc, selectors, getConfig(configs))
	return result, err
}

// GetDataFirst - extract data by CSS-selectors, get first entry for each selector or ""
//  texts, err := doc.GetDataFirst(map[string]string{"h1": "h1"})
func (doc Doc) GetDataFirst(selectors map[string]string, configs ...Cfg) (result map[string]string, err error) {
	resultRaw, err := doc.getDataFromDocOrSelection(doc.doc, selectors, getConfig(configs))
	if err != nil {
		return result, err
	}

	result = map[string]string{}
	for key := range resultRaw {
		if len(resultRaw[key]) > 0 {
			result[key] = resultRaw[key][0]
		} else {
			result[key] = ""
		}
	}

	return result, err
}

// GetDataNested - extract nested data by CSS-selectors from another CSS-selector
//  texts, err := doc.GetDataNested("CSS.selector", map[string]string{"h1": "h1"}) - get h1 from CSS.selector
func (doc Doc) GetDataNested(selectorRaw string, nestedSelectors map[string]string, configs ...Cfg) (result []map[string][]string, err error) {
	selector := parseSelector(selectorRaw)
	defer func() {
		if errRecoverRaw := recover(); errRecoverRaw != nil {
			result, err = []map[string][]string{}, fmt.Errorf("%s", errRecoverRaw)
		}
	}()
	result = []map[string][]string{}

	doc.doc.Find(selector.selector).Each(func(i int, selection *goquery.Selection) {
		if selector.getNth > 0 && selector.getNth != i+1 {
			return
		}

		nestedResult, nestedErr := doc.getDataFromDocOrSelection(selection, nestedSelectors, getConfig(configs))
		if nestedErr != nil {
			err = nestedErr
			return
		}

		result = append(result, nestedResult)
	})

	return result, err
}

// GetDataNestedFirst - extract nested data by CSS-selectors from another CSS-selector
// get first entry for each selector or ""
//  texts, err := doc.GetDataNestedFirst("CSS.selector", map[string]string{"h1": "h1"}) - get h1 from CSS.selector
func (doc Doc) GetDataNestedFirst(selectorRaw string, nestedSelectors map[string]string, configs ...Cfg) (result []map[string]string, err error) {
	resultRaw, err := doc.GetDataNested(selectorRaw, nestedSelectors, configs...)
	if err != nil {
		return result, err
	}

	result = []map[string]string{}
	for i, resultRawPart := range resultRaw {
		result = append(result, map[string]string{})
		for key := range resultRawPart {
			if len(resultRawPart[key]) > 0 {
				result[i][key] = resultRawPart[key][0]
			} else {
				result[i][key] = ""
			}
		}
	}

	return result, err
}

// GetDataSingle - extract data by one CSS-selector
//  title, err := doc.GetDataSingle("title")
func (doc Doc) GetDataSingle(selector string, configs ...Cfg) (result string, err error) {
	texts, err := doc.GetData(map[string]string{"single": selector}, getConfig(configs))
	if err != nil {
		return result, err
	}

	if textOne, ok := texts["single"]; ok && len(textOne) > 0 {
		result = textOne[0]
	}

	return result, err
}

// FromReader - get doc from io.Reader
func FromReader(reader io.Reader) Doc {
	doc, err := goquery.NewDocumentFromReader(reader)
	return Doc{doc, err}
}

// FromFile - get doc from file
func FromFile(fileName string) Doc {
	fileReader, err := os.Open(fileName)
	if err != nil {
		return Doc{Err: err}
	}

	doc := FromReader(fileReader)
	err = fileReader.Close()
	if err != nil {
		return Doc{Err: err}
	}

	return doc
}

// URLCfg - config for FromURL()
type URLCfg struct {
	UA                string // custom user-agent
	TimeOut           int    // timeout in seconds
	DontDetectCharset bool   // dont autoconvert to UTF8
}

// FromURL - get doc from URL
//
//  FromURL("https://url")
//  FromURL("https://url", URLCfg{UA: "Custom UA 1.0", TimeOut: 10})
func FromURL(URL string, config ...URLCfg) Doc {
	ua, timeout, dontDetectCharset := "", 0, false
	if len(config) == 1 {
		ua = config[0].UA
		timeout = config[0].TimeOut
		dontDetectCharset = config[0].DontDetectCharset
	} else if len(config) > 1 {
		panic("FromURL(): only one config argument allowed")
	}

	htmlReader, err := getHTMLPage(URL, ua, timeout, dontDetectCharset)
	if err != nil {
		return Doc{Err: err}
	}

	return FromReader(htmlReader)
}

// getHTMLPage - get html by http(s) as http.Response
func getHTMLPage(url string, ua string, timeout int, dontDetectCharset bool) (htmlReader io.Reader, err error) {
	cookie, _ := cookiejar.New(nil)
	client := &http.Client{
		Jar:     cookie,
		Timeout: time.Duration(timeout) * time.Second,
	}

	request, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return htmlReader, err
	}

	if ua != "" {
		request.Header.Set("User-Agent", ua)
	}

	response, err := client.Do(request)
	if err != nil {
		return htmlReader, err
	}

	if contentType := response.Header.Get("Content-Type"); contentType != "" && !dontDetectCharset {
		htmlReader, err = charset.NewReader(response.Body, contentType)
		if err != nil {
			return htmlReader, err
		}
	} else {
		return response.Body, nil
	}

	return htmlReader, nil
}
<|endoftext|>


<commit_before>package tls

import (
	"crypto/x509"
	"crypto/x509/pkix"

	"github.com/openshift/installer/pkg/asset"
)

// KubeletCertKey is the asset that generates the kubelet key/cert pair.
type KubeletCertKey struct {
	CertKey
}

var _ asset.Asset = (*KubeletCertKey)(nil)

// Dependencies returns the dependency of the the cert/key pair, which includes
// the parent CA, and install config if it depends on the install config for
// DNS names, etc.
func (a *KubeletCertKey) Dependencies() []asset.Asset {
	return []asset.Asset{
		&KubeCA{},
	}
}

// Generate generates the cert/key pair based on its dependencies.
func (a *KubeletCertKey) Generate(dependencies asset.Parents) error {
	kubeCA := &KubeCA{}
	dependencies.Get(kubeCA)

	cfg := &CertCfg{
		Subject:      pkix.Name{CommonName: "system:serviceaccount:kube-system:default", Organization: []string{"system:serviceaccounts:kube-system"}},
		KeyUsages:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
		Validity:     ValidityThirtyMinutes,
	}

	return a.CertKey.Generate(cfg, kubeCA, "kubelet", DoNotAppendParent)
}

// Name returns the human-friendly name of the asset.
func (a *KubeletCertKey) Name() string {
	return "Certificate (system:serviceaccount:kube-system:default)"
}
<commit_msg>tls: add system:masters to kubelet client certs<commit_after>package tls

import (
	"crypto/x509"
	"crypto/x509/pkix"

	"github.com/openshift/installer/pkg/asset"
)

// KubeletCertKey is the asset that generates the kubelet key/cert pair.
type KubeletCertKey struct {
	CertKey
}

var _ asset.Asset = (*KubeletCertKey)(nil)

// Dependencies returns the dependency of the the cert/key pair, which includes
// the parent CA, and install config if it depends on the install config for
// DNS names, etc.
func (a *KubeletCertKey) Dependencies() []asset.Asset {
	return []asset.Asset{
		&KubeCA{},
	}
}

// Generate generates the cert/key pair based on its dependencies.
func (a *KubeletCertKey) Generate(dependencies asset.Parents) error {
	kubeCA := &KubeCA{}
	dependencies.Get(kubeCA)

	cfg := &CertCfg{
		// system:masters is a hack to get the kubelet up without kube-core
		// TODO(node): make kubelet bootstrapping secure with minimal permissions eventually switching to system:node:* CommonName
		Subject:      pkix.Name{CommonName: "system:serviceaccount:kube-system:default", Organization: []string{"system:serviceaccounts:kube-system", "system:masters"}},
		KeyUsages:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
		ExtKeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
		Validity:     ValidityThirtyMinutes,
	}

	return a.CertKey.Generate(cfg, kubeCA, "kubelet", DoNotAppendParent)
}

// Name returns the human-friendly name of the asset.
func (a *KubeletCertKey) Name() string {
	return "Certificate (system:serviceaccount:kube-system:default)"
}
<|endoftext|>


<commit_before>// Package power provides algorithms for simulating the power dissipation of
// concurrent applications running on multiprocessor platforms.
package power

import (
	"github.com/ready-steady/persim/system"
	"github.com/ready-steady/persim/time"
)

// Distributer represents a power simulator configured for a particular system.
type Distributer struct {
	platform    *system.Platform
	application *system.Application
	Δt          float64
}

// New returns a power distributor for the given platform, application, and
// sampling period.
func New(platform *system.Platform, application *system.Application, Δt float64) *Distributer {
	return &Distributer{
		platform:    platform,
		application: application,
		Δt:          Δt,
	}
}

// Compute constructs the power profile of the given schedule and stores it in a
// cc-by-sc matrix P where cc is the number of cores and sc is the maximal
// number of steps (samples) that the matrix can accommodate. P is assumed to be
// zeroed.
func (self *Distributer) Compute(schedule *time.Schedule, P []float64, sc uint32) {
	cores, tasks := self.platform.Cores, self.application.Tasks
	Δt := self.Δt

	cc := uint32(len(cores))
	tc := uint16(len(tasks))
	if count := uint32(schedule.Span / Δt); count < sc {
		sc = count
	}

	var j, s, f uint32
	var p float64

	for i := uint16(0); i < tc; i++ {
		j = uint32(schedule.Mapping[i])
		p = cores[j].Power[tasks[i].Type]

		s = uint32(schedule.Start[i] / Δt)
		f = uint32(schedule.Finish[i] / Δt)
		if f > sc {
			f = sc
		}

		for ; s < f; s++ {
			P[s*cc+j] = p
		}
	}
}
<commit_msg>Renamed Distributer into Power<commit_after>// Package power provides algorithms for simulating the power dissipation of
// concurrent applications running on multiprocessor platforms.
package power

import (
	"github.com/ready-steady/persim/system"
	"github.com/ready-steady/persim/time"
)

// Power represents a power simulator configured for a particular system.
type Power struct {
	platform    *system.Platform
	application *system.Application
	Δt          float64
}

// New returns a power distributor for the given platform, application, and
// sampling period.
func New(platform *system.Platform, application *system.Application, Δt float64) *Power {
	return &Power{
		platform:    platform,
		application: application,
		Δt:          Δt,
	}
}

// Compute constructs the power profile of the given schedule and stores it in a
// cc-by-sc matrix P where cc is the number of cores and sc is the maximal
// number of steps (samples) that the matrix can accommodate. P is assumed to be
// zeroed.
func (p *Power) Compute(schedule *time.Schedule, P []float64, sc uint32) {
	cores, tasks := p.platform.Cores, p.application.Tasks
	Δt := p.Δt

	cc := uint32(len(cores))
	tc := uint16(len(tasks))
	if count := uint32(schedule.Span / Δt); count < sc {
		sc = count
	}

	for i := uint16(0); i < tc; i++ {
		j := uint32(schedule.Mapping[i])
		p := cores[j].Power[tasks[i].Type]

		s := uint32(schedule.Start[i] / Δt)
		f := uint32(schedule.Finish[i] / Δt)
		if f > sc {
			f = sc
		}

		for ; s < f; s++ {
			P[s*cc+j] = p
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2018-2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package multiboot implements bootloading multiboot kernels as defined by
// https://www.gnu.org/software/grub/manual/multiboot/multiboot.html.
//
// Package multiboot crafts kexec segments that can be used with the kexec_load
// system call.
package multiboot

import (
	"bytes"
	"debug/elf"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/boot/ibft"
	"github.com/u-root/u-root/pkg/boot/kexec"
	"github.com/u-root/u-root/pkg/boot/multiboot/internal/trampoline"
	"github.com/u-root/u-root/pkg/ubinary"
)

const bootloader = "u-root kexec"

// multiboot defines parameters for working with multiboot kernels.
type multiboot struct {
	mem kexec.Memory

	file    string
	modules []string

	cmdLine    string
	bootloader string

	// trampoline is a path to an executable blob, which contains a trampoline segment.
	// Trampoline sets machine to a specific state defined by multiboot v1 spec.
	// https://www.gnu.org/software/grub/manual/multiboot/multiboot.html#Machine-state.
	trampoline string

	header header

	// infoAddr is a pointer to multiboot info.
	infoAddr uintptr
	// kernelEntry is a pointer to entry point of kernel.
	kernelEntry uintptr
	// EntryPoint is a pointer to trampoline.
	entryPoint uintptr

	info          info
	loadedModules []module
}

var (
	rangeTypes = map[kexec.RangeType]uint32{
		kexec.RangeRAM:      1,
		kexec.RangeDefault:  2,
		kexec.RangeACPI:     3,
		kexec.RangeNVS:      4,
		kexec.RangeReserved: 2,
	}
)

var sizeofMemoryMap = uint(binary.Size(MemoryMap{}))

// MemoryMap represents a reserved range of memory passed via the multiboot Info header.
type MemoryMap struct {
	// Size is the size of the associated structure in bytes.
	Size uint32
	// BaseAddr is the starting address.
	BaseAddr uint64
	// Length is the size of the memory region in bytes.
	Length uint64
	// Type is the variety of address range represented.
	Type uint32
}

// String returns a readable representation of a MemoryMap entry.
func (m MemoryMap) String() string {
	return fmt.Sprintf("[0x%x, 0x%x) (len: 0x%x, size: 0x%x, type: %d)", m.BaseAddr, m.BaseAddr+m.Length, m.Length, m.Size, m.Type)
}

type memoryMaps []MemoryMap

// String returns a new-line-separated representation of the entire memory map.
func (m memoryMaps) String() string {
	var s []string
	for _, mm := range m {
		s = append(s, mm.String())
	}
	return strings.Join(s, "\n")
}

// Probe checks if file is multiboot v1 kernel.
func Probe(file string) error {
	b, err := readFile(file)
	if err != nil {
		return err
	}
	kernel := &kernelReader{buf: b}
	_, err = parseHeader(kernel)
	return err
}

// newMB returns a new multiboot instance.
func newMB(file, cmdLine string, modules []string) (*multiboot, error) {
	// Trampoline should be a part of current binary.
	p, err := os.Executable()
	if err != nil {
		return nil, fmt.Errorf("cannot find current executable path: %v", err)
	}
	trampoline, err := filepath.EvalSymlinks(p)
	if err != nil {
		return nil, fmt.Errorf("cannot eval symlinks for %v: %v", p, err)
	}

	return &multiboot{
		file:       file,
		modules:    modules,
		cmdLine:    cmdLine,
		trampoline: trampoline,
		bootloader: bootloader,
		mem:        kexec.Memory{},
	}, nil
}

// Load parses and loads a multiboot kernel `file` using kexec_load.
//
// Each module is a path followed by optional command-line arguments, e.g.
// []string{"./module arg1 arg2", "./module2 arg3 arg4"}.
//
// debug turns on debug logging.
//
// Load can set up an arbitrary number of modules, and takes care of the
// multiboot info structure, including the memory map.
//
// After Load is called, kexec.Reboot() is ready to be called any time to stop
// Linux and execute the loaded kernel.
func Load(debug bool, file, cmdline string, modules []string, ibft *ibft.IBFT) error {
	m, err := newMB(file, cmdline, modules)
	if err != nil {
		return err
	}
	if err := m.load(debug, ibft); err != nil {
		return err
	}
	if err := kexec.Load(m.entryPoint, m.mem.Segments, 0); err != nil {
		return fmt.Errorf("kexec.Load() error: %v", err)
	}
	return nil
}

// load loads and parses multiboot information from m.file.
func (m *multiboot) load(debug bool, ibft *ibft.IBFT) error {
	log.Printf("Parsing file %v", m.file)
	b, err := readFile(m.file)
	if err != nil {
		return err
	}
	kernel := kernelReader{buf: b}
	log.Println("Parsing multiboot header")
	if m.header, err = parseHeader(&kernel); err != nil {
		return fmt.Errorf("error parsing headers: %v", err)
	}

	log.Printf("Getting kernel entry point")
	if m.kernelEntry, err = getEntryPoint(kernel); err != nil {
		return fmt.Errorf("error getting kernel entry point: %v", err)
	}
	log.Printf("Kernel entry point at %#x", m.kernelEntry)

	log.Printf("Parsing ELF segments")
	if err := m.mem.LoadElfSegments(kernel); err != nil {
		return fmt.Errorf("error loading ELF segments: %v", err)
	}

	log.Printf("Parsing memory map")
	if err := m.mem.ParseMemoryMap(); err != nil {
		return fmt.Errorf("error parsing memory map: %v", err)
	}

	// Insert the iBFT now, since nothing else has been allocated and this
	// is the most restricted allocation we're gonna have to make.
	if ibft != nil {
		ibuf := ibft.Marshal()

		// The iBFT may sit between 512K and 1M in physical memory. The
		// loaded OS finds it by scanning this region.
		allowedRange := kexec.Range{
			Start: 0x80000,
			Size:  0x80000,
		}
		r, err := m.mem.ReservePhys(uint(len(ibuf)), allowedRange)
		if err != nil {
			return fmt.Errorf("reserving space for the iBFT in %s failed: %v", allowedRange, err)
		}
		log.Printf("iBFT was allocated at %s: %#v", r, ibft)
		m.mem.Segments.Insert(kexec.NewSegment(ibuf, r))
	}

	log.Printf("Preparing multiboot info")
	if m.infoAddr, err = m.addInfo(); err != nil {
		return fmt.Errorf("error preparing multiboot info: %v", err)
	}

	log.Printf("Adding trampoline")
	if m.entryPoint, err = m.addTrampoline(); err != nil {
		return fmt.Errorf("error adding trampoline: %v", err)
	}
	log.Printf("Trampoline entry point at %#x", m.entryPoint)

	if debug {
		info, err := m.description()
		if err != nil {
			log.Printf("%v cannot create debug info: %v", DebugPrefix, err)
		}
		log.Printf("%v %v", DebugPrefix, info)
	}

	return nil
}

func getEntryPoint(r io.ReaderAt) (uintptr, error) {
	f, err := elf.NewFile(r)
	if err != nil {
		return 0, err
	}
	return uintptr(f.Entry), err
}

func (m *multiboot) addInfo() (addr uintptr, err error) {
	iw, err := m.newMultibootInfo()
	if err != nil {
		return 0, err
	}
	infoSize, err := iw.size()
	if err != nil {
		return 0, err
	}

	r, err := m.mem.FindSpace(infoSize)
	if err != nil {
		return 0, err
	}

	d, err := iw.marshal(r.Start)
	if err != nil {
		return 0, err
	}
	m.info = iw.info

	m.mem.Segments.Insert(kexec.NewSegment(d, r))
	return r.Start, nil
}

func (m multiboot) memoryMap() memoryMaps {
	var ret memoryMaps
	for _, r := range m.mem.Phys {
		typ, ok := rangeTypes[r.Type]
		if !ok {
			typ = rangeTypes[kexec.RangeDefault]
		}
		v := MemoryMap{
			// Size is really used for skipping to the next pair.
			Size:     uint32(sizeofMemoryMap) - 4,
			BaseAddr: uint64(r.Start),
			Length:   uint64(r.Size),
			Type:     typ,
		}
		ret = append(ret, v)
	}
	return ret
}

func (m *multiboot) addMmap() (addr uintptr, size uint, err error) {
	mmap := m.memoryMap()
	log.Printf("Memory map:\n%s", mmap)
	d, err := mmap.marshal()
	if err != nil {
		return 0, 0, err
	}
	r, err := m.mem.AddKexecSegment(d)
	if err != nil {
		return 0, 0, err
	}
	return r.Start, uint(len(mmap)) * sizeofMemoryMap, nil
}

func (m multiboot) memoryBoundaries() (lower, upper uint32) {
	const M1 = 1048576
	const K640 = 640 * 1024
	for _, r := range m.mem.Phys {
		if r.Type != kexec.RangeRAM {
			continue
		}
		end := uint32(r.Start) + uint32(r.Size)
		// Lower memory starts at address 0, and upper memory starts at address 1 megabyte.
		// The maximum possible value for lower memory is 640 kilobytes.
		// The value returned for upper memory is maximally the address of the first upper memory hole minus 1 megabyte.
		// It is not guaranteed to be this value.
		if r.Start <= K640 && end > lower {
			lower = end
		}
		if r.Start <= M1 && end > upper+M1 {
			upper = end - M1
		}
	}
	return
}

func min(a, b uint32) uint32 {
	if a < b {
		return a
	}
	return b
}

func (m *multiboot) newMultibootInfo() (*infoWrapper, error) {
	mmapAddr, mmapSize, err := m.addMmap()
	if err != nil {
		return nil, err
	}
	var inf info
	if m.header.Flags&flagHeaderMemoryInfo != 0 {
		lower, upper := m.memoryBoundaries()
		inf = info{
			Flags:      flagInfoMemMap | flagInfoMemory,
			MemLower:   min(uint32(lower>>10), 0xFFFFFFFF),
			MemUpper:   min(uint32(upper>>10), 0xFFFFFFFF),
			MmapLength: uint32(mmapSize),
			MmapAddr:   uint32(mmapAddr),
		}
	}

	if len(m.modules) > 0 {
		modAddr, err := m.addModules()
		if err != nil {
			return nil, err
		}
		inf.Flags |= flagInfoMods
		inf.ModsAddr = uint32(modAddr)
		inf.ModsCount = uint32(len(m.modules))
	}

	inf.CmdLine = sizeofInfo
	inf.BootLoaderName = sizeofInfo + uint32(len(m.cmdLine)) + 1
	inf.Flags |= flagInfoCmdLine | flagInfoBootLoaderName
	return &infoWrapper{
		info:           inf,
		CmdLine:        m.cmdLine,
		BootLoaderName: m.bootloader,
	}, nil
}

// marshal writes out the exact bytes expected by the multiboot info header
// specified in
// https://www.gnu.org/software/grub/manual/multiboot/multiboot.html#Boot-information-format.
func (m memoryMaps) marshal() ([]byte, error) {
	buf := bytes.Buffer{}
	err := binary.Write(&buf, ubinary.NativeEndian, m)
	return buf.Bytes(), err
}

func (m *multiboot) addTrampoline() (entry uintptr, err error) {
	// Trampoline setups the machine registers to desired state
	// and executes the loaded kernel.
	d, err := trampoline.Setup(m.trampoline, m.infoAddr, m.kernelEntry)
	if err != nil {
		return 0, err
	}

	r, err := m.mem.AddKexecSegment(d)
	if err != nil {
		return 0, err
	}
	return r.Start, nil
}
<commit_msg>Revert "Revert "pkg/boot/multiboot: store kernel as ReaderAt not filename""<commit_after>// Copyright 2018-2019 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package multiboot implements bootloading multiboot kernels as defined by
// https://www.gnu.org/software/grub/manual/multiboot/multiboot.html.
//
// Package multiboot crafts kexec segments that can be used with the kexec_load
// system call.
package multiboot

import (
	"bytes"
	"debug/elf"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/u-root/u-root/pkg/boot/ibft"
	"github.com/u-root/u-root/pkg/boot/kexec"
	"github.com/u-root/u-root/pkg/boot/multiboot/internal/trampoline"
	"github.com/u-root/u-root/pkg/ubinary"
	"github.com/u-root/u-root/pkg/uio"
)

const bootloader = "u-root kexec"

// multiboot defines parameters for working with multiboot kernels.
type multiboot struct {
	mem kexec.Memory

	kernel  io.ReaderAt
	modules []string

	cmdLine    string
	bootloader string

	// trampoline is a path to an executable blob, which contains a trampoline segment.
	// Trampoline sets machine to a specific state defined by multiboot v1 spec.
	// https://www.gnu.org/software/grub/manual/multiboot/multiboot.html#Machine-state.
	trampoline string

	header header

	// infoAddr is a pointer to multiboot info.
	infoAddr uintptr
	// kernelEntry is a pointer to entry point of kernel.
	kernelEntry uintptr
	// EntryPoint is a pointer to trampoline.
	entryPoint uintptr

	info          info
	loadedModules []module
}

var (
	rangeTypes = map[kexec.RangeType]uint32{
		kexec.RangeRAM:      1,
		kexec.RangeDefault:  2,
		kexec.RangeACPI:     3,
		kexec.RangeNVS:      4,
		kexec.RangeReserved: 2,
	}
)

var sizeofMemoryMap = uint(binary.Size(MemoryMap{}))

// MemoryMap represents a reserved range of memory passed via the multiboot Info header.
type MemoryMap struct {
	// Size is the size of the associated structure in bytes.
	Size uint32
	// BaseAddr is the starting address.
	BaseAddr uint64
	// Length is the size of the memory region in bytes.
	Length uint64
	// Type is the variety of address range represented.
	Type uint32
}

// String returns a readable representation of a MemoryMap entry.
func (m MemoryMap) String() string {
	return fmt.Sprintf("[0x%x, 0x%x) (len: 0x%x, size: 0x%x, type: %d)", m.BaseAddr, m.BaseAddr+m.Length, m.Length, m.Size, m.Type)
}

type memoryMaps []MemoryMap

// String returns a new-line-separated representation of the entire memory map.
func (m memoryMaps) String() string {
	var s []string
	for _, mm := range m {
		s = append(s, mm.String())
	}
	return strings.Join(s, "\n")
}

// Probe checks if file is multiboot v1 kernel.
func Probe(file string) error {
	b, err := readFile(file)
	if err != nil {
		return err
	}
	kernel := &kernelReader{buf: b}
	_, err = parseHeader(kernel)
	return err
}

// newMB returns a new multiboot instance.
func newMB(kernel io.ReaderAt, cmdLine string, modules []string) (*multiboot, error) {
	// Trampoline should be a part of current binary.
	p, err := os.Executable()
	if err != nil {
		return nil, fmt.Errorf("cannot find current executable path: %v", err)
	}
	trampoline, err := filepath.EvalSymlinks(p)
	if err != nil {
		return nil, fmt.Errorf("cannot eval symlinks for %v: %v", p, err)
	}

	return &multiboot{
		kernel:     kernel,
		modules:    modules,
		cmdLine:    cmdLine,
		trampoline: trampoline,
		bootloader: bootloader,
		mem:        kexec.Memory{},
	}, nil
}

// Load parses and loads a multiboot kernel `file` using kexec_load.
//
// Each module is a path followed by optional command-line arguments, e.g.
// []string{"./module arg1 arg2", "./module2 arg3 arg4"}.
//
// debug turns on debug logging.
//
// Load can set up an arbitrary number of modules, and takes care of the
// multiboot info structure, including the memory map.
//
// After Load is called, kexec.Reboot() is ready to be called any time to stop
// Linux and execute the loaded kernel.
func Load(debug bool, file, cmdline string, modules []string, ibft *ibft.IBFT) error {
	b, err := readFile(file)
	if err != nil {
		return err
	}
	kernel := kernelReader{buf: b}
	m, err := newMB(kernel, cmdline, modules)
	if err != nil {
		return err
	}
	if err := m.load(debug, ibft); err != nil {
		return err
	}
	if err := kexec.Load(m.entryPoint, m.mem.Segments, 0); err != nil {
		return fmt.Errorf("kexec.Load() error: %v", err)
	}
	return nil
}

// load loads and parses multiboot information from m.kernel.
func (m *multiboot) load(debug bool, ibft *ibft.IBFT) error {
	var err error
	log.Println("Parsing multiboot header")
	if m.header, err = parseHeader(uio.Reader(m.kernel)); err != nil {
		return fmt.Errorf("error parsing headers: %v", err)
	}

	log.Printf("Getting kernel entry point")
	if m.kernelEntry, err = getEntryPoint(m.kernel); err != nil {
		return fmt.Errorf("error getting kernel entry point: %v", err)
	}
	log.Printf("Kernel entry point at %#x", m.kernelEntry)

	log.Printf("Parsing ELF segments")
	if err := m.mem.LoadElfSegments(m.kernel); err != nil {
		return fmt.Errorf("error loading ELF segments: %v", err)
	}

	log.Printf("Parsing memory map")
	if err := m.mem.ParseMemoryMap(); err != nil {
		return fmt.Errorf("error parsing memory map: %v", err)
	}

	// Insert the iBFT now, since nothing else has been allocated and this
	// is the most restricted allocation we're gonna have to make.
	if ibft != nil {
		ibuf := ibft.Marshal()

		// The iBFT may sit between 512K and 1M in physical memory. The
		// loaded OS finds it by scanning this region.
		allowedRange := kexec.Range{
			Start: 0x80000,
			Size:  0x80000,
		}
		r, err := m.mem.ReservePhys(uint(len(ibuf)), allowedRange)
		if err != nil {
			return fmt.Errorf("reserving space for the iBFT in %s failed: %v", allowedRange, err)
		}
		log.Printf("iBFT was allocated at %s: %#v", r, ibft)
		m.mem.Segments.Insert(kexec.NewSegment(ibuf, r))
	}

	log.Printf("Preparing multiboot info")
	if m.infoAddr, err = m.addInfo(); err != nil {
		return fmt.Errorf("error preparing multiboot info: %v", err)
	}

	log.Printf("Adding trampoline")
	if m.entryPoint, err = m.addTrampoline(); err != nil {
		return fmt.Errorf("error adding trampoline: %v", err)
	}
	log.Printf("Trampoline entry point at %#x", m.entryPoint)

	if debug {
		info, err := m.description()
		if err != nil {
			log.Printf("%v cannot create debug info: %v", DebugPrefix, err)
		}
		log.Printf("%v %v", DebugPrefix, info)
	}

	return nil
}

func getEntryPoint(r io.ReaderAt) (uintptr, error) {
	f, err := elf.NewFile(r)
	if err != nil {
		return 0, err
	}
	return uintptr(f.Entry), err
}

func (m *multiboot) addInfo() (addr uintptr, err error) {
	iw, err := m.newMultibootInfo()
	if err != nil {
		return 0, err
	}
	infoSize, err := iw.size()
	if err != nil {
		return 0, err
	}

	r, err := m.mem.FindSpace(infoSize)
	if err != nil {
		return 0, err
	}

	d, err := iw.marshal(r.Start)
	if err != nil {
		return 0, err
	}
	m.info = iw.info

	m.mem.Segments.Insert(kexec.NewSegment(d, r))
	return r.Start, nil
}

func (m multiboot) memoryMap() memoryMaps {
	var ret memoryMaps
	for _, r := range m.mem.Phys {
		typ, ok := rangeTypes[r.Type]
		if !ok {
			typ = rangeTypes[kexec.RangeDefault]
		}
		v := MemoryMap{
			// Size is really used for skipping to the next pair.
			Size:     uint32(sizeofMemoryMap) - 4,
			BaseAddr: uint64(r.Start),
			Length:   uint64(r.Size),
			Type:     typ,
		}
		ret = append(ret, v)
	}
	return ret
}

func (m *multiboot) addMmap() (addr uintptr, size uint, err error) {
	mmap := m.memoryMap()
	log.Printf("Memory map:\n%s", mmap)
	d, err := mmap.marshal()
	if err != nil {
		return 0, 0, err
	}
	r, err := m.mem.AddKexecSegment(d)
	if err != nil {
		return 0, 0, err
	}
	return r.Start, uint(len(mmap)) * sizeofMemoryMap, nil
}

func (m multiboot) memoryBoundaries() (lower, upper uint32) {
	const M1 = 1048576
	const K640 = 640 * 1024
	for _, r := range m.mem.Phys {
		if r.Type != kexec.RangeRAM {
			continue
		}
		end := uint32(r.Start) + uint32(r.Size)
		// Lower memory starts at address 0, and upper memory starts at address 1 megabyte.
		// The maximum possible value for lower memory is 640 kilobytes.
		// The value returned for upper memory is maximally the address of the first upper memory hole minus 1 megabyte.
		// It is not guaranteed to be this value.
		if r.Start <= K640 && end > lower {
			lower = end
		}
		if r.Start <= M1 && end > upper+M1 {
			upper = end - M1
		}
	}
	return
}

func min(a, b uint32) uint32 {
	if a < b {
		return a
	}
	return b
}

func (m *multiboot) newMultibootInfo() (*infoWrapper, error) {
	mmapAddr, mmapSize, err := m.addMmap()
	if err != nil {
		return nil, err
	}
	var inf info
	if m.header.Flags&flagHeaderMemoryInfo != 0 {
		lower, upper := m.memoryBoundaries()
		inf = info{
			Flags:      flagInfoMemMap | flagInfoMemory,
			MemLower:   min(uint32(lower>>10), 0xFFFFFFFF),
			MemUpper:   min(uint32(upper>>10), 0xFFFFFFFF),
			MmapLength: uint32(mmapSize),
			MmapAddr:   uint32(mmapAddr),
		}
	}

	if len(m.modules) > 0 {
		modAddr, err := m.addModules()
		if err != nil {
			return nil, err
		}
		inf.Flags |= flagInfoMods
		inf.ModsAddr = uint32(modAddr)
		inf.ModsCount = uint32(len(m.modules))
	}

	inf.CmdLine = sizeofInfo
	inf.BootLoaderName = sizeofInfo + uint32(len(m.cmdLine)) + 1
	inf.Flags |= flagInfoCmdLine | flagInfoBootLoaderName
	return &infoWrapper{
		info:           inf,
		CmdLine:        m.cmdLine,
		BootLoaderName: m.bootloader,
	}, nil
}

// marshal writes out the exact bytes expected by the multiboot info header
// specified in
// https://www.gnu.org/software/grub/manual/multiboot/multiboot.html#Boot-information-format.
func (m memoryMaps) marshal() ([]byte, error) {
	buf := bytes.Buffer{}
	err := binary.Write(&buf, ubinary.NativeEndian, m)
	return buf.Bytes(), err
}

func (m *multiboot) addTrampoline() (entry uintptr, err error) {
	// Trampoline setups the machine registers to desired state
	// and executes the loaded kernel.
	d, err := trampoline.Setup(m.trampoline, m.infoAddr, m.kernelEntry)
	if err != nil {
		return 0, err
	}

	r, err := m.mem.AddKexecSegment(d)
	if err != nil {
		return 0, err
	}
	return r.Start, nil
}
<|endoftext|>


<commit_before>package template

import (
	"fmt"
	html "html/template"
	"sort"
	text "text/template"
	"text/template/parse"
)

type Validator struct {
	allowRangeNode    bool
	allowTemplateNode bool
	maxDepth          int
}

type ValidatorOption func(*Validator)

func AllowRangeNode(b bool) ValidatorOption {
	return func(v *Validator) {
		v.allowRangeNode = b
	}
}

func AllowTemplateNode(b bool) ValidatorOption {
	return func(v *Validator) {
		v.allowTemplateNode = b
	}
}

func MaxDepth(d int) ValidatorOption {
	return func(v *Validator) {
		v.maxDepth = d
	}
}

func NewValidator(opts ...ValidatorOption) *Validator {
	v := &Validator{}
	for _, opt := range opts {
		opt(v)
	}
	return v
}

func (v *Validator) ValidateTextTemplate(template *text.Template) error {
	tpls := template.Templates()
	sort.Slice(tpls, func(i, j int) bool {
		return tpls[i].Name() < tpls[j].Name()
	})

	for _, tpl := range tpls {
		if err := v.validateTree(tpl.Tree); err != nil {
			return err
		}
	}
	return nil
}

func (v *Validator) ValidateHTMLTemplate(template *html.Template) error {
	tpls := template.Templates()
	sort.Slice(tpls, func(i, j int) bool {
		return tpls[i].Name() < tpls[j].Name()
	})

	for _, tpl := range tpls {
		if err := v.validateTree(tpl.Tree); err != nil {
			return err
		}
	}
	return nil
}

func (v *Validator) validateTree(tree *parse.Tree) (err error) {
	validateFn := func(n parse.Node, depth int) (cont bool) {
		maxDepth := v.maxDepth
		if maxDepth == 0 {
			maxDepth = 4
		}
		if depth > maxDepth {
			err = fmt.Errorf("%s: template nested too deep", formatLocation(tree, n))
		} else {
			switch n := n.(type) {
			case *parse.IfNode, *parse.ListNode, *parse.ActionNode, *parse.TextNode:
				break
			case *parse.PipeNode:
				if len(n.Decl) > 0 {
					err = fmt.Errorf("%s: declaration is forbidden", formatLocation(tree, n))
				} else if len(n.Cmds) > 1 {
					err = fmt.Errorf("%s: pipeline is forbidden", formatLocation(tree, n))
				}
			case *parse.CommandNode:
				for _, arg := range n.Args {
					if ident, ok := arg.(*parse.IdentifierNode); ok {
						if !checkIdentifier(ident.Ident) {
							err = fmt.Errorf("%s: forbidden identifier %s", formatLocation(tree, n), ident.Ident)
							break
						}
					}
				}
			case *parse.RangeNode:
				if v.allowRangeNode {
					break
				} else {
					err = fmt.Errorf("%s: forbidden construct %T", formatLocation(tree, n), n)
				}
			case *parse.TemplateNode:
				if v.allowTemplateNode {
					break
				} else {
					err = fmt.Errorf("%s: forbidden construct %T", formatLocation(tree, n), n)
				}
			default:
				err = fmt.Errorf("%s: forbidden construct %T", formatLocation(tree, n), n)
			}
		}

		return err == nil
	}

	traverseTree(tree, validateFn)
	return
}

var badIdentifiers = []string{
	"print",
	"printf",
	"println",
}

func checkIdentifier(id string) bool {
	for _, badID := range badIdentifiers {
		if id == badID {
			return false
		}
	}
	return true
}

func formatLocation(tree *parse.Tree, n parse.Node) string {
	location, _ := tree.ErrorContext(n)
	return location
}

func traverseTree(tree *parse.Tree, fn func(n parse.Node, depth int) (cont bool)) {
	var visit func(n parse.Node, depth int) (cont bool)
	visitBranch := func(n *parse.BranchNode, depth int) (cont bool) {
		if cont = visit(n.Pipe, depth); !cont {
			return
		}
		if cont = visit(n.List, depth); !cont {
			return
		}
		if n.ElseList != nil {
			if cont = visit(n.ElseList, depth); !cont {
				return false
			}
		}
		return
	}
	visit = func(n parse.Node, depth int) (cont bool) {
		cont = fn(n, depth)
		if !cont {
			return
		}

		switch n := n.(type) {
		case *parse.IfNode:
			cont = visitBranch(&n.BranchNode, depth)
		case *parse.RangeNode:
			cont = visitBranch(&n.BranchNode, depth)
		case *parse.WithNode:
			cont = visitBranch(&n.BranchNode, depth)
		case *parse.ListNode:
			for _, n := range n.Nodes {
				if cont = visit(n, depth+1); !cont {
					break
				}
			}
		case *parse.ActionNode:
			cont = visit(n.Pipe, depth)
		case *parse.PipeNode:
			for _, cmd := range n.Cmds {
				if cont = visit(cmd, depth); !cont {
					break
				}
			}
		case *parse.CommandNode:
			for _, arg := range n.Args {
				if pipe, ok := arg.(*parse.PipeNode); ok {
					if cont = visit(pipe, depth+1); !cont {
						break
					}
				}
			}
		case *parse.TemplateNode, *parse.TextNode:
			break
		default:
			panic("unknown node type")
		}
		return
	}
	visit(tree.Root, 0)
}
<commit_msg>Allow declaration in template<commit_after>package template

import (
	"fmt"
	html "html/template"
	"sort"
	text "text/template"
	"text/template/parse"
)

type Validator struct {
	allowRangeNode    bool
	allowTemplateNode bool
	allowDeclaration  bool
	maxDepth          int
}

type ValidatorOption func(*Validator)

func AllowRangeNode(b bool) ValidatorOption {
	return func(v *Validator) {
		v.allowRangeNode = b
	}
}

func AllowTemplateNode(b bool) ValidatorOption {
	return func(v *Validator) {
		v.allowTemplateNode = b
	}
}

func AllowDeclaration(b bool) ValidatorOption {
	return func(v *Validator) {
		v.allowDeclaration = b
	}
}

func MaxDepth(d int) ValidatorOption {
	return func(v *Validator) {
		v.maxDepth = d
	}
}

func NewValidator(opts ...ValidatorOption) *Validator {
	v := &Validator{}
	for _, opt := range opts {
		opt(v)
	}
	return v
}

func (v *Validator) ValidateTextTemplate(template *text.Template) error {
	tpls := template.Templates()
	sort.Slice(tpls, func(i, j int) bool {
		return tpls[i].Name() < tpls[j].Name()
	})

	for _, tpl := range tpls {
		if err := v.validateTree(tpl.Tree); err != nil {
			return err
		}
	}
	return nil
}

func (v *Validator) ValidateHTMLTemplate(template *html.Template) error {
	tpls := template.Templates()
	sort.Slice(tpls, func(i, j int) bool {
		return tpls[i].Name() < tpls[j].Name()
	})

	for _, tpl := range tpls {
		if err := v.validateTree(tpl.Tree); err != nil {
			return err
		}
	}
	return nil
}

func (v *Validator) validateTree(tree *parse.Tree) (err error) {
	validateFn := func(n parse.Node, depth int) (cont bool) {
		maxDepth := v.maxDepth
		if maxDepth == 0 {
			maxDepth = 4
		}
		if depth > maxDepth {
			err = fmt.Errorf("%s: template nested too deep", formatLocation(tree, n))
		} else {
			switch n := n.(type) {
			case *parse.IfNode, *parse.ListNode, *parse.ActionNode, *parse.TextNode:
				break
			case *parse.PipeNode:
				if len(n.Decl) > 0 && !v.allowDeclaration {
					err = fmt.Errorf("%s: declaration is forbidden", formatLocation(tree, n))
				} else if len(n.Cmds) > 1 {
					err = fmt.Errorf("%s: pipeline is forbidden", formatLocation(tree, n))
				}
			case *parse.CommandNode:
				for _, arg := range n.Args {
					if ident, ok := arg.(*parse.IdentifierNode); ok {
						if !checkIdentifier(ident.Ident) {
							err = fmt.Errorf("%s: forbidden identifier %s", formatLocation(tree, n), ident.Ident)
							break
						}
					}
				}
			case *parse.RangeNode:
				if v.allowRangeNode {
					break
				} else {
					err = fmt.Errorf("%s: forbidden construct %T", formatLocation(tree, n), n)
				}
			case *parse.TemplateNode:
				if v.allowTemplateNode {
					break
				} else {
					err = fmt.Errorf("%s: forbidden construct %T", formatLocation(tree, n), n)
				}
			default:
				err = fmt.Errorf("%s: forbidden construct %T", formatLocation(tree, n), n)
			}
		}

		return err == nil
	}

	traverseTree(tree, validateFn)
	return
}

var badIdentifiers = []string{
	"print",
	"printf",
	"println",
}

func checkIdentifier(id string) bool {
	for _, badID := range badIdentifiers {
		if id == badID {
			return false
		}
	}
	return true
}

func formatLocation(tree *parse.Tree, n parse.Node) string {
	location, _ := tree.ErrorContext(n)
	return location
}

func traverseTree(tree *parse.Tree, fn func(n parse.Node, depth int) (cont bool)) {
	var visit func(n parse.Node, depth int) (cont bool)
	visitBranch := func(n *parse.BranchNode, depth int) (cont bool) {
		if cont = visit(n.Pipe, depth); !cont {
			return
		}
		if cont = visit(n.List, depth); !cont {
			return
		}
		if n.ElseList != nil {
			if cont = visit(n.ElseList, depth); !cont {
				return false
			}
		}
		return
	}
	visit = func(n parse.Node, depth int) (cont bool) {
		cont = fn(n, depth)
		if !cont {
			return
		}

		switch n := n.(type) {
		case *parse.IfNode:
			cont = visitBranch(&n.BranchNode, depth)
		case *parse.RangeNode:
			cont = visitBranch(&n.BranchNode, depth)
		case *parse.WithNode:
			cont = visitBranch(&n.BranchNode, depth)
		case *parse.ListNode:
			for _, n := range n.Nodes {
				if cont = visit(n, depth+1); !cont {
					break
				}
			}
		case *parse.ActionNode:
			cont = visit(n.Pipe, depth)
		case *parse.PipeNode:
			for _, cmd := range n.Cmds {
				if cont = visit(cmd, depth); !cont {
					break
				}
			}
		case *parse.CommandNode:
			for _, arg := range n.Args {
				if pipe, ok := arg.(*parse.PipeNode); ok {
					if cont = visit(pipe, depth+1); !cont {
						break
					}
				}
			}
		case *parse.TemplateNode, *parse.TextNode:
			break
		default:
			panic("unknown node type")
		}
		return
	}
	visit(tree.Root, 0)
}
<|endoftext|>


<commit_before>//go:build linux || freebsd || darwin
// +build linux freebsd darwin

package directory // import "github.com/docker/docker/pkg/directory"

import (
	"context"
	"os"
	"path/filepath"
	"syscall"
)

// Size walks a directory tree and returns its total size in bytes.
func Size(ctx context.Context, dir string) (size int64, err error) {
	data := make(map[uint64]struct{})
	err = filepath.Walk(dir, func(d string, fileInfo os.FileInfo, err error) error {
		if err != nil {
			// if dir does not exist, Size() returns the error.
			// if dir/x disappeared while walking, Size() ignores dir/x.
			if os.IsNotExist(err) && d != dir {
				return nil
			}
			return err
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		// Ignore directory sizes
		if fileInfo == nil {
			return nil
		}

		s := fileInfo.Size()
		if fileInfo.IsDir() || s == 0 {
			return nil
		}

		// Check inode to handle hard links correctly
		inode := fileInfo.Sys().(*syscall.Stat_t).Ino
		// inode is not a uint64 on all platforms. Cast it to avoid issues.
		if _, exists := data[inode]; exists {
			return nil
		}
		// inode is not a uint64 on all platforms. Cast it to avoid issues.
		data[inode] = struct{}{}

		size += s

		return nil
	})
	return
}
<commit_msg>pkg/directory: Size(): add back type-casts to account for platform differences<commit_after>//go:build linux || freebsd || darwin
// +build linux freebsd darwin

package directory // import "github.com/docker/docker/pkg/directory"

import (
	"context"
	"os"
	"path/filepath"
	"syscall"
)

// Size walks a directory tree and returns its total size in bytes.
func Size(ctx context.Context, dir string) (size int64, err error) {
	data := make(map[uint64]struct{})
	err = filepath.Walk(dir, func(d string, fileInfo os.FileInfo, err error) error {
		if err != nil {
			// if dir does not exist, Size() returns the error.
			// if dir/x disappeared while walking, Size() ignores dir/x.
			if os.IsNotExist(err) && d != dir {
				return nil
			}
			return err
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
		}

		// Ignore directory sizes
		if fileInfo == nil {
			return nil
		}

		s := fileInfo.Size()
		if fileInfo.IsDir() || s == 0 {
			return nil
		}

		// Check inode to handle hard links correctly
		inode := fileInfo.Sys().(*syscall.Stat_t).Ino
		//nolint:unconvert // inode is not an uint64 on all platforms.
		if _, exists := data[uint64(inode)]; exists {
			return nil
		}

		data[uint64(inode)] = struct{}{} //nolint:unconvert // inode is not an uint64 on all platforms.

		size += s

		return nil
	})
	return
}
<|endoftext|>


<commit_before>//
// daemon.go
//
package main

import (
  "bufio"
  "log"
  "net"
  "strconv"
  "strings"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store *ArticleStore
  api *SRNdAPI
  database *Database
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  feeds map[NNTPConnection]bool
  infeed chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  feed := NNTPConnection{conn, bufio.NewReader(conn), inbound, self.debug, new(ConnectionInfo), policy, make(chan *NNTPMessage)}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self *NNTPDaemon) Run() {
  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return
  }
  defer self.listener.Close()

  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  self.syncAllFeeds()

  // start accepting incoming connections
  go self.mainloop()

  // start api
  go self.api.Mainloop()

  // loop over messages
  for {
    message := <- self.infeed
    // load message
    nntp := self.store.GetMessage(message, false)
    self.store.StoreArticle(nntp.Newsgroup, nntp.MessageID)
    // send to all outfeeds
    if nntp != nil {
      for feed , use := range self.feeds {
        if use {
          feed.send <- nntp
        }
      }
    }
  }
}

func (self *NNTPDaemon) syncAllFeeds() {
  var err error
  if self.sync_on_start {
    log.Println("sync all feeds")
    self.store.IterateAllArticles(func(messageID string) bool {
      msg := self.store.GetMessage(messageID, false)
      if msg != nil {
        for feed, use := range self.feeds {
            if use {
              err = feed.SendMessage(msg, self)
            }
        }
      }
      return false
    })
    log.Println("sync done")
  }

}

func (self *NNTPDaemon) mainloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunConnection(nntp)
  }
}

func (self *NNTPDaemon) RunConnection(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Println("SRNd NNTPD bound at", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {
  CheckConfig()
  log.Println("load config")
  self.conf = ReadConf()
  if self.conf == nil {
    log.Println("cannot load config")
    return false
  }
  self.infeed = make(chan string, 200)
  self.feeds = make(map[NNTPConnection]bool)

  self.database = new(Database)

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  err := self.database.Init(db_host, db_port, db_user, db_passwd)
  if err != nil {
    log.Println("failed to initialize database", err)
    return false
  }

  self.store = new(ArticleStore)
  self.store.directory = self.conf.store["base_dir"]
  self.store.Init()

  self.api = new(SRNdAPI)
  self.api.Init(self)

  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  if self.sync_on_start {
    log.Println("sync on start")
  }
  self.bind_addr = self.conf.daemon["bind"]
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  return true
}
<commit_msg>add sleep between reconnects so stuff does not spam<commit_after>//
// daemon.go
//
package main

import (
  "bufio"
  "log"
  "net"
  "strconv"
  "strings"
  "time"
)

type NNTPDaemon struct {
  instance_name string
  bind_addr string
  conf *SRNdConfig
  store *ArticleStore
  api *SRNdAPI
  database *Database
  listener net.Listener
  debug bool
  sync_on_start bool
  running bool
  feeds map[NNTPConnection]bool
  infeed chan string
}

func (self *NNTPDaemon) End() {
  self.listener.Close()
}

// register a new connection
// can be either inbound or outbound
func (self *NNTPDaemon) newConnection(conn net.Conn, inbound bool, policy *FeedPolicy) NNTPConnection {
  feed := NNTPConnection{conn, bufio.NewReader(conn), inbound, self.debug, new(ConnectionInfo), policy, make(chan *NNTPMessage)}
  self.feeds[feed] = ! inbound
  return feed
}

func (self *NNTPDaemon) persistFeed(conf FeedConfig) {
  for {
    if self.running {

      var conn net.Conn
      var err error
      proxy_type := strings.ToLower(conf.proxy_type)

      if proxy_type ==  "" || proxy_type == "none" {
        // connect out without proxy
        log.Println("dial out to ", conf.addr)
        conn, err = net.Dial("tcp", conf.addr)
        if err != nil {
          log.Println("cannot connect to outfeed", conf.addr, err)
          continue
        }
      } else if proxy_type == "socks4a" {
        // connect via socks4a
        log.Println("dial out via proxy", conf.proxy_addr)
        conn, err = net.Dial("tcp", conf.proxy_addr)
        if err != nil {
          log.Println("cannot connect to proxy", conf.proxy_addr)
          continue
        }
        // generate request
        idx := strings.LastIndex(conf.addr, ":")
        if idx == -1 {
          log.Fatal("invalid outfeed address")
        }
        var port uint64
        addr := conf.addr[:idx]
        port, err = strconv.ParseUint(conf.addr[idx+1:], 10, 16)
        if port >= 25536 {
          log.Fatal("bad proxy port" , port)
        }
        var proxy_port uint16
        proxy_port = uint16(port)
        proxy_ident := "srndv2"
        req_len := len(addr) + 1 + len(proxy_ident) + 1 + 8

        req := make([]byte, req_len)
        // pack request
        req[0] = '\x04'
        req[1] = '\x01'
        req[2] = byte(proxy_port & 0xff00 >> 8)
        req[3] = byte(proxy_port & 0x00ff)
        req[7] = '\x01'
        idx = 8

        proxy_ident_b := []byte(proxy_ident)
        addr_b := []byte(addr)

        var bi int
        for bi = range proxy_ident_b {
          req[idx] = proxy_ident_b[bi]
          idx += 1
        }
        idx += 1
        for bi = range addr_b {
          req[idx] = addr_b[bi]
          idx += 1
        }

        // send request
        conn.Write(req)
        resp := make([]byte, 8)

        // receive response
        conn.Read(resp)
        if resp[1] == '\x5a' {
          // success
          log.Println("connected to", conf.addr)
        } else {
          log.Println("failed to connect to", conf.addr)
					time.Sleep(5)
          continue
        }
      }
      policy := &conf.policy
      nntp := self.newConnection(conn, false, policy)
      nntp.HandleOutbound(self)
      log.Println("remove outfeed")
      delete(self.feeds, nntp)
    }
  }
  time.Sleep(1 * time.Second)
}

// run daemon
func (self *NNTPDaemon) Run() {
  err := self.Bind()
  if err != nil {
    log.Println("failed to bind:", err)
    return
  }
  defer self.listener.Close()

  // we are now running
  self.running = true

  // persist outfeeds
  for idx := range self.conf.feeds {
    go self.persistFeed(self.conf.feeds[idx])
  }

  self.syncAllFeeds()

  // start accepting incoming connections
  go self.mainloop()

  // start api
  go self.api.Mainloop()

  // loop over messages
  for {
    message := <- self.infeed
    // load message
    nntp := self.store.GetMessage(message, false)
    self.store.StoreArticle(nntp.Newsgroup, nntp.MessageID)
    // send to all outfeeds
    if nntp != nil {
      for feed , use := range self.feeds {
        if use {
          feed.send <- nntp
        }
      }
    }
  }
}

func (self *NNTPDaemon) syncAllFeeds() {
  var err error
  if self.sync_on_start {
    log.Println("sync all feeds")
    self.store.IterateAllArticles(func(messageID string) bool {
      msg := self.store.GetMessage(messageID, false)
      if msg != nil {
        for feed, use := range self.feeds {
            if use {
              err = feed.SendMessage(msg, self)
            }
        }
      }
      return false
    })
    log.Println("sync done")
  }

}

func (self *NNTPDaemon) mainloop() {
  for {
    // accept
    conn, err := self.listener.Accept()
    if err != nil {
      log.Fatal(err)
    }
    // make a new inbound nntp connection handler
    nntp := self.newConnection(conn, true, nil)
    go self.RunConnection(nntp)
  }
}

func (self *NNTPDaemon) RunConnection(nntp NNTPConnection) {
  nntp.HandleInbound(self)
  delete(self.feeds, nntp)
}

// bind to address
func (self *NNTPDaemon) Bind() error {
  listener , err := net.Listen("tcp", self.bind_addr)
  if err != nil {
    log.Println("failed to bind to", self.bind_addr, err)
    return err
  }
  self.listener = listener
  log.Println("SRNd NNTPD bound at", listener.Addr())
  return nil
}

// load configuration
// bind to interface
func (self *NNTPDaemon) Init() bool {
  CheckConfig()
  log.Println("load config")
  self.conf = ReadConf()
  if self.conf == nil {
    log.Println("cannot load config")
    return false
  }
  self.infeed = make(chan string, 200)
  self.feeds = make(map[NNTPConnection]bool)

  self.database = new(Database)

  db_host := self.conf.database["host"]
  db_port := self.conf.database["port"]
  db_user := self.conf.database["user"]
  db_passwd := self.conf.database["password"]

  err := self.database.Init(db_host, db_port, db_user, db_passwd)
  if err != nil {
    log.Println("failed to initialize database", err)
    return false
  }

  self.store = new(ArticleStore)
  self.store.directory = self.conf.store["base_dir"]
  self.store.Init()

  self.api = new(SRNdAPI)
  self.api.Init(self)

  self.sync_on_start = self.conf.daemon["sync_on_start"] == "1"
  if self.sync_on_start {
    log.Println("sync on start")
  }
  self.bind_addr = self.conf.daemon["bind"]
  self.debug = self.conf.daemon["log"] == "debug"
  self.instance_name = self.conf.daemon["instance_name"]
  if self.debug {
    log.Println("debug mode activated")
  }

  return true
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Bitnami

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package webhook

import (
	"fmt"
	"log"
	"os"

	"bytes"
	"encoding/json"
	"net/http"
	"time"

	"github.com/bitnami-labs/kubewatch/config"
	kbEvent "github.com/bitnami-labs/kubewatch/pkg/event"
)

var webhookErrMsg = `
%s

You need to set Webhook url
using "--url/-u" or using environment variables:

export KW_WEBHOOK_URL=webhook_url

Command line flags will override environment variables

`

// Webhook handler implements handler.Handler interface,
// Notify event to Webhook channel
type Webhook struct {
	Url string
}

// WebhookMessage for messages
type WebhookMessage struct {
	EventMeta EventMeta `json:"eventmeta"`
	Message   string    `json:"message"`
	Time      time.Time `json:"time"`
}

// EventMeta containes the meta data about the event occurred
type EventMeta struct {
	Kind      string `json:"kind"`
	Name      string `json:"name"`
	Namespace string `json:"namespace"`
	Reason    string `json:"reason"`
}

// Init prepares Webhook configuration
func (m *Webhook) Init(c *config.Config) error {
	url := c.Handler.Webhook.Url

	if url == "" {
		url = os.Getenv("KW_WEBHOOK_URL")
	}

	m.Url = url

	return checkMissingWebhookVars(m)
}

// ObjectCreated calls notifyWebhook on event creation
func (m *Webhook) ObjectCreated(obj interface{}) {
	notifyWebhook(m, obj, "created")
}

// ObjectDeleted calls notifyWebhook on event creation
func (m *Webhook) ObjectDeleted(obj interface{}) {
	notifyWebhook(m, obj, "deleted")
}

// ObjectUpdated calls notifyWebhook on event creation
func (m *Webhook) ObjectUpdated(oldObj, newObj interface{}) {
	notifyWebhook(m, newObj, "updated")
}

// TestHandler tests the handler configurarion by sending test messages.
func (m *Webhook) TestHandler() {

	webhookMessage := &WebhookMessage{
		EventMeta: EventMeta{
			Kind:      "object",
			Name:      "Test",
			Namespace: "default",
			Reason:    "Tested",
		},
		Message: "Testing Handler Configuration. This is a Test message.",
		Time:    time.Now(),
	}

	err := postMessage(m.Url, webhookMessage)
	if err != nil {
		log.Printf("%s\n", err)
		return
	}

	log.Printf("Message successfully sent to %s at %s ", m.Url, time.Now())
}

func notifyWebhook(m *Webhook, obj interface{}, action string) {
	e := kbEvent.New(obj, action)

	webhookMessage := prepareWebhookMessage(e, m)

	err := postMessage(m.Url, webhookMessage)
	if err != nil {
		log.Printf("%s\n", err)
		return
	}

	log.Printf("Message successfully sent to %s at %s ", m.Url, time.Now())
}

func checkMissingWebhookVars(s *Webhook) error {
	if s.Url == "" {
		return fmt.Errorf(webhookErrMsg, "Missing Webhook url")
	}

	return nil
}

func prepareWebhookMessage(e kbEvent.Event, m *Webhook) *WebhookMessage {
	return &WebhookMessage{
		EventMeta: EventMeta{
			Kind:      e.Kind,
			Name:      e.Name,
			Namespace: e.Namespace,
			Reason:    e.Reason,
		},
		Message: e.Message(),
		Time:    time.Now(),
	}
}

func postMessage(url string, webhookMessage *WebhookMessage) error {
	message, err := json.Marshal(webhookMessage)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(message))
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")

	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Update pkg/handlers/webhook/webhook.go<commit_after>/*
Copyright 2018 Bitnami

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package webhook

import (
	"fmt"
	"log"
	"os"

	"bytes"
	"encoding/json"
	"net/http"
	"time"

	"github.com/bitnami-labs/kubewatch/config"
	kbEvent "github.com/bitnami-labs/kubewatch/pkg/event"
)

var webhookErrMsg = `
%s

You need to set Webhook url
using "--url/-u" or using environment variables:

export KW_WEBHOOK_URL=webhook_url

Command line flags will override environment variables

`

// Webhook handler implements handler.Handler interface,
// Notify event to Webhook channel
type Webhook struct {
	Url string
}

// WebhookMessage for messages
type WebhookMessage struct {
	EventMeta EventMeta `json:"eventmeta"`
	Text      string    `json:"text"`
	Time      time.Time `json:"time"`
}

// EventMeta containes the meta data about the event occurred
type EventMeta struct {
	Kind      string `json:"kind"`
	Name      string `json:"name"`
	Namespace string `json:"namespace"`
	Reason    string `json:"reason"`
}

// Init prepares Webhook configuration
func (m *Webhook) Init(c *config.Config) error {
	url := c.Handler.Webhook.Url

	if url == "" {
		url = os.Getenv("KW_WEBHOOK_URL")
	}

	m.Url = url

	return checkMissingWebhookVars(m)
}

// ObjectCreated calls notifyWebhook on event creation
func (m *Webhook) ObjectCreated(obj interface{}) {
	notifyWebhook(m, obj, "created")
}

// ObjectDeleted calls notifyWebhook on event creation
func (m *Webhook) ObjectDeleted(obj interface{}) {
	notifyWebhook(m, obj, "deleted")
}

// ObjectUpdated calls notifyWebhook on event creation
func (m *Webhook) ObjectUpdated(oldObj, newObj interface{}) {
	notifyWebhook(m, newObj, "updated")
}

// TestHandler tests the handler configurarion by sending test messages.
func (m *Webhook) TestHandler() {

	webhookMessage := &WebhookMessage{
		EventMeta: EventMeta{
			Kind:      "object",
			Name:      "Test",
			Namespace: "default",
			Reason:    "Tested",
		},
		Text: "Testing Handler Configuration. This is a Test message.",
		Time: time.Now(),
	}

	err := postMessage(m.Url, webhookMessage)
	if err != nil {
		log.Printf("%s\n", err)
		return
	}

	log.Printf("Message successfully sent to %s at %s ", m.Url, time.Now())
}

func notifyWebhook(m *Webhook, obj interface{}, action string) {
	e := kbEvent.New(obj, action)

	webhookMessage := prepareWebhookMessage(e, m)

	err := postMessage(m.Url, webhookMessage)
	if err != nil {
		log.Printf("%s\n", err)
		return
	}

	log.Printf("Message successfully sent to %s at %s ", m.Url, time.Now())
}

func checkMissingWebhookVars(s *Webhook) error {
	if s.Url == "" {
		return fmt.Errorf(webhookErrMsg, "Missing Webhook url")
	}

	return nil
}

func prepareWebhookMessage(e kbEvent.Event, m *Webhook) *WebhookMessage {
	return &WebhookMessage{
		EventMeta: EventMeta{
			Kind:      e.Kind,
			Name:      e.Name,
			Namespace: e.Namespace,
			Reason:    e.Reason,
		},
		Text: e.Message(),
		Time: time.Now(),
	}
}

func postMessage(url string, webhookMessage *WebhookMessage) error {
	message, err := json.Marshal(webhookMessage)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", url, bytes.NewBuffer(message))
	if err != nil {
		return err
	}
	req.Header.Add("Content-Type", "application/json")

	client := &http.Client{}
	_, err = client.Do(req)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"text/template"
	"time"
)

type Processor struct {
	Func func(page *Page, args []string)
	Desc string
}

// PreProcessors is a list of processor necessary to be executed beforehand to
// fill out information, which can be required by fellow pages
var PreProcessors = CommandList{"config", "rename", "ext", "directorify",
	"tags", "ignore"}

var Processors = map[string]*Processor{
	"inner-template": &Processor{
		ProcessInnerTemplate,
		"process content as a Go template",
	},
	"template": &Processor{
		ProcessTemplate,
		"put content in a template (by default in 'page' template)",
	},
	"markdown": &Processor{
		ProcessMarkdown,
		"process content as a markdown",
	},
	"rename": &Processor{
		ProcessRename,
		"rename resulting file (argument - pattern for renaming)",
	},
	"ext": &Processor{
		ProcessExt,
		"change extension",
	},
	"ignore": &Processor{
		ProcessIgnore,
		"ignore file",
	},
	"directorify": &Processor{
		ProcessDirectorify,
		"path/name.html -> path/name/index.html",
	},
	"external": &Processor{
		ProcessExternal,
		"run external command to process content (shortcut ':')",
	},
	"config": &Processor{
		ProcessConfig,
		"read config from content (separated by '----\\n')",
	},
	"tags": &Processor{
		ProcessTags,
		("generate tags pages for tags mentioned in page header " +
			"(argument - tag template)"),
	},
}

func ProcessorSummary() {
	keys := make([]string, 0)
	for k := range Processors {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		p := Processors[k]
		pre := ""
		if PreProcessors.MatchedIndex(Command(k)) != -1 {
			pre = "(preprocessor)"
		}
		fmt.Printf("%s %s\n\t%s\n", k, pre, p.Desc)
	}
}

func ProcessCommand(page *Page, cmd *Command) {
	c := string(*cmd)
	if strings.HasPrefix(c, ":") {
		c = "external " + c[1:]
	}
	bits := strings.Split(c, " ")
	processor := Processors[bits[0]]
	if processor == nil {
		errhandle(fmt.Errorf("processor '%s' not found", bits[0]))
	}
	processor.Func(page, bits[1:])
}

func ProcessInnerTemplate(page *Page, args []string) {
	t, err := page.Site.Template.Clone()
	errhandle(err)
	t, err = t.New("ad-hoc").Parse(page.Content())
	if err != nil {
		errhandle(fmt.Errorf("Page %s: %s", page.Source, err))
	}

	var buffer bytes.Buffer
	err = t.ExecuteTemplate(&buffer, "ad-hoc", page)
	if err != nil {
		errhandle(fmt.Errorf("Page %s: %s", page.Source, err))
	}

	page.SetContent(buffer.String())
}

func ProcessTemplate(page *Page, args []string) {
	if len(args) < 1 {
		errhandle(errors.New("'template' rule needs an argument"))
	}
	pagetype := args[0]

	var buffer bytes.Buffer
	err := page.Site.Template.ExecuteTemplate(&buffer, pagetype, page)
	if err != nil {
		errhandle(fmt.Errorf("%s: %s", page.Source, err))
	}

	page.SetContent(buffer.String())
}

func ProcessMarkdown(page *Page, args []string) {
	result := Markdown(page.Content())
	page.SetContent(result)
}

func ProcessRename(page *Page, args []string) {
	if len(args) < 1 {
		errhandle(errors.New("'rename' rule needs an argument"))
	}
	dest := strings.Replace(args[0], "*", "", -1)
	pattern := strings.Replace(page.Pattern, "*", "", -1)

	page.Path = strings.Replace(page.Path, pattern, dest, -1)
}

func ProcessExt(page *Page, args []string) {
	if len(args) < 1 {
		errhandle(errors.New(
			"'ext' rule requires an extension prefixed with dot"))
	}
	ext := filepath.Ext(page.Path)
	if ext == "" {
		page.Path = page.Path + args[0]
	} else {
		page.Path = strings.Replace(page.Path, ext, args[0], 1)
	}
}

func ProcessIgnore(page *Page, args []string) {
	page.state = StateIgnored
}

func ProcessDirectorify(page *Page, args []string) {
	if filepath.Base(page.Path) != "index.html" {
		page.Path = strings.Replace(page.Path, ".html", "/index.html", 1)
	}
}

func ProcessExternal(page *Page, args []string) {
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdin = strings.NewReader(page.Content())
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	out, err := cmd.Output()
	if err != nil {
		errhandle(fmt.Errorf("Error executing '%s': %s",
			strings.Join(args, " "), stderr.String()))
	}

	page.SetContent(string(out))
}

func ProcessConfig(page *Page, args []string) {
	parts := strings.SplitN(page.Content(), "----\n", 2)
	if len(parts) != 2 {
		// no configuration, well then...
		page.PageHeader = *NewPageHeader()
		return
	}

	page.PageHeader = *ParseHeader(parts[0])
	page.SetContent(parts[1])
}

func ProcessTags(page *Page, args []string) {
	if len(args) < 1 {
		errhandle(errors.New("'tags' rule needs an argument"))
	}

	if page.Tags == nil {
		return
	}

	site := page.Site

	for _, tag := range page.Tags {
		if !site.Pages.HasPage(func(inner *Page) bool {
			return inner.Title == tag
		}) {
			path := strings.Replace(args[0], "*", tag, 1)
			pattern, rule := site.Rules.MatchedRule(path)
			tagpage := &Page{
				PageHeader: PageHeader{Title: tag},
				Site:       site,
				Pattern:    pattern,
				Rule:       rule,
				Source:     path,
				Path:       path,
				// tags are never new, because they only depend on pages and
				// have not a bit of original content
				ModTime: time.Unix(0, 0),
			}
			page.Site.Pages = append(page.Site.Pages, tagpage)
			tagpage.peek()
		}
	}

}

// template utilities

var inventory = map[string]interface{}{}

func HasChanged(name string, value interface{}) bool {
	changed := true

	if inventory[name] == value {
		changed = false
	} else {
		inventory[name] = value
	}

	return changed
}

func Cut(value, begin, end string) (string, error) {
	bre, err := regexp.Compile(begin)
	if err != nil {
		return "", err
	}
	ere, err := regexp.Compile(end)
	if err != nil {
		return "", err
	}

	bloc := bre.FindIndex([]byte(value))
	eloc := ere.FindIndex([]byte(value))

	return value[bloc[1]:eloc[0]], nil
}

var funcMap = template.FuncMap{
	"changed": HasChanged,
	"cut":     Cut,
}
<commit_msg>always set working directory for external script<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"os/exec"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"text/template"
	"time"
)

type Processor struct {
	Func func(page *Page, args []string)
	Desc string
}

// PreProcessors is a list of processor necessary to be executed beforehand to
// fill out information, which can be required by fellow pages
var PreProcessors = CommandList{"config", "rename", "ext", "directorify",
	"tags", "ignore"}

var Processors = map[string]*Processor{
	"inner-template": &Processor{
		ProcessInnerTemplate,
		"process content as a Go template",
	},
	"template": &Processor{
		ProcessTemplate,
		"put content in a template (by default in 'page' template)",
	},
	"markdown": &Processor{
		ProcessMarkdown,
		"process content as a markdown",
	},
	"rename": &Processor{
		ProcessRename,
		"rename resulting file (argument - pattern for renaming)",
	},
	"ext": &Processor{
		ProcessExt,
		"change extension",
	},
	"ignore": &Processor{
		ProcessIgnore,
		"ignore file",
	},
	"directorify": &Processor{
		ProcessDirectorify,
		"path/name.html -> path/name/index.html",
	},
	"external": &Processor{
		ProcessExternal,
		"run external command to process content (shortcut ':')",
	},
	"config": &Processor{
		ProcessConfig,
		"read config from content (separated by '----\\n')",
	},
	"tags": &Processor{
		ProcessTags,
		("generate tags pages for tags mentioned in page header " +
			"(argument - tag template)"),
	},
}

func ProcessorSummary() {
	keys := make([]string, 0)
	for k := range Processors {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, k := range keys {
		p := Processors[k]
		pre := ""
		if PreProcessors.MatchedIndex(Command(k)) != -1 {
			pre = "(preprocessor)"
		}
		fmt.Printf("%s %s\n\t%s\n", k, pre, p.Desc)
	}
}

func ProcessCommand(page *Page, cmd *Command) {
	c := string(*cmd)
	if strings.HasPrefix(c, ":") {
		c = "external " + c[1:]
	}
	bits := strings.Split(c, " ")
	processor := Processors[bits[0]]
	if processor == nil {
		errhandle(fmt.Errorf("processor '%s' not found", bits[0]))
	}
	processor.Func(page, bits[1:])
}

func ProcessInnerTemplate(page *Page, args []string) {
	t, err := page.Site.Template.Clone()
	errhandle(err)
	t, err = t.New("ad-hoc").Parse(page.Content())
	if err != nil {
		errhandle(fmt.Errorf("Page %s: %s", page.Source, err))
	}

	var buffer bytes.Buffer
	err = t.ExecuteTemplate(&buffer, "ad-hoc", page)
	if err != nil {
		errhandle(fmt.Errorf("Page %s: %s", page.Source, err))
	}

	page.SetContent(buffer.String())
}

func ProcessTemplate(page *Page, args []string) {
	if len(args) < 1 {
		errhandle(errors.New("'template' rule needs an argument"))
	}
	pagetype := args[0]

	var buffer bytes.Buffer
	err := page.Site.Template.ExecuteTemplate(&buffer, pagetype, page)
	if err != nil {
		errhandle(fmt.Errorf("%s: %s", page.Source, err))
	}

	page.SetContent(buffer.String())
}

func ProcessMarkdown(page *Page, args []string) {
	result := Markdown(page.Content())
	page.SetContent(result)
}

func ProcessRename(page *Page, args []string) {
	if len(args) < 1 {
		errhandle(errors.New("'rename' rule needs an argument"))
	}
	dest := strings.Replace(args[0], "*", "", -1)
	pattern := strings.Replace(page.Pattern, "*", "", -1)

	page.Path = strings.Replace(page.Path, pattern, dest, -1)
}

func ProcessExt(page *Page, args []string) {
	if len(args) < 1 {
		errhandle(errors.New(
			"'ext' rule requires an extension prefixed with dot"))
	}
	ext := filepath.Ext(page.Path)
	if ext == "" {
		page.Path = page.Path + args[0]
	} else {
		page.Path = strings.Replace(page.Path, ext, args[0], 1)
	}
}

func ProcessIgnore(page *Page, args []string) {
	page.state = StateIgnored
}

func ProcessDirectorify(page *Page, args []string) {
	if filepath.Base(page.Path) != "index.html" {
		page.Path = strings.Replace(page.Path, ".html", "/index.html", 1)
	}
}

func ProcessExternal(page *Page, args []string) {
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdin = strings.NewReader(page.Content())
	cmd.Dir = page.Site.Source
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	out, err := cmd.Output()
	if err != nil {
		errhandle(fmt.Errorf("Error executing '%s': %s",
			strings.Join(args, " "), stderr.String()))
	}

	page.SetContent(string(out))
}

func ProcessConfig(page *Page, args []string) {
	parts := strings.SplitN(page.Content(), "----\n", 2)
	if len(parts) != 2 {
		// no configuration, well then...
		page.PageHeader = *NewPageHeader()
		return
	}

	page.PageHeader = *ParseHeader(parts[0])
	page.SetContent(parts[1])
}

func ProcessTags(page *Page, args []string) {
	if len(args) < 1 {
		errhandle(errors.New("'tags' rule needs an argument"))
	}

	if page.Tags == nil {
		return
	}

	site := page.Site

	for _, tag := range page.Tags {
		if !site.Pages.HasPage(func(inner *Page) bool {
			return inner.Title == tag
		}) {
			path := strings.Replace(args[0], "*", tag, 1)
			pattern, rule := site.Rules.MatchedRule(path)
			tagpage := &Page{
				PageHeader: PageHeader{Title: tag},
				Site:       site,
				Pattern:    pattern,
				Rule:       rule,
				Source:     path,
				Path:       path,
				// tags are never new, because they only depend on pages and
				// have not a bit of original content
				ModTime: time.Unix(0, 0),
			}
			page.Site.Pages = append(page.Site.Pages, tagpage)
			tagpage.peek()
		}
	}

}

// template utilities

var inventory = map[string]interface{}{}

func HasChanged(name string, value interface{}) bool {
	changed := true

	if inventory[name] == value {
		changed = false
	} else {
		inventory[name] = value
	}

	return changed
}

func Cut(value, begin, end string) (string, error) {
	bre, err := regexp.Compile(begin)
	if err != nil {
		return "", err
	}
	ere, err := regexp.Compile(end)
	if err != nil {
		return "", err
	}

	bloc := bre.FindIndex([]byte(value))
	eloc := ere.FindIndex([]byte(value))

	return value[bloc[1]:eloc[0]], nil
}

var funcMap = template.FuncMap{
	"changed": HasChanged,
	"cut":     Cut,
}
<|endoftext|>