<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package client provides a general purpose Omaha update client implementation.
package client

import (
	"errors"
	"fmt"
	"net/url"
	"time"

	"github.com/satori/go.uuid"

	"github.com/coreos/go-omaha/omaha"
)

const (
	defaultClientVersion = "go-omaha"

	// periodic update check and ping intervals
	pingFuzz     = 10 * time.Minute
	pingDelay    = 7 * time.Minute  // first check after 2-12 minutes
	pingInterval = 45 * time.Minute // check in every 40-50 minutes
)

// Client supports managing multiple apps using a single server.
type Client struct {
	apiClient     *httpClient
	apiEndpoint   string
	clientVersion string
	userID        string
	sessionID     string
	isMachine     bool
	sentPing      bool
	apps          map[string]*AppClient
}

// AppClient supports managing a single application.
type AppClient struct {
	*Client
	appID   string
	track   string
	version string
	oem     string
}

// New creates an omaha client for updating one or more applications.
// userID must be a persistent unique identifier of this update client.
func New(serverURL, userID string) (*Client, error) {
	if userID == "" {
		return nil, errors.New("omaha: empty user identifier")
	}

	c := &Client{
		apiClient:     newHTTPClient(),
		clientVersion: defaultClientVersion,
		userID:        userID,
		sessionID:     uuid.NewV4().String(),
		apps:          make(map[string]*AppClient),
	}

	if err := c.SetServerURL(serverURL); err != nil {
		return nil, err
	}

	return c, nil
}

// SetServerURL changes the Omaha server this client talks to.
// If the URL does not include a path component /v1/update/ is assumed.
func (c *Client) SetServerURL(serverURL string) error {
	u, err := url.Parse(serverURL)
	if err != nil {
		return fmt.Errorf("omaha: invalid server URL: %v", err)
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		return fmt.Errorf("omaha: invalid server protocol: %s", u)
	}
	if u.Host == "" {
		return fmt.Errorf("omaha: invalid server host: %s", u)
	}
	if u.Path == "" || u.Path == "/" {
		u.Path = "/v1/update/"
	}

	c.apiEndpoint = u.String()
	return nil
}

// SetClientVersion sets the identifier of this updater application.
// e.g. "update_engine-0.1.0".  Default is "go-omaha".
func (c *Client) SetClientVersion(clientVersion string) {
	c.clientVersion = clientVersion
}

// NextPing returns a timer channel that will fire when the next update
// check or ping should be sent.
func (c *Client) NextPing() <-chan time.Time {
	d := pingDelay
	if c.sentPing {
		d = pingInterval
	}
	return FuzzyAfter(d, pingFuzz)
}

// AppClient gets the application client for the given application ID.
func (c *Client) AppClient(appID string) (*AppClient, error) {
	if app, ok := c.apps[appID]; ok {
		return app, nil
	}

	return nil, fmt.Errorf("omaha: missing app client %q", appID)
}

// NewAppClient creates a new application client.
func (c *Client) NewAppClient(appID, appVersion string) (*AppClient, error) {
	if _, ok := c.apps[appID]; ok {
		return nil, fmt.Errorf("omaha: duplicate app client %q", appID)
	}

	ac := &AppClient{
		Client: c,
		appID:  appID,
	}
	c.apps[appID] = ac

	return ac, nil
}

// NewAppClient creates a single application client.
// Shorthand for New(serverURL, userID).NewAppClient(appID, appVersion).
func NewAppClient(serverURL, userID, appID, appVersion string) (*AppClient, error) {
	c, err := New(serverURL, userID)
	if err != nil {
		return nil, err
	}

	ac, err := c.NewAppClient(appID, appVersion)
	if err := ac.SetVersion(appVersion); err != nil {
		return nil, err
	}

	return ac, nil
}

// SetVersion changes the application version.
func (ac *AppClient) SetVersion(version string) error {
	if version == "" {
		return errors.New("omaha: empty application version")
	}

	ac.version = version
	return nil
}

// SetTrack sets the application update track or group.
// This is a update_engine/Core Update protocol extension.
func (ac *AppClient) SetTrack(track string) error {
	// Although track is an omaha extension and theoretically not required
	// our Core Update server requires track to be set to a valid id/name.
	// TODO: deprecate track and use the standard cohort protocol fields.
	if track == "" {
		return errors.New("omaha: empty application update track/group")
	}

	ac.track = track
	return nil
}

// SetOEM sets the application OEM name.
// This is a update_engine/Core Update protocol extension.
func (ac *AppClient) SetOEM(oem string) {
	ac.oem = oem
}

func (ac *AppClient) UpdateCheck() (*omaha.UpdateResponse, error) {
	req := ac.newReq()
	app := req.Apps[0]
	app.AddPing()
	app.AddUpdateCheck()

	ac.sentPing = true

	appResp, err := ac.doReq(ac.apiEndpoint, req)
	if err != nil {
		return nil, err
	}

	if appResp.Ping == nil {
		return nil, fmt.Errorf("omaha: ping status missing from response")
	}

	if appResp.Ping.Status != "ok" {
		return nil, fmt.Errorf("omaha: ping status %s", appResp.Ping.Status)
	}

	if appResp.UpdateCheck == nil {
		return nil, fmt.Errorf("omaha: update check missing from response")
	}

	if appResp.UpdateCheck.Status != omaha.UpdateOK {
		return nil, appResp.UpdateCheck.Status
	}

	return appResp.UpdateCheck, nil
}

func (ac *AppClient) Ping() error {
	req := ac.newReq()
	app := req.Apps[0]
	app.AddPing()

	ac.sentPing = true

	appResp, err := ac.doReq(ac.apiEndpoint, req)
	if err != nil {
		return err
	}

	if appResp.Ping == nil {
		return fmt.Errorf("omaha: ping status missing from response")
	}

	if appResp.Ping.Status != "ok" {
		return fmt.Errorf("omaha: ping status %s", appResp.Ping.Status)
	}

	return nil
}

func (ac *AppClient) Event(event *omaha.EventRequest) error {
	req := ac.newReq()
	app := req.Apps[0]
	app.Events = append(app.Events, event)

	appResp, err := ac.doReq(ac.apiEndpoint, req)
	if err != nil {
		return err
	}

	if len(appResp.Events) == 0 {
		return fmt.Errorf("omaha: event status missing from response")
	}

	if appResp.Events[0].Status != "ok" {
		return fmt.Errorf("omaha: event status %s", appResp.Events[0].Status)
	}

	return nil
}

func (ac *AppClient) newReq() *omaha.Request {
	req := omaha.NewRequest()
	req.Version = ac.clientVersion
	req.UserID = ac.userID
	req.SessionID = ac.sessionID
	if ac.isMachine {
		req.IsMachine = 1
	}

	app := req.AddApp(ac.appID, ac.version)
	app.Track = ac.track
	app.OEM = ac.oem

	// MachineID and BootID are non-standard fields used by CoreOS'
	// update_engine and Core Update. Copy their values from the
	// standard UserID and SessionID. Eventually the non-standard
	// fields should be deprecated.
	app.MachineID = req.UserID
	app.BootID = req.SessionID

	return req
}

func (ac *AppClient) doReq(url string, req *omaha.Request) (*omaha.AppResponse, error) {
	if len(req.Apps) != 1 {
		panic(fmt.Errorf("unexpected number of apps: %d", len(req.Apps)))
	}
	appID := req.Apps[0].ID
	resp, err := ac.apiClient.Omaha(url, req)
	if err != nil {
		return nil, err
	}

	appResp := resp.GetApp(appID)
	if appResp == nil {
		return nil, fmt.Errorf("omaha: app %s missing from response", appID)
	}

	if appResp.Status != omaha.AppOK {
		return nil, appResp.Status
	}

	return appResp, nil
}
<commit_msg>client: support runtime changes of app id<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package client provides a general purpose Omaha update client implementation.
package client

import (
	"errors"
	"fmt"
	"net/url"
	"time"

	"github.com/satori/go.uuid"

	"github.com/coreos/go-omaha/omaha"
)

const (
	defaultClientVersion = "go-omaha"

	// periodic update check and ping intervals
	pingFuzz     = 10 * time.Minute
	pingDelay    = 7 * time.Minute  // first check after 2-12 minutes
	pingInterval = 45 * time.Minute // check in every 40-50 minutes
)

// Client supports managing multiple apps using a single server.
type Client struct {
	apiClient     *httpClient
	apiEndpoint   string
	clientVersion string
	userID        string
	sessionID     string
	isMachine     bool
	sentPing      bool
	apps          map[string]*AppClient
}

// AppClient supports managing a single application.
type AppClient struct {
	*Client
	appID   string
	track   string
	version string
	oem     string
}

// New creates an omaha client for updating one or more applications.
// userID must be a persistent unique identifier of this update client.
func New(serverURL, userID string) (*Client, error) {
	if userID == "" {
		return nil, errors.New("omaha: empty user identifier")
	}

	c := &Client{
		apiClient:     newHTTPClient(),
		clientVersion: defaultClientVersion,
		userID:        userID,
		sessionID:     uuid.NewV4().String(),
		apps:          make(map[string]*AppClient),
	}

	if err := c.SetServerURL(serverURL); err != nil {
		return nil, err
	}

	return c, nil
}

// SetServerURL changes the Omaha server this client talks to.
// If the URL does not include a path component /v1/update/ is assumed.
func (c *Client) SetServerURL(serverURL string) error {
	u, err := url.Parse(serverURL)
	if err != nil {
		return fmt.Errorf("omaha: invalid server URL: %v", err)
	}
	if u.Scheme != "http" && u.Scheme != "https" {
		return fmt.Errorf("omaha: invalid server protocol: %s", u)
	}
	if u.Host == "" {
		return fmt.Errorf("omaha: invalid server host: %s", u)
	}
	if u.Path == "" || u.Path == "/" {
		u.Path = "/v1/update/"
	}

	c.apiEndpoint = u.String()
	return nil
}

// SetClientVersion sets the identifier of this updater application.
// e.g. "update_engine-0.1.0".  Default is "go-omaha".
func (c *Client) SetClientVersion(clientVersion string) {
	c.clientVersion = clientVersion
}

// NextPing returns a timer channel that will fire when the next update
// check or ping should be sent.
func (c *Client) NextPing() <-chan time.Time {
	d := pingDelay
	if c.sentPing {
		d = pingInterval
	}
	return FuzzyAfter(d, pingFuzz)
}

// AppClient gets the application client for the given application ID.
func (c *Client) AppClient(appID string) (*AppClient, error) {
	if app, ok := c.apps[appID]; ok {
		return app, nil
	}

	return nil, fmt.Errorf("omaha: missing app client %q", appID)
}

// NewAppClient creates a new application client.
func (c *Client) NewAppClient(appID, appVersion string) (*AppClient, error) {
	if _, ok := c.apps[appID]; ok {
		return nil, fmt.Errorf("omaha: duplicate app client %q", appID)
	}

	ac := &AppClient{
		Client: c,
		appID:  appID,
	}
	c.apps[appID] = ac

	return ac, nil
}

// NewAppClient creates a single application client.
// Shorthand for New(serverURL, userID).NewAppClient(appID, appVersion).
func NewAppClient(serverURL, userID, appID, appVersion string) (*AppClient, error) {
	c, err := New(serverURL, userID)
	if err != nil {
		return nil, err
	}

	ac, err := c.NewAppClient(appID, appVersion)
	if err := ac.SetVersion(appVersion); err != nil {
		return nil, err
	}

	return ac, nil
}

func (ac *AppClient) SetAppID(appID string) error {
	if appID == ac.appID {
		return nil
	}

	if _, ok := ac.apps[appID]; ok {
		return fmt.Errorf("omaha: duplicate app %q", appID)
	}

	delete(ac.apps, ac.appID)
	ac.appID = appID
	ac.apps[appID] = ac
	return nil
}

// SetVersion changes the application version.
func (ac *AppClient) SetVersion(version string) error {
	if version == "" {
		return errors.New("omaha: empty application version")
	}

	ac.version = version
	return nil
}

// SetTrack sets the application update track or group.
// This is a update_engine/Core Update protocol extension.
func (ac *AppClient) SetTrack(track string) error {
	// Although track is an omaha extension and theoretically not required
	// our Core Update server requires track to be set to a valid id/name.
	// TODO: deprecate track and use the standard cohort protocol fields.
	if track == "" {
		return errors.New("omaha: empty application update track/group")
	}

	ac.track = track
	return nil
}

// SetOEM sets the application OEM name.
// This is a update_engine/Core Update protocol extension.
func (ac *AppClient) SetOEM(oem string) {
	ac.oem = oem
}

func (ac *AppClient) UpdateCheck() (*omaha.UpdateResponse, error) {
	req := ac.newReq()
	app := req.Apps[0]
	app.AddPing()
	app.AddUpdateCheck()

	ac.sentPing = true

	appResp, err := ac.doReq(ac.apiEndpoint, req)
	if err != nil {
		return nil, err
	}

	if appResp.Ping == nil {
		return nil, fmt.Errorf("omaha: ping status missing from response")
	}

	if appResp.Ping.Status != "ok" {
		return nil, fmt.Errorf("omaha: ping status %s", appResp.Ping.Status)
	}

	if appResp.UpdateCheck == nil {
		return nil, fmt.Errorf("omaha: update check missing from response")
	}

	if appResp.UpdateCheck.Status != omaha.UpdateOK {
		return nil, appResp.UpdateCheck.Status
	}

	return appResp.UpdateCheck, nil
}

func (ac *AppClient) Ping() error {
	req := ac.newReq()
	app := req.Apps[0]
	app.AddPing()

	ac.sentPing = true

	appResp, err := ac.doReq(ac.apiEndpoint, req)
	if err != nil {
		return err
	}

	if appResp.Ping == nil {
		return fmt.Errorf("omaha: ping status missing from response")
	}

	if appResp.Ping.Status != "ok" {
		return fmt.Errorf("omaha: ping status %s", appResp.Ping.Status)
	}

	return nil
}

func (ac *AppClient) Event(event *omaha.EventRequest) error {
	req := ac.newReq()
	app := req.Apps[0]
	app.Events = append(app.Events, event)

	appResp, err := ac.doReq(ac.apiEndpoint, req)
	if err != nil {
		return err
	}

	if len(appResp.Events) == 0 {
		return fmt.Errorf("omaha: event status missing from response")
	}

	if appResp.Events[0].Status != "ok" {
		return fmt.Errorf("omaha: event status %s", appResp.Events[0].Status)
	}

	return nil
}

func (ac *AppClient) newReq() *omaha.Request {
	req := omaha.NewRequest()
	req.Version = ac.clientVersion
	req.UserID = ac.userID
	req.SessionID = ac.sessionID
	if ac.isMachine {
		req.IsMachine = 1
	}

	app := req.AddApp(ac.appID, ac.version)
	app.Track = ac.track
	app.OEM = ac.oem

	// MachineID and BootID are non-standard fields used by CoreOS'
	// update_engine and Core Update. Copy their values from the
	// standard UserID and SessionID. Eventually the non-standard
	// fields should be deprecated.
	app.MachineID = req.UserID
	app.BootID = req.SessionID

	return req
}

func (ac *AppClient) doReq(url string, req *omaha.Request) (*omaha.AppResponse, error) {
	if len(req.Apps) != 1 {
		panic(fmt.Errorf("unexpected number of apps: %d", len(req.Apps)))
	}
	appID := req.Apps[0].ID
	resp, err := ac.apiClient.Omaha(url, req)
	if err != nil {
		return nil, err
	}

	appResp := resp.GetApp(appID)
	if appResp == nil {
		return nil, fmt.Errorf("omaha: app %s missing from response", appID)
	}

	if appResp.Status != omaha.AppOK {
		return nil, appResp.Status
	}

	return appResp, nil
}
<|endoftext|>


<commit_before>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package perfevents

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
)

const (
	// Name of plugin
	name = "perfevents"
	// Version of plugin
	version = 4
	// Type of plugin
	pluginType = plugin.CollectorPluginType
	// Namespace definition
	ns_vendor  = "intel"
	ns_class   = "linux"
	ns_type    = "perfevents"
	ns_subtype = "cgroup"
)

type event struct {
	id    string
	etype string
	value uint64
}

type Perfevents struct {
	cgroup_events []event
	Init          func() error
}

var CGROUP_EVENTS = []string{"cycles", "instructions", "cache-references", "cache-misses",
	"branch-instructions", "branch-misses", "stalled-cycles-frontend",
	"stalled-cycles-backend", "ref-cycles"}

func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(name, version, pluginType, []string{plugin.SnapGOBContentType}, []string{plugin.SnapGOBContentType})
}

// CollectMetrics returns HW metrics from perf events subsystem
// for Cgroups present on the host.
func (p *Perfevents) CollectMetrics(mts []plugin.PluginMetricType) ([]plugin.PluginMetricType, error) {
	if len(mts) == 0 {
		return nil, nil
	}
	events := []string{}
	cgroups := []string{}

	// Get list of events and cgroups from Namespace
	// Replace "_" with "/" in cgroup name
	for _, m := range mts {
		err := validateNamespace(m.Namespace())
		if err != nil {
			return nil, err
		}
		events = append(events, m.Namespace()[4])
		cgroups = append(cgroups, strings.Replace(m.Namespace()[5], "_", "/", -1))
	}

	// Prepare events (-e) and Cgroups (-G) switches for "perf stat"
	cgroups_switch := "-G" + strings.Join(cgroups, ",")
	events_switch := "-e" + strings.Join(events, ",")

	// Prepare "perf stat" command
	cmd := exec.Command("perf", "stat", "--log-fd", "1", `-x;`, "-a", events_switch, cgroups_switch, "--", "sleep", "1")

	cmdReader, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error creating StdoutPipe", err)
		return nil, err
	}

	// Parse "perf stat" output
	p.cgroup_events = make([]event, len(mts))
	scanner := bufio.NewScanner(cmdReader)
	go func() {
		for i := 0; scanner.Scan(); i++ {
			line := strings.Split(scanner.Text(), ";")
			value, err := strconv.ParseUint(line[0], 10, 64)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Invalid metric value", err)
			}
			etype := line[2]
			id := line[3]
			e := event{id: id, etype: etype, value: value}
			p.cgroup_events[i] = e
		}
	}()

	// Run command and wait (up to 2 secs) for completion
	err = cmd.Start()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error starting perf stat", err)
		return nil, err
	}

	st := time.Now()
	for {
		if len(p.cgroup_events) == cap(p.cgroup_events) {
			break
		}
		if time.Since(st) > time.Second*2 {
			return nil, fmt.Errorf("Timed out waiting for metrics from perf stat")
		}
	}
	err = cmd.Wait()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error waiting for perf stat", err)
		return nil, err
	}

	// Populate metrics
	metrics := make([]plugin.PluginMetricType, len(mts))
	i := 0
	for _, m := range mts {
		metric, err := populate_metric(m.Namespace(), p.cgroup_events[i])
		if err != nil {
			return nil, err
		}
		metrics[i] = *metric
		metrics[i].Source_, _ = os.Hostname()
		i++
	}
	return metrics, nil
}

// GetMetricTypes returns the metric types exposed by perf events subsystem
func (p *Perfevents) GetMetricTypes(_ plugin.PluginConfigType) ([]plugin.PluginMetricType, error) {
	err := p.Init()
	if err != nil {
		return nil, err
	}
	cgroups, err := list_cgroups()
	if err != nil {
		return nil, err
	}
	if len(cgroups) == 0 {
		return nil, nil
	}
	mts := []plugin.PluginMetricType{}
	mts = append(mts, set_supported_metrics(ns_subtype, cgroups, CGROUP_EVENTS)...)

	return mts, nil
}

// GetConfigPolicy returns a ConfigPolicy
func (p *Perfevents) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	return c, nil
}

// New initializes Perfevents plugin
func NewPerfeventsCollector() *Perfevents {
	return &Perfevents{Init: initialize}
}

func initialize() error {
	file, err := os.Open("/proc/sys/kernel/perf_event_paranoid")
	if err != nil {
		if os.IsExist(err) {
			return errors.New("perf_event_paranoid file exists but couldn't be opened")
		}
		return errors.New("perf event system not enabled")
	}

	scanner := bufio.NewScanner(file)
	if !scanner.Scan() {
		return errors.New("cannot read from perf_event_paranoid")
	}

	i, err := strconv.ParseInt(scanner.Text(), 10, 64)
	if err != nil {
		return errors.New("invalid value in perf_event_paranoid file")
	}

	if i >= 1 {
		return errors.New("insufficient perf event subsystem capabilities")
	}
	return nil
}

func set_supported_metrics(source string, cgroups []string, events []string) []plugin.PluginMetricType {
	mts := make([]plugin.PluginMetricType, len(events)*len(cgroups))
	for _, e := range events {
		for _, c := range flatten_cg_name(cgroups) {
			mts = append(mts, plugin.PluginMetricType{Namespace_: []string{ns_vendor, ns_class, ns_type, source, e, c}})
		}
	}
	return mts
}
func flatten_cg_name(cg []string) []string {
	flat_cg := []string{}
	for _, c := range cg {
		flat_cg = append(flat_cg, strings.Replace(c, "/", "_", -1))
	}
	return flat_cg
}

func populate_metric(ns []string, e event) (*plugin.PluginMetricType, error) {
	return &plugin.PluginMetricType{
		Namespace_: ns,
		Data_:      e.value,
		Timestamp_: time.Now(),
	}, nil
}

func list_cgroups() ([]string, error) {
	cgroups := []string{}
	base_path := "/sys/fs/cgroup/perf_event/"
	err := filepath.Walk(base_path, func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			cgroup_name := strings.TrimPrefix(path, base_path)
			if len(cgroup_name) > 0 {
				cgroups = append(cgroups, cgroup_name)
			}
		}
		return nil

	})
	if err != nil {
		return nil, err
	}
	return cgroups, nil
}

func validateNamespace(namespace []string) error {
	if len(namespace) != 6 {
		return errors.New(fmt.Sprintf("unknown metricType %s (should containt exactly 6 segments)", namespace))
	}
	if namespace[0] != ns_vendor {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 1st segment %s)", namespace, ns_vendor))
	}

	if namespace[1] != ns_class {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 2nd segment %s)", namespace, ns_class))
	}
	if namespace[2] != ns_type {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 3rd segment %s)", namespace, ns_type))
	}
	if namespace[3] != ns_subtype {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 4th segment %s)", namespace, ns_subtype))
	}
	if !namespaceContains(namespace[4], CGROUP_EVENTS) {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 5th segment %v)", namespace, CGROUP_EVENTS))
	}
	return nil
}

func namespaceContains(element string, slice []string) bool {
	for _, v := range slice {
		if v == element {
			return true
		}
	}
	return false
}
<commit_msg>Update version to reflect change in snap dependency<commit_after>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package perfevents

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
)

const (
	// Name of plugin
	name = "perfevents"
	// Version of plugin
	version = 5
	// Type of plugin
	pluginType = plugin.CollectorPluginType
	// Namespace definition
	ns_vendor  = "intel"
	ns_class   = "linux"
	ns_type    = "perfevents"
	ns_subtype = "cgroup"
)

type event struct {
	id    string
	etype string
	value uint64
}

type Perfevents struct {
	cgroup_events []event
	Init          func() error
}

var CGROUP_EVENTS = []string{"cycles", "instructions", "cache-references", "cache-misses",
	"branch-instructions", "branch-misses", "stalled-cycles-frontend",
	"stalled-cycles-backend", "ref-cycles"}

func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(name, version, pluginType, []string{plugin.SnapGOBContentType}, []string{plugin.SnapGOBContentType})
}

// CollectMetrics returns HW metrics from perf events subsystem
// for Cgroups present on the host.
func (p *Perfevents) CollectMetrics(mts []plugin.PluginMetricType) ([]plugin.PluginMetricType, error) {
	if len(mts) == 0 {
		return nil, nil
	}
	events := []string{}
	cgroups := []string{}

	// Get list of events and cgroups from Namespace
	// Replace "_" with "/" in cgroup name
	for _, m := range mts {
		err := validateNamespace(m.Namespace())
		if err != nil {
			return nil, err
		}
		events = append(events, m.Namespace()[4])
		cgroups = append(cgroups, strings.Replace(m.Namespace()[5], "_", "/", -1))
	}

	// Prepare events (-e) and Cgroups (-G) switches for "perf stat"
	cgroups_switch := "-G" + strings.Join(cgroups, ",")
	events_switch := "-e" + strings.Join(events, ",")

	// Prepare "perf stat" command
	cmd := exec.Command("perf", "stat", "--log-fd", "1", `-x;`, "-a", events_switch, cgroups_switch, "--", "sleep", "1")

	cmdReader, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error creating StdoutPipe", err)
		return nil, err
	}

	// Parse "perf stat" output
	p.cgroup_events = make([]event, len(mts))
	scanner := bufio.NewScanner(cmdReader)
	go func() {
		for i := 0; scanner.Scan(); i++ {
			line := strings.Split(scanner.Text(), ";")
			value, err := strconv.ParseUint(line[0], 10, 64)
			if err != nil {
				fmt.Fprintln(os.Stderr, "Invalid metric value", err)
			}
			etype := line[2]
			id := line[3]
			e := event{id: id, etype: etype, value: value}
			p.cgroup_events[i] = e
		}
	}()

	// Run command and wait (up to 2 secs) for completion
	err = cmd.Start()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error starting perf stat", err)
		return nil, err
	}

	st := time.Now()
	for {
		if len(p.cgroup_events) == cap(p.cgroup_events) {
			break
		}
		if time.Since(st) > time.Second*2 {
			return nil, fmt.Errorf("Timed out waiting for metrics from perf stat")
		}
	}
	err = cmd.Wait()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Error waiting for perf stat", err)
		return nil, err
	}

	// Populate metrics
	metrics := make([]plugin.PluginMetricType, len(mts))
	i := 0
	for _, m := range mts {
		metric, err := populate_metric(m.Namespace(), p.cgroup_events[i])
		if err != nil {
			return nil, err
		}
		metrics[i] = *metric
		metrics[i].Source_, _ = os.Hostname()
		i++
	}
	return metrics, nil
}

// GetMetricTypes returns the metric types exposed by perf events subsystem
func (p *Perfevents) GetMetricTypes(_ plugin.PluginConfigType) ([]plugin.PluginMetricType, error) {
	err := p.Init()
	if err != nil {
		return nil, err
	}
	cgroups, err := list_cgroups()
	if err != nil {
		return nil, err
	}
	if len(cgroups) == 0 {
		return nil, nil
	}
	mts := []plugin.PluginMetricType{}
	mts = append(mts, set_supported_metrics(ns_subtype, cgroups, CGROUP_EVENTS)...)

	return mts, nil
}

// GetConfigPolicy returns a ConfigPolicy
func (p *Perfevents) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	c := cpolicy.New()
	return c, nil
}

// New initializes Perfevents plugin
func NewPerfeventsCollector() *Perfevents {
	return &Perfevents{Init: initialize}
}

func initialize() error {
	file, err := os.Open("/proc/sys/kernel/perf_event_paranoid")
	if err != nil {
		if os.IsExist(err) {
			return errors.New("perf_event_paranoid file exists but couldn't be opened")
		}
		return errors.New("perf event system not enabled")
	}

	scanner := bufio.NewScanner(file)
	if !scanner.Scan() {
		return errors.New("cannot read from perf_event_paranoid")
	}

	i, err := strconv.ParseInt(scanner.Text(), 10, 64)
	if err != nil {
		return errors.New("invalid value in perf_event_paranoid file")
	}

	if i >= 1 {
		return errors.New("insufficient perf event subsystem capabilities")
	}
	return nil
}

func set_supported_metrics(source string, cgroups []string, events []string) []plugin.PluginMetricType {
	mts := make([]plugin.PluginMetricType, len(events)*len(cgroups))
	for _, e := range events {
		for _, c := range flatten_cg_name(cgroups) {
			mts = append(mts, plugin.PluginMetricType{Namespace_: []string{ns_vendor, ns_class, ns_type, source, e, c}})
		}
	}
	return mts
}
func flatten_cg_name(cg []string) []string {
	flat_cg := []string{}
	for _, c := range cg {
		flat_cg = append(flat_cg, strings.Replace(c, "/", "_", -1))
	}
	return flat_cg
}

func populate_metric(ns []string, e event) (*plugin.PluginMetricType, error) {
	return &plugin.PluginMetricType{
		Namespace_: ns,
		Data_:      e.value,
		Timestamp_: time.Now(),
	}, nil
}

func list_cgroups() ([]string, error) {
	cgroups := []string{}
	base_path := "/sys/fs/cgroup/perf_event/"
	err := filepath.Walk(base_path, func(path string, info os.FileInfo, _ error) error {
		if info.IsDir() {
			cgroup_name := strings.TrimPrefix(path, base_path)
			if len(cgroup_name) > 0 {
				cgroups = append(cgroups, cgroup_name)
			}
		}
		return nil

	})
	if err != nil {
		return nil, err
	}
	return cgroups, nil
}

func validateNamespace(namespace []string) error {
	if len(namespace) != 6 {
		return errors.New(fmt.Sprintf("unknown metricType %s (should containt exactly 6 segments)", namespace))
	}
	if namespace[0] != ns_vendor {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 1st segment %s)", namespace, ns_vendor))
	}

	if namespace[1] != ns_class {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 2nd segment %s)", namespace, ns_class))
	}
	if namespace[2] != ns_type {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 3rd segment %s)", namespace, ns_type))
	}
	if namespace[3] != ns_subtype {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 4th segment %s)", namespace, ns_subtype))
	}
	if !namespaceContains(namespace[4], CGROUP_EVENTS) {
		return errors.New(fmt.Sprintf("unknown metricType %s (expected 5th segment %v)", namespace, CGROUP_EVENTS))
	}
	return nil
}

func namespaceContains(element string, slice []string) bool {
	for _, v := range slice {
		if v == element {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package perfsession

import (
	"debug/dwarf"
	"debug/elf"
	"fmt"
	"io"
	"log"
	"os/user"
	"sort"
)

type Symbolic struct {
	FuncName string
	Line     dwarf.LineEntry
}

func Symbolize(session *Session, mmap *Mmap, ip uint64, out *Symbolic) bool {
	s := getSymbolicExtra(session, mmap.Filename)
	if s == nil {
		return false
	}
	f, l := s.findIP(ip)
	if f == nil {
		out.FuncName = ""
	} else {
		out.FuncName = f.name
	}
	if l == nil {
		out.Line = dwarf.LineEntry{}
	} else {
		out.Line = *l
	}
	return true
}

var symbolicExtraKey = NewExtraKey("perfsession.symbolicExtra")

var buildIDDir = (func() string {
	// See set_buildid_dir in tools/perf/util/config.c.
	u, err := user.Current()
	if err != nil {
		return ".debug"
	}
	return fmt.Sprintf("%s/.debug", u.HomeDir)
})()

func getSymbolicExtra(session *Session, filename string) *symbolicExtra {
	tables, ok := session.Extra[symbolicExtraKey].(map[string]*symbolicExtra)
	if !ok {
		tables = make(map[string]*symbolicExtra)
		session.Extra[symbolicExtraKey] = tables
	}

	extra, ok := tables[filename]
	if ok {
		return extra
	}
	tables[filename] = (*symbolicExtra)(nil)

	// See dso__data_fd in toosl/perf/util/dso.c.

	// TODO: Handle kernel symbols. See dso__find_kallsyms.

	// Try build ID cache first.
	//
	// TODO: Cache filename to build ID mapping.
	bids, err := session.File.BuildIDs()
	if err != nil {
		log.Fatal(err)
	}
	for _, bid := range bids {
		if bid.Filename == filename {
			nfilename := fmt.Sprintf("%s/.build-id/%.2s/%s", buildIDDir, bid.BuildID, bid.BuildID.String()[2:])
			extra, err = newSymbolicExtra(nfilename)
			if err == nil {
				break
			}
		}
	}

	// Try original path.
	if extra == nil {
		extra, err = newSymbolicExtra(filename)
		if err != nil {
			log.Println(err)
		}
	}

	tables[filename] = extra
	return extra
}

func newSymbolicExtra(filename string) (*symbolicExtra, error) {
	// Load ELF
	//
	// TODO: Relocate ELF.
	elff, err := elf.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("error loading ELF file %s: %s", filename, err)
	}
	defer elff.Close()

	// Load DWARF
	//
	// TODO: Support build IDs and split DWARF
	if elff.Section(".debug_info") == nil {
		return nil, fmt.Errorf("no DWARF info for %s", filename)
	}
	dwarff, err := elff.DWARF()
	if err != nil {
		return nil, fmt.Errorf("error loading DWARF from %s: %s", filename, err)
	}

	return &symbolicExtra{
		dwarfFuncTable(dwarff),
		dwarfLineTable(dwarff),
	}, nil
}

type symbolicExtra struct {
	functab []funcRange
	linetab []dwarf.LineEntry
}

func (s *symbolicExtra) findIP(ip uint64) (f *funcRange, l *dwarf.LineEntry) {
	if s.functab == nil || s.linetab == nil {
		return nil, nil
	}

	i := sort.Search(len(s.functab), func(i int) bool {
		return ip < s.functab[i].highpc
	})
	if i < len(s.functab) && s.functab[i].lowpc <= ip && ip < s.functab[i].highpc {
		f = &s.functab[i]
	}

	i = sort.Search(len(s.linetab), func(i int) bool {
		return ip < s.linetab[i].Address
	})
	if i != 0 && !s.linetab[i-1].EndSequence {
		l = &s.linetab[i-1]
	}

	return
}

type funcRange struct {
	name          string
	lowpc, highpc uint64
}

func dwarfFuncTable(dwarff *dwarf.Data) []funcRange {
	// TODO Use ELF symbol table if DWARF isn't available.

	// Walk DWARF for functions
	// TODO: Use .debug_pubnames (not supported by dwarf package)
	r := dwarff.Reader()
	out := make([]funcRange, 0)
	for {
		ent, err := r.Next()
		if ent == nil || err != nil {
			break
		}
		// TODO: We should process TagInlinedSubroutine, but
		// apparently gc doesn't produce these.
		//
		// TODO: Support DW_AT_ranges.
	tag:
		switch ent.Tag {
		case dwarf.TagSubprogram:
			r.SkipChildren()
			name, ok := ent.Val(dwarf.AttrName).(string)
			if !ok {
				break
			}
			lowpc, ok := ent.Val(dwarf.AttrLowpc).(uint64)
			if !ok {
				break
			}
			var highpc uint64
			switch highpcx := ent.Val(dwarf.AttrHighpc).(type) {
			case uint64:
				highpc = highpcx
			case int64:
				highpc = lowpc + uint64(highpcx)
			default:
				break tag
			}
			out = append(out, funcRange{name, lowpc, highpc})

		case dwarf.TagCompileUnit, dwarf.TagModule, dwarf.TagNamespace:
			break

		default:
			r.SkipChildren()
		}
	}

	sort.Sort(funcRangeSorter(out))

	return out
}

type funcRangeSorter []funcRange

func (s funcRangeSorter) Len() int {
	return len(s)
}

func (s funcRangeSorter) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s funcRangeSorter) Less(i, j int) bool {
	return s[i].lowpc < s[j].lowpc
}

func dwarfLineTable(dwarff *dwarf.Data) []dwarf.LineEntry {
	out := make([]dwarf.LineEntry, 0)

	// Iterate over compilation units
	dr := dwarff.Reader()
	for {
		ent, err := dr.Next()
		if ent == nil || err != nil {
			break
		}

		if ent.Tag != dwarf.TagCompileUnit {
			dr.SkipChildren()
			continue
		}

		// Decode CU's line table
		lr, err := dwarff.LineReader(ent)
		if err != nil {
			log.Fatal(err)
		} else if lr == nil {
			continue
		}

		for {
			var lent dwarf.LineEntry
			err := lr.Next(&lent)
			if err != nil {
				if err == io.EOF {
					break
				}
				log.Fatal(err)
			}
			out = append(out, lent)
		}
	}
	return out
}
<commit_msg>perfsession: use ELF function symbols if DWARF isn't available<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package perfsession

import (
	"debug/dwarf"
	"debug/elf"
	"fmt"
	"io"
	"log"
	"os/user"
	"sort"
)

type Symbolic struct {
	FuncName string
	Line     dwarf.LineEntry
}

func Symbolize(session *Session, mmap *Mmap, ip uint64, out *Symbolic) bool {
	s := getSymbolicExtra(session, mmap.Filename)
	if s == nil {
		return false
	}
	f, l := s.findIP(ip)
	if f == nil {
		out.FuncName = ""
	} else {
		out.FuncName = f.name
	}
	if l == nil {
		out.Line = dwarf.LineEntry{}
	} else {
		out.Line = *l
	}
	return true
}

var symbolicExtraKey = NewExtraKey("perfsession.symbolicExtra")

var buildIDDir = (func() string {
	// See set_buildid_dir in tools/perf/util/config.c.
	u, err := user.Current()
	if err != nil {
		return ".debug"
	}
	return fmt.Sprintf("%s/.debug", u.HomeDir)
})()

func getSymbolicExtra(session *Session, filename string) *symbolicExtra {
	tables, ok := session.Extra[symbolicExtraKey].(map[string]*symbolicExtra)
	if !ok {
		tables = make(map[string]*symbolicExtra)
		session.Extra[symbolicExtraKey] = tables
	}

	extra, ok := tables[filename]
	if ok {
		return extra
	}
	tables[filename] = (*symbolicExtra)(nil)

	// See dso__data_fd in toosl/perf/util/dso.c.

	// TODO: Handle kernel symbols. See dso__find_kallsyms.

	// Try build ID cache first.
	//
	// TODO: Cache filename to build ID mapping.
	bids, err := session.File.BuildIDs()
	if err != nil {
		log.Fatal(err)
	}
	for _, bid := range bids {
		if bid.Filename == filename {
			nfilename := fmt.Sprintf("%s/.build-id/%.2s/%s", buildIDDir, bid.BuildID, bid.BuildID.String()[2:])
			extra, err = newSymbolicExtra(nfilename)
			if err == nil {
				break
			}
		}
	}

	// Try original path.
	if extra == nil {
		extra, err = newSymbolicExtra(filename)
		if err != nil {
			log.Println(err)
		}
	}

	tables[filename] = extra
	return extra
}

func newSymbolicExtra(filename string) (*symbolicExtra, error) {
	// Load ELF
	//
	// TODO: Relocate ELF.
	elff, err := elf.Open(filename)
	if err != nil {
		return nil, fmt.Errorf("error loading ELF file %s: %s", filename, err)
	}
	defer elff.Close()

	// Load DWARF
	//
	// TODO: Support build IDs and split DWARF
	if elff.Section(".debug_info") != nil {
		dwarff, err := elff.DWARF()
		if err != nil {
			return nil, fmt.Errorf("error loading DWARF from %s: %s", filename, err)
		}

		return &symbolicExtra{
			dwarfFuncTable(dwarff),
			dwarfLineTable(dwarff),
		}, nil
	}

	// Make do with the ELF symbols.
	return &symbolicExtra{elfFuncTable(filename, elff), nil}, nil
}

type symbolicExtra struct {
	functab []funcRange
	linetab []dwarf.LineEntry
}

func (s *symbolicExtra) findIP(ip uint64) (f *funcRange, l *dwarf.LineEntry) {
	if s.functab != nil {
		i := sort.Search(len(s.functab), func(i int) bool {
			return ip < s.functab[i].highpc
		})
		if i < len(s.functab) && s.functab[i].lowpc <= ip && ip < s.functab[i].highpc {
			f = &s.functab[i]
		}
	}

	if s.linetab != nil {
		i := sort.Search(len(s.linetab), func(i int) bool {
			return ip < s.linetab[i].Address
		})
		if i != 0 && !s.linetab[i-1].EndSequence {
			l = &s.linetab[i-1]
		}
	}

	return
}

type funcRange struct {
	name          string
	lowpc, highpc uint64
}

func dwarfFuncTable(dwarff *dwarf.Data) []funcRange {
	// Walk DWARF for functions
	// TODO: Use .debug_pubnames (not supported by dwarf package)
	r := dwarff.Reader()
	out := make([]funcRange, 0)
	for {
		ent, err := r.Next()
		if ent == nil || err != nil {
			break
		}
		// TODO: We should process TagInlinedSubroutine, but
		// apparently gc doesn't produce these.
		//
		// TODO: Support DW_AT_ranges.
	tag:
		switch ent.Tag {
		case dwarf.TagSubprogram:
			r.SkipChildren()
			name, ok := ent.Val(dwarf.AttrName).(string)
			if !ok {
				break
			}
			lowpc, ok := ent.Val(dwarf.AttrLowpc).(uint64)
			if !ok {
				break
			}
			var highpc uint64
			switch highpcx := ent.Val(dwarf.AttrHighpc).(type) {
			case uint64:
				highpc = highpcx
			case int64:
				highpc = lowpc + uint64(highpcx)
			default:
				break tag
			}
			out = append(out, funcRange{name, lowpc, highpc})

		case dwarf.TagCompileUnit, dwarf.TagModule, dwarf.TagNamespace:
			break

		default:
			r.SkipChildren()
		}
	}

	sort.Sort(funcRangeSorter(out))

	return out
}

func elfFuncTable(filename string, elff *elf.File) []funcRange {
	if elff.Type != elf.ET_EXEC {
		// TODO: Deal with relocatable shared objects. (ET_DYN)
		// Note that PIE binaries are ET_DYN.
		return nil
	}
	out := make([]funcRange, 0)
	syms, err := elff.Symbols()
	if err != nil {
		if err != elf.ErrNoSymbols {
			log.Fatal("%s: %s", filename, err)
		}
		return nil
	}
	for _, sym := range syms {
		if elf.SymType(sym.Info&0xF) != elf.STT_FUNC {
			continue
		}
		out = append(out, funcRange{sym.Name, sym.Value, sym.Value + sym.Size})
	}

	sort.Sort(funcRangeSorter(out))

	// Assign symbols highpcs if they don't have them.
	for i := range out {
		if out[i].highpc == out[i].lowpc {
			if i == len(out)-1 {
				out[i].highpc++
			} else {
				out[i].highpc = out[i+1].lowpc
			}
		}
	}

	return out
}

type funcRangeSorter []funcRange

func (s funcRangeSorter) Len() int {
	return len(s)
}

func (s funcRangeSorter) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s funcRangeSorter) Less(i, j int) bool {
	return s[i].lowpc < s[j].lowpc
}

func dwarfLineTable(dwarff *dwarf.Data) []dwarf.LineEntry {
	out := make([]dwarf.LineEntry, 0)

	// Iterate over compilation units
	dr := dwarff.Reader()
	for {
		ent, err := dr.Next()
		if ent == nil || err != nil {
			break
		}

		if ent.Tag != dwarf.TagCompileUnit {
			dr.SkipChildren()
			continue
		}

		// Decode CU's line table
		lr, err := dwarff.LineReader(ent)
		if err != nil {
			log.Fatal(err)
		} else if lr == nil {
			continue
		}

		for {
			var lent dwarf.LineEntry
			err := lr.Next(&lent)
			if err != nil {
				if err == io.EOF {
					break
				}
				log.Fatal(err)
			}
			out = append(out, lent)
		}
	}
	return out
}
<|endoftext|>


<commit_before>package dir_test

import (
	"os"
	"path"
	"testing"

	"github.com/mlafeldt/chef-runner/resolver/dir"
	"github.com/mlafeldt/chef-runner/util"
	"github.com/stretchr/testify/assert"
)

const CookbookPath = "test-cookbooks"

func TestResolve(t *testing.T) {
	if err := os.Chdir("../../testdata"); err != nil {
		panic(err)
	}

	defer os.RemoveAll(CookbookPath)

	assert.NoError(t, dir.Resolver{}.Resolve(CookbookPath))

	expectFiles := []string{
		"practicingruby/README.md",
		"practicingruby/attributes",
		"practicingruby/metadata.rb",
		"practicingruby/recipes",
	}
	for _, f := range expectFiles {
		assert.True(t, util.FileExist(path.Join(CookbookPath, f)))
	}
}
<commit_msg>Fix Dir resolver test to succeed with multiple CPUs<commit_after>package dir_test

import (
	"os"
	"path"
	"testing"

	"github.com/mlafeldt/chef-runner/resolver/dir"
	"github.com/mlafeldt/chef-runner/util"
	"github.com/stretchr/testify/assert"
)

func TestResolve(t *testing.T) {
	util.InDir("../../testdata", func() {
		cookbookPath := "test-cookbooks"
		defer os.RemoveAll(cookbookPath)

		assert.NoError(t, dir.Resolver{}.Resolve(cookbookPath))

		expectFiles := []string{
			"practicingruby/README.md",
			"practicingruby/attributes",
			"practicingruby/metadata.rb",
			"practicingruby/recipes",
		}
		for _, f := range expectFiles {
			assert.True(t, util.FileExist(path.Join(cookbookPath, f)))
		}
	})
}
<|endoftext|>


<commit_before>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"strings"

	"github.com/TheThingsNetwork/go-account-lib/account"
	"github.com/TheThingsNetwork/go-account-lib/rights"
	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/gosuri/uitable"
)

func printCollaborators(collaborators []account.Collaborator) {
	table := uitable.New()
	table.MaxColWidth = 70
	table.AddRow("", "Username", "Rights")
	for i, collaborator := range collaborators {
		table.AddRow(i+1, collaborator.Username, joinRights(collaborator.Rights, ","))
	}
	fmt.Println()
	fmt.Println(table)
	fmt.Println()
}

var applicationRights = []types.Right{
	rights.AppSettings,
	rights.AppCollaborators,
	rights.AppDelete,
	rights.ReadUplink,
	rights.WriteUplink,
	rights.WriteDownlink,
	rights.Devices,
}
var gatewayRights = []types.Right{
	rights.GatewaySettings,
	rights.GatewayCollaborators,
	rights.GatewayDelete,
	rights.GatewayLocation,
	rights.GatewayStatus,
	rights.GatewayOwner,
	rights.GatewayMessages,
}
var componentsRights = []types.Right{
	rights.ComponentSettings,
	rights.ComponentDelete,
	rights.ComponentCollaborators,
}

func validRight(available []types.Right, right types.Right) bool {
	for _, available := range available {
		if right == available {
			return true
		}
	}
	return false
}

func joinRights(rights []types.Right, sep string) string {
	rightStrings := make([]string, len(rights))
	for i, right := range rights {
		rightStrings[i] = string(right)
	}
	return strings.Join(rightStrings, sep)
}
<commit_msg>Remove non-grantable rights from app default rights<commit_after>// Copyright © 2017 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"
	"strings"

	"github.com/TheThingsNetwork/go-account-lib/account"
	"github.com/TheThingsNetwork/go-account-lib/rights"
	"github.com/TheThingsNetwork/ttn/core/types"
	"github.com/gosuri/uitable"
)

func printCollaborators(collaborators []account.Collaborator) {
	table := uitable.New()
	table.MaxColWidth = 70
	table.AddRow("", "Username", "Rights")
	for i, collaborator := range collaborators {
		table.AddRow(i+1, collaborator.Username, joinRights(collaborator.Rights, ","))
	}
	fmt.Println()
	fmt.Println(table)
	fmt.Println()
}

var applicationRights = []types.Right{
	rights.AppSettings,
	rights.AppCollaborators,
	rights.AppDelete,
	rights.Devices,
}
var gatewayRights = []types.Right{
	rights.GatewaySettings,
	rights.GatewayCollaborators,
	rights.GatewayDelete,
	rights.GatewayLocation,
	rights.GatewayStatus,
	rights.GatewayOwner,
	rights.GatewayMessages,
}
var componentsRights = []types.Right{
	rights.ComponentSettings,
	rights.ComponentDelete,
	rights.ComponentCollaborators,
}

func validRight(available []types.Right, right types.Right) bool {
	for _, available := range available {
		if right == available {
			return true
		}
	}
	return false
}

func joinRights(rights []types.Right, sep string) string {
	rightStrings := make([]string, len(rights))
	for i, right := range rights {
		rightStrings[i] = string(right)
	}
	return strings.Join(rightStrings, sep)
}
<|endoftext|>


<commit_before>package qingcloud

import (
	"github.com/yunify/qingcloud-sdk-go/client"
	qc "github.com/yunify/qingcloud-sdk-go/service"
	"time"
)

func updateLoadBalancer(lbID *string, meta interface{}) error {
	clt := meta.(*QingCloudClient).loadbalancer
	input := new(qc.UpdateLoadBalancersInput)
	input.LoadBalancers = []*string{lbID}
	var output *qc.UpdateLoadBalancersOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.UpdateLoadBalancers(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	client.WaitJob(meta.(*QingCloudClient).job,
		qc.StringValue(output.JobID),
		time.Duration(10)*time.Second, time.Duration(1)*time.Second)
	if _, err := LoadBalancerTransitionStateRefresh(clt, lbID); err != nil {
		return err
	}
	return nil
}

func resizeLoadBalancer(lbID *string, lbType *int, meta interface{}) error {
	clt := meta.(*QingCloudClient).loadbalancer
	input := new(qc.ResizeLoadBalancersInput)
	input.LoadBalancers = []*string{lbID}
	input.LoadBalancerType = lbType
	var output *qc.ResizeLoadBalancersOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.ResizeLoadBalancers(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	client.WaitJob(meta.(*QingCloudClient).job,
		qc.StringValue(output.JobID),
		time.Duration(10)*time.Second, time.Duration(1)*time.Second)
	if _, err := LoadBalancerTransitionStateRefresh(clt, lbID); err != nil {
		return err
	}
	return nil
}
<commit_msg>associateEipsToLoadBalancer & dissociateEipsToLoadBalancer<commit_after>package qingcloud

import (
	"github.com/yunify/qingcloud-sdk-go/client"
	qc "github.com/yunify/qingcloud-sdk-go/service"
	"time"
)

func updateLoadBalancer(lbID *string, meta interface{}) error {
	clt := meta.(*QingCloudClient).loadbalancer
	input := new(qc.UpdateLoadBalancersInput)
	input.LoadBalancers = []*string{lbID}
	var output *qc.UpdateLoadBalancersOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.UpdateLoadBalancers(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	client.WaitJob(meta.(*QingCloudClient).job,
		qc.StringValue(output.JobID),
		time.Duration(10)*time.Second, time.Duration(1)*time.Second)
	if _, err := LoadBalancerTransitionStateRefresh(clt, lbID); err != nil {
		return err
	}
	return nil
}

func resizeLoadBalancer(lbID *string, lbType *int, meta interface{}) error {
	clt := meta.(*QingCloudClient).loadbalancer
	input := new(qc.ResizeLoadBalancersInput)
	input.LoadBalancers = []*string{lbID}
	input.LoadBalancerType = lbType
	var output *qc.ResizeLoadBalancersOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.ResizeLoadBalancers(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	client.WaitJob(meta.(*QingCloudClient).job,
		qc.StringValue(output.JobID),
		time.Duration(10)*time.Second, time.Duration(1)*time.Second)
	if _, err := LoadBalancerTransitionStateRefresh(clt, lbID); err != nil {
		return err
	}
	return nil
}

func associateEipsToLoadBalancer(lbID *string, eips []*string, meta interface{}) error {
	clt := meta.(*QingCloudClient).loadbalancer
	input := new(qc.AssociateEIPsToLoadBalancerInput)
	input.LoadBalancer = lbID
	input.EIPs = eips
	var output *qc.AssociateEIPsToLoadBalancerOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.AssociateEIPsToLoadBalancer(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	client.WaitJob(meta.(*QingCloudClient).job,
		qc.StringValue(output.JobID),
		time.Duration(10)*time.Second, time.Duration(1)*time.Second)
	if _, err := LoadBalancerTransitionStateRefresh(clt, lbID); err != nil {
		return err
	}
	return nil
}

func dissociateEipsToLoadBalancer(lbID *string, eips []*string, meta interface{}) error {
	clt := meta.(*QingCloudClient).loadbalancer
	input := new(qc.DissociateEIPsFromLoadBalancerInput)
	input.LoadBalancer = lbID
	input.EIPs = eips
	var output *qc.DissociateEIPsFromLoadBalancerOutput
	var err error
	simpleRetry(func() error {
		output, err = clt.DissociateEIPsFromLoadBalancer(input)
		return isServerBusy(err)
	})
	if err != nil {
		return err
	}
	client.WaitJob(meta.(*QingCloudClient).job,
		qc.StringValue(output.JobID),
		time.Duration(10)*time.Second, time.Duration(1)*time.Second)
	if _, err := LoadBalancerTransitionStateRefresh(clt, lbID); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Package rng implements a series of pseudo-random number generator
// based on a variety of common probability distributions
// Author: Leesper
// Email: pascal7718@gmail.com 394683518@qq.com
package rng

import (
	"math"
)

// GaussianGenerator is a random number generator for gaussian distribution.
// The zero value is invalid, use NewGaussianGenerator to create a generator
type GaussianGenerator struct {
	uniform *UniformGenerator
}

// NewGaussianGenerator returns a gaussian-distribution generator
// it is recommended using time.Now().UnixNano() as the seed, for example:
// crng := rng.NewGaussianGenerator(time.Now().UnixNano())
func NewGaussianGenerator(seed int64) *GaussianGenerator {
	urng := NewUniformGenerator(seed)
	return &GaussianGenerator{urng}
}

// StdGaussian returns a random number of standard gaussian distribution
func (grng GaussianGenerator) StdGaussian() float64 {
	return grng.gaussian()
}

// Gaussian returns a random number of gaussian distribution Gauss(mean, stddev^2)
func (grng GaussianGenerator) Gaussian(mean, stddev float64) float64 {
	return mean + stddev*grng.gaussian()
}

func (grng GaussianGenerator) gaussian() float64 {
	// Box-Muller Transform
	var r, x, y float64
	for r >= 1 || r == 0 {
		x = grng.uniform.Float64Range(-1.0, 1.0)
		y = grng.uniform.Float64Range(-1.0, 1.0)
		r = x*x + y*y
	}
	return x * math.Sqrt(-2*math.Log(r)/r)
}
<commit_msg>remove author comment<commit_after>// Package rng implements a series of pseudo-random number generator
// based on a variety of common probability distributions
package rng

import (
	"math"
)

// GaussianGenerator is a random number generator for gaussian distribution.
// The zero value is invalid, use NewGaussianGenerator to create a generator
type GaussianGenerator struct {
	uniform *UniformGenerator
}

// NewGaussianGenerator returns a gaussian-distribution generator
// it is recommended using time.Now().UnixNano() as the seed, for example:
// crng := rng.NewGaussianGenerator(time.Now().UnixNano())
func NewGaussianGenerator(seed int64) *GaussianGenerator {
	urng := NewUniformGenerator(seed)
	return &GaussianGenerator{urng}
}

// StdGaussian returns a random number of standard gaussian distribution
func (grng GaussianGenerator) StdGaussian() float64 {
	return grng.gaussian()
}

// Gaussian returns a random number of gaussian distribution Gauss(mean, stddev^2)
func (grng GaussianGenerator) Gaussian(mean, stddev float64) float64 {
	return mean + stddev*grng.gaussian()
}

func (grng GaussianGenerator) gaussian() float64 {
	// Box-Muller Transform
	var r, x, y float64
	for r >= 1 || r == 0 {
		x = grng.uniform.Float64Range(-1.0, 1.0)
		y = grng.uniform.Float64Range(-1.0, 1.0)
		r = x*x + y*y
	}
	return x * math.Sqrt(-2*math.Log(r)/r)
}
<|endoftext|>


<commit_before>package backend

import (
	"bytes"
	"errors"
	"fmt"
	"image/gif"
	"os/exec"

	"github.com/thoas/picfit/image"
)

// Gifsicle is the gifsicle backend.
type Gifsicle struct {
	Path string
}

func (b *Gifsicle) String() string {
	return "gifsicle"
}

// Fit implements Backend.
func (b *Gifsicle) Fit(*image.ImageFile, *Options) ([]byte, error) {
	return nil, MethodNotImplementedError
}

// Flat implements Backend.
func (b *Gifsicle) Flat(*image.ImageFile, *Options) ([]byte, error) {
	return nil, MethodNotImplementedError
}

// Flip implements Backend.
func (b *Gifsicle) Flip(*image.ImageFile, *Options) ([]byte, error) {
	return nil, MethodNotImplementedError
}

// Resize implements Backend.
func (b *Gifsicle) Resize(imgfile *image.ImageFile, opts *Options) ([]byte, error) {
	cmd := exec.Command(b.Path,
		"--resize", fmt.Sprintf("%dx%d", opts.Width, opts.Height),
	)
	cmd.Stdin = bytes.NewReader(imgfile.Source)
	stdout := new(bytes.Buffer)
	cmd.Stdout = stdout
	stderr := new(bytes.Buffer)
	cmd.Stderr = stderr

	var target *exec.ExitError
	if err := cmd.Run(); errors.As(err, &target) && target.Exited() {
		return nil, errors.New(stderr.String())
	} else if err != nil {
		return nil, err
	}
	return stdout.Bytes(), nil
}

// Rotate implements Backend.
func (b *Gifsicle) Rotate(*image.ImageFile, *Options) ([]byte, error) {
	return nil, MethodNotImplementedError
}

// Thumbnail implements Backend.
func (b *Gifsicle) Thumbnail(imgfile *image.ImageFile, opts *Options) ([]byte, error) {
	img, err := gif.Decode(bytes.NewReader(imgfile.Source))
	if err != nil {
		return nil, err
	}
	bounds := img.Bounds()
	left, top, cropw, croph := computecrop(bounds.Dx(), bounds.Dy(), opts.Width, opts.Height)

	cmd := exec.Command(b.Path,
		"--crop", fmt.Sprintf("%d,%d+%dx%d", left, top, cropw, croph),
		"--resize", fmt.Sprintf("%dx%d", opts.Width, opts.Height),
	)
	cmd.Stdin = bytes.NewReader(imgfile.Source)
	stdout := new(bytes.Buffer)
	cmd.Stdout = stdout
	stderr := new(bytes.Buffer)
	cmd.Stderr = stderr

	var target *exec.ExitError
	if err := cmd.Run(); errors.As(err, &target) && target.Exited() {
		return nil, errors.New(stderr.String())
	} else if err != nil {
		return nil, err
	}
	return stdout.Bytes(), nil
}

func computecrop(srcw, srch, destw, desth int) (left, top, cropw, croph int) {
	srcratio := float64(srcw) / float64(srch)
	destratio := float64(destw) / float64(desth)

	if srcratio > destratio {
		cropw = int((destratio * float64(srch)) + 0.5)
		croph = srch
	} else {
		croph = int((float64(srcw) / destratio) + 0.5)
		cropw = srcw
	}

	left = int(float64(srcw-cropw) * 0.5)
	if left < 0 {
		left = 0
	}

	top = int(float64(srch-croph) * 0.5)
	if top < 0 {
		top = 0
	}
	return
}
<commit_msg>feat: add upscale option to gifsicle<commit_after>package backend

import (
	"bytes"
	"errors"
	"fmt"
	"image/gif"
	"os/exec"

	"github.com/thoas/picfit/image"
)

// Gifsicle is the gifsicle backend.
type Gifsicle struct {
	Path string
}

func (b *Gifsicle) String() string {
	return "gifsicle"
}

// Resize implements Backend.
func (b *Gifsicle) Resize(imgfile *image.ImageFile, opts *Options) ([]byte, error) {
	img, err := gif.Decode(bytes.NewReader(imgfile.Source))
	if err != nil {
		return nil, err
	}
	factor := scalingFactorImage(img, opts.Width, opts.Height)
	if factor > 1 && !opts.Upscale {
		return imgfile.Source, nil
	}

	cmd := exec.Command(b.Path,
		"--resize", fmt.Sprintf("%dx%d", opts.Width, opts.Height),
	)
	cmd.Stdin = bytes.NewReader(imgfile.Source)
	stdout := new(bytes.Buffer)
	cmd.Stdout = stdout
	stderr := new(bytes.Buffer)
	cmd.Stderr = stderr

	var target *exec.ExitError
	if err := cmd.Run(); errors.As(err, &target) && target.Exited() {
		return nil, errors.New(stderr.String())
	} else if err != nil {
		return nil, err
	}
	return stdout.Bytes(), nil
}

// Thumbnail implements Backend.
func (b *Gifsicle) Thumbnail(imgfile *image.ImageFile, opts *Options) ([]byte, error) {
	img, err := gif.Decode(bytes.NewReader(imgfile.Source))
	if err != nil {
		return nil, err
	}
	factor := scalingFactorImage(img, opts.Width, opts.Height)
	if factor > 1 && !opts.Upscale {
		return imgfile.Source, nil
	}

	bounds := img.Bounds()
	left, top, cropw, croph := computecrop(bounds.Dx(), bounds.Dy(), opts.Width, opts.Height)

	cmd := exec.Command(b.Path,
		"--crop", fmt.Sprintf("%d,%d+%dx%d", left, top, cropw, croph),
		"--resize", fmt.Sprintf("%dx%d", opts.Width, opts.Height),
	)
	cmd.Stdin = bytes.NewReader(imgfile.Source)
	stdout := new(bytes.Buffer)
	cmd.Stdout = stdout
	stderr := new(bytes.Buffer)
	cmd.Stderr = stderr

	var target *exec.ExitError
	if err := cmd.Run(); errors.As(err, &target) && target.Exited() {
		return nil, errors.New(stderr.String())
	} else if err != nil {
		return nil, err
	}
	return stdout.Bytes(), nil
}

// Rotate implements Backend.
func (b *Gifsicle) Rotate(*image.ImageFile, *Options) ([]byte, error) {
	return nil, MethodNotImplementedError
}

// Fit implements Backend.
func (b *Gifsicle) Fit(*image.ImageFile, *Options) ([]byte, error) {
	return nil, MethodNotImplementedError
}

// Flat implements Backend.
func (b *Gifsicle) Flat(*image.ImageFile, *Options) ([]byte, error) {
	return nil, MethodNotImplementedError
}

// Flip implements Backend.
func (b *Gifsicle) Flip(*image.ImageFile, *Options) ([]byte, error) {
	return nil, MethodNotImplementedError
}

func computecrop(srcw, srch, destw, desth int) (left, top, cropw, croph int) {
	srcratio := float64(srcw) / float64(srch)
	destratio := float64(destw) / float64(desth)

	if srcratio > destratio {
		cropw = int((destratio * float64(srch)) + 0.5)
		croph = srch
	} else {
		croph = int((float64(srcw) / destratio) + 0.5)
		cropw = srcw
	}

	left = int(float64(srcw-cropw) * 0.5)
	if left < 0 {
		left = 0
	}

	top = int(float64(srch-croph) * 0.5)
	if top < 0 {
		top = 0
	}
	return
}
<|endoftext|>


<commit_before>// +build !appengine

package fasthttp

import (
	"bytes"
	"io"
	"mime/multipart"

	"github.com/labstack/echo/engine"
	"github.com/labstack/gommon/log"
	"github.com/valyala/fasthttp"
)

type (
	// Request implements `engine.Request`.
	Request struct {
		*fasthttp.RequestCtx
		url    engine.URL
		header engine.Header
		logger *log.Logger
	}
)

// NewRequest returns `Request` instance.
func NewRequest(c *fasthttp.RequestCtx, l *log.Logger) *Request {
	return &Request{
		RequestCtx: c,
		url:        &URL{URI: c.URI()},
		header:     &RequestHeader{RequestHeader: &c.Request.Header},
		logger:     l,
	}
}

// IsTLS implements `engine.Request#TLS` function.
func (r *Request) IsTLS() bool {
	return r.IsTLS()
}

// Scheme implements `engine.Request#Scheme` function.
func (r *Request) Scheme() string {
	return string(r.RequestCtx.URI().Scheme())
}

// Host implements `engine.Request#Host` function.
func (r *Request) Host() string {
	return string(r.RequestCtx.Host())
}

// URL implements `engine.Request#URL` function.
func (r *Request) URL() engine.URL {
	return r.url
}

// Header implements `engine.Request#Header` function.
func (r *Request) Header() engine.Header {
	return r.header
}

// ContentLength implements `engine.Request#ContentLength` function.
func (r *Request) ContentLength() int {
	return r.Request.Header.ContentLength()
}

// UserAgent implements `engine.Request#UserAgent` function.
func (r *Request) UserAgent() string {
	return string(r.RequestCtx.UserAgent())
}

// RemoteAddress implements `engine.Request#RemoteAddress` function.
func (r *Request) RemoteAddress() string {
	return r.RemoteAddr().String()
}

// Method implements `engine.Request#Method` function.
func (r *Request) Method() string {
	return string(r.RequestCtx.Method())
}

// SetMethod implements `engine.Request#SetMethod` function.
func (r *Request) SetMethod(method string) {
	r.Request.Header.SetMethod(method)
}

// URI implements `engine.Request#URI` function.
func (r *Request) URI() string {
	return string(r.RequestURI())
}

// SetURI implements `engine.Request#SetURI` function.
func (r *Request) SetURI(uri string) {
	r.Request.Header.SetRequestURI(uri)
}

// Body implements `engine.Request#Body` function.
func (r *Request) Body() io.Reader {
	return bytes.NewBuffer(r.PostBody())
}

// FormValue implements `engine.Request#FormValue` function.
func (r *Request) FormValue(name string) string {
	return string(r.RequestCtx.FormValue(name))
}

// FormParams implements `engine.Request#FormParams` function.
func (r *Request) FormParams() (params map[string][]string) {
	params = make(map[string][]string)
	r.PostArgs().VisitAll(func(k, v []byte) {
		// TODO: Filling with only first value
		params[string(k)] = []string{string(v)}
	})
	return
}

// FormFile implements `engine.Request#FormFile` function.
func (r *Request) FormFile(name string) (*multipart.FileHeader, error) {
	return r.RequestCtx.FormFile(name)
}

// MultipartForm implements `engine.Request#MultipartForm` function.
func (r *Request) MultipartForm() (*multipart.Form, error) {
	return r.RequestCtx.MultipartForm()
}

func (r *Request) reset(c *fasthttp.RequestCtx, h engine.Header, u engine.URL) {
	r.RequestCtx = c
	r.header = h
	r.url = u
}
<commit_msg>Fix infinite recursion<commit_after>// +build !appengine

package fasthttp

import (
	"bytes"
	"io"
	"mime/multipart"

	"github.com/labstack/echo/engine"
	"github.com/labstack/gommon/log"
	"github.com/valyala/fasthttp"
)

type (
	// Request implements `engine.Request`.
	Request struct {
		*fasthttp.RequestCtx
		url    engine.URL
		header engine.Header
		logger *log.Logger
	}
)

// NewRequest returns `Request` instance.
func NewRequest(c *fasthttp.RequestCtx, l *log.Logger) *Request {
	return &Request{
		RequestCtx: c,
		url:        &URL{URI: c.URI()},
		header:     &RequestHeader{RequestHeader: &c.Request.Header},
		logger:     l,
	}
}

// IsTLS implements `engine.Request#TLS` function.
func (r *Request) IsTLS() bool {
	return r.RequestCtx.IsTLS()
}

// Scheme implements `engine.Request#Scheme` function.
func (r *Request) Scheme() string {
	return string(r.RequestCtx.URI().Scheme())
}

// Host implements `engine.Request#Host` function.
func (r *Request) Host() string {
	return string(r.RequestCtx.Host())
}

// URL implements `engine.Request#URL` function.
func (r *Request) URL() engine.URL {
	return r.url
}

// Header implements `engine.Request#Header` function.
func (r *Request) Header() engine.Header {
	return r.header
}

// ContentLength implements `engine.Request#ContentLength` function.
func (r *Request) ContentLength() int {
	return r.Request.Header.ContentLength()
}

// UserAgent implements `engine.Request#UserAgent` function.
func (r *Request) UserAgent() string {
	return string(r.RequestCtx.UserAgent())
}

// RemoteAddress implements `engine.Request#RemoteAddress` function.
func (r *Request) RemoteAddress() string {
	return r.RemoteAddr().String()
}

// Method implements `engine.Request#Method` function.
func (r *Request) Method() string {
	return string(r.RequestCtx.Method())
}

// SetMethod implements `engine.Request#SetMethod` function.
func (r *Request) SetMethod(method string) {
	r.Request.Header.SetMethod(method)
}

// URI implements `engine.Request#URI` function.
func (r *Request) URI() string {
	return string(r.RequestURI())
}

// SetURI implements `engine.Request#SetURI` function.
func (r *Request) SetURI(uri string) {
	r.Request.Header.SetRequestURI(uri)
}

// Body implements `engine.Request#Body` function.
func (r *Request) Body() io.Reader {
	return bytes.NewBuffer(r.PostBody())
}

// FormValue implements `engine.Request#FormValue` function.
func (r *Request) FormValue(name string) string {
	return string(r.RequestCtx.FormValue(name))
}

// FormParams implements `engine.Request#FormParams` function.
func (r *Request) FormParams() (params map[string][]string) {
	params = make(map[string][]string)
	r.PostArgs().VisitAll(func(k, v []byte) {
		// TODO: Filling with only first value
		params[string(k)] = []string{string(v)}
	})
	return
}

// FormFile implements `engine.Request#FormFile` function.
func (r *Request) FormFile(name string) (*multipart.FileHeader, error) {
	return r.RequestCtx.FormFile(name)
}

// MultipartForm implements `engine.Request#MultipartForm` function.
func (r *Request) MultipartForm() (*multipart.Form, error) {
	return r.RequestCtx.MultipartForm()
}

func (r *Request) reset(c *fasthttp.RequestCtx, h engine.Header, u engine.URL) {
	r.RequestCtx = c
	r.header = h
	r.url = u
}
<|endoftext|>


<commit_before>// geomi, 거미, is a basic spider for crawling specific webs. It was designed to crawl
// either a site, or a subset of a site, indexing the pages and where they link to.
// Any links that go outside of the provided base URL are noted but not followed.
//
//
// Even though geomi was designed with the intended use being crawling ones own site,
// or a client's site, it does come with some basic behavior configuration to make it
// a friendly bot:
//   * respects ROBOTS.txt TODO
//   * configurable concurrent walkers TODO
//   * configurable wait interval range TODO
//   * configurable max requests per: TODO
//     * min
//     * hour
//     * day
//     * month
//
// To start, go get the geomi package:
//    go get github.com/mohae/geomi
//
// Import it into your code:
//    import github.com/mohae/geomi
//
// Set up the site for the spider:
//   s := geomi.NewSite("http://example.com")
//
// The url that is passed to the NewSite() function is the base case, after which
// a BFS traversal is done.
//
package geomi

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/mohae/utilitybelt/queue"
	"github.com/temoto/robotstxt.go"
	"golang.org/x/net/html"
)

// Fetcher is an interface that makes it easier to test. In the future, it may be
// useful for other purposes, but that would require exporting the method that uses
// it first.
type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page
	Fetch(url string) (body string, urls []string, err error)
}

//}
// fetched tracks URLs that have been (or are being) fetched.
// The lock must be held while reading from or writing to the map.
var fetched = struct {
	m map[string]error
	sync.Mutex
}{m: make(map[string]error)}

var loading = errors.New("url load in progress") // sentinel value

// a page is a url. This is usually some content wiht a number of elements, but it
// could be something as simple as the url for cat.jpg on your site and represent that
// image.
type Page struct {
	*url.URL
	distance int
	body     string
	links    []string // immediate children
}

// Site is a type that implements fetcher
type Site struct {
	*url.URL
}

// Implements fetcher.
// TODO: make the design cleaner
func (s Site) Fetch(url string) (body string, urls []string, err error) {
	// see if the passed url is outside of the baseURL
	resp, err := http.Get(url)
	if err != nil {
		return "", nil, err
	}
	defer resp.Body.Close()
	buff := &bytes.Buffer{}
	tee := io.TeeReader(resp.Body, buff)
	tokens := getTokens(tee)
	if len(tokens) == 0 {
		return "", nil, fmt.Errorf("%s: nothing in body", url)
	}
	urls, err = s.linksFromTokens(tokens)
	if err != nil {
		return "", nil, err
	}
	return buff.String(), urls, nil
}

// linksFromTokens returns a list of links (href a) found in the token slice
// TODO should internal links be tracked separatly? i.e. record them in a
// separate var (so they don't get fetched)
func (s *Site) linksFromTokens(tokens []html.Token) ([]string, error) {
	var links []string
	for _, token := range tokens {
		if token.Type == html.StartTagToken && token.DataAtom.String() == "a" {
			for _, attr := range token.Attr {
				// We only care about links that aren't #
				if attr.Key == "href" && !strings.HasPrefix(attr.Val, "#") {
					// parse the url
					url, err := url.Parse(attr.Val)
					if err != nil {
						return nil, err
					}
					link := s.URL.ResolveReference(url)
					links = append(links, link.String())
				}
			}
		}
	}
	return links, nil
}

// Spider crawls the target. It contains all information needed to manage the crawl
// including keeping track of work to do, work done, and the results.
type Spider struct {
	*queue.Queue
	sync.Mutex
	wg                 sync.WaitGroup
	*url.URL                  // the start url
	bot                string // the name of the bot for robots
	concurrency        int    // concurrency level of crawling
	fetchInterval      int64  // if > 0, interval, in milliseconds to wait between gets
	intervalJitter     int64  // the max jitter to be added, per fetch. Jitter is a rand with this as max.
	RestrictToScheme   bool   // if true, only crawls urls with the same scheme as baseURL
	RespectRobots      bool   // whether or not to respect the robots.txt
	CheckExternalLinks bool   // Check the status of any links that go to other domains
	robots             *robotstxt.Group
	maxDepth           int
	Pages              map[string]Page
	foundURLs          map[string]struct{} // keeps track of urls found to prevent recrawling
	fetchedURLs        map[string]error    // urls that have been fetched with their status
	skippedURLs        map[string]struct{} // urls within the same domain that are not retrieved
	externalHosts      map[string]struct{} // list of external hosts TODO: elide?
	externalLinks      map[string]struct{} // list of external links; not fetched
}

// returns a Spider with the its site's baseUrl set. The baseUrl is the start point for
// the crawl. It is also the restriction on the crawl:
func NewSpider(start string) (*Spider, error) {
	if start == "" {
		return nil, errors.New("newSpider: the start url cannot be empty")
	}
	var err error
	spider := &Spider{Queue: queue.New(128, 0),
		bot:           "geomi",
		RespectRobots: true,
		Pages:         make(map[string]Page),
		foundURLs:     make(map[string]struct{}),
		fetchedURLs:   make(map[string]error),
		skippedURLs:   make(map[string]struct{}),
		externalHosts: make(map[string]struct{}),
		externalLinks: make(map[string]struct{}),
	}
	spider.URL, err = url.Parse(start)
	if err != nil {
		return nil, err
	}
	return spider, nil
}

// SetFetchInterval set's the spiders wait interval, and calculates the intervalJitter
// value for the random jitter added to the wait. This value is in milliseconds, 1000
// means 1 second.
//
// When fetchInterval > 0:
// time between fetches = fetchInterval + rand(intervalJitter)
// max time between fetches = fetchInterval + intervalJitter
func (s *Spider) SetFetchInterval(i int64) {
	s.fetchInterval = i
	jitter := i / 5 // jitter is, at most 20% of the fetchInterval, Min interval is 10ms
	// 10ms is used as the floor because arbitrary. A larger floor might be reasonable
	if jitter >= 10 {
		s.intervalJitter = jitter
	}
}

// Crawl is the exposed method for starting a crawl at baseURL. The crawl private method
// does the actual work. The depth is the maximum depth, or distance, to crawl from the
// baseURL. If depth == -1, no limits are set and it is expected that the entire site
// will be crawled.
func (s *Spider) Crawl(depth int) (message string, err error) {
	s.maxDepth = depth
	S := Site{URL: s.URL}
	// if we are to respect the robots.txt, set up the info
	if s.RespectRobots {
		s.getRobotsTxt()
	}
	s.Queue.Enqueue(Page{URL: s.URL})
	err = s.crawl(S)
	return fmt.Sprintf("%d nodes were processed; %d external links linking to %d external hosts were not processed", len(s.Pages), len(s.externalLinks), len(s.externalHosts)), err
}

// This crawl does all the work.
func (s *Spider) crawl(fetcher Fetcher) error {
	var err error
	for !s.Queue.IsEmpty() {
		// get next item from queue
		page := s.Queue.Dequeue().(Page)
		// if a depth value was passed and the distance is > depth, we are done
		// depth of 0 means no limit
		if s.maxDepth != -1 && page.distance > s.maxDepth {
			return nil
		}
		// see if this is an external url
		if s.externalURL(page.URL) {
			if s.CheckExternalLinks {
				s.fetchExternalLink(page.URL)
			}
			continue
		}
		// check to see if this url should be skipped for other reasons
		if s.skip(page.URL) {
			// see if the skipped is external and process accordingly
			continue
		}
		s.foundURLs[page.URL.String()] = struct{}{}
		// get the url
		page.body, page.links, err = fetcher.Fetch(page.URL.String())
		// add the page and status to the map. map isn't checked for membership becuase we don't
		// fetch found urls.
		s.Lock()
		s.Pages[page.URL.String()] = page
		s.fetchedURLs[page.URL.String()] = err
		s.Unlock()
		// add the urls that the node contains to the queue
		for _, l := range page.links {
			u, _ := url.Parse(l)
			s.Queue.Enqueue(Page{URL: u, distance: page.distance + 1})
		}
		// if their is a wait between fetches, sleep for that + random jitter
		if s.fetchInterval > 0 {
			wait := s.fetchInterval
			// if there is a value for jitter, add a random jitter
			if s.intervalJitter > 0 {
				wait += rand.Int63n(s.intervalJitter)
			}
			time.Sleep(time.Duration(wait) * time.Millisecond)
		}
	}
	return nil
}

// skip determines whether the url should be skipped.
//   * skip urls that have already been fetched
//   * skip urls that are outside of the basePath
//   * conditionally skip urls that are outside of the current scheme, even if
//     they are within the current pasePath
//   * skip if not allowed by robots
func (s *Spider) skip(u *url.URL) bool {
	s.Lock()
	_, ok := s.foundURLs[u.String()]
	s.Unlock()
	if ok { // if it was found, skip it
		s.addSkippedURL(u)
		return true
	}

	// skip if we are restricted to current scheme
	if s.RestrictToScheme {
		if u.Scheme != s.URL.Scheme {
			s.addSkippedURL(u)
			return true
		}
	}
	if s.RespectRobots {
		ok := s.robotsAllowed(u)
		if !ok {
			return false
		}
	}
	// skip if the url is outside of base
	// remove the scheme + schemePrefix so just the rest of the url is being compared
	if !strings.HasPrefix(u.Path, s.URL.Path) {
		s.addSkippedURL(u)
		return true
	}
	return false
}

// addSkippedURL add's the url info too the skipped info
func (s *Spider) addSkippedURL(u *url.URL) {
	s.Lock()
	s.skippedURLs[u.Path] = struct{}{}
	s.Unlock()
}

// externalURL check's to see if the url is external to the site, host isn't the same.
// and add's that info to the ext structs.
func (s *Spider) externalURL(u *url.URL) bool {
	if u.Host != s.URL.Host {
		s.Lock()
		// see if the host is already in the map
		_, ok := s.externalHosts[u.Host]
		if !ok {
			s.externalHosts[u.Host] = struct{}{}
		}
		// same with url
		_, ok = s.externalLinks[u.String()]
		if !ok {
			s.externalLinks[u.String()] = struct{}{}
		}
		s.Unlock()
		return true
	}
	return false
}

// getRobotsTxt retrieves and processes the site's robot.txt. If the robots.txt doesn't
// exist, it is assumed that everything is allowed.
func (s *Spider) getRobotsTxt() error {
	resp, err := http.Get("http://" + s.URL.Host)
	if err != nil {
		return err
	}
	robots, err := robotstxt.FromResponse(resp)
	if err != nil {
		return err
	}
	s.robots = robots.FindGroup(s.bot)
	return nil
}

// robotsAllowed checks to see if the passed path is allowed by Robots.txt. If the
// robots isn't set, it's always true
func (s *Spider) robotsAllowed(u *url.URL) bool {
	if s.robots != nil {
		return s.robots.Test(u.Path)
	}
	return true
}

// getTokens returns all tokens in the body
func getTokens(body io.Reader) []html.Token {
	tokens := make([]html.Token, 0)
	page := html.NewTokenizer(body)
	for {
		typ := page.Next()
		if typ == html.ErrorToken {
			return tokens
		}
		tokens = append(tokens, page.Token())
	}
}

func init() {
	// We just use math/rand because it's good enough for our purpose.
	rand.Seed(time.Now().UTC().UnixNano())
}
<commit_msg>add support for checking status of external links<commit_after>// geomi, 거미, is a basic spider for crawling specific webs. It was designed to crawl
// either a site, or a subset of a site, indexing the pages and where they link to.
// Any links that go outside of the provided base URL are noted but not followed.
//
//
// Even though geomi was designed with the intended use being crawling ones own site,
// or a client's site, it does come with some basic behavior configuration to make it
// a friendly bot:
//   * respects ROBOTS.txt TODO
//   * configurable concurrent walkers TODO
//   * configurable wait interval range TODO
//   * configurable max requests per: TODO
//     * min
//     * hour
//     * day
//     * month
//
// To start, go get the geomi package:
//    go get github.com/mohae/geomi
//
// Import it into your code:
//    import github.com/mohae/geomi
//
// Set up the site for the spider:
//   s := geomi.NewSite("http://example.com")
//
// The url that is passed to the NewSite() function is the base case, after which
// a BFS traversal is done.
//
package geomi

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/mohae/utilitybelt/queue"
	"github.com/temoto/robotstxt.go"
	"golang.org/x/net/html"
)

// Fetcher is an interface that makes it easier to test. In the future, it may be
// useful for other purposes, but that would require exporting the method that uses
// it first.
type Fetcher interface {
	// Fetch returns the body of URL and
	// a slice of URLs found on that page
	Fetch(url string) (body string, urls []string, err error)
}

//}
// fetched tracks URLs that have been (or are being) fetched.
// The lock must be held while reading from or writing to the map.
var fetched = struct {
	m map[string]error
	sync.Mutex
}{m: make(map[string]error)}

var loading = errors.New("url load in progress") // sentinel value

// a page is a url. This is usually some content wiht a number of elements, but it
// could be something as simple as the url for cat.jpg on your site and represent that
// image.
type Page struct {
	*url.URL
	distance int
	body     string
	links    []string // immediate children
}

// responseInfo contains the status and error information from a get
type responseInfo struct {
	Status     string
	StatusCode int
	Err        error
}

// Site is a type that implements fetcher
type Site struct {
	*url.URL
}

// Implements fetcher.
// TODO: make the design cleaner
func (s Site) Fetch(url string) (body string, urls []string, err error) {
	// see if the passed url is outside of the baseURL
	resp, err := http.Get(url)
	if err != nil {
		return "", nil, err
	}
	defer resp.Body.Close()
	buff := &bytes.Buffer{}
	tee := io.TeeReader(resp.Body, buff)
	tokens := getTokens(tee)
	if len(tokens) == 0 {
		return "", nil, fmt.Errorf("%s: nothing in body", url)
	}
	urls, err = s.linksFromTokens(tokens)
	if err != nil {
		return "", nil, err
	}
	return buff.String(), urls, nil
}

// linksFromTokens returns a list of links (href a) found in the token slice
// TODO should internal links be tracked separatly? i.e. record them in a
// separate var (so they don't get fetched)
func (s *Site) linksFromTokens(tokens []html.Token) ([]string, error) {
	var links []string
	for _, token := range tokens {
		if token.Type == html.StartTagToken && token.DataAtom.String() == "a" {
			for _, attr := range token.Attr {
				// We only care about links that aren't #
				if attr.Key == "href" && !strings.HasPrefix(attr.Val, "#") {
					// parse the url
					url, err := url.Parse(attr.Val)
					if err != nil {
						return nil, err
					}
					link := s.URL.ResolveReference(url)
					links = append(links, link.String())
				}
			}
		}
	}
	return links, nil
}

// Spider crawls the target. It contains all information needed to manage the crawl
// including keeping track of work to do, work done, and the results.
type Spider struct {
	*queue.Queue
	sync.Mutex
	wg                 sync.WaitGroup
	*url.URL                  // the start url
	bot                string // the name of the bot for robots
	concurrency        int    // concurrency level of crawling
	fetchInterval      int64  // if > 0, interval, in milliseconds to wait between gets
	intervalJitter     int64  // the max jitter to be added, per fetch. Jitter is a rand with this as max.
	RestrictToScheme   bool   // if true, only crawls urls with the same scheme as baseURL
	RespectRobots      bool   // whether or not to respect the robots.txt
	CheckExternalLinks bool   // Check the status of any links that go to other domains
	robots             *robotstxt.Group
	maxDepth           int
	Pages              map[string]Page
	foundURLs          map[string]struct{}     // keeps track of urls found to prevent recrawling
	fetchedURLs        map[string]error        // urls that have been fetched with their status
	skippedURLs        map[string]struct{}     // urls within the same domain that are not retrieved
	externalHosts      map[string]struct{}     // list of external hosts TODO: elide?
	externalLinks      map[string]responseInfo // list of external links; if fetched,
}

// returns a Spider with the its site's baseUrl set. The baseUrl is the start point for
// the crawl. It is also the restriction on the crawl:
func NewSpider(start string) (*Spider, error) {
	if start == "" {
		return nil, errors.New("newSpider: the start url cannot be empty")
	}
	var err error
	spider := &Spider{Queue: queue.New(128, 0),
		bot:           "geomi",
		RespectRobots: true,
		Pages:         make(map[string]Page),
		foundURLs:     make(map[string]struct{}),
		fetchedURLs:   make(map[string]error),
		skippedURLs:   make(map[string]struct{}),
		externalHosts: make(map[string]struct{}),
		externalLinks: make(map[string]responseInfo),
	}
	spider.URL, err = url.Parse(start)
	if err != nil {
		return nil, err
	}
	return spider, nil
}

// SetFetchInterval set's the spiders wait interval, and calculates the intervalJitter
// value for the random jitter added to the wait. This value is in milliseconds, 1000
// means 1 second.
//
// When fetchInterval > 0:
// time between fetches = fetchInterval + rand(intervalJitter)
// max time between fetches = fetchInterval + intervalJitter
func (s *Spider) SetFetchInterval(i int64) {
	s.fetchInterval = i
	jitter := i / 5 // jitter is, at most 20% of the fetchInterval, Min interval is 10ms
	// 10ms is used as the floor because arbitrary. A larger floor might be reasonable
	if jitter >= 10 {
		s.intervalJitter = jitter
	}
}

// Crawl is the exposed method for starting a crawl at baseURL. The crawl private method
// does the actual work. The depth is the maximum depth, or distance, to crawl from the
// baseURL. If depth == -1, no limits are set and it is expected that the entire site
// will be crawled.
func (s *Spider) Crawl(depth int) (message string, err error) {
	s.maxDepth = depth
	S := Site{URL: s.URL}
	// if we are to respect the robots.txt, set up the info
	if s.RespectRobots {
		s.getRobotsTxt()
	}
	s.Queue.Enqueue(Page{URL: s.URL})
	err = s.crawl(S)
	return fmt.Sprintf("%d nodes were processed; %d external links linking to %d external hosts were not processed", len(s.Pages), len(s.externalLinks), len(s.externalHosts)), err
}

// This crawl does all the work.
func (s *Spider) crawl(fetcher Fetcher) error {
	var err error
	for !s.Queue.IsEmpty() {
		// get next item from queue
		page := s.Queue.Dequeue().(Page)
		// if a depth value was passed and the distance is > depth, we are done
		// depth of 0 means no limit
		if s.maxDepth != -1 && page.distance > s.maxDepth {
			return nil
		}
		// see if this is an external url
		if s.externalURL(page.URL) {
			if s.CheckExternalLinks {
				s.fetchExternalLink(page.URL)
			}
			continue
		}
		// check to see if this url should be skipped for other reasons
		if s.skip(page.URL) {
			// see if the skipped is external and process accordingly
			continue
		}
		s.foundURLs[page.URL.String()] = struct{}{}
		// get the url
		page.body, page.links, err = fetcher.Fetch(page.URL.String())
		// add the page and status to the map. map isn't checked for membership becuase we don't
		// fetch found urls.
		s.Lock()
		s.Pages[page.URL.String()] = page
		s.fetchedURLs[page.URL.String()] = err
		s.Unlock()
		// add the urls that the node contains to the queue
		for _, l := range page.links {
			u, _ := url.Parse(l)
			s.Queue.Enqueue(Page{URL: u, distance: page.distance + 1})
		}
		// if their is a wait between fetches, sleep for that + random jitter
		if s.fetchInterval > 0 {
			wait := s.fetchInterval
			// if there is a value for jitter, add a random jitter
			if s.intervalJitter > 0 {
				wait += rand.Int63n(s.intervalJitter)
			}
			time.Sleep(time.Duration(wait) * time.Millisecond)
		}
	}
	return nil
}

// skip determines whether the url should be skipped.
//   * skip urls that have already been fetched
//   * skip urls that are outside of the basePath
//   * conditionally skip urls that are outside of the current scheme, even if
//     they are within the current pasePath
//   * skip if not allowed by robots
func (s *Spider) skip(u *url.URL) bool {
	s.Lock()
	_, ok := s.foundURLs[u.String()]
	s.Unlock()
	if ok { // if it was found, skip it
		s.addSkippedURL(u)
		return true
	}

	// skip if we are restricted to current scheme
	if s.RestrictToScheme {
		if u.Scheme != s.URL.Scheme {
			s.addSkippedURL(u)
			return true
		}
	}
	if s.RespectRobots {
		ok := s.robotsAllowed(u)
		if !ok {
			return false
		}
	}
	// skip if the url is outside of base
	// remove the scheme + schemePrefix so just the rest of the url is being compared
	if !strings.HasPrefix(u.Path, s.URL.Path) {
		s.addSkippedURL(u)
		return true
	}
	return false
}

// addSkippedURL add's the url info too the skipped info
func (s *Spider) addSkippedURL(u *url.URL) {
	s.Lock()
	s.skippedURLs[u.Path] = struct{}{}
	s.Unlock()
}

// externalURL check's to see if the url is external to the site, host isn't the same.
// and add's that info to the ext structs.
func (s *Spider) externalURL(u *url.URL) bool {
	if u.Host != s.URL.Host {
		s.Lock()
		// see if the host is already in the map
		_, ok := s.externalHosts[u.Host]
		if !ok {
			s.externalHosts[u.Host] = struct{}{}
		}
		// same with url
		_, ok = s.externalLinks[u.String()]
		if !ok {
			s.externalLinks[u.String()] = responseInfo{}
		}
		s.Unlock()
		return true
	}
	return false
}

// fetchExternalLink: fetches an external link and check's it status. Note, this
// does not implement fetcher
func (s *Spider) fetchExternalLink(u *url.URL) error {
	// if this has already benn fetched, don't
	var ri responseInfo
	s.Lock()
	r, _ := s.externalLinks[u.String()]
	if r != ri { // if !0 value, it's been retrieved
		s.Unlock()
		return nil
	}
	s.Unlock()
	resp, err := http.Get(u.String())
	resp.Body.Close()
	if err != nil {
		r.Err = err
		s.Lock()
		s.externalLinks[u.String()] = r
		s.Unlock()
		return err
	}
	r.Status = resp.Status
	r.StatusCode = resp.StatusCode
	s.Lock()
	s.externalLinks[u.String()] = r
	s.Unlock()
	return nil
}

// getRobotsTxt retrieves and processes the site's robot.txt. If the robots.txt doesn't
// exist, it is assumed that everything is allowed.
func (s *Spider) getRobotsTxt() error {
	resp, err := http.Get("http://" + s.URL.Host)
	if err != nil {
		return err
	}
	robots, err := robotstxt.FromResponse(resp)
	if err != nil {
		return err
	}
	s.robots = robots.FindGroup(s.bot)
	return nil
}

// robotsAllowed checks to see if the passed path is allowed by Robots.txt. If the
// robots isn't set, it's always true
func (s *Spider) robotsAllowed(u *url.URL) bool {
	if s.robots != nil {
		return s.robots.Test(u.Path)
	}
	return true
}

// getTokens returns all tokens in the body
func getTokens(body io.Reader) []html.Token {
	tokens := make([]html.Token, 0)
	page := html.NewTokenizer(body)
	for {
		typ := page.Next()
		if typ == html.ErrorToken {
			return tokens
		}
		tokens = append(tokens, page.Token())
	}
}

func init() {
	// We just use math/rand because it's good enough for our purpose.
	rand.Seed(time.Now().UTC().UnixNano())
}
<|endoftext|>