<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcloud

import (
	"fmt"
	"math/rand"
	"strings"
	"time"

	"golang.org/x/crypto/ssh/agent"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/googleapi"

	"github.com/coreos/mantle/util"
)

func (a *API) vmname() string {
	return fmt.Sprintf("mantle-%x", rand.Int63())
}

// Taken from: https://github.com/golang/build/blob/master/buildlet/gce.go
func (a *API) mkinstance(userdata, name string, keys []*agent.Key) *compute.Instance {
	var metadataItems []*compute.MetadataItems
	if len(keys) > 0 {
		var sshKeys string
		for i, key := range keys {
			sshKeys += fmt.Sprintf("%d:%s\n", i, key)
		}

		metadataItems = append(metadataItems, &compute.MetadataItems{
			Key:   "ssh-keys",
			Value: &sshKeys,
		})
	}

	prefix := "https://www.googleapis.com/compute/v1/projects/" + a.options.Project
	instance := &compute.Instance{
		Name:        name,
		MachineType: prefix + "/zones/" + a.options.Zone + "/machineTypes/" + a.options.MachineType,
		Metadata: &compute.Metadata{
			Items: metadataItems,
		},
		Disks: []*compute.AttachedDisk{
			{
				AutoDelete: true,
				Boot:       true,
				Type:       "PERSISTENT",
				InitializeParams: &compute.AttachedDiskInitializeParams{
					DiskName:    name,
					SourceImage: prefix + "/global/images/" + a.options.Image,
					DiskType:    "/zones/" + a.options.Zone + "/diskTypes/" + a.options.DiskType,
				},
			},
		},
		NetworkInterfaces: []*compute.NetworkInterface{
			&compute.NetworkInterface{
				AccessConfigs: []*compute.AccessConfig{
					&compute.AccessConfig{
						Type: "ONE_TO_ONE_NAT",
						Name: "External NAT",
					},
				},
				Network: prefix + "/global/networks/" + a.options.Network,
			},
		},
	}
	// add cloud config
	if userdata != "" {
		instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{
			Key:   "user-data",
			Value: &userdata,
		})
	}

	return instance

}

type doable interface {
	Do(opts ...googleapi.CallOption) (*compute.Operation, error)
}

func (a *API) waitop(operation string, do doable) error {
	retry := func() error {
		op, err := do.Do()
		if err != nil {
			return err
		}

		switch op.Status {
		case "PENDING", "RUNNING":
			return fmt.Errorf("Operation %q is %q", operation, op.Status)
		case "DONE":
			if op.Error != nil {
				for _, operr := range op.Error.Errors {
					return fmt.Errorf("Error creating instance: %+v", operr)
				}
				return fmt.Errorf("Operation %q failed to start", op.Status)
			}

			return nil
		}

		return fmt.Errorf("Unknown operation status %q: %+v", op.Status, op)
	}

	// 5 minutes
	if err := util.Retry(30, 10*time.Second, retry); err != nil {
		return fmt.Errorf("Failed to wait for operation %q: %v", operation, err)
	}

	return nil
}

// CreateInstance creates a Google Compute Engine instance.
func (a *API) CreateInstance(userdata string, keys []*agent.Key) (*compute.Instance, error) {
	name := a.vmname()
	inst := a.mkinstance(userdata, name, keys)

	plog.Debugf("Creating instance %q", name)

	op, err := a.compute.Instances.Insert(a.options.Project, a.options.Zone, inst).Do()
	if err != nil {
		return nil, fmt.Errorf("failed to request new GCE instance: %v\n", err)
	}

	doable := a.compute.ZoneOperations.Get(a.options.Project, a.options.Zone, op.Name)
	if err := a.waitop(op.Name, doable); err != nil {
		return nil, err
	}

	inst, err = a.compute.Instances.Get(a.options.Project, a.options.Zone, name).Do()
	if err != nil {
		return nil, fmt.Errorf("failed getting instance %s details after creation: %v", name, err)
	}

	plog.Debugf("Created instance %q", name)

	return inst, nil
}

func (a *API) TerminateInstance(name string) error {
	plog.Debugf("Terminating instance %q", name)

	_, err := a.compute.Instances.Delete(a.options.Project, a.options.Zone, name).Do()
	return err
}

func (a *API) ListInstances(prefix string) ([]*compute.Instance, error) {
	var instances []*compute.Instance

	list, err := a.compute.Instances.List(a.options.Project, a.options.Zone).Do()
	if err != nil {
		return nil, err
	}

	for _, inst := range list.Items {
		if !strings.HasPrefix(inst.Name, prefix) {
			continue
		}

		instances = append(instances, inst)
	}

	return instances, nil
}

// Taken from: https://github.com/golang/build/blob/master/buildlet/gce.go
func InstanceIPs(inst *compute.Instance) (intIP, extIP string) {
	for _, iface := range inst.NetworkInterfaces {
		if strings.HasPrefix(iface.NetworkIP, "10.") {
			intIP = iface.NetworkIP
		}
		for _, accessConfig := range iface.AccessConfigs {
			if accessConfig.Type == "ONE_TO_ONE_NAT" {
				extIP = accessConfig.NatIP
			}
		}
	}
	return
}
<commit_msg>platform/api/gcloud: allow http and https<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcloud

import (
	"fmt"
	"math/rand"
	"strings"
	"time"

	"golang.org/x/crypto/ssh/agent"
	"google.golang.org/api/compute/v1"
	"google.golang.org/api/googleapi"

	"github.com/coreos/mantle/util"
)

func (a *API) vmname() string {
	return fmt.Sprintf("mantle-%x", rand.Int63())
}

// Taken from: https://github.com/golang/build/blob/master/buildlet/gce.go
func (a *API) mkinstance(userdata, name string, keys []*agent.Key) *compute.Instance {
	var metadataItems []*compute.MetadataItems
	if len(keys) > 0 {
		var sshKeys string
		for i, key := range keys {
			sshKeys += fmt.Sprintf("%d:%s\n", i, key)
		}

		metadataItems = append(metadataItems, &compute.MetadataItems{
			Key:   "ssh-keys",
			Value: &sshKeys,
		})
	}

	prefix := "https://www.googleapis.com/compute/v1/projects/" + a.options.Project
	instance := &compute.Instance{
		Name:        name,
		MachineType: prefix + "/zones/" + a.options.Zone + "/machineTypes/" + a.options.MachineType,
		Metadata: &compute.Metadata{
			Items: metadataItems,
		},
		Tags: &compute.Tags{
			// Apparently you need this tag in addition to the
			// firewall rules to open the port because these ports
			// are special?
			Items: []string{"https-server", "http-server"},
		},
		Disks: []*compute.AttachedDisk{
			{
				AutoDelete: true,
				Boot:       true,
				Type:       "PERSISTENT",
				InitializeParams: &compute.AttachedDiskInitializeParams{
					DiskName:    name,
					SourceImage: prefix + "/global/images/" + a.options.Image,
					DiskType:    "/zones/" + a.options.Zone + "/diskTypes/" + a.options.DiskType,
				},
			},
		},
		NetworkInterfaces: []*compute.NetworkInterface{
			&compute.NetworkInterface{
				AccessConfigs: []*compute.AccessConfig{
					&compute.AccessConfig{
						Type: "ONE_TO_ONE_NAT",
						Name: "External NAT",
					},
				},
				Network: prefix + "/global/networks/" + a.options.Network,
			},
		},
	}
	// add cloud config
	if userdata != "" {
		instance.Metadata.Items = append(instance.Metadata.Items, &compute.MetadataItems{
			Key:   "user-data",
			Value: &userdata,
		})
	}

	return instance

}

type doable interface {
	Do(opts ...googleapi.CallOption) (*compute.Operation, error)
}

func (a *API) waitop(operation string, do doable) error {
	retry := func() error {
		op, err := do.Do()
		if err != nil {
			return err
		}

		switch op.Status {
		case "PENDING", "RUNNING":
			return fmt.Errorf("Operation %q is %q", operation, op.Status)
		case "DONE":
			if op.Error != nil {
				for _, operr := range op.Error.Errors {
					return fmt.Errorf("Error creating instance: %+v", operr)
				}
				return fmt.Errorf("Operation %q failed to start", op.Status)
			}

			return nil
		}

		return fmt.Errorf("Unknown operation status %q: %+v", op.Status, op)
	}

	// 5 minutes
	if err := util.Retry(30, 10*time.Second, retry); err != nil {
		return fmt.Errorf("Failed to wait for operation %q: %v", operation, err)
	}

	return nil
}

// CreateInstance creates a Google Compute Engine instance.
func (a *API) CreateInstance(userdata string, keys []*agent.Key) (*compute.Instance, error) {
	name := a.vmname()
	inst := a.mkinstance(userdata, name, keys)

	plog.Debugf("Creating instance %q", name)

	op, err := a.compute.Instances.Insert(a.options.Project, a.options.Zone, inst).Do()
	if err != nil {
		return nil, fmt.Errorf("failed to request new GCE instance: %v\n", err)
	}

	doable := a.compute.ZoneOperations.Get(a.options.Project, a.options.Zone, op.Name)
	if err := a.waitop(op.Name, doable); err != nil {
		return nil, err
	}

	inst, err = a.compute.Instances.Get(a.options.Project, a.options.Zone, name).Do()
	if err != nil {
		return nil, fmt.Errorf("failed getting instance %s details after creation: %v", name, err)
	}

	plog.Debugf("Created instance %q", name)

	return inst, nil
}

func (a *API) TerminateInstance(name string) error {
	plog.Debugf("Terminating instance %q", name)

	_, err := a.compute.Instances.Delete(a.options.Project, a.options.Zone, name).Do()
	return err
}

func (a *API) ListInstances(prefix string) ([]*compute.Instance, error) {
	var instances []*compute.Instance

	list, err := a.compute.Instances.List(a.options.Project, a.options.Zone).Do()
	if err != nil {
		return nil, err
	}

	for _, inst := range list.Items {
		if !strings.HasPrefix(inst.Name, prefix) {
			continue
		}

		instances = append(instances, inst)
	}

	return instances, nil
}

// Taken from: https://github.com/golang/build/blob/master/buildlet/gce.go
func InstanceIPs(inst *compute.Instance) (intIP, extIP string) {
	for _, iface := range inst.NetworkInterfaces {
		if strings.HasPrefix(iface.NetworkIP, "10.") {
			intIP = iface.NetworkIP
		}
		for _, accessConfig := range iface.AccessConfigs {
			if accessConfig.Type == "ONE_TO_ONE_NAT" {
				extIP = accessConfig.NatIP
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package editdistance

import "testing"

var distanceTests = []struct {
	i string
	j string
	d int
}{
	{"kitten", "sitting", 3},
	{"sitting", "kitten", 3},
	{"cat", "dog", 3},
	{"hog", "dog", 1},
	{"frog", "frogfrog", 4},
	{"hog", "frog", 2},
	{"frog", "log", 2},
	{"intention", "execution", 5},
}

func TestDistance(t *testing.T) {
	for _, tt := range distanceTests {
		d := distance(tt.i, tt.j)
		if d != tt.d {
			t.Errorf("distance(%v, %v) Got %v, want %v", tt.i, tt.j, d, tt.d)
		}
	}
}
<commit_msg>break break break<commit_after>package editdistance

import "testing"

var distanceTests = []struct {
	i string
	j string
	d int
}{
	{"kitten", "sitting", 3},
	{"sitting", "kitten", 3},
	{"cat", "dog", 3},
	{"hog", "dog", 1},
	{"frog", "frogfrog", 4},
	{"hog", "frog", 2},
	{"frog", "log", 2},
	{"intention", "execution", 5},
	{"abcdefgh", "1ab2cd3ef4gh5", 5},
}

func TestDistance(t *testing.T) {
	for _, tt := range distanceTests {
		d := distance(tt.i, tt.j)
		if d != tt.d {
			t.Errorf("distance(%v, %v) Got %v, want %v", tt.i, tt.j, d, tt.d)
		}
	}
}
<|endoftext|>


<commit_before>package plugins_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/ArjenSchwarz/igor/config"
	"github.com/ArjenSchwarz/igor/plugins"
	"github.com/ArjenSchwarz/igor/slack"
)

// TestNaughtyStrings calls every plugin with the list of naughtystrings
// (https://github.com/minimaxir/big-list-of-naughty-strings).
func TestNaughtyStrings(t *testing.T) {
	err := os.Setenv("IGOR_CONFIG", "{\"token\": \"testtoken\"}")
	if err != nil {
		t.Error("Problem setting environment variable")
	}
	var list []string

	filename, _ := filepath.Abs("../devtools/blns.json")
	c, _ := ioutil.ReadFile(filename)
	dec := json.NewDecoder(bytes.NewReader(c))
	dec.Decode(&list)
	config, err := config.GeneralConfig()
	if err != nil {
		t.Error("Problem getting config")
	}
	request := slack.Request{Text: "string"}
	for _, plugin := range plugins.GetPlugins(request, config) {
		for _, string := range list {
			_, err := plugin.Work()
			if err != nil {
				switch err.(type) {
				case *plugins.NoMatchError:
				default:
					t.Error(fmt.Sprintf("Failed naughty string: %s - %s > %s",
						string,
						plugin.Name(),
						err.Error()))
				}
			}
		}
	}
}
<commit_msg>Split up the tests for 1.7<commit_after>package plugins_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/ArjenSchwarz/igor/config"
	"github.com/ArjenSchwarz/igor/plugins"
	"github.com/ArjenSchwarz/igor/slack"
)

// TestNaughtyStrings calls every plugin with the list of naughtystrings
// (https://github.com/minimaxir/big-list-of-naughty-strings).
func TestNaughtyStrings(t *testing.T) {
	err := os.Setenv("IGOR_CONFIG", "{\"token\": \"testtoken\"}")
	if err != nil {
		t.Error("Problem setting environment variable")
	}
	var list []string

	filename, _ := filepath.Abs("../devtools/blns.json")
	c, _ := ioutil.ReadFile(filename)
	dec := json.NewDecoder(bytes.NewReader(c))
	dec.Decode(&list)
	config, err := config.GeneralConfig()
	if err != nil {
		t.Error("Problem getting config")
	}
	request := slack.Request{Text: "string"}
	for _, plugin := range plugins.GetPlugins(request, config) {
		t.Run("Plugin="+plugin.Name(), func(t *testing.T) {
			for _, string := range list {
				_, err := plugin.Work()
				if err != nil {
					switch err.(type) {
					case *plugins.NoMatchError:
					default:
						t.Error(fmt.Sprintf("Failed naughty string: %s - %s > %s",
							string,
							plugin.Name(),
							err.Error()))
					}
				}
			}

		})
	}
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2016 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package simple

import (
	"context"
	"crypto/tls"
	"io"
	"io/ioutil"
	"math"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"time"

	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/lib/netext"
	"github.com/loadimpact/k6/stats"
)

type Runner struct {
	URL       *url.URL
	Transport *http.Transport
	Options   lib.Options

	defaultGroup *lib.Group
}

func New(u *url.URL) (*Runner, error) {
	return &Runner{
		URL: u,
		Transport: &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			DialContext: netext.NewDialer(net.Dialer{
				Timeout:   10 * time.Second,
				KeepAlive: 60 * time.Second,
				DualStack: true,
			}).DialContext,
			TLSClientConfig:     &tls.Config{},
			MaxIdleConns:        math.MaxInt32,
			MaxIdleConnsPerHost: math.MaxInt32,
		},
		defaultGroup: &lib.Group{},
	}, nil
}

func (r *Runner) MakeArchive() *lib.Archive {
	return &lib.Archive{
		Type:     "url",
		Filename: r.URL.String(),
	}
}

func (r *Runner) NewVU() (lib.VU, error) {
	return &VU{
		Runner:    r,
		URLString: r.URL.String(),
		Request: &http.Request{
			Method: "GET",
			URL:    r.URL,
		},
		Client: &http.Client{
			Transport: r.Transport,
		},
		tracer: &netext.Tracer{},
	}, nil
}

func (r *Runner) GetDefaultGroup() *lib.Group {
	return &lib.Group{}
}

func (r Runner) GetOptions() lib.Options {
	return r.Options
}

func (r *Runner) ApplyOptions(opts lib.Options) {
	r.Options = r.Options.Apply(opts)
	r.Transport.TLSClientConfig.InsecureSkipVerify = opts.InsecureSkipTLSVerify.Bool
}

type VU struct {
	Runner   *Runner
	ID       int64
	IDString string

	URLString string
	Request   *http.Request
	Client    *http.Client

	tracer *netext.Tracer
}

func (u *VU) RunOnce(ctx context.Context) ([]stats.Sample, error) {
	tags := map[string]string{
		"vu":     u.IDString,
		"status": "0",
		"method": "GET",
		"url":    u.URLString,
	}

	resp, err := u.Client.Do(u.Request.WithContext(netext.WithTracer(ctx, u.tracer)))
	if err != nil {
		return u.tracer.Done().Samples(tags), err
	}
	tags["status"] = strconv.Itoa(resp.StatusCode)

	if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
		return u.tracer.Done().Samples(tags), err
	}
	_ = resp.Body.Close()

	return u.tracer.Done().Samples(tags), nil
}

func (u *VU) Reconfigure(id int64) error {
	u.ID = id
	u.IDString = strconv.FormatInt(id, 10)
	return nil
}
<commit_msg>Dropped the old Simple runner code<commit_after><|endoftext|>


<commit_before>package control

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/ondevice/ondevice/config"
	"github.com/sirupsen/logrus"
)

// request -- control socket request
type request struct {
	endpoint string

	body io.Reader
}

// Do -- run request request
func (r request) Do(method string) response {
	transport := &http.Transport{
		Dial: func(proto, addr string) (conn net.Conn, err error) {
			urls := getSocketURLs()
			var firstError error

			for _, url := range urls {
				var protocol, path string

				if url.Scheme == "unix" || url.Scheme == "" {
					protocol = "unix"
					path = url.Path
				} else if url.Scheme == "http" {
					protocol = "tcp"
					path = url.Host
				}

				c, err := net.Dial(protocol, path)
				if err == nil {
					// it worked
					return c, nil
				} else if firstError == nil {
					firstError = err
				}
			}

			return nil, firstError
		},
	}

	// remove leading slashes
	for strings.HasPrefix(r.endpoint, "/") {
		r.endpoint = r.endpoint[1:]
	}

	// TODO do proper URL parsing
	client := &http.Client{Transport: transport}

	var req, err = http.NewRequest(method, "http://ondevice/"+r.endpoint, r.body)
	if err != nil {
		return response{err: err}
	}

	var resp *http.Response
	if resp, err = client.Do(req); err != nil {
		return response{err: err}
	}

	if resp.StatusCode != 200 {
		return response{err: fmt.Errorf("Unexpected device request response code: %s", resp.Status)}
	}

	return response{resp: resp}
}

func (r request) Get() response {
	return r.Do("GET")
}

func getSocketURLs() []url.URL {
	if env := os.Getenv("ONDEVICE_HOST"); env != "" {
		// e.g.:
		// - unix:///var/run/ondevice/ondevice.sock
		// - /var/run/ondevice/ondevice.sock
		// - http://localhost:1234/

		u, err := url.Parse(env)
		if err != nil {
			logrus.WithError(err).Fatal("failed to parse ONDEVICE_HOST")
		}

		return []url.URL{*u}
	}

	return []url.URL{
		{Scheme: "unix", Path: config.MustLoad().GetFilePath(config.PathOndeviceSock)},
		{Scheme: "unix", Path: "/var/run/ondevice/ondevice.sock"},
	}
}
<commit_msg>control: added request.PostForm()<commit_after>package control

import (
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/ondevice/ondevice/config"
	"github.com/sirupsen/logrus"
)

// request -- control socket request
type request struct {
	endpoint string

	body io.Reader
}

// Do -- run request request
func (r request) Do(method string) response {
	transport := &http.Transport{
		Dial: func(proto, addr string) (conn net.Conn, err error) {
			urls := getSocketURLs()
			var firstError error

			for _, url := range urls {
				var protocol, path string

				if url.Scheme == "unix" || url.Scheme == "" {
					protocol = "unix"
					path = url.Path
				} else if url.Scheme == "http" {
					protocol = "tcp"
					path = url.Host
				}

				c, err := net.Dial(protocol, path)
				if err == nil {
					// it worked
					return c, nil
				} else if firstError == nil {
					firstError = err
				}
			}

			return nil, firstError
		},
	}

	// remove leading slashes
	for strings.HasPrefix(r.endpoint, "/") {
		r.endpoint = r.endpoint[1:]
	}

	// TODO do proper URL parsing
	client := &http.Client{Transport: transport}

	var req, err = http.NewRequest(method, "http://ondevice/"+r.endpoint, r.body)
	if err != nil {
		return response{err: err}
	}

	var resp *http.Response
	if resp, err = client.Do(req); err != nil {
		return response{err: err}
	}

	if resp.StatusCode != 200 {
		return response{err: fmt.Errorf("Unexpected device request response code: %s", resp.Status)}
	}

	return response{resp: resp}
}

func (r request) Get() response {
	return r.Do("GET")
}

func (r request) PostForm(form url.Values) response {
	r.body = strings.NewReader(form.Encode())
	return r.Do("POST")
}

func getSocketURLs() []url.URL {
	if env := os.Getenv("ONDEVICE_HOST"); env != "" {
		// e.g.:
		// - unix:///var/run/ondevice/ondevice.sock
		// - /var/run/ondevice/ondevice.sock
		// - http://localhost:1234/

		u, err := url.Parse(env)
		if err != nil {
			logrus.WithError(err).Fatal("failed to parse ONDEVICE_HOST")
		}

		return []url.URL{*u}
	}

	return []url.URL{
		{Scheme: "unix", Path: config.MustLoad().GetFilePath(config.PathOndeviceSock)},
		{Scheme: "unix", Path: "/var/run/ondevice/ondevice.sock"},
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"bytes"
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"
	"github.com/eirka/eirka-libs/user"

	local "github.com/eirka/eirka-post/config"
)

func init() {

	// Database connection settings
	dbase := db.Database{

		User:           local.Settings.Database.User,
		Password:       local.Settings.Database.Password,
		Proto:          local.Settings.Database.Proto,
		Host:           local.Settings.Database.Host,
		Database:       local.Settings.Database.Database,
		MaxIdle:        local.Settings.Database.MaxIdle,
		MaxConnections: local.Settings.Database.MaxConnections,
	}

	// Set up DB connection
	dbase.NewDb()

	// Get limits and stuff from database
	config.GetDatabaseSettings()

	user.Secret = "secret"
}

func performRequest(r http.Handler, method, path string) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, nil)
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func performJwtJsonRequest(r http.Handler, method, path, token string, body []byte) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, bytes.NewBuffer(body))
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("Content-Type", "application/json")
	fmt.Println(req)
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func performJwtFormRequest(r http.Handler, method, path, token string, body bytes.Buffer) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, &body)
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("Content-Type", "multipart/form-data")
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func TestEmailController(t *testing.T) {

	var err error

	gin.SetMode(gin.ReleaseMode)

	router := gin.New()

	router.Use(user.Auth(true))

	router.POST("/email", EmailController)

	first := performRequest(router, "POST", "/email")

	assert.Equal(t, first.Code, 401, "HTTP request code should match")

	u := user.DefaultUser()
	u.SetId(2)
	u.SetAuthenticated()
	u.Password()

	assert.True(t, u.ComparePassword("testpassword"), "Test user password should be set")

	token, err := u.CreateToken()
	if assert.NoError(t, err, "An error was not expected") {
		assert.NotEmpty(t, token, "token should be returned")
	}

	request := []byte(`{"ib":"1","email":"test@test.com"}`)

	second := performJwtJsonRequest(router, "POST", "/email", token, request)

	fmt.Println(second)

	assert.Equal(t, second.Code, 200, "HTTP request code should match")

}
<commit_msg>add email test<commit_after>package controllers

import (
	"bytes"
	"fmt"
	"github.com/gin-gonic/gin"
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/eirka/eirka-libs/config"
	"github.com/eirka/eirka-libs/db"
	"github.com/eirka/eirka-libs/user"

	local "github.com/eirka/eirka-post/config"
)

func init() {

	// Database connection settings
	dbase := db.Database{

		User:           local.Settings.Database.User,
		Password:       local.Settings.Database.Password,
		Proto:          local.Settings.Database.Proto,
		Host:           local.Settings.Database.Host,
		Database:       local.Settings.Database.Database,
		MaxIdle:        local.Settings.Database.MaxIdle,
		MaxConnections: local.Settings.Database.MaxConnections,
	}

	// Set up DB connection
	dbase.NewDb()

	// Get limits and stuff from database
	config.GetDatabaseSettings()

	user.Secret = "secret"
}

func performRequest(r http.Handler, method, path string) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, nil)
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func performJwtJsonRequest(r http.Handler, method, path, token string, body []byte) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, bytes.NewBuffer(body))
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("Content-Type", "application/json")
	fmt.Println(req)
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func performJwtFormRequest(r http.Handler, method, path, token string, body bytes.Buffer) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(method, path, &body)
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
	req.Header.Set("Content-Type", "multipart/form-data")
	w := httptest.NewRecorder()
	r.ServeHTTP(w, req)
	return w
}

func TestEmailController(t *testing.T) {

	var err error

	gin.SetMode(gin.ReleaseMode)

	router := gin.New()

	router.Use(user.Auth(true))

	router.POST("/email", EmailController)

	first := performRequest(router, "POST", "/email")

	assert.Equal(t, first.Code, 401, "HTTP request code should match")

	u := user.DefaultUser()
	u.SetId(2)
	u.SetAuthenticated()
	u.Password()

	assert.True(t, u.ComparePassword("testpassword"), "Test user password should be set")

	token, err := u.CreateToken()
	if assert.NoError(t, err, "An error was not expected") {
		assert.NotEmpty(t, token, "token should be returned")
	}

	request := []byte(`{ib: 1, email: "test@test.com"}`)

	second := performJwtJsonRequest(router, "POST", "/email", token, request)

	fmt.Println(second)

	assert.Equal(t, second.Code, 200, "HTTP request code should match")

}
<|endoftext|>


<commit_before>package kafkafs

import (
	"fmt"
	"log"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"
)

// Implements a FUSE filesystem backed by a Kafka installation.
//
// This version is read only, so it cannot post to topics, only read
// from them.
type KafkaRoFs struct {
	pathfs.FileSystem

	KafkaClient KafkaClient
	userFiles   map[string]bool
}

type parsedPath struct {
	IsValid   bool
	IsRoot    bool
	Topic     string
	Partition int32
	Offset    int64
}

func (fs *KafkaRoFs) parseAndValidate(name string) (parsedPath, error) {
	parsed := parsedPath{IsValid: true, IsRoot: false, Topic: "", Partition: -1,
		Offset: -1}
	slashed := filepath.ToSlash(name)
	re := regexp.MustCompile("/{2,}")
	normal := re.ReplaceAllString(slashed, "/")
	if normal == "" {
		parsed.IsRoot = true
		return parsed, nil
	}

	splits := strings.Split(normal, "/")

	if len(splits) > 3 {
		parsed.IsValid = false
		return parsed, nil
	}

	if len(splits) >= 1 {
		maybeTopic := splits[0]
		isTop, err := fs.isTopic(maybeTopic)
		if err != nil {
			return parsed, err
		}
		if !isTop {
			parsed.IsValid = false
			return parsed, nil
		}
		parsed.Topic = maybeTopic
	}

	if len(splits) >= 2 {
		maybePartition := splits[1]
		isPart, err := fs.isPartition(parsed.Topic, maybePartition)
		if err != nil {
			return parsed, err
		}
		if !isPart {
			parsed.IsValid = false
			return parsed, nil
		}
		// this should always succeed if isPartition returned true.
		partition64, _ := strconv.ParseInt(maybePartition, 10, 32)
		parsed.Partition = int32(partition64)
	}

	if len(splits) == 3 {
		maybeOffset := splits[2]
		offset, err := strconv.ParseInt(maybeOffset, 10, 64)
		if err != nil {
			parsed.IsValid = false
			return parsed, nil
		}
		parsed.Offset = offset
	}

	return parsed, nil
}

func (fs *KafkaRoFs) isTopic(maybeTopic string) (bool, error) {
	topics, err := fs.KafkaClient.GetTopics()
	if err != nil {
		return false, err
	}

	for _, topic := range topics {
		if topic == maybeTopic {
			return true, nil
		}
	}

	return false, nil
}

func (fs *KafkaRoFs) isPartition(topic string, maybePartition string) (bool, error) {
	maybePartition64, err := strconv.ParseInt(maybePartition, 10, 32)
	if err != nil {
		return false, nil
	}
	maybePartition32 := int32(maybePartition64)
	partitions, err := fs.KafkaClient.GetPartitions(topic)
	if err != nil {
		return false, err
	}

	for _, partition := range partitions {
		if partition == maybePartition32 {
			return true, nil
		}
	}

	return false, nil
}

func (fs *KafkaRoFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr,
	fuse.Status) {
	log.Printf("GetAttr name: %s", name)

	parsed, err := fs.parseAndValidate(name)

	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid {
		return nil, fuse.ENOENT
	}

	switch {
	// root or a topic
	case parsed.IsRoot, parsed.Offset == -1 && parsed.Partition == -1:
		return &fuse.Attr{Mode: fuse.S_IFDIR | 0500}, fuse.OK
	// partition
	case parsed.Offset == -1:
		return &fuse.Attr{Mode: fuse.S_IFDIR | 0700}, fuse.OK
	// offset / msg
	case true:
		msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return nil, fuse.EIO
		}
		if msgBytes == nil {
			return nil, fuse.ENOENT
		}
		fs.userFiles[name] = true
		return &fuse.Attr{Mode: fuse.S_IFREG | 0400,
			Size: uint64(len(msgBytes))}, fuse.OK
	}

	return nil, fuse.ENOENT
}

func (fs *KafkaRoFs) OpenDir(name string, context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	log.Printf("OpenDir name: %s", name)

	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid {
		return nil, fuse.ENOENT
	}

	switch {
	case parsed.IsRoot:
		return fs.openRoot(context)
	case parsed.Partition == -1:
		return fs.openTopic(parsed.Topic, context)
	case parsed.Offset == -1:
		return fs.openPartition(parsed.Topic, parsed.Partition, context)
	}

	return nil, fuse.ENOENT
}

func (fs *KafkaRoFs) openPartition(topic string, partition int32,
	context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	earliest, next, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return nil, fuse.EIO
	}
	if next == int64(0) {
		// this means an empty partition
		return []fuse.DirEntry{}, fuse.OK
	}

	entries := []fuse.DirEntry{fuse.DirEntry{Name: strconv.FormatInt(earliest, 10),
		Mode: fuse.S_IFREG}}
	if earliest != next-1 {
		entries = append(entries, fuse.DirEntry{Name: strconv.FormatInt(next-1, 10),
			Mode: fuse.S_IFREG})
	}

	for path, _ := range fs.userFiles {
		parsed, err := fs.parseAndValidate(path)
		if err != nil {
			log.Panicf("Error was non-nil, bad user path %s", err)
		}
		if parsed.Partition == partition && parsed.Offset != earliest &&
			parsed.Offset != next-1 {
			entries = append(entries, fuse.DirEntry{
				Name: strconv.FormatInt(parsed.Offset, 10), Mode: fuse.S_IFREG})
		}
	}

	return entries, fuse.OK
}

func (fs *KafkaRoFs) openRoot(context *fuse.Context) ([]fuse.DirEntry, fuse.Status) {
	// root, show all the topics
	var topicDirs []fuse.DirEntry
	topics, err := fs.KafkaClient.GetTopics()

	if err != nil {
		return nil, fuse.EIO
	}

	for _, topic := range topics {
		topicDirs = append(topicDirs, fuse.DirEntry{Name: topic, Mode: fuse.S_IFDIR})
	}

	return topicDirs, fuse.OK
}

func (fs *KafkaRoFs) openTopic(name string, context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	var partitionDirs []fuse.DirEntry
	partitions, err := fs.KafkaClient.GetPartitions(name)

	if err != nil {
		return nil, fuse.EIO
	}

	for _, partition := range partitions {
		partitionDirs = append(partitionDirs,
			fuse.DirEntry{Name: strconv.FormatInt(int64(partition), 10),
				Mode: fuse.S_IFDIR})
	}

	return partitionDirs, fuse.OK
}

func (fs *KafkaRoFs) Open(name string, flags uint32,
	context *fuse.Context) (nodefs.File, fuse.Status) {
	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid || parsed.Offset == -1 {
		return nil, fuse.ENOENT
	}

	if flags&fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}

	msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
		parsed.Offset)
	if err != nil {
		return nil, fuse.EIO
	}

	return nodefs.NewDataFile(msgBytes), fuse.OK
}

func (fs *KafkaRoFs) Mknod(name string, mode uint32, dev uint32,
	context *fuse.Context) fuse.Status {

	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return fuse.EIO
	}

	if !parsed.IsValid || parsed.Offset == -1 {
		return fuse.ENOENT
	}

	// we don't want to block waiting on an offset that is in the past
	// and no longer available from kafka

	nE := func() (bool, error) {
		return fs.offsetNotExpired(parsed.Topic, parsed.Partition, parsed.Offset)
	}

	for notExpired, err := nE(); notExpired; notExpired, err = nE() {
		if err != nil {
			return fuse.EIO
		}

		isFuture, err := fs.offsetIsFuture(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		if isFuture {
			time.Sleep(1 * time.Second)
			continue
		}

		msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		if msgBytes == nil {
			// this shouldn't happen unless the messages are expiring
			// very fast, but who knows
			return fuse.ENOENT
		}

		err = fs.addUserFile(parsed.Topic, parsed.Partition, parsed.Offset, msgBytes)
		if err != nil {
			return fuse.EIO
		}

		return fuse.OK
	}

	return fuse.ENOENT
}

func (fs *KafkaRoFs) addUserFile(topic string, partition int32, offset int64,
	msgBytes []byte) error {
	path := fmt.Sprintf("%s/%d/%d", topic, partition, offset)
	fs.userFiles[path] = true
	return nil
}

func (fs *KafkaRoFs) offsetNotExpired(topic string, partition int32,
	offset int64) (bool, error) {
	earliest, _, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return false, err
	}
	return offset >= earliest, nil
}

func (fs *KafkaRoFs) offsetIsFuture(topic string, partition int32,
	offset int64) (bool, error) {
	_, next, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return false, err
	}
	return offset >= next, nil
}

// just pretend we set the times to keep touch happy
func (fs *KafkaRoFs) Utimens(name string, Atime *time.Time, Mtime *time.Time,
	context *fuse.Context) fuse.Status {
	return fuse.OK
}

func NewKafkaRoFs(kClient KafkaClient) *KafkaRoFs {
	return &KafkaRoFs{FileSystem: pathfs.NewDefaultFileSystem(), KafkaClient: kClient,
		userFiles: make(map[string]bool)}
}

func (fs *KafkaRoFs) Unlink(name string, context *fuse.Context) fuse.Status {
	_, ok := fs.userFiles[name]
	if !ok {
		return fuse.EPERM
	}
	delete(fs.userFiles, name)
	return fuse.OK
}
<commit_msg>Protect map in goroutines.<commit_after>package kafkafs

import (
	"fmt"
	"log"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"
)

// Implements a FUSE filesystem backed by a Kafka installation.
//
// This version is read only, so it cannot post to topics, only read
// from them.
type KafkaRoFs struct {
	pathfs.FileSystem

	KafkaClient KafkaClient
	userFiles   map[string]bool
	userFilesM  sync.RWMutex
}

type parsedPath struct {
	IsValid   bool
	IsRoot    bool
	Topic     string
	Partition int32
	Offset    int64
}

func (fs *KafkaRoFs) parseAndValidate(name string) (parsedPath, error) {
	parsed := parsedPath{IsValid: true, IsRoot: false, Topic: "", Partition: -1,
		Offset: -1}
	slashed := filepath.ToSlash(name)
	re := regexp.MustCompile("/{2,}")
	normal := re.ReplaceAllString(slashed, "/")
	if normal == "" {
		parsed.IsRoot = true
		return parsed, nil
	}

	splits := strings.Split(normal, "/")

	if len(splits) > 3 {
		parsed.IsValid = false
		return parsed, nil
	}

	if len(splits) >= 1 {
		maybeTopic := splits[0]
		isTop, err := fs.isTopic(maybeTopic)
		if err != nil {
			return parsed, err
		}
		if !isTop {
			parsed.IsValid = false
			return parsed, nil
		}
		parsed.Topic = maybeTopic
	}

	if len(splits) >= 2 {
		maybePartition := splits[1]
		isPart, err := fs.isPartition(parsed.Topic, maybePartition)
		if err != nil {
			return parsed, err
		}
		if !isPart {
			parsed.IsValid = false
			return parsed, nil
		}
		// this should always succeed if isPartition returned true.
		partition64, _ := strconv.ParseInt(maybePartition, 10, 32)
		parsed.Partition = int32(partition64)
	}

	if len(splits) == 3 {
		maybeOffset := splits[2]
		offset, err := strconv.ParseInt(maybeOffset, 10, 64)
		if err != nil {
			parsed.IsValid = false
			return parsed, nil
		}
		parsed.Offset = offset
	}

	return parsed, nil
}

func (fs *KafkaRoFs) isTopic(maybeTopic string) (bool, error) {
	topics, err := fs.KafkaClient.GetTopics()
	if err != nil {
		return false, err
	}

	for _, topic := range topics {
		if topic == maybeTopic {
			return true, nil
		}
	}

	return false, nil
}

func (fs *KafkaRoFs) isPartition(topic string, maybePartition string) (bool, error) {
	maybePartition64, err := strconv.ParseInt(maybePartition, 10, 32)
	if err != nil {
		return false, nil
	}
	maybePartition32 := int32(maybePartition64)
	partitions, err := fs.KafkaClient.GetPartitions(topic)
	if err != nil {
		return false, err
	}

	for _, partition := range partitions {
		if partition == maybePartition32 {
			return true, nil
		}
	}

	return false, nil
}

func (fs *KafkaRoFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr,
	fuse.Status) {
	log.Printf("GetAttr name: %s", name)

	parsed, err := fs.parseAndValidate(name)

	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid {
		return nil, fuse.ENOENT
	}

	switch {
	// root or a topic
	case parsed.IsRoot, parsed.Offset == -1 && parsed.Partition == -1:
		return &fuse.Attr{Mode: fuse.S_IFDIR | 0500}, fuse.OK
	// partition
	case parsed.Offset == -1:
		return &fuse.Attr{Mode: fuse.S_IFDIR | 0700}, fuse.OK
	// offset / msg
	case true:
		msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return nil, fuse.EIO
		}
		if msgBytes == nil {
			return nil, fuse.ENOENT
		}
		fs.addUserFile(parsed.Topic, parsed.Partition, parsed.Offset)
		return &fuse.Attr{Mode: fuse.S_IFREG | 0400,
			Size: uint64(len(msgBytes))}, fuse.OK
	}

	return nil, fuse.ENOENT
}

func (fs *KafkaRoFs) OpenDir(name string, context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	log.Printf("OpenDir name: %s", name)

	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid {
		return nil, fuse.ENOENT
	}

	switch {
	case parsed.IsRoot:
		return fs.openRoot(context)
	case parsed.Partition == -1:
		return fs.openTopic(parsed.Topic, context)
	case parsed.Offset == -1:
		return fs.openPartition(parsed.Topic, parsed.Partition, context)
	}

	return nil, fuse.ENOENT
}

func (fs *KafkaRoFs) openPartition(topic string, partition int32,
	context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	earliest, next, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return nil, fuse.EIO
	}
	if next == int64(0) {
		// this means an empty partition
		return []fuse.DirEntry{}, fuse.OK
	}

	entries := []fuse.DirEntry{fuse.DirEntry{Name: strconv.FormatInt(earliest, 10),
		Mode: fuse.S_IFREG}}
	if earliest != next-1 {
		entries = append(entries, fuse.DirEntry{Name: strconv.FormatInt(next-1, 10),
			Mode: fuse.S_IFREG})
	}

	var paths []string
	fs.userFilesM.RLock()
	for path, _ := range fs.userFiles {
		paths = append(paths, path)
	}
	fs.userFilesM.RUnlock()

	for _, path := range paths {
		parsed, err := fs.parseAndValidate(path)
		if err != nil {
			log.Panicf("Error was non-nil, bad user path %s", err)
		}
		if parsed.Partition == partition && parsed.Offset != earliest &&
			parsed.Offset != next-1 {
			entries = append(entries, fuse.DirEntry{
				Name: strconv.FormatInt(parsed.Offset, 10), Mode: fuse.S_IFREG})
		}
	}

	return entries, fuse.OK
}

func (fs *KafkaRoFs) openRoot(context *fuse.Context) ([]fuse.DirEntry, fuse.Status) {
	// root, show all the topics
	var topicDirs []fuse.DirEntry
	topics, err := fs.KafkaClient.GetTopics()

	if err != nil {
		return nil, fuse.EIO
	}

	for _, topic := range topics {
		topicDirs = append(topicDirs, fuse.DirEntry{Name: topic, Mode: fuse.S_IFDIR})
	}

	return topicDirs, fuse.OK
}

func (fs *KafkaRoFs) openTopic(name string, context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	var partitionDirs []fuse.DirEntry
	partitions, err := fs.KafkaClient.GetPartitions(name)

	if err != nil {
		return nil, fuse.EIO
	}

	for _, partition := range partitions {
		partitionDirs = append(partitionDirs,
			fuse.DirEntry{Name: strconv.FormatInt(int64(partition), 10),
				Mode: fuse.S_IFDIR})
	}

	return partitionDirs, fuse.OK
}

func (fs *KafkaRoFs) Open(name string, flags uint32,
	context *fuse.Context) (nodefs.File, fuse.Status) {
	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid || parsed.Offset == -1 {
		return nil, fuse.ENOENT
	}

	if flags&fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}

	msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
		parsed.Offset)
	if err != nil {
		return nil, fuse.EIO
	}

	return nodefs.NewDataFile(msgBytes), fuse.OK
}

func (fs *KafkaRoFs) Mknod(name string, mode uint32, dev uint32,
	context *fuse.Context) fuse.Status {

	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return fuse.EIO
	}

	if !parsed.IsValid || parsed.Offset == -1 {
		return fuse.ENOENT
	}

	// we don't want to block waiting on an offset that is in the past
	// and no longer available from kafka

	nE := func() (bool, error) {
		return fs.offsetNotExpired(parsed.Topic, parsed.Partition, parsed.Offset)
	}

	for notExpired, err := nE(); notExpired; notExpired, err = nE() {
		if err != nil {
			return fuse.EIO
		}

		isFuture, err := fs.offsetIsFuture(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		if isFuture {
			time.Sleep(1 * time.Second)
			continue
		}

		msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		if msgBytes == nil {
			// this shouldn't happen unless the messages are expiring
			// very fast, but who knows
			return fuse.ENOENT
		}

		err = fs.addUserFile(parsed.Topic, parsed.Partition, parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		return fuse.OK
	}

	return fuse.ENOENT
}

func (fs *KafkaRoFs) addUserFile(topic string, partition int32, offset int64) error {
	path := fmt.Sprintf("%s/%d/%d", topic, partition, offset)
	fs.userFilesM.Lock()
	fs.userFiles[path] = true
	fs.userFilesM.Unlock()
	return nil
}

func (fs *KafkaRoFs) offsetNotExpired(topic string, partition int32,
	offset int64) (bool, error) {
	earliest, _, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return false, err
	}
	return offset >= earliest, nil
}

func (fs *KafkaRoFs) offsetIsFuture(topic string, partition int32,
	offset int64) (bool, error) {
	_, next, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return false, err
	}
	return offset >= next, nil
}

// just pretend we set the times to keep touch happy
func (fs *KafkaRoFs) Utimens(name string, Atime *time.Time, Mtime *time.Time,
	context *fuse.Context) fuse.Status {
	return fuse.OK
}

func NewKafkaRoFs(kClient KafkaClient) *KafkaRoFs {
	return &KafkaRoFs{FileSystem: pathfs.NewDefaultFileSystem(), KafkaClient: kClient,
		userFiles: make(map[string]bool)}
}

func (fs *KafkaRoFs) Unlink(name string, context *fuse.Context) fuse.Status {
	_, ok := fs.userFiles[name]
	if !ok {
		return fuse.EPERM
	}
	delete(fs.userFiles, name)
	return fuse.OK
}
<|endoftext|>


<commit_before>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.
package gregex_test

import (
	"fmt"
	"github.com/gogf/gf/v2/frame/g"
	"github.com/gogf/gf/v2/text/gregex"
)

func ExampleIsMatch() {
	var str = "hello 94 easy gf!"
	patternStr := `[1-9]\d*`
	fmt.Println(gregex.IsMatch(patternStr, []byte(str)))

	// output
	// true
}

func ExampleIsMatchString() {
	var str = "hello 94 easy gf!"
	patternStr := `[1-9]\d*`
	fmt.Println(gregex.IsMatchString(patternStr, str))

	// output
	// true
}

func ExampleMatch() {
	var str = "hello 94 easy gf!"
	patternStr := `[1-9]\d*`
	result, err := gregex.Match(patternStr, []byte(str))
	if err != nil {
		fmt.Println(err.Error())
	}
	g.Dump(result)
	g.Dump(result[0])

	// output
	// ["OTQ="]
	// 94
}

func ExampleMatchAll() {
	var str = "hello 94 easy gf!"
	patternStr := `[1-9]\d*`
	results, err := gregex.MatchAll(patternStr, []byte(str))
	if err != nil {
		fmt.Println(err.Error())
	}
	g.Dump(results)

	// output
	// [["OTQ="]]
	//
}

func ExampleMatchAllString() {
	var str = "hello 94 98 easy gf!"
	patternStr := `[1-9]\d*`
	results, err := gregex.MatchAllString(patternStr, str)
	if err != nil {
		fmt.Println(err.Error())
	}

	g.Dump(results)

	// output
	// [["94"],["98"]]
}

func ExampleMatchString() {
	var str = "hello 94 98 easy gf!"
	patternStr := `[1-9]\d*`

	// if you need a greed match, should use <..all> methods
	results, err := gregex.MatchString(patternStr, str)
	if err != nil {
		fmt.Println(err.Error())
	}

	g.Dump(results)

	// output
	// ["94"]
}

func ExampleQuote() {
	patternStr := `[1-9]\d*`
	result := gregex.Quote(patternStr)
	fmt.Println(result)

	// output
	// \[1-9\]\\d\*
}

func ExampleReplace() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2020!"
	repStr := "2021"
	result, err := gregex.Replace(patternStr, []byte(repStr), []byte(str))

	g.Dump(err)
	g.Dump(result)

	// output
	// null
	// hello gf 2021!
}

func ExampleReplaceFunc() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2018~2020!"

	// When the regular statement can match multiple results
	// func can be used to further control the value that needs to be modified
	result, err := gregex.ReplaceFunc(patternStr, []byte(str), func(b []byte) []byte {

		g.Dump(b)

		replaceStr := "2021"
		if string(b) == "2020" {
			return []byte(replaceStr)
		}
		return b
	})

	g.Dump(result)
	g.Dump(err)

	// output
	// 2018
	// 2020
	// hello gf 2018~2021!
	// null
}

func ExampleReplaceFuncMatch() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2018~2020!"

	// In contrast to [ExampleReplaceFunc]
	// the result contains the `pattern' of all subpatterns that use the matching function
	result, err := gregex.ReplaceFuncMatch(patternStr, []byte(str), func(match [][]byte) []byte {

		g.Dump(match)

		replaceStr := "2021"
		for _, v := range match {
			if string(v) == "2020" {
				return []byte(replaceStr)
			}
		}
		return match[0]
	})

	g.Dump(result)
	g.Dump(err)

	// output
	// [
	//	"MjAxOA=="
	// ]
	//
	// [
	//	"MjAyMA=="
	// ]
	//
	// hello gf 2018~2021!
	// null
}

func ExampleReplaceString() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2020!"
	replaceStr := "2021"
	result, err := gregex.ReplaceString(patternStr, replaceStr, str)

	g.Dump(result)
	g.Dump(err)

	// output
	// hello gf 2021!
	// null
}

func ExampleReplaceStringFunc() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2018~2020!"

	// When the regular statement can match multiple results
	// func can be used to further control the value that needs to be modified
	result, err := gregex.ReplaceStringFunc(patternStr, str, func(b string) string {

		g.Dump(b)

		replaceStr := "2021"
		if b == "2020" {
			return replaceStr
		}
		return b
	})

	g.Dump(result)
	g.Dump(err)

	// output
	// 2018
	// 2020
	// hello gf 2018~2021!
	// null
}

func ExampleReplaceStringFuncMatch() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2018~2020!"

	// When the regular statement can match multiple results
	// func can be used to further control the value that needs to be modified
	result, err := gregex.ReplaceStringFuncMatch(patternStr, str, func(b []string) string {

		g.Dump(b)

		replaceStr := "2021"
		for _, v := range b {
			if v == "2020" {
				return replaceStr
			}
		}
		return b[0]
	})

	g.Dump(result)
	g.Dump(err)

	// output
	// ["2018"]
	// ["2020"]
	// hello gf 2018~2021!
	// null
}

func ExampleSplit() {
	patternStr := `[1-9]\d*`
	str := "hello2020gf"
	result := gregex.Split(patternStr, str)

	g.Dump(result)

	// output
	// ["hello","gf"]
}

func ExampleValidate() {
	// Valid match statement
	g.Dump(gregex.Validate(`[1-9]\d*`))
	// Mismatched statement
	g.Dump(gregex.Validate(`[a-9]\d*`))

	// output
	// null
	// {
	//	"Code": "invalid character class range",
	//	"Expr": "a-9"
	// }
}
<commit_msg>:memo: add example<commit_after>// Copyright GoFrame Author(https://goframe.org). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.
package gregex_test

import (
	"github.com/gogf/gf/v2/frame/g"
	"github.com/gogf/gf/v2/text/gregex"
)

func ExampleIsMatch() {
	patternStr := `[1-9]\d*`
	g.Dump(gregex.IsMatch(patternStr, []byte("hello 94 easy gf!")))
	g.Dump(gregex.IsMatch(patternStr, nil))
	g.Dump(gregex.IsMatch(patternStr, []byte("hello easy gf!")))

	// output
	// true
	// false
	// false
}

func ExampleIsMatchString() {
	patternStr := `[1-9]\d*`
	g.Dump(gregex.IsMatchString(patternStr, "hello 94 easy gf!"))
	g.Dump(gregex.IsMatchString(patternStr, "hello easy gf!"))
	g.Dump(gregex.IsMatchString(patternStr, ""))

	// output
	// true
	// false
	// false
}

func ExampleMatch() {
	patternStr := `[1-9]\d*`
	result, err := gregex.Match(patternStr, []byte("hello 94 98 easy gf!"))
	g.Dump(result)
	g.Dump(err)

	result, err = gregex.Match(patternStr, nil)
	g.Dump(result)
	g.Dump(err)

	result, err = gregex.Match(patternStr, []byte("hello easy gf!"))
	g.Dump(result)
	g.Dump(err)

	// output
	// ["OTQ="]
	// null
	// []
	// null
	// []
	// null
}

func ExampleMatchAll() {
	patternStr := `[1-9]\d*`
	results, err := gregex.MatchAll(patternStr, []byte("hello 94 98 easy gf!"))
	g.Dump(results)
	g.Dump(err)

	results, err = gregex.MatchAll(patternStr, []byte("hello easy gf!"))
	g.Dump(results)
	g.Dump(err)

	results, err = gregex.MatchAll(patternStr, nil)
	g.Dump(results)
	g.Dump(err)

	// output
	// [["OTQ="],["OTg="]]
	// null
	// []
	// null
	// []
	// null
}

func ExampleMatchAllString() {
	patternStr := `[1-9]\d*`
	results, err := gregex.MatchAllString(patternStr, "hello 94 98 easy gf!")
	g.Dump(results)
	g.Dump(err)

	results, err = gregex.MatchAllString(patternStr, "hello easy gf!")
	g.Dump(results)
	g.Dump(err)

	results, err = gregex.MatchAllString(patternStr, "")
	g.Dump(results)
	g.Dump(err)

	// output
	// [["94"],["98"]]
	// null
	// []
	// null
	// []
	// null
}

func ExampleMatchString() {
	var str = "hello 94 98 easy gf!"
	patternStr := `[1-9]\d*`

	// if you need a greed match, should use <..all> methods
	results, err := gregex.MatchString(patternStr, str)

	g.Dump(results)
	g.Dump(err)

	// output
	// ["94"]
	// null
}

func ExampleQuote() {
	patternStr := `[1-9]\d*`
	result := gregex.Quote(patternStr)

	g.Dump(result)

	// output
	// \[1-9\]\\d\*
}

func ExampleReplace() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2020!"
	repStr := "2021"
	result, err := gregex.Replace(patternStr, []byte(repStr), []byte(str))

	g.Dump(err)
	g.Dump(result)

	// output
	// null
	// hello gf 2021!
}

func ExampleReplaceFunc() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2018~2020!"

	// When the regular statement can match multiple results
	// func can be used to further control the value that needs to be modified
	result, err := gregex.ReplaceFunc(patternStr, []byte(str), func(b []byte) []byte {

		g.Dump(b)

		replaceStr := "2021"
		if string(b) == "2020" {
			return []byte(replaceStr)
		}
		return b
	})

	g.Dump(result)
	g.Dump(err)

	// output
	// 2018
	// 2020
	// hello gf 2018~2021!
	// null
}

func ExampleReplaceFuncMatch() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2018~2020!"

	// In contrast to [ExampleReplaceFunc]
	// the result contains the `pattern' of all subpatterns that use the matching function
	result, err := gregex.ReplaceFuncMatch(patternStr, []byte(str), func(match [][]byte) []byte {

		g.Dump(match)

		replaceStr := "2021"
		for _, v := range match {
			if string(v) == "2020" {
				return []byte(replaceStr)
			}
		}
		return match[0]
	})

	g.Dump(result)
	g.Dump(err)

	// output
	// [
	//	"MjAxOA=="
	// ]
	//
	// [
	//	"MjAyMA=="
	// ]
	//
	// hello gf 2018~2021!
	// null
}

func ExampleReplaceString() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2020!"
	replaceStr := "2021"
	result, err := gregex.ReplaceString(patternStr, replaceStr, str)

	g.Dump(result)
	g.Dump(err)

	// output
	// hello gf 2021!
	// null
}

func ExampleReplaceStringFunc() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2018~2020!"

	// When the regular statement can match multiple results
	// func can be used to further control the value that needs to be modified
	result, err := gregex.ReplaceStringFunc(patternStr, str, func(b string) string {

		g.Dump(b)

		replaceStr := "2021"
		if b == "2020" {
			return replaceStr
		}
		return b
	})

	g.Dump(result)
	g.Dump(err)

	// output
	// 2018
	// 2020
	// hello gf 2018~2021!
	// null
}

func ExampleReplaceStringFuncMatch() {
	patternStr := `[1-9]\d*`
	str := "hello gf 2018~2020!"

	// When the regular statement can match multiple results
	// func can be used to further control the value that needs to be modified
	result, err := gregex.ReplaceStringFuncMatch(patternStr, str, func(b []string) string {

		g.Dump(b)

		replaceStr := "2021"
		for _, v := range b {
			if v == "2020" {
				return replaceStr
			}
		}
		return b[0]
	})

	g.Dump(result)
	g.Dump(err)

	// output
	// ["2018"]
	// ["2020"]
	// hello gf 2018~2021!
	// null
}

func ExampleSplit() {
	patternStr := `[1-9]\d*`
	str := "hello2020gf"
	result := gregex.Split(patternStr, str)

	g.Dump(result)

	// output
	// ["hello","gf"]
}

func ExampleValidate() {
	// Valid match statement
	g.Dump(gregex.Validate(`[1-9]\d*`))
	// Mismatched statement
	g.Dump(gregex.Validate(`[a-9]\d*`))

	// output
	// null
	// {
	//	"Code": "invalid character class range",
	//	"Expr": "a-9"
	// }
}
<|endoftext|>


<commit_before>package mppostgres

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/jmoiron/sqlx"
	// PostgreSQL Driver
	_ "github.com/lib/pq"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
	"github.com/mackerelio/golib/logging"
)

var logger = logging.GetLogger("metrics.plugin.postgres")

// PostgresPlugin mackerel plugin for PostgreSQL
type PostgresPlugin struct {
	Host     string
	Port     string
	Username string
	Password string
	SSLmode  string
	Prefix   string
	Timeout  int
	Tempfile string
	Option   string
}

func fetchStatDatabase(db *sqlx.DB) (map[string]interface{}, error) {
	db = db.Unsafe()
	rows, err := db.Queryx(`SELECT * FROM pg_stat_database`)
	if err != nil {
		logger.Errorf("Failed to select pg_stat_database. %s", err)
		return nil, err
	}

	type pgStat struct {
		XactCommit   uint64   `db:"xact_commit"`
		XactRollback uint64   `db:"xact_rollback"`
		BlksRead     uint64   `db:"blks_read"`
		BlksHit      uint64   `db:"blks_hit"`
		BlkReadTime  *float64 `db:"blk_read_time"`
		BlkWriteTime *float64 `db:"blk_write_time"`
		TupReturned  uint64   `db:"tup_returned"`
		TupFetched   uint64   `db:"tup_fetched"`
		TupInserted  uint64   `db:"tup_inserted"`
		TupUpdated   uint64   `db:"tup_updated"`
		TupDeleted   uint64   `db:"tup_deleted"`
		Deadlocks    *uint64  `db:"deadlocks"`
		TempBytes    *uint64  `db:"temp_bytes"`
	}

	totalStat := pgStat{}
	for rows.Next() {
		p := pgStat{}
		if err := rows.StructScan(&p); err != nil {
			logger.Warningf("Failed to scan. %s", err)
			continue
		}
		totalStat.XactCommit += p.XactCommit
		totalStat.XactRollback += p.XactRollback
		totalStat.BlksRead += p.BlksRead
		totalStat.BlksHit += p.BlksHit
		if p.BlkReadTime != nil {
			if totalStat.BlkReadTime == nil {
				totalStat.BlkReadTime = p.BlkReadTime
			} else {
				*totalStat.BlkReadTime += *p.BlkReadTime
			}
		}
		if p.BlkWriteTime != nil {
			if totalStat.BlkWriteTime == nil {
				totalStat.BlkWriteTime = p.BlkWriteTime
			} else {
				*totalStat.BlkWriteTime += *p.BlkWriteTime
			}
		}
		totalStat.TupReturned += p.TupReturned
		totalStat.TupFetched += p.TupFetched
		totalStat.TupInserted += p.TupInserted
		totalStat.TupUpdated += p.TupUpdated
		totalStat.TupDeleted += p.TupDeleted
		if p.Deadlocks != nil {
			if totalStat.Deadlocks == nil {
				totalStat.Deadlocks = p.Deadlocks
			} else {
				*totalStat.Deadlocks += *p.Deadlocks
			}
		}
		if p.TempBytes != nil {
			if totalStat.TempBytes == nil {
				totalStat.TempBytes = p.TempBytes
			} else {
				*totalStat.TempBytes += *p.TempBytes
			}
		}
	}
	stat := make(map[string]interface{})
	stat["xact_commit"] = totalStat.XactCommit
	stat["xact_rollback"] = totalStat.XactRollback
	stat["blks_read"] = totalStat.BlksRead
	stat["blks_hit"] = totalStat.BlksHit
	if totalStat.BlkReadTime != nil {
		stat["blk_read_time"] = *totalStat.BlkReadTime
	}
	if totalStat.BlkWriteTime != nil {
		stat["blk_write_time"] = *totalStat.BlkWriteTime
	}
	stat["tup_returned"] = totalStat.TupReturned
	stat["tup_fetched"] = totalStat.TupFetched
	stat["tup_inserted"] = totalStat.TupInserted
	stat["tup_updated"] = totalStat.TupUpdated
	stat["tup_deleted"] = totalStat.TupDeleted
	if totalStat.Deadlocks != nil {
		stat["deadlocks"] = *totalStat.Deadlocks
	}
	if totalStat.TempBytes != nil {
		stat["temp_bytes"] = *totalStat.TempBytes
	}
	return stat, nil
}

func fetchConnections(db *sqlx.DB, version version) (map[string]interface{}, error) {
	var query string

	if version.first > 9 || version.first == 9 && version.second >= 6 {
		query = `select count(*), state, wait_event is not null from pg_stat_activity group by state, wait_event is not null`
	} else {
		query = `select count(*), state, waiting from pg_stat_activity group by state, waiting`
	}
	rows, err := db.Query(query)
	if err != nil {
		logger.Errorf("Failed to select pg_stat_activity. %s", err)
		return nil, err
	}

	stat := map[string]interface{}{
		"active":                      0.0,
		"active_waiting":              0.0,
		"idle":                        0.0,
		"idle_in_transaction":         0.0,
		"idle_in_transaction_aborted": 0.0,
	}

	normalizeRe := regexp.MustCompile("[^a-zA-Z0-9_-]+")

	for rows.Next() {
		var count float64
		var waiting bool
		var state string
		if err := rows.Scan(&count, &state, &waiting); err != nil {
			logger.Warningf("Failed to scan %s", err)
			continue
		}
		state = normalizeRe.ReplaceAllString(state, "_")
		state = strings.TrimRight(state, "_")
		if waiting {
			state += "_waiting"
		}
		stat[state] = float64(count)
	}

	return stat, nil
}

func fetchDatabaseSize(db *sqlx.DB) (map[string]interface{}, error) {
	rows, err := db.Query("select sum(pg_database_size(datname)) as dbsize from pg_database where has_database_privilege(datname, 'connect')")
	if err != nil {
		logger.Errorf("Failed to select pg_database_size. %s", err)
		return nil, err
	}

	var totalSize float64
	for rows.Next() {
		var dbsize float64
		if err := rows.Scan(&dbsize); err != nil {
			logger.Warningf("Failed to scan %s", err)
			continue
		}
		totalSize += dbsize
	}

	return map[string]interface{}{
		"total_size": totalSize,
	}, nil
}

var versionRe = regexp.MustCompile("PostgreSQL (\\d+)\\.(\\d+)(\\.(\\d+))? ")

type version struct {
	first  uint
	second uint
	thrird uint
}

func fetchVersion(db *sqlx.DB) (version, error) {

	res := version{}

	rows, err := db.Query("select version()")
	if err != nil {
		logger.Errorf("Failed to select version(). %s", err)
		return res, err
	}

	for rows.Next() {
		var versionStr string
		var first, second, third uint64
		if err := rows.Scan(&versionStr); err != nil {
			return res, err
		}

		// ref. https://www.postgresql.org/support/versioning/

		submatch := versionRe.FindStringSubmatch(versionStr)
		if len(submatch) >= 4 {
			first, err = strconv.ParseUint(submatch[1], 10, 0)
			if err != nil {
				return res, err
			}
			second, err = strconv.ParseUint(submatch[2], 10, 0)
			if err != nil {
				return res, err
			}
			if len(submatch) == 5 {
				third, err = strconv.ParseUint(submatch[4], 10, 0)
				if err != nil {
					return res, err
				}
			}
			res = version{uint(first), uint(second), uint(third)}
			return res, err
		}
	}
	return res, errors.New("failed to select version()")
}

func mergeStat(dst, src map[string]interface{}) {
	for k, v := range src {
		dst[k] = v
	}
}

// MetricKeyPrefix retruns the metrics key prefix
func (p PostgresPlugin) MetricKeyPrefix() string {
	if p.Prefix == "" {
		p.Prefix = "postgres"
	}
	return p.Prefix
}

// FetchMetrics interface for mackerelplugin
func (p PostgresPlugin) FetchMetrics() (map[string]interface{}, error) {

	db, err := sqlx.Connect("postgres", fmt.Sprintf("user=%s password=%s host=%s port=%s sslmode=%s connect_timeout=%d %s", p.Username, p.Password, p.Host, p.Port, p.SSLmode, p.Timeout, p.Option))
	if err != nil {
		logger.Errorf("FetchMetrics: %s", err)
		return nil, err
	}
	defer db.Close()

	version, err := fetchVersion(db)
	if err != nil {
		logger.Warningf("FetchMetrics: %s", err)
		return nil, err
	}

	statStatDatabase, err := fetchStatDatabase(db)
	if err != nil {
		return nil, err
	}
	statConnections, err := fetchConnections(db, version)
	if err != nil {
		return nil, err
	}
	statDatabaseSize, err := fetchDatabaseSize(db)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]interface{})
	mergeStat(stat, statStatDatabase)
	mergeStat(stat, statConnections)
	mergeStat(stat, statDatabaseSize)

	return stat, err
}

// GraphDefinition interface for mackerelplugin
func (p PostgresPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.MetricKeyPrefix())

	var graphdef = map[string]mp.Graphs{
		"connections": {
			Label: (labelPrefix + " Connections"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "active", Label: "Active", Diff: false, Stacked: true},
				{Name: "active_waiting", Label: "Active waiting", Diff: false, Stacked: true},
				{Name: "idle", Label: "Idle", Diff: false, Stacked: true},
				{Name: "idle_in_transaction", Label: "Idle in transaction", Diff: false, Stacked: true},
				{Name: "idle_in_transaction_aborted_", Label: "Idle in transaction (aborted)", Diff: false, Stacked: true},
				{Name: "fastpath_function_call", Label: "fast-path function call", Diff: false, Stacked: true},
				{Name: "disabled", Label: "Disabled", Diff: false, Stacked: true},
			},
		},
		"commits": {
			Label: (labelPrefix + " Commits"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "xact_commit", Label: "Xact Commit", Diff: true, Stacked: false},
				{Name: "xact_rollback", Label: "Xact Rollback", Diff: true, Stacked: false},
			},
		},
		"blocks": {
			Label: (labelPrefix + " Blocks"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "blks_read", Label: "Blocks Read", Diff: true, Stacked: false},
				{Name: "blks_hit", Label: "Blocks Hit", Diff: true, Stacked: false},
			},
		},
		"rows": {
			Label: (labelPrefix + " Rows"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "tup_returned", Label: "Returned Rows", Diff: true, Stacked: false},
				{Name: "tup_fetched", Label: "Fetched Rows", Diff: true, Stacked: true},
				{Name: "tup_inserted", Label: "Inserted Rows", Diff: true, Stacked: true},
				{Name: "tup_updated", Label: "Updated Rows", Diff: true, Stacked: true},
				{Name: "tup_deleted", Label: "Deleted Rows", Diff: true, Stacked: true},
			},
		},
		"size": {
			Label: (labelPrefix + " Data Size"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "total_size", Label: "Total Size", Diff: false, Stacked: false},
			},
		},
		"deadlocks": {
			Label: (labelPrefix + " Dead Locks"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "deadlocks", Label: "Deadlocks", Diff: true, Stacked: false},
			},
		},
		"iotime": {
			Label: (labelPrefix + " Block I/O time"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "blk_read_time", Label: "Block Read Time (ms)", Diff: true, Stacked: false},
				{Name: "blk_write_time", Label: "Block Write Time (ms)", Diff: true, Stacked: false},
			},
		},
		"tempfile": {
			Label: (labelPrefix + " Temporary file"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "temp_bytes", Label: "Temporary file size (byte)", Diff: true, Stacked: false},
			},
		},
	}

	return graphdef
}

// Do the plugin
func Do() {
	optHost := flag.String("hostname", "localhost", "Hostname to login to")
	optPort := flag.String("port", "5432", "Database port")
	optUser := flag.String("user", "", "Postgres User")
	optDatabase := flag.String("database", "", "Database name")
	optPass := flag.String("password", "", "Postgres Password")
	optPrefix := flag.String("metric-key-prefix", "postgres", "Metric key prefix")
	optSSLmode := flag.String("sslmode", "disable", "Whether or not to use SSL")
	optConnectTimeout := flag.Int("connect_timeout", 5, "Maximum wait for connection, in seconds.")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optUser == "" {
		logger.Warningf("user is required")
		flag.PrintDefaults()
		os.Exit(1)
	}
	if *optPass == "" {
		logger.Warningf("password is required")
		flag.PrintDefaults()
		os.Exit(1)
	}
	option := ""
	if *optDatabase != "" {
		option = fmt.Sprintf("dbname=%s", *optDatabase)
	}

	var postgres PostgresPlugin
	postgres.Host = *optHost
	postgres.Port = *optPort
	postgres.Username = *optUser
	postgres.Password = *optPass
	postgres.Prefix = *optPrefix
	postgres.SSLmode = *optSSLmode
	postgres.Timeout = *optConnectTimeout
	postgres.Option = option

	helper := mp.NewMackerelPlugin(postgres)

	helper.Tempfile = *optTempfile
	helper.Run()
}
<commit_msg>s/thrird/third/<commit_after>package mppostgres

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/jmoiron/sqlx"
	// PostgreSQL Driver
	_ "github.com/lib/pq"
	mp "github.com/mackerelio/go-mackerel-plugin-helper"
	"github.com/mackerelio/golib/logging"
)

var logger = logging.GetLogger("metrics.plugin.postgres")

// PostgresPlugin mackerel plugin for PostgreSQL
type PostgresPlugin struct {
	Host     string
	Port     string
	Username string
	Password string
	SSLmode  string
	Prefix   string
	Timeout  int
	Tempfile string
	Option   string
}

func fetchStatDatabase(db *sqlx.DB) (map[string]interface{}, error) {
	db = db.Unsafe()
	rows, err := db.Queryx(`SELECT * FROM pg_stat_database`)
	if err != nil {
		logger.Errorf("Failed to select pg_stat_database. %s", err)
		return nil, err
	}

	type pgStat struct {
		XactCommit   uint64   `db:"xact_commit"`
		XactRollback uint64   `db:"xact_rollback"`
		BlksRead     uint64   `db:"blks_read"`
		BlksHit      uint64   `db:"blks_hit"`
		BlkReadTime  *float64 `db:"blk_read_time"`
		BlkWriteTime *float64 `db:"blk_write_time"`
		TupReturned  uint64   `db:"tup_returned"`
		TupFetched   uint64   `db:"tup_fetched"`
		TupInserted  uint64   `db:"tup_inserted"`
		TupUpdated   uint64   `db:"tup_updated"`
		TupDeleted   uint64   `db:"tup_deleted"`
		Deadlocks    *uint64  `db:"deadlocks"`
		TempBytes    *uint64  `db:"temp_bytes"`
	}

	totalStat := pgStat{}
	for rows.Next() {
		p := pgStat{}
		if err := rows.StructScan(&p); err != nil {
			logger.Warningf("Failed to scan. %s", err)
			continue
		}
		totalStat.XactCommit += p.XactCommit
		totalStat.XactRollback += p.XactRollback
		totalStat.BlksRead += p.BlksRead
		totalStat.BlksHit += p.BlksHit
		if p.BlkReadTime != nil {
			if totalStat.BlkReadTime == nil {
				totalStat.BlkReadTime = p.BlkReadTime
			} else {
				*totalStat.BlkReadTime += *p.BlkReadTime
			}
		}
		if p.BlkWriteTime != nil {
			if totalStat.BlkWriteTime == nil {
				totalStat.BlkWriteTime = p.BlkWriteTime
			} else {
				*totalStat.BlkWriteTime += *p.BlkWriteTime
			}
		}
		totalStat.TupReturned += p.TupReturned
		totalStat.TupFetched += p.TupFetched
		totalStat.TupInserted += p.TupInserted
		totalStat.TupUpdated += p.TupUpdated
		totalStat.TupDeleted += p.TupDeleted
		if p.Deadlocks != nil {
			if totalStat.Deadlocks == nil {
				totalStat.Deadlocks = p.Deadlocks
			} else {
				*totalStat.Deadlocks += *p.Deadlocks
			}
		}
		if p.TempBytes != nil {
			if totalStat.TempBytes == nil {
				totalStat.TempBytes = p.TempBytes
			} else {
				*totalStat.TempBytes += *p.TempBytes
			}
		}
	}
	stat := make(map[string]interface{})
	stat["xact_commit"] = totalStat.XactCommit
	stat["xact_rollback"] = totalStat.XactRollback
	stat["blks_read"] = totalStat.BlksRead
	stat["blks_hit"] = totalStat.BlksHit
	if totalStat.BlkReadTime != nil {
		stat["blk_read_time"] = *totalStat.BlkReadTime
	}
	if totalStat.BlkWriteTime != nil {
		stat["blk_write_time"] = *totalStat.BlkWriteTime
	}
	stat["tup_returned"] = totalStat.TupReturned
	stat["tup_fetched"] = totalStat.TupFetched
	stat["tup_inserted"] = totalStat.TupInserted
	stat["tup_updated"] = totalStat.TupUpdated
	stat["tup_deleted"] = totalStat.TupDeleted
	if totalStat.Deadlocks != nil {
		stat["deadlocks"] = *totalStat.Deadlocks
	}
	if totalStat.TempBytes != nil {
		stat["temp_bytes"] = *totalStat.TempBytes
	}
	return stat, nil
}

func fetchConnections(db *sqlx.DB, version version) (map[string]interface{}, error) {
	var query string

	if version.first > 9 || version.first == 9 && version.second >= 6 {
		query = `select count(*), state, wait_event is not null from pg_stat_activity group by state, wait_event is not null`
	} else {
		query = `select count(*), state, waiting from pg_stat_activity group by state, waiting`
	}
	rows, err := db.Query(query)
	if err != nil {
		logger.Errorf("Failed to select pg_stat_activity. %s", err)
		return nil, err
	}

	stat := map[string]interface{}{
		"active":                      0.0,
		"active_waiting":              0.0,
		"idle":                        0.0,
		"idle_in_transaction":         0.0,
		"idle_in_transaction_aborted": 0.0,
	}

	normalizeRe := regexp.MustCompile("[^a-zA-Z0-9_-]+")

	for rows.Next() {
		var count float64
		var waiting bool
		var state string
		if err := rows.Scan(&count, &state, &waiting); err != nil {
			logger.Warningf("Failed to scan %s", err)
			continue
		}
		state = normalizeRe.ReplaceAllString(state, "_")
		state = strings.TrimRight(state, "_")
		if waiting {
			state += "_waiting"
		}
		stat[state] = float64(count)
	}

	return stat, nil
}

func fetchDatabaseSize(db *sqlx.DB) (map[string]interface{}, error) {
	rows, err := db.Query("select sum(pg_database_size(datname)) as dbsize from pg_database where has_database_privilege(datname, 'connect')")
	if err != nil {
		logger.Errorf("Failed to select pg_database_size. %s", err)
		return nil, err
	}

	var totalSize float64
	for rows.Next() {
		var dbsize float64
		if err := rows.Scan(&dbsize); err != nil {
			logger.Warningf("Failed to scan %s", err)
			continue
		}
		totalSize += dbsize
	}

	return map[string]interface{}{
		"total_size": totalSize,
	}, nil
}

var versionRe = regexp.MustCompile("PostgreSQL (\\d+)\\.(\\d+)(\\.(\\d+))? ")

type version struct {
	first  uint
	second uint
	third  uint
}

func fetchVersion(db *sqlx.DB) (version, error) {

	res := version{}

	rows, err := db.Query("select version()")
	if err != nil {
		logger.Errorf("Failed to select version(). %s", err)
		return res, err
	}

	for rows.Next() {
		var versionStr string
		var first, second, third uint64
		if err := rows.Scan(&versionStr); err != nil {
			return res, err
		}

		// ref. https://www.postgresql.org/support/versioning/

		submatch := versionRe.FindStringSubmatch(versionStr)
		if len(submatch) >= 4 {
			first, err = strconv.ParseUint(submatch[1], 10, 0)
			if err != nil {
				return res, err
			}
			second, err = strconv.ParseUint(submatch[2], 10, 0)
			if err != nil {
				return res, err
			}
			if len(submatch) == 5 {
				third, err = strconv.ParseUint(submatch[4], 10, 0)
				if err != nil {
					return res, err
				}
			}
			res = version{uint(first), uint(second), uint(third)}
			return res, err
		}
	}
	return res, errors.New("failed to select version()")
}

func mergeStat(dst, src map[string]interface{}) {
	for k, v := range src {
		dst[k] = v
	}
}

// MetricKeyPrefix retruns the metrics key prefix
func (p PostgresPlugin) MetricKeyPrefix() string {
	if p.Prefix == "" {
		p.Prefix = "postgres"
	}
	return p.Prefix
}

// FetchMetrics interface for mackerelplugin
func (p PostgresPlugin) FetchMetrics() (map[string]interface{}, error) {

	db, err := sqlx.Connect("postgres", fmt.Sprintf("user=%s password=%s host=%s port=%s sslmode=%s connect_timeout=%d %s", p.Username, p.Password, p.Host, p.Port, p.SSLmode, p.Timeout, p.Option))
	if err != nil {
		logger.Errorf("FetchMetrics: %s", err)
		return nil, err
	}
	defer db.Close()

	version, err := fetchVersion(db)
	if err != nil {
		logger.Warningf("FetchMetrics: %s", err)
		return nil, err
	}

	statStatDatabase, err := fetchStatDatabase(db)
	if err != nil {
		return nil, err
	}
	statConnections, err := fetchConnections(db, version)
	if err != nil {
		return nil, err
	}
	statDatabaseSize, err := fetchDatabaseSize(db)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]interface{})
	mergeStat(stat, statStatDatabase)
	mergeStat(stat, statConnections)
	mergeStat(stat, statDatabaseSize)

	return stat, err
}

// GraphDefinition interface for mackerelplugin
func (p PostgresPlugin) GraphDefinition() map[string]mp.Graphs {
	labelPrefix := strings.Title(p.MetricKeyPrefix())

	var graphdef = map[string]mp.Graphs{
		"connections": {
			Label: (labelPrefix + " Connections"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "active", Label: "Active", Diff: false, Stacked: true},
				{Name: "active_waiting", Label: "Active waiting", Diff: false, Stacked: true},
				{Name: "idle", Label: "Idle", Diff: false, Stacked: true},
				{Name: "idle_in_transaction", Label: "Idle in transaction", Diff: false, Stacked: true},
				{Name: "idle_in_transaction_aborted_", Label: "Idle in transaction (aborted)", Diff: false, Stacked: true},
				{Name: "fastpath_function_call", Label: "fast-path function call", Diff: false, Stacked: true},
				{Name: "disabled", Label: "Disabled", Diff: false, Stacked: true},
			},
		},
		"commits": {
			Label: (labelPrefix + " Commits"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "xact_commit", Label: "Xact Commit", Diff: true, Stacked: false},
				{Name: "xact_rollback", Label: "Xact Rollback", Diff: true, Stacked: false},
			},
		},
		"blocks": {
			Label: (labelPrefix + " Blocks"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "blks_read", Label: "Blocks Read", Diff: true, Stacked: false},
				{Name: "blks_hit", Label: "Blocks Hit", Diff: true, Stacked: false},
			},
		},
		"rows": {
			Label: (labelPrefix + " Rows"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "tup_returned", Label: "Returned Rows", Diff: true, Stacked: false},
				{Name: "tup_fetched", Label: "Fetched Rows", Diff: true, Stacked: true},
				{Name: "tup_inserted", Label: "Inserted Rows", Diff: true, Stacked: true},
				{Name: "tup_updated", Label: "Updated Rows", Diff: true, Stacked: true},
				{Name: "tup_deleted", Label: "Deleted Rows", Diff: true, Stacked: true},
			},
		},
		"size": {
			Label: (labelPrefix + " Data Size"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "total_size", Label: "Total Size", Diff: false, Stacked: false},
			},
		},
		"deadlocks": {
			Label: (labelPrefix + " Dead Locks"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "deadlocks", Label: "Deadlocks", Diff: true, Stacked: false},
			},
		},
		"iotime": {
			Label: (labelPrefix + " Block I/O time"),
			Unit:  "float",
			Metrics: []mp.Metrics{
				{Name: "blk_read_time", Label: "Block Read Time (ms)", Diff: true, Stacked: false},
				{Name: "blk_write_time", Label: "Block Write Time (ms)", Diff: true, Stacked: false},
			},
		},
		"tempfile": {
			Label: (labelPrefix + " Temporary file"),
			Unit:  "integer",
			Metrics: []mp.Metrics{
				{Name: "temp_bytes", Label: "Temporary file size (byte)", Diff: true, Stacked: false},
			},
		},
	}

	return graphdef
}

// Do the plugin
func Do() {
	optHost := flag.String("hostname", "localhost", "Hostname to login to")
	optPort := flag.String("port", "5432", "Database port")
	optUser := flag.String("user", "", "Postgres User")
	optDatabase := flag.String("database", "", "Database name")
	optPass := flag.String("password", "", "Postgres Password")
	optPrefix := flag.String("metric-key-prefix", "postgres", "Metric key prefix")
	optSSLmode := flag.String("sslmode", "disable", "Whether or not to use SSL")
	optConnectTimeout := flag.Int("connect_timeout", 5, "Maximum wait for connection, in seconds.")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	if *optUser == "" {
		logger.Warningf("user is required")
		flag.PrintDefaults()
		os.Exit(1)
	}
	if *optPass == "" {
		logger.Warningf("password is required")
		flag.PrintDefaults()
		os.Exit(1)
	}
	option := ""
	if *optDatabase != "" {
		option = fmt.Sprintf("dbname=%s", *optDatabase)
	}

	var postgres PostgresPlugin
	postgres.Host = *optHost
	postgres.Port = *optPort
	postgres.Username = *optUser
	postgres.Password = *optPass
	postgres.Prefix = *optPrefix
	postgres.SSLmode = *optSSLmode
	postgres.Timeout = *optConnectTimeout
	postgres.Option = option

	helper := mp.NewMackerelPlugin(postgres)

	helper.Tempfile = *optTempfile
	helper.Run()
}
<|endoftext|>


<commit_before>// +build integration

package spaces_test

import (
	"io/ioutil"
	"os"
	"reflect"

	"github.com/DMarby/picsum-photos/database"
	"github.com/DMarby/picsum-photos/storage/spaces"

	"testing"
)

func TestSpaces(t *testing.T) {
	provider, err := spaces.New(
		os.Getenv("PICSUM_SPACE"),
		os.Getenv("PICSUM_SPACES_REGION"),
		os.Getenv("PICSUM_SPACES_ACCESS_KEY"),
		os.Getenv("PICSUM_SPACES_SECRET_KEY"),
	)

	if err != nil {
		t.Fatal(err)
	}

	t.Run("Get an image by id", func(t *testing.T) {
		buf, err := provider.Get("1")
		if err != nil {
			t.Fatal(err)
		}

		resultFixture, _ := ioutil.ReadFile("../../test/fixtures/fixture.jpg")
		if !reflect.DeepEqual(buf, resultFixture) {
			t.Error("image data doesn't match")
		}
	})

	t.Run("Returns error on a nonexistant image", func(t *testing.T) {
		_, err := provider.Get("nonexistant")
		if err != database.ErrNotFound {
			t.FailNow()
		}
	})
}

func TestNew(t *testing.T) {
	_, err := spaces.New("", "", "", "")
	if err == nil {
		t.Fatal("no error")
	}
}
<commit_msg>Add setup/teardown for spaces<commit_after>// +build integration

package spaces_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"reflect"

	"github.com/DMarby/picsum-photos/database"
	"github.com/DMarby/picsum-photos/storage/spaces"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"

	"testing"
)

func TestSpaces(t *testing.T) {
	var (
		space     = os.Getenv("PICSUM_SPACE")
		region    = os.Getenv("PICSUM_SPACES_REGION")
		accessKey = os.Getenv("PICSUM_SPACES_ACCESS_KEY")
		secretKey = os.Getenv("PICSUM_SPACES_SECRET_KEY")
	)

	provider, err := spaces.New(
		space,
		region,
		accessKey,
		secretKey,
	)

	if err != nil {
		t.Fatal(err)
	}

	fixture, _ := ioutil.ReadFile("../../test/fixtures/fixture.jpg")

	// Upload a fixture to the bucket
	spacesSession := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
		Endpoint:    aws.String(fmt.Sprintf("https://%s.digitaloceanspaces.com", region)),
		Region:      aws.String("us-east-1"), // Needs to be us-east-1 for Spaces, or it'll fail
	})
	spaces := s3.New(spacesSession)
	object := s3.PutObjectInput{
		Bucket: &space,
		Key:    aws.String("/1.jpg"),
		Body:   bytes.NewReader(fixture),
	}
	_, err = spaces.PutObject(&object)
	if err != nil {
		t.Fatal(err)
	}

	t.Run("Get an image by id", func(t *testing.T) {
		buf, err := provider.Get("1")
		if err != nil {
			t.Fatal(err)
		}

		if !reflect.DeepEqual(buf, fixture) {
			t.Error("image data doesn't match")
		}
	})

	t.Run("Returns error on a nonexistant image", func(t *testing.T) {
		_, err := provider.Get("nonexistant")
		if err != database.ErrNotFound {
			t.FailNow()
		}
	})

	// Cleanup
	delObject := s3.DeleteObjectInput{
		Bucket: &space,
		Key:    aws.String("/1.jpg"),
	}
	_, err = spaces.DeleteObject(&delObject)
}

func TestNew(t *testing.T) {
	_, err := spaces.New("", "", "", "")
	if err == nil {
		t.Fatal("no error")
	}
}
<|endoftext|>