<commit_before>package esitag

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"
	"text/template"
	"time"

	"github.com/SchumacherFM/caddyesi/bufpool"
	"github.com/SchumacherFM/caddyesi/helpers"
	"github.com/corestoreio/errors"
	"golang.org/x/sync/errgroup"
)

// TemplateIdentifier if some strings contain these characters then a
// template.Template will be created. For now a resource key or an URL is
// supported.
const TemplateIdentifier = "{{"

// Conditioner does not represent your favorite shampoo but it gives you the
// possibility to define an expression which gets executed for every request to
// include the ESI resource or not.
type Conditioner interface {
	OK(r *http.Request) bool
}

type condition struct {
	*template.Template
}

func (c condition) OK(r *http.Request) bool {
	// todo
	return false
}

// Tag identifies an ESI tag by its start and end position in the HTML byte
// stream for replacing. If the HTML changes there needs to be a refresh call to
// re-parse the HTML.
type Tag struct {
	// Data from the micro service gathered in a goroutine.
	Data  []byte
	Start int // start position in the stream
	End   int // end position in the stream
}

// Entity represents a single fully parsed ESI tag
type Entity struct {
	RawTag            []byte
	Tag               Tag
	Resources         // Any 3rd party servers
	TTL               time.Duration
	Timeout           time.Duration
	OnError           string
	ForwardHeaders    []string
	ForwardHeadersAll bool
	ReturnHeaders     []string
	ReturnHeadersAll  bool
	// Key defines a key in a KeyValue server to fetch the value from.
	Key string
	// KeyTemplate gets created when the Key field contains the template
	// identifier. Then the Key field would be empty.
	KeyTemplate *template.Template
	Conditioner // todo
}

// todo split into two regexs for better performance and use the single quote regex only then when the first one returns nothing
var regexESITagDouble = regexp.MustCompile(`([a-z]+)="([^"\r\n]+)"|([a-z]+)='([^'\r\n]+)'`)

// ParseRaw parses the RawTag field and fills the remaining fields of the
// struct.
func (et *Entity) ParseRaw() error {
	if len(et.RawTag) == 0 {
		return nil
	}
	et.Resources.Logf = log.Printf

	// it's kinda ridiculous because the ESI tag parser uses even sync.Pool to
	// reduce allocs and speed up processing and here we're relying on regex.
	// Usually those function for ESI tag parsing will only be called once and
	// then cached. we can optimize it later.
	matches := regexESITagDouble.FindAllSubmatch(et.RawTag, -1)

	srcCounter := 0
	for _, subs := range matches {

		// 1+2 defines the double quotes: key="product_234234"
		subsAttr := subs[1]
		subsVal := subs[2]
		if len(subsAttr) == 0 {
			// fall back to enclosed in single quotes: key='product_234234_{{ .r.Header.Get "myHeaderKey" }}'
			subsAttr = subs[3]
			subsVal = subs[4]
		}
		attr := string(bytes.ToLower(subsAttr)) // must be lower because we use lower case here
		value := string(bytes.TrimSpace(subsVal))

		switch attr {
		case "src":
			if err := et.parseResource(srcCounter, value); err != nil {
				return errors.Wrapf(err, "[caddyesi] Failed to parse src %q in tag %q", value, et.RawTag)
			}
			srcCounter++
		case "key":
			if err := et.parseKey(value); err != nil {
				return errors.Wrapf(err, "[caddyesi] Failed to parse src %q in tag %q", value, et.RawTag)
			}
		case "condition":
			if err := et.parseCondition(value); err != nil {
				return errors.Wrapf(err, "[caddyesi] Failed to parse condition %q in tag %q", value, et.RawTag)
			}
		case "onerror":
			et.OnError = value
		case "timeout":
			var err error
			et.Timeout, err = time.ParseDuration(value)
			if err != nil {
				return errors.NewNotValidf("[caddyesi] ESITag.ParseRaw. Cannot parse duration in timeout: %s => %q\nTag: %q", err, value, et.RawTag)
			}
		case "ttl":
			var err error
			et.TTL, err = time.ParseDuration(value)
			if err != nil {
				return errors.NewNotValidf("[caddyesi] ESITag.ParseRaw. Cannot parse duration in ttl: %s => %q\nTag: %q", err, value, et.RawTag)
			}
		case "forwardheaders":
			if value == "all" {
				et.ForwardHeadersAll = true
			} else {
				et.ForwardHeaders = helpers.CommaListToSlice(value)
			}
		case "returnheaders":
			if value == "all" {
				et.ReturnHeadersAll = true
			} else {
				et.ReturnHeaders = helpers.CommaListToSlice(value)
			}
			// default: ignore all other tags
		}
	}
	if len(et.Resources.Items) == 0 || srcCounter == 0 {
		return errors.NewEmptyf("[caddyesi] ESITag.ParseRaw. src (Items: %d/Src: %d) cannot be empty in Tag which requires at least one resource: %q", len(et.Resources.Items), srcCounter, et.RawTag)
	}
	return nil
}

func (et *Entity) parseCondition(s string) error {
	tpl, err := template.New("condition_tpl").Parse(s)
	if err != nil {
		return errors.NewFatalf("[caddyesi] ESITag.ParseRaw. Failed to parse %q as template with error: %s\nTag: %q", s, err, et.RawTag)
	}
	et.Conditioner = condition{Template: tpl}
	return nil
}

func (et *Entity) parseResource(idx int, val string) (err error) {
	r := &Resource{
		Index: idx,
		URL:   val,
		IsURL: strings.Contains(val, "://"),
	}
	if r.IsURL && strings.Contains(val, TemplateIdentifier) {
		r.URLTemplate, err = template.New("resource_tpl").Parse(val)
		if err != nil {
			return errors.NewFatalf("[caddyesi] ESITag.ParseRaw. Failed to parse %q as template with error: %s\nTag: %q", val, err, et.RawTag)
		}
		r.URL = ""
	}
	et.Items = append(et.Items, r)
	return nil
}

func (et *Entity) parseKey(val string) (err error) {
	et.Key = val
	if strings.Contains(val, TemplateIdentifier) {
		et.KeyTemplate, err = template.New("key_tpl").Parse(val)
		if err != nil {
			return errors.NewFatalf("[caddyesi] ESITag.ParseRaw. Failed to parse %q as template with error: %s\nTag: %q", val, err, et.RawTag)
		}
		et.Key = "" // unset Key because we have a template
	}
	return nil
}

// Entities represents a list of ESI tags found in one HTML page.
type Entities []*Entity

// ParseRaw parses all ESI tags
func (et Entities) ParseRaw() error {
	for i := range et {
		if err := et[i].ParseRaw(); err != nil {
			return errors.Wrapf(err, "[caddyesi] Entities ParseRaw failed at index %d", i)
		}
	}
	return nil
}

// String for debugging only!
func (et Entities) String() string {
	buf := bufpool.Get()
	defer bufpool.Put(buf)

	for i, e := range et {
		raw := e.RawTag
		e.RawTag = nil
		_, _ = fmt.Fprintf(buf, "%d: %#v\n", i, e)
		_, _ = fmt.Fprintf(buf, "%d: RawTag: %q\n\n", i, raw)
	}
	return buf.String()
}

// QueryResources runs in parallel to query all available backend services /
// resources which are available in the current page. The returned Tag slice
// does not guarantee to be ordered. If the request gets canceled via its
// context then all resource requests gets canceled too.
func (et Entities) QueryResources(r *http.Request) ([]Tag, error) {

	tags := make([]Tag, 0, len(et))
	g, ctx := errgroup.WithContext(r.Context())
	cTag := make(chan Tag)
	for _, e := range et {
		e := e
		g.Go(func() error {
			data, err := e.Resources.DoRequest(e.Timeout, r)
			if err != nil {
				return errors.Wrapf(err, "[esitag] QueryResources.Resources.DoRequest failed for Tag %q", e.RawTag)
			}
			t := e.Tag
			t.Data = data

			select {
			case cTag <- t:
			case <-ctx.Done():
				return errors.Wrap(ctx.Err(), "[esitag] Context Done!")
			}
			return nil
		})
	}
	go func() {
		g.Wait()
		close(cTag)
	}()

	for t := range cTag {
		tags = append(tags, t)
	}

	// Check whether any of the goroutines failed. Since g is accumulating the
	// errors, we don't need to send them (or check for them) in the individual
	// results sent on the channel.
	if err := g.Wait(); err != nil {
		return nil, errors.Wrap(err, "[esitag]")
	}

	return tags, nil
}
<commit_msg>esitag: Reduce allocs by switching from bytes to strings<commit_after>package esitag

import (
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strings"
	"text/template"
	"time"

	"github.com/SchumacherFM/caddyesi/bufpool"
	"github.com/SchumacherFM/caddyesi/helpers"
	"github.com/corestoreio/errors"
	"golang.org/x/sync/errgroup"
)

// TemplateIdentifier if some strings contain these characters then a
// template.Template will be created. For now a resource key or an URL is
// supported.
const TemplateIdentifier = "{{"

// Conditioner does not represent your favorite shampoo but it gives you the
// possibility to define an expression which gets executed for every request to
// include the ESI resource or not.
type Conditioner interface {
	OK(r *http.Request) bool
}

type condition struct {
	*template.Template
}

func (c condition) OK(r *http.Request) bool {
	// todo
	return false
}

// Tag identifies an ESI tag by its start and end position in the HTML byte
// stream for replacing. If the HTML changes there needs to be a refresh call to
// re-parse the HTML.
type Tag struct {
	// Data from the micro service gathered in a goroutine.
	Data  []byte
	Start int // start position in the stream
	End   int // end position in the stream
}

// Entity represents a single fully parsed ESI tag
type Entity struct {
	RawTag            []byte
	Tag               Tag
	Resources         // Any 3rd party servers
	TTL               time.Duration
	Timeout           time.Duration
	OnError           string
	ForwardHeaders    []string
	ForwardHeadersAll bool
	ReturnHeaders     []string
	ReturnHeadersAll  bool
	// Key defines a key in a KeyValue server to fetch the value from.
	Key string
	// KeyTemplate gets created when the Key field contains the template
	// identifier. Then the Key field would be empty.
	KeyTemplate *template.Template
	Conditioner // todo
}

// todo split into two regexs for better performance and use the single quote regex only then when the first one returns nothing
var regexESITagDouble = regexp.MustCompile(`([a-z]+)="([^"\r\n]+)"|([a-z]+)='([^'\r\n]+)'`)

// ParseRaw parses the RawTag field and fills the remaining fields of the
// struct.
func (et *Entity) ParseRaw() error {
	if len(et.RawTag) == 0 {
		return nil
	}
	et.Resources.Logf = log.Printf

	// it's kinda ridiculous because the ESI tag parser uses even sync.Pool to
	// reduce allocs and speed up processing and here we're relying on regex.
	// Usually those function for ESI tag parsing will only be called once and
	// then cached. we can optimize it later.
	matches := regexESITagDouble.FindAllStringSubmatch(string(et.RawTag), -1)

	srcCounter := 0
	for _, subs := range matches {

		// 1+2 defines the double quotes: key="product_234234"
		subsAttr := subs[1]
		subsVal := subs[2]
		if subsAttr == "" {
			// fall back to enclosed in single quotes: key='product_234234_{{ .r.Header.Get "myHeaderKey" }}'
			subsAttr = subs[3]
			subsVal = subs[4]
		}
		attr := strings.ToLower(subsAttr) // must be lower because we use lower case here
		value := strings.TrimSpace(subsVal)

		switch attr {
		case "src":
			if err := et.parseResource(srcCounter, value); err != nil {
				return errors.Wrapf(err, "[caddyesi] Failed to parse src %q in tag %q", value, et.RawTag)
			}
			srcCounter++
		case "key":
			if err := et.parseKey(value); err != nil {
				return errors.Wrapf(err, "[caddyesi] Failed to parse src %q in tag %q", value, et.RawTag)
			}
		case "condition":
			if err := et.parseCondition(value); err != nil {
				return errors.Wrapf(err, "[caddyesi] Failed to parse condition %q in tag %q", value, et.RawTag)
			}
		case "onerror":
			et.OnError = value
		case "timeout":
			var err error
			et.Timeout, err = time.ParseDuration(value)
			if err != nil {
				return errors.NewNotValidf("[caddyesi] ESITag.ParseRaw. Cannot parse duration in timeout: %s => %q\nTag: %q", err, value, et.RawTag)
			}
		case "ttl":
			var err error
			et.TTL, err = time.ParseDuration(value)
			if err != nil {
				return errors.NewNotValidf("[caddyesi] ESITag.ParseRaw. Cannot parse duration in ttl: %s => %q\nTag: %q", err, value, et.RawTag)
			}
		case "forwardheaders":
			if value == "all" {
				et.ForwardHeadersAll = true
			} else {
				et.ForwardHeaders = helpers.CommaListToSlice(value)
			}
		case "returnheaders":
			if value == "all" {
				et.ReturnHeadersAll = true
			} else {
				et.ReturnHeaders = helpers.CommaListToSlice(value)
			}
			// default: ignore all other tags
		}
	}
	if len(et.Resources.Items) == 0 || srcCounter == 0 {
		return errors.NewEmptyf("[caddyesi] ESITag.ParseRaw. src (Items: %d/Src: %d) cannot be empty in Tag which requires at least one resource: %q", len(et.Resources.Items), srcCounter, et.RawTag)
	}
	return nil
}

func (et *Entity) parseCondition(s string) error {
	tpl, err := template.New("condition_tpl").Parse(s)
	if err != nil {
		return errors.NewFatalf("[caddyesi] ESITag.ParseRaw. Failed to parse %q as template with error: %s\nTag: %q", s, err, et.RawTag)
	}
	et.Conditioner = condition{Template: tpl}
	return nil
}

func (et *Entity) parseResource(idx int, val string) (err error) {
	r := &Resource{
		Index: idx,
		URL:   val,
		IsURL: strings.Contains(val, "://"),
	}
	if r.IsURL && strings.Contains(val, TemplateIdentifier) {
		r.URLTemplate, err = template.New("resource_tpl").Parse(val)
		if err != nil {
			return errors.NewFatalf("[caddyesi] ESITag.ParseRaw. Failed to parse %q as template with error: %s\nTag: %q", val, err, et.RawTag)
		}
		r.URL = ""
	}
	et.Items = append(et.Items, r)
	return nil
}

func (et *Entity) parseKey(val string) (err error) {
	et.Key = val
	if strings.Contains(val, TemplateIdentifier) {
		et.KeyTemplate, err = template.New("key_tpl").Parse(val)
		if err != nil {
			return errors.NewFatalf("[caddyesi] ESITag.ParseRaw. Failed to parse %q as template with error: %s\nTag: %q", val, err, et.RawTag)
		}
		et.Key = "" // unset Key because we have a template
	}
	return nil
}

// Entities represents a list of ESI tags found in one HTML page.
type Entities []*Entity

// ParseRaw parses all ESI tags
func (et Entities) ParseRaw() error {
	for i := range et {
		if err := et[i].ParseRaw(); err != nil {
			return errors.Wrapf(err, "[caddyesi] Entities ParseRaw failed at index %d", i)
		}
	}
	return nil
}

// String for debugging only!
func (et Entities) String() string {
	buf := bufpool.Get()
	defer bufpool.Put(buf)

	for i, e := range et {
		raw := e.RawTag
		e.RawTag = nil
		_, _ = fmt.Fprintf(buf, "%d: %#v\n", i, e)
		_, _ = fmt.Fprintf(buf, "%d: RawTag: %q\n\n", i, raw)
	}
	return buf.String()
}

// QueryResources runs in parallel to query all available backend services /
// resources which are available in the current page. The returned Tag slice
// does not guarantee to be ordered. If the request gets canceled via its
// context then all resource requests gets canceled too.
func (et Entities) QueryResources(r *http.Request) ([]Tag, error) {

	tags := make([]Tag, 0, len(et))
	g, ctx := errgroup.WithContext(r.Context())
	cTag := make(chan Tag)
	for _, e := range et {
		e := e
		g.Go(func() error {
			data, err := e.Resources.DoRequest(e.Timeout, r)
			if err != nil {
				return errors.Wrapf(err, "[esitag] QueryResources.Resources.DoRequest failed for Tag %q", e.RawTag)
			}
			t := e.Tag
			t.Data = data

			select {
			case cTag <- t:
			case <-ctx.Done():
				return errors.Wrap(ctx.Err(), "[esitag] Context Done!")
			}
			return nil
		})
	}
	go func() {
		g.Wait()
		close(cTag)
	}()

	for t := range cTag {
		tags = append(tags, t)
	}

	// Check whether any of the goroutines failed. Since g is accumulating the
	// errors, we don't need to send them (or check for them) in the individual
	// results sent on the channel.
	if err := g.Wait(); err != nil {
		return nil, errors.Wrap(err, "[esitag]")
	}

	return tags, nil
}
<|endoftext|>


<commit_before>package io

import (
	"fmt"
	"go-bots/ev3"
	"go-bots/seeker2/config"
	"go-bots/seeker2/logic"
	"go-bots/seeker2/vision"
	"os"
	"time"
)

func colorIsOut(v int) bool {
	return v > 20
}

var devs *ev3.Devices
var data chan<- logic.Data
var commands <-chan logic.Commands

var pme, pmesp, ml, mr, mf *ev3.Attribute
var dme, dmf string
var colR, colL, irR, irL *ev3.Attribute

var ledRR, ledRG, ledLR, ledLG *ev3.Attribute

var start time.Time

func getEyesDirection() ev3.Direction {
	if pmesp.Value > 0 {
		return ev3.Right
	} else if pmesp.Value < 0 {
		return ev3.Left
	} else {
		return ev3.NoDirection
	}
}
func setEyesDirection(dir ev3.Direction) {
	desiredSetPosition := int(config.VisionMaxPosition * dir)
	if pmesp.Value != desiredSetPosition {
		pmesp.Value = desiredSetPosition
		pmesp.Sync()
		ev3.RunCommand(dme, ev3.CmdRunToAbsPos)

		fmt.Fprintln(os.Stderr, "setEyesDirection", dir, desiredSetPosition, pmesp.Value)

	}
}

// StartTime gets the time when the bot started
func StartTime() time.Time {
	return start
}

// Init initializes the io module
func Init(d chan<- logic.Data, s time.Time) {
	devs = ev3.Scan(&ev3.OutPortModes{
		OutA: ev3.OutPortModeAuto,
		OutB: ev3.OutPortModeAuto,
		OutC: ev3.OutPortModeDcMotor,
		OutD: ev3.OutPortModeDcMotor,
	})
	data = d
	start = s

	// Col L
	ev3.CheckDriver(devs.In1, ev3.DriverColor, ev3.In1)
	// Col R
	ev3.CheckDriver(devs.In2, ev3.DriverColor, ev3.In2)
	// Ir L
	ev3.CheckDriver(devs.In3, ev3.DriverIr, ev3.In3)
	// Ir R
	ev3.CheckDriver(devs.In4, ev3.DriverIr, ev3.In4)

	// A front
	ev3.CheckDriver(devs.OutA, ev3.DriverTachoMotorMedium, ev3.OutA)
	// B eyes
	ev3.CheckDriver(devs.OutB, ev3.DriverTachoMotorMedium, ev3.OutB)
	// C right direct
	ev3.CheckDriver(devs.OutC, ev3.DriverRcxMotor, ev3.OutC)
	// D left inverted
	ev3.CheckDriver(devs.OutD, ev3.DriverRcxMotor, ev3.OutD)

	ev3.SetMode(devs.In1, ev3.ColorModeReflect)
	ev3.SetMode(devs.In2, ev3.ColorModeReflect)
	ev3.SetMode(devs.In3, ev3.IrModeProx)
	ev3.SetMode(devs.In4, ev3.IrModeProx)

	ev3.RunCommand(devs.OutA, ev3.CmdReset)
	ev3.RunCommand(devs.OutB, ev3.CmdReset)
	ev3.RunCommand(devs.OutC, ev3.CmdStop)
	ev3.RunCommand(devs.OutD, ev3.CmdStop)

	colL = ev3.OpenByteR(devs.In1, ev3.BinData)
	colR = ev3.OpenByteR(devs.In2, ev3.BinData)
	irL = ev3.OpenByteR(devs.In3, ev3.BinData)
	irR = ev3.OpenByteR(devs.In4, ev3.BinData)
	// C right direct
	mr = ev3.OpenTextW(devs.OutC, ev3.DutyCycleSp)
	// D left inverted
	ml = ev3.OpenTextW(devs.OutD, ev3.DutyCycleSp)
	// B eyes
	dme = devs.OutB
	pme = ev3.OpenTextR(devs.OutB, ev3.Position)
	pmesp = ev3.OpenTextW(devs.OutB, ev3.PositionSp)
	// A front
	dmf = devs.OutA
	mf = ev3.OpenTextW(devs.OutA, ev3.DutyCycleSp)

	ledLG = ev3.OpenTextW(devs.LedLeftGreen, ev3.Brightness)
	ledLR = ev3.OpenTextW(devs.LedLeftRed, ev3.Brightness)
	ledRG = ev3.OpenTextW(devs.LedRightGreen, ev3.Brightness)
	ledRR = ev3.OpenTextW(devs.LedRightRed, ev3.Brightness)
	ledLG.Value = 0
	ledLR.Value = 0
	ledRG.Value = 0
	ledRR.Value = 0
	ledLG.Sync()
	ledLR.Sync()
	ledRG.Sync()
	ledRR.Sync()

	// Wheels
	mr.Value = 0
	ml.Value = 0
	mr.Sync()
	ml.Sync()
	ev3.RunCommand(devs.OutC, ev3.CmdStop)
	ev3.RunCommand(devs.OutD, ev3.CmdStop)
	ev3.RunCommand(devs.OutC, ev3.CmdRunDirect)
	ev3.RunCommand(devs.OutD, ev3.CmdRunDirect)

	// Front
	ev3.RunCommand(dmf, ev3.CmdReset)
	mf.Value = 0
	mf.Sync()
	ev3.RunCommand(dmf, ev3.CmdRunDirect)

	// Eyes
	ev3.RunCommand(dme, ev3.CmdReset)
	ev3.WriteStringAttribute(dme, ev3.Position, "0")
	ev3.WriteStringAttribute(dme, ev3.SpeedSp, config.VisionSpeed)
	ev3.WriteStringAttribute(dme, ev3.StopAction, "hold")
	setEyesDirection(ev3.NoDirection)
}

var speedL, speedR int
var lastMillis, currentMillis int

func computeSpeed(currentSpeed int, targetSpeed int, millis int) int {
	if currentSpeed < targetSpeed {
		currentSpeed += (config.ForwardAcceleration * millis)
		if currentSpeed > targetSpeed {
			currentSpeed = targetSpeed
		}
	}
	if currentSpeed > targetSpeed {
		currentSpeed -= (config.ReverseAcceleration * millis)
		if currentSpeed < targetSpeed {
			currentSpeed = targetSpeed
		}
	}
	return currentSpeed
}

func ProcessCommand(c *logic.Commands) {
	currentMillis = c.Millis
	millis := currentMillis - lastMillis
	speedL = computeSpeed(speedL, c.SpeedLeft, millis)
	speedR = computeSpeed(speedR, c.SpeedRight, millis)
	lastMillis = currentMillis

	ml.Value = -speedL / 100
	mr.Value = speedR / 100
	ml.Sync()
	mr.Sync()

	ledLG.Value = c.LedLeftGreen
	ledLR.Value = c.LedLeftRed
	ledRG.Value = c.LedRightGreen
	ledRR.Value = c.LedRightRed
	ledLG.Sync()
	ledLR.Sync()
	ledRG.Sync()
	ledRR.Sync()

	if c.FrontActive {
		mf.Value = config.FrontWheelsSpeed
	} else {
		mf.Value = 0
	}
	mf.Sync()

	// fmt.Fprintln(os.Stderr, "DATA EYES ACTIVE", c.EyesActive)

	if !c.EyesActive {
		setEyesDirection(ev3.NoDirection)
	} else {
		if getEyesDirection() == ev3.NoDirection {
			setEyesDirection(ev3.Right)
		}
	}
}

// Loop contains the io loop
func Loop() {
	for {
		now := time.Now()
		millis := ev3.TimespanAsMillis(start, now)

		pme.Sync()
		colR.Sync()
		colL.Sync()
		irR.Sync()
		irL.Sync()

		visionIntensity, visionAngle, eyesDirection := 0, 0, getEyesDirection()
		if eyesDirection != ev3.NoDirection {
			// fmt.Fprintln(os.Stderr, "EYES PROCESS", eyesDirection)
			visionIntensity, visionAngle, eyesDirection = vision.Process(millis, eyesDirection, pme.Value, irR.Value, irL.Value)
			setEyesDirection(eyesDirection)
		}

		// fmt.Fprintln(os.Stderr, "DATA", colL.Value, colR.Value, irL.Value, irR.Value)

		data <- logic.Data{
			Start:            start,
			Millis:           millis,
			CornerRightIsOut: colorIsOut(colR.Value),
			CornerLeftIsOut:  colorIsOut(colL.Value),
			CornerRight:      colR.Value,
			CornerLeft:       colL.Value,
			IrValueRight:     irR.Value,
			IrValueLeft:      irL.Value,
			VisionIntensity:  visionIntensity,
			VisionAngle:      visionAngle,
		}
	}
}

// Close terminates and cleans up the io module
func Close() {
	defer ev3.RunCommand(devs.OutA, ev3.CmdReset)
	defer ev3.RunCommand(devs.OutB, ev3.CmdReset)
	defer ev3.RunCommand(devs.OutC, ev3.CmdStop)
	defer ev3.RunCommand(devs.OutD, ev3.CmdStop)

	defer ev3.RunCommand(devs.OutA, ev3.CmdStop)
	defer ev3.RunCommand(devs.OutB, ev3.CmdStop)

	ledLG.Value = 0
	ledLR.Value = 0
	ledRG.Value = 0
	ledRR.Value = 0
	ledLG.Sync()
	ledLR.Sync()
	ledRG.Sync()
	ledRR.Sync()

	// TODO: close all files
	// pf, mf, ml, mc, mr
	// colR, colL, irR, irL
}
<commit_msg>Fixed seeker2 motors.<commit_after>package io

import (
	"fmt"
	"go-bots/ev3"
	"go-bots/seeker2/config"
	"go-bots/seeker2/logic"
	"go-bots/seeker2/vision"
	"os"
	"time"
)

func colorIsOut(v int) bool {
	return v > 20
}

var devs *ev3.Devices
var data chan<- logic.Data
var commands <-chan logic.Commands

var pme, pmesp, ml, mr, mf *ev3.Attribute
var dme, dmf string
var colR, colL, irR, irL *ev3.Attribute

var ledRR, ledRG, ledLR, ledLG *ev3.Attribute

var start time.Time

func getEyesDirection() ev3.Direction {
	if pmesp.Value > 0 {
		return ev3.Right
	} else if pmesp.Value < 0 {
		return ev3.Left
	} else {
		return ev3.NoDirection
	}
}
func setEyesDirection(dir ev3.Direction) {
	desiredSetPosition := int(config.VisionMaxPosition * dir)
	if pmesp.Value != desiredSetPosition {
		pmesp.Value = desiredSetPosition
		pmesp.Sync()
		ev3.RunCommand(dme, ev3.CmdRunToAbsPos)

		fmt.Fprintln(os.Stderr, "setEyesDirection", dir, desiredSetPosition, pmesp.Value)

	}
}

// StartTime gets the time when the bot started
func StartTime() time.Time {
	return start
}

// Init initializes the io module
func Init(d chan<- logic.Data, s time.Time) {
	devs = ev3.Scan(&ev3.OutPortModes{
		OutA: ev3.OutPortModeAuto,
		OutB: ev3.OutPortModeAuto,
		OutC: ev3.OutPortModeDcMotor,
		OutD: ev3.OutPortModeDcMotor,
	})
	data = d
	start = s

	// Col L
	ev3.CheckDriver(devs.In1, ev3.DriverColor, ev3.In1)
	// Col R
	ev3.CheckDriver(devs.In2, ev3.DriverColor, ev3.In2)
	// Ir L
	ev3.CheckDriver(devs.In3, ev3.DriverIr, ev3.In3)
	// Ir R
	ev3.CheckDriver(devs.In4, ev3.DriverIr, ev3.In4)

	// A front
	ev3.CheckDriver(devs.OutA, ev3.DriverTachoMotorMedium, ev3.OutA)
	// B eyes
	ev3.CheckDriver(devs.OutB, ev3.DriverTachoMotorMedium, ev3.OutB)
	// C left direct
	ev3.CheckDriver(devs.OutC, ev3.DriverRcxMotor, ev3.OutC)
	// D right inverted
	ev3.CheckDriver(devs.OutD, ev3.DriverRcxMotor, ev3.OutD)

	ev3.SetMode(devs.In1, ev3.ColorModeReflect)
	ev3.SetMode(devs.In2, ev3.ColorModeReflect)
	ev3.SetMode(devs.In3, ev3.IrModeProx)
	ev3.SetMode(devs.In4, ev3.IrModeProx)

	ev3.RunCommand(devs.OutA, ev3.CmdReset)
	ev3.RunCommand(devs.OutB, ev3.CmdReset)
	ev3.RunCommand(devs.OutC, ev3.CmdStop)
	ev3.RunCommand(devs.OutD, ev3.CmdStop)

	colL = ev3.OpenByteR(devs.In1, ev3.BinData)
	colR = ev3.OpenByteR(devs.In2, ev3.BinData)
	irL = ev3.OpenByteR(devs.In3, ev3.BinData)
	irR = ev3.OpenByteR(devs.In4, ev3.BinData)
	// C left direct
	ml = ev3.OpenTextW(devs.OutC, ev3.DutyCycleSp)
	// D right inverted
	mr = ev3.OpenTextW(devs.OutD, ev3.DutyCycleSp)
	// B eyes
	dme = devs.OutB
	pme = ev3.OpenTextR(devs.OutB, ev3.Position)
	pmesp = ev3.OpenTextW(devs.OutB, ev3.PositionSp)
	// A front
	dmf = devs.OutA
	mf = ev3.OpenTextW(devs.OutA, ev3.DutyCycleSp)

	ledLG = ev3.OpenTextW(devs.LedLeftGreen, ev3.Brightness)
	ledLR = ev3.OpenTextW(devs.LedLeftRed, ev3.Brightness)
	ledRG = ev3.OpenTextW(devs.LedRightGreen, ev3.Brightness)
	ledRR = ev3.OpenTextW(devs.LedRightRed, ev3.Brightness)
	ledLG.Value = 0
	ledLR.Value = 0
	ledRG.Value = 0
	ledRR.Value = 0
	ledLG.Sync()
	ledLR.Sync()
	ledRG.Sync()
	ledRR.Sync()

	// Wheels
	mr.Value = 0
	ml.Value = 0
	mr.Sync()
	ml.Sync()
	ev3.RunCommand(devs.OutC, ev3.CmdStop)
	ev3.RunCommand(devs.OutD, ev3.CmdStop)
	ev3.RunCommand(devs.OutC, ev3.CmdRunDirect)
	ev3.RunCommand(devs.OutD, ev3.CmdRunDirect)

	// Front
	ev3.RunCommand(dmf, ev3.CmdReset)
	mf.Value = 0
	mf.Sync()
	ev3.RunCommand(dmf, ev3.CmdRunDirect)

	// Eyes
	ev3.RunCommand(dme, ev3.CmdReset)
	ev3.WriteStringAttribute(dme, ev3.Position, "0")
	ev3.WriteStringAttribute(dme, ev3.SpeedSp, config.VisionSpeed)
	ev3.WriteStringAttribute(dme, ev3.StopAction, "hold")
	setEyesDirection(ev3.NoDirection)
}

var speedL, speedR int
var lastMillis, currentMillis int

func computeSpeed(currentSpeed int, targetSpeed int, millis int) int {
	if currentSpeed < targetSpeed {
		currentSpeed += (config.ForwardAcceleration * millis)
		if currentSpeed > targetSpeed {
			currentSpeed = targetSpeed
		}
	}
	if currentSpeed > targetSpeed {
		currentSpeed -= (config.ReverseAcceleration * millis)
		if currentSpeed < targetSpeed {
			currentSpeed = targetSpeed
		}
	}
	return currentSpeed
}

func ProcessCommand(c *logic.Commands) {
	currentMillis = c.Millis
	millis := currentMillis - lastMillis
	speedL = computeSpeed(speedL, c.SpeedLeft, millis)
	speedR = computeSpeed(speedR, c.SpeedRight, millis)
	lastMillis = currentMillis

	ml.Value = speedL / 100
	mr.Value = -speedR / 100
	ml.Sync()
	mr.Sync()

	ledLG.Value = c.LedLeftGreen
	ledLR.Value = c.LedLeftRed
	ledRG.Value = c.LedRightGreen
	ledRR.Value = c.LedRightRed
	ledLG.Sync()
	ledLR.Sync()
	ledRG.Sync()
	ledRR.Sync()

	if c.FrontActive {
		mf.Value = config.FrontWheelsSpeed
	} else {
		mf.Value = 0
	}
	mf.Sync()

	// fmt.Fprintln(os.Stderr, "DATA EYES ACTIVE", c.EyesActive)

	if !c.EyesActive {
		setEyesDirection(ev3.NoDirection)
	} else {
		if getEyesDirection() == ev3.NoDirection {
			setEyesDirection(ev3.Right)
		}
	}
}

// Loop contains the io loop
func Loop() {
	for {
		now := time.Now()
		millis := ev3.TimespanAsMillis(start, now)

		pme.Sync()
		colR.Sync()
		colL.Sync()
		irR.Sync()
		irL.Sync()

		visionIntensity, visionAngle, eyesDirection := 0, 0, getEyesDirection()
		if eyesDirection != ev3.NoDirection {
			// fmt.Fprintln(os.Stderr, "EYES PROCESS", eyesDirection)
			visionIntensity, visionAngle, eyesDirection = vision.Process(millis, eyesDirection, pme.Value, irR.Value, irL.Value)
			setEyesDirection(eyesDirection)
		}

		// fmt.Fprintln(os.Stderr, "DATA", colL.Value, colR.Value, irL.Value, irR.Value)

		data <- logic.Data{
			Start:            start,
			Millis:           millis,
			CornerRightIsOut: colorIsOut(colR.Value),
			CornerLeftIsOut:  colorIsOut(colL.Value),
			CornerRight:      colR.Value,
			CornerLeft:       colL.Value,
			IrValueRight:     irR.Value,
			IrValueLeft:      irL.Value,
			VisionIntensity:  visionIntensity,
			VisionAngle:      visionAngle,
		}
	}
}

// Close terminates and cleans up the io module
func Close() {
	defer ev3.RunCommand(devs.OutA, ev3.CmdReset)
	defer ev3.RunCommand(devs.OutB, ev3.CmdReset)
	defer ev3.RunCommand(devs.OutC, ev3.CmdStop)
	defer ev3.RunCommand(devs.OutD, ev3.CmdStop)

	defer ev3.RunCommand(devs.OutA, ev3.CmdStop)
	defer ev3.RunCommand(devs.OutB, ev3.CmdStop)

	ledLG.Value = 0
	ledLR.Value = 0
	ledRG.Value = 0
	ledRR.Value = 0
	ledLG.Sync()
	ledLR.Sync()
	ledRG.Sync()
	ledRR.Sync()

	// TODO: close all files
	// pf, mf, ml, mc, mr
	// colR, colL, irR, irL
}
<|endoftext|>


<commit_before>package serf

import (
	"fmt"
)

// delegate is the memberlist.Delegate implementation that Serf uses.
type delegate struct {
	serf *Serf
}

func (d *delegate) NodeMeta(limit int) []byte {
	roleBytes := []byte(d.serf.config.Role)
	if len(roleBytes) > limit {
		panic(fmt.Errorf("role '%s' exceeds length limit of %d bytes", d.serf.config.Role, limit))
	}

	return roleBytes
}

func (d *delegate) NotifyMsg(buf []byte) {
	// If we didn't actually receive any data, then ignore it.
	if len(buf) == 0 {
		return
	}

	rebroadcast := false
	rebroadcastQueue := d.serf.broadcasts
	t := messageType(buf[0])
	switch t {
	case messageLeaveType:
		var leave messageLeave
		if err := decodeMessage(buf[1:], &leave); err != nil {
			d.serf.logger.Printf("[ERR] Error decoding leave message: %s", err)
			break
		}

		d.serf.logger.Printf("[DEBUG] serf-delegate: messageLeaveType: %s", leave.Node)
		rebroadcast = d.serf.handleNodeLeaveIntent(&leave)

	case messageJoinType:
		var join messageJoin
		if err := decodeMessage(buf[1:], &join); err != nil {
			d.serf.logger.Printf("[ERR] Error decoding join message: %s", err)
			break
		}

		d.serf.logger.Printf("[DEBUG] serf-delegate: messageJoinType: %s", join.Node)
		rebroadcast = d.serf.handleNodeJoinIntent(&join)

	case messageUserEventType:
		var event messageUserEvent
		if err := decodeMessage(buf[1:], &event); err != nil {
			d.serf.logger.Printf("[ERR] Error decoding user event message: %s", err)
			break
		}

		d.serf.logger.Printf("[DEBUG] serf-delegate: messageUserEventType: %s", event.Name)
		rebroadcast = d.serf.handleUserEvent(&event)
		rebroadcastQueue = d.serf.eventBroadcasts

	default:
		d.serf.logger.Printf("[WARN] Received message of unknown type: %d", t)
	}

	if rebroadcast {
		// Copy the buffer since it we cannot rely on the slice not changing
		newBuf := make([]byte, len(buf))
		copy(newBuf, buf)

		rebroadcastQueue.QueueBroadcast(&broadcast{
			msg:    newBuf,
			notify: nil,
		})
	}
}

func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
	msgs := d.serf.broadcasts.GetBroadcasts(overhead, limit)

	// Determine the bytes used already
	bytesUsed := 0
	for _, msg := range msgs {
		bytesUsed += len(msg) + overhead
	}

	// Get any additional event broadcasts
	eventMsgs := d.serf.eventBroadcasts.GetBroadcasts(overhead, limit-bytesUsed)
	if eventMsgs != nil {
		msgs = append(msgs, eventMsgs...)
	}

	return msgs
}

func (d *delegate) LocalState() []byte {
	d.serf.memberLock.RLock()
	defer d.serf.memberLock.RUnlock()
	d.serf.eventLock.RLock()
	defer d.serf.eventLock.RUnlock()

	// Create the message to send
	pp := messagePushPull{
		LTime:        d.serf.clock.Time(),
		StatusLTimes: make(map[string]LamportTime, len(d.serf.members)),
		LeftMembers:  make([]string, 0, len(d.serf.leftMembers)),
		EventLTime:   d.serf.eventClock.Time(),
		Events:       d.serf.eventBuffer,
	}

	// Add all the join LTimes
	for name, member := range d.serf.members {
		pp.StatusLTimes[name] = member.statusLTime
	}

	// Add all the left nodes
	for _, member := range d.serf.leftMembers {
		pp.LeftMembers = append(pp.LeftMembers, member.Name)
	}

	// Encode the push pull state
	buf, err := encodeMessage(messagePushPullType, &pp)
	if err != nil {
		d.serf.logger.Printf("[ERR] serf: Failed to encode local state: %v", err)
		return nil
	}
	return buf
}

func (d *delegate) MergeRemoteState(buf []byte) {
	// Check the message type
	if messageType(buf[0]) != messagePushPullType {
		d.serf.logger.Printf("[ERR] serf: Remote state has bad type prefix: %v", buf[0])
		return
	}

	// Attempt a decode
	pp := messagePushPull{}
	if err := decodeMessage(buf[1:], &pp); err != nil {
		d.serf.logger.Printf("[ERR] serf: Failed to decode remote state: %v", err)
		return
	}

	// Witness the Lamport clocks first.
	// We subtract 1 since no message with that clock has been sent yet
	d.serf.clock.Witness(pp.LTime - 1)
	d.serf.eventClock.Witness(pp.EventLTime - 1)

	// Process the left nodes first to avoid the LTimes from being increment
	// in the wrong order
	leftMap := make(map[string]struct{}, len(pp.LeftMembers))
	leave := messageLeave{}
	for _, name := range pp.LeftMembers {
		leftMap[name] = struct{}{}
		leave.LTime = pp.StatusLTimes[name]
		leave.Node = name
		d.serf.handleNodeLeaveIntent(&leave)
	}

	// Update any other LTimes
	join := messageJoin{}
	for name, statusLTime := range pp.StatusLTimes {
		// Skip the left nodes
		if _, ok := leftMap[name]; ok {
			continue
		}

		// Create an artificial join message
		join.LTime = statusLTime
		join.Node = name
		d.serf.handleNodeJoinIntent(&join)
	}

	// Process all the events
	userEvent := messageUserEvent{}
	for _, events := range pp.Events {
		if events == nil {
			continue
		}
		userEvent.LTime = events.LTime
		for _, e := range events.Events {
			userEvent.Name = e.Name
			userEvent.Payload = e.Payload
			d.serf.handleUserEvent(&userEvent)
		}
	}
}
<commit_msg>serf: update to new Memberlist delegate interface<commit_after>package serf

import (
	"fmt"
)

// delegate is the memberlist.Delegate implementation that Serf uses.
type delegate struct {
	serf *Serf
}

func (d *delegate) NodeMeta(limit int) []byte {
	roleBytes := []byte(d.serf.config.Role)
	if len(roleBytes) > limit {
		panic(fmt.Errorf("role '%s' exceeds length limit of %d bytes", d.serf.config.Role, limit))
	}

	return roleBytes
}

func (d *delegate) NotifyMsg(buf []byte) {
	// If we didn't actually receive any data, then ignore it.
	if len(buf) == 0 {
		return
	}

	rebroadcast := false
	rebroadcastQueue := d.serf.broadcasts
	t := messageType(buf[0])
	switch t {
	case messageLeaveType:
		var leave messageLeave
		if err := decodeMessage(buf[1:], &leave); err != nil {
			d.serf.logger.Printf("[ERR] Error decoding leave message: %s", err)
			break
		}

		d.serf.logger.Printf("[DEBUG] serf-delegate: messageLeaveType: %s", leave.Node)
		rebroadcast = d.serf.handleNodeLeaveIntent(&leave)

	case messageJoinType:
		var join messageJoin
		if err := decodeMessage(buf[1:], &join); err != nil {
			d.serf.logger.Printf("[ERR] Error decoding join message: %s", err)
			break
		}

		d.serf.logger.Printf("[DEBUG] serf-delegate: messageJoinType: %s", join.Node)
		rebroadcast = d.serf.handleNodeJoinIntent(&join)

	case messageUserEventType:
		var event messageUserEvent
		if err := decodeMessage(buf[1:], &event); err != nil {
			d.serf.logger.Printf("[ERR] Error decoding user event message: %s", err)
			break
		}

		d.serf.logger.Printf("[DEBUG] serf-delegate: messageUserEventType: %s", event.Name)
		rebroadcast = d.serf.handleUserEvent(&event)
		rebroadcastQueue = d.serf.eventBroadcasts

	default:
		d.serf.logger.Printf("[WARN] Received message of unknown type: %d", t)
	}

	if rebroadcast {
		// Copy the buffer since it we cannot rely on the slice not changing
		newBuf := make([]byte, len(buf))
		copy(newBuf, buf)

		rebroadcastQueue.QueueBroadcast(&broadcast{
			msg:    newBuf,
			notify: nil,
		})
	}
}

func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
	msgs := d.serf.broadcasts.GetBroadcasts(overhead, limit)

	// Determine the bytes used already
	bytesUsed := 0
	for _, msg := range msgs {
		bytesUsed += len(msg) + overhead
	}

	// Get any additional event broadcasts
	eventMsgs := d.serf.eventBroadcasts.GetBroadcasts(overhead, limit-bytesUsed)
	if eventMsgs != nil {
		msgs = append(msgs, eventMsgs...)
	}

	return msgs
}

func (d *delegate) LocalState(join bool) []byte {
	d.serf.memberLock.RLock()
	defer d.serf.memberLock.RUnlock()
	d.serf.eventLock.RLock()
	defer d.serf.eventLock.RUnlock()

	// Create the message to send
	pp := messagePushPull{
		LTime:        d.serf.clock.Time(),
		StatusLTimes: make(map[string]LamportTime, len(d.serf.members)),
		LeftMembers:  make([]string, 0, len(d.serf.leftMembers)),
		EventLTime:   d.serf.eventClock.Time(),
		Events:       d.serf.eventBuffer,
	}

	// Add all the join LTimes
	for name, member := range d.serf.members {
		pp.StatusLTimes[name] = member.statusLTime
	}

	// Add all the left nodes
	for _, member := range d.serf.leftMembers {
		pp.LeftMembers = append(pp.LeftMembers, member.Name)
	}

	// Encode the push pull state
	buf, err := encodeMessage(messagePushPullType, &pp)
	if err != nil {
		d.serf.logger.Printf("[ERR] serf: Failed to encode local state: %v", err)
		return nil
	}
	return buf
}

func (d *delegate) MergeRemoteState(buf []byte, isJoin bool) {
	// Check the message type
	if messageType(buf[0]) != messagePushPullType {
		d.serf.logger.Printf("[ERR] serf: Remote state has bad type prefix: %v", buf[0])
		return
	}

	// Attempt a decode
	pp := messagePushPull{}
	if err := decodeMessage(buf[1:], &pp); err != nil {
		d.serf.logger.Printf("[ERR] serf: Failed to decode remote state: %v", err)
		return
	}

	// Witness the Lamport clocks first.
	// We subtract 1 since no message with that clock has been sent yet
	d.serf.clock.Witness(pp.LTime - 1)
	d.serf.eventClock.Witness(pp.EventLTime - 1)

	// Process the left nodes first to avoid the LTimes from being increment
	// in the wrong order
	leftMap := make(map[string]struct{}, len(pp.LeftMembers))
	leave := messageLeave{}
	for _, name := range pp.LeftMembers {
		leftMap[name] = struct{}{}
		leave.LTime = pp.StatusLTimes[name]
		leave.Node = name
		d.serf.handleNodeLeaveIntent(&leave)
	}

	// Update any other LTimes
	join := messageJoin{}
	for name, statusLTime := range pp.StatusLTimes {
		// Skip the left nodes
		if _, ok := leftMap[name]; ok {
			continue
		}

		// Create an artificial join message
		join.LTime = statusLTime
		join.Node = name
		d.serf.handleNodeJoinIntent(&join)
	}

	// Process all the events
	userEvent := messageUserEvent{}
	for _, events := range pp.Events {
		if events == nil {
			continue
		}
		userEvent.LTime = events.LTime
		for _, e := range events.Events {
			userEvent.Name = e.Name
			userEvent.Payload = e.Payload
			d.serf.handleUserEvent(&userEvent)
		}
	}
}
<|endoftext|>


<commit_before>// Run tests for all the remotes.  Run this with package names which
// need integration testing.
//
// See the `test` target in the Makefile.
//
package main

/* FIXME

Make TesTrun have a []string of flags to try - that then makes it generic

*/

import (
	"flag"
	"log"
	"math/rand"
	"os"
	"path"
	"regexp"
	"strings"
	"time"

	_ "github.com/rclone/rclone/backend/all" // import all fs
	"github.com/rclone/rclone/lib/pacer"
)

var (
	// Flags
	maxTries     = flag.Int("maxtries", 5, "Number of times to try each test")
	maxN         = flag.Int("n", 20, "Maximum number of tests to run at once")
	testRemotes  = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
	testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage")
	testTests    = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'")
	clean        = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
	runOnly      = flag.String("run", "", "Run only those tests matching the regexp supplied")
	timeout      = flag.Duration("timeout", 30*time.Minute, "Maximum time to run each test for before giving up")
	configFile   = flag.String("config", "fstest/test_all/config.yaml", "Path to config file")
	outputDir    = flag.String("output", path.Join(os.TempDir(), "rclone-integration-tests"), "Place to store results")
	emailReport  = flag.String("email", "", "Set to email the report to the address supplied")
	dryRun       = flag.Bool("dry-run", false, "Print commands which would be executed only")
	urlBase      = flag.String("url-base", "https://pub.rclone.org/integration-tests/", "Base for the online version")
	uploadPath   = flag.String("upload", "", "Set this to an rclone path to upload the results here")
	verbose      = flag.Bool("verbose", false, "Set to enable verbose logging in the tests")
	listRetries  = flag.Int("list-retries", -1, "Number or times to retry listing - set to override the default")
)

// if matches then is definitely OK in the shell
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")

// converts a argv style input into a shell command
func toShell(args []string) (result string) {
	for _, arg := range args {
		if result != "" {
			result += " "
		}
		if shellOK.MatchString(arg) {
			result += arg
		} else {
			result += "'" + arg + "'"
		}
	}
	return result
}

func main() {
	flag.Parse()
	conf, err := NewConfig(*configFile)
	if err != nil {
		log.Println("test_all should be run from the root of the rclone source code")
		log.Fatal(err)
	}

	// Seed the random number generator
	rand.Seed(time.Now().UTC().UnixNano())

	// Filter selection
	if *testRemotes != "" {
		conf.filterBackendsByRemotes(strings.Split(*testRemotes, ","))
	}
	if *testBackends != "" {
		conf.filterBackendsByBackends(strings.Split(*testBackends, ","))
	}
	if *testTests != "" {
		conf.filterTests(strings.Split(*testTests, ","))
	}

	// Just clean the directories if required
	if *clean {
		err := cleanRemotes(conf)
		if err != nil {
			log.Fatalf("Failed to clean: %v", err)
		}
		return
	}

	var names []string
	for _, remote := range conf.Backends {
		names = append(names, remote.Remote)
	}
	log.Printf("Testing remotes: %s", strings.Join(names, ", "))

	// Runs we will do for this test in random order
	runs := conf.MakeRuns()
	rand.Shuffle(len(runs), runs.Swap)

	// Create Report
	report := NewReport()

	// Make the test binaries, one per Path found in the tests
	done := map[string]struct{}{}
	for _, run := range runs {
		if _, found := done[run.Path]; !found {
			done[run.Path] = struct{}{}
			if !run.NoBinary {
				run.MakeTestBinary()
				defer run.RemoveTestBinary()
			}
		}
	}

	// workaround for cache backend as we run simultaneous tests
	_ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m")

	// start the tests
	results := make(chan *Run, len(runs))
	awaiting := 0
	tokens := pacer.NewTokenDispenser(*maxN)
	for _, run := range runs {
		tokens.Get()
		go func(run *Run) {
			defer tokens.Put()
			run.Run(report.LogDir, results)
		}(run)
		awaiting++
	}

	// Wait for the tests to finish
	for ; awaiting > 0; awaiting-- {
		t := <-results
		report.RecordResult(t)
	}

	// Log and exit
	report.End()
	report.LogSummary()
	report.LogJSON()
	report.LogHTML()
	report.EmailHTML()
	report.Upload()
	if !report.AllPassed() {
		os.Exit(1)
	}
}
<commit_msg>test_all: increase the test timeout to 60m from 30m<commit_after>// Run tests for all the remotes.  Run this with package names which
// need integration testing.
//
// See the `test` target in the Makefile.
//
package main

/* FIXME

Make TesTrun have a []string of flags to try - that then makes it generic

*/

import (
	"flag"
	"log"
	"math/rand"
	"os"
	"path"
	"regexp"
	"strings"
	"time"

	_ "github.com/rclone/rclone/backend/all" // import all fs
	"github.com/rclone/rclone/lib/pacer"
)

var (
	// Flags
	maxTries     = flag.Int("maxtries", 5, "Number of times to try each test")
	maxN         = flag.Int("n", 20, "Maximum number of tests to run at once")
	testRemotes  = flag.String("remotes", "", "Comma separated list of remotes to test, eg 'TestSwift:,TestS3'")
	testBackends = flag.String("backends", "", "Comma separated list of backends to test, eg 's3,googlecloudstorage")
	testTests    = flag.String("tests", "", "Comma separated list of tests to test, eg 'fs/sync,fs/operations'")
	clean        = flag.Bool("clean", false, "Instead of testing, clean all left over test directories")
	runOnly      = flag.String("run", "", "Run only those tests matching the regexp supplied")
	timeout      = flag.Duration("timeout", 60*time.Minute, "Maximum time to run each test for before giving up")
	configFile   = flag.String("config", "fstest/test_all/config.yaml", "Path to config file")
	outputDir    = flag.String("output", path.Join(os.TempDir(), "rclone-integration-tests"), "Place to store results")
	emailReport  = flag.String("email", "", "Set to email the report to the address supplied")
	dryRun       = flag.Bool("dry-run", false, "Print commands which would be executed only")
	urlBase      = flag.String("url-base", "https://pub.rclone.org/integration-tests/", "Base for the online version")
	uploadPath   = flag.String("upload", "", "Set this to an rclone path to upload the results here")
	verbose      = flag.Bool("verbose", false, "Set to enable verbose logging in the tests")
	listRetries  = flag.Int("list-retries", -1, "Number or times to retry listing - set to override the default")
)

// if matches then is definitely OK in the shell
var shellOK = regexp.MustCompile("^[A-Za-z0-9./_:-]+$")

// converts a argv style input into a shell command
func toShell(args []string) (result string) {
	for _, arg := range args {
		if result != "" {
			result += " "
		}
		if shellOK.MatchString(arg) {
			result += arg
		} else {
			result += "'" + arg + "'"
		}
	}
	return result
}

func main() {
	flag.Parse()
	conf, err := NewConfig(*configFile)
	if err != nil {
		log.Println("test_all should be run from the root of the rclone source code")
		log.Fatal(err)
	}

	// Seed the random number generator
	rand.Seed(time.Now().UTC().UnixNano())

	// Filter selection
	if *testRemotes != "" {
		conf.filterBackendsByRemotes(strings.Split(*testRemotes, ","))
	}
	if *testBackends != "" {
		conf.filterBackendsByBackends(strings.Split(*testBackends, ","))
	}
	if *testTests != "" {
		conf.filterTests(strings.Split(*testTests, ","))
	}

	// Just clean the directories if required
	if *clean {
		err := cleanRemotes(conf)
		if err != nil {
			log.Fatalf("Failed to clean: %v", err)
		}
		return
	}

	var names []string
	for _, remote := range conf.Backends {
		names = append(names, remote.Remote)
	}
	log.Printf("Testing remotes: %s", strings.Join(names, ", "))

	// Runs we will do for this test in random order
	runs := conf.MakeRuns()
	rand.Shuffle(len(runs), runs.Swap)

	// Create Report
	report := NewReport()

	// Make the test binaries, one per Path found in the tests
	done := map[string]struct{}{}
	for _, run := range runs {
		if _, found := done[run.Path]; !found {
			done[run.Path] = struct{}{}
			if !run.NoBinary {
				run.MakeTestBinary()
				defer run.RemoveTestBinary()
			}
		}
	}

	// workaround for cache backend as we run simultaneous tests
	_ = os.Setenv("RCLONE_CACHE_DB_WAIT_TIME", "30m")

	// start the tests
	results := make(chan *Run, len(runs))
	awaiting := 0
	tokens := pacer.NewTokenDispenser(*maxN)
	for _, run := range runs {
		tokens.Get()
		go func(run *Run) {
			defer tokens.Put()
			run.Run(report.LogDir, results)
		}(run)
		awaiting++
	}

	// Wait for the tests to finish
	for ; awaiting > 0; awaiting-- {
		t := <-results
		report.RecordResult(t)
	}

	// Log and exit
	report.End()
	report.LogSummary()
	report.LogJSON()
	report.LogHTML()
	report.EmailHTML()
	report.Upload()
	if !report.AllPassed() {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// 文件监控.
// 使用时需要注意的是，一旦一个文件被删除，那么对其的监控将会失效；如果删除的是目录，那么该目录及其下的文件都将被递归删除监控。
package gfsnotify

import (
    "container/list"
    "errors"
    "fmt"
    "gitee.com/johng/gf/g/container/glist"
    "gitee.com/johng/gf/g/container/gmap"
    "gitee.com/johng/gf/g/container/gqueue"
    "gitee.com/johng/gf/g/container/gtype"
    "gitee.com/johng/gf/g/encoding/ghash"
    "gitee.com/johng/gf/g/os/gcache"
    "gitee.com/johng/gf/third/github.com/fsnotify/fsnotify"
)

// 监听管理对象
type Watcher struct {
    watcher       *fsnotify.Watcher        // 底层fsnotify对象
    events        *gqueue.Queue            // 过滤后的事件通知，不会出现重复事件
    closeChan     chan struct{}            // 关闭事件
    callbacks     *gmap.StringInterfaceMap // 监听的回调函数
    cache         *gcache.Cache            // 缓存对象，用于事件重复过滤
}

// 注册的监听回调方法
type Callback struct {
    Id     int                 // 唯一ID
    Func   func(event *Event)  // 回调方法
    Path   string              // 监听的文件/目录
    elem   *list.Element       // 指向监听链表中的元素项位置
    parent *Callback           // 父级callback，有这个属性表示该callback为被自动管理的callback
    subs   *glist.List         // 子级回调对象指针列表
}

// 监听事件对象
type Event struct {
    event   fsnotify.Event   // 底层事件对象
    Path    string           // 文件绝对路径
    Op      Op               // 触发监听的文件操作
    Watcher *Watcher         // 事件对应的监听对象
}

// 按位进行识别的操作集合
type Op uint32

// 必须放到一个const分组里面
const (
    CREATE Op = 1 << iota
    WRITE
    REMOVE
    RENAME
    CHMOD
)

const (
    REPEAT_EVENT_FILTER_INTERVAL = 1 // (毫秒)重复事件过滤间隔
    DEFAULT_WATCHER_COUNT        = 8 // 默认创建的监控对象数量(使用哈希取模)
)

var (
    // 全局监听对象，方便应用端调用
    watchers       = make([]*Watcher, DEFAULT_WATCHER_COUNT)
    // 默认的watchers是否初始化，使用时才创建
    watcherInited  = gtype.NewBool()
    // 回调方法ID与对象指针的映射哈希表，用于根据ID快速查找回调对象
    callbackIdMap  = gmap.NewIntInterfaceMap()
)

// 初始化创建8个watcher对象，用于包默认管理监听
func initWatcher() {
    if !watcherInited.Set(true) {
        for i := 0; i < DEFAULT_WATCHER_COUNT; i++ {
            if w, err := New(); err == nil {
                watchers[i] = w
            } else {
                panic(err)
            }
        }
    }
}

// 创建监听管理对象，主要注意的是创建监听对象会占用系统的inotify句柄数量，受到 fs.inotify.max_user_instances 的限制
func New() (*Watcher, error) {
    if watch, err := fsnotify.NewWatcher(); err == nil {
        w := &Watcher {
            cache         : gcache.New(),
            watcher       : watch,
            events        : gqueue.New(),
            closeChan     : make(chan struct{}),
            callbacks     : gmap.NewStringInterfaceMap(),
        }
        w.startWatchLoop()
        w.startEventLoop()
        return w, nil
    } else {
        return nil, err
    }
}

// 添加对指定文件/目录的监听，并给定回调函数；如果给定的是一个目录，默认递归监控。
func Add(path string, callbackFunc func(event *Event), recursive...bool) (callback *Callback, err error) {
    return getWatcherByPath(path).Add(path, callbackFunc, recursive...)
}

// 递归移除对指定文件/目录的所有监听回调
func Remove(path string) error {
    return getWatcherByPath(path).Remove(path)
}

// 根据指定的回调函数ID，移出指定的inotify回调函数
func RemoveCallback(callbackId int) error {
    callback := (*Callback)(nil)
    if r := callbackIdMap.Get(callbackId); r != nil {
        callback = r.(*Callback)
    }
    if callback == nil {
        return errors.New(fmt.Sprintf(`callback for id %d not found`, callbackId))
    }
    return getWatcherByPath(callback.Path).RemoveCallback(callbackId)
}

// 根据path计算对应的watcher对象
func getWatcherByPath(path string) *Watcher {
    initWatcher()
    return watchers[ghash.BKDRHash([]byte(path)) % DEFAULT_WATCHER_COUNT]
}
<commit_msg>修改gfsnotify默认的Watcher数量，并可以通过命令行参数或者环境变量进行修改<commit_after>// Copyright 2018 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

// 文件监控.
// 使用时需要注意的是，一旦一个文件被删除，那么对其的监控将会失效；如果删除的是目录，那么该目录及其下的文件都将被递归删除监控。
package gfsnotify

import (
    "container/list"
    "errors"
    "fmt"
    "gitee.com/johng/gf/g/container/glist"
    "gitee.com/johng/gf/g/container/gmap"
    "gitee.com/johng/gf/g/container/gqueue"
    "gitee.com/johng/gf/g/container/gtype"
    "gitee.com/johng/gf/g/encoding/ghash"
    "gitee.com/johng/gf/g/os/gcache"
    "gitee.com/johng/gf/third/github.com/fsnotify/fsnotify"
)

// 监听管理对象
type Watcher struct {
    watcher       *fsnotify.Watcher        // 底层fsnotify对象
    events        *gqueue.Queue            // 过滤后的事件通知，不会出现重复事件
    closeChan     chan struct{}            // 关闭事件
    callbacks     *gmap.StringInterfaceMap // 监听的回调函数
    cache         *gcache.Cache            // 缓存对象，用于事件重复过滤
}

// 注册的监听回调方法
type Callback struct {
    Id     int                 // 唯一ID
    Func   func(event *Event)  // 回调方法
    Path   string              // 监听的文件/目录
    elem   *list.Element       // 指向监听链表中的元素项位置
    parent *Callback           // 父级callback，有这个属性表示该callback为被自动管理的callback
    subs   *glist.List         // 子级回调对象指针列表
}

// 监听事件对象
type Event struct {
    event   fsnotify.Event   // 底层事件对象
    Path    string           // 文件绝对路径
    Op      Op               // 触发监听的文件操作
    Watcher *Watcher         // 事件对应的监听对象
}

// 按位进行识别的操作集合
type Op uint32

// 必须放到一个const分组里面
const (
    CREATE Op = 1 << iota
    WRITE
    REMOVE
    RENAME
    CHMOD
)

const (
    REPEAT_EVENT_FILTER_INTERVAL = 1 // (毫秒)重复事件过滤间隔
    DEFAULT_WATCHER_COUNT        = 2 // 默认创建的监控对象数量(使用哈希取模)
)

var (
    // 全局监听对象，方便应用端调用
    watchers       = make([]*Watcher, DEFAULT_WATCHER_COUNT)
    // 默认的watchers是否初始化，使用时才创建
    watcherInited  = gtype.NewBool()
    // 回调方法ID与对象指针的映射哈希表，用于根据ID快速查找回调对象
    callbackIdMap  = gmap.NewIntInterfaceMap()
)

// 初始化创建8个watcher对象，用于包默认管理监听
func initWatcher() {
    if !watcherInited.Set(true) {
        for i := 0; i < DEFAULT_WATCHER_COUNT; i++ {
            if w, err := New(); err == nil {
                watchers[i] = w
            } else {
                panic(err)
            }
        }
    }
}

// 创建监听管理对象，主要注意的是创建监听对象会占用系统的inotify句柄数量，受到 fs.inotify.max_user_instances 的限制
func New() (*Watcher, error) {
    if watch, err := fsnotify.NewWatcher(); err == nil {
        w := &Watcher {
            cache         : gcache.New(),
            watcher       : watch,
            events        : gqueue.New(),
            closeChan     : make(chan struct{}),
            callbacks     : gmap.NewStringInterfaceMap(),
        }
        w.startWatchLoop()
        w.startEventLoop()
        return w, nil
    } else {
        return nil, err
    }
}

// 添加对指定文件/目录的监听，并给定回调函数；如果给定的是一个目录，默认递归监控。
func Add(path string, callbackFunc func(event *Event), recursive...bool) (callback *Callback, err error) {
    return getWatcherByPath(path).Add(path, callbackFunc, recursive...)
}

// 递归移除对指定文件/目录的所有监听回调
func Remove(path string) error {
    return getWatcherByPath(path).Remove(path)
}

// 根据指定的回调函数ID，移出指定的inotify回调函数
func RemoveCallback(callbackId int) error {
    callback := (*Callback)(nil)
    if r := callbackIdMap.Get(callbackId); r != nil {
        callback = r.(*Callback)
    }
    if callback == nil {
        return errors.New(fmt.Sprintf(`callback for id %d not found`, callbackId))
    }
    return getWatcherByPath(callback.Path).RemoveCallback(callbackId)
}

// 根据path计算对应的watcher对象
func getWatcherByPath(path string) *Watcher {
    initWatcher()
    return watchers[ghash.BKDRHash([]byte(path)) % DEFAULT_WATCHER_COUNT]
}
<|endoftext|>


<commit_before>package postgres

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"testing"

	"github.com/nuveo/prest/api"
	"github.com/nuveo/prest/statements"
	. "github.com/smartystreets/goconvey/convey"
)

func TestWhereByRequest(t *testing.T) {
	Convey("Where by request without paginate", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool", nil)
		So(err, ShouldBeNil)

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "dbname=$")
		So(where, ShouldContainSubstring, "test=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "prest")
		So(values, ShouldContain, "cool")
	})

	Convey("Where by request with jsonb field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?name=nuveo&data->>description:jsonb=bla", nil)
		So(err, ShouldBeNil)

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "name=$")
		So(where, ShouldContainSubstring, "data->>'description'=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "nuveo")
		So(values, ShouldContain, "bla")
	})
}

func TestQuery(t *testing.T) {
	Convey("Query execution", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata ORDER BY schema_name ASC"
		json, err := Query(sql)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Query execution with params", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata WHERE schema_name = $1 ORDER BY schema_name ASC"
		json, err := Query(sql, "public")
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Query with invalid characters", t, func() {
		sql := "SELECT ~~, ``, ˜ schema_name FROM information_schema.schemata WHERE schema_name = $1 ORDER BY schema_name ASC"
		json, err := Query(sql, "public")
		So(err, ShouldNotBeNil)
		So(json, ShouldBeNil)
	})

}

func TestPaginateIfPossible(t *testing.T) {
	Convey("Paginate if possible", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool&_page=1&_page_size=20", nil)
		So(err, ShouldBeNil)
		where, err := PaginateIfPossible(r)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "LIMIT 20 OFFSET(1 - 1) * 20")
	})
}

func TestInsert(t *testing.T) {
	Convey("Insert data into a table", t, func() {
		m := make(map[string]interface{}, 0)
		m["name"] = "prest"

		r := api.Request{
			Data: m,
		}
		jsonByte, err := Insert("prest", "public", "test4", r)
		So(err, ShouldBeNil)
		So(len(jsonByte), ShouldBeGreaterThan, 0)

		var toJSON map[string]interface{}
		err = json.Unmarshal(jsonByte, &toJSON)
		So(err, ShouldBeNil)

		So(toJSON["id"], ShouldEqual, 1)
	})

	Convey("Insert data into a table with contraints", t, func() {
		m := make(map[string]interface{}, 0)
		m["name"] = "prest"

		r := api.Request{
			Data: m,
		}
		_, err := Insert("prest", "public", "test3", r)
		So(err, ShouldNotBeNil)
	})
}

func TestDelete(t *testing.T) {
	Convey("Delete data from table", t, func() {
		json, err := Delete("prest", "public", "test", "name=$1", []interface{}{"nuveo"})
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestUpdate(t *testing.T) {
	Convey("Update data into a table", t, func() {

		m := make(map[string]interface{}, 0)
		m["name"] = "prest"

		r := api.Request{
			Data: m,
		}
		json, err := Update("prest", "public", "test", "name=$1", []interface{}{"prest"}, r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Update data into a table with constraints", t, func() {
		m := make(map[string]interface{}, 0)
		m["name"] = "prest"

		r := api.Request{
			Data: m,
		}
		_, err := Update("prest", "public", "test3", "name=$1", []interface{}{"prest tester"}, r)
		So(err, ShouldNotBeNil)
	})
}

func TestChkInvaidIdentifier(t *testing.T) {
	Convey("Check invalid character on identifier", t, func() {
		chk := chkInvalidIdentifier("fildName")
		So(chk, ShouldBeFalse)
		chk = chkInvalidIdentifier("_9fildName")
		So(chk, ShouldBeFalse)
		chk = chkInvalidIdentifier("_fild.Name")
		So(chk, ShouldBeFalse)

		chk = chkInvalidIdentifier("0fildName")
		So(chk, ShouldBeTrue)
		chk = chkInvalidIdentifier("fild'Name")
		So(chk, ShouldBeTrue)
		chk = chkInvalidIdentifier("fild\"Name")
		So(chk, ShouldBeTrue)
		chk = chkInvalidIdentifier("fild;Name")
		So(chk, ShouldBeTrue)
		chk = chkInvalidIdentifier("_123456789_123456789_123456789_123456789_123456789_123456789_12345")
		So(chk, ShouldBeTrue)

	})
}

func TestJoinByRequest(t *testing.T) {
	Convey("Join by request", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_join=inner:test2:test2.name:$eq:test.name", nil)
		So(err, ShouldBeNil)

		join, err := JoinByRequest(r)
		joinStr := strings.Join(join, " ")

		So(err, ShouldBeNil)
		So(joinStr, ShouldContainSubstring, "INNER JOIN test2 ON test2.name = test.name")
	})
	Convey("Join missing param", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_join=inner:test2:test2.name:$eq", nil)
		So(err, ShouldNotBeNil)

		_, err = JoinByRequest(r)
		So(err, ShouldNotBeNil)
	})
	Convey("Join invalid operator", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_join=inner:test2:test2.name:notexist:test.name", nil)
		So(err, ShouldNotBeNil)

		_, err = JoinByRequest(r)
		So(err, ShouldNotBeNil)
	})
	Convey("Join with where", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_join=inner:test2:test2.name:$eq:test.name&name=nuveo&data->>description:jsonb=bla", nil)
		So(err, ShouldBeNil)

		join, err := JoinByRequest(r)
		joinStr := strings.Join(join, " ")

		So(err, ShouldBeNil)
		So(joinStr, ShouldContainSubstring, "INNER JOIN test2 ON test2.name = test.name")

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "name=$")
		So(where, ShouldContainSubstring, "data->>'description'=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "nuveo")
		So(values, ShouldContain, "bla")
	})

}

func TestSelectFields(t *testing.T) {
	Convey("Select fields from table", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_select=celphone", nil)
		So(err, ShouldBeNil)

		selectQuery := SelectByRequest(r)
		So(selectQuery, ShouldContainSubstring, "SELECT celphone FROM")
	})

	Convey("Select all from table", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_select=*", nil)
		So(err, ShouldBeNil)

		selectQuery := SelectByRequest(r)
		So(selectQuery, ShouldContainSubstring, "SELECT * FROM")
	})

	Convey("Try Select with empty '_select' field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_select=", nil)
		So(err, ShouldBeNil)

		selectQuery := SelectByRequest(r)
		So(selectQuery, ShouldEqual, "")
	})
}

func TestCountFields(t *testing.T) {
	Convey("Count fields from table", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_count=celphone", nil)
		So(err, ShouldBeNil)

		countQuery := CountByRequest(r)
		So(countQuery, ShouldContainSubstring, "SELECT COUNT(celphone) FROM")
	})

	Convey("Count all from table", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_count=*", nil)
		So(err, ShouldBeNil)

		countQuery := CountByRequest(r)
		So(countQuery, ShouldContainSubstring, "SELECT COUNT(*) FROM")
	})

	Convey("Try Count with empty '_count' field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_count=", nil)
		So(err, ShouldBeNil)

		countQuery := CountByRequest(r)
		So(countQuery, ShouldEqual, "")
	})
}

func TestDatabaseClause(t *testing.T) {
	Convey("Return appropriate SELECT clause", t, func() {
		r, err := http.NewRequest("GET", "/databases", nil)
		So(err, ShouldBeNil)

		countQuery := DatabaseClause(r)
		So(countQuery, ShouldEqual, fmt.Sprintf(statements.DatabasesSelect, statements.FieldDatabaseName))
	})

	Convey("Return appropriate COUNT clause", t, func() {
		r, err := http.NewRequest("GET", "/databases?_count=*", nil)
		So(err, ShouldBeNil)

		countQuery := DatabaseClause(r)
		So(countQuery, ShouldEqual, fmt.Sprintf(statements.DatabasesSelect, statements.FieldCountDatabaseName))
	})
}

func TestSchemaClause(t *testing.T) {
	Convey("Return appropriate SELECT clause", t, func() {
		r, err := http.NewRequest("GET", "/schemas", nil)
		So(err, ShouldBeNil)

		countQuery := SchemaClause(r)
		So(countQuery, ShouldEqual, fmt.Sprintf(statements.SchemasSelect, statements.FieldSchemaName))
	})

	Convey("Return appropriate COUNT clause", t, func() {
		r, err := http.NewRequest("GET", "/schemas?_count=*", nil)
		So(err, ShouldBeNil)

		countQuery := SchemaClause(r)
		So(countQuery, ShouldEqual, fmt.Sprintf(statements.SchemasSelect, statements.FieldCountSchemaName))
	})
}

func TestGetQueryOperator(t *testing.T) {
	Convey("Query operator eq", t, func() {
		op, err := GetQueryOperator("$eq")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "=")
	})
	Convey("Query operator gt", t, func() {
		op, err := GetQueryOperator("$gt")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, ">")
	})
	Convey("Query operator gte", t, func() {
		op, err := GetQueryOperator("$gte")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, ">=")
	})

	Convey("Query operator lt", t, func() {
		op, err := GetQueryOperator("$lt")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "<")
	})
	Convey("Query operator lte", t, func() {
		op, err := GetQueryOperator("$lte")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "<=")
	})
	Convey("Query operator IN", t, func() {
		op, err := GetQueryOperator("$in")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "IN")
	})
	Convey("Query operator NIN", t, func() {
		op, err := GetQueryOperator("$nin")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "NOT IN")
	})
}

func TestOrderByRequest(t *testing.T) {
	Convey("Query ORDER BY", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_order=name,-number", nil)
		So(err, ShouldBeNil)

		order, err := OrderByRequest(r)
		So(err, ShouldBeNil)
		So(order, ShouldContainSubstring, "ORDER BY")
		So(order, ShouldContainSubstring, "name")
		So(order, ShouldContainSubstring, "number DESC")
	})
}
<commit_msg>fix assertion<commit_after>package postgres

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"testing"

	"github.com/nuveo/prest/api"
	"github.com/nuveo/prest/statements"
	. "github.com/smartystreets/goconvey/convey"
)

func TestWhereByRequest(t *testing.T) {
	Convey("Where by request without paginate", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool", nil)
		So(err, ShouldBeNil)

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "dbname=$")
		So(where, ShouldContainSubstring, "test=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "prest")
		So(values, ShouldContain, "cool")
	})

	Convey("Where by request with jsonb field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?name=nuveo&data->>description:jsonb=bla", nil)
		So(err, ShouldBeNil)

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "name=$")
		So(where, ShouldContainSubstring, "data->>'description'=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "nuveo")
		So(values, ShouldContain, "bla")
	})
}

func TestQuery(t *testing.T) {
	Convey("Query execution", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata ORDER BY schema_name ASC"
		json, err := Query(sql)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Query execution with params", t, func() {
		sql := "SELECT schema_name FROM information_schema.schemata WHERE schema_name = $1 ORDER BY schema_name ASC"
		json, err := Query(sql, "public")
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Query with invalid characters", t, func() {
		sql := "SELECT ~~, ``, ˜ schema_name FROM information_schema.schemata WHERE schema_name = $1 ORDER BY schema_name ASC"
		json, err := Query(sql, "public")
		So(err, ShouldNotBeNil)
		So(json, ShouldBeNil)
	})

}

func TestPaginateIfPossible(t *testing.T) {
	Convey("Paginate if possible", t, func() {
		r, err := http.NewRequest("GET", "/databases?dbname=prest&test=cool&_page=1&_page_size=20", nil)
		So(err, ShouldBeNil)
		where, err := PaginateIfPossible(r)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "LIMIT 20 OFFSET(1 - 1) * 20")
	})
}

func TestInsert(t *testing.T) {
	Convey("Insert data into a table", t, func() {
		m := make(map[string]interface{}, 0)
		m["name"] = "prest"

		r := api.Request{
			Data: m,
		}
		jsonByte, err := Insert("prest", "public", "test4", r)
		So(err, ShouldBeNil)
		So(len(jsonByte), ShouldBeGreaterThan, 0)

		var toJSON map[string]interface{}
		err = json.Unmarshal(jsonByte, &toJSON)
		So(err, ShouldBeNil)

		So(toJSON["id"], ShouldEqual, 1)
	})

	Convey("Insert data into a table with contraints", t, func() {
		m := make(map[string]interface{}, 0)
		m["name"] = "prest"

		r := api.Request{
			Data: m,
		}
		_, err := Insert("prest", "public", "test3", r)
		So(err, ShouldNotBeNil)
	})
}

func TestDelete(t *testing.T) {
	Convey("Delete data from table", t, func() {
		json, err := Delete("prest", "public", "test", "name=$1", []interface{}{"nuveo"})
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})
}

func TestUpdate(t *testing.T) {
	Convey("Update data into a table", t, func() {

		m := make(map[string]interface{}, 0)
		m["name"] = "prest"

		r := api.Request{
			Data: m,
		}
		json, err := Update("prest", "public", "test", "name=$1", []interface{}{"prest"}, r)
		So(err, ShouldBeNil)
		So(len(json), ShouldBeGreaterThan, 0)
	})

	Convey("Update data into a table with constraints", t, func() {
		m := make(map[string]interface{}, 0)
		m["name"] = "prest"

		r := api.Request{
			Data: m,
		}
		_, err := Update("prest", "public", "test3", "name=$1", []interface{}{"prest tester"}, r)
		So(err, ShouldNotBeNil)
	})
}

func TestChkInvaidIdentifier(t *testing.T) {
	Convey("Check invalid character on identifier", t, func() {
		chk := chkInvalidIdentifier("fildName")
		So(chk, ShouldBeFalse)
		chk = chkInvalidIdentifier("_9fildName")
		So(chk, ShouldBeFalse)
		chk = chkInvalidIdentifier("_fild.Name")
		So(chk, ShouldBeFalse)

		chk = chkInvalidIdentifier("0fildName")
		So(chk, ShouldBeTrue)
		chk = chkInvalidIdentifier("fild'Name")
		So(chk, ShouldBeTrue)
		chk = chkInvalidIdentifier("fild\"Name")
		So(chk, ShouldBeTrue)
		chk = chkInvalidIdentifier("fild;Name")
		So(chk, ShouldBeTrue)
		chk = chkInvalidIdentifier("_123456789_123456789_123456789_123456789_123456789_123456789_12345")
		So(chk, ShouldBeTrue)

	})
}

func TestJoinByRequest(t *testing.T) {
	Convey("Join by request", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_join=inner:test2:test2.name:$eq:test.name", nil)
		So(err, ShouldBeNil)

		join, err := JoinByRequest(r)
		joinStr := strings.Join(join, " ")

		So(err, ShouldBeNil)
		So(joinStr, ShouldContainSubstring, "INNER JOIN test2 ON test2.name = test.name")
	})
	Convey("Join missing param", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_join=inner:test2:test2.name:$eq", nil)
		So(err, ShouldNotBeNil)

		_, err = JoinByRequest(r)
		So(err, ShouldNotBeNil)
	})
	Convey("Join invalid operator", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_join=inner:test2:test2.name:notexist:test.name", nil)
		So(err, ShouldBeNil)

		_, err = JoinByRequest(r)
		So(err, ShouldNotBeNil)
	})
	Convey("Join with where", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_join=inner:test2:test2.name:$eq:test.name&name=nuveo&data->>description:jsonb=bla", nil)
		So(err, ShouldBeNil)

		join, err := JoinByRequest(r)
		joinStr := strings.Join(join, " ")

		So(err, ShouldBeNil)
		So(joinStr, ShouldContainSubstring, "INNER JOIN test2 ON test2.name = test.name")

		where, values, err := WhereByRequest(r, 1)
		So(err, ShouldBeNil)
		So(where, ShouldContainSubstring, "name=$")
		So(where, ShouldContainSubstring, "data->>'description'=$")
		So(where, ShouldContainSubstring, " AND ")
		So(values, ShouldContain, "nuveo")
		So(values, ShouldContain, "bla")
	})

}

func TestSelectFields(t *testing.T) {
	Convey("Select fields from table", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_select=celphone", nil)
		So(err, ShouldBeNil)

		selectQuery := SelectByRequest(r)
		So(selectQuery, ShouldContainSubstring, "SELECT celphone FROM")
	})

	Convey("Select all from table", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_select=*", nil)
		So(err, ShouldBeNil)

		selectQuery := SelectByRequest(r)
		So(selectQuery, ShouldContainSubstring, "SELECT * FROM")
	})

	Convey("Try Select with empty '_select' field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_select=", nil)
		So(err, ShouldBeNil)

		selectQuery := SelectByRequest(r)
		So(selectQuery, ShouldEqual, "")
	})
}

func TestCountFields(t *testing.T) {
	Convey("Count fields from table", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_count=celphone", nil)
		So(err, ShouldBeNil)

		countQuery := CountByRequest(r)
		So(countQuery, ShouldContainSubstring, "SELECT COUNT(celphone) FROM")
	})

	Convey("Count all from table", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_count=*", nil)
		So(err, ShouldBeNil)

		countQuery := CountByRequest(r)
		So(countQuery, ShouldContainSubstring, "SELECT COUNT(*) FROM")
	})

	Convey("Try Count with empty '_count' field", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test5?_count=", nil)
		So(err, ShouldBeNil)

		countQuery := CountByRequest(r)
		So(countQuery, ShouldEqual, "")
	})
}

func TestDatabaseClause(t *testing.T) {
	Convey("Return appropriate SELECT clause", t, func() {
		r, err := http.NewRequest("GET", "/databases", nil)
		So(err, ShouldBeNil)

		countQuery := DatabaseClause(r)
		So(countQuery, ShouldEqual, fmt.Sprintf(statements.DatabasesSelect, statements.FieldDatabaseName))
	})

	Convey("Return appropriate COUNT clause", t, func() {
		r, err := http.NewRequest("GET", "/databases?_count=*", nil)
		So(err, ShouldBeNil)

		countQuery := DatabaseClause(r)
		So(countQuery, ShouldEqual, fmt.Sprintf(statements.DatabasesSelect, statements.FieldCountDatabaseName))
	})
}

func TestSchemaClause(t *testing.T) {
	Convey("Return appropriate SELECT clause", t, func() {
		r, err := http.NewRequest("GET", "/schemas", nil)
		So(err, ShouldBeNil)

		countQuery := SchemaClause(r)
		So(countQuery, ShouldEqual, fmt.Sprintf(statements.SchemasSelect, statements.FieldSchemaName))
	})

	Convey("Return appropriate COUNT clause", t, func() {
		r, err := http.NewRequest("GET", "/schemas?_count=*", nil)
		So(err, ShouldBeNil)

		countQuery := SchemaClause(r)
		So(countQuery, ShouldEqual, fmt.Sprintf(statements.SchemasSelect, statements.FieldCountSchemaName))
	})
}

func TestGetQueryOperator(t *testing.T) {
	Convey("Query operator eq", t, func() {
		op, err := GetQueryOperator("$eq")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "=")
	})
	Convey("Query operator gt", t, func() {
		op, err := GetQueryOperator("$gt")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, ">")
	})
	Convey("Query operator gte", t, func() {
		op, err := GetQueryOperator("$gte")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, ">=")
	})

	Convey("Query operator lt", t, func() {
		op, err := GetQueryOperator("$lt")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "<")
	})
	Convey("Query operator lte", t, func() {
		op, err := GetQueryOperator("$lte")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "<=")
	})
	Convey("Query operator IN", t, func() {
		op, err := GetQueryOperator("$in")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "IN")
	})
	Convey("Query operator NIN", t, func() {
		op, err := GetQueryOperator("$nin")
		So(err, ShouldBeNil)
		So(op, ShouldEqual, "NOT IN")
	})
}

func TestOrderByRequest(t *testing.T) {
	Convey("Query ORDER BY", t, func() {
		r, err := http.NewRequest("GET", "/prest/public/test?_order=name,-number", nil)
		So(err, ShouldBeNil)

		order, err := OrderByRequest(r)
		So(err, ShouldBeNil)
		So(order, ShouldContainSubstring, "ORDER BY")
		So(order, ShouldContainSubstring, "name")
		So(order, ShouldContainSubstring, "number DESC")
	})
}
<|endoftext|>


<commit_before>package database

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"log"

	"bytes"

	"time"

	"github.com/ChristianNorbertBraun/seaweed-banking/seaweed-banking-backend/config"
	"github.com/ChristianNorbertBraun/seaweed-banking/seaweed-banking-backend/model"
)

// ReadAccount returns for a given bic and iban an account or an error if there
// is no matching account
func ReadAccount(bic string, iban string) (*model.Account, error) {
	account := model.Account{}

	if err := Connection.
		QueryRow("SELECT bic, iban, balance FROM accountbalance WHERE bic = $1 AND iban = $2",
			bic,
			iban).Scan(&account.BIC, &account.IBAN, &account.Balance); err != nil {
		log.Printf("Unable to read accounts with bic %s and iban %s: %s", bic, iban, err)
		return nil, err
	}

	return &account, nil
}

// ReadAccounts returns all accounts created so far with their balance
func ReadAccounts() ([]*model.Account, error) {
	rows, err := Connection.Query("SELECT bic, iban, balance FROM accountbalance")

	if err != nil {
		log.Printf("Unable to read all accounts: %s", err)
		return nil, err
	}

	accounts := []*model.Account{}

	for rows.Next() {
		current := model.Account{}
		err := rows.Scan(&current.BIC, &current.IBAN, &current.Balance)
		if err != nil {
			return nil, err
		}
		accounts = append(accounts, &current)
	}

	return accounts, nil
}

// UpdateAccountBalance takes a transaction and applies the
// transaction value to the given account
//
// If the the transaction value would make the account balance go below zero
// there will be returned an error an the transaction will be canceld
func UpdateAccountBalance(transaction model.Transaction) error {
	account := model.Account{}
	tx, err := Connection.Begin()
	if err != nil {
		return err
	}
	defer rollback(err, tx)

	row := tx.QueryRow("SELECT bic, iban, balance FROM accountbalance WHERE bic = $1 AND IBAN = $2",
		transaction.BIC,
		transaction.IBAN)
	if err = row.Scan(&account.BIC, &account.IBAN, &account.Balance); err != nil {
		return err
	}

	if (account.Balance + transaction.ValueInSmallestUnit) < 0 {
		err = fmt.Errorf("Tried to withdraw %d from account bic: %s iban: %s with balance: %d",
			transaction.ValueInSmallestUnit,
			transaction.BIC,
			transaction.IBAN,
			account.Balance)

		return err
	}

	_, err = tx.Exec("UPDATE accountbalance SET balance = $1 where bic = $2 AND iban = $3",
		(account.Balance + transaction.ValueInSmallestUnit),
		transaction.BIC,
		transaction.IBAN)

	return err
}

// CreateAccount creates an account with the given data
func CreateAccount(account model.Account) error {
	tx, err := Connection.Begin()
	if err != nil {
		return err
	}
	defer rollback(err, tx)

	if _, err := Connection.Exec("INSERT INTO accountbalance(bic, iban, balance) VALUES ($1, $2, $3)",
		account.BIC,
		account.IBAN,
		account.Balance); err != nil {
		log.Printf("Unable to create account %s", err)
		return err
	}

	if err := createAccountInfo(account); err != nil {
		log.Printf("Unable to create account info for bic %s, iban %s",
			account.BIC,
			account.IBAN)

		return err
	}

	return nil
}

func createAccountInfo(account model.Account) error {
	buffer := bytes.Buffer{}
	fileName := time.Now().UTC().Format(time.RFC3339Nano)
	path := fmt.Sprintf("%s/%s/%s",
		config.Configuration.Seaweed.AccountFolder,
		account.BIC,
		account.IBAN)

	if err := json.NewEncoder(&buffer).Encode(account); err != nil {
		return err
	}

	return filer.Create(&buffer, fileName, path)
}

func rollback(err error, tx *sql.Tx) {
	if err != nil {
		tx.Rollback()

		return
	}
	err = tx.Commit()
}
<commit_msg>Replace sql.tx with driver.tx<commit_after>package database

import (
	"database/sql/driver"
	"encoding/json"
	"fmt"
	"log"

	"bytes"

	"time"

	"github.com/ChristianNorbertBraun/seaweed-banking/seaweed-banking-backend/config"
	"github.com/ChristianNorbertBraun/seaweed-banking/seaweed-banking-backend/model"
)

// ReadAccount returns for a given bic and iban an account or an error if there
// is no matching account
func ReadAccount(bic string, iban string) (*model.Account, error) {
	account := model.Account{}

	if err := Connection.
		QueryRow("SELECT bic, iban, balance FROM accountbalance WHERE bic = $1 AND iban = $2",
			bic,
			iban).Scan(&account.BIC, &account.IBAN, &account.Balance); err != nil {
		log.Printf("Unable to read accounts with bic %s and iban %s: %s", bic, iban, err)
		return nil, err
	}

	return &account, nil
}

// ReadAccounts returns all accounts created so far with their balance
func ReadAccounts() ([]*model.Account, error) {
	rows, err := Connection.Query("SELECT bic, iban, balance FROM accountbalance")

	if err != nil {
		log.Printf("Unable to read all accounts: %s", err)
		return nil, err
	}

	accounts := []*model.Account{}

	for rows.Next() {
		current := model.Account{}
		err := rows.Scan(&current.BIC, &current.IBAN, &current.Balance)
		if err != nil {
			return nil, err
		}
		accounts = append(accounts, &current)
	}

	return accounts, nil
}

// UpdateAccountBalance takes a transaction and applies the
// transaction value to the given account
//
// If the the transaction value would make the account balance go below zero
// there will be returned an error an the transaction will be canceld
func UpdateAccountBalance(transaction model.Transaction) error {
	account := model.Account{}
	tx, err := Connection.Begin()
	if err != nil {
		return err
	}
	defer rollback(err, tx)

	row := tx.QueryRow("SELECT bic, iban, balance FROM accountbalance WHERE bic = $1 AND IBAN = $2",
		transaction.BIC,
		transaction.IBAN)
	if err = row.Scan(&account.BIC, &account.IBAN, &account.Balance); err != nil {
		return err
	}

	if (account.Balance + transaction.ValueInSmallestUnit) < 0 {
		err = fmt.Errorf("Tried to withdraw %d from account bic: %s iban: %s with balance: %d",
			transaction.ValueInSmallestUnit,
			transaction.BIC,
			transaction.IBAN,
			account.Balance)

		return err
	}

	_, err = tx.Exec("UPDATE accountbalance SET balance = $1 where bic = $2 AND iban = $3",
		(account.Balance + transaction.ValueInSmallestUnit),
		transaction.BIC,
		transaction.IBAN)

	return err
}

// CreateAccount creates an account with the given data
func CreateAccount(account model.Account) error {
	tx, err := Connection.Begin()
	if err != nil {
		return err
	}
	defer rollback(err, tx)

	if _, err := Connection.Exec("INSERT INTO accountbalance(bic, iban, balance) VALUES ($1, $2, $3)",
		account.BIC,
		account.IBAN,
		account.Balance); err != nil {
		log.Printf("Unable to create account %s", err)
		return err
	}

	if err := createAccountInfo(account); err != nil {
		log.Printf("Unable to create account info for bic %s, iban %s",
			account.BIC,
			account.IBAN)

		return err
	}

	return nil
}

func createAccountInfo(account model.Account) error {
	buffer := bytes.Buffer{}
	fileName := time.Now().UTC().Format(time.RFC3339Nano)
	path := fmt.Sprintf("%s/%s/%s",
		config.Configuration.Seaweed.AccountFolder,
		account.BIC,
		account.IBAN)

	if err := json.NewEncoder(&buffer).Encode(account); err != nil {
		return err
	}

	return filer.Create(&buffer, fileName, path)
}

func rollback(err error, tx driver.Tx) {
	if err != nil {
		tx.Rollback()

		return
	}
	err = tx.Commit()
}
<|endoftext|>