<commit_before>package indicators

import (
	"container/list"
	"errors"
	"github.com/thetruetrade/gotrade"
	"math"
)

// A Williamns Percent R Indicator
type WillRWithoutStorage struct {
	*baseIndicator
	*baseFloatBounds

	// private variables
	periodHighHistory    *list.List
	periodLowHistory     *list.List
	periodCounter        int
	valueAvailableAction ValueAvailableActionFloat
	timePeriod           int
}

// NewWillRWithoutStorage creates a Williams Percent R Indicator (WillR) without storage
func NewWillRWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *WillRWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 2
	if timePeriod < 2 {
		return nil, errors.New("timePeriod is less than the minimum (2)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := timePeriod - 1
	ind := WillRWithoutStorage{
		baseIndicator:        newBaseIndicator(lookback),
		baseFloatBounds:      newBaseFloatBounds(),
		periodCounter:        timePeriod * -1,
		periodHighHistory:    list.New(),
		periodLowHistory:     list.New(),
		valueAvailableAction: valueAvailableAction,
		timePeriod:           timePeriod,
	}

	return &ind, nil
}

// A Simple Moving Average Indicator
type WillR struct {
	*WillRWithoutStorage

	// public variables
	Data []float64
}

// NewWillR creates a Williams Percent R Indicator (WillR) for online usage
func NewWillR(timePeriod int) (indicator *WillR, err error) {
	ind := WillR{}
	ind.WillRWithoutStorage, err = NewWillRWithoutStorage(timePeriod, func(dataItem float64, streamBarIndex int) {
		ind.Data = append(ind.Data, dataItem)
	})

	return &ind, err
}

// NewDefaultWillR creates a Williams Percent R Indicator (WillR) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultWillR() (indicator *WillR, err error) {
	timePeriod := 14
	return NewWillR(timePeriod)
}

// NewWillRWithSrcLen creates a Williams Percent R Indicator (WillR) for offline usage
func NewWillRWithSrcLen(sourceLength uint, timePeriod int) (indicator *WillR, err error) {
	ind, err := NewWillR(timePeriod)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultWillRWithSrcLen creates a Williams Percent R Indicator (WillR) for offline usage with default parameters
func NewDefaultWillRWithSrcLen(sourceLength uint) (indicator *WillR, err error) {
	ind, err := NewDefaultWillR()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewWillRForStream creates a Williams Percent R Indicator (WillR) for online usage with a source data stream
func NewWillRForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *WillR, err error) {
	ind, err := NewWillR(timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultWillRForStream creates a Williams Percent R Indicator (WillR) for online usage with a source data stream
func NewDefaultWillRForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *WillR, err error) {
	ind, err := NewDefaultWillR()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewWillRForStreamWithSrcLen creates a Williams Percent R Indicator (WillR) for offline usage with a source data stream
func NewWillRForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *WillR, err error) {
	ind, err := NewWillRWithSrcLen(sourceLength, timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultWillRForStreamWithSrcLen creates a Williams Percent R Indicator (WillR) for offline usage with a source data stream
func NewDefaultWillRForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *WillR, err error) {
	ind, err := NewDefaultWillRWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *WillRWithoutStorage) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {

	ind.periodCounter += 1
	ind.periodHighHistory.PushBack(tickData.H())
	ind.periodLowHistory.PushBack(tickData.L())

	highestHigh, _ := highestHighofPeriod(ind.periodHighHistory)
	lowestLow, _ := lowestLowofPeriod(ind.periodLowHistory)

	var result float64 = (highestHigh - tickData.C()) / (highestHigh - lowestLow) * -100.0
	if ind.periodCounter >= 0 {

		// increment the number of results this indicator can be expected to return
		ind.dataLength += 1

		if ind.validFromBar == -1 {
			// set the streamBarIndex from which this indicator returns valid results
			ind.validFromBar = streamBarIndex
		}

		// update the maximum result value
		if result > ind.maxValue {
			ind.maxValue = result
		}

		// update the minimum result value
		if result < ind.minValue {
			ind.minValue = result
		}

		// notify of a new result value though the value available action
		ind.valueAvailableAction(result, streamBarIndex)
	}

	if ind.periodHighHistory.Len() >= ind.timePeriod {
		var first = ind.periodHighHistory.Front()
		ind.periodHighHistory.Remove(first)
	}
	if ind.periodLowHistory.Len() >= ind.timePeriod {
		var first = ind.periodLowHistory.Front()
		ind.periodLowHistory.Remove(first)
	}
}

func highestHighofPeriod(l *list.List) (result float64, err error) {
	if l.Len() == 0 {
		err = errors.New("list is empty no high can be calculated.")
	}

	high := math.SmallestNonzeroFloat64
	for e := l.Front(); e != nil; e = e.Next() {
		value := e.Value.(float64)
		if value > high {
			high = value
		}
	}
	return high, err
}

func lowestLowofPeriod(l *list.List) (result float64, err error) {
	if l.Len() == 0 {
		err = errors.New("list is empty no low can be calculated.")
	}

	low := math.MaxFloat64
	for e := l.Front(); e != nil; e = e.Next() {
		value := e.Value.(float64)
		if value < low {
			low = value
		}
	}

	return low, err
}
<commit_msg>#76 Remove duplication - williamsr<commit_after>package indicators

import (
	"container/list"
	"errors"
	"github.com/thetruetrade/gotrade"
	"math"
)

// A Williamns Percent R Indicator
type WillRWithoutStorage struct {
	*baseIndicatorWithFloatBounds

	// private variables
	periodHighHistory *list.List
	periodLowHistory  *list.List
	periodCounter     int
	timePeriod        int
}

// NewWillRWithoutStorage creates a Williams Percent R Indicator (WillR) without storage
func NewWillRWithoutStorage(timePeriod int, valueAvailableAction ValueAvailableActionFloat) (indicator *WillRWithoutStorage, err error) {

	// an indicator without storage MUST have a value available action
	if valueAvailableAction == nil {
		return nil, ErrValueAvailableActionIsNil
	}

	// the minimum timeperiod for this indicator is 2
	if timePeriod < 2 {
		return nil, errors.New("timePeriod is less than the minimum (2)")
	}

	// check the maximum timeperiod
	if timePeriod > MaximumLookbackPeriod {
		return nil, errors.New("timePeriod is greater than the maximum (100000)")
	}

	lookback := timePeriod - 1
	ind := WillRWithoutStorage{
		baseIndicatorWithFloatBounds: newBaseIndicatorWithFloatBounds(lookback, valueAvailableAction),
		periodCounter:                timePeriod * -1,
		periodHighHistory:            list.New(),
		periodLowHistory:             list.New(),
		timePeriod:                   timePeriod,
	}

	return &ind, nil
}

// A Simple Moving Average Indicator
type WillR struct {
	*WillRWithoutStorage

	// public variables
	Data []float64
}

// NewWillR creates a Williams Percent R Indicator (WillR) for online usage
func NewWillR(timePeriod int) (indicator *WillR, err error) {
	ind := WillR{}
	ind.WillRWithoutStorage, err = NewWillRWithoutStorage(timePeriod, func(dataItem float64, streamBarIndex int) {
		ind.Data = append(ind.Data, dataItem)
	})

	return &ind, err
}

// NewDefaultWillR creates a Williams Percent R Indicator (WillR) for online usage with default parameters
//	- timePeriod: 14
func NewDefaultWillR() (indicator *WillR, err error) {
	timePeriod := 14
	return NewWillR(timePeriod)
}

// NewWillRWithSrcLen creates a Williams Percent R Indicator (WillR) for offline usage
func NewWillRWithSrcLen(sourceLength uint, timePeriod int) (indicator *WillR, err error) {
	ind, err := NewWillR(timePeriod)

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewDefaultWillRWithSrcLen creates a Williams Percent R Indicator (WillR) for offline usage with default parameters
func NewDefaultWillRWithSrcLen(sourceLength uint) (indicator *WillR, err error) {
	ind, err := NewDefaultWillR()

	// only initialise the storage if there is enough source data to require it
	if sourceLength-uint(ind.GetLookbackPeriod()) > 1 {
		ind.Data = make([]float64, 0, sourceLength-uint(ind.GetLookbackPeriod()))
	}

	return ind, err
}

// NewWillRForStream creates a Williams Percent R Indicator (WillR) for online usage with a source data stream
func NewWillRForStream(priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *WillR, err error) {
	ind, err := NewWillR(timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultWillRForStream creates a Williams Percent R Indicator (WillR) for online usage with a source data stream
func NewDefaultWillRForStream(priceStream gotrade.DOHLCVStreamSubscriber) (indicator *WillR, err error) {
	ind, err := NewDefaultWillR()
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewWillRForStreamWithSrcLen creates a Williams Percent R Indicator (WillR) for offline usage with a source data stream
func NewWillRForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber, timePeriod int) (indicator *WillR, err error) {
	ind, err := NewWillRWithSrcLen(sourceLength, timePeriod)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// NewDefaultWillRForStreamWithSrcLen creates a Williams Percent R Indicator (WillR) for offline usage with a source data stream
func NewDefaultWillRForStreamWithSrcLen(sourceLength uint, priceStream gotrade.DOHLCVStreamSubscriber) (indicator *WillR, err error) {
	ind, err := NewDefaultWillRWithSrcLen(sourceLength)
	priceStream.AddTickSubscription(ind)
	return ind, err
}

// ReceiveDOHLCVTick consumes a source data DOHLCV price tick
func (ind *WillRWithoutStorage) ReceiveDOHLCVTick(tickData gotrade.DOHLCV, streamBarIndex int) {

	ind.periodCounter += 1
	ind.periodHighHistory.PushBack(tickData.H())
	ind.periodLowHistory.PushBack(tickData.L())

	highestHigh, _ := highestHighofPeriod(ind.periodHighHistory)
	lowestLow, _ := lowestLowofPeriod(ind.periodLowHistory)

	var result float64 = (highestHigh - tickData.C()) / (highestHigh - lowestLow) * -100.0
	if ind.periodCounter >= 0 {

		ind.UpdateIndicatorWithNewValue(result, streamBarIndex)
	}

	if ind.periodHighHistory.Len() >= ind.timePeriod {
		var first = ind.periodHighHistory.Front()
		ind.periodHighHistory.Remove(first)
	}
	if ind.periodLowHistory.Len() >= ind.timePeriod {
		var first = ind.periodLowHistory.Front()
		ind.periodLowHistory.Remove(first)
	}
}

func highestHighofPeriod(l *list.List) (result float64, err error) {
	if l.Len() == 0 {
		err = errors.New("list is empty no high can be calculated.")
	}

	high := math.SmallestNonzeroFloat64
	for e := l.Front(); e != nil; e = e.Next() {
		value := e.Value.(float64)
		if value > high {
			high = value
		}
	}
	return high, err
}

func lowestLowofPeriod(l *list.List) (result float64, err error) {
	if l.Len() == 0 {
		err = errors.New("list is empty no low can be calculated.")
	}

	low := math.MaxFloat64
	for e := l.Front(); e != nil; e = e.Next() {
		value := e.Value.(float64)
		if value < low {
			low = value
		}
	}

	return low, err
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration-cli/requirement"
	"github.com/docker/docker/internal/test/registry"
)

func ArchitectureIsNot(arch string) bool {
	return os.Getenv("DOCKER_ENGINE_GOARCH") != arch
}

func DaemonIsWindows() bool {
	return testEnv.OSType == "windows"
}

func DaemonIsWindowsAtLeastBuild(buildNumber int) func() bool {
	return func() bool {
		if testEnv.OSType != "windows" {
			return false
		}
		version := testEnv.DaemonInfo.KernelVersion
		numVersion, _ := strconv.Atoi(strings.Split(version, " ")[1])
		return numVersion >= buildNumber
	}
}

func DaemonIsLinux() bool {
	return testEnv.OSType == "linux"
}

func MinimumAPIVersion(version string) func() bool {
	return func() bool {
		return versions.GreaterThanOrEqualTo(testEnv.DaemonAPIVersion(), version)
	}
}

func OnlyDefaultNetworks() bool {
	cli, err := client.NewEnvClient()
	if err != nil {
		return false
	}
	networks, err := cli.NetworkList(context.TODO(), types.NetworkListOptions{})
	if err != nil || len(networks) > 0 {
		return false
	}
	return true
}

// Deprecated: use skip.If(t, !testEnv.DaemonInfo.ExperimentalBuild)
func ExperimentalDaemon() bool {
	return testEnv.DaemonInfo.ExperimentalBuild
}

func IsAmd64() bool {
	return os.Getenv("DOCKER_ENGINE_GOARCH") == "amd64"
}

func NotArm() bool {
	return ArchitectureIsNot("arm")
}

func NotArm64() bool {
	return ArchitectureIsNot("arm64")
}

func NotPpc64le() bool {
	return ArchitectureIsNot("ppc64le")
}

func NotS390X() bool {
	return ArchitectureIsNot("s390x")
}

func SameHostDaemon() bool {
	return testEnv.IsLocalDaemon()
}

func UnixCli() bool {
	return isUnixCli
}

func ExecSupport() bool {
	return supportsExec
}

func Network() bool {
	// Set a timeout on the GET at 15s
	var timeout = time.Duration(15 * time.Second)
	var url = "https://hub.docker.com"

	client := http.Client{
		Timeout: timeout,
	}

	resp, err := client.Get(url)
	if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
		panic(fmt.Sprintf("Timeout for GET request on %s", url))
	}
	if resp != nil {
		resp.Body.Close()
	}
	return err == nil
}

func Apparmor() bool {
	if strings.HasPrefix(testEnv.DaemonInfo.OperatingSystem, "SUSE Linux Enterprise Server ") {
		return false
	}
	buf, err := ioutil.ReadFile("/sys/module/apparmor/parameters/enabled")
	return err == nil && len(buf) > 1 && buf[0] == 'Y'
}

func Devicemapper() bool {
	return strings.HasPrefix(testEnv.DaemonInfo.Driver, "devicemapper")
}

func IPv6() bool {
	cmd := exec.Command("test", "-f", "/proc/net/if_inet6")
	return cmd.Run() != nil
}

func UserNamespaceROMount() bool {
	// quick case--userns not enabled in this test run
	if os.Getenv("DOCKER_REMAP_ROOT") == "" {
		return true
	}
	if _, _, err := dockerCmdWithError("run", "--rm", "--read-only", "busybox", "date"); err != nil {
		return false
	}
	return true
}

func NotUserNamespace() bool {
	root := os.Getenv("DOCKER_REMAP_ROOT")
	return root == ""
}

func UserNamespaceInKernel() bool {
	if _, err := os.Stat("/proc/self/uid_map"); os.IsNotExist(err) {
		/*
		 * This kernel-provided file only exists if user namespaces are
		 * supported
		 */
		return false
	}

	// We need extra check on redhat based distributions
	if f, err := os.Open("/sys/module/user_namespace/parameters/enable"); err == nil {
		defer f.Close()
		b := make([]byte, 1)
		_, _ = f.Read(b)
		return string(b) != "N"
	}

	return true
}

func IsPausable() bool {
	if testEnv.OSType == "windows" {
		return testEnv.DaemonInfo.Isolation == "hyperv"
	}
	return true
}

func NotPausable() bool {
	if testEnv.OSType == "windows" {
		return testEnv.DaemonInfo.Isolation == "process"
	}
	return false
}

func IsolationIs(expectedIsolation string) bool {
	return testEnv.OSType == "windows" && string(testEnv.DaemonInfo.Isolation) == expectedIsolation
}

func IsolationIsHyperv() bool {
	return IsolationIs("hyperv")
}

func IsolationIsProcess() bool {
	return IsolationIs("process")
}

// RegistryHosting returns wether the host can host a registry (v2) or not
func RegistryHosting() bool {
	// for now registry binary is built only if we're running inside
	// container through `make test`. Figure that out by testing if
	// registry binary is in PATH.
	_, err := exec.LookPath(registry.V2binary)
	return err == nil
}

func SwarmInactive() bool {
	return testEnv.DaemonInfo.Swarm.LocalNodeState == swarm.LocalNodeStateInactive
}

func TODOBuildkit() bool {
	return os.Getenv("DOCKER_BUILDKIT") == ""
}

// testRequires checks if the environment satisfies the requirements
// for the test to run or skips the tests.
func testRequires(c requirement.SkipT, requirements ...requirement.Test) {
	requirement.Is(c, requirements...)
}
<commit_msg>fix typos:  wether -> whether<commit_after>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/swarm"
	"github.com/docker/docker/api/types/versions"
	"github.com/docker/docker/client"
	"github.com/docker/docker/integration-cli/requirement"
	"github.com/docker/docker/internal/test/registry"
)

func ArchitectureIsNot(arch string) bool {
	return os.Getenv("DOCKER_ENGINE_GOARCH") != arch
}

func DaemonIsWindows() bool {
	return testEnv.OSType == "windows"
}

func DaemonIsWindowsAtLeastBuild(buildNumber int) func() bool {
	return func() bool {
		if testEnv.OSType != "windows" {
			return false
		}
		version := testEnv.DaemonInfo.KernelVersion
		numVersion, _ := strconv.Atoi(strings.Split(version, " ")[1])
		return numVersion >= buildNumber
	}
}

func DaemonIsLinux() bool {
	return testEnv.OSType == "linux"
}

func MinimumAPIVersion(version string) func() bool {
	return func() bool {
		return versions.GreaterThanOrEqualTo(testEnv.DaemonAPIVersion(), version)
	}
}

func OnlyDefaultNetworks() bool {
	cli, err := client.NewEnvClient()
	if err != nil {
		return false
	}
	networks, err := cli.NetworkList(context.TODO(), types.NetworkListOptions{})
	if err != nil || len(networks) > 0 {
		return false
	}
	return true
}

// Deprecated: use skip.If(t, !testEnv.DaemonInfo.ExperimentalBuild)
func ExperimentalDaemon() bool {
	return testEnv.DaemonInfo.ExperimentalBuild
}

func IsAmd64() bool {
	return os.Getenv("DOCKER_ENGINE_GOARCH") == "amd64"
}

func NotArm() bool {
	return ArchitectureIsNot("arm")
}

func NotArm64() bool {
	return ArchitectureIsNot("arm64")
}

func NotPpc64le() bool {
	return ArchitectureIsNot("ppc64le")
}

func NotS390X() bool {
	return ArchitectureIsNot("s390x")
}

func SameHostDaemon() bool {
	return testEnv.IsLocalDaemon()
}

func UnixCli() bool {
	return isUnixCli
}

func ExecSupport() bool {
	return supportsExec
}

func Network() bool {
	// Set a timeout on the GET at 15s
	var timeout = time.Duration(15 * time.Second)
	var url = "https://hub.docker.com"

	client := http.Client{
		Timeout: timeout,
	}

	resp, err := client.Get(url)
	if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
		panic(fmt.Sprintf("Timeout for GET request on %s", url))
	}
	if resp != nil {
		resp.Body.Close()
	}
	return err == nil
}

func Apparmor() bool {
	if strings.HasPrefix(testEnv.DaemonInfo.OperatingSystem, "SUSE Linux Enterprise Server ") {
		return false
	}
	buf, err := ioutil.ReadFile("/sys/module/apparmor/parameters/enabled")
	return err == nil && len(buf) > 1 && buf[0] == 'Y'
}

func Devicemapper() bool {
	return strings.HasPrefix(testEnv.DaemonInfo.Driver, "devicemapper")
}

func IPv6() bool {
	cmd := exec.Command("test", "-f", "/proc/net/if_inet6")
	return cmd.Run() != nil
}

func UserNamespaceROMount() bool {
	// quick case--userns not enabled in this test run
	if os.Getenv("DOCKER_REMAP_ROOT") == "" {
		return true
	}
	if _, _, err := dockerCmdWithError("run", "--rm", "--read-only", "busybox", "date"); err != nil {
		return false
	}
	return true
}

func NotUserNamespace() bool {
	root := os.Getenv("DOCKER_REMAP_ROOT")
	return root == ""
}

func UserNamespaceInKernel() bool {
	if _, err := os.Stat("/proc/self/uid_map"); os.IsNotExist(err) {
		/*
		 * This kernel-provided file only exists if user namespaces are
		 * supported
		 */
		return false
	}

	// We need extra check on redhat based distributions
	if f, err := os.Open("/sys/module/user_namespace/parameters/enable"); err == nil {
		defer f.Close()
		b := make([]byte, 1)
		_, _ = f.Read(b)
		return string(b) != "N"
	}

	return true
}

func IsPausable() bool {
	if testEnv.OSType == "windows" {
		return testEnv.DaemonInfo.Isolation == "hyperv"
	}
	return true
}

func NotPausable() bool {
	if testEnv.OSType == "windows" {
		return testEnv.DaemonInfo.Isolation == "process"
	}
	return false
}

func IsolationIs(expectedIsolation string) bool {
	return testEnv.OSType == "windows" && string(testEnv.DaemonInfo.Isolation) == expectedIsolation
}

func IsolationIsHyperv() bool {
	return IsolationIs("hyperv")
}

func IsolationIsProcess() bool {
	return IsolationIs("process")
}

// RegistryHosting returns whether the host can host a registry (v2) or not
func RegistryHosting() bool {
	// for now registry binary is built only if we're running inside
	// container through `make test`. Figure that out by testing if
	// registry binary is in PATH.
	_, err := exec.LookPath(registry.V2binary)
	return err == nil
}

func SwarmInactive() bool {
	return testEnv.DaemonInfo.Swarm.LocalNodeState == swarm.LocalNodeStateInactive
}

func TODOBuildkit() bool {
	return os.Getenv("DOCKER_BUILDKIT") == ""
}

// testRequires checks if the environment satisfies the requirements
// for the test to run or skips the tests.
func testRequires(c requirement.SkipT, requirements ...requirement.Test) {
	requirement.Is(c, requirements...)
}
<|endoftext|>


<commit_before>package lifecycle_test

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/cloudfoundry-incubator/garden"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/types"
)

var dockerRegistryRootFSPath = os.Getenv("GARDEN_DOCKER_REGISTRY_TEST_ROOTFS")

var _ = Describe("Rootfs container create parameter", func() {
	var container garden.Container
	var args []string

	BeforeEach(func() {
		container = nil
		args = []string{}
	})

	JustBeforeEach(func() {
		client = startGarden(args...)
	})

	AfterEach(func() {
		if container != nil {
			Expect(client.Destroy(container.Handle())).To(Succeed())
		}
	})

	Context("without a default rootfs", func() {
		BeforeEach(func() {
			args = []string{"--rootfs", ""}
		})

		It("without a rootfs in container spec, the container creation fails", func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{RootFSPath: ""})
			Ω(err).Should(HaveOccurred())
			Ω(err).Should(MatchError(ContainSubstring(
				"RootFSPath: is a required parameter, since no default rootfs was provided to the server. To provide a default rootfs, use the --rootfs flag on startup.",
			)))
		})

		It("with a rootfs in container spec, the container is created successfully", func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{RootFSPath: os.Getenv("GARDEN_TEST_ROOTFS")})
			Ω(err).ShouldNot(HaveOccurred())
		})
	})

	Context("with a default rootfs", func() {
		It("the container is created successfully", func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{RootFSPath: ""})
			Expect(err).ToNot(HaveOccurred())
		})
	})

	Context("with a docker rootfs URI", func() {
		Context("not containing a host", func() {
			It("the container is created successfully", func() {
				var err error

				container, err = client.Create(garden.ContainerSpec{RootFSPath: "docker:///busybox"})
				Expect(err).ToNot(HaveOccurred())
			})
		})

		Context("containing a host", func() {
			Context("which is valid", func() {
				It("the container is created successfully", func() {
					var err error

					container, err = client.Create(garden.ContainerSpec{RootFSPath: "docker://index.docker.io/busybox"})
					Expect(err).ToNot(HaveOccurred())
				})
			})

			Context("which is invalid", func() {
				It("the container is not created successfully", func() {
					var err error

					container, err = client.Create(garden.ContainerSpec{RootFSPath: "docker://xindex.docker.io/busybox"})
					Expect(err.Error()).To(MatchRegexp("could not resolve"))
				})
			})

			Context("which is insecure", func() {
				var dockerRegistry garden.Container

				dockerRegistryIP := "10.0.0.1"
				dockerRegistryPort := "5001"

				if dockerRegistryRootFSPath == "" {
					log.Println("GARDEN_DOCKER_REGISTRY_TEST_ROOTFS undefined; skipping")
					return
				}

				JustBeforeEach(func() {
					dockerRegistry = startDockerRegistry(dockerRegistryIP, dockerRegistryPort)
				})

				AfterEach(func() {
					if dockerRegistry != nil {
						Expect(client.Destroy(dockerRegistry.Handle())).To(Succeed())
					}
				})

				Context("when the host is listed in -insecureDockerRegistryList", func() {
					BeforeEach(func() {
						args = []string{
							"-insecureDockerRegistryList", dockerRegistryIP + ":" + dockerRegistryPort,
							"-allowHostAccess=true",
						}
					})

					It("creates the container successfully ", func() {
						_, err := client.Create(garden.ContainerSpec{
							RootFSPath: fmt.Sprintf("docker://%s:%s/busybox", dockerRegistryIP, dockerRegistryPort),
						})
						Expect(err).ToNot(HaveOccurred())
					})
				})

				Context("when the host is NOT listed in -insecureDockerRegistryList", func() {
					It("fails, and suggests the -insecureDockerRegistryList flag", func() {
						_, err := client.Create(garden.ContainerSpec{
							RootFSPath: fmt.Sprintf("docker://%s:%s/busybox", dockerRegistryIP, dockerRegistryPort),
						})

						Expect(err).To(MatchError(ContainSubstring(
							"Registry %s:%s is missing from -insecureDockerRegistryList ([])", dockerRegistryIP, dockerRegistryPort,
						)))
					})
				})
			})
		})
	})
})

func startDockerRegistry(dockerRegistryIP string, dockerRegistryPort string) garden.Container {
	dockerRegistry, err := client.Create(
		garden.ContainerSpec{
			RootFSPath: dockerRegistryRootFSPath,
			Network:    dockerRegistryIP,
		},
	)
	Expect(err).ToNot(HaveOccurred())

	_, err = dockerRegistry.Run(garden.ProcessSpec{
		User: "vcap",
		Env: []string{
			"DOCKER_REGISTRY_CONFIG=/docker-registry/config/config_sample.yml",
			fmt.Sprintf("REGISTRY_PORT=%s", dockerRegistryPort),
			"STANDALONE=true",
			"MIRROR_SOURCE=https://registry-1.docker.io",
			"MIRROR_SOURCE_INDEX=https://index.docker.io",
		},
		Path: "docker-registry",
	}, garden.ProcessIO{Stdout: GinkgoWriter, Stderr: GinkgoWriter})
	Expect(err).ToNot(HaveOccurred())

	Eventually(
		fmt.Sprintf("http://%s:%s/_ping", dockerRegistryIP, dockerRegistryPort),
		"5s",
	).Should(RespondToGETWith(200))

	return dockerRegistry
}

type statusMatcher struct {
	expectedStatus int

	httpError    error
	actualStatus int
}

func RespondToGETWith(expected int) types.GomegaMatcher {
	return &statusMatcher{expected, nil, 200}
}

func (m *statusMatcher) Match(actual interface{}) (success bool, err error) {
	response, err := http.Get(fmt.Sprintf("%s", actual))
	if err != nil {
		m.httpError = err
		return false, nil
	}

	m.httpError = nil
	m.actualStatus = response.StatusCode
	return response.StatusCode == m.expectedStatus, nil
}

func (m *statusMatcher) FailureMessage(actual interface{}) string {
	if m.httpError != nil {
		return fmt.Sprintf("Expected http request to have status %d but got error: %s", m.expectedStatus, m.httpError.Error())
	}

	return fmt.Sprintf("Expected http status code to be %d but was %d", m.expectedStatus, m.actualStatus)
}

func (m *statusMatcher) NegatedFailureMessage(actual interface{}) string {
	if m.httpError != nil {
		return fmt.Sprintf("Expected http request to have status %d, but got error: %s", m.expectedStatus, m.httpError.Error())
	}

	return fmt.Sprintf("Expected http status code not to be %d", m.expectedStatus)
}
<commit_msg>bump docker registry startup timeout in tests to reduce flakes<commit_after>package lifecycle_test

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/cloudfoundry-incubator/garden"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/types"
)

var dockerRegistryRootFSPath = os.Getenv("GARDEN_DOCKER_REGISTRY_TEST_ROOTFS")

var _ = Describe("Rootfs container create parameter", func() {
	var container garden.Container
	var args []string

	BeforeEach(func() {
		container = nil
		args = []string{}
	})

	JustBeforeEach(func() {
		client = startGarden(args...)
	})

	AfterEach(func() {
		if container != nil {
			Expect(client.Destroy(container.Handle())).To(Succeed())
		}
	})

	Context("without a default rootfs", func() {
		BeforeEach(func() {
			args = []string{"--rootfs", ""}
		})

		It("without a rootfs in container spec, the container creation fails", func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{RootFSPath: ""})
			Ω(err).Should(HaveOccurred())
			Ω(err).Should(MatchError(ContainSubstring(
				"RootFSPath: is a required parameter, since no default rootfs was provided to the server. To provide a default rootfs, use the --rootfs flag on startup.",
			)))
		})

		It("with a rootfs in container spec, the container is created successfully", func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{RootFSPath: os.Getenv("GARDEN_TEST_ROOTFS")})
			Ω(err).ShouldNot(HaveOccurred())
		})
	})

	Context("with a default rootfs", func() {
		It("the container is created successfully", func() {
			var err error

			container, err = client.Create(garden.ContainerSpec{RootFSPath: ""})
			Expect(err).ToNot(HaveOccurred())
		})
	})

	Context("with a docker rootfs URI", func() {
		Context("not containing a host", func() {
			It("the container is created successfully", func() {
				var err error

				container, err = client.Create(garden.ContainerSpec{RootFSPath: "docker:///busybox"})
				Expect(err).ToNot(HaveOccurred())
			})
		})

		Context("containing a host", func() {
			Context("which is valid", func() {
				It("the container is created successfully", func() {
					var err error

					container, err = client.Create(garden.ContainerSpec{RootFSPath: "docker://index.docker.io/busybox"})
					Expect(err).ToNot(HaveOccurred())
				})
			})

			Context("which is invalid", func() {
				It("the container is not created successfully", func() {
					var err error

					container, err = client.Create(garden.ContainerSpec{RootFSPath: "docker://xindex.docker.io/busybox"})
					Expect(err.Error()).To(MatchRegexp("could not resolve"))
				})
			})

			Context("which is insecure", func() {
				var dockerRegistry garden.Container

				dockerRegistryIP := "10.0.0.1"
				dockerRegistryPort := "5001"

				if dockerRegistryRootFSPath == "" {
					log.Println("GARDEN_DOCKER_REGISTRY_TEST_ROOTFS undefined; skipping")
					return
				}

				JustBeforeEach(func() {
					dockerRegistry = startDockerRegistry(dockerRegistryIP, dockerRegistryPort)
				})

				AfterEach(func() {
					if dockerRegistry != nil {
						Expect(client.Destroy(dockerRegistry.Handle())).To(Succeed())
					}
				})

				Context("when the host is listed in -insecureDockerRegistryList", func() {
					BeforeEach(func() {
						args = []string{
							"-insecureDockerRegistryList", dockerRegistryIP + ":" + dockerRegistryPort,
							"-allowHostAccess=true",
						}
					})

					It("creates the container successfully ", func() {
						_, err := client.Create(garden.ContainerSpec{
							RootFSPath: fmt.Sprintf("docker://%s:%s/busybox", dockerRegistryIP, dockerRegistryPort),
						})
						Expect(err).ToNot(HaveOccurred())
					})
				})

				Context("when the host is NOT listed in -insecureDockerRegistryList", func() {
					It("fails, and suggests the -insecureDockerRegistryList flag", func() {
						_, err := client.Create(garden.ContainerSpec{
							RootFSPath: fmt.Sprintf("docker://%s:%s/busybox", dockerRegistryIP, dockerRegistryPort),
						})

						Expect(err).To(MatchError(ContainSubstring(
							"Registry %s:%s is missing from -insecureDockerRegistryList ([])", dockerRegistryIP, dockerRegistryPort,
						)))
					})
				})
			})
		})
	})
})

func startDockerRegistry(dockerRegistryIP string, dockerRegistryPort string) garden.Container {
	dockerRegistry, err := client.Create(
		garden.ContainerSpec{
			RootFSPath: dockerRegistryRootFSPath,
			Network:    dockerRegistryIP,
		},
	)
	Expect(err).ToNot(HaveOccurred())

	_, err = dockerRegistry.Run(garden.ProcessSpec{
		User: "vcap",
		Env: []string{
			"DOCKER_REGISTRY_CONFIG=/docker-registry/config/config_sample.yml",
			fmt.Sprintf("REGISTRY_PORT=%s", dockerRegistryPort),
			"STANDALONE=true",
			"MIRROR_SOURCE=https://registry-1.docker.io",
			"MIRROR_SOURCE_INDEX=https://index.docker.io",
		},
		Path: "docker-registry",
	}, garden.ProcessIO{Stdout: GinkgoWriter, Stderr: GinkgoWriter})
	Expect(err).ToNot(HaveOccurred())

	Eventually(
		fmt.Sprintf("http://%s:%s/_ping", dockerRegistryIP, dockerRegistryPort),
		"10s",
	).Should(RespondToGETWith(200))

	return dockerRegistry
}

type statusMatcher struct {
	expectedStatus int

	httpError    error
	actualStatus int
}

func RespondToGETWith(expected int) types.GomegaMatcher {
	return &statusMatcher{expected, nil, 200}
}

func (m *statusMatcher) Match(actual interface{}) (success bool, err error) {
	response, err := http.Get(fmt.Sprintf("%s", actual))
	if err != nil {
		m.httpError = err
		return false, nil
	}

	m.httpError = nil
	m.actualStatus = response.StatusCode
	return response.StatusCode == m.expectedStatus, nil
}

func (m *statusMatcher) FailureMessage(actual interface{}) string {
	if m.httpError != nil {
		return fmt.Sprintf("Expected http request to have status %d but got error: %s", m.expectedStatus, m.httpError.Error())
	}

	return fmt.Sprintf("Expected http status code to be %d but was %d", m.expectedStatus, m.actualStatus)
}

func (m *statusMatcher) NegatedFailureMessage(actual interface{}) string {
	if m.httpError != nil {
		return fmt.Sprintf("Expected http request to have status %d, but got error: %s", m.expectedStatus, m.httpError.Error())
	}

	return fmt.Sprintf("Expected http status code not to be %d", m.expectedStatus)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"time"

	"github.com/gin-gonic/gin"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var (
	router   *gin.Engine
	response *httptest.ResponseRecorder
)

func sendRequest(method, path string, body ...string) (err error) {
	var req *http.Request

	response = httptest.NewRecorder()

	if method == "POST" {
		reqBody := strings.NewReader(body[0])
		req, err = http.NewRequest(method, "http://logbook.test"+path, reqBody)
	} else {
		req, err = http.NewRequest(method, "http://logbook.test"+path, nil)
	}

	if err != nil {
		return err
	}

	req.SetBasicAuth(config.Auth.User, config.Auth.Password)

	if method == "POST" {
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	router.ServeHTTP(response, req)

	return nil
}

var _ = Describe("Actions", func() {
	var query string

	BeforeEach(func() {
		gin.SetMode(gin.TestMode)
		router = setupRouter()
	})

	AssertSuccess := func() {
		It("should respond with 200", func() {
			Expect(response.Code).To(Equal(200))
		})
	}

	AssertUnprocessable := func() {
		It("should respond with 422", func() {
			Expect(response.Code).To(Equal(422))
		})

		It("should respond with error", func() {
			parsedRes := ErrorResponse{}
			Expect(
				json.Unmarshal(response.Body.Bytes(), &parsedRes),
			).To(Succeed())

			Expect(parsedRes.Error).NotTo(BeEmpty())
		})
	}

	Describe("/:application/put", func() {
		BeforeEach(func() {
			query = "message=Lorem%20ipsum&level=1&tags=tag1,tag2"
		})

		JustBeforeEach(func() {
			Expect(
				sendRequest("POST", "/apptest/put", query),
			).To(Succeed())
		})

		AssertSuccess()

		It("should respond with created log record", func() {
			parsedRes := LogRecord{}
			Expect(
				json.Unmarshal(response.Body.Bytes(), &parsedRes),
			).To(Succeed())

			Expect(parsedRes.Message).To(Equal("Lorem ipsum"))
			Expect(parsedRes.Level).To(Equal(1))
			Expect(parsedRes.CreatedAt).To(BeAssignableToTypeOf(time.Now()))
			Expect(parsedRes.Tags).To(Equal([]string{"tag1", "tag2"}))
		})

		Context("without tags", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=1"
			})
			AssertSuccess()
		})

		Context("with level 5", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=5&tags=tag1,tag2"
			})
			AssertSuccess()
		})

		Context("without message", func() {
			BeforeEach(func() {
				query = "message=level=5&tags=tag1,tag2"
			})
			AssertUnprocessable()
		})

		Context("without level", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=&tags=tag1,tag2"
			})
			AssertUnprocessable()
		})

		Context("with level < 0", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=-1&tags=tag1,tag2"
			})
			AssertUnprocessable()
		})

		Context("with level > 6", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=6&tags=tag1,tag2"
			})
			AssertUnprocessable()
		})

		Context("with empty string tag", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=1&tags=tag1,,tag2"
			})
			AssertUnprocessable()
		})

		Context("with created_at", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=1&tags=tag1,tag2&created_at=2015-10-16T08:10:11.123"
			})

			It("should create log record with provided datetime", func() {
				parsedRes := LogRecord{}
				Expect(
					json.Unmarshal(response.Body.Bytes(), &parsedRes),
				).To(Succeed())

				Expect(parsedRes.CreatedAt.Truncate(time.Millisecond)).To(
					Equal(time.Date(2015, 10, 16, 8, 10, 11, 123000000, time.Local)),
				)
			})
		})

		Context("with invalid created_at", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=1&tags=tag1,tag2&created_at=2015-10-16T08:10:111"
			})
			AssertUnprocessable()
		})
	})

	Describe("/:application/get", func() {
		generateLogRecord := func(application, message string, level int, tags ...string) {
			logRecord := LogRecord{
				Message: message,
				Level:   level,
				Tags:    tags,
			}
			Expect(
				saveLogRecord(application, &logRecord),
			).To(Succeed())
		}

		BeforeEach(func() {
			generateLogRecord("testapp1", "Message one", 1, "tag2, tag3", "tag4")
			generateLogRecord("testapp1", "Message two", 2, "tag2", "tag3", "tag4")
			generateLogRecord("testapp1", "Message three", 3, "tag3", "tag4", "tag5")
			generateLogRecord("testapp1", "Message four", 3, "tag1", "tag2", "tag3")
			generateLogRecord("testapp2", "Message five", 3, "tag3", "tag4")

			query = fmt.Sprintf(
				"level=2&tags=tag3,tag4&start_time=%v&end_time=%v&page=1",
				time.Now().Format("2006-01-02"),
				time.Now().Format("2006-01-02"),
			)
		})

		JustBeforeEach(func() {
			Expect(
				sendRequest("GET", "/testapp1/get?"+query),
			).To(Succeed())
		})

		AssertSuccess()

		It("should respond with found log records", func() {
			parsedRes := LogRecords{}
			Expect(
				json.Unmarshal(response.Body.Bytes(), &parsedRes),
			).To(Succeed())

			Expect(parsedRes).To(HaveLen(2))

			Expect(parsedRes[0].Message).To(Equal("Message two"))
			Expect(parsedRes[0].Level).To(Equal(2))
			Expect(parsedRes[0].CreatedAt).To(BeAssignableToTypeOf(time.Now()))
			Expect(parsedRes[0].Tags).To(ConsistOf("tag2", "tag3", "tag4"))

			Expect(parsedRes[1].Message).To(Equal("Message three"))
			Expect(parsedRes[1].Level).To(Equal(3))
			Expect(parsedRes[1].CreatedAt).To(BeAssignableToTypeOf(time.Now()))
			Expect(parsedRes[1].Tags).To(ConsistOf("tag3", "tag4", "tag5"))
		})

		Context("without tags", func() {
			BeforeEach(func() {
				query = "level=1&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertSuccess()
		})

		Context("with level 5", func() {
			BeforeEach(func() {
				query = "level=5&tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertSuccess()
		})

		Context("with datetime as start_time and end_time", func() {
			BeforeEach(func() {
				query = "level=1&tags=tag3,tag4&start_time=2006-01-02T01:02:03&end_time=2006-01-02T01:02:03&page=1"
			})
			AssertSuccess()
		})

		Context("without page", func() {
			BeforeEach(func() {
				query = "level=1&tags=tag3,tag4&start_time=2006-01-02T01:02:03&end_time=2006-01-02T01:02:03"
			})
			AssertSuccess()
		})

		Context("without level", func() {
			BeforeEach(func() {
				query = "tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with level < 0", func() {
			BeforeEach(func() {
				query = "level=-1tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with level > 5", func() {
			BeforeEach(func() {
				query = "level=6tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with empty string tag", func() {
			BeforeEach(func() {
				query = "level=1tags=tag3,,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with invalid start_time", func() {
			BeforeEach(func() {
				query = "level=1tags=tag3,tag4&start_time=2006-01-022&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with invalid end_time", func() {
			BeforeEach(func() {
				query = "level=1tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-022&page=1"
			})
			AssertUnprocessable()
		})

		Context("with invalid page", func() {
			BeforeEach(func() {
				query = "level=1tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=a"
			})
			AssertUnprocessable()
		})
	})
})
<commit_msg>Fix test<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"time"

	"github.com/gin-gonic/gin"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var (
	router   *gin.Engine
	response *httptest.ResponseRecorder
)

func sendRequest(method, path string, body ...string) (err error) {
	var req *http.Request

	response = httptest.NewRecorder()

	if method == "POST" {
		reqBody := strings.NewReader(body[0])
		req, err = http.NewRequest(method, "http://logbook.test"+path, reqBody)
	} else {
		req, err = http.NewRequest(method, "http://logbook.test"+path, nil)
	}

	if err != nil {
		return err
	}

	req.SetBasicAuth(config.Auth.User, config.Auth.Password)

	if method == "POST" {
		req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	}

	router.ServeHTTP(response, req)

	return nil
}

var _ = Describe("Actions", func() {
	var query string

	BeforeEach(func() {
		gin.SetMode(gin.TestMode)
		router = setupRouter()
	})

	AssertSuccess := func() {
		It("should respond with 200", func() {
			Expect(response.Code).To(Equal(200))
		})
	}

	AssertUnprocessable := func() {
		It("should respond with 422", func() {
			Expect(response.Code).To(Equal(422))
		})

		It("should respond with error", func() {
			parsedRes := ErrorResponse{}
			Expect(
				json.Unmarshal(response.Body.Bytes(), &parsedRes),
			).To(Succeed())

			Expect(parsedRes.Error).NotTo(BeEmpty())
		})
	}

	Describe("/:application/put", func() {
		BeforeEach(func() {
			query = "message=Lorem%20ipsum&level=1&tags=tag1,tag2"
		})

		JustBeforeEach(func() {
			Expect(
				sendRequest("POST", "/apptest/put", query),
			).To(Succeed())
		})

		AssertSuccess()

		It("should respond with created log record", func() {
			parsedRes := LogRecord{}
			Expect(
				json.Unmarshal(response.Body.Bytes(), &parsedRes),
			).To(Succeed())

			Expect(parsedRes.Message).To(Equal("Lorem ipsum"))
			Expect(parsedRes.Level).To(Equal(1))
			Expect(parsedRes.CreatedAt).To(BeAssignableToTypeOf(time.Now()))
			Expect(parsedRes.Tags).To(Equal([]string{"tag1", "tag2"}))
		})

		Context("without tags", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=1"
			})
			AssertSuccess()
		})

		Context("with level 5", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=5&tags=tag1,tag2"
			})
			AssertSuccess()
		})

		Context("without message", func() {
			BeforeEach(func() {
				query = "message=level=5&tags=tag1,tag2"
			})
			AssertUnprocessable()
		})

		Context("without level", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=&tags=tag1,tag2"
			})
			AssertUnprocessable()
		})

		Context("with level < 0", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=-1&tags=tag1,tag2"
			})
			AssertUnprocessable()
		})

		Context("with level > 6", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=6&tags=tag1,tag2"
			})
			AssertUnprocessable()
		})

		Context("with empty string tag", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=1&tags=tag1,,tag2"
			})
			AssertUnprocessable()
		})

		Context("with created_at", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=1&tags=tag1,tag2&created_at=2015-10-16T08:10:11.123"
			})

			It("should create log record with provided datetime", func() {
				parsedRes := LogRecord{}
				Expect(
					json.Unmarshal(response.Body.Bytes(), &parsedRes),
				).To(Succeed())

				Expect(parsedRes.CreatedAt.Local().Truncate(time.Millisecond)).To(
					Equal(time.Date(2015, 10, 16, 8, 10, 11, 123000000, time.Local)),
				)
			})
		})

		Context("with invalid created_at", func() {
			BeforeEach(func() {
				query = "message=Lorem%20ipsum&level=1&tags=tag1,tag2&created_at=2015-10-16T08:10:111"
			})
			AssertUnprocessable()
		})
	})

	Describe("/:application/get", func() {
		generateLogRecord := func(application, message string, level int, tags ...string) {
			logRecord := LogRecord{
				Message: message,
				Level:   level,
				Tags:    tags,
			}
			Expect(
				saveLogRecord(application, &logRecord),
			).To(Succeed())
		}

		BeforeEach(func() {
			generateLogRecord("testapp1", "Message one", 1, "tag2, tag3", "tag4")
			generateLogRecord("testapp1", "Message two", 2, "tag2", "tag3", "tag4")
			generateLogRecord("testapp1", "Message three", 3, "tag3", "tag4", "tag5")
			generateLogRecord("testapp1", "Message four", 3, "tag1", "tag2", "tag3")
			generateLogRecord("testapp2", "Message five", 3, "tag3", "tag4")

			query = fmt.Sprintf(
				"level=2&tags=tag3,tag4&start_time=%v&end_time=%v&page=1",
				time.Now().Format("2006-01-02"),
				time.Now().Format("2006-01-02"),
			)
		})

		JustBeforeEach(func() {
			Expect(
				sendRequest("GET", "/testapp1/get?"+query),
			).To(Succeed())
		})

		AssertSuccess()

		It("should respond with found log records", func() {
			parsedRes := LogRecords{}
			Expect(
				json.Unmarshal(response.Body.Bytes(), &parsedRes),
			).To(Succeed())

			Expect(parsedRes).To(HaveLen(2))

			Expect(parsedRes[0].Message).To(Equal("Message two"))
			Expect(parsedRes[0].Level).To(Equal(2))
			Expect(parsedRes[0].CreatedAt).To(BeAssignableToTypeOf(time.Now()))
			Expect(parsedRes[0].Tags).To(ConsistOf("tag2", "tag3", "tag4"))

			Expect(parsedRes[1].Message).To(Equal("Message three"))
			Expect(parsedRes[1].Level).To(Equal(3))
			Expect(parsedRes[1].CreatedAt).To(BeAssignableToTypeOf(time.Now()))
			Expect(parsedRes[1].Tags).To(ConsistOf("tag3", "tag4", "tag5"))
		})

		Context("without tags", func() {
			BeforeEach(func() {
				query = "level=1&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertSuccess()
		})

		Context("with level 5", func() {
			BeforeEach(func() {
				query = "level=5&tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertSuccess()
		})

		Context("with datetime as start_time and end_time", func() {
			BeforeEach(func() {
				query = "level=1&tags=tag3,tag4&start_time=2006-01-02T01:02:03&end_time=2006-01-02T01:02:03&page=1"
			})
			AssertSuccess()
		})

		Context("without page", func() {
			BeforeEach(func() {
				query = "level=1&tags=tag3,tag4&start_time=2006-01-02T01:02:03&end_time=2006-01-02T01:02:03"
			})
			AssertSuccess()
		})

		Context("without level", func() {
			BeforeEach(func() {
				query = "tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with level < 0", func() {
			BeforeEach(func() {
				query = "level=-1tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with level > 5", func() {
			BeforeEach(func() {
				query = "level=6tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with empty string tag", func() {
			BeforeEach(func() {
				query = "level=1tags=tag3,,tag4&start_time=2006-01-02&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with invalid start_time", func() {
			BeforeEach(func() {
				query = "level=1tags=tag3,tag4&start_time=2006-01-022&end_time=2006-01-02&page=1"
			})
			AssertUnprocessable()
		})

		Context("with invalid end_time", func() {
			BeforeEach(func() {
				query = "level=1tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-022&page=1"
			})
			AssertUnprocessable()
		})

		Context("with invalid page", func() {
			BeforeEach(func() {
				query = "level=1tags=tag3,tag4&start_time=2006-01-02&end_time=2006-01-02&page=a"
			})
			AssertUnprocessable()
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"html/template"
	"net/http"
	"time"

	// [START imports]
	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	// [END imports]
)

var (
	indexTemplate = template.Must(template.ParseFiles("index.html"))
)

// [START post_struct]
type Post struct {
	Author  string
	Message string
	Posted  time.Time
}

// [END post_struct]

type templateParams struct {
	Notice string

	Name string
	// [START added_templateParams_fields]
	Message string

	Posts []Post
	// [END added_templateParams_fields]

}

func main() {
	http.HandleFunc("/", indexHandler)
	appengine.Main()
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.Redirect(w, r, "/", http.StatusFound)
		return
	}
	// [START new_context]
	ctx := appengine.NewContext(r)
	// [END new_context]
	params := templateParams{}

	// [START new_query]
	q := datastore.NewQuery("Post").Order("-Posted").Limit(20)
	// [END new_query]
	// [START get_posts]
	if _, err := q.GetAll(ctx, &params.Posts); err != nil {
		log.Errorf(ctx, "Getting posts: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		params.Notice = "Couldn't get latest posts. Refresh?"
		indexTemplate.Execute(w, params)
		return
	}
	// [END get_posts]

	if r.Method == "GET" {
		indexTemplate.Execute(w, params)
		return
	}

	// It's a POST request, so handle the form submission.
	// [START new_post]
	post := Post{
		Author:  r.FormValue("name"),
		Message: r.FormValue("message"),
		Posted:  time.Now(),
	}
	// [END new_post]
	if post.Author == "" {
		post.Author = "Anonymous Gopher"
	}
	params.Name = post.Author

	if post.Message == "" {
		w.WriteHeader(http.StatusBadRequest)
		params.Notice = "No message provided"
		indexTemplate.Execute(w, params)
		return
	}
	// [START new_key]
	key := datastore.NewIncompleteKey(ctx, "Post", nil)
	// [END new_key]
	// [START add_post]
	if _, err := datastore.Put(ctx, key, &post); err != nil {
		log.Errorf(ctx, "datastore.Put: %v", err)

		w.WriteHeader(http.StatusInternalServerError)
		params.Notice = "Couldn't add new post. Try again?"
		params.Message = post.Message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// [END add_post]

	// Prepend the post that was just added.
	// [START prepend_post]
	params.Posts = append([]Post{post}, params.Posts...)
	// [END prepend_post]

	params.Notice = fmt.Sprintf("Thank you for your submission, %s!", post.Author)
	indexTemplate.Execute(w, params)
}
<commit_msg>appengine/gophers: change location of time package in list of imports (#411)<commit_after>// Copyright 2018 Google Inc. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"html/template"
	"net/http"

	"google.golang.org/appengine"

	// [START imports]
	"time"

	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	// [END imports]
)

var (
	indexTemplate = template.Must(template.ParseFiles("index.html"))
)

// [START post_struct]
type Post struct {
	Author  string
	Message string
	Posted  time.Time
}

// [END post_struct]

type templateParams struct {
	Notice string

	Name string
	// [START added_templateParams_fields]
	Message string

	Posts []Post
	// [END added_templateParams_fields]

}

func main() {
	http.HandleFunc("/", indexHandler)
	appengine.Main()
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.Redirect(w, r, "/", http.StatusFound)
		return
	}
	// [START new_context]
	ctx := appengine.NewContext(r)
	// [END new_context]
	params := templateParams{}

	// [START new_query]
	q := datastore.NewQuery("Post").Order("-Posted").Limit(20)
	// [END new_query]
	// [START get_posts]
	if _, err := q.GetAll(ctx, &params.Posts); err != nil {
		log.Errorf(ctx, "Getting posts: %v", err)
		w.WriteHeader(http.StatusInternalServerError)
		params.Notice = "Couldn't get latest posts. Refresh?"
		indexTemplate.Execute(w, params)
		return
	}
	// [END get_posts]

	if r.Method == "GET" {
		indexTemplate.Execute(w, params)
		return
	}

	// It's a POST request, so handle the form submission.
	// [START new_post]
	post := Post{
		Author:  r.FormValue("name"),
		Message: r.FormValue("message"),
		Posted:  time.Now(),
	}
	// [END new_post]
	if post.Author == "" {
		post.Author = "Anonymous Gopher"
	}
	params.Name = post.Author

	if post.Message == "" {
		w.WriteHeader(http.StatusBadRequest)
		params.Notice = "No message provided"
		indexTemplate.Execute(w, params)
		return
	}
	// [START new_key]
	key := datastore.NewIncompleteKey(ctx, "Post", nil)
	// [END new_key]
	// [START add_post]
	if _, err := datastore.Put(ctx, key, &post); err != nil {
		log.Errorf(ctx, "datastore.Put: %v", err)

		w.WriteHeader(http.StatusInternalServerError)
		params.Notice = "Couldn't add new post. Try again?"
		params.Message = post.Message // Preserve their message so they can try again.
		indexTemplate.Execute(w, params)
		return
	}
	// [END add_post]

	// Prepend the post that was just added.
	// [START prepend_post]
	params.Posts = append([]Post{post}, params.Posts...)
	// [END prepend_post]

	params.Notice = fmt.Sprintf("Thank you for your submission, %s!", post.Author)
	indexTemplate.Execute(w, params)
}
<|endoftext|>


<commit_before>//
// policy.go
//
package srnd

import (
  "log"
  "regexp"
)

type FeedPolicy struct {
  rules map[string]string
}

// do we allow this newsgroup?
func (self *FeedPolicy) AllowsNewsgroup(newsgroup string) bool {
  var k string
  for k = range self.rules {

    inverse := k[0] == '!'
    if inverse {
      k = k[1:]
    }
    match, err := regexp.MatchString(k, newsgroup)
    if err != nil {
      log.Fatal(err)
    }
    if match {
      if inverse {
        return false
      }
      return true
    }
  }
  return false
}

<commit_msg>try fixing policy<commit_after>//
// policy.go
//
package srnd

import (
  "log"
  "regexp"
)

type FeedPolicy struct {
  rules map[string]string
}

// do we allow this newsgroup?
func (self *FeedPolicy) AllowsNewsgroup(newsgroup string) bool {
  var k, v string
  for k, v = range self.rules {
    match, err := regexp.MatchString(k, newsgroup)
    if err != nil {
      log.Fatal(err)
    }
    if match {
      return v == "1"
    }
  }
  return false
}

<|endoftext|>


<commit_before>package testfixtures

import (
	"database/sql/driver"
	"fmt"
)

type CustomType []byte

func CustomTypeFromString(s string) CustomType {
	return CustomType(s)
}

func (u CustomType) Value() (driver.Value, error) {
	return []byte(u), nil
}

func (u *CustomType) Scan(value interface{}) error {
	switch value := value.(type) {
	case nil:
		u = nil
	case []byte:
		*u = make(CustomType, len(value))
		copy(*u, value)
	case string:
		*u = CustomType(value)
	default:
		return fmt.Errorf("cannot convert %v (of type %T) to %T", value, value, u)
	}
	return nil
}
<commit_msg>go testfixtures: test non-native SQL type in sqlgen/livesql<commit_after>package testfixtures

import (
	"database/sql/driver"
	"fmt"
)

type CustomType [16]byte

func CustomTypeFromString(s string) CustomType {
	b := []byte(s)
	c := CustomType{}
	copy(c[:], b)
	return c
}

func (u CustomType) Value() (driver.Value, error) {
	return []byte(u[:]), nil
}

func (u *CustomType) Scan(value interface{}) error {
	switch value := value.(type) {
	case nil:
		u = nil
	case string:
		b := []byte(value)
		copy(u[:], b)
	case []byte:
		copy(u[:], value)
	default:
		return fmt.Errorf("cannot convert %v (of type %T) to %T", value, value, u)
	}
	return nil
}
<|endoftext|>


<commit_before>package ssdb

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/seefan/goerr"
	"net"
	"strconv"
	log "github.com/thinkboy/log4go"
)

type SSDBClient struct {
	isOpen    bool
	Password  string
	addr      string
	sock      *net.TCPConn
	buf       *bufio.ReadWriter
	packetBuf bytes.Buffer
}

var (
	byt              = []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
	maxByteSize byte = 58
)

func ToNum(bs []byte) int {
	re := 0
	for _, v := range bs {
		if v >= maxByteSize {
			return 0
		}
		re = re*10 + byt[v]
	}
	return re
}

func Connect(addr string) (*SSDBClient, error) {
	l := &SSDBClient{addr: addr}
	l.Start()
	return l, nil

}

//打开连接
func (s *SSDBClient) Start() error {
	addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s", s.addr))
	if err != nil {
		log.Error("Can't connect to (\"%s\")", addr)
		return err
	}
	sock, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return err
	}
	sock.SetReadBuffer(8192)
	//sock.SetWriteBuffer(1024)
	s.buf = bufio.NewReadWriter(bufio.NewReader(sock), bufio.NewWriter(sock))
	s.sock = sock

	s.isOpen = true
	return nil
}
func (s *SSDBClient) Close() error {
	s.isOpen = false
	s.buf = nil
	s.packetBuf.Reset()
	return s.sock.Close()
}
func (s *SSDBClient) IsOpen() bool {
	return s.isOpen
}
func (s *SSDBClient) Ping() bool {
	_, err := s.Do("info")
	return err == nil
}
func (s *SSDBClient) do(args ...interface{}) ([]string, error) {
	err := s.send(args)
	if err != nil {
		return nil, err
	}
	resp, err := s.Recv()
	return resp, err
}

//通用调用方法，如果有需要在所有方法前执行的，可以在这里执行
func (s *SSDBClient) Do(args ...interface{}) ([]string, error) {
	if s.Password != "" {
		resp, err := s.do("auth", []string{s.Password})
		if err != nil {
			s.sock.Close()
			s.isOpen = false
			return nil, goerr.NewError(err, "authentication failed")
		}
		if len(resp) > 0 && resp[0] == "ok" {
			//验证成功
			s.Password = ""
		} else {
			return nil, goerr.New("Authentication failed,password is wrong")
		}
	}
	resp, err := s.do(args...)
	if err != nil {
		s.sock.Close()
		s.isOpen = false
	}
	return resp, err
}

func (s *SSDBClient) Send(args ...interface{}) error {
	return s.send(args)
}

func (s *SSDBClient) send(args []interface{}) error {

	for _, arg := range args {
		switch arg := arg.(type) {
		case string:
			s.buf.Write(strconv.AppendInt(nil, int64(len(arg)), 10))
			s.buf.WriteByte('\n')
			s.buf.WriteString(arg)
		case []byte:
			s.buf.Write(strconv.AppendInt(nil, int64(len(arg)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(arg)
		case int:
			bs := strconv.AppendInt(nil, int64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case int8:
			bs := strconv.AppendInt(nil, int64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case int16:
			bs := strconv.AppendInt(nil, int64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case int32:
			bs := strconv.AppendInt(nil, int64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case int64:
			bs := strconv.AppendInt(nil, arg, 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case uint8:
			bs := strconv.AppendUint(nil, uint64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case uint16:
			bs := strconv.AppendUint(nil, uint64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case uint32:
			bs := strconv.AppendUint(nil, uint64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case uint64:
			bs := strconv.AppendUint(nil, uint64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case float32:
			bs := strconv.AppendFloat(nil, float64(arg), 'g', -1, 32)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case float64:
			bs := strconv.AppendFloat(nil, arg, 'g', -1, 64)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case bool:
			s.buf.WriteByte(1)
			s.buf.WriteByte('\n')
			if arg {
				s.buf.WriteByte(1)
			} else {
				s.buf.WriteByte(0)
			}
		case nil:
			s.buf.WriteByte(0)
			s.buf.WriteByte('\n')
			s.buf.WriteString("")
		default:
			fmt.Errorf("bad arguments type,can not json marshal")
		}
		s.buf.WriteByte('\n')
	}
	s.buf.WriteByte('\n')
	return s.buf.Flush()
}

func (s *SSDBClient) Recv() (resp []string, err error) {
	packetSize := -1
	drop := 1
	bufSize := 0
	s.packetBuf.Reset()
	for {
		buf, err := s.buf.ReadBytes('\n')
		if err != nil {
			return nil, err
		}
		bufSize = len(buf)
		if packetSize == -1 && (bufSize == 1 || bufSize == 2 && buf[0] == '\r') { //空行，说明一个数据包结束
			return resp, nil
		}
		if bufSize > 2 && buf[bufSize-2] == '\r' { // drop end
			drop = 2
		} else {
			drop = 1
		}
		if packetSize == -1 {
			packetSize = ToNum(buf[:(bufSize - drop)])
		} else {
			if s.packetBuf.Len()+bufSize == packetSize+drop {
				s.packetBuf.Write(buf[:bufSize-drop])
				resp = append(resp, s.packetBuf.String())
				s.packetBuf.Reset()
				packetSize = -1
			} else {
				s.packetBuf.Write(buf)
			}
		}
	}
	return resp, nil
}

func (s *SSDBClient) RecvBinlog() (resp *Binlog, err error) {
	packetSize := -1
	drop := 1
	bufSize := 0
	s.packetBuf.Reset()
	var body [][]byte
	for {
		//fmt.Printf("RecvBinlog\n")
		buf, err := s.buf.ReadBytes('\n')
		if err != nil {
			return nil, err
		}
		bufSize = len(buf)
		//fmt.Printf("RecvBinlog:packetSize(%d),packetSize(%d),packetSize(%d)\n", packetSize, bufSize, buf[0])
		if packetSize == -1 && (bufSize == 1 || (bufSize == 2 && buf[0] == '\r')) { //空行，说明一个数据包结束
			resp,err = LoadBinlog(body)
			return resp, nil
		}
		if bufSize > 2 && buf[bufSize-2] == '\r' { // drop end
			drop = 2	//end with \n\n
		} else {
			drop = 1	//end with \n
		}
		if packetSize == -1 {
			packetSize = ToNum(buf[:(bufSize - drop)])
		} else {
			if s.packetBuf.Len()+bufSize == packetSize+drop {
				s.packetBuf.Write(buf[:bufSize-drop])
				packet := make([]byte, s.packetBuf.Len())
				copy(packet, s.packetBuf.Bytes())
				body = append(body, packet)
				s.packetBuf.Reset()
				packetSize = -1
			} else {
				s.packetBuf.Write(buf)
			}
		}
	}
	return
}<commit_msg>fix ssdb protocol parser bug. [more detail](https://github.com/seefan/gossdb/pull/35)<commit_after>package ssdb

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/seefan/goerr"
	"net"
	"strconv"
	log "github.com/thinkboy/log4go"
)

type SSDBClient struct {
	isOpen    bool
	Password  string
	addr      string
	sock      *net.TCPConn
	buf       *bufio.ReadWriter
	packetBuf bytes.Buffer
}

var (
	byt              = []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
	maxByteSize byte = 58
)

func ToNum(bs []byte) int {
	re := 0
	for _, v := range bs {
		if v >= maxByteSize {
			return 0
		}
		re = re*10 + byt[v]
	}
	return re
}

func Connect(addr string) (*SSDBClient, error) {
	l := &SSDBClient{addr: addr}
	l.Start()
	return l, nil

}

//打开连接
func (s *SSDBClient) Start() error {
	addr, err := net.ResolveTCPAddr("tcp", fmt.Sprintf("%s", s.addr))
	if err != nil {
		log.Error("Can't connect to (\"%s\")", addr)
		return err
	}
	sock, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		return err
	}
	sock.SetReadBuffer(8192)
	//sock.SetWriteBuffer(1024)
	s.buf = bufio.NewReadWriter(bufio.NewReader(sock), bufio.NewWriter(sock))
	s.sock = sock

	s.isOpen = true
	return nil
}
func (s *SSDBClient) Close() error {
	s.isOpen = false
	s.buf = nil
	s.packetBuf.Reset()
	return s.sock.Close()
}
func (s *SSDBClient) IsOpen() bool {
	return s.isOpen
}
func (s *SSDBClient) Ping() bool {
	_, err := s.Do("info")
	return err == nil
}
func (s *SSDBClient) do(args ...interface{}) ([]string, error) {
	err := s.send(args)
	if err != nil {
		return nil, err
	}
	resp, err := s.Recv()
	return resp, err
}

//通用调用方法，如果有需要在所有方法前执行的，可以在这里执行
func (s *SSDBClient) Do(args ...interface{}) ([]string, error) {
	if s.Password != "" {
		resp, err := s.do("auth", []string{s.Password})
		if err != nil {
			s.sock.Close()
			s.isOpen = false
			return nil, goerr.NewError(err, "authentication failed")
		}
		if len(resp) > 0 && resp[0] == "ok" {
			//验证成功
			s.Password = ""
		} else {
			return nil, goerr.New("Authentication failed,password is wrong")
		}
	}
	resp, err := s.do(args...)
	if err != nil {
		s.sock.Close()
		s.isOpen = false
	}
	return resp, err
}

func (s *SSDBClient) Send(args ...interface{}) error {
	return s.send(args)
}

func (s *SSDBClient) send(args []interface{}) error {

	for _, arg := range args {
		switch arg := arg.(type) {
		case string:
			s.buf.Write(strconv.AppendInt(nil, int64(len(arg)), 10))
			s.buf.WriteByte('\n')
			s.buf.WriteString(arg)
		case []byte:
			s.buf.Write(strconv.AppendInt(nil, int64(len(arg)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(arg)
		case int:
			bs := strconv.AppendInt(nil, int64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case int8:
			bs := strconv.AppendInt(nil, int64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case int16:
			bs := strconv.AppendInt(nil, int64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case int32:
			bs := strconv.AppendInt(nil, int64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case int64:
			bs := strconv.AppendInt(nil, arg, 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case uint8:
			bs := strconv.AppendUint(nil, uint64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case uint16:
			bs := strconv.AppendUint(nil, uint64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case uint32:
			bs := strconv.AppendUint(nil, uint64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case uint64:
			bs := strconv.AppendUint(nil, uint64(arg), 10)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case float32:
			bs := strconv.AppendFloat(nil, float64(arg), 'g', -1, 32)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case float64:
			bs := strconv.AppendFloat(nil, arg, 'g', -1, 64)
			s.buf.Write(strconv.AppendInt(nil, int64(len(bs)), 10))
			s.buf.WriteByte('\n')
			s.buf.Write(bs)
		case bool:
			s.buf.WriteByte(1)
			s.buf.WriteByte('\n')
			if arg {
				s.buf.WriteByte(1)
			} else {
				s.buf.WriteByte(0)
			}
		case nil:
			s.buf.WriteByte(0)
			s.buf.WriteByte('\n')
			s.buf.WriteString("")
		default:
			fmt.Errorf("bad arguments type,can not json marshal")
		}
		s.buf.WriteByte('\n')
	}
	s.buf.WriteByte('\n')
	return s.buf.Flush()
}

func (s *SSDBClient) Recv() (resp []string, err error) {
	packetSize := -1
	drop := 1
	bufSize := 0
	s.packetBuf.Reset()
	for {
		buf, err := s.buf.ReadBytes('\n')
		if err != nil {
			return nil, err
		}
		bufSize = len(buf)
		if packetSize == -1 && (bufSize == 1 || bufSize == 2 && buf[0] == '\r') { //空行，说明一个数据包结束
			return resp, nil
		}
		if bufSize > 2 && buf[bufSize-2] == '\r' { // drop end
			drop = 2
		} else {
			drop = 1
		}
		if packetSize == -1 {
			packetSize = ToNum(buf[:(bufSize - drop)])
		} else {
			//data endwith '\n' or '\r\n'
			drop = 0
			if s.packetBuf.Len()+bufSize == packetSize+1 { //
				drop = 1
			}
			if s.packetBuf.Len()+bufSize == packetSize+2 {
				drop = 2
			}
			if drop > 0 {
				s.packetBuf.Write(buf[:bufSize-drop])
				resp = append(resp, s.packetBuf.String())
				s.packetBuf.Reset()
				packetSize = -1
			} else {
				s.packetBuf.Write(buf)
			}
		}
	}
	return resp, nil
}

func (s *SSDBClient) RecvBinlog() (resp *Binlog, err error) {
	packetSize := -1
	drop := 1
	bufSize := 0
	s.packetBuf.Reset()
	var body [][]byte
	for {
		buf, err := s.buf.ReadBytes('\n')
		if err != nil {
			return nil, err
		}
		bufSize = len(buf)
		//fmt.Printf("RecvBinlog:packetSize(%d),packetSize(%d),packetSize(%d)\n", packetSize, bufSize, buf[0])
		if packetSize == -1 && (bufSize == 1 || (bufSize == 2 && buf[0] == '\r')) { //空行，说明一个数据包结束
			resp,err = LoadBinlog(body)
			return resp, nil
		}

		if bufSize > 2 && buf[bufSize-2] == '\r' { // drop end
			drop = 2	//end with \n\n
		} else {
			drop = 1	//end with \n
		}
		if packetSize == -1 {
			packetSize = ToNum(buf[:(bufSize - drop)])
		} else {
			//data endwith '\n' or '\r\n'
			drop = 0
			if s.packetBuf.Len()+bufSize == packetSize+1 { //
				drop = 1
			}
			if s.packetBuf.Len()+bufSize == packetSize+2 {
				drop = 2
			}
			if drop > 0 {
				s.packetBuf.Write(buf[:bufSize-drop])
				packet := make([]byte, s.packetBuf.Len())
				copy(packet, s.packetBuf.Bytes())
				body = append(body, packet)
				s.packetBuf.Reset()
				packetSize = -1
			}else {
				s.packetBuf.Write(buf)
			}
		}
	}
	return
}<|endoftext|>


<commit_before>// +build windows

package memory_map

import (
	"os"
	"reflect"
	"syscall"
	"unsafe"

	"golang.org/x/sys/windows"
)

type MemoryBuffer struct {
	aligned_length uint64
	length         uint64
	aligned_ptr    uintptr
	ptr            uintptr
	Buffer         []byte
}

type MemoryMap struct {
	File                   *os.File
	file_memory_map_handle uintptr
	write_map_views        []MemoryBuffer
	max_length             uint64
	End_of_file            int64
}

var FileMemoryMap = make(map[string]*MemoryMap)

type DWORD = uint32
type WORD = uint16

var (
	modkernel32 = syscall.NewLazyDLL("kernel32.dll")

	procGetSystemInfo            = modkernel32.NewProc("GetSystemInfo")
	procGetProcessWorkingSetSize = modkernel32.NewProc("GetProcessWorkingSetSize")
	procSetProcessWorkingSetSize = modkernel32.NewProc("SetProcessWorkingSetSize")
)

var currentProcess, _ = windows.GetCurrentProcess()
var currentMinWorkingSet uint64 = 0
var currentMaxWorkingSet uint64 = 0
var _ = getProcessWorkingSetSize(uintptr(currentProcess), &currentMinWorkingSet, &currentMaxWorkingSet)

var systemInfo, _ = getSystemInfo()
var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 256

func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) {

	chunks := (maxLength / chunkSize)
	if chunks*chunkSize < maxLength {
		chunks = chunks + 1
	}

	alignedMaxLength := chunks * chunkSize

	maxLength_high := uint32(alignedMaxLength >> 32)
	maxLength_low := uint32(alignedMaxLength & 0xFFFFFFFF)
	file_memory_map_handle, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, maxLength_high, maxLength_low, nil)

	if err == nil {
		mMap.File = file
		mMap.file_memory_map_handle = uintptr(file_memory_map_handle)
		mMap.write_map_views = make([]MemoryBuffer, 0, alignedMaxLength/chunkSize)
		mMap.max_length = alignedMaxLength
		mMap.End_of_file = -1
	}
}

func (mMap *MemoryMap) DeleteFileAndMemoryMap() {
	//First we close the file handles first to delete the file,
	//Then we unmap the memory to ensure the unmapping process doesn't write the data to disk
	windows.CloseHandle(windows.Handle(mMap.file_memory_map_handle))
	windows.CloseHandle(windows.Handle(mMap.File.Fd()))

	for _, view := range mMap.write_map_views {
		view.ReleaseMemory()
	}

	mMap.write_map_views = nil
	mMap.max_length = 0
}

func min(x, y uint64) uint64 {
	if x < y {
		return x
	}
	return y
}

func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {

	for {
		if ((offset+length)/chunkSize)+1 > uint64(len(mMap.write_map_views)) {
			allocateChunk(mMap)
		} else {
			break
		}
	}

	remaining_length := length
	sliceIndex := offset / chunkSize
	sliceOffset := offset - (sliceIndex * chunkSize)
	dataOffset := uint64(0)

	for {
		writeEnd := min((remaining_length + sliceOffset), chunkSize)
		copy(mMap.write_map_views[sliceIndex].Buffer[sliceOffset:writeEnd], data[dataOffset:])
		remaining_length -= (writeEnd - sliceOffset)
		dataOffset += (writeEnd - sliceOffset)

		if remaining_length > 0 {
			sliceIndex += 1
			sliceOffset = 0
		} else {
			break
		}
	}

	if mMap.End_of_file < int64(offset+length-1) {
		mMap.End_of_file = int64(offset + length - 1)
	}
}

func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) {
	return allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false)
}

func (mBuffer *MemoryBuffer) ReleaseMemory() {

	currentMinWorkingSet = currentMinWorkingSet - mBuffer.aligned_length
	currentMaxWorkingSet = currentMaxWorkingSet - mBuffer.aligned_length

	windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length))
	windows.UnmapViewOfFile(mBuffer.aligned_ptr)

	var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet)

	mBuffer.ptr = 0
	mBuffer.aligned_ptr = 0
	mBuffer.length = 0
	mBuffer.aligned_length = 0
	mBuffer.Buffer = nil
}

func allocateChunk(mMap *MemoryMap) {
	start := uint64(len(mMap.write_map_views)) * chunkSize
	mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), start, chunkSize, true)

	if err == nil {
		mMap.write_map_views = append(mMap.write_map_views, mBuffer)
		windows.VirtualLock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length))
	}
}

func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) (MemoryBuffer, error) {

	mBuffer := MemoryBuffer{}

	//align memory allocations to the minium virtal memory allocation size
	dwSysGran := systemInfo.dwAllocationGranularity

	start := (offset / uint64(dwSysGran)) * uint64(dwSysGran)
	diff := offset - start
	aligned_length := diff + length

	offset_high := uint32(start >> 32)
	offset_low := uint32(start & 0xFFFFFFFF)

	access := windows.FILE_MAP_READ

	if write {
		access = windows.FILE_MAP_WRITE
	}

	currentMinWorkingSet = currentMinWorkingSet + aligned_length
	currentMaxWorkingSet = currentMaxWorkingSet + aligned_length

	// increase the process working set size to hint to windows memory manager to
	// prioritise keeping this memory mapped in physical memory over other standby memory
	var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet)

	addr_ptr, errno := windows.MapViewOfFile(hMapFile,
		uint32(access), // read/write permission
		offset_high,
		offset_low,
		uintptr(aligned_length))

	if addr_ptr == 0 {
		return mBuffer, errno
	}

	mBuffer.aligned_ptr = addr_ptr
	mBuffer.aligned_length = aligned_length
	mBuffer.ptr = addr_ptr + uintptr(diff)
	mBuffer.length = length

	slice_header := (*reflect.SliceHeader)(unsafe.Pointer(&mBuffer.Buffer))
	slice_header.Data = addr_ptr + uintptr(diff)
	slice_header.Len = int(length)
	slice_header.Cap = int(length)

	return mBuffer, nil
}

// typedef struct _SYSTEM_INFO {
//   union {
//     DWORD  dwOemId;
//     struct {
//       WORD wProcessorArchitecture;
//       WORD wReserved;
//     };
//   };
//   DWORD     dwPageSize;
//   LPVOID    lpMinimumApplicationAddress;
//   LPVOID    lpMaximumApplicationAddress;
//   DWORD_PTR dwActiveProcessorMask;
//   DWORD     dwNumberOfProcessors;
//   DWORD     dwProcessorType;
//   DWORD     dwAllocationGranularity;
//   WORD      wProcessorLevel;
//   WORD      wProcessorRevision;
// } SYSTEM_INFO;
// https://msdn.microsoft.com/en-us/library/ms724958(v=vs.85).aspx
type _SYSTEM_INFO struct {
	dwOemId                     DWORD
	dwPageSize                  DWORD
	lpMinimumApplicationAddress uintptr
	lpMaximumApplicationAddress uintptr
	dwActiveProcessorMask       uintptr
	dwNumberOfProcessors        DWORD
	dwProcessorType             DWORD
	dwAllocationGranularity     DWORD
	wProcessorLevel             WORD
	wProcessorRevision          WORD
}

// void WINAPI GetSystemInfo(
//   _Out_ LPSYSTEM_INFO lpSystemInfo
// );
// https://msdn.microsoft.com/en-us/library/ms724381(VS.85).aspx
func getSystemInfo() (_SYSTEM_INFO, error) {
	var si _SYSTEM_INFO
	_, _, err := procGetSystemInfo.Call(
		uintptr(unsafe.Pointer(&si)),
	)
	if err != syscall.Errno(0) {
		return si, err
	}
	return si, nil
}

// BOOL GetProcessWorkingSetSize(
//   HANDLE  hProcess,
//   PSIZE_T lpMinimumWorkingSetSize,
//   PSIZE_T lpMaximumWorkingSetSize
// );
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getprocessworkingsetsize

func getProcessWorkingSetSize(process uintptr, dwMinWorkingSet *uint64, dwMaxWorkingSet *uint64) error {
	r1, _, err := syscall.Syscall(procGetProcessWorkingSetSize.Addr(), 3, process, uintptr(unsafe.Pointer(dwMinWorkingSet)), uintptr(unsafe.Pointer(dwMaxWorkingSet)))
	if r1 == 0 {
		if err != syscall.Errno(0) {
			return err
		}
	}
	return nil
}

// BOOL SetProcessWorkingSetSize(
//   HANDLE hProcess,
//   SIZE_T dwMinimumWorkingSetSize,
//   SIZE_T dwMaximumWorkingSetSize
// );
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-setprocessworkingsetsize

func setProcessWorkingSetSize(process uintptr, dwMinWorkingSet uint64, dwMaxWorkingSet uint64) error {
	r1, _, err := syscall.Syscall(procSetProcessWorkingSetSize.Addr(), 3, process, uintptr(dwMinWorkingSet), uintptr(dwMaxWorkingSet))
	if r1 == 0 {
		if err != syscall.Errno(0) {
			return err
		}
	}
	return nil
}
<commit_msg>limit locking physical memory to 80% of max physical memory<commit_after>// +build windows

package memory_map

import (
	"os"
	"reflect"
	"syscall"
	"unsafe"

	"golang.org/x/sys/windows"
)

type MemoryBuffer struct {
	aligned_length uint64
	length         uint64
	aligned_ptr    uintptr
	ptr            uintptr
	Buffer         []byte
}

type MemoryMap struct {
	File                   *os.File
	file_memory_map_handle uintptr
	write_map_views        []MemoryBuffer
	max_length             uint64
	End_of_file            int64
}

var FileMemoryMap = make(map[string]*MemoryMap)

type DWORDLONG = uint64
type DWORD = uint32
type WORD = uint16

var (
	modkernel32 = syscall.NewLazyDLL("kernel32.dll")

	procGetSystemInfo            = modkernel32.NewProc("GetSystemInfo")
	procGlobalMemoryStatusEx     = modkernel32.NewProc("GlobalMemoryStatusEx")
	procGetProcessWorkingSetSize = modkernel32.NewProc("GetProcessWorkingSetSize")
	procSetProcessWorkingSetSize = modkernel32.NewProc("SetProcessWorkingSetSize")
)

var currentProcess, _ = windows.GetCurrentProcess()
var currentMinWorkingSet uint64 = 0
var currentMaxWorkingSet uint64 = 0
var _ = getProcessWorkingSetSize(uintptr(currentProcess), &currentMinWorkingSet, &currentMaxWorkingSet)

var systemInfo, _ = getSystemInfo()
var chunkSize = uint64(systemInfo.dwAllocationGranularity) * 256

var memoryStatusEx, _ = globalMemoryStatusEx()
var maxMemoryLimitBytes = uint64(float64(memoryStatusEx.ullTotalPhys) * 0.8)

func (mMap *MemoryMap) CreateMemoryMap(file *os.File, maxLength uint64) {

	chunks := (maxLength / chunkSize)
	if chunks*chunkSize < maxLength {
		chunks = chunks + 1
	}

	alignedMaxLength := chunks * chunkSize

	maxLength_high := uint32(alignedMaxLength >> 32)
	maxLength_low := uint32(alignedMaxLength & 0xFFFFFFFF)
	file_memory_map_handle, err := windows.CreateFileMapping(windows.Handle(file.Fd()), nil, windows.PAGE_READWRITE, maxLength_high, maxLength_low, nil)

	if err == nil {
		mMap.File = file
		mMap.file_memory_map_handle = uintptr(file_memory_map_handle)
		mMap.write_map_views = make([]MemoryBuffer, 0, alignedMaxLength/chunkSize)
		mMap.max_length = alignedMaxLength
		mMap.End_of_file = -1
	}
}

func (mMap *MemoryMap) DeleteFileAndMemoryMap() {
	//First we close the file handles first to delete the file,
	//Then we unmap the memory to ensure the unmapping process doesn't write the data to disk
	windows.CloseHandle(windows.Handle(mMap.file_memory_map_handle))
	windows.CloseHandle(windows.Handle(mMap.File.Fd()))

	for _, view := range mMap.write_map_views {
		view.ReleaseMemory()
	}

	mMap.write_map_views = nil
	mMap.max_length = 0
}

func min(x, y uint64) uint64 {
	if x < y {
		return x
	}
	return y
}

func (mMap *MemoryMap) WriteMemory(offset uint64, length uint64, data []byte) {

	for {
		if ((offset+length)/chunkSize)+1 > uint64(len(mMap.write_map_views)) {
			allocateChunk(mMap)
		} else {
			break
		}
	}

	remaining_length := length
	sliceIndex := offset / chunkSize
	sliceOffset := offset - (sliceIndex * chunkSize)
	dataOffset := uint64(0)

	for {
		writeEnd := min((remaining_length + sliceOffset), chunkSize)
		copy(mMap.write_map_views[sliceIndex].Buffer[sliceOffset:writeEnd], data[dataOffset:])
		remaining_length -= (writeEnd - sliceOffset)
		dataOffset += (writeEnd - sliceOffset)

		if remaining_length > 0 {
			sliceIndex += 1
			sliceOffset = 0
		} else {
			break
		}
	}

	if mMap.End_of_file < int64(offset+length-1) {
		mMap.End_of_file = int64(offset + length - 1)
	}
}

func (mMap *MemoryMap) ReadMemory(offset uint64, length uint64) (MemoryBuffer, error) {
	return allocate(windows.Handle(mMap.file_memory_map_handle), offset, length, false)
}

func (mBuffer *MemoryBuffer) ReleaseMemory() {

	windows.VirtualUnlock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length))
	windows.UnmapViewOfFile(mBuffer.aligned_ptr)

	currentMinWorkingSet = currentMinWorkingSet - mBuffer.aligned_length
	currentMaxWorkingSet = currentMaxWorkingSet - mBuffer.aligned_length

	if currentMinWorkingSet < maxMemoryLimitBytes {
		var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet)
	}

	mBuffer.ptr = 0
	mBuffer.aligned_ptr = 0
	mBuffer.length = 0
	mBuffer.aligned_length = 0
	mBuffer.Buffer = nil
}

func allocateChunk(mMap *MemoryMap) {
	start := uint64(len(mMap.write_map_views)) * chunkSize
	mBuffer, err := allocate(windows.Handle(mMap.file_memory_map_handle), start, chunkSize, true)

	if err == nil {
		mMap.write_map_views = append(mMap.write_map_views, mBuffer)
	}
}

func allocate(hMapFile windows.Handle, offset uint64, length uint64, write bool) (MemoryBuffer, error) {

	mBuffer := MemoryBuffer{}

	//align memory allocations to the minium virtal memory allocation size
	dwSysGran := systemInfo.dwAllocationGranularity

	start := (offset / uint64(dwSysGran)) * uint64(dwSysGran)
	diff := offset - start
	aligned_length := diff + length

	offset_high := uint32(start >> 32)
	offset_low := uint32(start & 0xFFFFFFFF)

	access := windows.FILE_MAP_READ

	if write {
		access = windows.FILE_MAP_WRITE
	}

	currentMinWorkingSet = currentMinWorkingSet + aligned_length
	currentMaxWorkingSet = currentMaxWorkingSet + aligned_length

	if currentMinWorkingSet < maxMemoryLimitBytes {
		// increase the process working set size to hint to windows memory manager to
		// prioritise keeping this memory mapped in physical memory over other standby memory
		var _ = setProcessWorkingSetSize(uintptr(currentProcess), currentMinWorkingSet, currentMaxWorkingSet)
	}

	addr_ptr, errno := windows.MapViewOfFile(hMapFile,
		uint32(access), // read/write permission
		offset_high,
		offset_low,
		uintptr(aligned_length))

	if addr_ptr == 0 {
		return mBuffer, errno
	}

	if currentMinWorkingSet < maxMemoryLimitBytes {
		windows.VirtualLock(mBuffer.aligned_ptr, uintptr(mBuffer.aligned_length))
	}

	mBuffer.aligned_ptr = addr_ptr
	mBuffer.aligned_length = aligned_length
	mBuffer.ptr = addr_ptr + uintptr(diff)
	mBuffer.length = length

	slice_header := (*reflect.SliceHeader)(unsafe.Pointer(&mBuffer.Buffer))
	slice_header.Data = addr_ptr + uintptr(diff)
	slice_header.Len = int(length)
	slice_header.Cap = int(length)

	return mBuffer, nil
}

//typedef struct _MEMORYSTATUSEX {
//	DWORD     dwLength;
//	DWORD     dwMemoryLoad;
//	DWORDLONG ullTotalPhys;
//	DWORDLONG ullAvailPhys;
//	DWORDLONG ullTotalPageFile;
//	DWORDLONG ullAvailPageFile;
//	DWORDLONG ullTotalVirtual;
//	DWORDLONG ullAvailVirtual;
//	DWORDLONG ullAvailExtendedVirtual;
//  } MEMORYSTATUSEX, *LPMEMORYSTATUSEX;
//https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex

type _MEMORYSTATUSEX struct {
	dwLength                DWORD
	dwMemoryLoad            DWORD
	ullTotalPhys            DWORDLONG
	ullAvailPhys            DWORDLONG
	ullTotalPageFile        DWORDLONG
	ullAvailPageFile        DWORDLONG
	ullTotalVirtual         DWORDLONG
	ullAvailVirtual         DWORDLONG
	ullAvailExtendedVirtual DWORDLONG
}

// BOOL GlobalMemoryStatusEx(
//  LPMEMORYSTATUSEX lpBuffer
// );
// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex
func globalMemoryStatusEx() (_MEMORYSTATUSEX, error) {
	var mem_status _MEMORYSTATUSEX

	mem_status.dwLength = uint32(unsafe.Sizeof(mem_status))
	_, _, err := procGlobalMemoryStatusEx.Call(uintptr(unsafe.Pointer(&mem_status)))

	if err != syscall.Errno(0) {
		return mem_status, err
	}
	return mem_status, nil
}

// typedef struct _SYSTEM_INFO {
//   union {
//     DWORD  dwOemId;
//     struct {
//       WORD wProcessorArchitecture;
//       WORD wReserved;
//     };
//   };
//   DWORD     dwPageSize;
//   LPVOID    lpMinimumApplicationAddress;
//   LPVOID    lpMaximumApplicationAddress;
//   DWORD_PTR dwActiveProcessorMask;
//   DWORD     dwNumberOfProcessors;
//   DWORD     dwProcessorType;
//   DWORD     dwAllocationGranularity;
//   WORD      wProcessorLevel;
//   WORD      wProcessorRevision;
// } SYSTEM_INFO;
// https://docs.microsoft.com/en-gb/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info
type _SYSTEM_INFO struct {
	dwOemId                     DWORD
	dwPageSize                  DWORD
	lpMinimumApplicationAddress uintptr
	lpMaximumApplicationAddress uintptr
	dwActiveProcessorMask       uintptr
	dwNumberOfProcessors        DWORD
	dwProcessorType             DWORD
	dwAllocationGranularity     DWORD
	wProcessorLevel             WORD
	wProcessorRevision          WORD
}

// void WINAPI GetSystemInfo(
//   _Out_ LPSYSTEM_INFO lpSystemInfo
// );
// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getsysteminfo
func getSystemInfo() (_SYSTEM_INFO, error) {
	var si _SYSTEM_INFO
	_, _, err := procGetSystemInfo.Call(uintptr(unsafe.Pointer(&si)))
	if err != syscall.Errno(0) {
		return si, err
	}
	return si, nil
}

// BOOL GetProcessWorkingSetSize(
//   HANDLE  hProcess,
//   PSIZE_T lpMinimumWorkingSetSize,
//   PSIZE_T lpMaximumWorkingSetSize
// );
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-getprocessworkingsetsize

func getProcessWorkingSetSize(process uintptr, dwMinWorkingSet *uint64, dwMaxWorkingSet *uint64) error {
	r1, _, err := syscall.Syscall(procGetProcessWorkingSetSize.Addr(), 3, process, uintptr(unsafe.Pointer(dwMinWorkingSet)), uintptr(unsafe.Pointer(dwMaxWorkingSet)))
	if r1 == 0 {
		if err != syscall.Errno(0) {
			return err
		}
	}
	return nil
}

// BOOL SetProcessWorkingSetSize(
//   HANDLE hProcess,
//   SIZE_T dwMinimumWorkingSetSize,
//   SIZE_T dwMaximumWorkingSetSize
// );
// https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-setprocessworkingsetsize

func setProcessWorkingSetSize(process uintptr, dwMinWorkingSet uint64, dwMaxWorkingSet uint64) error {
	r1, _, err := syscall.Syscall(procSetProcessWorkingSetSize.Addr(), 3, process, uintptr(dwMinWorkingSet), uintptr(dwMaxWorkingSet))
	if r1 == 0 {
		if err != syscall.Errno(0) {
			return err
		}
	}
	return nil
}
<|endoftext|>