<commit_before>package integration_test

import (
	"bufio"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"

	"github.com/cloudfoundry/bosh-bootloader/integration-test"
	"github.com/cloudfoundry/bosh-bootloader/integration-test/aws/actors"
	"github.com/pivotal-cf-experimental/bosh-test/bosh"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	ConcourseExampleManifestURL = "https://raw.githubusercontent.com/concourse/concourse/master/docs/setting-up/installing.any"
	ConcourseReleaseURL         = "https://bosh.io/d/github.com/concourse/concourse"
	GardenReleaseURL            = "https://bosh.io/d/github.com/cloudfoundry-incubator/garden-runc-release"
	GardenReleaseName           = "garden-runc"
	StemcellURL                 = "https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent"
	StemcellName                = "bosh-aws-xen-hvm-ubuntu-trusty-go_agent"
)

var _ = Describe("bosh deployment tests", func() {
	var (
		bbl   actors.BBL
		aws   actors.AWS
		state integration.State
	)

	BeforeEach(func() {
		var err error
		configuration, err := integration.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		bbl = actors.NewBBL(configuration.StateFileDir, pathToBBL, configuration)
		aws = actors.NewAWS(configuration)
		state = integration.NewState(configuration.StateFileDir)
	})

	It("is able to deploy concourse", func() {
		bbl.Up()

		certPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_CERT)
		Expect(err).NotTo(HaveOccurred())

		keyPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_KEY)
		Expect(err).NotTo(HaveOccurred())

		bbl.CreateLB("concourse", certPath, keyPath, "")

		boshClient := bosh.NewClient(bosh.Config{
			URL:              bbl.DirectorAddress(),
			Username:         bbl.DirectorUsername(),
			Password:         bbl.DirectorPassword(),
			AllowInsecureSSL: true,
		})

		err = downloadAndUploadRelease(boshClient, ConcourseReleaseURL)
		Expect(err).NotTo(HaveOccurred())

		err = downloadAndUploadRelease(boshClient, GardenReleaseURL)
		Expect(err).NotTo(HaveOccurred())

		err = downloadAndUploadStemcell(boshClient, StemcellURL)
		Expect(err).NotTo(HaveOccurred())

		concourseExampleManifest, err := downloadConcourseExampleManifest()
		Expect(err).NotTo(HaveOccurred())

		info, err := boshClient.Info()
		Expect(err).NotTo(HaveOccurred())

		lbURL := fmt.Sprintf("http://%s", aws.LoadBalancers(state.StackName())["ConcourseLoadBalancerURL"])

		stemcell, err := boshClient.Stemcell(StemcellName)
		Expect(err).NotTo(HaveOccurred())

		concourseRelease, err := boshClient.Release("concourse")
		Expect(err).NotTo(HaveOccurred())

		gardenRelease, err := boshClient.Release(GardenReleaseName)
		Expect(err).NotTo(HaveOccurred())

		concourseManifestInputs := concourseManifestInputs{
			boshDirectorUUID:        info.UUID,
			webExternalURL:          lbURL,
			stemcellVersion:         stemcell.Latest(),
			concourseReleaseVersion: concourseRelease.Latest(),
			gardenReleaseVersion:    gardenRelease.Latest(),
		}
		concourseManifest, err := populateManifest(concourseExampleManifest, concourseManifestInputs)
		Expect(err).NotTo(HaveOccurred())

		_, err = boshClient.Deploy([]byte(concourseManifest))
		Expect(err).NotTo(HaveOccurred())

		Eventually(func() ([]bosh.VM, error) {
			return boshClient.DeploymentVMs("concourse")
		}, "1m", "10s").Should(ConsistOf([]bosh.VM{
			{JobName: "worker", Index: 0, State: "running"},
			{JobName: "db", Index: 0, State: "running"},
			{JobName: "web", Index: 0, State: "running"},
		}))

		resp, err := http.Get(lbURL)
		Expect(err).NotTo(HaveOccurred())

		Expect(resp.StatusCode).To(Equal(http.StatusOK))

		body, err := ioutil.ReadAll(resp.Body)
		Expect(err).NotTo(HaveOccurred())

		Expect(string(body)).To(ContainSubstring("<title>Concourse</title>"))

		err = boshClient.DeleteDeployment("concourse")
		Expect(err).NotTo(HaveOccurred())

		bbl.Destroy()
	})
})

func downloadAndUploadStemcell(boshClient bosh.Client, stemcell string) error {
	file, size, err := download(stemcell)
	if err != nil {
		return err
	}

	_, err = boshClient.UploadStemcell(bosh.NewSizeReader(file, size))
	return err
}

func downloadAndUploadRelease(boshClient bosh.Client, release string) error {
	file, size, err := download(release)
	if err != nil {
		return err
	}

	_, err = boshClient.UploadRelease(bosh.NewSizeReader(file, size))
	return err
}

func downloadConcourseExampleManifest() (string, error) {
	resp, _, err := download(ConcourseExampleManifestURL)
	if err != nil {
		return "", err
	}

	var lines []string
	scanner := bufio.NewScanner(resp)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	startIndexOfYamlCode := -1
	endIndexOfYamlCode := -1

	for index, line := range lines {
		startMatched, startErr := regexp.MatchString(`^\s*\\codeblock{yaml}{$`, line)
		endMatched, endErr := regexp.MatchString(`^\s*}$`, line)
		if endErr != nil {
			return "", endErr
		}

		if startErr != nil {
			panic(startErr)
		}

		if startMatched && startIndexOfYamlCode < 0 {
			startIndexOfYamlCode = index + 1
		}
		if endMatched && endIndexOfYamlCode < 0 && startIndexOfYamlCode > 0 {
			endIndexOfYamlCode = index
		}
	}

	yamlDocument := lines[startIndexOfYamlCode:endIndexOfYamlCode]

	re := regexp.MustCompile(`^(\s*)---`)
	results := re.FindAllStringSubmatch(yamlDocument[0], -1)
	indentation := results[0][1]
	for index, line := range yamlDocument {
		indentationRegexp := regexp.MustCompile(fmt.Sprintf(`^%s`, indentation))
		escapesRegexp := regexp.MustCompile(`\\([{}])`)
		tlsRegexp := regexp.MustCompile("^.*(tls_key|tls_cert).*$")

		line = indentationRegexp.ReplaceAllString(line, "")
		line = escapesRegexp.ReplaceAllString(line, "$1")
		line = tlsRegexp.ReplaceAllString(line, "")

		yamlDocument[index] = line

	}

	yamlString := strings.Join(yamlDocument, "\n")
	return yamlString, nil
}

func download(location string) (io.Reader, int64, error) {
	resp, err := http.Get(location)
	if err != nil {
		return nil, 0, err
	}

	return resp.Body, resp.ContentLength, nil
}
<commit_msg>Fix other imports for AWS integration test package rename<commit_after>package integration_test

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"

	integration "github.com/cloudfoundry/bosh-bootloader/integration-test"
	"github.com/cloudfoundry/bosh-bootloader/integration-test/aws/actors"
	"github.com/cloudfoundry/bosh-bootloader/testhelpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/pivotal-cf-experimental/bosh-test/bosh"
)

const (
	ConcourseExampleManifestURL = "https://raw.githubusercontent.com/concourse/concourse/master/docs/setting-up/installing.any"
	ConcourseReleaseURL         = "https://bosh.io/d/github.com/concourse/concourse"
	GardenReleaseURL            = "https://bosh.io/d/github.com/cloudfoundry-incubator/garden-runc-release"
	GardenReleaseName           = "garden-runc"
	StemcellURL                 = "https://bosh.io/d/stemcells/bosh-aws-xen-hvm-ubuntu-trusty-go_agent"
	StemcellName                = "bosh-aws-xen-hvm-ubuntu-trusty-go_agent"
)

var _ = Describe("bosh deployment tests", func() {
	var (
		bbl   actors.BBL
		aws   actors.AWS
		state integration.State
	)

	BeforeEach(func() {
		var err error
		configuration, err := integration.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		bbl = actors.NewBBL(configuration.StateFileDir, pathToBBL, configuration)
		aws = actors.NewAWS(configuration)
		state = integration.NewState(configuration.StateFileDir)
	})

	It("is able to deploy concourse", func() {
		bbl.Up()

		certPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_CERT)
		Expect(err).NotTo(HaveOccurred())

		keyPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_KEY)
		Expect(err).NotTo(HaveOccurred())

		bbl.CreateLB("concourse", certPath, keyPath, "")

		boshClient := bosh.NewClient(bosh.Config{
			URL:              bbl.DirectorAddress(),
			Username:         bbl.DirectorUsername(),
			Password:         bbl.DirectorPassword(),
			AllowInsecureSSL: true,
		})

		err = downloadAndUploadRelease(boshClient, ConcourseReleaseURL)
		Expect(err).NotTo(HaveOccurred())

		err = downloadAndUploadRelease(boshClient, GardenReleaseURL)
		Expect(err).NotTo(HaveOccurred())

		err = downloadAndUploadStemcell(boshClient, StemcellURL)
		Expect(err).NotTo(HaveOccurred())

		concourseExampleManifest, err := downloadConcourseExampleManifest()
		Expect(err).NotTo(HaveOccurred())

		info, err := boshClient.Info()
		Expect(err).NotTo(HaveOccurred())

		lbURL := fmt.Sprintf("http://%s", aws.LoadBalancers(state.StackName())["ConcourseLoadBalancerURL"])

		stemcell, err := boshClient.Stemcell(StemcellName)
		Expect(err).NotTo(HaveOccurred())

		concourseRelease, err := boshClient.Release("concourse")
		Expect(err).NotTo(HaveOccurred())

		gardenRelease, err := boshClient.Release(GardenReleaseName)
		Expect(err).NotTo(HaveOccurred())

		concourseManifestInputs := concourseManifestInputs{
			boshDirectorUUID:        info.UUID,
			webExternalURL:          lbURL,
			stemcellVersion:         stemcell.Latest(),
			concourseReleaseVersion: concourseRelease.Latest(),
			gardenReleaseVersion:    gardenRelease.Latest(),
		}
		concourseManifest, err := populateManifest(concourseExampleManifest, concourseManifestInputs)
		Expect(err).NotTo(HaveOccurred())

		_, err = boshClient.Deploy([]byte(concourseManifest))
		Expect(err).NotTo(HaveOccurred())

		Eventually(func() ([]bosh.VM, error) {
			return boshClient.DeploymentVMs("concourse")
		}, "1m", "10s").Should(ConsistOf([]bosh.VM{
			{JobName: "worker", Index: 0, State: "running"},
			{JobName: "db", Index: 0, State: "running"},
			{JobName: "web", Index: 0, State: "running"},
		}))

		resp, err := http.Get(lbURL)
		Expect(err).NotTo(HaveOccurred())

		Expect(resp.StatusCode).To(Equal(http.StatusOK))

		body, err := ioutil.ReadAll(resp.Body)
		Expect(err).NotTo(HaveOccurred())

		Expect(string(body)).To(ContainSubstring("<title>Concourse</title>"))

		err = boshClient.DeleteDeployment("concourse")
		Expect(err).NotTo(HaveOccurred())

		bbl.Destroy()
	})
})

func downloadAndUploadStemcell(boshClient bosh.Client, stemcell string) error {
	file, size, err := download(stemcell)
	if err != nil {
		return err
	}

	_, err = boshClient.UploadStemcell(bosh.NewSizeReader(file, size))
	return err
}

func downloadAndUploadRelease(boshClient bosh.Client, release string) error {
	file, size, err := download(release)
	if err != nil {
		return err
	}

	_, err = boshClient.UploadRelease(bosh.NewSizeReader(file, size))
	return err
}

func downloadConcourseExampleManifest() (string, error) {
	resp, _, err := download(ConcourseExampleManifestURL)
	if err != nil {
		return "", err
	}

	var lines []string
	scanner := bufio.NewScanner(resp)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	startIndexOfYamlCode := -1
	endIndexOfYamlCode := -1

	for index, line := range lines {
		startMatched, startErr := regexp.MatchString(`^\s*\\codeblock{yaml}{$`, line)
		endMatched, endErr := regexp.MatchString(`^\s*}$`, line)
		if endErr != nil {
			return "", endErr
		}

		if startErr != nil {
			panic(startErr)
		}

		if startMatched && startIndexOfYamlCode < 0 {
			startIndexOfYamlCode = index + 1
		}
		if endMatched && endIndexOfYamlCode < 0 && startIndexOfYamlCode > 0 {
			endIndexOfYamlCode = index
		}
	}

	yamlDocument := lines[startIndexOfYamlCode:endIndexOfYamlCode]

	re := regexp.MustCompile(`^(\s*)---`)
	results := re.FindAllStringSubmatch(yamlDocument[0], -1)
	indentation := results[0][1]
	for index, line := range yamlDocument {
		indentationRegexp := regexp.MustCompile(fmt.Sprintf(`^%s`, indentation))
		escapesRegexp := regexp.MustCompile(`\\([{}])`)
		tlsRegexp := regexp.MustCompile("^.*(tls_key|tls_cert).*$")

		line = indentationRegexp.ReplaceAllString(line, "")
		line = escapesRegexp.ReplaceAllString(line, "$1")
		line = tlsRegexp.ReplaceAllString(line, "")

		yamlDocument[index] = line

	}

	yamlString := strings.Join(yamlDocument, "\n")
	return yamlString, nil
}

func download(location string) (io.Reader, int64, error) {
	resp, err := http.Get(location)
	if err != nil {
		return nil, 0, err
	}

	return resp.Body, resp.ContentLength, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package performance

import (
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/knative/test-infra/shared/junit"
	"github.com/knative/test-infra/shared/mysql"
)

const (
	presubmit  = "presubmit"
	dbName     = "knative_performance"
	dbInstance = "knative-tests:us-central1:knative-monitoring"
	insertStmt = `
	INSERT INTO METRICS (
		RunId, JobType, TestName, MetricName, MetricValue
		) VALUES (?, ?, ?, ?, ?)`

	// Path to secrets for username and password
	userSecret = "/secrets/cloudsql/monitoringdb/username"
	passSecret = "/secrets/cloudsql/monitoringdb/password"

	// Property name used by testgrid
	perfLatency = "perf_latency"
)

// CreatePerfTestCase creates a perf test case with the provided name and value
func CreatePerfTestCase(metricValue float32, metricName, testName string) junit.TestCase {
	tp := []junit.TestProperty{{Name: perfLatency, Value: fmt.Sprintf("%f", metricValue)}}
	tc := junit.TestCase{
		ClassName:  testName,
		Name:       fmt.Sprintf("%s/%s", testName, metricName),
		Properties: junit.TestProperties{Properties: tp}}

	db, err := ConfigureDB()
	if err == nil {
		if err = db.StoreMetrics(testName, metricName, metricValue); err != nil {
			log.Printf("Cannot store metrics %s for %s due to: %v", metricName, testName, err)
		}
	} else {
		log.Printf("Cannot configure db: %v", err)
	}
	return tc
}

type DBConfig struct {
	*mysql.DBConfig
}

// Configure the db instance to store metrics information.
// This will be later used to show the trending metrics on our grafana dashboard.
func ConfigureDB() (*DBConfig, error) {
	config, err := mysql.ConfigureDB(userSecret, passSecret, dbName, dbInstance)
	return &DBConfig{config}, err
}

func (c *DBConfig) StoreMetrics(tName, metricName string, metricValue float32) error {
	// Get values of env vars set up by Prow. Ignore storage for local runs
	runId := os.Getenv("BUILD_ID")
	jobType := os.Getenv("JOB_TYPE")
	if len(runId) == 0 || len(jobType) == 0 {
		log.Printf("Build id or job type not set. Not storing metric %s", metricName)
		return nil
	}

	if strings.ToLower(jobType) == presubmit {
		log.Printf("Ignoring metric %s storage as this is a pre-submit job", metricName)
		return nil
	}

	// Store the metrics if the perf tests are run by prow(ci/cd) and if its periodic runs
	db, err := c.Connect()
	if err != nil {
		return err
	}
	defer db.Close()

	// Execute the insert statement to add metric information
	res, err := db.Exec(insertStmt, runId, jobType, tName, metricName, metricValue)
	if err != nil {
		return err
	}
	rows, err := res.RowsAffected()
	if err != nil {
		return err
	}
	if rows != 1 {
		return fmt.Errorf("expected to affect 1 row, affected %d", rows)
	}

	return nil
}
<commit_msg>do not store perf metrics in sql (#951)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package performance

import (
	"fmt"

	"github.com/knative/test-infra/shared/junit"
)

const (
	// Property name used by testgrid
	perfLatency = "perf_latency"
)

// CreatePerfTestCase creates a perf test case with the provided name and value
func CreatePerfTestCase(metricValue float32, metricName, testName string) junit.TestCase {
	tp := []junit.TestProperty{{Name: perfLatency, Value: fmt.Sprintf("%f", metricValue)}}
	tc := junit.TestCase{
		ClassName:  testName,
		Name:       fmt.Sprintf("%s/%s", testName, metricName),
		Properties: junit.TestProperties{Properties: tp}}

	return tc
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sync"
	"syscall"
)

type ksmSetting struct {
	// run describes if we want KSM to be on or off.
	run bool

	// pagesPerScanFactor describes how many pages we want
	// to scan per KSM run.
	// ksmd will san N pages, where N*pagesPerScanFactor is
	// equal to the number of anonymous pages.
	pagesPerScanFactor uint32

	// scanIntervalMS is the KSM scan interval in milliseconds.
	scanIntervalMS uint32
}

func (s ksmSetting) pagesToScan() (string, error) {
	return "10000", nil
}

const (
	ksmInitial    = "initial"
	ksmOff        = "off"
	ksmSlow       = "slow"
	ksmStandard   = "standard"
	ksmAggressive = "aggressive"
)

var ksmSettings = map[string]ksmSetting{
	ksmOff:        {false, 1000, 500}, // Turn KSM off
	ksmSlow:       {true, 500, 100},   // Every 100ms, we scan 1 page for every 500 pages available in the system
	ksmStandard:   {true, 100, 10},    // Every 10ms, we scan 1 page for every 100 pages available in the system
	ksmAggressive: {true, 10, 1},      // Every ms, we scan 1 page for every 10 pages available in the system
}

var defaultKSMRoot = "/sys/kernel/mm/ksm/"
var errKSMUnavailable = errors.New("KSM is unavailable")

const (
	ksmRunFile        = "run"
	ksmPagesToScan    = "pages_to_scan"
	ksmSleepMillisec  = "sleep_millisecs"
	ksmStart          = "1"
	ksmStop           = "0"
	defaultKSMSetting = ksmInitial
)

type sysfsAttribute struct {
	path string
	file *os.File
}

func (attr *sysfsAttribute) open() error {
	file, err := os.OpenFile(attr.path, os.O_RDWR|syscall.O_NONBLOCK, 0660)
	attr.file = file
	return err
}

func (attr *sysfsAttribute) close() error {
	err := attr.file.Close()
	attr.file = nil
	return err
}

func (attr *sysfsAttribute) read() (string, error) {
	_, err := attr.file.Seek(0, os.SEEK_SET)
	if err != nil {
		return "", err
	}

	data, err := ioutil.ReadAll(attr.file)
	if err != nil {
		return "", err
	}

	return string(data), nil
}

func (attr *sysfsAttribute) write(value string) error {
	_, err := attr.file.Seek(0, os.SEEK_SET)
	if err != nil {
		return err
	}
	_, err = attr.file.WriteString(value)

	return err
}

type ksm struct {
	root          string
	run           sysfsAttribute
	pagesToScan   sysfsAttribute
	sleepInterval sysfsAttribute
	initialized   bool

	initialPagesToScan   string
	initialSleepInterval string
	initialKSMRun        string

	sync.Mutex
}

func (k *ksm) isAvailable() error {
	info, err := os.Stat(k.root)
	if err != nil || !info.IsDir() {
		return fmt.Errorf("%s is not available", k.root)
	}

	return nil
}

func newKSM(root string) (*ksm, error) {
	var err error
	var k ksm

	k.initialized = false
	k.root = root

	if root == "" {
		return nil, errors.New("Invalid KSM root")
	}

	if err := k.isAvailable(); err != nil {
		return nil, err
	}

	k.pagesToScan = sysfsAttribute{
		path: filepath.Join(k.root, ksmPagesToScan),
	}

	k.sleepInterval = sysfsAttribute{
		path: filepath.Join(k.root, ksmSleepMillisec),
	}

	k.run = sysfsAttribute{
		path: filepath.Join(k.root, ksmRunFile),
	}

	defer func(err error) {
		if err != nil {
			_ = k.run.close()
			_ = k.sleepInterval.close()
			_ = k.pagesToScan.close()
		}
	}(err)

	if err := k.run.open(); err != nil {
		return nil, err
	}

	if err := k.sleepInterval.open(); err != nil {
		return nil, err
	}

	if err := k.pagesToScan.open(); err != nil {
		return nil, err
	}

	k.initialPagesToScan, err = k.pagesToScan.read()
	if err != nil {
		return nil, err
	}

	k.initialSleepInterval, err = k.sleepInterval.read()
	if err != nil {
		return nil, err
	}

	k.initialKSMRun, err = k.run.read()
	if err != nil {
		return nil, err
	}

	k.initialized = true
	return &k, nil
}

func (k *ksm) restore() error {
	var err error

	k.Lock()
	defer k.Unlock()

	if !k.initialized {
		return errKSMUnavailable
	}

	if err = k.pagesToScan.write(k.initialPagesToScan); err != nil {
		return err
	}

	if err = k.sleepInterval.write(k.initialSleepInterval); err != nil {
		return err
	}

	if err = k.run.write(k.initialKSMRun); err != nil {
		return err
	}

	if err := k.run.close(); err != nil {
		return err
	}

	if err := k.sleepInterval.close(); err != nil {
		return err
	}

	if err := k.pagesToScan.close(); err != nil {
		return err
	}

	k.initialized = false
	return nil
}

func (k *ksm) tune(s ksmSetting) error {
	k.Lock()
	defer k.Unlock()

	if !k.initialized {
		return errKSMUnavailable
	}

	if !s.run {
		return k.run.write(ksmStop)
	}

	newPagesToScan, err := s.pagesToScan()
	if err != nil {
		return err
	}

	if err = k.run.write(ksmStop); err != nil {
		return err
	}

	if err = k.pagesToScan.write(newPagesToScan); err != nil {
		return err
	}

	if err = k.sleepInterval.write(fmt.Sprintf("%v", s.scanIntervalMS)); err != nil {
		return err
	}

	if err = k.run.write(ksmStart); err != nil {
		return err
	}

	return nil
}
<commit_msg>ksm: Compute number of pages to scan<commit_after>//
// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"syscall"
)

type ksmSetting struct {
	// run describes if we want KSM to be on or off.
	run bool

	// pagesPerScanFactor describes how many pages we want
	// to scan per KSM run.
	// ksmd will san N pages, where N*pagesPerScanFactor is
	// equal to the number of anonymous pages.
	pagesPerScanFactor int64

	// scanIntervalMS is the KSM scan interval in milliseconds.
	scanIntervalMS uint32
}

func anonPages() (int64, error) {
	// We're going to parse meminfo
	f, err := os.Open(memInfo)
	if err != nil {
		return -1, err
	}
	defer f.Close()

	scan := bufio.NewScanner(f)
	for scan.Scan() {
		line := scan.Text()

		// We only care about anonymous pages
		if !strings.HasPrefix(line, "AnonPages:") {
			continue
		}

		// Extract the before last (value) and last (unit) fields
		fields := strings.Split(line, " ")
		value := fields[len(fields)-2]
		totalMemory, err := strconv.ParseInt(value, 10, 64)
		if err != nil {
			return -1, fmt.Errorf("Invalid integer")
		}

		// meminfo gives us kB
		totalMemory *= 1024

		// Fetch the system page size
		pageSize := (int64)(os.Getpagesize())

		nPages := totalMemory / pageSize
		return nPages, nil
	}

	return 0, fmt.Errorf("Could not compute number of pages")
}

func (s ksmSetting) pagesToScan() (string, error) {
	nPages, err := anonPages()
	if err != nil {
		return "", err
	}

	pagesToScan := nPages / s.pagesPerScanFactor

	return fmt.Sprintf("%v", pagesToScan), nil
}

const (
	ksmInitial    = "initial"
	ksmOff        = "off"
	ksmSlow       = "slow"
	ksmStandard   = "standard"
	ksmAggressive = "aggressive"
)

var ksmSettings = map[string]ksmSetting{
	ksmOff:        {false, 1000, 500}, // Turn KSM off
	ksmSlow:       {true, 500, 100},   // Every 100ms, we scan 1 page for every 500 pages available in the system
	ksmStandard:   {true, 100, 10},    // Every 10ms, we scan 1 page for every 100 pages available in the system
	ksmAggressive: {true, 10, 1},      // Every ms, we scan 1 page for every 10 pages available in the system
}

var defaultKSMRoot = "/sys/kernel/mm/ksm/"
var errKSMUnavailable = errors.New("KSM is unavailable")
var memInfo = "/proc/meminfo"

const (
	ksmRunFile        = "run"
	ksmPagesToScan    = "pages_to_scan"
	ksmSleepMillisec  = "sleep_millisecs"
	ksmStart          = "1"
	ksmStop           = "0"
	defaultKSMSetting = ksmInitial
)

type sysfsAttribute struct {
	path string
	file *os.File
}

func (attr *sysfsAttribute) open() error {
	file, err := os.OpenFile(attr.path, os.O_RDWR|syscall.O_NONBLOCK, 0660)
	attr.file = file
	return err
}

func (attr *sysfsAttribute) close() error {
	err := attr.file.Close()
	attr.file = nil
	return err
}

func (attr *sysfsAttribute) read() (string, error) {
	_, err := attr.file.Seek(0, os.SEEK_SET)
	if err != nil {
		return "", err
	}

	data, err := ioutil.ReadAll(attr.file)
	if err != nil {
		return "", err
	}

	return string(data), nil
}

func (attr *sysfsAttribute) write(value string) error {
	_, err := attr.file.Seek(0, os.SEEK_SET)
	if err != nil {
		return err
	}
	_, err = attr.file.WriteString(value)

	return err
}

type ksm struct {
	root          string
	run           sysfsAttribute
	pagesToScan   sysfsAttribute
	sleepInterval sysfsAttribute
	initialized   bool

	initialPagesToScan   string
	initialSleepInterval string
	initialKSMRun        string

	sync.Mutex
}

func (k *ksm) isAvailable() error {
	info, err := os.Stat(k.root)
	if err != nil || !info.IsDir() {
		return fmt.Errorf("%s is not available", k.root)
	}

	return nil
}

func newKSM(root string) (*ksm, error) {
	var err error
	var k ksm

	k.initialized = false
	k.root = root

	if root == "" {
		return nil, errors.New("Invalid KSM root")
	}

	if err := k.isAvailable(); err != nil {
		return nil, err
	}

	k.pagesToScan = sysfsAttribute{
		path: filepath.Join(k.root, ksmPagesToScan),
	}

	k.sleepInterval = sysfsAttribute{
		path: filepath.Join(k.root, ksmSleepMillisec),
	}

	k.run = sysfsAttribute{
		path: filepath.Join(k.root, ksmRunFile),
	}

	defer func(err error) {
		if err != nil {
			_ = k.run.close()
			_ = k.sleepInterval.close()
			_ = k.pagesToScan.close()
		}
	}(err)

	if err := k.run.open(); err != nil {
		return nil, err
	}

	if err := k.sleepInterval.open(); err != nil {
		return nil, err
	}

	if err := k.pagesToScan.open(); err != nil {
		return nil, err
	}

	k.initialPagesToScan, err = k.pagesToScan.read()
	if err != nil {
		return nil, err
	}

	k.initialSleepInterval, err = k.sleepInterval.read()
	if err != nil {
		return nil, err
	}

	k.initialKSMRun, err = k.run.read()
	if err != nil {
		return nil, err
	}

	k.initialized = true
	return &k, nil
}

func (k *ksm) restore() error {
	var err error

	k.Lock()
	defer k.Unlock()

	if !k.initialized {
		return errKSMUnavailable
	}

	if err = k.pagesToScan.write(k.initialPagesToScan); err != nil {
		return err
	}

	if err = k.sleepInterval.write(k.initialSleepInterval); err != nil {
		return err
	}

	if err = k.run.write(k.initialKSMRun); err != nil {
		return err
	}

	if err := k.run.close(); err != nil {
		return err
	}

	if err := k.sleepInterval.close(); err != nil {
		return err
	}

	if err := k.pagesToScan.close(); err != nil {
		return err
	}

	k.initialized = false
	return nil
}

func (k *ksm) tune(s ksmSetting) error {
	k.Lock()
	defer k.Unlock()

	if !k.initialized {
		return errKSMUnavailable
	}

	if !s.run {
		return k.run.write(ksmStop)
	}

	newPagesToScan, err := s.pagesToScan()
	if err != nil {
		return err
	}

	if err = k.run.write(ksmStop); err != nil {
		return err
	}

	if err = k.pagesToScan.write(newPagesToScan); err != nil {
		return err
	}

	if err = k.sleepInterval.write(fmt.Sprintf("%v", s.scanIntervalMS)); err != nil {
		return err
	}

	if err = k.run.write(ksmStart); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package command

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/hashicorp/consul/command/agent"
	"github.com/hashicorp/consul/command/base"
	"github.com/hashicorp/consul/watch"
)

// WatchCommand is a Command implementation that is used to setup
// a "watch" which uses a sub-process
type WatchCommand struct {
	base.Command
	ShutdownCh <-chan struct{}
}

func (c *WatchCommand) Help() string {
	helpText := `
Usage: consul watch [options] [child...]

  Watches for changes in a given data view from Consul. If a child process
  is specified, it will be invoked with the latest results on changes. Otherwise,
  the latest values are dumped to stdout and the watch terminates.

  Providing the watch type is required, and other parameters may be required
  or supported depending on the watch type.

` + c.Command.Help()

	return strings.TrimSpace(helpText)
}

func (c *WatchCommand) Run(args []string) int {
	var watchType, key, prefix, service, tag, passingOnly, state, name string

	f := c.Command.NewFlagSet(c)
	f.StringVar(&watchType, "type", "",
		"Specifies the watch type. One of key, keyprefix services, nodes, "+
			"service, checks, or event.")
	f.StringVar(&key, "key", "",
		"Specifies the key to watch. Only for 'key' type.")
	f.StringVar(&prefix, "prefix", "",
		"Specifies the key prefix to watch. Only for 'keyprefix' type.")
	f.StringVar(&service, "service", "",
		"Specifies the service to watch. Required for 'service' type, "+
			"optional for 'checks' type.")
	f.StringVar(&tag, "tag", "",
		"Specifies the service tag to filter on. Optional for 'service' type.")
	f.StringVar(&passingOnly, "passingonly", "",
		"Specifies if only hosts passing all checks are displayed. "+
			"Optional for 'service' type, must be one of `[true|false]`. Defaults false.")
	f.StringVar(&state, "state", "",
		"Specifies the states to watch. Optional for 'checks' type.")
	f.StringVar(&name, "name", "",
		"Specifies an event name to watch. Only for 'event' type.")

	if err := c.Command.Parse(args); err != nil {
		return 1
	}

	// Check for a type
	if watchType == "" {
		c.UI.Error("Watch type must be specified")
		c.UI.Error("")
		c.UI.Error(c.Help())
		return 1
	}

	// Grab the script to execute if any
	script := strings.Join(f.Args(), " ")

	// Compile the watch parameters
	params := make(map[string]interface{})
	if watchType != "" {
		params["type"] = watchType
	}
	if c.Command.HTTPDatacenter() != "" {
		params["datacenter"] = c.Command.HTTPDatacenter()
	}
	if c.Command.HTTPToken() != "" {
		params["token"] = c.Command.HTTPToken()
	}
	if key != "" {
		params["key"] = key
	}
	if prefix != "" {
		params["prefix"] = prefix
	}
	if service != "" {
		params["service"] = service
	}
	if tag != "" {
		params["tag"] = tag
	}
	if c.Command.HTTPStale() {
		params["stale"] = c.Command.HTTPStale()
	}
	if state != "" {
		params["state"] = state
	}
	if name != "" {
		params["name"] = name
	}
	if passingOnly != "" {
		b, err := strconv.ParseBool(passingOnly)
		if err != nil {
			c.UI.Error(fmt.Sprintf("Failed to parse passingonly flag: %s", err))
			return 1
		}
		params["passingonly"] = b
	}

	// Create the watch
	wp, err := watch.Parse(params)
	if err != nil {
		c.UI.Error(fmt.Sprintf("%s", err))
		return 1
	}

	// Create and test the HTTP client
	client, err := c.Command.HTTPClient()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
		return 1
	}
	_, err = client.Agent().NodeName()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error querying Consul agent: %s", err))
		return 1
	}

	// Setup handler

	// errExit:
	//	0: false
	//	1: true
	errExit := 0
	if script == "" {
		wp.Handler = func(idx uint64, data interface{}) {
			defer wp.Stop()
			buf, err := json.MarshalIndent(data, "", "    ")
			if err != nil {
				c.UI.Error(fmt.Sprintf("Error encoding output: %s", err))
				errExit = 1
			}
			c.UI.Output(string(buf))
		}
	} else {
		wp.Handler = func(idx uint64, data interface{}) {
			// Create the command
			var buf bytes.Buffer
			var err error
			cmd, err := agent.ExecScript(script)
			if err != nil {
				c.UI.Error(fmt.Sprintf("Error executing handler: %s", err))
				goto ERR
			}
			cmd.Env = append(os.Environ(),
				"CONSUL_INDEX="+strconv.FormatUint(idx, 10),
			)

			// Encode the input
			if err = json.NewEncoder(&buf).Encode(data); err != nil {
				c.UI.Error(fmt.Sprintf("Error encoding output: %s", err))
				goto ERR
			}
			cmd.Stdin = &buf
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr

			// Run the handler
			if err := cmd.Run(); err != nil {
				c.UI.Error(fmt.Sprintf("Error executing handler: %s", err))
				goto ERR
			}
			return
		ERR:
			wp.Stop()
			errExit = 1
		}
	}

	// Watch for a shutdown
	go func() {
		<-c.ShutdownCh
		wp.Stop()
		os.Exit(0)
	}()

	// Run the watch
	if err := wp.Run(c.Command.HTTPAddr()); err != nil {
		c.UI.Error(fmt.Sprintf("Error querying Consul agent: %s", err))
		return 1
	}

	return errExit
}

func (c *WatchCommand) Synopsis() string {
	return "Watch for changes in Consul"
}
<commit_msg>Itty bitty typo<commit_after>package command

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/hashicorp/consul/command/agent"
	"github.com/hashicorp/consul/command/base"
	"github.com/hashicorp/consul/watch"
)

// WatchCommand is a Command implementation that is used to setup
// a "watch" which uses a sub-process
type WatchCommand struct {
	base.Command
	ShutdownCh <-chan struct{}
}

func (c *WatchCommand) Help() string {
	helpText := `
Usage: consul watch [options] [child...]

  Watches for changes in a given data view from Consul. If a child process
  is specified, it will be invoked with the latest results on changes. Otherwise,
  the latest values are dumped to stdout and the watch terminates.

  Providing the watch type is required, and other parameters may be required
  or supported depending on the watch type.

` + c.Command.Help()

	return strings.TrimSpace(helpText)
}

func (c *WatchCommand) Run(args []string) int {
	var watchType, key, prefix, service, tag, passingOnly, state, name string

	f := c.Command.NewFlagSet(c)
	f.StringVar(&watchType, "type", "",
		"Specifies the watch type. One of key, keyprefix, services, nodes, "+
			"service, checks, or event.")
	f.StringVar(&key, "key", "",
		"Specifies the key to watch. Only for 'key' type.")
	f.StringVar(&prefix, "prefix", "",
		"Specifies the key prefix to watch. Only for 'keyprefix' type.")
	f.StringVar(&service, "service", "",
		"Specifies the service to watch. Required for 'service' type, "+
			"optional for 'checks' type.")
	f.StringVar(&tag, "tag", "",
		"Specifies the service tag to filter on. Optional for 'service' type.")
	f.StringVar(&passingOnly, "passingonly", "",
		"Specifies if only hosts passing all checks are displayed. "+
			"Optional for 'service' type, must be one of `[true|false]`. Defaults false.")
	f.StringVar(&state, "state", "",
		"Specifies the states to watch. Optional for 'checks' type.")
	f.StringVar(&name, "name", "",
		"Specifies an event name to watch. Only for 'event' type.")

	if err := c.Command.Parse(args); err != nil {
		return 1
	}

	// Check for a type
	if watchType == "" {
		c.UI.Error("Watch type must be specified")
		c.UI.Error("")
		c.UI.Error(c.Help())
		return 1
	}

	// Grab the script to execute if any
	script := strings.Join(f.Args(), " ")

	// Compile the watch parameters
	params := make(map[string]interface{})
	if watchType != "" {
		params["type"] = watchType
	}
	if c.Command.HTTPDatacenter() != "" {
		params["datacenter"] = c.Command.HTTPDatacenter()
	}
	if c.Command.HTTPToken() != "" {
		params["token"] = c.Command.HTTPToken()
	}
	if key != "" {
		params["key"] = key
	}
	if prefix != "" {
		params["prefix"] = prefix
	}
	if service != "" {
		params["service"] = service
	}
	if tag != "" {
		params["tag"] = tag
	}
	if c.Command.HTTPStale() {
		params["stale"] = c.Command.HTTPStale()
	}
	if state != "" {
		params["state"] = state
	}
	if name != "" {
		params["name"] = name
	}
	if passingOnly != "" {
		b, err := strconv.ParseBool(passingOnly)
		if err != nil {
			c.UI.Error(fmt.Sprintf("Failed to parse passingonly flag: %s", err))
			return 1
		}
		params["passingonly"] = b
	}

	// Create the watch
	wp, err := watch.Parse(params)
	if err != nil {
		c.UI.Error(fmt.Sprintf("%s", err))
		return 1
	}

	// Create and test the HTTP client
	client, err := c.Command.HTTPClient()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error connecting to Consul agent: %s", err))
		return 1
	}
	_, err = client.Agent().NodeName()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error querying Consul agent: %s", err))
		return 1
	}

	// Setup handler

	// errExit:
	//	0: false
	//	1: true
	errExit := 0
	if script == "" {
		wp.Handler = func(idx uint64, data interface{}) {
			defer wp.Stop()
			buf, err := json.MarshalIndent(data, "", "    ")
			if err != nil {
				c.UI.Error(fmt.Sprintf("Error encoding output: %s", err))
				errExit = 1
			}
			c.UI.Output(string(buf))
		}
	} else {
		wp.Handler = func(idx uint64, data interface{}) {
			// Create the command
			var buf bytes.Buffer
			var err error
			cmd, err := agent.ExecScript(script)
			if err != nil {
				c.UI.Error(fmt.Sprintf("Error executing handler: %s", err))
				goto ERR
			}
			cmd.Env = append(os.Environ(),
				"CONSUL_INDEX="+strconv.FormatUint(idx, 10),
			)

			// Encode the input
			if err = json.NewEncoder(&buf).Encode(data); err != nil {
				c.UI.Error(fmt.Sprintf("Error encoding output: %s", err))
				goto ERR
			}
			cmd.Stdin = &buf
			cmd.Stdout = os.Stdout
			cmd.Stderr = os.Stderr

			// Run the handler
			if err := cmd.Run(); err != nil {
				c.UI.Error(fmt.Sprintf("Error executing handler: %s", err))
				goto ERR
			}
			return
		ERR:
			wp.Stop()
			errExit = 1
		}
	}

	// Watch for a shutdown
	go func() {
		<-c.ShutdownCh
		wp.Stop()
		os.Exit(0)
	}()

	// Run the watch
	if err := wp.Run(c.Command.HTTPAddr()); err != nil {
		c.UI.Error(fmt.Sprintf("Error querying Consul agent: %s", err))
		return 1
	}

	return errExit
}

func (c *WatchCommand) Synopsis() string {
	return "Watch for changes in Consul"
}
<|endoftext|>


<commit_before>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by the license that can be found in the
// LICENSE file.

// Package postgresstore implements a store that saves all the segments in a
// PostgreSQL database. It requires PostgreSQL >= 9.5 for
// "ON CONFLICT DO UPDATE" support.
package postgresstore

import (
	"database/sql"
	"encoding/json"

	"github.com/lib/pq"
	"github.com/stratumn/go/cs"
	"github.com/stratumn/go/store"
	"github.com/stratumn/go/types"
)

const (
	// Name is the name set in the store's information.
	Name = "postgres"

	// Description is the description set in the store's information.
	Description = "Stratumn PostgreSQL Store"

	// DefaultURL is the default URL of the database.
	DefaultURL = "postgres://postgres@postgres/postgres?sslmode=disable"
)

const notFoundError = "sql: no rows in result set"

// Config contains configuration options for the store.
type Config struct {
	// A version string that will be set in the store's information.
	Version string

	// A git commit hash that will be set in the store's information.
	Commit string

	// The URL of the PostgreSQL database, such as
	// "postgres://postgres@localhost/store?sslmode=disable".
	URL string
}

// Info is the info returned by GetInfo.
type Info struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Version     string `json:"version"`
	Commit      string `json:"commit"`
}

// Store is the type that implements github.com/stratumn/go/store.Adapter.
type Store struct {
	config *Config
	db     *sql.DB
	stmts  *stmts
}

// New creates an instance of a Store.
func New(config *Config) (*Store, error) {
	db, err := sql.Open("postgres", config.URL)
	if err != nil {
		return nil, err
	}
	return &Store{config: config, db: db}, nil
}

// GetInfo implements github.com/stratumn/go/store.Adapter.GetInfo.
func (a *Store) GetInfo() (interface{}, error) {
	return &Info{
		Name:        Name,
		Description: Description,
		Version:     a.config.Version,
		Commit:      a.config.Commit,
	}, nil
}

// SaveSegment implements github.com/stratumn/go/store.Adapter.SaveSegment.
func (a *Store) SaveSegment(segment *cs.Segment) error {
	var (
		err          error
		linkHash     = segment.GetLinkHash()
		priority     = segment.Link.GetPriority()
		mapID        = segment.Link.GetMapID()
		prevLinkHash = segment.Link.GetPrevLinkHash()
		tags         = segment.Link.GetTags()
	)

	data, err := json.Marshal(segment)
	if err != nil {
		return err
	}

	if prevLinkHash == nil {
		_, err = a.stmts.SaveSegment.Exec(linkHash[:], priority, mapID, nil, pq.Array(tags), string(data))
	} else {
		_, err = a.stmts.SaveSegment.Exec(linkHash[:], priority, mapID, prevLinkHash[:], pq.Array(tags), string(data))
	}

	return err
}

// GetSegment implements github.com/stratumn/go/store.Adapter.GetSegment.
func (a *Store) GetSegment(linkHash *types.Bytes32) (*cs.Segment, error) {
	var data string

	if err := a.stmts.GetSegment.QueryRow(linkHash[:]).Scan(&data); err != nil {
		if err.Error() == notFoundError {
			return nil, nil
		}
		return nil, err
	}

	var segment cs.Segment
	if err := json.Unmarshal([]byte(data), &segment); err != nil {
		return nil, err
	}

	return &segment, nil
}

// DeleteSegment implements github.com/stratumn/go/store.Adapter.DeleteSegment.
func (a *Store) DeleteSegment(linkHash *types.Bytes32) (*cs.Segment, error) {
	var (
		data    string
		segment cs.Segment
	)

	if err := a.stmts.DeleteSegment.QueryRow(linkHash[:]).Scan(&data); err != nil {
		if err.Error() == notFoundError {
			return nil, nil
		}
		return nil, err
	}

	if err := json.Unmarshal([]byte(data), &segment); err != nil {
		return nil, err
	}

	return &segment, nil
}

// FindSegments implements github.com/stratumn/go/store.Adapter.FindSegments.
func (a *Store) FindSegments(filter *store.Filter) (cs.SegmentSlice, error) {
	var (
		rows     *sql.Rows
		err      error
		limit    = filter.Limit
		offset   = filter.Offset
		segments = make(cs.SegmentSlice, 0, limit)
	)

	if filter.PrevLinkHash != nil {
		prevLinkHash := filter.PrevLinkHash[:]
		if len(filter.Tags) > 0 {
			tags := pq.Array(filter.Tags)
			rows, err = a.stmts.FindSegmentsWithPrevLinkHashAndTags.Query(prevLinkHash, tags, offset, limit)
		} else {
			rows, err = a.stmts.FindSegmentsWithPrevLinkHash.Query(prevLinkHash, offset, limit)
		}
	} else if mapID := filter.MapID; mapID != "" {
		if len(filter.Tags) > 0 {
			tags := pq.Array(filter.Tags)
			rows, err = a.stmts.FindSegmentsWithMapIDAndTags.Query(mapID, tags, offset, limit)
		} else {
			rows, err = a.stmts.FindSegmentsWithMapID.Query(mapID, offset, limit)
		}
	} else if len(filter.Tags) > 0 {
		tags := pq.Array(filter.Tags)
		rows, err = a.stmts.FindSegmentsWithTags.Query(tags, offset, limit)
	} else {
		rows, err = a.stmts.FindSegments.Query(offset, limit)
	}

	if err != nil {
		return nil, err
	}

	defer rows.Close()

	for rows.Next() {
		var (
			data    string
			segment cs.Segment
		)

		if err = rows.Scan(&data); err != nil {
			return nil, err
		}

		if err = json.Unmarshal([]byte(data), &segment); err != nil {
			return nil, err
		}

		segments = append(segments, &segment)
	}

	return segments, nil
}

// GetMapIDs implements github.com/stratumn/go/store.Adapter.GetMapIDs.
func (a *Store) GetMapIDs(pagination *store.Pagination) ([]string, error) {
	rows, err := a.stmts.GetMapIDs.Query(pagination.Offset, pagination.Limit)
	if err != nil {
		return nil, err
	}

	defer rows.Close()

	mapIDs := make([]string, 0, pagination.Limit)

	for rows.Next() {
		var mapID string
		if err = rows.Scan(&mapID); err != nil {
			return nil, err
		}

		mapIDs = append(mapIDs, mapID)
	}

	return mapIDs, nil
}

// Create creates the database tables and indexes.
func (a *Store) Create() error {
	for _, query := range sqlCreate {
		if _, err := a.db.Exec(query); err != nil {
			return err
		}
	}
	return nil
}

// Prepare prepares the database stmts.
// It should be called once before interacting with segments.
// It assumes the tables have been created using Create().
func (a *Store) Prepare() (err error) {
	a.stmts, err = newStmts(a.db)
	return
}

// Drop drops the database tables and indexes.
func (a *Store) Drop() error {
	for _, query := range sqlDrop {
		if _, err := a.db.Exec(query); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>postgresstore: add web socket support to stores<commit_after>// Copyright 2016 Stratumn SAS. All rights reserved.
// Use of this source code is governed by the license that can be found in the
// LICENSE file.

// Package postgresstore implements a store that saves all the segments in a
// PostgreSQL database. It requires PostgreSQL >= 9.5 for
// "ON CONFLICT DO UPDATE" support.
package postgresstore

import (
	"database/sql"
	"encoding/json"

	"github.com/lib/pq"
	"github.com/stratumn/go/cs"
	"github.com/stratumn/go/store"
	"github.com/stratumn/go/types"
)

const (
	// Name is the name set in the store's information.
	Name = "postgres"

	// Description is the description set in the store's information.
	Description = "Stratumn PostgreSQL Store"

	// DefaultURL is the default URL of the database.
	DefaultURL = "postgres://postgres@postgres/postgres?sslmode=disable"
)

const notFoundError = "sql: no rows in result set"

// Config contains configuration options for the store.
type Config struct {
	// A version string that will be set in the store's information.
	Version string

	// A git commit hash that will be set in the store's information.
	Commit string

	// The URL of the PostgreSQL database, such as
	// "postgres://postgres@localhost/store?sslmode=disable".
	URL string
}

// Info is the info returned by GetInfo.
type Info struct {
	Name        string `json:"name"`
	Description string `json:"description"`
	Version     string `json:"version"`
	Commit      string `json:"commit"`
}

// Store is the type that implements github.com/stratumn/go/store.Adapter.
type Store struct {
	config       *Config
	didSaveChans []chan *cs.Segment
	db           *sql.DB
	stmts        *stmts
}

// New creates an instance of a Store.
func New(config *Config) (*Store, error) {
	db, err := sql.Open("postgres", config.URL)
	if err != nil {
		return nil, err
	}
	return &Store{config: config, db: db}, nil
}

// AddDidSaveChannel implements
// github.com/stratumn/go/fossilizer.Store.AddDidSaveChannel.
func (a *Store) AddDidSaveChannel(saveChan chan *cs.Segment) {
	a.didSaveChans = append(a.didSaveChans, saveChan)
}

// GetInfo implements github.com/stratumn/go/store.Adapter.GetInfo.
func (a *Store) GetInfo() (interface{}, error) {
	return &Info{
		Name:        Name,
		Description: Description,
		Version:     a.config.Version,
		Commit:      a.config.Commit,
	}, nil
}

// SaveSegment implements github.com/stratumn/go/store.Adapter.SaveSegment.
func (a *Store) SaveSegment(segment *cs.Segment) error {
	var (
		err          error
		linkHash     = segment.GetLinkHash()
		priority     = segment.Link.GetPriority()
		mapID        = segment.Link.GetMapID()
		prevLinkHash = segment.Link.GetPrevLinkHash()
		tags         = segment.Link.GetTags()
	)

	data, err := json.Marshal(segment)
	if err != nil {
		return err
	}

	if prevLinkHash == nil {
		_, err = a.stmts.SaveSegment.Exec(linkHash[:], priority, mapID, nil, pq.Array(tags), string(data))
	} else {
		_, err = a.stmts.SaveSegment.Exec(linkHash[:], priority, mapID, prevLinkHash[:], pq.Array(tags), string(data))
	}

	if err != nil {
		return err
	}

	// Send saved segment to all the save channels without blocking.
	go func(chans []chan *cs.Segment) {
		for _, c := range chans {
			c <- segment
		}
	}(a.didSaveChans)

	return nil
}

// GetSegment implements github.com/stratumn/go/store.Adapter.GetSegment.
func (a *Store) GetSegment(linkHash *types.Bytes32) (*cs.Segment, error) {
	var data string

	if err := a.stmts.GetSegment.QueryRow(linkHash[:]).Scan(&data); err != nil {
		if err.Error() == notFoundError {
			return nil, nil
		}
		return nil, err
	}

	var segment cs.Segment
	if err := json.Unmarshal([]byte(data), &segment); err != nil {
		return nil, err
	}

	return &segment, nil
}

// DeleteSegment implements github.com/stratumn/go/store.Adapter.DeleteSegment.
func (a *Store) DeleteSegment(linkHash *types.Bytes32) (*cs.Segment, error) {
	var (
		data    string
		segment cs.Segment
	)

	if err := a.stmts.DeleteSegment.QueryRow(linkHash[:]).Scan(&data); err != nil {
		if err.Error() == notFoundError {
			return nil, nil
		}
		return nil, err
	}

	if err := json.Unmarshal([]byte(data), &segment); err != nil {
		return nil, err
	}

	return &segment, nil
}

// FindSegments implements github.com/stratumn/go/store.Adapter.FindSegments.
func (a *Store) FindSegments(filter *store.Filter) (cs.SegmentSlice, error) {
	var (
		rows     *sql.Rows
		err      error
		limit    = filter.Limit
		offset   = filter.Offset
		segments = make(cs.SegmentSlice, 0, limit)
	)

	if filter.PrevLinkHash != nil {
		prevLinkHash := filter.PrevLinkHash[:]
		if len(filter.Tags) > 0 {
			tags := pq.Array(filter.Tags)
			rows, err = a.stmts.FindSegmentsWithPrevLinkHashAndTags.Query(prevLinkHash, tags, offset, limit)
		} else {
			rows, err = a.stmts.FindSegmentsWithPrevLinkHash.Query(prevLinkHash, offset, limit)
		}
	} else if mapID := filter.MapID; mapID != "" {
		if len(filter.Tags) > 0 {
			tags := pq.Array(filter.Tags)
			rows, err = a.stmts.FindSegmentsWithMapIDAndTags.Query(mapID, tags, offset, limit)
		} else {
			rows, err = a.stmts.FindSegmentsWithMapID.Query(mapID, offset, limit)
		}
	} else if len(filter.Tags) > 0 {
		tags := pq.Array(filter.Tags)
		rows, err = a.stmts.FindSegmentsWithTags.Query(tags, offset, limit)
	} else {
		rows, err = a.stmts.FindSegments.Query(offset, limit)
	}

	if err != nil {
		return nil, err
	}

	defer rows.Close()

	for rows.Next() {
		var (
			data    string
			segment cs.Segment
		)

		if err = rows.Scan(&data); err != nil {
			return nil, err
		}

		if err = json.Unmarshal([]byte(data), &segment); err != nil {
			return nil, err
		}

		segments = append(segments, &segment)
	}

	return segments, nil
}

// GetMapIDs implements github.com/stratumn/go/store.Adapter.GetMapIDs.
func (a *Store) GetMapIDs(pagination *store.Pagination) ([]string, error) {
	rows, err := a.stmts.GetMapIDs.Query(pagination.Offset, pagination.Limit)
	if err != nil {
		return nil, err
	}

	defer rows.Close()

	mapIDs := make([]string, 0, pagination.Limit)

	for rows.Next() {
		var mapID string
		if err = rows.Scan(&mapID); err != nil {
			return nil, err
		}

		mapIDs = append(mapIDs, mapID)
	}

	return mapIDs, nil
}

// Create creates the database tables and indexes.
func (a *Store) Create() error {
	for _, query := range sqlCreate {
		if _, err := a.db.Exec(query); err != nil {
			return err
		}
	}
	return nil
}

// Prepare prepares the database stmts.
// It should be called once before interacting with segments.
// It assumes the tables have been created using Create().
func (a *Store) Prepare() (err error) {
	a.stmts, err = newStmts(a.db)
	return
}

// Drop drops the database tables and indexes.
func (a *Store) Drop() error {
	for _, query := range sqlDrop {
		if _, err := a.db.Exec(query); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package lcs

import (
	"bufio"
	"io"
	"regexp"
	"strings"

	"sync"
)

type prose interface {
	new() prose
	len() int
	word(i int) word
	slice(i, j int) prose
	append(op prose)
	// prepend(op prose)
	appendWord(w word)
	// prependWord(w word)
	wrapDel()
	wrapIns()
}

type word interface {
	isEqual(ow word) bool
	indexIn(p prose, i int) int
}

type article struct {
	id int
	// origin string
	terms []*term
}

var (
	// puncs = ",!;:\"?.\n"
	puncs = regexp.MustCompile("[,!;:\"?\\.]$")
)

func newArticle(ori io.Reader) *article {
	a := article{id: genSpId()}
	scanner := bufio.NewScanner(ori)
	scanner.Split(scanWords)

	for scanner.Scan() {
		w := scanner.Text()
		a.terms = append(a.terms, distillTerms(w)...)
	}
	return &a
}

func distillTerms(w string) (r []*term) {
	if i := strings.Index(w, "\n"); i != -1 {
		if i == 0 {
			r = append(r, newTerm(string(w[i])))
		} else {
			r = append(r, distillTerms(w[:i])...)
			r = append(r, newTerm(string(w[i])))
		}

		if i+1 < len(w) {
			r = append(r, distillTerms(w[i+1:])...)
		}

		return
	}

	terms := []*term{}

	for puncs.MatchString(w) {
		terms = append(terms, newTerm(string(w[len(w)-1])))
		w = w[:len(w)-1]
	}

	if w != "" {
		terms = append(terms, newTerm(w))
	}

	for i, _ := range terms {
		r = append(r, terms[len(terms)-i-1])
	}

	return
}

func (a *article) String() (str string) {
	for i, term := range a.terms {
		if !term.isPuncs() {
			if i == 0 {
				if term.isLF() || a.terms[i+1].isLF() {
					goto appendTerm
				}
			} else {
				if a.terms[i-1].isLF() || a.terms[i+1].isLF() {
					goto appendTerm
				}

				if !a.terms[i-1].isOpenMarks() && !term.isCloseMarks() {
					str += " "
				}
			}
		}
	appendTerm:
		str += term.string
	}

	return
}

func (t *term) isPuncs() bool {
	return puncs.MatchString(t.string)
}

func (t term) isOpenMarks() bool {
	return t.string == "[" || t.string == "("
}

func (t term) isLF() bool {
	return t.string == "\n"
}

func (t term) isCloseMarks() bool {
	return t.string == "]" || t.string == ")"
}

func (a article) new() prose {
	return &article{id: genSpId()}
}

func (a article) len() int {
	return len(a.terms)
}

func (a article) word(i int) word {
	return a.terms[i]
}

func (a article) slice(i, j int) prose {
	na := new(article)
	na.id = a.id
	na.terms = a.terms[i:j]
	return na
}

func (a *article) append(op prose) {
	oa := op.(*article)
	// a.origin += oa.origin
	a.terms = append(a.terms, oa.terms...)
}

func (a *article) prepend(op prose) {
	oa := op.(*article)
	// a.origin = oa.origin + a.origin
	a.terms = append(oa.terms, a.terms...)
}

func (a *article) appendWord(w word) {
	t := w.(*term)
	// a.origin += t.string
	a.terms = append(a.terms, t)
}

func (a *article) prependWord(w word) {
	t := w.(*term)
	// a.origin = t.string + a.origin
	a.terms = append([]*term{t}, a.terms...)
}

func (a *article) wrapDel() {
	a.appendWord(newTerm("]"))
	// a.origin = "[" + a.origin
	a.terms = append([]*term{newTerm("[")}, a.terms...)
}

func (a *article) wrapIns() {
	a.appendWord(newTerm(")"))
	// a.origin = "(" + a.origin
	a.terms = append([]*term{newTerm("(")}, a.terms...)
}

type term struct {
	string
	pos map[int]int
}

func newTerm(cont string) *term {
	return &term{string: cont, pos: map[int]int{}}
}

func (t term) isEqual(ow word) bool {
	return t.string == ow.(*term).string
}

func (t *term) indexIn(p prose, i int) int {
	if i == -1 {
		return t.pos[p.(*article).id]
	}
	t.pos[p.(*article).id] = i
	return i
}

func (t *term) String() string {
	return t.string
}

type stringProse struct {
	// string
	words []*byteWord
	id    int
}

var (
	spId     = 0
	spIdLock = sync.Mutex{}
)

func genSpId() int {
	spIdLock.Lock()
	defer spIdLock.Unlock()
	spId += 1

	return spId
}

func newStringProse(content string) (sp *stringProse) {
	sp = new(stringProse)
	sp.id = genSpId()
	for _, b := range content {
		w := newByteWord(byte(b))
		// w.indexIn(sp, i)
		sp.words = append(sp.words, w)
	}

	return
}

func (s stringProse) len() int {
	// return len(s.string)
	return len(s.words)
}

func (s stringProse) word(i int) word {
	return s.words[i]
}

func (s stringProse) slice(i, j int) prose {
	// sp := stringProse(s[i:j])
	// return &stringProse{s.string[i:j], genSpId()}
	// p.words = make([]byteWord, j-i)
	// copy(p.words, s.words[i:j])
	p := newStringProse(s.sliceString(i, j))
	p.id = s.id
	return p
}

func (s stringProse) sliceString(i, j int) (str string) {
	for _, w := range s.words[i:j] {
		str += string(w.byte)
	}
	return
}

func (s *stringProse) append(op prose) {
	// s.string += op.(*stringProse).string
	s.words = append(s.words, op.(*stringProse).words...)
}

func (s *stringProse) prepend(op prose) {
	// *s = stringProse(string(*op.(*stringProse)) + string(*s))
	// s.string = op.(*stringProse).string + s.string
	s.words = append(op.(*stringProse).words, s.words...)
}

func (s *stringProse) prependWord(w word) {
	// *s = stringProse(string(byte(w.(byteWord).byte)) + string(*s))
	// s.string = string(w.(*byteWord).byte) + s.string
	s.words = append([]*byteWord{w.(*byteWord)}, s.words...)
}

func (s *stringProse) appendWord(w word) {
	s.words = append(s.words, w.(*byteWord))
}

func (s stringProse) new() prose {
	// np := stringProse("")
	return &stringProse{[]*byteWord{}, genSpId()}
}

func (s *stringProse) wrapDel() {
	// s.string = "[" + s.string + "]"
	s.prependWord(newByteWord('['))
	s.appendWord(newByteWord(']'))
}

func (s *stringProse) wrapIns() {
	// s.string = "(" + s.string + ")"
	s.prependWord(newByteWord('('))
	s.appendWord(newByteWord(')'))
}

func (s stringProse) String() (str string) {
	for _, w := range s.words {
		str += string(w.byte)
	}
	return
}

type byteWord struct {
	byte byte
	pos  map[int]int
}

func newByteWord(b byte) (bw *byteWord) {
	bw = new(byteWord)
	bw.byte = b
	bw.pos = map[int]int{}
	return
}

func (b byteWord) isEqual(ow word) bool {
	return byte(b.byte) == byte(ow.(*byteWord).byte)
}

func (b *byteWord) indexIn(p prose, i int) int {
	if i == -1 {
		return b.pos[p.(*stringProse).id]
	}
	b.pos[p.(*stringProse).id] = i
	return i
}

func (b byteWord) String() string {
	return string(b.byte)
}

func LCS(a, b prose) prose {
	aLen := a.len()
	bLen := b.len()
	lengths := make([][]int, aLen+1)
	for i := 0; i <= aLen; i++ {
		lengths[i] = make([]int, bLen+1)
	}

	// row 0 and column 0 are initialized to 0 already
	for i := 0; i < aLen; i++ {
		for j := 0; j < bLen; j++ {
			if a.word(i).isEqual(b.word(j)) {
				lengths[i+1][j+1] = lengths[i][j] + 1
			} else if lengths[i+1][j] > lengths[i][j+1] {
				lengths[i+1][j+1] = lengths[i+1][j]
			} else {
				lengths[i+1][j+1] = lengths[i][j+1]
			}
		}
	}

	// for _, r := range lengths {
	// 	fmt.Println(r)
	// }

	// read the substring out from the matrix
	s := a.new()
	for x, y := aLen, bLen; x != 0 && y != 0; {
		if lengths[x][y] == lengths[x-1][y] {
			x--
		} else if lengths[x][y] == lengths[x][y-1] {
			y--
		} else {
			w := a.word(x - 1)
			w.indexIn(a, x-1)
			w.indexIn(b, y-1)
			s.appendWord(w)
			x--
			y--
		}
	}

	// reverse string
	r := a.new()
	for i := 0; i < s.len(); i++ {
		r.appendWord(s.word(s.len() - 1 - i))
	}
	return r
}

func Diff(ori, edit, lcs prose) prose {
	// println("---->")
	// fmt.Println(ori, edit, lcs)
	if lcs.len() == 0 {
		ori.wrapDel()
		edit.wrapIns()
		ori.append(edit)
		return ori
	}

	diff := ori.new()
	lastIOri := -1
	lastIEdit := -1
	lenOri := ori.len()
	lenEdit := edit.len()
	lenLCS := lcs.len()
	for i := 0; i < lenLCS; i++ {
		w := lcs.word(i)
		oi := w.indexIn(ori, -1)
		if lenOri > i && oi != i {
			// deletion
			// fmt.Printf("--> %+v\n", w.(*term).pos)
			// println(w.(*term).string)
			// println(lastIOri+1, oi)
			// fmt.Println(ori.(*article).id)
			del := ori.slice(lastIOri+1, oi)
			if del.len() > 0 {
				del.wrapDel()
				diff.append(del)
			}
		}
		lastIOri = oi
		ei := w.indexIn(edit, -1)
		if lenEdit > i && ei != i {
			// insertion
			ins := edit.slice(lastIEdit+1, ei)
			if ins.len() > 0 {
				ins.wrapIns()
				diff.append(ins)
			}
		}
		diff.appendWord(w)
		lastIEdit = ei
	}

	// deletion and insertion after last word of ori and edit
	lastw := lcs.word(lcs.len() - 1)
	lastIOri = lastw.indexIn(ori, -1)
	if lenOri > lastIOri+1 {
		del := ori.slice(lastIOri+1, lenOri)
		if del.len() > 0 {
			del.wrapDel()
			diff.append(del)
		}
	}
	lastIEdit = lastw.indexIn(edit, -1)
	if lenEdit > lastIEdit+1 {
		ins := edit.slice(lastIEdit+1, lenEdit)
		if ins.len() > 0 {
			ins.wrapIns()
			diff.append(ins)
		}
	}

	return diff
}
<commit_msg>expose prose, word interfaces<commit_after>package lcs

import (
	"bufio"
	"io"
	"regexp"
	"strings"

	"sync"
)

type Prose interface {
	New() Prose
	Len() int
	Word(i int) Word
	Slice(i, j int) Prose
	Append(op Prose)
	AppendWord(w Word)
	WrapDel()
	WrapIns()
}

type Word interface {
	IsEqual(ow Word) bool
	IndexIn(p Prose, i int) int
}

type article struct {
	id    int
	terms []*term
}

var puncs = regexp.MustCompile("[,!;:\"?\\.]$")

func newArticle(ori io.Reader) *article {
	a := article{id: genSpId()}
	scanner := bufio.NewScanner(ori)
	scanner.Split(scanWords)

	for scanner.Scan() {
		w := scanner.Text()
		a.terms = append(a.terms, distillTerms(w)...)
	}
	return &a
}

func distillTerms(w string) (r []*term) {
	if i := strings.Index(w, "\n"); i != -1 {
		if i == 0 {
			r = append(r, newTerm(string(w[i])))
		} else {
			r = append(r, distillTerms(w[:i])...)
			r = append(r, newTerm(string(w[i])))
		}

		if i+1 < len(w) {
			r = append(r, distillTerms(w[i+1:])...)
		}

		return
	}

	terms := []*term{}

	for puncs.MatchString(w) {
		terms = append(terms, newTerm(string(w[len(w)-1])))
		w = w[:len(w)-1]
	}

	if w != "" {
		terms = append(terms, newTerm(w))
	}

	for i, _ := range terms {
		r = append(r, terms[len(terms)-i-1])
	}

	return
}

func (a *article) String() (str string) {
	for i, term := range a.terms {
		if !term.isPuncs() {
			if i == 0 {
				if term.isLF() || a.terms[i+1].isLF() {
					goto appendTerm
				}
			} else {
				if a.terms[i-1].isLF() || a.terms[i+1].isLF() {
					goto appendTerm
				}

				if !a.terms[i-1].isOpenMarks() && !term.isCloseMarks() {
					str += " "
				}
			}
		}
	appendTerm:
		str += term.string
	}

	return
}

func (t *term) isPuncs() bool {
	return puncs.MatchString(t.string)
}

func (t term) isOpenMarks() bool {
	return t.string == "[" || t.string == "("
}

func (t term) isLF() bool {
	return t.string == "\n"
}

func (t term) isCloseMarks() bool {
	return t.string == "]" || t.string == ")"
}

func (a article) New() Prose {
	return &article{id: genSpId()}
}

func (a article) Len() int {
	return len(a.terms)
}

func (a article) Word(i int) Word {
	return a.terms[i]
}

func (a article) Slice(i, j int) Prose {
	na := new(article)
	na.id = a.id
	na.terms = a.terms[i:j]
	return na
}

func (a *article) Append(op Prose) {
	oa := op.(*article)
	// a.origin += oa.origin
	a.terms = append(a.terms, oa.terms...)
}

func (a *article) Prepend(op Prose) {
	oa := op.(*article)
	// a.origin = oa.origin + a.origin
	a.terms = append(oa.terms, a.terms...)
}

func (a *article) AppendWord(w Word) {
	t := w.(*term)
	// a.origin += t.string
	a.terms = append(a.terms, t)
}

func (a *article) PrependWord(w Word) {
	t := w.(*term)
	// a.origin = t.string + a.origin
	a.terms = append([]*term{t}, a.terms...)
}

func (a *article) WrapDel() {
	a.AppendWord(newTerm("]"))
	// a.origin = "[" + a.origin
	a.terms = append([]*term{newTerm("[")}, a.terms...)
}

func (a *article) WrapIns() {
	a.AppendWord(newTerm(")"))
	// a.origin = "(" + a.origin
	a.terms = append([]*term{newTerm("(")}, a.terms...)
}

type term struct {
	string
	pos map[int]int
}

func newTerm(cont string) *term {
	return &term{string: cont, pos: map[int]int{}}
}

func (t term) IsEqual(ow Word) bool {
	return t.string == ow.(*term).string
}

func (t *term) IndexIn(p Prose, i int) int {
	if i == -1 {
		return t.pos[p.(*article).id]
	}
	t.pos[p.(*article).id] = i
	return i
}

func (t *term) String() string {
	return t.string
}

type stringProse struct {
	// string
	words []*byteWord
	id    int
}

var (
	spId     = 0
	spIdLock = sync.Mutex{}
)

func genSpId() int {
	spIdLock.Lock()
	defer spIdLock.Unlock()
	spId += 1

	return spId
}

func newStringProse(content string) (sp *stringProse) {
	sp = new(stringProse)
	sp.id = genSpId()
	for _, b := range content {
		w := newByteWord(byte(b))
		// w.indexIn(sp, i)
		sp.words = append(sp.words, w)
	}

	return
}

func (s stringProse) Len() int {
	return len(s.words)
}

func (s stringProse) Word(i int) Word {
	return s.words[i]
}

func (s stringProse) Slice(i, j int) Prose {
	p := newStringProse(s.SliceString(i, j))
	p.id = s.id
	return p
}

func (s stringProse) SliceString(i, j int) (str string) {
	for _, w := range s.words[i:j] {
		str += string(w.byte)
	}
	return
}

func (s *stringProse) Append(op Prose) {
	s.words = append(s.words, op.(*stringProse).words...)
}

func (s *stringProse) Prepend(op Prose) {
	s.words = append(op.(*stringProse).words, s.words...)
}

func (s *stringProse) PrependWord(w Word) {
	s.words = append([]*byteWord{w.(*byteWord)}, s.words...)
}

func (s *stringProse) AppendWord(w Word) {
	s.words = append(s.words, w.(*byteWord))
}

func (s stringProse) New() Prose {
	return &stringProse{[]*byteWord{}, genSpId()}
}

func (s *stringProse) WrapDel() {
	s.PrependWord(newByteWord('['))
	s.AppendWord(newByteWord(']'))
}

func (s *stringProse) WrapIns() {
	s.PrependWord(newByteWord('('))
	s.AppendWord(newByteWord(')'))
}

func (s stringProse) String() (str string) {
	for _, w := range s.words {
		str += string(w.byte)
	}
	return
}

type byteWord struct {
	byte byte
	pos  map[int]int
}

func newByteWord(b byte) (bw *byteWord) {
	bw = new(byteWord)
	bw.byte = b
	bw.pos = map[int]int{}
	return
}

func (b byteWord) IsEqual(ow Word) bool {
	return byte(b.byte) == byte(ow.(*byteWord).byte)
}

func (b *byteWord) IndexIn(p Prose, i int) int {
	if i == -1 {
		return b.pos[p.(*stringProse).id]
	}
	b.pos[p.(*stringProse).id] = i
	return i
}

func (b byteWord) String() string {
	return string(b.byte)
}

func LCS(a, b Prose) Prose {
	aLen := a.Len()
	bLen := b.Len()
	lengths := make([][]int, aLen+1)
	for i := 0; i <= aLen; i++ {
		lengths[i] = make([]int, bLen+1)
	}

	// row 0 and column 0 are initialized to 0 already
	for i := 0; i < aLen; i++ {
		for j := 0; j < bLen; j++ {
			if a.Word(i).IsEqual(b.Word(j)) {
				lengths[i+1][j+1] = lengths[i][j] + 1
			} else if lengths[i+1][j] > lengths[i][j+1] {
				lengths[i+1][j+1] = lengths[i+1][j]
			} else {
				lengths[i+1][j+1] = lengths[i][j+1]
			}
		}
	}

	// for _, r := range lengths {
	// 	fmt.Println(r)
	// }

	// read the substring out from the matrix
	s := a.New()
	for x, y := aLen, bLen; x != 0 && y != 0; {
		if lengths[x][y] == lengths[x-1][y] {
			x--
		} else if lengths[x][y] == lengths[x][y-1] {
			y--
		} else {
			w := a.Word(x - 1)
			w.IndexIn(a, x-1)
			w.IndexIn(b, y-1)
			s.AppendWord(w)
			x--
			y--
		}
	}

	// reverse string
	r := a.New()
	for i := 0; i < s.Len(); i++ {
		r.AppendWord(s.Word(s.Len() - 1 - i))
	}
	return r
}

func Diff(ori, edit, lcs Prose) Prose {
	// println("---->")
	// fmt.Println(ori, edit, lcs)
	if lcs.Len() == 0 {
		ori.WrapDel()
		edit.WrapIns()
		ori.Append(edit)
		return ori
	}

	diff := ori.New()
	lastIOri := -1
	lastIEdit := -1
	lenOri := ori.Len()
	lenEdit := edit.Len()
	lenLCS := lcs.Len()
	for i := 0; i < lenLCS; i++ {
		w := lcs.Word(i)
		oi := w.IndexIn(ori, -1)
		if lenOri > i && oi != i {
			// deletion
			// fmt.Printf("--> %+v\n", w.(*term).pos)
			// println(w.(*term).string)
			// println(lastIOri+1, oi)
			// fmt.Println(ori.(*article).id)
			del := ori.Slice(lastIOri+1, oi)
			if del.Len() > 0 {
				del.WrapDel()
				diff.Append(del)
			}
		}
		lastIOri = oi
		ei := w.IndexIn(edit, -1)
		if lenEdit > i && ei != i {
			// insertion
			ins := edit.Slice(lastIEdit+1, ei)
			if ins.Len() > 0 {
				ins.WrapIns()
				diff.Append(ins)
			}
		}
		diff.AppendWord(w)
		lastIEdit = ei
	}

	// deletion and insertion after last word of ori and edit
	lastw := lcs.Word(lcs.Len() - 1)
	lastIOri = lastw.IndexIn(ori, -1)
	if lenOri > lastIOri+1 {
		del := ori.Slice(lastIOri+1, lenOri)
		if del.Len() > 0 {
			del.WrapDel()
			diff.Append(del)
		}
	}
	lastIEdit = lastw.IndexIn(edit, -1)
	if lenEdit > lastIEdit+1 {
		ins := edit.Slice(lastIEdit+1, lenEdit)
		if ins.Len() > 0 {
			ins.WrapIns()
			diff.Append(ins)
		}
	}

	return diff
}
<|endoftext|>


<commit_before>package commands

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"

	"github.com/inconshreveable/go-update"
	"github.com/tedsuo/rata"

	"github.com/concourse/atc"
	"github.com/concourse/fly/rc"
)

type SyncCommand struct{}

var syncCommand SyncCommand

func init() {
	sync, err := Parser.AddCommand(
		"sync",
		"Download and replace the current fly from the target",
		"",
		&syncCommand,
	)
	if err != nil {
		panic(err)
	}

	sync.Aliases = []string{"s"}
}

func (command *SyncCommand) Execute(args []string) error {
	target, err := rc.SelectTarget(globalOptions.Target, globalOptions.Insecure)
	if err != nil {
		log.Fatalln(err)
		return nil
	}

	reqGenerator := rata.NewRequestGenerator(target.URL(), atc.Routes)

	request, err := reqGenerator.CreateRequest(
		atc.DownloadCLI, rata.Params{}, nil,
	)
	if err != nil {
		fmt.Printf("building request failed: %v\n", err)
		os.Exit(1)
	}

	request.URL.RawQuery = url.Values{
		"arch":     []string{runtime.GOARCH},
		"platform": []string{runtime.GOOS},
	}.Encode()

	tlsConfig := &tls.Config{InsecureSkipVerify: target.Insecure}

	transport := &http.Transport{TLSClientConfig: tlsConfig}

	client := &http.Client{Transport: transport}

	updateCustom := &update.Update{HTTPClient: client}

	fmt.Printf("downloading fly from %s... ", request.URL.Host)

	err, errRecover := updateCustom.FromUrl(request.URL.String())
	if err != nil {
		fmt.Printf("update failed: %v\n", err)
		if errRecover != nil {
			fmt.Printf("failed to recover previous executable: %v!\n", errRecover)
			fmt.Printf("things are probably in a bad state on your machine now.\n")
		}

		return err
	}

	fmt.Println("update successful!")
	return nil
}
<commit_msg>fix for new go-update interfaces<commit_after>package commands

import (
	"crypto/tls"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"

	"github.com/inconshreveable/go-update"
	"github.com/tedsuo/rata"

	"github.com/concourse/atc"
	"github.com/concourse/fly/rc"
)

type SyncCommand struct{}

var syncCommand SyncCommand

func init() {
	sync, err := Parser.AddCommand(
		"sync",
		"Download and replace the current fly from the target",
		"",
		&syncCommand,
	)
	if err != nil {
		panic(err)
	}

	sync.Aliases = []string{"s"}
}

func (command *SyncCommand) Execute(args []string) error {
	target, err := rc.SelectTarget(globalOptions.Target, globalOptions.Insecure)
	if err != nil {
		log.Fatalln(err)
		return nil
	}

	reqGenerator := rata.NewRequestGenerator(target.URL(), atc.Routes)

	request, err := reqGenerator.CreateRequest(
		atc.DownloadCLI, rata.Params{}, nil,
	)
	if err != nil {
		fmt.Printf("building request failed: %v\n", err)
		os.Exit(1)
	}

	request.URL.RawQuery = url.Values{
		"arch":     []string{runtime.GOARCH},
		"platform": []string{runtime.GOOS},
	}.Encode()

	tlsConfig := &tls.Config{InsecureSkipVerify: target.Insecure}

	transport := &http.Transport{TLSClientConfig: tlsConfig}

	client := &http.Client{Transport: transport}

	response, err := client.Do(request)
	if err != nil {
		log.Fatalln(err)
	}

	if response.StatusCode != http.StatusOK {
		failf("bad response: %s", response.Status)
	}

	fmt.Printf("downloading fly from %s... ", request.URL.Host)

	err = update.Apply(response.Body, update.Options{})
	if err != nil {
		failf("update failed: %s", err)
	}

	fmt.Println("update successful!")
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package common

import (
	"bytes"
	"encoding/binary"
	"io"
)

const (
	EBHeaderSize = 140 // 32+32+32+32+4+4+4
)

// EBlock is the Entry Block. It holds the hashes of the Entries and its Merkel
// Root is written into the Directory Blocks. Each Entry Block represents all
// of the entries for a paticular Chain during a 10 minute period.
type EBlock struct {
	Header *EBlockHeader
	Body   *EBlockBody
}

func MakeEBlock(echain *EChain, prev *EBlock) *EBlock {
	e := NewEBlock()
	e.Header.ChainID = echain.ChainID
	if prev != nil {
		e.Header.PrevKeyMR = prev.KeyMR()
		e.Header.PrevFullHash = prev.Hash()
	}
	e.Header.EBSequence = echain.NextBlockHeight
	return e
}

// NewEBlock returns a blank initialized Entry Block
func NewEBlock() *EBlock {
	e := new(EBlock)
	e.Header = NewEBlockHeader()
	e.Body = NewEBlockBody()
	return e
}

// AddEBEntry
func (e *EBlock) AddEBEntry(entry *Entry) error {
	e.Body.EBEntries = append(e.Body.EBEntries, entry.Hash())
	return nil
}

func (e *EBlock) AddEndOfMinuteMarker(m byte) {
	h := make([]byte, 32)
	h[len(h)-1] = m
	hash := NewHash()
	hash.SetBytes(h)
	e.Body.EBEntries = append(e.Body.EBEntries, hash)
}

// BuildHeader updates the Entry Block Header to include information about the
// Entry Block Body. BuildHeader should be run after the Entry Block Body has
// included all of its Entries.
func (e *EBlock) BuildHeader() error {
	e.Header.BodyMR = e.Body.MR()
	e.Header.EntryCount = uint32(len(e.Body.EBEntries))
	return nil
}

func (e *EBlock) Hash() *Hash {
	p, err := e.MarshalBinary()
	if err != nil {
		return NewHash()
	}
	return Sha(p)
}

func (e *EBlock) KeyMR() *Hash {
	// h(h(header) + BodyMR)
	e.BuildHeader()
	header, err := e.Header.MarshalBinary()
	if err != nil {
		return NewHash()
	}
	h1 := Sha(header)
	return Sha(append(h1.Bytes(), e.Header.BodyMR.Bytes()...))
}

func (e *EBlock) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)

	if err := e.BuildHeader(); err != nil {
		return buf.Bytes(), err
	}
	if p, err := e.Header.MarshalBinary(); err != nil {
		return buf.Bytes(), err
	} else {
		buf.Write(p)
	}

	if p, err := e.Body.MarshalBinary(); err != nil {
		return buf.Bytes(), err
	} else {
		buf.Write(p)
	}

	return buf.Bytes(), nil
}

func (e *EBlock) UnmarshalBinary(data []byte) error {
	buf := bytes.NewBuffer(data)

	if err := e.Header.UnmarshalBinary(buf.Next(EBHeaderSize)); err != nil {
		return err
	}

	if err := e.Body.UnmarshalBinary(buf.Bytes()); err != nil {
		return err
	}

	return nil
}

type EBlockBody struct {
	EBEntries []*Hash
}

func NewEBlockBody() *EBlockBody {
	e := new(EBlockBody)
	e.EBEntries = make([]*Hash, 0)
	return e
}

func (e *EBlockBody) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)

	for _, v := range e.EBEntries {
		buf.Write(v.Bytes())
	}

	return buf.Bytes(), nil
}

func (e *EBlockBody) MR() *Hash {
	mrs := BuildMerkleTreeStore(e.EBEntries)
	r := mrs[len(mrs)-1]
	return r
}

func (e *EBlockBody) UnmarshalBinary(data []byte) error {
	buf := bytes.NewBuffer(data)
	hash := make([]byte, 32)

	for {
		if _, err := buf.Read(hash); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}

		h := NewHash()
		h.SetBytes(hash)
		e.EBEntries = append(e.EBEntries, h)
	}

	return nil
}

type EBlockHeader struct {
	ChainID      *Hash
	BodyMR       *Hash
	PrevKeyMR    *Hash
	PrevFullHash *Hash
	EBSequence   uint32
	DBHeight     uint32
	EntryCount   uint32
}

func NewEBlockHeader() *EBlockHeader {
	e := new(EBlockHeader)
	e.ChainID = NewHash()
	e.BodyMR = NewHash()
	e.PrevKeyMR = NewHash()
	e.PrevFullHash = NewHash()
	return e
}

func (e *EBlockHeader) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)

	// 32 byte ChainID
	buf.Write(e.ChainID.Bytes())

	// 32 byte Body MR
	buf.Write(e.BodyMR.Bytes())

	// 32 byte Previous Key MR
	buf.Write(e.PrevKeyMR.Bytes())

	// 32 byte Previous Full Hash
	buf.Write(e.PrevFullHash.Bytes())

	if err := binary.Write(buf, binary.BigEndian, e.EBSequence); err != nil {
		return buf.Bytes(), err
	}

	if err := binary.Write(buf, binary.BigEndian, e.DBHeight); err != nil {
		return buf.Bytes(), err
	}

	if err := binary.Write(buf, binary.BigEndian, e.EntryCount); err != nil {
		return buf.Bytes(), err
	}

	return buf.Bytes(), nil
}

func (e *EBlockHeader) UnmarshalBinary(data []byte) error {
	buf := bytes.NewBuffer(data)
	hash := make([]byte, 32)

	if _, err := buf.Read(hash); err != nil {
		return err
	} else {
		e.ChainID.SetBytes(hash)
	}

	if _, err := buf.Read(hash); err != nil {
		return err
	} else {
		e.BodyMR.SetBytes(hash)
	}

	if _, err := buf.Read(hash); err != nil {
		return err
	} else {
		e.PrevKeyMR.SetBytes(hash)
	}

	if _, err := buf.Read(hash); err != nil {
		return err
	} else {
		e.PrevFullHash.SetBytes(hash)
	}

	if err := binary.Read(buf, binary.BigEndian, &e.EBSequence); err != nil {
		return err
	}

	if err := binary.Read(buf, binary.BigEndian, &e.DBHeight); err != nil {
		return err
	}

	if err := binary.Read(buf, binary.BigEndian, &e.EntryCount); err != nil {
		return err
	}

	return nil
}
<commit_msg>added documentation to eblock<commit_after>// Copyright 2015 Factom Foundation
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package common

import (
	"bytes"
	"encoding/binary"
	"io"
)

const (
	EBHeaderSize = 140 // 32+32+32+32+4+4+4
)

// EBlock is the Entry Block. It holds the hashes of the Entries and its Merkel
// Root is written into the Directory Blocks. Each Entry Block represents all
// of the entries for a paticular Chain during a 10 minute period.
type EBlock struct {
	Header *EBlockHeader
	Body   *EBlockBody
}

// MakeEBlock creates a new Entry Block belonging to the provieded Entry Chain.
// Its PrevKeyMR and PrevFullHash are populated by the provided previous Entry
// Block. If The previous Entry Block is nil (the new Entry Block is first in
// the Chain) the relevent Entry Block Header fields will contain zeroed Hashes.
func MakeEBlock(echain *EChain, prev *EBlock) *EBlock {
	e := NewEBlock()
	e.Header.ChainID = echain.ChainID
	if prev != nil {
		e.Header.PrevKeyMR = prev.KeyMR()
		e.Header.PrevFullHash = prev.Hash()
	}
	e.Header.EBSequence = echain.NextBlockHeight
	return e
}

// NewEBlock returns a blank initialized Entry Block with all of its fields
// zeroed.
func NewEBlock() *EBlock {
	e := new(EBlock)
	e.Header = NewEBlockHeader()
	e.Body = NewEBlockBody()
	return e
}

// AddEBEntry creates a new Entry Block Entry from the provided Factom Entry
// and adds it to the Entry Block Body.
func (e *EBlock) AddEBEntry(entry *Entry) error {
	e.Body.EBEntries = append(e.Body.EBEntries, entry.Hash())
	return nil
}

// AddEndOfMinuteMarker adds the End of Minute to the Entry Block. The End of
// Minut byte becomes the last byte in a 32 byte slice that is added to the
// Entry Block Body as an Entry Block Entry.
func (e *EBlock) AddEndOfMinuteMarker(m byte) {
	h := make([]byte, 32)
	h[len(h)-1] = m
	hash := NewHash()
	hash.SetBytes(h)
	e.Body.EBEntries = append(e.Body.EBEntries, hash)
}

// BuildHeader updates the Entry Block Header to include information about the
// Entry Block Body. BuildHeader should be run after the Entry Block Body has
// included all of its EntryEntries.
func (e *EBlock) BuildHeader() error {
	e.Header.BodyMR = e.Body.MR()
	e.Header.EntryCount = uint32(len(e.Body.EBEntries))
	return nil
}

// Hash returns the simple Sha256 hash of the serialized Entry Block. Hash is
// used to provide the PrevFullHash to the next Entry Block in a Chain.
func (e *EBlock) Hash() *Hash {
	p, err := e.MarshalBinary()
	if err != nil {
		return NewHash()
	}
	return Sha(p)
}

// KeyMR returns the hash of the hash of the Entry Block Header concatinated
// with the Merkle Root of the Entry Block Body. The Body Merkle Root is
// calculated by the func (e *EBlockBody) MR() which is called by the func
// (e *EBlock) BuildHeader().
func (e *EBlock) KeyMR() *Hash {
	// Sha(Sha(header) + BodyMR)
	e.BuildHeader()
	header, err := e.Header.MarshalBinary()
	if err != nil {
		return NewHash()
	}
	h := Sha(header)
	return Sha(append(h.Bytes(), e.Header.BodyMR.Bytes()...))
}

// MarshalBinary returns the serialized binary form of the Entry Block.
func (e *EBlock) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)

	if err := e.BuildHeader(); err != nil {
		return buf.Bytes(), err
	}
	if p, err := e.Header.MarshalBinary(); err != nil {
		return buf.Bytes(), err
	} else {
		buf.Write(p)
	}

	if p, err := e.Body.MarshalBinary(); err != nil {
		return buf.Bytes(), err
	} else {
		buf.Write(p)
	}

	return buf.Bytes(), nil
}

// UnmarshalBinary populates the Entry Block object from the serialized binary
// data.
func (e *EBlock) UnmarshalBinary(data []byte) error {
	buf := bytes.NewBuffer(data)

	if err := e.Header.UnmarshalBinary(buf.Next(EBHeaderSize)); err != nil {
		return err
	}

	if err := e.Body.UnmarshalBinary(buf.Bytes()); err != nil {
		return err
	}

	return nil
}

// EBlockBody is the series of Hashes that form the Entry Block Body.
type EBlockBody struct {
	EBEntries []*Hash
}

// NewEBlockBody initalizes an empty Entry Block Body.
func NewEBlockBody() *EBlockBody {
	e := new(EBlockBody)
	e.EBEntries = make([]*Hash, 0)
	return e
}

// MarshalBinary returns a serialized binary Entry Block Body
func (e *EBlockBody) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)

	for _, v := range e.EBEntries {
		buf.Write(v.Bytes())
	}

	return buf.Bytes(), nil
}

// MR calculates the Merkle Root of the Entry Block Body. See func
// BuildMerkleTreeStore(hashes []*Hash) (merkles []*Hash) in common/merkle.go.
func (e *EBlockBody) MR() *Hash {
	mrs := BuildMerkleTreeStore(e.EBEntries)
	r := mrs[len(mrs)-1]
	return r
}

// UnmarshalBinary builds the Entry Block Body from the serialized binary.
func (e *EBlockBody) UnmarshalBinary(data []byte) error {
	buf := bytes.NewBuffer(data)
	hash := make([]byte, 32)

	for {
		if _, err := buf.Read(hash); err != nil {
			if err == io.EOF {
				break
			}
			return err
		}

		h := NewHash()
		h.SetBytes(hash)
		e.EBEntries = append(e.EBEntries, h)
	}

	return nil
}

// EBlockHeader holds relevent metadata about the Entry Block and the data
// nessisary to verify the previous block in the Entry Block Chain.
type EBlockHeader struct {
	ChainID      *Hash
	BodyMR       *Hash
	PrevKeyMR    *Hash
	PrevFullHash *Hash
	EBSequence   uint32
	DBHeight     uint32
	EntryCount   uint32
}

// NewEBlockHeader initializes a new empty Entry Block Header.
func NewEBlockHeader() *EBlockHeader {
	e := new(EBlockHeader)
	e.ChainID = NewHash()
	e.BodyMR = NewHash()
	e.PrevKeyMR = NewHash()
	e.PrevFullHash = NewHash()
	return e
}

// MarshalBinary returns a serialized binary Entry Block Header
func (e *EBlockHeader) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)

	// 32 byte ChainID
	buf.Write(e.ChainID.Bytes())

	// 32 byte Body MR
	buf.Write(e.BodyMR.Bytes())

	// 32 byte Previous Key MR
	buf.Write(e.PrevKeyMR.Bytes())

	// 32 byte Previous Full Hash
	buf.Write(e.PrevFullHash.Bytes())

	if err := binary.Write(buf, binary.BigEndian, e.EBSequence); err != nil {
		return buf.Bytes(), err
	}

	if err := binary.Write(buf, binary.BigEndian, e.DBHeight); err != nil {
		return buf.Bytes(), err
	}

	if err := binary.Write(buf, binary.BigEndian, e.EntryCount); err != nil {
		return buf.Bytes(), err
	}

	return buf.Bytes(), nil
}

// UnmarshalBinary builds the Entry Block Header from the serialized binary.
func (e *EBlockHeader) UnmarshalBinary(data []byte) error {
	buf := bytes.NewBuffer(data)
	hash := make([]byte, 32)

	if _, err := buf.Read(hash); err != nil {
		return err
	} else {
		e.ChainID.SetBytes(hash)
	}

	if _, err := buf.Read(hash); err != nil {
		return err
	} else {
		e.BodyMR.SetBytes(hash)
	}

	if _, err := buf.Read(hash); err != nil {
		return err
	} else {
		e.PrevKeyMR.SetBytes(hash)
	}

	if _, err := buf.Read(hash); err != nil {
		return err
	} else {
		e.PrevFullHash.SetBytes(hash)
	}

	if err := binary.Read(buf, binary.BigEndian, &e.EBSequence); err != nil {
		return err
	}

	if err := binary.Read(buf, binary.BigEndian, &e.DBHeight); err != nil {
		return err
	}

	if err := binary.Read(buf, binary.BigEndian, &e.EntryCount); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package redis

import (
	"reflect"
	"testing"
	"time"

	"github.com/alicebob/miniredis"
	"github.com/kylelemons/godebug/pretty"
	goredis "gopkg.in/redis.v5"

	"github.com/yuuki/diamondb/lib/config"
	"github.com/yuuki/diamondb/lib/series"
)

func NewTestRedis(t *testing.T) {
	tests := []struct {
		desc         string
		in           []string
		expectedType reflect.Type
	}{
		{
			"redis not cluster",
			[]string{"dummy:6379"},
			reflect.TypeOf((*goredis.Client)(nil)),
		},
		{
			"redis cluster",
			[]string{"dummy01:6379", "dummy02:6379"},
			reflect.TypeOf((*goredis.Client)(nil)),
		},
	}
	for _, tc := range tests {
		config.Config.RedisAddrs = tc.in
		r := NewRedis()
		if v := reflect.TypeOf(r.client); v != tc.expectedType {
			t.Fatalf("desc: %s , Redis client type should be %s, not %s",
				tc.desc, tc.expectedType, v)
		}
	}
}

func TestPing(t *testing.T) {
	s, err := miniredis.Run()
	if err != nil {
		panic(err)
	}
	defer s.Close()

	// Set mock
	config.Config.RedisAddrs = []string{s.Addr()}
	r := NewRedis()

	err = r.Ping()
	if err != nil {
		t.Fatalf("unexpected error %s", err)
	}
}

func TestFetchSeriesMap(t *testing.T) {
	s, err := miniredis.Run()
	if err != nil {
		panic(err)
	}
	defer s.Close()

	// Set mock
	config.Config.RedisAddrs = []string{s.Addr()}
	r := NewRedis()

	_, err = r.client.HMSet("1m:server1.loadavg5", map[string]string{
		"100": "10.0", "160": "10.2", "220": "11.0",
	}).Result()
	if err != nil {
		panic(err)
	}
	_, err = r.client.HMSet("1m:server2.loadavg5", map[string]string{
		"100": "8.0", "160": "5.0", "220": "6.0",
	}).Result()
	if err != nil {
		panic(err)
	}

	name := "server{1,2}.loadavg5"
	sm, err := r.Fetch(name, time.Unix(100, 0), time.Unix(1000, 0))
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	expected := series.SeriesMap{
		"server1.loadavg5": series.NewSeriesPoint("server1.loadavg5", series.DataPoints{
			series.NewDataPoint(100, 10.0),
			series.NewDataPoint(160, 10.2),
			series.NewDataPoint(220, 11.0),
		}, 60),
		"server2.loadavg5": series.NewSeriesPoint("server2.loadavg5", series.DataPoints{
			series.NewDataPoint(100, 8.0),
			series.NewDataPoint(160, 5.0),
			series.NewDataPoint(220, 6.0),
		}, 60),
	}
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestBatchGet(t *testing.T) {
	s, err := miniredis.Run()
	if err != nil {
		panic(err)
	}
	defer s.Close()

	// Set mock
	config.Config.RedisAddrs = []string{s.Addr()}
	r := NewRedis()

	_, err = r.client.HMSet("1m:server1.loadavg5", map[string]string{
		"100": "10.0", "130": "10.2", "160": "11.0",
	}).Result()
	if err != nil {
		panic(err)
	}
	_, err = r.client.HMSet("1m:server2.loadavg5", map[string]string{
		"100": "8.0", "130": "5.0", "160": "6.0",
	}).Result()
	if err != nil {
		panic(err)
	}

	metrics, err := r.batchGet(&query{
		names: []string{"server1.loadavg5", "server2.loadavg5"},
		slot:  "1m",
		start: time.Unix(100, 0),
		end:   time.Unix(200, 0),
		step:  30,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	expected := series.SeriesMap{
		"server1.loadavg5": series.NewSeriesPoint("server1.loadavg5", series.DataPoints{
			series.NewDataPoint(100, 10.0),
			series.NewDataPoint(130, 10.2),
			series.NewDataPoint(160, 11.0),
		}, 30),
		"server2.loadavg5": series.NewSeriesPoint("server2.loadavg5", series.DataPoints{
			series.NewDataPoint(100, 8.0),
			series.NewDataPoint(130, 5.0),
			series.NewDataPoint(160, 6.0),
		}, 30),
	}
	if diff := pretty.Compare(metrics, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestBatchGet_Empty(t *testing.T) {
	s, err := miniredis.Run()
	if err != nil {
		panic(err)
	}
	defer s.Close()

	// Set mock
	config.Config.RedisAddrs = []string{s.Addr()}
	r := NewRedis()

	metrics, err := r.batchGet(&query{
		names: []string{"server1.loadavg5", "server2.loadavg5"},
		slot:  "1m",
		start: time.Unix(100, 0),
		end:   time.Unix(200, 0),
		step:  30,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if len(metrics) != 0 {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", 0, len(metrics))
	}
}

var testHGetAllToMapTests = []struct {
	desc     string
	name     string
	tsval    map[string]string
	query    *query
	expected *series.SeriesPoint
}{
	{
		"all datapoints within time range",
		"server1.loadavg5",
		map[string]string{"100": "10.0", "160": "11.0", "240": "12.0"},
		&query{
			names: []string{"server1.loadavg5"},
			start: time.Unix(100, 0),
			end:   time.Unix(240, 0),
			slot:  "1m",
			step:  60,
		},
		series.NewSeriesPoint(
			"server1.loadavg5", series.DataPoints{
				series.NewDataPoint(100, 10.0),
				series.NewDataPoint(160, 11.0),
				series.NewDataPoint(240, 12.0),
			}, 60,
		),
	},
	{
		"some datapoints out of time range",
		"server1.loadavg5",
		map[string]string{"40": "9.0", "100": "10.0", "160": "11.0", "240": "12.0", "300": "13.0"},
		&query{
			names: []string{"server1.loadavg5"},
			start: time.Unix(100, 0),
			end:   time.Unix(240, 0),
			slot:  "1m",
			step:  60,
		},
		series.NewSeriesPoint(
			"server1.loadavg5", series.DataPoints{
				series.NewDataPoint(100, 10.0),
				series.NewDataPoint(160, 11.0),
				series.NewDataPoint(240, 12.0),
			}, 60,
		),
	},
}

func TestHGetAllToMap(t *testing.T) {
	for _, tc := range testHGetAllToMapTests {
		got, err := hGetAllToMap(tc.name, tc.tsval, tc.query)
		if err != nil {
			t.Fatalf("err: %s", err)
		}
		if diff := pretty.Compare(got, tc.expected); diff != "" {
			t.Fatalf("diff: (-actual +expected)\n%s", diff)
		}
	}
}

var selectTimeSlotTests = []struct {
	start time.Time
	end   time.Time
	slot  string
	step  int
}{
	{time.Unix(100, 0), time.Unix(6000, 0), "1m", 60},
	{time.Unix(10000, 0), time.Unix(100000, 0), "5m", 300},
	{time.Unix(100000, 0), time.Unix(1000000, 0), "1h", 3600},
	{time.Unix(1000000, 0), time.Unix(100000000, 0), "1d", 86400},
}

func TestSelectTimeSlot(t *testing.T) {
	for i, lc := range selectTimeSlotTests {
		slot, step := selectTimeSlot(lc.start, lc.end)
		if slot != lc.slot {
			t.Fatalf("\nExpected: %+v\nActual:   %+v (#%d)", lc.slot, slot, i)
		}
		if step != lc.step {
			t.Fatalf("\nExpected: %+v\nActual:   %+v (#%d)", lc.step, step, i)
		}
	}
}
<commit_msg>Fix test for NewRedis not running<commit_after>package redis

import (
	"reflect"
	"testing"
	"time"

	"github.com/alicebob/miniredis"
	"github.com/kylelemons/godebug/pretty"
	goredis "gopkg.in/redis.v5"

	"github.com/yuuki/diamondb/lib/config"
	"github.com/yuuki/diamondb/lib/series"
)

func TestNewRedis(t *testing.T) {
	tests := []struct {
		desc         string
		in           []string
		expectedType reflect.Type
	}{
		{
			"redis not cluster",
			[]string{"dummy:6379"},
			reflect.TypeOf((*goredis.Client)(nil)),
		},
		{
			"redis cluster",
			[]string{"dummy01:6379", "dummy02:6379"},
			reflect.TypeOf((*goredis.ClusterClient)(nil)),
		},
	}
	for _, tc := range tests {
		config.Config.RedisAddrs = tc.in
		r := NewRedis()
		if v := reflect.TypeOf(r.client); v != tc.expectedType {
			t.Fatalf("desc: %s , Redis client type should be %s, not %s",
				tc.desc, tc.expectedType, v)
		}
	}
}

func TestPing(t *testing.T) {
	s, err := miniredis.Run()
	if err != nil {
		panic(err)
	}
	defer s.Close()

	// Set mock
	config.Config.RedisAddrs = []string{s.Addr()}
	r := NewRedis()

	err = r.Ping()
	if err != nil {
		t.Fatalf("unexpected error %s", err)
	}
}

func TestFetchSeriesMap(t *testing.T) {
	s, err := miniredis.Run()
	if err != nil {
		panic(err)
	}
	defer s.Close()

	// Set mock
	config.Config.RedisAddrs = []string{s.Addr()}
	r := NewRedis()

	_, err = r.client.HMSet("1m:server1.loadavg5", map[string]string{
		"100": "10.0", "160": "10.2", "220": "11.0",
	}).Result()
	if err != nil {
		panic(err)
	}
	_, err = r.client.HMSet("1m:server2.loadavg5", map[string]string{
		"100": "8.0", "160": "5.0", "220": "6.0",
	}).Result()
	if err != nil {
		panic(err)
	}

	name := "server{1,2}.loadavg5"
	sm, err := r.Fetch(name, time.Unix(100, 0), time.Unix(1000, 0))
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	expected := series.SeriesMap{
		"server1.loadavg5": series.NewSeriesPoint("server1.loadavg5", series.DataPoints{
			series.NewDataPoint(100, 10.0),
			series.NewDataPoint(160, 10.2),
			series.NewDataPoint(220, 11.0),
		}, 60),
		"server2.loadavg5": series.NewSeriesPoint("server2.loadavg5", series.DataPoints{
			series.NewDataPoint(100, 8.0),
			series.NewDataPoint(160, 5.0),
			series.NewDataPoint(220, 6.0),
		}, 60),
	}
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestBatchGet(t *testing.T) {
	s, err := miniredis.Run()
	if err != nil {
		panic(err)
	}
	defer s.Close()

	// Set mock
	config.Config.RedisAddrs = []string{s.Addr()}
	r := NewRedis()

	_, err = r.client.HMSet("1m:server1.loadavg5", map[string]string{
		"100": "10.0", "130": "10.2", "160": "11.0",
	}).Result()
	if err != nil {
		panic(err)
	}
	_, err = r.client.HMSet("1m:server2.loadavg5", map[string]string{
		"100": "8.0", "130": "5.0", "160": "6.0",
	}).Result()
	if err != nil {
		panic(err)
	}

	metrics, err := r.batchGet(&query{
		names: []string{"server1.loadavg5", "server2.loadavg5"},
		slot:  "1m",
		start: time.Unix(100, 0),
		end:   time.Unix(200, 0),
		step:  30,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	expected := series.SeriesMap{
		"server1.loadavg5": series.NewSeriesPoint("server1.loadavg5", series.DataPoints{
			series.NewDataPoint(100, 10.0),
			series.NewDataPoint(130, 10.2),
			series.NewDataPoint(160, 11.0),
		}, 30),
		"server2.loadavg5": series.NewSeriesPoint("server2.loadavg5", series.DataPoints{
			series.NewDataPoint(100, 8.0),
			series.NewDataPoint(130, 5.0),
			series.NewDataPoint(160, 6.0),
		}, 30),
	}
	if diff := pretty.Compare(metrics, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestBatchGet_Empty(t *testing.T) {
	s, err := miniredis.Run()
	if err != nil {
		panic(err)
	}
	defer s.Close()

	// Set mock
	config.Config.RedisAddrs = []string{s.Addr()}
	r := NewRedis()

	metrics, err := r.batchGet(&query{
		names: []string{"server1.loadavg5", "server2.loadavg5"},
		slot:  "1m",
		start: time.Unix(100, 0),
		end:   time.Unix(200, 0),
		step:  30,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if len(metrics) != 0 {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", 0, len(metrics))
	}
}

var testHGetAllToMapTests = []struct {
	desc     string
	name     string
	tsval    map[string]string
	query    *query
	expected *series.SeriesPoint
}{
	{
		"all datapoints within time range",
		"server1.loadavg5",
		map[string]string{"100": "10.0", "160": "11.0", "240": "12.0"},
		&query{
			names: []string{"server1.loadavg5"},
			start: time.Unix(100, 0),
			end:   time.Unix(240, 0),
			slot:  "1m",
			step:  60,
		},
		series.NewSeriesPoint(
			"server1.loadavg5", series.DataPoints{
				series.NewDataPoint(100, 10.0),
				series.NewDataPoint(160, 11.0),
				series.NewDataPoint(240, 12.0),
			}, 60,
		),
	},
	{
		"some datapoints out of time range",
		"server1.loadavg5",
		map[string]string{"40": "9.0", "100": "10.0", "160": "11.0", "240": "12.0", "300": "13.0"},
		&query{
			names: []string{"server1.loadavg5"},
			start: time.Unix(100, 0),
			end:   time.Unix(240, 0),
			slot:  "1m",
			step:  60,
		},
		series.NewSeriesPoint(
			"server1.loadavg5", series.DataPoints{
				series.NewDataPoint(100, 10.0),
				series.NewDataPoint(160, 11.0),
				series.NewDataPoint(240, 12.0),
			}, 60,
		),
	},
}

func TestHGetAllToMap(t *testing.T) {
	for _, tc := range testHGetAllToMapTests {
		got, err := hGetAllToMap(tc.name, tc.tsval, tc.query)
		if err != nil {
			t.Fatalf("err: %s", err)
		}
		if diff := pretty.Compare(got, tc.expected); diff != "" {
			t.Fatalf("diff: (-actual +expected)\n%s", diff)
		}
	}
}

var selectTimeSlotTests = []struct {
	start time.Time
	end   time.Time
	slot  string
	step  int
}{
	{time.Unix(100, 0), time.Unix(6000, 0), "1m", 60},
	{time.Unix(10000, 0), time.Unix(100000, 0), "5m", 300},
	{time.Unix(100000, 0), time.Unix(1000000, 0), "1h", 3600},
	{time.Unix(1000000, 0), time.Unix(100000000, 0), "1d", 86400},
}

func TestSelectTimeSlot(t *testing.T) {
	for i, lc := range selectTimeSlotTests {
		slot, step := selectTimeSlot(lc.start, lc.end)
		if slot != lc.slot {
			t.Fatalf("\nExpected: %+v\nActual:   %+v (#%d)", lc.slot, slot, i)
		}
		if step != lc.step {
			t.Fatalf("\nExpected: %+v\nActual:   %+v (#%d)", lc.step, step, i)
		}
	}
}
<|endoftext|>