<commit_before>package main

import (
	"database/sql"
	"github.com/mattn/go-sqlite3"
	"log"
	"os"
)

func main() {
	sqlite3conn := []*sqlite3.SQLiteConn{}
	sql.Register("sqlite3_with_hook_example",
			&sqlite3.SQLiteDriver{
					ConnectHook: func(conn *sqlite3.SQLiteConn) error {
						sqlite3conn = append(sqlite3conn, conn)
						return nil
					},
			})
	os.Remove("./foo.db")
	os.Remove("./bar.db")

	destDb, err := sql.Open("sqlite3_with_hook_example", "./foo.db")
	if err != nil {
		log.Fatal(err)
	}
	defer destDb.Close()
	destDb.Ping()

	_, err = destDb.Exec("create table foo(id int, value text)")
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Exec("insert into foo values(1, 'foo')")
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Exec("insert into foo values(2, 'bar')")
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Query("select * from foo")
	if err != nil {
		log.Fatal(err)
	}
	srcDb, err := sql.Open("sqlite3_with_hook_example", "./bar.db")
	if err != nil {
		log.Fatal(err)
	}
	defer srcDb.Close()
	srcDb.Ping()

	bk, err := sqlite3conn[1].Backup("main", sqlite3conn[0], "main")
	if err != nil {
		log.Fatal(err)
	}

	bk.Step(-1)
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Query("select * from foo")
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Exec("insert into foo values(3, 'bar')")
	if err != nil {
		log.Fatal(err)
	}

	bk.Finish()
}
<commit_msg>Fix example to check the correct error value<commit_after>package main

import (
	"database/sql"
	"github.com/mattn/go-sqlite3"
	"log"
	"os"
)

func main() {
	sqlite3conn := []*sqlite3.SQLiteConn{}
	sql.Register("sqlite3_with_hook_example",
		&sqlite3.SQLiteDriver{
			ConnectHook: func(conn *sqlite3.SQLiteConn) error {
				sqlite3conn = append(sqlite3conn, conn)
				return nil
			},
		})
	os.Remove("./foo.db")
	os.Remove("./bar.db")

	destDb, err := sql.Open("sqlite3_with_hook_example", "./foo.db")
	if err != nil {
		log.Fatal(err)
	}
	defer destDb.Close()
	destDb.Ping()

	_, err = destDb.Exec("create table foo(id int, value text)")
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Exec("insert into foo values(1, 'foo')")
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Exec("insert into foo values(2, 'bar')")
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Query("select * from foo")
	if err != nil {
		log.Fatal(err)
	}
	srcDb, err := sql.Open("sqlite3_with_hook_example", "./bar.db")
	if err != nil {
		log.Fatal(err)
	}
	defer srcDb.Close()
	srcDb.Ping()

	bk, err := sqlite3conn[1].Backup("main", sqlite3conn[0], "main")
	if err != nil {
		log.Fatal(err)
	}

	_, err = bk.Step(-1)
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Query("select * from foo")
	if err != nil {
		log.Fatal(err)
	}
	_, err = destDb.Exec("insert into foo values(3, 'bar')")
	if err != nil {
		log.Fatal(err)
	}

	bk.Finish()
}
<|endoftext|>


<commit_before>package options

type Options struct {
	Output   string
	Filter   []FilterOption
	Query    string
	User     string
	Password string
	Profile  string
	Trace    bool
	Help     bool
}

type FilterOption struct {
	PropertyName string
	Operation    string
	Value        string
}
<commit_msg>Treat filters as strings on the option parsing stage<commit_after>package options

type Options struct {
	Output   string
	Filter   string
	Query    string
	User     string
	Password string
	Profile  string
	Trace    bool
	Help     bool
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mishudark/eventhus/examples/bank"
	"github.com/mishudark/eventhus/utils"
	"os"
	"time"
)

func main() {
	end := make(chan bool)
	commandBus, err := config()
	if err != nil {
		os.Exit(1)
	}

	//Create Account
	for i := 0; i < 3; i++ {
		go func() {
			uuid, err := utils.UUID()
			if err != nil {
				return
			}

			//1) Create an account
			var account bank.CreateAccount
			account.AggregateID = uuid
			account.Owner = "mishudark"

			commandBus.HandleCommand(account)

			time.Sleep(time.Millisecond * 100)

			//2) Perform a deposit
			deposit := bank.PerformDeposit{
				Ammount: 300,
			}

			deposit.AggregateID = uuid
			deposit.Version = 1

			commandBus.HandleCommand(deposit)

			//3) Perform a withdrawl
			time.Sleep(time.Millisecond * 100)
			withdrawl := bank.PerformWithdrawal{
				Ammount: 249,
			}

			withdrawl.AggregateID = uuid
			withdrawl.Version = 2

			commandBus.HandleCommand(withdrawl)
		}()
	}
	<-end
}
<commit_msg>chore(main): better interline<commit_after>package main

import (
	"github.com/mishudark/eventhus/examples/bank"
	"github.com/mishudark/eventhus/utils"
	"os"
	"time"
)

func main() {
	end := make(chan bool)
	commandBus, err := config()
	if err != nil {
		os.Exit(1)
	}

	//Create Account
	for i := 0; i < 3; i++ {
		go func() {
			uuid, err := utils.UUID()
			if err != nil {
				return
			}

			//1) Create an account
			var account bank.CreateAccount
			account.AggregateID = uuid
			account.Owner = "mishudark"

			commandBus.HandleCommand(account)

			//2) Perform a deposit
			time.Sleep(time.Millisecond * 100)
			deposit := bank.PerformDeposit{
				Ammount: 300,
			}

			deposit.AggregateID = uuid
			deposit.Version = 1

			commandBus.HandleCommand(deposit)

			//3) Perform a withdrawl
			time.Sleep(time.Millisecond * 100)
			withdrawl := bank.PerformWithdrawal{
				Ammount: 249,
			}

			withdrawl.AggregateID = uuid
			withdrawl.Version = 2

			commandBus.HandleCommand(withdrawl)
		}()
	}
	<-end
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// load is an example client of the bigquery client library.
// It loads a file from Google Cloud Storage into a BigQuery table.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"cloud.google.com/go/bigquery"
	"golang.org/x/net/context"
)

var (
	project  = flag.String("project", "", "The ID of a Google Cloud Platform project")
	dataset  = flag.String("dataset", "", "The ID of a BigQuery dataset")
	table    = flag.String("table", "", "The ID of a BigQuery table to load data into")
	bucket   = flag.String("bucket", "", "The name of a Google Cloud Storage bucket to load data from")
	object   = flag.String("object", "", "The name of a Google Cloud Storage object to load data from. Must exist within the bucket specified by --bucket")
	skiprows = flag.Int64("skiprows", 0, "The number of rows of the source data to skip when loading")
	pollint  = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
)

func main() {
	flag.Parse()

	flagsOk := true
	for _, f := range []string{"project", "dataset", "table", "bucket", "object"} {
		if flag.Lookup(f).Value.String() == "" {
			fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
			flagsOk = false
		}
	}
	if !flagsOk {
		os.Exit(1)
	}

	ctx := context.Background()
	client, err := bigquery.NewClient(ctx, *project)
	if err != nil {
		log.Fatalf("Creating bigquery client: %v", err)
	}

	table := client.Dataset(*dataset).Table(*table)

	gcs := client.NewGCSReference(fmt.Sprintf("gs://%s/%s", *bucket, *object))
	gcs.SkipLeadingRows = *skiprows
	gcs.MaxBadRecords = 1
	gcs.AllowQuotedNewlines = true

	// Load data from Google Cloud Storage into a BigQuery table.
	loader := table.LoaderFrom(gcs)
	loader.TableWriteDisposition = bigquery.WriteTruncate
	job, err := loader.Run(ctx)

	if err != nil {
		log.Fatalf("Loading data: %v", err)
	}

	fmt.Printf("Job for data load operation: %+v\n", job)
	fmt.Printf("Waiting for job to complete.\n")

	for range time.Tick(*pollint) {
		status, err := job.Status(ctx)
		if err != nil {
			fmt.Printf("Failure determining status: %v", err)
			break
		}
		if !status.Done() {
			continue
		}
		if err := status.Err(); err == nil {
			fmt.Printf("Success\n")
		} else {
			fmt.Printf("Failure: %+v\n", err)
		}
		break
	}
}
<commit_msg>examples/bigquery: update WriteDisposition field<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// load is an example client of the bigquery client library.
// It loads a file from Google Cloud Storage into a BigQuery table.
package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"cloud.google.com/go/bigquery"
	"golang.org/x/net/context"
)

var (
	project  = flag.String("project", "", "The ID of a Google Cloud Platform project")
	dataset  = flag.String("dataset", "", "The ID of a BigQuery dataset")
	table    = flag.String("table", "", "The ID of a BigQuery table to load data into")
	bucket   = flag.String("bucket", "", "The name of a Google Cloud Storage bucket to load data from")
	object   = flag.String("object", "", "The name of a Google Cloud Storage object to load data from. Must exist within the bucket specified by --bucket")
	skiprows = flag.Int64("skiprows", 0, "The number of rows of the source data to skip when loading")
	pollint  = flag.Duration("pollint", 10*time.Second, "Polling interval for checking job status")
)

func main() {
	flag.Parse()

	flagsOk := true
	for _, f := range []string{"project", "dataset", "table", "bucket", "object"} {
		if flag.Lookup(f).Value.String() == "" {
			fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
			flagsOk = false
		}
	}
	if !flagsOk {
		os.Exit(1)
	}

	ctx := context.Background()
	client, err := bigquery.NewClient(ctx, *project)
	if err != nil {
		log.Fatalf("Creating bigquery client: %v", err)
	}

	table := client.Dataset(*dataset).Table(*table)

	gcs := client.NewGCSReference(fmt.Sprintf("gs://%s/%s", *bucket, *object))
	gcs.SkipLeadingRows = *skiprows
	gcs.MaxBadRecords = 1
	gcs.AllowQuotedNewlines = true

	// Load data from Google Cloud Storage into a BigQuery table.
	loader := table.LoaderFrom(gcs)
	loader.WriteDisposition = bigquery.WriteTruncate
	job, err := loader.Run(ctx)

	if err != nil {
		log.Fatalf("Loading data: %v", err)
	}

	fmt.Printf("Job for data load operation: %+v\n", job)
	fmt.Printf("Waiting for job to complete.\n")

	for range time.Tick(*pollint) {
		status, err := job.Status(ctx)
		if err != nil {
			fmt.Printf("Failure determining status: %v", err)
			break
		}
		if !status.Done() {
			continue
		}
		if err := status.Err(); err == nil {
			fmt.Printf("Success\n")
		} else {
			fmt.Printf("Failure: %+v\n", err)
		}
		break
	}
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package perm

import (
	"regexp"
	"strings"

	"github.com/admpub/nging/application/registry/navigate"
)

type Map struct {
	V   map[string]*Map
	Nav *navigate.Item
}

//Import 导入菜单（用户缓存结果）
func (m *Map) Import(navList navigate.List) *Map {
	for _, nav := range navList {
		item := NewMap()
		item.Nav = nav
		m.V[nav.Action] = item
		item.Import(nav.Children)
	}
	return m
}

func BuildPermActions(values []string) string {
	var permActions string

	if len(values) > 0 && values[0] == `*` {
		permActions = `*`
		return permActions
	}
	var prefix string
	for _, v := range values {
		length := len(v)
		var suffix string
		if length > 2 {
			suffix = v[length-2:]
		}
		if suffix == `/*` {
			if len(prefix) > 0 {
				prefix += `|`
			}
			prefix += regexp.QuoteMeta(v[0 : length-2])
			if len(permActions) > 0 {
				permActions += `,`
			}
			permActions += v
			continue
		}
		if len(prefix) > 0 {
			re := regexp.MustCompile(`^(` + prefix + `)`)
			if re.MatchString(v) {
				continue
			}
		}
		if len(permActions) > 0 {
			permActions += `,`
		}
		permActions += v
	}
	return permActions
}

//Parse 解析用户获取的权限
func (m *Map) Parse(permActions string, navTree *Map) *Map {
	perms := strings.Split(permActions, `,`)
	for _, perm := range perms {
		arr := strings.Split(perm, `/`)
		amap := m
		result := m.V
		var spath string
		for _, a := range arr {
			if mp, y := navTree.V[a]; y {
				amap.Nav = m.Nav
				spath = ``
				amap = mp
			} else {
				if len(spath) > 0 {
					spath += `/`
				}
				spath += a
				if mp, y := navTree.V[spath]; y {
					amap.Nav = m.Nav
					spath = ``
					amap = mp
				}
			}
			if _, y := result[a]; !y {
				result[a] = NewMap()
			}
			result = result[a].V
		}
	}
	return m
}

//Check 检测权限
func (m *Map) Check(perm string) bool {
	if m.Nav != nil && m.Nav.Unlimited {
		return true
	}
	arr := strings.Split(perm, `/`)
	result := m.V
	for _, a := range arr {
		v, y := result[a]
		if !y {
			return false
		}
		if v.Nav != nil && v.Nav.Unlimited {
			return true
		}
		if _, y := v.V[`*`]; y {
			return true
		}
		result = v.V
	}
	return true
}

func NewMap() *Map {
	return &Map{V: map[string]*Map{}}
}
<commit_msg>update<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package perm

import (
	"regexp"
	"strings"

	"github.com/admpub/nging/application/registry/navigate"
)

type Map struct {
	V   map[string]*Map
	Nav *navigate.Item
}

//Import 导入菜单（用户缓存结果）
func (m *Map) Import(navList *navigate.List) *Map {
	for _, nav := range *navList {
		item := NewMap()
		item.Nav = nav
		m.V[nav.Action] = item
		item.Import(nav.Children)
	}
	return m
}

func BuildPermActions(values []string) string {
	var permActions string

	if len(values) > 0 && values[0] == `*` {
		permActions = `*`
		return permActions
	}
	var prefix string
	for _, v := range values {
		length := len(v)
		var suffix string
		if length > 2 {
			suffix = v[length-2:]
		}
		if suffix == `/*` {
			if len(prefix) > 0 {
				prefix += `|`
			}
			prefix += regexp.QuoteMeta(v[0 : length-2])
			if len(permActions) > 0 {
				permActions += `,`
			}
			permActions += v
			continue
		}
		if len(prefix) > 0 {
			re := regexp.MustCompile(`^(` + prefix + `)`)
			if re.MatchString(v) {
				continue
			}
		}
		if len(permActions) > 0 {
			permActions += `,`
		}
		permActions += v
	}
	return permActions
}

//Parse 解析用户获取的权限
func (m *Map) Parse(permActions string, navTree *Map) *Map {
	perms := strings.Split(permActions, `,`)
	for _, perm := range perms {
		arr := strings.Split(perm, `/`)
		amap := m
		result := m.V
		var spath string
		for _, a := range arr {
			if mp, y := navTree.V[a]; y {
				amap.Nav = m.Nav
				spath = ``
				amap = mp
			} else {
				if len(spath) > 0 {
					spath += `/`
				}
				spath += a
				if mp, y := navTree.V[spath]; y {
					amap.Nav = m.Nav
					spath = ``
					amap = mp
				}
			}
			if _, y := result[a]; !y {
				result[a] = NewMap()
			}
			result = result[a].V
		}
	}
	return m
}

//Check 检测权限
func (m *Map) Check(perm string) bool {
	if m.Nav != nil && m.Nav.Unlimited {
		return true
	}
	arr := strings.Split(perm, `/`)
	result := m.V
	for _, a := range arr {
		v, y := result[a]
		if !y {
			return false
		}
		if v.Nav != nil && v.Nav.Unlimited {
			return true
		}
		if _, y := v.V[`*`]; y {
			return true
		}
		result = v.V
	}
	return true
}

func NewMap() *Map {
	return &Map{V: map[string]*Map{}}
}
<|endoftext|>


<commit_before>package webdav

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"bytes"

	"time"

	"github.com/petergtz/bitsgo"
	"github.com/petergtz/bitsgo/config"
	"github.com/petergtz/bitsgo/httputil"
	"github.com/petergtz/bitsgo/logger"
	"github.com/pkg/errors"
)

type Blobstore struct {
	httpClient            *http.Client
	webdavPrivateEndpoint string
	webdavPublicEndpoint  string
	webdavUsername        string
	webdavPassword        string
}

func NewBlobstore(c config.WebdavBlobstoreConfig) *Blobstore {
	return &Blobstore{
		webdavPrivateEndpoint: c.PrivateEndpoint,
		webdavPublicEndpoint:  c.PublicEndpoint,
		httpClient:            NewHttpClient(c.CACert(), c.SkipCertVerify),
		webdavUsername:        c.Username,
		webdavPassword:        c.Password,
	}
}

func (blobstore *Blobstore) Exists(path string) (bool, error) {
	url := blobstore.webdavPrivateEndpoint + "/" + path
	logger.Log.Debugw("Exists", "path", path, "url", url)
	response, e := blobstore.httpClient.Do(blobstore.newRequestWithBasicAuth("HEAD", url, nil))
	if e != nil {
		return false, errors.Wrapf(e, "Error in Exists, path=%v", path)
	}
	if response.StatusCode == http.StatusOK {
		logger.Log.Debugw("Exists", "result", true)
		return true, nil
	}
	logger.Log.Debugw("Exists", "result", false)
	return false, nil
}

func (blobstore *Blobstore) HeadOrRedirectAsGet(path string) (redirectLocation string, err error) {
	_, redirectLocation, e := blobstore.GetOrRedirect(path)
	return redirectLocation, e
}

func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) {
	exists, e := blobstore.Exists(path)
	if e != nil {
		return nil, e
	}
	if !exists {
		return nil, bitsgo.NewNotFoundError()
	}

	response, e := blobstore.httpClient.Get(blobstore.webdavPrivateEndpoint + "/" + path)

	if e != nil {
		return nil, errors.Wrapf(e, "path=%v")
	}
	if response.StatusCode != http.StatusOK {
		return nil, errors.Errorf("Unexpected status code %v. Expected status OK", response.Status)
	}

	return response.Body, nil
}

func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) {
	exists, e := blobstore.Exists(path)
	if e != nil {
		return nil, "", e
	}
	if !exists {
		return nil, "", bitsgo.NewNotFoundError()
	}
	// TODO use clock instead
	signedUrl := blobstore.Sign(path, "get", time.Now().Add(1*time.Hour))
	return nil, signedUrl, nil
}

func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error {
	response, e := blobstore.httpClient.Do(
		blobstore.newRequestWithBasicAuth("PUT", blobstore.webdavPrivateEndpoint+"/admin/"+path, src))
	if e != nil {
		return errors.Wrapf(e, "Request failed. path=%v", path)
	}
	if response.StatusCode < 200 || response.StatusCode > 204 {
		return errors.Errorf("Expected StatusCreated, but got status code: " + response.Status)
	}
	return nil
}

func (blobstore *Blobstore) PutOrRedirect(path string, src io.ReadSeeker) (redirectLocation string, err error) {
	return "", blobstore.Put(path, src)
}

func (blobstore *Blobstore) Copy(src, dest string) error {
	_, e := blobstore.PutOrRedirect(dest, bytes.NewReader(nil))
	if e != nil {
		return e
	}
	response, e := blobstore.httpClient.Do(
		httputil.NewRequest("COPY", blobstore.webdavPrivateEndpoint+"/admin/"+src, nil).
			WithHeader("Destination", blobstore.webdavPrivateEndpoint+"/admin/"+dest).
			WithBasicAuth(blobstore.webdavUsername, blobstore.webdavPassword).
			Build())
	if e != nil {
		return errors.Wrapf(e, "Request failed. src=%v, dest=%v", src, dest)
	}
	if response.StatusCode == http.StatusNotFound {
		return bitsgo.NewNotFoundError()
	}
	if response.StatusCode < 200 || response.StatusCode > 204 {
		return errors.Errorf("Expected HTTP status code 200-204, but got status code: " + response.Status)
	}
	return nil
}

func (blobstore *Blobstore) Delete(path string) error {
	response, e := blobstore.httpClient.Do(
		blobstore.newRequestWithBasicAuth("DELETE", blobstore.webdavPrivateEndpoint+"/admin/"+path, nil))
	if e != nil {
		return errors.Wrapf(e, "Request failed. path=%v", path)
	}
	if response.StatusCode < 200 || response.StatusCode > 204 {
		return errors.Errorf("Expected HTTP status code 200-204, but got status code: " + response.Status)
	}
	return nil
}

func (blobstore *Blobstore) DeleteDir(prefix string) error {
	if prefix != "" {
		prefix += "/"
	}
	response, e := blobstore.httpClient.Do(
		blobstore.newRequestWithBasicAuth("DELETE", blobstore.webdavPrivateEndpoint+"/admin/"+prefix, nil))
	if e != nil {
		return errors.Wrapf(e, "Request failed. prefix=%v", prefix)
	}

	if response.StatusCode == http.StatusNotFound {
		return bitsgo.NewNotFoundError()
	}

	if response.StatusCode < 200 || response.StatusCode > 204 {
		return errors.Errorf("Expected HTTP status code 200-204, but got status code: " + response.Status)
	}
	return nil
}

func (signer *Blobstore) Sign(resource string, method string, expirationTime time.Time) string {
	var url string
	switch strings.ToLower(method) {
	case "put":
		// TODO why do we need a "/" before the resource?
		url = fmt.Sprintf(signer.webdavPrivateEndpoint+"/sign_for_put?path=/%v&expires=%v", resource, expirationTime.Unix())
	case "get":
		url = fmt.Sprintf(signer.webdavPrivateEndpoint+"/sign?path=/%v&expires=%v", resource, expirationTime.Unix())
	}
	response, e := signer.httpClient.Do(
		httputil.NewRequest("GET", url, nil).
			WithBasicAuth(signer.webdavUsername, signer.webdavPassword).
			Build())
	if e != nil {
		return "Error during signing. Error: " + e.Error()
	}
	if response.StatusCode != http.StatusOK {
		return "Error during signing. Error code: " + response.Status
	}
	defer response.Body.Close()
	content, e := ioutil.ReadAll(response.Body)
	if e != nil {
		return "Error reading response body. Error: " + e.Error()
	}

	signedUrl := httputil.MustParse(string(content))

	// TODO Is this really what we want to do?
	signedUrl.Host = httputil.MustParse(signer.webdavPublicEndpoint).Host
	signedUrl.Scheme = httputil.MustParse(signer.webdavPublicEndpoint).Scheme

	return signedUrl.String()
}

func (blobstore *Blobstore) newRequestWithBasicAuth(method string, urlStr string, body io.Reader) *http.Request {
	logger.Log.Debugw("Building HTTP request", "method", method, "url", urlStr, "has-body", body != nil, "user", blobstore.webdavUsername)
	return httputil.NewRequest(method, urlStr, body).
		WithBasicAuth(blobstore.webdavUsername, blobstore.webdavPassword).
		Build()
}
<commit_msg>Hard-code blobstore scheme to http<commit_after>package webdav

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"bytes"

	"time"

	"github.com/petergtz/bitsgo"
	"github.com/petergtz/bitsgo/config"
	"github.com/petergtz/bitsgo/httputil"
	"github.com/petergtz/bitsgo/logger"
	"github.com/pkg/errors"
)

type Blobstore struct {
	httpClient            *http.Client
	webdavPrivateEndpoint string
	webdavPublicEndpoint  string
	webdavUsername        string
	webdavPassword        string
}

func NewBlobstore(c config.WebdavBlobstoreConfig) *Blobstore {
	return &Blobstore{
		webdavPrivateEndpoint: c.PrivateEndpoint,
		webdavPublicEndpoint:  c.PublicEndpoint,
		httpClient:            NewHttpClient(c.CACert(), c.SkipCertVerify),
		webdavUsername:        c.Username,
		webdavPassword:        c.Password,
	}
}

func (blobstore *Blobstore) Exists(path string) (bool, error) {
	url := blobstore.webdavPrivateEndpoint + "/" + path
	logger.Log.Debugw("Exists", "path", path, "url", url)
	response, e := blobstore.httpClient.Do(blobstore.newRequestWithBasicAuth("HEAD", url, nil))
	if e != nil {
		return false, errors.Wrapf(e, "Error in Exists, path=%v", path)
	}
	if response.StatusCode == http.StatusOK {
		logger.Log.Debugw("Exists", "result", true)
		return true, nil
	}
	logger.Log.Debugw("Exists", "result", false)
	return false, nil
}

func (blobstore *Blobstore) HeadOrRedirectAsGet(path string) (redirectLocation string, err error) {
	_, redirectLocation, e := blobstore.GetOrRedirect(path)
	return redirectLocation, e
}

func (blobstore *Blobstore) Get(path string) (body io.ReadCloser, err error) {
	exists, e := blobstore.Exists(path)
	if e != nil {
		return nil, e
	}
	if !exists {
		return nil, bitsgo.NewNotFoundError()
	}

	response, e := blobstore.httpClient.Get(blobstore.webdavPrivateEndpoint + "/" + path)

	if e != nil {
		return nil, errors.Wrapf(e, "path=%v")
	}
	if response.StatusCode != http.StatusOK {
		return nil, errors.Errorf("Unexpected status code %v. Expected status OK", response.Status)
	}

	return response.Body, nil
}

func (blobstore *Blobstore) GetOrRedirect(path string) (body io.ReadCloser, redirectLocation string, err error) {
	exists, e := blobstore.Exists(path)
	if e != nil {
		return nil, "", e
	}
	if !exists {
		return nil, "", bitsgo.NewNotFoundError()
	}
	// TODO use clock instead
	signedUrl := blobstore.Sign(path, "get", time.Now().Add(1*time.Hour))
	return nil, signedUrl, nil
}

func (blobstore *Blobstore) Put(path string, src io.ReadSeeker) error {
	response, e := blobstore.httpClient.Do(
		blobstore.newRequestWithBasicAuth("PUT", blobstore.webdavPrivateEndpoint+"/admin/"+path, src))
	if e != nil {
		return errors.Wrapf(e, "Request failed. path=%v", path)
	}
	if response.StatusCode < 200 || response.StatusCode > 204 {
		return errors.Errorf("Expected StatusCreated, but got status code: " + response.Status)
	}
	return nil
}

func (blobstore *Blobstore) PutOrRedirect(path string, src io.ReadSeeker) (redirectLocation string, err error) {
	return "", blobstore.Put(path, src)
}

func (blobstore *Blobstore) Copy(src, dest string) error {
	_, e := blobstore.PutOrRedirect(dest, bytes.NewReader(nil))
	if e != nil {
		return e
	}
	response, e := blobstore.httpClient.Do(
		httputil.NewRequest("COPY", blobstore.webdavPrivateEndpoint+"/admin/"+src, nil).
			WithHeader("Destination", blobstore.webdavPrivateEndpoint+"/admin/"+dest).
			WithBasicAuth(blobstore.webdavUsername, blobstore.webdavPassword).
			Build())
	if e != nil {
		return errors.Wrapf(e, "Request failed. src=%v, dest=%v", src, dest)
	}
	if response.StatusCode == http.StatusNotFound {
		return bitsgo.NewNotFoundError()
	}
	if response.StatusCode < 200 || response.StatusCode > 204 {
		return errors.Errorf("Expected HTTP status code 200-204, but got status code: " + response.Status)
	}
	return nil
}

func (blobstore *Blobstore) Delete(path string) error {
	response, e := blobstore.httpClient.Do(
		blobstore.newRequestWithBasicAuth("DELETE", blobstore.webdavPrivateEndpoint+"/admin/"+path, nil))
	if e != nil {
		return errors.Wrapf(e, "Request failed. path=%v", path)
	}
	if response.StatusCode < 200 || response.StatusCode > 204 {
		return errors.Errorf("Expected HTTP status code 200-204, but got status code: " + response.Status)
	}
	return nil
}

func (blobstore *Blobstore) DeleteDir(prefix string) error {
	if prefix != "" {
		prefix += "/"
	}
	response, e := blobstore.httpClient.Do(
		blobstore.newRequestWithBasicAuth("DELETE", blobstore.webdavPrivateEndpoint+"/admin/"+prefix, nil))
	if e != nil {
		return errors.Wrapf(e, "Request failed. prefix=%v", prefix)
	}

	if response.StatusCode == http.StatusNotFound {
		return bitsgo.NewNotFoundError()
	}

	if response.StatusCode < 200 || response.StatusCode > 204 {
		return errors.Errorf("Expected HTTP status code 200-204, but got status code: " + response.Status)
	}
	return nil
}

func (signer *Blobstore) Sign(resource string, method string, expirationTime time.Time) string {
	var url string
	switch strings.ToLower(method) {
	case "put":
		// TODO why do we need a "/" before the resource?
		url = fmt.Sprintf(signer.webdavPrivateEndpoint+"/sign_for_put?path=/%v&expires=%v", resource, expirationTime.Unix())
	case "get":
		url = fmt.Sprintf(signer.webdavPrivateEndpoint+"/sign?path=/%v&expires=%v", resource, expirationTime.Unix())
	}
	response, e := signer.httpClient.Do(
		httputil.NewRequest("GET", url, nil).
			WithBasicAuth(signer.webdavUsername, signer.webdavPassword).
			Build())
	if e != nil {
		return "Error during signing. Error: " + e.Error()
	}
	if response.StatusCode != http.StatusOK {
		return "Error during signing. Error code: " + response.Status
	}
	defer response.Body.Close()
	content, e := ioutil.ReadAll(response.Body)
	if e != nil {
		return "Error reading response body. Error: " + e.Error()
	}

	signedUrl := httputil.MustParse(string(content))

	// TODO Is this really what we want to do?
	signedUrl.Host = httputil.MustParse(signer.webdavPublicEndpoint).Host

	// TODO: in the legacy bits-service, this is hard-coded to http, although it should probably be:
	//       httputil.MustParse(signer.webdavPublicEndpoint).Scheme
	//       However, certificates currently do not work out yet, when Stager (Rep) tries to access the URL.
	//       It will then error, because it cannot verify the certificate. Hence, keeping the hard-coded value
	//       for now to be functinally equivalent.
	signedUrl.Scheme = "http"

	return signedUrl.String()
}

func (blobstore *Blobstore) newRequestWithBasicAuth(method string, urlStr string, body io.Reader) *http.Request {
	logger.Log.Debugw("Building HTTP request", "method", method, "url", urlStr, "has-body", body != nil, "user", blobstore.webdavUsername)
	return httputil.NewRequest(method, urlStr, body).
		WithBasicAuth(blobstore.webdavUsername, blobstore.webdavPassword).
		Build()
}
<|endoftext|>


<commit_before>package ga

import "testing"

func TestNewUserEvent(t *testing.T) {
	s := &DefaultAnnotations{}
	e := NewUserEvent(s)
	if e.Category != "user" {
		t.Errorf("Expected user event to has user category (%s)", e.Category)
	}
}

func TestUser_Validate(t *testing.T) {
	e := &User{Category: "other"}
	err := e.Validate()
	if err == nil {
		t.Errorf("Expected user category to be invalid (%s)", e.Category)
	}

	e.Category = "user"
	err = e.Validate()
	if err != nil {
		t.Errorf("Expected user category to be valid (%v)", e)
	}
}

func TestNewSessionEndEvent(t *testing.T) {
	s := &DefaultAnnotations{}
	e := NewSessionEndEvent(s)
	if e.Category != "session_end" {
		t.Errorf("Expected session_end event to has 'session_end' category (%s)",
			e.Category)
	}
}

func TestSessionEnd_Validate(t *testing.T) {
	e := &SessionEnd{Category: "other"}
	err := e.Validate()
	if err == nil {
		t.Errorf("Expected session_end category to be invalid (%s)", e.Category)
	}
	e.Category = "session_end"

	e.Length = -1
	err = e.Validate()
	if err == nil {
		t.Errorf("Expected session_end length to be invalid (%d)", e.Length)
	}

	e.Length = 0
	err = e.Validate()
	if err != nil {
		t.Errorf("Expected session_end category to be valid (%v)", e)
	}
}

func TestNewBusinessEvent(t *testing.T) {
	s := &DefaultAnnotations{}
	e := NewBusinessEvent(s)
	if e.Category != "business" {
		t.Errorf("Expected business event to has business category (%s)", e.Category)
	}
}

func TestBusiness_Validate(t *testing.T) {
	e := &Business{Category: "other"}
	err := e.Validate()
	if err == nil {
		t.Errorf("Expected business category to be invalid (%s)", e.Category)
	}
	e.Category = "business"

	e.EventID = "WrongPattern"
	err = e.Validate()
	if err == nil {
		t.Errorf("Expected business event_id to be invalid (%s)", e.EventID)
	}
	e.EventID = "Correct:Pattern"

	e.Currency = "AAA"
	err = e.Validate()
	if err == nil {
		t.Errorf("Expected business currency to be invalid (%s)", e.Currency)
	}
	e.Currency = "USD"

	err = e.Validate()
	if err != nil {
		t.Errorf("Expected business to be valid (%v)", e)
	}
}
<commit_msg>Refactor tests using tables<commit_after>package ga

import "testing"

type eventCase struct {
	event Event
	valid bool
}

func TestNewUserEvent(t *testing.T) {
	s := &DefaultAnnotations{}
	e := NewUserEvent(s)
	if e.Category != "user" {
		t.Errorf("Expected user event to has user category (%s)", e.Category)
	}
}

func validateEvents(cases []eventCase, t *testing.T) {
	for _, c := range cases {
		err := c.event.Validate()
		if c.valid {
			if err != nil {
				t.Errorf("Expected event (%v) to be valid", c.event)
			}
		} else {
			if err == nil {
				t.Errorf("Expected event (%v) to be invalid", c.event)
			}
		}
	}
}

func TestUser_Validate(t *testing.T) {
	var cases = []eventCase{
		{&User{Category: "other"}, false},
		{&User{Category: "user"}, true},
	}
	validateEvents(cases, t)
}

func TestNewSessionEndEvent(t *testing.T) {
	s := &DefaultAnnotations{}
	e := NewSessionEndEvent(s)
	if e.Category != "session_end" {
		t.Errorf("Expected session_end event to has 'session_end' category (%s)",
			e.Category)
	}
}

func TestSessionEnd_Validate(t *testing.T) {
	var cases = []eventCase{
		{&SessionEnd{Category: "other"}, false},
		{&SessionEnd{Category: "session_end", Length: -1}, false},
		{&SessionEnd{Category: "session_end", Length: 0}, true},
	}
	validateEvents(cases, t)
}

func TestNewBusinessEvent(t *testing.T) {
	s := &DefaultAnnotations{}
	e := NewBusinessEvent(s)
	if e.Category != "business" {
		t.Errorf("Expected business event to has business category (%s)", e.Category)
	}
}

func TestBusiness_Validate(t *testing.T) {
	var cases = []eventCase{
		{&Business{Category: "other"}, false},
		{&Business{Category: "business", EventID: "WrongPattern"}, false},
		{&Business{Category: "business", EventID: "Correct:Pattern", Currency: ""}, false},
		{&Business{Category: "business", EventID: "Correct:Pattern", Currency: "AAA"}, false},
		{&Business{Category: "business", EventID: "Correct:Pattern", Currency: "USD"}, true},
	}
	validateEvents(cases, t)
}
<|endoftext|>


<commit_before>package postgis

import (
	"fmt"
	"github.com/aodin/aspect"
)

type LatLong struct {
	Latitude, Longitude float64
}

// A point with the implied SRID of 4326
// TODO parameterization
func (p LatLong) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return fmt.Sprintf(
		`ST_SetSRID(ST_Point(%f %f), 4326)::geometry`,
		p.Longitude,
		p.Latitude,
	), nil
}

func (p LatLong) Create(d aspect.Dialect) (string, error) {
	return "POINT", nil
}

// TODO Shapes implement both the Compiles interface and dbType (which
// is not exported but probably should be)
type Shape interface {
	aspect.Compiles
	Create(aspect.Dialect) (string, error)
}

type Point struct {
	X, Y float64
}

func (p Point) String() string {
	return fmt.Sprintf(`POINT(%f %f)`, p.X, p.Y)
}

func (p Point) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return fmt.Sprintf(`ST_Point(%f %f)`, p.X, p.Y), nil
}

func (p Point) Create(d aspect.Dialect) (string, error) {
	return "POINT", nil
}

type MultiPoint struct {
	Points []Point
}

// TODO
func (p MultiPoint) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return "", nil
}

func (p MultiPoint) Create(d aspect.Dialect) (string, error) {
	return "MULTIPOINT", nil
}

type Linestring struct {
	Points []Point
}

func (p Linestring) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return "", nil
}

func (p Linestring) Create(d aspect.Dialect) (string, error) {
	return "LINESTRING", nil
}

type Polygon struct {
	Exterior  Linestring
	Interiors []Linestring
}

func (p Polygon) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return "", nil
}

func (p Polygon) Create(d aspect.Dialect) (string, error) {
	return "POLYGON", nil
}
<commit_msg>Importing PostGis now implies PostGres<commit_after>package postgis

import (
	"fmt"
	"github.com/aodin/aspect"
	_ "github.com/aodin/aspect/postgres"
)

// Importing PostGIS implies you'll be using PostGres

type LatLong struct {
	Latitude, Longitude float64
}

// A point with the implied SRID of 4326
// TODO parameterization
func (p LatLong) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return fmt.Sprintf(
		`ST_SetSRID(ST_Point(%f %f), 4326)::geometry`,
		p.Longitude,
		p.Latitude,
	), nil
}

func (p LatLong) Create(d aspect.Dialect) (string, error) {
	return "POINT", nil
}

// TODO Shapes implement both the Compiles interface and dbType (which
// is not exported but probably should be)
type Shape interface {
	aspect.Compiles
	Create(aspect.Dialect) (string, error)
}

type Point struct {
	X, Y float64
}

func (p Point) String() string {
	return fmt.Sprintf(`POINT(%f %f)`, p.X, p.Y)
}

func (p Point) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return fmt.Sprintf(`ST_Point(%f %f)`, p.X, p.Y), nil
}

func (p Point) Create(d aspect.Dialect) (string, error) {
	return "POINT", nil
}

type MultiPoint struct {
	Points []Point
}

// TODO
func (p MultiPoint) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return "", nil
}

func (p MultiPoint) Create(d aspect.Dialect) (string, error) {
	return "MULTIPOINT", nil
}

type Linestring struct {
	Points []Point
}

func (p Linestring) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return "", nil
}

func (p Linestring) Create(d aspect.Dialect) (string, error) {
	return "LINESTRING", nil
}

type Polygon struct {
	Exterior  Linestring
	Interiors []Linestring
}

func (p Polygon) Compile(d aspect.Dialect, params *aspect.Parameters) (string, error) {
	return "", nil
}

func (p Polygon) Create(d aspect.Dialect) (string, error) {
	return "POLYGON", nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v3io

import (
	"fmt"
	"time"

	"github.com/nuclio/nuclio/pkg/errors"
	"github.com/nuclio/nuclio/pkg/processor/trigger/partitioned"

	"github.com/nuclio/logger"
	v3iohttp "github.com/v3io/v3io-go-http"
)

type partition struct {
	*partitioned.AbstractPartition
	partitionID int
	event       Event
	v3ioTrigger *v3io
}

func newPartition(parentLogger logger.Logger, v3ioTrigger *v3io, partitionID int) (*partition, error) {
	var err error

	partitionName := fmt.Sprintf("partition-%d", partitionID)

	// create a partition
	newPartition := &partition{
		partitionID: partitionID,
		v3ioTrigger: v3ioTrigger,
	}

	newPartition.AbstractPartition, err = partitioned.NewAbstractPartition(parentLogger.GetChild(partitionName),
		v3ioTrigger.AbstractStream)

	if err != nil {
		return nil, errors.Wrap(err, "Failed to create abstract partition")
	}
	return newPartition, nil
}

func (p *partition) Read() error {
	partitionPath := fmt.Sprintf("%s/%d", p.v3ioTrigger.streamPath, p.partitionID)

	location, err := p.seek(partitionPath)
	if err != nil {
		return err
	}

	p.Logger.DebugWith("Starting to read from partition",
		"location", location,
		"pollingInterval", p.v3ioTrigger.configuration.PollingIntervalMs)

	pollingInterval := time.Duration(p.v3ioTrigger.configuration.PollingIntervalMs) * time.Millisecond

	for {
		getRecordsOutput, err := p.getRecords(partitionPath, location, pollingInterval)

		if err != nil {
			p.Logger.ErrorWith("Failed to read from partition", "error", err)
			time.Sleep(pollingInterval)
			continue
		}

		// set next location
		location = getRecordsOutput.NextLocation

		// handle records by processing them in the function
		for _, record := range getRecordsOutput.Records {

			// set the record in the event
			p.event.record = &record

			// submit to worker
			p.Stream.SubmitEventToWorker(nil, p.Worker, &p.event) // nolint: errcheck
		}

		if len(getRecordsOutput.Records) == 0 {
			time.Sleep(pollingInterval)
		}
	}
}

func (p *partition) seek(partitionPath string) (string, error) {
	p.Logger.DebugWith("Seeking partition",
		"partitionPath", partitionPath,
		"seekType", p.v3ioTrigger.seekType)

	response, err := p.v3ioTrigger.container.Sync.SeekShard(&v3iohttp.SeekShardInput{
		Path: partitionPath,
		Type: p.v3ioTrigger.seekType,
	})
	if err != nil {
		return "", errors.Wrap(err, "Failed to seek partition")
	}
	response.Release()

	location := response.Output.(*v3iohttp.SeekShardOutput).Location
	if location == "" {
		return "", errors.New("Got empty location from seek")
	}
	return location, nil
}

func (p *partition) getRecords(partitionPath string, location string, pollingInterval time.Duration) (*v3iohttp.GetRecordsOutput, error) {
	response, err := p.v3ioTrigger.container.Sync.GetRecords(&v3iohttp.GetRecordsInput{
		Path:     partitionPath,
		Location: location,
		Limit:    p.v3ioTrigger.configuration.ReadBatchSize,
	})
	if err != nil {
		return nil, err
	}
	response.Release()

	return response.Output.(*v3iohttp.GetRecordsOutput), nil

}
<commit_msg>Fix issue with v3io partition not existing when creating trigger (#1150)<commit_after>/*
Copyright 2017 The Nuclio Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v3io

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/nuclio/nuclio/pkg/errors"
	"github.com/nuclio/nuclio/pkg/processor/trigger/partitioned"

	"github.com/nuclio/logger"
	v3iohttp "github.com/v3io/v3io-go-http"
)

const NotFoundError = "Failed GET with status 404"

type partition struct {
	*partitioned.AbstractPartition
	partitionID int
	event       Event
	v3ioTrigger *v3io
}

func newPartition(parentLogger logger.Logger, v3ioTrigger *v3io, partitionID int) (*partition, error) {
	var err error

	partitionName := fmt.Sprintf("partition-%d", partitionID)

	// create a partition
	newPartition := &partition{
		partitionID: partitionID,
		v3ioTrigger: v3ioTrigger,
	}

	newPartition.AbstractPartition, err = partitioned.NewAbstractPartition(parentLogger.GetChild(partitionName),
		v3ioTrigger.AbstractStream)

	if err != nil {
		return nil, errors.Wrap(err, "Failed to create abstract partition")
	}
	return newPartition, nil
}

func (p *partition) Read() error {
	partitionPath := fmt.Sprintf("%s/%d", strings.TrimRight(p.v3ioTrigger.streamPath, "/"), p.partitionID)
	pollingInterval := time.Duration(p.v3ioTrigger.configuration.PollingIntervalMs) * time.Millisecond

	err := p.waitPartitionAvailable(partitionPath, pollingInterval)
	if err != nil {
		return errors.Wrap(err, "Failed to wait for partition availability")
	}

	location, err := p.seek(partitionPath)
	if err != nil {
		return err
	}

	p.Logger.DebugWith("Starting to read from partition",
		"location", location,
		"pollingInterval", p.v3ioTrigger.configuration.PollingIntervalMs)

	for {
		getRecordsOutput, err := p.getRecords(partitionPath, location, pollingInterval)

		if err != nil {
			p.Logger.ErrorWith("Failed to read from partition", "error", err)
			time.Sleep(pollingInterval)
			continue
		}

		// set next location
		location = getRecordsOutput.NextLocation

		// handle records by processing them in the function
		for _, record := range getRecordsOutput.Records {

			// set the record in the event
			p.event.record = &record

			// submit to worker
			p.Stream.SubmitEventToWorker(nil, p.Worker, &p.event) // nolint: errcheck
		}

		if len(getRecordsOutput.Records) == 0 {
			time.Sleep(pollingInterval)
		}
	}
}

func (p *partition) seek(partitionPath string) (string, error) {
	p.Logger.DebugWith("Seeking partition",
		"partitionPath", partitionPath,
		"seekType", p.v3ioTrigger.seekType)

	response, err := p.v3ioTrigger.container.Sync.SeekShard(&v3iohttp.SeekShardInput{
		Path: partitionPath,
		Type: p.v3ioTrigger.seekType,
	})
	if err != nil {
		return "", errors.Wrap(err, "Failed to seek partition")
	}
	response.Release()

	location := response.Output.(*v3iohttp.SeekShardOutput).Location
	if location == "" {
		return "", errors.New("Got empty location from seek")
	}
	return location, nil
}

func (p *partition) getRecords(partitionPath string, location string, pollingInterval time.Duration) (*v3iohttp.GetRecordsOutput, error) {
	response, err := p.v3ioTrigger.container.Sync.GetRecords(&v3iohttp.GetRecordsInput{
		Path:     partitionPath,
		Location: location,
		Limit:    p.v3ioTrigger.configuration.ReadBatchSize,
	})
	if err != nil {
		return nil, err
	}
	response.Release()

	return response.Output.(*v3iohttp.GetRecordsOutput), nil
}

func (p *partition) waitPartitionAvailable(partitionPath string, pollingInterval time.Duration) error {
	for {
		listBucketResult, err := p.getListBucket()

		if err != nil {

			// print only if error is something else other than 404
			if err.Error() != NotFoundError {
				p.Logger.ErrorWith("Failed to list bucket", "err", err)
			}
			time.Sleep(pollingInterval)
			continue
		}

		// look to see if the partition inside the stream path
		for _, partition := range listBucketResult.Contents {
			splittedPartitionKey := strings.Split(strings.TrimRight(partition.Key, "/"), "/")
			if splittedPartitionKey[len(splittedPartitionKey)-1] == strconv.Itoa(p.partitionID) {
				return nil
			}
		}
		time.Sleep(pollingInterval)
	}
}

func (p *partition) getListBucket() (*v3iohttp.ListBucketOutput, error) {
	request := &v3iohttp.ListBucketInput{
		Path: p.v3ioTrigger.streamPath,
	}
	response, err := p.v3ioTrigger.container.Sync.ListBucket(request)

	// upon errors, response is being released by ListBucket
	if err != nil {
		return nil, err
	}
	response.Release()

	listBucketResult, ok := response.Output.(*v3iohttp.ListBucketOutput)
	if !ok {
		return nil, errors.New("Failed to cast response to ListBucketOutput")
	}

	return listBucketResult, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package descriptors

import (
	"github.com/caicloud/nirvana/definition"
	"github.com/caicloud/nirvana/operators/validator"

	handler "github.com/caicloud/cyclone/pkg/server/handler/v1alpha1"
	httputil "github.com/caicloud/cyclone/pkg/util/http"
)

func init() {
	register(workflowrun...)
}

var workflowrun = []definition.Descriptor{
	{
		Path:        "/projects/{project}/workflows/{workflow}/workflowruns",
		Description: "workflowrun APIs",
		Definitions: []definition.Definition{
			{
				Method:      definition.Create,
				Function:    handler.CreateWorkflowRun,
				Description: "Create workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source:      definition.Body,
						Description: "JSON body to describe the new workflowrun",
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
			{
				Method:      definition.Get,
				Function:    handler.ListWorkflowRuns,
				Description: "List workflowruns",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source:      definition.Auto,
						Name:        httputil.PaginationAutoParameter,
						Description: "pagination",
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
		},
	},
	{
		Path:        "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}",
		Description: "workflowrun APIs",
		Definitions: []definition.Definition{
			{
				Method:      definition.Get,
				Function:    handler.GetWorkflowRun,
				Description: "Get workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
			{
				Method:      definition.Update,
				Function:    handler.UpdateWorkflowRun,
				Description: "Update workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source:      definition.Body,
						Description: "JSON body to describe the updated workflowrun",
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
			{
				Method:      definition.Delete,
				Function:    handler.DeleteWorkflowRun,
				Description: "Delete workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
				},
				Results: []definition.Result{definition.ErrorResult()},
			},
		},
	},
	{
		Path: "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}/pause",
		Definitions: []definition.Definition{
			{
				Method:      definition.Update,
				Function:    handler.PauseWorkflowRun,
				Description: "Pause a workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
		},
	},
	{
		Path: "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}/continue",
		Definitions: []definition.Definition{
			{
				Method:      definition.Update,
				Function:    handler.ContinueWorkflowRun,
				Description: "Continue ro run workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
		},
	},
	{
		Path: "/workflowruns/{workflowrun}/streamlogs",
		Definitions: []definition.Definition{
			{
				Method:      definition.Get,
				Function:    handler.ReceiveContainerLogStream,
				Description: "Used for collecting stage logs",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Query,
						Name:   httputil.NamespaceQueryParameter,
					},
					{
						Source: definition.Query,
						Name:   httputil.StageNameQueryParameter,
					},
					{
						Source: definition.Query,
						Name:   httputil.ContainerNameQueryParameter,
					},
				},
				Results: []definition.Result{
					{
						Destination: definition.Error,
					},
				},
			},
		},
	},
	{
		Path: "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}/logstream",
		Definitions: []definition.Definition{
			{
				Method:      definition.Get,
				Function:    handler.GetContainerLogStream,
				Description: "Get log stream of containers",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source: definition.Query,
						Name:   httputil.StageNameQueryParameter,
					},
					{
						Source: definition.Query,
						Name:   httputil.ContainerNameQueryParameter,
					},
				},
				Results: []definition.Result{definition.ErrorResult()},
			},
		},
	},
	{
		Path: "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}/logs",
		Definitions: []definition.Definition{
			{
				Method:      definition.Get,
				Function:    handler.GetContainerLogs,
				Description: "Get log of containers",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source: definition.Query,
						Name:   httputil.StageNameQueryParameter,
					},
					{
						Source: definition.Query,
						Name:   httputil.ContainerNameQueryParameter,
					},
					{
						Source:    definition.Query,
						Name:      httputil.DownloadQueryParameter,
						Operators: []definition.Operator{validator.Bool("")},
					},
				},
				Results: []definition.Result{
					{
						Destination: definition.Data,
						Description: "container log",
					},
					{
						Destination: definition.Meta,
					},
					{
						Destination: definition.Error,
					},
				},
			},
		},
	},
}
<commit_msg>fix: correct produce content type of workflowrun logs (#833)<commit_after>/*
Copyright 2018 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package descriptors

import (
	"github.com/caicloud/nirvana/definition"
	"github.com/caicloud/nirvana/operators/validator"

	handler "github.com/caicloud/cyclone/pkg/server/handler/v1alpha1"
	httputil "github.com/caicloud/cyclone/pkg/util/http"
)

func init() {
	register(workflowrun...)
}

var workflowrun = []definition.Descriptor{
	{
		Path:        "/projects/{project}/workflows/{workflow}/workflowruns",
		Description: "workflowrun APIs",
		Definitions: []definition.Definition{
			{
				Method:      definition.Create,
				Function:    handler.CreateWorkflowRun,
				Description: "Create workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source:      definition.Body,
						Description: "JSON body to describe the new workflowrun",
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
			{
				Method:      definition.Get,
				Function:    handler.ListWorkflowRuns,
				Description: "List workflowruns",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source:      definition.Auto,
						Name:        httputil.PaginationAutoParameter,
						Description: "pagination",
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
		},
	},
	{
		Path:        "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}",
		Description: "workflowrun APIs",
		Definitions: []definition.Definition{
			{
				Method:      definition.Get,
				Function:    handler.GetWorkflowRun,
				Description: "Get workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
			{
				Method:      definition.Update,
				Function:    handler.UpdateWorkflowRun,
				Description: "Update workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source:      definition.Body,
						Description: "JSON body to describe the updated workflowrun",
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
			{
				Method:      definition.Delete,
				Function:    handler.DeleteWorkflowRun,
				Description: "Delete workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
				},
				Results: []definition.Result{definition.ErrorResult()},
			},
		},
	},
	{
		Path: "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}/pause",
		Definitions: []definition.Definition{
			{
				Method:      definition.Update,
				Function:    handler.PauseWorkflowRun,
				Description: "Pause a workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
		},
	},
	{
		Path: "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}/continue",
		Definitions: []definition.Definition{
			{
				Method:      definition.Update,
				Function:    handler.ContinueWorkflowRun,
				Description: "Continue ro run workflowrun",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
				},
				Results: definition.DataErrorResults("workflowrun"),
			},
		},
	},
	{
		Path: "/workflowruns/{workflowrun}/streamlogs",
		Definitions: []definition.Definition{
			{
				Method:      definition.Get,
				Function:    handler.ReceiveContainerLogStream,
				Description: "Used for collecting stage logs",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Query,
						Name:   httputil.NamespaceQueryParameter,
					},
					{
						Source: definition.Query,
						Name:   httputil.StageNameQueryParameter,
					},
					{
						Source: definition.Query,
						Name:   httputil.ContainerNameQueryParameter,
					},
				},
				Results: []definition.Result{
					{
						Destination: definition.Error,
					},
				},
			},
		},
	},
	{
		Path: "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}/logstream",
		Definitions: []definition.Definition{
			{
				Method:      definition.Get,
				Function:    handler.GetContainerLogStream,
				Description: "Get log stream of containers",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source: definition.Query,
						Name:   httputil.StageNameQueryParameter,
					},
					{
						Source: definition.Query,
						Name:   httputil.ContainerNameQueryParameter,
					},
				},
				Results: []definition.Result{definition.ErrorResult()},
			},
		},
	},
	{
		Path: "/projects/{project}/workflows/{workflow}/workflowruns/{workflowrun}/logs",
		Definitions: []definition.Definition{
			{
				Method:      definition.Get,
				Produces:    []string{definition.MIMEText},
				Function:    handler.GetContainerLogs,
				Description: "Get log of containers",
				Parameters: []definition.Parameter{
					{
						Source: definition.Path,
						Name:   httputil.ProjectNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowNamePathParameterName,
					},
					{
						Source: definition.Path,
						Name:   httputil.WorkflowRunNamePathParameterName,
					},
					{
						Source: definition.Header,
						Name:   httputil.TenantHeaderName,
					},
					{
						Source: definition.Query,
						Name:   httputil.StageNameQueryParameter,
					},
					{
						Source: definition.Query,
						Name:   httputil.ContainerNameQueryParameter,
					},
					{
						Source:    definition.Query,
						Name:      httputil.DownloadQueryParameter,
						Operators: []definition.Operator{validator.Bool("")},
					},
				},
				Results: []definition.Result{
					{
						Destination: definition.Data,
						Description: "container log",
					},
					{
						Destination: definition.Meta,
					},
					{
						Destination: definition.Error,
					},
				},
			},
		},
	},
}
<|endoftext|>


<commit_before>// +build integrationtest

package integrationtest

import (
	"flag"
	"testing"

	storkdriver "github.com/libopenstorage/stork/drivers/volume"
	_ "github.com/libopenstorage/stork/drivers/volume/portworx"
	"github.com/portworx/torpedo/drivers/node"
	_ "github.com/portworx/torpedo/drivers/node/ssh"
	"github.com/portworx/torpedo/drivers/scheduler"
	_ "github.com/portworx/torpedo/drivers/scheduler/k8s"
	"github.com/portworx/torpedo/drivers/volume"
	_ "github.com/portworx/torpedo/drivers/volume/portworx"
	"github.com/sirupsen/logrus"
	"github.com/skyrings/skyring-common/tools/uuid"
	"github.com/stretchr/testify/require"
)

const (
	nodeDriverName      = "ssh"
	volumeDriverName    = "pxd"
	schedulerDriverName = "k8s"

	nodeScore   = 100
	rackScore   = 50
	zoneScore   = 25
	regionScore = 10
)

var nodeDriver node.Driver
var schedulerDriver scheduler.Driver
var volumeDriver volume.Driver
var storkVolumeDriver storkdriver.Driver

var snapshotScaleCount int

// TODO: Start stork scheduler and stork extender
// TODO: Take driver name from input
// TODO: Parse storageclass specs based on driver name
func setup(t *testing.T) {
	var err error

	storkVolumeDriver, err = storkdriver.Get(volumeDriverName)
	require.NoError(t, err, "Error getting stork driver %v", volumeDriverName)

	err = storkVolumeDriver.Init(nil)
	require.NoError(t, err, "Error initializing stork driver %v", volumeDriverName)

	nodeDriver, err = node.Get(nodeDriverName)
	require.NoError(t, err, "Error getting node driver %v", nodeDriverName)

	err = nodeDriver.Init()
	require.NoError(t, err, "Error initializing node driver %v", nodeDriverName)

	schedulerDriver, err = scheduler.Get(schedulerDriverName)
	require.NoError(t, err, "Error getting scheduler driver %v", schedulerDriverName)

	err = schedulerDriver.Init("/specs", volumeDriverName, nodeDriverName)
	require.NoError(t, err, "Error initializing scheduler driver %v", schedulerDriverName)

	volumeDriver, err = volume.Get(volumeDriverName)
	require.NoError(t, err, "Error getting volume driver %v", volumeDriverName)

	err = volumeDriver.Init(schedulerDriverName, nodeDriverName)
	require.NoError(t, err, "Error initializing volume driver %v", volumeDriverName)
}

func TestMain(t *testing.T) {
	// If setup fails stop the test
	if passed := t.Run("setup", setup); !passed {
		t.FailNow()
	}
	t.Run("Extender", testExtender)
	t.Run("HealthMonitor", testHealthMonitor)
	t.Run("Snapshot", testSnapshot)
	t.Run("CmdExecutor", asyncPodCommandTest)
}

func generateInstanceID(t *testing.T, testName string) string {
	id, err := uuid.New()
	require.NoError(t, err, "Error generating uuid for task")
	return testName + "-" + id.String()
}

func destroyAndWait(t *testing.T, ctxs []*scheduler.Context) {
	for _, ctx := range ctxs {
		err := schedulerDriver.Destroy(ctx, nil)
		require.NoError(t, err, "Error destroying ctx: %+v", ctx)
		err = schedulerDriver.WaitForDestroy(ctx)
		require.NoError(t, err, "Error waiting for destroy of ctx: %+v", ctx)
		_, err = schedulerDriver.DeleteVolumes(ctx)
		require.NoError(t, err, "Error deleting volumes in ctx: %+v", ctx)
	}
}

func getVolumeNames(t *testing.T, ctx *scheduler.Context) []string {
	volumeParams, err := schedulerDriver.GetVolumeParameters(ctx)
	require.NoError(t, err, "Error getting volume Parameters")

	var volumes []string
	for vol := range volumeParams {
		volumes = append(volumes, vol)
	}
	return volumes
}

func verifyScheduledNode(t *testing.T, appNode node.Node, volumes []string) {
	driverNodes, err := storkVolumeDriver.GetNodes()
	require.NoError(t, err, "Error getting nodes from stork driver")

	found := false
	for _, dNode := range driverNodes {
		if dNode.Hostname == appNode.Name {
			found = true
			break
		}
	}
	require.Equal(t, true, found, "Scheduled node not found in driver node list")

	scores := make(map[string]int)
	idMap := make(map[string]*storkdriver.NodeInfo)
	rackMap := make(map[string][]string)
	zoneMap := make(map[string][]string)
	regionMap := make(map[string][]string)
	for _, dNode := range driverNodes {
		scores[dNode.Hostname] = 0
		idMap[dNode.ID] = dNode
		if dNode.Status == storkdriver.NodeOnline {
			if dNode.Rack != "" {
				rackMap[dNode.Rack] = append(rackMap[dNode.Rack], dNode.Hostname)
			}
			if dNode.Zone != "" {
				zoneMap[dNode.Zone] = append(rackMap[dNode.Zone], dNode.Hostname)
			}
			if dNode.Region != "" {
				regionMap[dNode.Region] = append(rackMap[dNode.Region], dNode.Hostname)
			}
		}
	}

	// Calculate scores for each node
	for _, vol := range volumes {
		volInfo, err := storkVolumeDriver.InspectVolume(vol)
		require.NoError(t, err, "Error inspecting volume %v", vol)

		for _, dataNode := range volInfo.DataNodes {
			hostname := idMap[dataNode].Hostname
			scores[hostname] += nodeScore

			if idMap[dataNode].Rack != "" {
				for _, node := range rackMap[idMap[dataNode].Rack] {
					if dataNode != node {
						scores[node] += rackScore
					}
				}
			}
			if idMap[dataNode].Zone != "" {
				for _, node := range zoneMap[idMap[dataNode].Zone] {
					if dataNode != node {
						scores[node] += zoneScore
					}
				}
			}
			if idMap[dataNode].Rack != "" {
				for _, node := range regionMap[idMap[dataNode].Region] {
					if dataNode != node {
						scores[node] += regionScore
					}
				}
			}
		}
	}

	highScore := 0
	for _, score := range scores {
		if score > highScore {
			highScore = score
		}
	}

	logrus.Infof("Scores: %v", scores)
	require.Equal(t, highScore, scores[appNode.Name], "Scheduled node does not have the highest score")
}

func init() {
	flag.IntVar(&snapshotScaleCount,
		"snapshot-scale-count",
		10,
		"Number of volumes to use for scale snapshot test")
	flag.Parse()
}
<commit_msg>Print node info on failure when checking scheduled node<commit_after>// +build integrationtest

package integrationtest

import (
	"flag"
	"testing"

	storkdriver "github.com/libopenstorage/stork/drivers/volume"
	_ "github.com/libopenstorage/stork/drivers/volume/portworx"
	"github.com/portworx/torpedo/drivers/node"
	_ "github.com/portworx/torpedo/drivers/node/ssh"
	"github.com/portworx/torpedo/drivers/scheduler"
	_ "github.com/portworx/torpedo/drivers/scheduler/k8s"
	"github.com/portworx/torpedo/drivers/volume"
	_ "github.com/portworx/torpedo/drivers/volume/portworx"
	"github.com/sirupsen/logrus"
	"github.com/skyrings/skyring-common/tools/uuid"
	"github.com/stretchr/testify/require"
)

const (
	nodeDriverName      = "ssh"
	volumeDriverName    = "pxd"
	schedulerDriverName = "k8s"

	nodeScore   = 100
	rackScore   = 50
	zoneScore   = 25
	regionScore = 10
)

var nodeDriver node.Driver
var schedulerDriver scheduler.Driver
var volumeDriver volume.Driver
var storkVolumeDriver storkdriver.Driver

var snapshotScaleCount int

// TODO: Start stork scheduler and stork extender
// TODO: Take driver name from input
// TODO: Parse storageclass specs based on driver name
func setup(t *testing.T) {
	var err error

	storkVolumeDriver, err = storkdriver.Get(volumeDriverName)
	require.NoError(t, err, "Error getting stork driver %v", volumeDriverName)

	err = storkVolumeDriver.Init(nil)
	require.NoError(t, err, "Error initializing stork driver %v", volumeDriverName)

	nodeDriver, err = node.Get(nodeDriverName)
	require.NoError(t, err, "Error getting node driver %v", nodeDriverName)

	err = nodeDriver.Init()
	require.NoError(t, err, "Error initializing node driver %v", nodeDriverName)

	schedulerDriver, err = scheduler.Get(schedulerDriverName)
	require.NoError(t, err, "Error getting scheduler driver %v", schedulerDriverName)

	err = schedulerDriver.Init("/specs", volumeDriverName, nodeDriverName)
	require.NoError(t, err, "Error initializing scheduler driver %v", schedulerDriverName)

	volumeDriver, err = volume.Get(volumeDriverName)
	require.NoError(t, err, "Error getting volume driver %v", volumeDriverName)

	err = volumeDriver.Init(schedulerDriverName, nodeDriverName)
	require.NoError(t, err, "Error initializing volume driver %v", volumeDriverName)
}

func TestMain(t *testing.T) {
	// If setup fails stop the test
	if passed := t.Run("setup", setup); !passed {
		t.FailNow()
	}
	t.Run("Extender", testExtender)
	t.Run("HealthMonitor", testHealthMonitor)
	t.Run("Snapshot", testSnapshot)
	t.Run("CmdExecutor", asyncPodCommandTest)
}

func generateInstanceID(t *testing.T, testName string) string {
	id, err := uuid.New()
	require.NoError(t, err, "Error generating uuid for task")
	return testName + "-" + id.String()
}

func destroyAndWait(t *testing.T, ctxs []*scheduler.Context) {
	for _, ctx := range ctxs {
		err := schedulerDriver.Destroy(ctx, nil)
		require.NoError(t, err, "Error destroying ctx: %+v", ctx)
		err = schedulerDriver.WaitForDestroy(ctx)
		require.NoError(t, err, "Error waiting for destroy of ctx: %+v", ctx)
		_, err = schedulerDriver.DeleteVolumes(ctx)
		require.NoError(t, err, "Error deleting volumes in ctx: %+v", ctx)
	}
}

func getVolumeNames(t *testing.T, ctx *scheduler.Context) []string {
	volumeParams, err := schedulerDriver.GetVolumeParameters(ctx)
	require.NoError(t, err, "Error getting volume Parameters")

	var volumes []string
	for vol := range volumeParams {
		volumes = append(volumes, vol)
	}
	return volumes
}

func verifyScheduledNode(t *testing.T, appNode node.Node, volumes []string) {
	driverNodes, err := storkVolumeDriver.GetNodes()
	require.NoError(t, err, "Error getting nodes from stork driver")

	found := false
	for _, dNode := range driverNodes {
		if dNode.Hostname == appNode.Name {
			found = true
			break
		}
	}
	require.Equal(t, true, found, "Scheduled node not found in driver node list. DriverNodes: %v ScheduledNode: %v", driverNodes, appNode)

	scores := make(map[string]int)
	idMap := make(map[string]*storkdriver.NodeInfo)
	rackMap := make(map[string][]string)
	zoneMap := make(map[string][]string)
	regionMap := make(map[string][]string)
	for _, dNode := range driverNodes {
		scores[dNode.Hostname] = 0
		idMap[dNode.ID] = dNode
		if dNode.Status == storkdriver.NodeOnline {
			if dNode.Rack != "" {
				rackMap[dNode.Rack] = append(rackMap[dNode.Rack], dNode.Hostname)
			}
			if dNode.Zone != "" {
				zoneMap[dNode.Zone] = append(rackMap[dNode.Zone], dNode.Hostname)
			}
			if dNode.Region != "" {
				regionMap[dNode.Region] = append(rackMap[dNode.Region], dNode.Hostname)
			}
		}
	}

	// Calculate scores for each node
	for _, vol := range volumes {
		volInfo, err := storkVolumeDriver.InspectVolume(vol)
		require.NoError(t, err, "Error inspecting volume %v", vol)

		for _, dataNode := range volInfo.DataNodes {
			hostname := idMap[dataNode].Hostname
			scores[hostname] += nodeScore

			if idMap[dataNode].Rack != "" {
				for _, node := range rackMap[idMap[dataNode].Rack] {
					if dataNode != node {
						scores[node] += rackScore
					}
				}
			}
			if idMap[dataNode].Zone != "" {
				for _, node := range zoneMap[idMap[dataNode].Zone] {
					if dataNode != node {
						scores[node] += zoneScore
					}
				}
			}
			if idMap[dataNode].Rack != "" {
				for _, node := range regionMap[idMap[dataNode].Region] {
					if dataNode != node {
						scores[node] += regionScore
					}
				}
			}
		}
	}

	highScore := 0
	for _, score := range scores {
		if score > highScore {
			highScore = score
		}
	}

	logrus.Infof("Scores: %v", scores)
	require.Equal(t, highScore, scores[appNode.Name], "Scheduled node does not have the highest score")
}

func init() {
	flag.IntVar(&snapshotScaleCount,
		"snapshot-scale-count",
		10,
		"Number of volumes to use for scale snapshot test")
	flag.Parse()
}
<|endoftext|>


<commit_before>package queryrange

import (
	"context"
	"net/http"
	"time"

	"github.com/cortexproject/cortex/pkg/querier/queryrange"
	"github.com/opentracing/opentracing-go"
	otlog "github.com/opentracing/opentracing-go/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/weaveworks/common/httpgrpc"
	"github.com/weaveworks/common/user"

	"github.com/grafana/loki/pkg/logproto"
)

type lokiResult struct {
	req queryrange.Request
	ch  chan *packedResp
}

type packedResp struct {
	resp queryrange.Response
	err  error
}

type SplitByMetrics struct {
	splits prometheus.Histogram
}

func NewSplitByMetrics(r prometheus.Registerer) *SplitByMetrics {
	return &SplitByMetrics{
		splits: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
			Namespace: "loki",
			Name:      "query_frontend_partitions",
			Help:      "Number of time-based partitions (sub-requests) per request",
			Buckets:   prometheus.ExponentialBuckets(1, 4, 5), // 1 -> 1024
		}),
	}
}

type splitByInterval struct {
	next    queryrange.Handler
	limits  Limits
	merger  queryrange.Merger
	metrics *SplitByMetrics
}

// SplitByIntervalMiddleware creates a new Middleware that splits log requests by a given interval.
func SplitByIntervalMiddleware(limits Limits, merger queryrange.Merger, metrics *SplitByMetrics) queryrange.Middleware {
	return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler {
		return &splitByInterval{
			next:    next,
			limits:  limits,
			merger:  merger,
			metrics: metrics,
		}
	})
}

func (h *splitByInterval) Feed(ctx context.Context, input []*lokiResult) chan *lokiResult {
	ch := make(chan *lokiResult)

	go func() {
		defer close(ch)
		for _, d := range input {
			select {
			case <-ctx.Done():
				return
			case ch <- d:
				continue
			}
		}
	}()

	return ch
}

func (h *splitByInterval) Process(
	ctx context.Context,
	parallelism int,
	threshold int64,
	input []*lokiResult,
) (responses []queryrange.Response, err error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	ch := h.Feed(ctx, input)

	// queries with 0 limits should not be exited early
	var unlimited bool
	if threshold == 0 {
		unlimited = true
	}

	// don't spawn unnecessary goroutines
	var p int = parallelism
	if len(input) < parallelism {
		p = len(input)
	}

	for i := 0; i < p; i++ {
		go h.loop(ctx, ch)
	}

	for _, x := range input {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case data := <-x.ch:
			if data.err != nil {
				return nil, err
			}

			responses = append(responses, data.resp)

			// see if we can exit early if a limit has been reached
			if casted, ok := data.resp.(*LokiResponse); !unlimited && ok {
				threshold -= casted.Count()

				if threshold <= 0 {
					return responses, nil
				}

			}

		}

	}

	return responses, nil
}

func (h *splitByInterval) loop(ctx context.Context, ch <-chan *lokiResult) {

	for data := range ch {

		sp, ctx := opentracing.StartSpanFromContext(ctx, "interval")
		data.req.LogToSpan(sp)

		resp, err := h.next.Do(ctx, data.req)

		select {
		case <-ctx.Done():
			sp.Finish()
			return
		case data.ch <- &packedResp{resp, err}:
			sp.Finish()
		}
	}
}

func (h *splitByInterval) Do(ctx context.Context, r queryrange.Request) (queryrange.Response, error) {

	userid, err := user.ExtractOrgID(ctx)
	if err != nil {
		return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error())
	}

	interval := h.limits.QuerySplitDuration(userid)
	// skip split by if unset
	if interval == 0 {
		return h.next.Do(ctx, r)
	}

	intervals := splitByTime(r, interval)
	h.metrics.splits.Observe(float64(len(intervals)))

	// no interval should not be processed by the frontend.
	if len(intervals) == 0 {
		return h.next.Do(ctx, r)
	}

	if sp := opentracing.SpanFromContext(ctx); sp != nil {
		sp.LogFields(otlog.Int("n_intervals", len(intervals)))

	}

	var limit int64
	switch req := r.(type) {
	case *LokiRequest:
		limit = int64(req.Limit)
		if req.Direction == logproto.BACKWARD {
			for i, j := 0, len(intervals)-1; i < j; i, j = i+1, j-1 {
				intervals[i], intervals[j] = intervals[j], intervals[i]
			}
		}
	case *LokiSeriesRequest, *LokiLabelNamesRequest:
		// Set this to 0 since this is not used in Series/Labels Request.
		limit = 0
	default:
		return nil, httpgrpc.Errorf(http.StatusBadRequest, "unknown request type")
	}

	input := make([]*lokiResult, 0, len(intervals))
	for _, interval := range intervals {
		input = append(input, &lokiResult{
			req: interval,
			ch:  make(chan *packedResp),
		})
	}

	resps, err := h.Process(ctx, h.limits.MaxQueryParallelism(userid), limit, input)
	if err != nil {
		return nil, err
	}
	return h.merger.MergeResponse(resps...)
}

func splitByTime(req queryrange.Request, interval time.Duration) []queryrange.Request {
	var reqs []queryrange.Request

	switch r := req.(type) {
	case *LokiRequest:
		forInterval(interval, r.StartTs, r.EndTs, func(start, end time.Time) {
			reqs = append(reqs, &LokiRequest{
				Query:     r.Query,
				Limit:     r.Limit,
				Step:      r.Step,
				Direction: r.Direction,
				Path:      r.Path,
				StartTs:   start,
				EndTs:     end,
			})
		})
	case *LokiSeriesRequest:
		forInterval(interval, r.StartTs, r.EndTs, func(start, end time.Time) {
			reqs = append(reqs, &LokiSeriesRequest{
				Match:   r.Match,
				Path:    r.Path,
				StartTs: start,
				EndTs:   end,
			})
		})
	case *LokiLabelNamesRequest:
		forInterval(interval, r.StartTs, r.EndTs, func(start, end time.Time) {
			reqs = append(reqs, &LokiLabelNamesRequest{
				Path:    r.Path,
				StartTs: start,
				EndTs:   end,
			})
		})
	default:
		return nil
	}
	return reqs

}

func forInterval(interval time.Duration, start, end time.Time, callback func(start, end time.Time)) {
	for start := start; start.Before(end); start = start.Add(interval) {
		newEnd := start.Add(interval)
		if newEnd.After(end) {
			newEnd = end
		}
		callback(start, newEnd)
	}
}
<commit_msg>Fix error swallowed in the frontend. (#2807)<commit_after>package queryrange

import (
	"context"
	"net/http"
	"time"

	"github.com/cortexproject/cortex/pkg/querier/queryrange"
	"github.com/opentracing/opentracing-go"
	otlog "github.com/opentracing/opentracing-go/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"github.com/weaveworks/common/httpgrpc"
	"github.com/weaveworks/common/user"

	"github.com/grafana/loki/pkg/logproto"
)

type lokiResult struct {
	req queryrange.Request
	ch  chan *packedResp
}

type packedResp struct {
	resp queryrange.Response
	err  error
}

type SplitByMetrics struct {
	splits prometheus.Histogram
}

func NewSplitByMetrics(r prometheus.Registerer) *SplitByMetrics {
	return &SplitByMetrics{
		splits: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
			Namespace: "loki",
			Name:      "query_frontend_partitions",
			Help:      "Number of time-based partitions (sub-requests) per request",
			Buckets:   prometheus.ExponentialBuckets(1, 4, 5), // 1 -> 1024
		}),
	}
}

type splitByInterval struct {
	next    queryrange.Handler
	limits  Limits
	merger  queryrange.Merger
	metrics *SplitByMetrics
}

// SplitByIntervalMiddleware creates a new Middleware that splits log requests by a given interval.
func SplitByIntervalMiddleware(limits Limits, merger queryrange.Merger, metrics *SplitByMetrics) queryrange.Middleware {
	return queryrange.MiddlewareFunc(func(next queryrange.Handler) queryrange.Handler {
		return &splitByInterval{
			next:    next,
			limits:  limits,
			merger:  merger,
			metrics: metrics,
		}
	})
}

func (h *splitByInterval) Feed(ctx context.Context, input []*lokiResult) chan *lokiResult {
	ch := make(chan *lokiResult)

	go func() {
		defer close(ch)
		for _, d := range input {
			select {
			case <-ctx.Done():
				return
			case ch <- d:
				continue
			}
		}
	}()

	return ch
}

func (h *splitByInterval) Process(
	ctx context.Context,
	parallelism int,
	threshold int64,
	input []*lokiResult,
) ([]queryrange.Response, error) {
	var responses []queryrange.Response
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	ch := h.Feed(ctx, input)

	// queries with 0 limits should not be exited early
	var unlimited bool
	if threshold == 0 {
		unlimited = true
	}

	// don't spawn unnecessary goroutines
	var p int = parallelism
	if len(input) < parallelism {
		p = len(input)
	}

	for i := 0; i < p; i++ {
		go h.loop(ctx, ch)
	}

	for _, x := range input {
		select {
		case <-ctx.Done():
			return nil, ctx.Err()
		case data := <-x.ch:
			if data.err != nil {
				return nil, data.err
			}

			responses = append(responses, data.resp)

			// see if we can exit early if a limit has been reached
			if casted, ok := data.resp.(*LokiResponse); !unlimited && ok {
				threshold -= casted.Count()

				if threshold <= 0 {
					return responses, nil
				}

			}

		}

	}

	return responses, nil
}

func (h *splitByInterval) loop(ctx context.Context, ch <-chan *lokiResult) {

	for data := range ch {

		sp, ctx := opentracing.StartSpanFromContext(ctx, "interval")
		data.req.LogToSpan(sp)

		resp, err := h.next.Do(ctx, data.req)

		select {
		case <-ctx.Done():
			sp.Finish()
			return
		case data.ch <- &packedResp{resp, err}:
			sp.Finish()
		}
	}
}

func (h *splitByInterval) Do(ctx context.Context, r queryrange.Request) (queryrange.Response, error) {

	userid, err := user.ExtractOrgID(ctx)
	if err != nil {
		return nil, httpgrpc.Errorf(http.StatusBadRequest, err.Error())
	}

	interval := h.limits.QuerySplitDuration(userid)
	// skip split by if unset
	if interval == 0 {
		return h.next.Do(ctx, r)
	}

	intervals := splitByTime(r, interval)
	h.metrics.splits.Observe(float64(len(intervals)))

	// no interval should not be processed by the frontend.
	if len(intervals) == 0 {
		return h.next.Do(ctx, r)
	}

	if sp := opentracing.SpanFromContext(ctx); sp != nil {
		sp.LogFields(otlog.Int("n_intervals", len(intervals)))

	}

	var limit int64
	switch req := r.(type) {
	case *LokiRequest:
		limit = int64(req.Limit)
		if req.Direction == logproto.BACKWARD {
			for i, j := 0, len(intervals)-1; i < j; i, j = i+1, j-1 {
				intervals[i], intervals[j] = intervals[j], intervals[i]
			}
		}
	case *LokiSeriesRequest, *LokiLabelNamesRequest:
		// Set this to 0 since this is not used in Series/Labels Request.
		limit = 0
	default:
		return nil, httpgrpc.Errorf(http.StatusBadRequest, "unknown request type")
	}

	input := make([]*lokiResult, 0, len(intervals))
	for _, interval := range intervals {
		input = append(input, &lokiResult{
			req: interval,
			ch:  make(chan *packedResp),
		})
	}

	resps, err := h.Process(ctx, h.limits.MaxQueryParallelism(userid), limit, input)
	if err != nil {
		return nil, err
	}
	return h.merger.MergeResponse(resps...)
}

func splitByTime(req queryrange.Request, interval time.Duration) []queryrange.Request {
	var reqs []queryrange.Request

	switch r := req.(type) {
	case *LokiRequest:
		forInterval(interval, r.StartTs, r.EndTs, func(start, end time.Time) {
			reqs = append(reqs, &LokiRequest{
				Query:     r.Query,
				Limit:     r.Limit,
				Step:      r.Step,
				Direction: r.Direction,
				Path:      r.Path,
				StartTs:   start,
				EndTs:     end,
			})
		})
	case *LokiSeriesRequest:
		forInterval(interval, r.StartTs, r.EndTs, func(start, end time.Time) {
			reqs = append(reqs, &LokiSeriesRequest{
				Match:   r.Match,
				Path:    r.Path,
				StartTs: start,
				EndTs:   end,
			})
		})
	case *LokiLabelNamesRequest:
		forInterval(interval, r.StartTs, r.EndTs, func(start, end time.Time) {
			reqs = append(reqs, &LokiLabelNamesRequest{
				Path:    r.Path,
				StartTs: start,
				EndTs:   end,
			})
		})
	default:
		return nil
	}
	return reqs

}

func forInterval(interval time.Duration, start, end time.Time, callback func(start, end time.Time)) {
	for start := start; start.Before(end); start = start.Add(interval) {
		newEnd := start.Add(interval)
		if newEnd.After(end) {
			newEnd = end
		}
		callback(start, newEnd)
	}
}
<|endoftext|>