<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Rob Murtha robmurtha@gmail.com [:robmurtha]

package main

import (
	"fmt"
	"testing"

	"gopkg.in/gcfg.v1"
)

func TestConfigLoadDefault(t *testing.T) {
	// loading the default config produces an error
	expect := `configLoad() -> config.Certs.Ca open /path/to/ca/cert: no such file or directory`
	err := configLoad("../conf/mig-agent.cfg.inc")
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
}

func TestConfigLoadCerts(t *testing.T) {
	// test that configured cert files are loaded
	path := `../conf/mig-agent.cfg.inc`
	var config config
	err := gcfg.ReadFileInto(&config, path)
	if err != nil {
		t.Error("expected to read", path, "got", err)
		t.FailNow()
	}

	globals := globals{}
	config.Certs.Key = "../conf/mig-agent.cfg.inc"
	config.Certs.Ca = "../conf/mig-agent.cfg.inc"
	config.Certs.Cert = "../conf/mig-agent.cfg.inc"
	expect := `no error`
	err = globals.parseConfig(config)
	if err != nil {
		t.Error("expected", expect, "got", err)
	}
	expect = `agentCert not empty`
	if len(AGENTCERT) == 0 {
		t.Error("expected", expect)
	}
	expect = `agentKey not empty`
	if len(AGENTKEY) == 0 {
		t.Error("expected", expect)
	}
	expect = `caCert not empty`
	if len(CACERT) == 0 {
		t.Error("expected", expect)
	}
}

func TestConfigLoadEmptyCerts(t *testing.T) {
	// test that empty certs are ok and the defaults are used
	globals := newGlobals()

	expect := "caCert not empty"
	if len(globals.caCert) == 0 {
		t.Error("expected", expect)
	}
	expect = "agentCert not empty"
	if len(globals.agentCert) == 0 {
		t.Error("expected", expect)
	}
	expect = "agentKey not empty"
	if len(globals.agentKey) == 0 {
		t.Error("expected", expect)
	}

	path := `../conf/mig-agent.cfg.inc`
	var config config
	err := gcfg.ReadFileInto(&config, path)
	if err != nil {
		t.Error("expected to read", path, "got", err)
		t.FailNow()
	}

	config.Certs.Ca = ""
	config.Certs.Cert = ""
	config.Certs.Key = ""

	expect = "no error"
	err = globals.parseConfig(config)
	if err != nil {
		t.Error("expected", expect, "got", err)
	}

	//verify defaults are intact
	expect = "caCert not empty"
	if len(globals.caCert) == 0 {
		t.Error("expected", expect)
	}
	expect = "agentCert not empty"
	if len(globals.agentCert) == 0 {
		t.Error("expected", expect)
	}
	expect = "agentKey not empty"
	if len(globals.agentKey) == 0 {
		t.Error("expected", expect)
	}
}

func TestConfigLoadCertErrors(t *testing.T) {
	// test that an informative error is returned for invalid cert paths
	path := `../conf/mig-agent.cfg.inc`
	var config config
	expect := `no error`
	err := gcfg.ReadFileInto(&config, path)
	if err != nil {
		t.Error("expected", expect, "got", err)
		t.FailNow()
	}

	// start with empty globals for testing vs populated via NewGlobals
	globals := &globals{}

	expect = `config.Certs.Ca open /path/to/ca/cert: no such file or directory`
	err = globals.parseConfig(config)
	if err.Error() != expect {
		t.Error(fmt.Sprintf("expected %v got %v", expect, err))
	}
	config.Certs.Ca = ""

	expect = `config.Certs.Cert open /path/to/client/cert: no such file or directory`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
	config.Certs.Cert = ""

	expect = `config.Certs.Key open /path/to/private/key: no such file or directory`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
	config.Certs.Key = ""

	expect = "no error"
	err = globals.parseConfig(config)
	if err != nil {
		t.Error("expected", expect, "got", err)
	}
}

func TestConfigParseDurationErrors(t *testing.T) {
	// test that an informative error is returned for invalid durations
	var config config
	var globals globals

	path := `../conf/mig-agent.cfg.inc`
	expect := `no error`
	err := gcfg.ReadFileInto(&config, path)
	if err != nil {
		t.Error("expected", expect, "got", err)
		t.FailNow()
	}
	config.Certs.Ca = ""
	config.Certs.Cert = ""
	config.Certs.Key = ""

	config.Agent.RefreshEnv = "300"
	expect = `config.Agent.RefreshEnv time: missing unit in duration 300`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
	config.Agent.RefreshEnv = "300s"

	config.Agent.HeartbeatFreq = "300"
	expect = `config.Agent.HeartbeatFreq time: missing unit in duration 300`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
	config.Agent.HeartbeatFreq = "300s"

	config.Agent.ModuleTimeout = "300"
	expect = `config.Agent.ModuleTimeout time: missing unit in duration 300`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
}
<commit_msg>[minor] initialize global.tags map in tests where we dont use newGlobals<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Rob Murtha robmurtha@gmail.com [:robmurtha]

package main

import (
	"fmt"
	"testing"

	"gopkg.in/gcfg.v1"
)

func TestConfigLoadDefault(t *testing.T) {
	// loading the default config produces an error
	expect := `configLoad() -> config.Certs.Ca open /path/to/ca/cert: no such file or directory`
	err := configLoad("../conf/mig-agent.cfg.inc")
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
}

func TestConfigLoadCerts(t *testing.T) {
	// test that configured cert files are loaded
	path := `../conf/mig-agent.cfg.inc`
	var config config
	err := gcfg.ReadFileInto(&config, path)
	if err != nil {
		t.Error("expected to read", path, "got", err)
		t.FailNow()
	}

	globals := globals{}
	globals.tags = make(map[string]string)
	config.Certs.Key = "../conf/mig-agent.cfg.inc"
	config.Certs.Ca = "../conf/mig-agent.cfg.inc"
	config.Certs.Cert = "../conf/mig-agent.cfg.inc"
	expect := `no error`
	err = globals.parseConfig(config)
	if err != nil {
		t.Error("expected", expect, "got", err)
	}
	expect = `agentCert not empty`
	if len(AGENTCERT) == 0 {
		t.Error("expected", expect)
	}
	expect = `agentKey not empty`
	if len(AGENTKEY) == 0 {
		t.Error("expected", expect)
	}
	expect = `caCert not empty`
	if len(CACERT) == 0 {
		t.Error("expected", expect)
	}
}

func TestConfigLoadEmptyCerts(t *testing.T) {
	// test that empty certs are ok and the defaults are used
	globals := newGlobals()

	expect := "caCert not empty"
	if len(globals.caCert) == 0 {
		t.Error("expected", expect)
	}
	expect = "agentCert not empty"
	if len(globals.agentCert) == 0 {
		t.Error("expected", expect)
	}
	expect = "agentKey not empty"
	if len(globals.agentKey) == 0 {
		t.Error("expected", expect)
	}

	path := `../conf/mig-agent.cfg.inc`
	var config config
	err := gcfg.ReadFileInto(&config, path)
	if err != nil {
		t.Error("expected to read", path, "got", err)
		t.FailNow()
	}

	config.Certs.Ca = ""
	config.Certs.Cert = ""
	config.Certs.Key = ""

	expect = "no error"
	err = globals.parseConfig(config)
	if err != nil {
		t.Error("expected", expect, "got", err)
	}

	//verify defaults are intact
	expect = "caCert not empty"
	if len(globals.caCert) == 0 {
		t.Error("expected", expect)
	}
	expect = "agentCert not empty"
	if len(globals.agentCert) == 0 {
		t.Error("expected", expect)
	}
	expect = "agentKey not empty"
	if len(globals.agentKey) == 0 {
		t.Error("expected", expect)
	}
}

func TestConfigLoadCertErrors(t *testing.T) {
	// test that an informative error is returned for invalid cert paths
	path := `../conf/mig-agent.cfg.inc`
	var config config
	expect := `no error`
	err := gcfg.ReadFileInto(&config, path)
	if err != nil {
		t.Error("expected", expect, "got", err)
		t.FailNow()
	}

	// start with empty globals for testing vs populated via NewGlobals
	globals := &globals{}
	globals.tags = make(map[string]string)

	expect = `config.Certs.Ca open /path/to/ca/cert: no such file or directory`
	err = globals.parseConfig(config)
	if err.Error() != expect {
		t.Error(fmt.Sprintf("expected %v got %v", expect, err))
	}
	config.Certs.Ca = ""

	expect = `config.Certs.Cert open /path/to/client/cert: no such file or directory`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
	config.Certs.Cert = ""

	expect = `config.Certs.Key open /path/to/private/key: no such file or directory`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
	config.Certs.Key = ""

	expect = "no error"
	err = globals.parseConfig(config)
	if err != nil {
		t.Error("expected", expect, "got", err)
	}
}

func TestConfigParseDurationErrors(t *testing.T) {
	// test that an informative error is returned for invalid durations
	var config config

	globals := &globals{}
	globals.tags = make(map[string]string)

	path := `../conf/mig-agent.cfg.inc`
	expect := `no error`
	err := gcfg.ReadFileInto(&config, path)
	if err != nil {
		t.Error("expected", expect, "got", err)
		t.FailNow()
	}
	config.Certs.Ca = ""
	config.Certs.Cert = ""
	config.Certs.Key = ""

	config.Agent.RefreshEnv = "300"
	expect = `config.Agent.RefreshEnv time: missing unit in duration 300`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
	config.Agent.RefreshEnv = "300s"

	config.Agent.HeartbeatFreq = "300"
	expect = `config.Agent.HeartbeatFreq time: missing unit in duration 300`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
	config.Agent.HeartbeatFreq = "300s"

	config.Agent.ModuleTimeout = "300"
	expect = `config.Agent.ModuleTimeout time: missing unit in duration 300`
	err = globals.parseConfig(config)
	if err == nil || err.Error() != expect {
		t.Error("expected", expect, "got", err)
	}
}
<|endoftext|>


<commit_before>package service_test

import (
	"encoding/json"
	"fmt"
	"os"
	"time"

	"github.com/pborman/uuid"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/runner"
	"github.com/cloudfoundry-incubator/cf-test-helpers/services"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

type rabbitmqTestConfig struct {
	services.Config

	ServiceName     string   `json:"service_name"`
	PlanNames       []string `json:"plan_names"`
	RabbitMQSkipSSL bool     `json:"rabbitmq_skip_ssl"`
}

func loadConfig() (testConfig rabbitmqTestConfig) {
	path := os.Getenv("CONFIG_PATH")
	configFile, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	decoder := json.NewDecoder(configFile)
	err = decoder.Decode(&testConfig)
	if err != nil {
		panic(err)
	}

	return testConfig
}

var config = loadConfig()

var _ = Describe("RabbitMQ Service", func() {
	var timeout = time.Second * 60
	var retryInterval = time.Second * 1
	var appPath = "../assets/cf-rabbitmq-example-app"

	var appName string

	randomName := func() string {
		return uuid.NewRandom().String()
	}

	appUri := func(appName string) string {
		return "https://" + appName + "." + config.AppsDomain
	}

	assertAppIsRunning := func(appName string) {
		pingUri := appUri(appName) + "/ping"
		fmt.Println("Checking that the app is responding at url: ", pingUri)
		Eventually(runner.Curl(pingUri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say("OK"))
		fmt.Println("\n")
	}

	BeforeSuite(func() {
		config.TimeoutScale = 3
		services.NewContext(config.Config, "rabbitmq-smoke-test").Setup()
	})

	AssertLifeCycleBehavior := func(planName string) {
		var serviceInstanceName string
		appPushed := false
		serviceCreated := false
		serviceBound := false
		appIsRunning := false

		It("Should be able to push the application", func() {
			appName = randomName()
			Eventually(cf.Cf("push", appName, "-m", "256M", "-p", appPath, "-s", "cflinuxfs2", "-no-start"), config.ScaledTimeout(timeout)).Should(Exit(0))
			appPushed = true
		})

		It("Can create the service instance", func() {
			Ω(appPushed).Should(BeTrue())
			serviceInstanceName = randomName()
			Eventually(cf.Cf("create-service", config.ServiceName, planName, serviceInstanceName), config.ScaledTimeout(timeout)).Should(Exit(0))
			serviceCreated = true
		})

		It("Can bind the service and start the application", func() {
			Ω(appPushed && serviceCreated).Should(BeTrue())
			Eventually(cf.Cf("bind-service", appName, serviceInstanceName), config.ScaledTimeout(timeout)).Should(Exit(0))
			serviceBound = true
			var RMQSkipSSLValue string = "0"
			if config.RabbitMQSkipSSL {
				RMQSkipSSLValue = "1"
			}
			Eventually(cf.Cf("set-env", appName, "RABBITMQ_SKIP_SSL", RMQSkipSSLValue), config.ScaledTimeout(5*time.Minute)).Should(Exit(0))
			Eventually(cf.Cf("start", appName), config.ScaledTimeout(5*time.Minute)).Should(Exit(0))
			assertAppIsRunning(appName)
			appIsRunning = true
		})

		It("can write to and read from a service instance using the "+planName+" plan", func() {
			Ω(appPushed && serviceCreated && serviceBound && appIsRunning).Should(BeTrue())
			/*
			   create a queue     (should 201)
			   list the queues    (should 200)
			   subscribe          (should 204)
			   publish            (should 201)
			   subscribe          (should 200)
			*/
			uri := appUri(appName) + "/queues"
			fmt.Println("Creating a new queue: ", uri)
			Eventually(runner.Curl(uri, "-k", "-X", "POST", "-d", "name=test-q"), config.ScaledTimeout(timeout), retryInterval).Should(Say("SUCCESS"))
			fmt.Println("\n")

			fmt.Println("Listing the queues: ", uri)
			Eventually(runner.Curl(uri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say("test-q\n"))
			fmt.Println("\n")

			uri = appUri(appName) + "/queue/test-q"
			fmt.Println("Reading from the (empty) queue: ", uri)
			Eventually(runner.Curl(uri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say(""))
			fmt.Println("\n")

			fmt.Println("Publishing to the queue: ", uri)
			Eventually(runner.Curl("-d", "data=test-message", "-X", "PUT", uri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say("SUCCESS"))
			fmt.Println("\n")

			fmt.Println("Reading from the (non-empty) queue: ", uri)
			Eventually(runner.Curl(uri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say("test-message"))
			fmt.Println("\n")
		})

		It("Should be able to clean up after itself", func() {
			if serviceBound {
				Eventually(cf.Cf("unbind-service", appName, serviceInstanceName), config.ScaledTimeout(timeout)).Should(Exit(0))
			}
			if serviceCreated {
				Eventually(cf.Cf("delete-service", "-f", serviceInstanceName), config.ScaledTimeout(timeout)).Should(Exit(0))
			}
			if appPushed {
				Eventually(cf.Cf("delete", appName, "-f"), config.ScaledTimeout(timeout)).Should(Exit(0))
			}
		})
	}

	Context("for each plan", func() {
		for _, planName := range config.PlanNames {
			AssertLifeCycleBehavior(planName)
		}
	})
})
<commit_msg>org, space, and users created now get cleaned up<commit_after>package service_test

import (
	"encoding/json"
	"fmt"
	"os"
	"time"

	"github.com/pborman/uuid"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/runner"
	"github.com/cloudfoundry-incubator/cf-test-helpers/services"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

type rabbitmqTestConfig struct {
	services.Config

	ServiceName     string   `json:"service_name"`
	PlanNames       []string `json:"plan_names"`
	RabbitMQSkipSSL bool     `json:"rabbitmq_skip_ssl"`
}

func loadConfig() (testConfig rabbitmqTestConfig) {
	path := os.Getenv("CONFIG_PATH")
	configFile, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	decoder := json.NewDecoder(configFile)
	err = decoder.Decode(&testConfig)
	if err != nil {
		panic(err)
	}

	return testConfig
}

var config = loadConfig()
var context services.Context

var _ = Describe("RabbitMQ Service", func() {
	var timeout = time.Second * 60
	var retryInterval = time.Second * 1
	var appPath = "../assets/cf-rabbitmq-example-app"

	var appName string

	randomName := func() string {
		return uuid.NewRandom().String()
	}

	appUri := func(appName string) string {
		return "https://" + appName + "." + config.AppsDomain
	}

	assertAppIsRunning := func(appName string) {
		pingUri := appUri(appName) + "/ping"
		fmt.Println("Checking that the app is responding at url: ", pingUri)
		Eventually(runner.Curl(pingUri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say("OK"))
		fmt.Println("\n")
	}

	BeforeSuite(func() {
		config.TimeoutScale = 3
		context = services.NewContext(config.Config, "rabbitmq-smoke-test")
		context.Setup()
	})

	AfterSuite(func() {
		context.Teardown()
	})

	AssertLifeCycleBehavior := func(planName string) {
		var serviceInstanceName string
		appPushed := false
		serviceCreated := false
		serviceBound := false
		appIsRunning := false

		It("Should be able to push the application", func() {
			appName = randomName()
			Eventually(cf.Cf("push", appName, "-m", "256M", "-p", appPath, "-s", "cflinuxfs2", "-no-start"), config.ScaledTimeout(timeout)).Should(Exit(0))
			appPushed = true
		})

		It("Can create the service instance", func() {
			Ω(appPushed).Should(BeTrue())
			serviceInstanceName = randomName()
			Eventually(cf.Cf("create-service", config.ServiceName, planName, serviceInstanceName), config.ScaledTimeout(timeout)).Should(Exit(0))
			serviceCreated = true
		})

		It("Can bind the service and start the application", func() {
			Ω(appPushed && serviceCreated).Should(BeTrue())
			Eventually(cf.Cf("bind-service", appName, serviceInstanceName), config.ScaledTimeout(timeout)).Should(Exit(0))
			serviceBound = true
			var RMQSkipSSLValue string = "0"
			if config.RabbitMQSkipSSL {
				RMQSkipSSLValue = "1"
			}
			Eventually(cf.Cf("set-env", appName, "RABBITMQ_SKIP_SSL", RMQSkipSSLValue), config.ScaledTimeout(5*time.Minute)).Should(Exit(0))
			Eventually(cf.Cf("start", appName), config.ScaledTimeout(5*time.Minute)).Should(Exit(0))
			assertAppIsRunning(appName)
			appIsRunning = true
		})

		It("can write to and read from a service instance using the "+planName+" plan", func() {
			Ω(appPushed && serviceCreated && serviceBound && appIsRunning).Should(BeTrue())
			/*
			   create a queue     (should 201)
			   list the queues    (should 200)
			   subscribe          (should 204)
			   publish            (should 201)
			   subscribe          (should 200)
			*/
			uri := appUri(appName) + "/queues"
			fmt.Println("Creating a new queue: ", uri)
			Eventually(runner.Curl(uri, "-k", "-X", "POST", "-d", "name=test-q"), config.ScaledTimeout(timeout), retryInterval).Should(Say("SUCCESS"))
			fmt.Println("\n")

			fmt.Println("Listing the queues: ", uri)
			Eventually(runner.Curl(uri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say("test-q\n"))
			fmt.Println("\n")

			uri = appUri(appName) + "/queue/test-q"
			fmt.Println("Reading from the (empty) queue: ", uri)
			Eventually(runner.Curl(uri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say(""))
			fmt.Println("\n")

			fmt.Println("Publishing to the queue: ", uri)
			Eventually(runner.Curl("-d", "data=test-message", "-X", "PUT", uri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say("SUCCESS"))
			fmt.Println("\n")

			fmt.Println("Reading from the (non-empty) queue: ", uri)
			Eventually(runner.Curl(uri, "-k"), config.ScaledTimeout(timeout), retryInterval).Should(Say("test-message"))
			fmt.Println("\n")
		})

		It("Should be able to clean up after itself", func() {
			if serviceBound {
				Eventually(cf.Cf("unbind-service", appName, serviceInstanceName), config.ScaledTimeout(timeout)).Should(Exit(0))
			}
			if serviceCreated {
				Eventually(cf.Cf("delete-service", "-f", serviceInstanceName), config.ScaledTimeout(timeout)).Should(Exit(0))
			}
			if appPushed {
				Eventually(cf.Cf("delete", appName, "-f"), config.ScaledTimeout(timeout)).Should(Exit(0))
			}
		})
	}

	Context("for each plan", func() {
		for _, planName := range config.PlanNames {
			AssertLifeCycleBehavior(planName)
		}
	})
})
<|endoftext|>


<commit_before>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package pull

import (
	"fmt"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
)

// Update updates pull request with base branch.
func Update(pull *models.PullRequest, doer *models.User, message string) error {
	//use merge functions but switch repo's and branch's
	pr := &models.PullRequest{
		HeadRepoID: pull.BaseRepoID,
		BaseRepoID: pull.HeadRepoID,
		HeadBranch: pull.BaseBranch,
		BaseBranch: pull.HeadBranch,
	}

	if err := pr.LoadHeadRepo(); err != nil {
		log.Error("LoadHeadRepo: %v", err)
		return fmt.Errorf("LoadHeadRepo: %v", err)
	} else if err = pr.LoadBaseRepo(); err != nil {
		log.Error("LoadBaseRepo: %v", err)
		return fmt.Errorf("LoadBaseRepo: %v", err)
	}

	diffCount, err := GetDiverging(pull)
	if err != nil {
		return err
	} else if diffCount.Behind == 0 {
		return fmt.Errorf("HeadBranch of PR %d is up to date", pull.Index)
	}

	_, err = rawMerge(pr, doer, models.MergeStyleMerge, message)

	defer func() {
		go AddTestPullRequestTask(doer, pr.HeadRepo.ID, pr.HeadBranch, false, "", "")
	}()

	return err
}

// IsUserAllowedToUpdate check if user is allowed to update PR with given permissions and branch protections
func IsUserAllowedToUpdate(pull *models.PullRequest, user *models.User) (bool, error) {
	headRepoPerm, err := models.GetUserRepoPermission(pull.HeadRepo, user)
	if err != nil {
		return false, err
	}

	pr := &models.PullRequest{
		HeadRepoID: pull.BaseRepoID,
		BaseRepoID: pull.HeadRepoID,
		HeadBranch: pull.BaseBranch,
		BaseBranch: pull.HeadBranch,
	}

	err = pr.LoadProtectedBranch()
	if err != nil {
		return false, err
	}

	// Update function need push permission
	if pr.ProtectedBranch != nil && !pr.ProtectedBranch.CanUserPush(user.ID) {
		return false, nil
	}

	return IsUserAllowedToMerge(pr, headRepoPerm, user)
}

// GetDiverging determines how many commits a PR is ahead or behind the PR base branch
func GetDiverging(pr *models.PullRequest) (*git.DivergeObject, error) {
	log.Trace("GetDiverging[%d]: compare commits", pr.ID)
	if err := pr.LoadBaseRepo(); err != nil {
		return nil, err
	}
	if err := pr.LoadHeadRepo(); err != nil {
		return nil, err
	}

	tmpRepo, err := createTemporaryRepo(pr)
	if err != nil {
		log.Error("CreateTemporaryPath: %v", err)
		return nil, err
	}
	defer func() {
		if err := models.RemoveTemporaryPath(tmpRepo); err != nil {
			log.Error("Merge: RemoveTemporaryPath: %s", err)
		}
	}()

	diff, err := git.GetDivergingCommits(tmpRepo, "base", "tracking")
	return &diff, err
}
<commit_msg>IsUserAllowedToUpdate ignore igonre if user is nil (#14885)<commit_after>// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package pull

import (
	"fmt"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/git"
	"code.gitea.io/gitea/modules/log"
)

// Update updates pull request with base branch.
func Update(pull *models.PullRequest, doer *models.User, message string) error {
	//use merge functions but switch repo's and branch's
	pr := &models.PullRequest{
		HeadRepoID: pull.BaseRepoID,
		BaseRepoID: pull.HeadRepoID,
		HeadBranch: pull.BaseBranch,
		BaseBranch: pull.HeadBranch,
	}

	if err := pr.LoadHeadRepo(); err != nil {
		log.Error("LoadHeadRepo: %v", err)
		return fmt.Errorf("LoadHeadRepo: %v", err)
	} else if err = pr.LoadBaseRepo(); err != nil {
		log.Error("LoadBaseRepo: %v", err)
		return fmt.Errorf("LoadBaseRepo: %v", err)
	}

	diffCount, err := GetDiverging(pull)
	if err != nil {
		return err
	} else if diffCount.Behind == 0 {
		return fmt.Errorf("HeadBranch of PR %d is up to date", pull.Index)
	}

	_, err = rawMerge(pr, doer, models.MergeStyleMerge, message)

	defer func() {
		go AddTestPullRequestTask(doer, pr.HeadRepo.ID, pr.HeadBranch, false, "", "")
	}()

	return err
}

// IsUserAllowedToUpdate check if user is allowed to update PR with given permissions and branch protections
func IsUserAllowedToUpdate(pull *models.PullRequest, user *models.User) (bool, error) {
	if user == nil {
		return false, nil
	}
	headRepoPerm, err := models.GetUserRepoPermission(pull.HeadRepo, user)
	if err != nil {
		return false, err
	}

	pr := &models.PullRequest{
		HeadRepoID: pull.BaseRepoID,
		BaseRepoID: pull.HeadRepoID,
		HeadBranch: pull.BaseBranch,
		BaseBranch: pull.HeadBranch,
	}

	err = pr.LoadProtectedBranch()
	if err != nil {
		return false, err
	}

	// Update function need push permission
	if pr.ProtectedBranch != nil && !pr.ProtectedBranch.CanUserPush(user.ID) {
		return false, nil
	}

	return IsUserAllowedToMerge(pr, headRepoPerm, user)
}

// GetDiverging determines how many commits a PR is ahead or behind the PR base branch
func GetDiverging(pr *models.PullRequest) (*git.DivergeObject, error) {
	log.Trace("GetDiverging[%d]: compare commits", pr.ID)
	if err := pr.LoadBaseRepo(); err != nil {
		return nil, err
	}
	if err := pr.LoadHeadRepo(); err != nil {
		return nil, err
	}

	tmpRepo, err := createTemporaryRepo(pr)
	if err != nil {
		log.Error("CreateTemporaryPath: %v", err)
		return nil, err
	}
	defer func() {
		if err := models.RemoveTemporaryPath(tmpRepo); err != nil {
			log.Error("Merge: RemoveTemporaryPath: %s", err)
		}
	}()

	diff, err := git.GetDivergingCommits(tmpRepo, "base", "tracking")
	return &diff, err
}
<|endoftext|>


<commit_before>package sofa

import (
	"fmt"
	"testing"
	"time"

	"github.com/nbio/st"
	"gopkg.in/h2non/gock.v1"
)

var (
	ReplicatorTestDB       = "replicator_test"
	ReplicatorTestTargetDB = "replicator_test_target"
	ReplicatorTestName     = "my_repl"
)

func TestReplicatorSave(t *testing.T) {
	defer gock.Off()

	gock.New(fmt.Sprintf("https://%s", globalTestConnections.Version1MockHost)).
		Put("/_replicator/_myrepl").
		Reply(201).
		JSON(map[string]interface{}{
			"ok":  true,
			"id":  "_myrepl",
			"rev": "1-801609c9fdb4c6d196820c5b1f3c26c9",
		}).
		SetHeader("Etag", `"1-801609c9fdb4c6d196820c5b1f3c26c9"`)

	con := globalTestConnections.Version1(t, true)

	repl := NewReplication("_myrepl", "mydb", "yourdb")
	repl.CreateTarget = true
	repl.Continuous = true

	rev, err := con.PutReplication(&repl)
	st.Assert(t, err, nil)

	st.Assert(t, rev, "1-801609c9fdb4c6d196820c5b1f3c26c9")
}

func TestReplicatorReal(t *testing.T) {
	con := globalTestConnections.Version1(t, false)

	// Create a new database
	_, err := con.CreateDatabase(ReplicatorTestDB)
	if err != nil {
		t.Fatal(err)
	}

	getRepl, err := con.Replication(ReplicatorTestName, "")
	if err != nil {
		if !ErrorStatus(err, 404) {
			t.Fatal(err)
		}
	} else {
		_, err := con.DeleteReplication(&getRepl)
		if err != nil {
			t.Fatal(err)
		}
	}

	putRepl := NewReplication(ReplicatorTestName, ReplicatorTestDB, ReplicatorTestTargetDB)
	putRepl.CreateTarget = true
	putRepl.Continuous = true
	putRepl.Owner = "admin"
	putRepl.UserContext = map[string]interface{}{"roles": []string{"_admin"}}

	_, err = con.PutReplication(&putRepl)
	st.Assert(t, err, nil)

	// This loop handles the case where CouchDB is slow to create the database after the replication
	// is created. This seems to get slower after the first time the replication is created but the
	// race condition is always present.
	getDBAttempts := 1
	for {
		_, err = con.EnsureDatabase(ReplicatorTestTargetDB)
		if err != nil {
			if getDBAttempts >= 10 {
				fmt.Println(time.Now())
				t.Fatalf("couldn't get the created database after %d attempts: %s", getDBAttempts, err)
			}

			time.Sleep(1 * time.Second)
			getDBAttempts++

			continue
		}

		break
	}

	// This loop is to handle the case where we were able to retrieve the DB before CouchDB had updated
	// the replicator document to account for this - in this case the replicator document can have the state
	// set to error with the reason: "db_not_found: could not open replicator_test_target"
	getReplAttempts := 1
	for {
		getRepl, err = con.Replication(ReplicatorTestName, "")
		st.Assert(t, err, nil)

		if getRepl.ReplicationState == "error" {
			if getReplAttempts >= 3 {
				fmt.Println(time.Now())
				t.Fatalf("replication still in error after %d attempts: %s", getReplAttempts, getRepl.ReplicationStateReason)
			}

			time.Sleep(1 * time.Second)
			getReplAttempts++

			continue
		}

		break
	}

	_, err = con.DeleteReplication(&getRepl)
	st.Assert(t, err, nil)

	err = con.DeleteDatabase(ReplicatorTestDB)
	st.Assert(t, err, nil)

	err = con.DeleteDatabase(ReplicatorTestTargetDB)
	st.Assert(t, err, nil)
}
<commit_msg>Defer the cleanup so that it always happens<commit_after>package sofa

import (
	"fmt"
	"testing"
	"time"

	"github.com/nbio/st"
	"gopkg.in/h2non/gock.v1"
)

var (
	ReplicatorTestDB       = "replicator_test"
	ReplicatorTestTargetDB = "replicator_test_target"
	ReplicatorTestName     = "my_repl"
)

func TestReplicatorSave(t *testing.T) {
	defer gock.Off()

	gock.New(fmt.Sprintf("https://%s", globalTestConnections.Version1MockHost)).
		Put("/_replicator/_myrepl").
		Reply(201).
		JSON(map[string]interface{}{
			"ok":  true,
			"id":  "_myrepl",
			"rev": "1-801609c9fdb4c6d196820c5b1f3c26c9",
		}).
		SetHeader("Etag", `"1-801609c9fdb4c6d196820c5b1f3c26c9"`)

	con := globalTestConnections.Version1(t, true)

	repl := NewReplication("_myrepl", "mydb", "yourdb")
	repl.CreateTarget = true
	repl.Continuous = true

	rev, err := con.PutReplication(&repl)
	st.Assert(t, err, nil)

	st.Assert(t, rev, "1-801609c9fdb4c6d196820c5b1f3c26c9")
}

func TestReplicatorReal(t *testing.T) {
	con := globalTestConnections.Version1(t, false)

	// Run the cleanup as a best effort task - failures here do not case the test to fail
	defer func() {
		if err := con.DeleteDatabase(ReplicatorTestDB); err != nil {
			t.Logf("error cleaning up source DB: %s", err)
		}

		if err := con.DeleteDatabase(ReplicatorTestTargetDB); err != nil {
			t.Logf("error cleaning up target DB: %s", err)
		}
	}()

	// Create a new database
	_, err := con.CreateDatabase(ReplicatorTestDB)
	if err != nil {
		t.Fatal(err)
	}

	getRepl, err := con.Replication(ReplicatorTestName, "")
	if err != nil {
		if !ErrorStatus(err, 404) {
			t.Fatal(err)
		}
	} else {
		_, err := con.DeleteReplication(&getRepl)
		if err != nil {
			t.Fatal(err)
		}
	}

	putRepl := NewReplication(ReplicatorTestName, ReplicatorTestDB, ReplicatorTestTargetDB)
	putRepl.CreateTarget = true
	putRepl.Continuous = true
	putRepl.Owner = "admin"
	putRepl.UserContext = map[string]interface{}{"roles": []string{"_admin"}}

	_, err = con.PutReplication(&putRepl)
	st.Assert(t, err, nil)

	// This loop handles the case where CouchDB is slow to create the database after the replication
	// is created. This seems to get slower after the first time the replication is created but the
	// race condition is always present.
	getDBAttempts := 1
	for {
		_, err = con.EnsureDatabase(ReplicatorTestTargetDB)
		if err != nil {
			if getDBAttempts >= 10 {
				fmt.Println(time.Now())
				t.Fatalf("couldn't get the created database after %d attempts: %s", getDBAttempts, err)
			}

			time.Sleep(1 * time.Second)
			getDBAttempts++

			continue
		}

		break
	}

	// This loop is to handle the case where we were able to retrieve the DB before CouchDB had updated
	// the replicator document to account for this - in this case the replicator document can have the state
	// set to error with the reason: "db_not_found: could not open replicator_test_target"
	getReplAttempts := 1
	for {
		getRepl, err = con.Replication(ReplicatorTestName, "")
		st.Assert(t, err, nil)

		if getRepl.ReplicationState != "triggered" {
			if getReplAttempts >= 3 {
				fmt.Println(time.Now())
				t.Fatalf(
					"replication still in state '%s' after %d attempts: %s",
					getRepl.ReplicationState,
					getReplAttempts,
					getRepl.ReplicationStateReason,
				)
			}

			time.Sleep(1 * time.Second)
			getReplAttempts++

			continue
		}

		break
	}

	_, err = con.DeleteReplication(&getRepl)
	st.Assert(t, err, nil)
}
<|endoftext|>


<commit_before>package parser_test

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strconv"
	"strings"
	"testing"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/parser"
)

// Returns nil if two IP Lists are equal
func isEqualIPLists(listComp, list []parser.IPNode) error {
	for index, element := range list {
		err := parser.IsEqualIPNodes(element, listComp[index])
		if err != nil {
			return err
		}
	}
	return nil
}

// Returns nil if two Location lists are equal
func isEqualLocLists(list, listComp []parser.LocationNode) error {
	for index, element := range list {
		if index >= len(listComp) {
			output := fmt.Sprint("Out of range:", index)
			log.Println(output)
			return errors.New(output)
		}
		if element.GeonameID != listComp[index].GeonameID {
			output := strings.Join([]string{"GeonameID inconsistent\ngot:", strconv.Itoa(element.GeonameID), " \nwanted:", strconv.Itoa(listComp[index].GeonameID)}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.ContinentCode != listComp[index].ContinentCode {
			output := strings.Join([]string{"Continent code inconsistent\ngot:", element.ContinentCode, " \nwanted:", listComp[index].ContinentCode}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.CountryCode != listComp[index].CountryCode {
			output := strings.Join([]string{"Country code inconsistent\ngot:", element.CountryCode, " \nwanted:", listComp[index].CountryCode}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.CountryName != listComp[index].CountryName {
			output := strings.Join([]string{"Country name inconsistent\ngot:", element.CountryName, " \nwanted:", listComp[index].CountryName}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.MetroCode != listComp[index].MetroCode {
			output := strings.Join([]string{"Longitude inconsistent\ngot:", strconv.FormatInt(element.MetroCode, 16), " \nwanted:", strconv.FormatInt(listComp[index].MetroCode, 16)}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.CityName != listComp[index].CityName {
			output := strings.Join([]string{"Longitude inconsistent\ngot:", element.CityName, " \nwanted:", listComp[index].CityName}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.RegionName != listComp[index].RegionName {
			output := strings.Join([]string{"RegionName inconsistent\ngot:", element.RegionName, " \nwanted:", listComp[index].RegionName}, "")
			log.Println(output)
			return errors.New(output)
		}
	}
	return nil
}

func TestConvertIPNodeToGeoData(t *testing.T) {
	tests := []struct {
		node parser.IPNode
		locs []parser.LocationNode
		res  *api.GeoData
	}{
		{
			node: parser.IPNode{LocationIndex: 0, PostalCode: "10583"},
			locs: []parser.LocationNode{{CityName: "Not A Real City", RegionCode: "ME"}},
			res: &api.GeoData{
				Geo: &api.GeolocationIP{City: "Not A Real City", Postal_code: "10583", Region: "ME"},
				ASN: &api.IPASNData{}},
		},
		{
			node: parser.IPNode{LocationIndex: -1, PostalCode: "10583"},
			locs: nil,
			res: &api.GeoData{
				Geo: &api.GeolocationIP{Postal_code: "10583"},
				ASN: &api.IPASNData{}},
		},
	}
	for _, test := range tests {
		res := parser.ConvertIPNodeToGeoData(test.node, test.locs)
		if !reflect.DeepEqual(res, test.res) {
			t.Errorf("Expected %v, got %v", test.res, res)
		}
	}
}
<commit_msg>Add TODO for TestGetAnnotation<commit_after>package parser_test

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strconv"
	"strings"
	"testing"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/parser"
)

// Returns nil if two IP Lists are equal
func isEqualIPLists(listComp, list []parser.IPNode) error {
	for index, element := range list {
		err := parser.IsEqualIPNodes(element, listComp[index])
		if err != nil {
			return err
		}
	}
	return nil
}

// Returns nil if two Location lists are equal
func isEqualLocLists(list, listComp []parser.LocationNode) error {
	for index, element := range list {
		if index >= len(listComp) {
			output := fmt.Sprint("Out of range:", index)
			log.Println(output)
			return errors.New(output)
		}
		if element.GeonameID != listComp[index].GeonameID {
			output := strings.Join([]string{"GeonameID inconsistent\ngot:", strconv.Itoa(element.GeonameID), " \nwanted:", strconv.Itoa(listComp[index].GeonameID)}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.ContinentCode != listComp[index].ContinentCode {
			output := strings.Join([]string{"Continent code inconsistent\ngot:", element.ContinentCode, " \nwanted:", listComp[index].ContinentCode}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.CountryCode != listComp[index].CountryCode {
			output := strings.Join([]string{"Country code inconsistent\ngot:", element.CountryCode, " \nwanted:", listComp[index].CountryCode}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.CountryName != listComp[index].CountryName {
			output := strings.Join([]string{"Country name inconsistent\ngot:", element.CountryName, " \nwanted:", listComp[index].CountryName}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.MetroCode != listComp[index].MetroCode {
			output := strings.Join([]string{"Longitude inconsistent\ngot:", strconv.FormatInt(element.MetroCode, 16), " \nwanted:", strconv.FormatInt(listComp[index].MetroCode, 16)}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.CityName != listComp[index].CityName {
			output := strings.Join([]string{"Longitude inconsistent\ngot:", element.CityName, " \nwanted:", listComp[index].CityName}, "")
			log.Println(output)
			return errors.New(output)
		}
		if element.RegionName != listComp[index].RegionName {
			output := strings.Join([]string{"RegionName inconsistent\ngot:", element.RegionName, " \nwanted:", listComp[index].RegionName}, "")
			log.Println(output)
			return errors.New(output)
		}
	}
	return nil
}

func TestConvertIPNodeToGeoData(t *testing.T) {
	tests := []struct {
		node parser.IPNode
		locs []parser.LocationNode
		res  *api.GeoData
	}{
		{
			node: parser.IPNode{LocationIndex: 0, PostalCode: "10583"},
			locs: []parser.LocationNode{{CityName: "Not A Real City", RegionCode: "ME"}},
			res: &api.GeoData{
				Geo: &api.GeolocationIP{City: "Not A Real City", Postal_code: "10583", Region: "ME"},
				ASN: &api.IPASNData{}},
		},
		{
			node: parser.IPNode{LocationIndex: -1, PostalCode: "10583"},
			locs: nil,
			res: &api.GeoData{
				Geo: &api.GeolocationIP{Postal_code: "10583"},
				ASN: &api.IPASNData{}},
		},
	}
	for _, test := range tests {
		res := parser.ConvertIPNodeToGeoData(test.node, test.locs)
		if !reflect.DeepEqual(res, test.res) {
			t.Errorf("Expected %v, got %v", test.res, res)
		}
	}
}

// TODO - add test for GetAnnotation.  Postponing to next PR, which merges this code with search package,
// and moves to geolite2 package.  The TestGetAnnotation will be build on the code currently in geo-search_test.go
<|endoftext|>


<commit_before>package jetstream

import (
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/sirupsen/logrus"

	saramajs "github.com/S7evinK/saramajetstream"
	natsserver "github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	natsclient "github.com/nats-io/nats.go"
)

var natsServer *natsserver.Server
var natsServerMutex sync.Mutex

func Prepare(cfg *config.JetStream) (nats.JetStreamContext, sarama.Consumer, sarama.SyncProducer) {
	// check if we need an in-process NATS Server
	if len(cfg.Addresses) != 0 {
		return setupNATS(cfg, nil)
	}
	natsServerMutex.Lock()
	if natsServer == nil {
		var err error
		natsServer, err = natsserver.NewServer(&natsserver.Options{
			ServerName:       "monolith",
			DontListen:       true,
			JetStream:        true,
			StoreDir:         string(cfg.StoragePath),
			NoSystemAccount:  true,
			AllowNewAccounts: false,
		})
		if err != nil {
			panic(err)
		}
		natsServer.ConfigureLogger()
		go natsServer.Start()
	}
	natsServerMutex.Unlock()
	if !natsServer.ReadyForConnections(time.Second * 10) {
		logrus.Fatalln("NATS did not start in time")
	}
	nc, err := natsclient.Connect("", natsclient.InProcessServer(natsServer))
	if err != nil {
		logrus.Fatalln("Failed to create NATS client")
	}
	return setupNATS(cfg, nc)
}

func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (nats.JetStreamContext, sarama.Consumer, sarama.SyncProducer) {
	if nc == nil {
		var err error
		nc, err = nats.Connect(strings.Join(cfg.Addresses, ","))
		if err != nil {
			logrus.WithError(err).Panic("Unable to connect to NATS")
			return nil, nil, nil
		}
	}

	s, err := nc.JetStream()
	if err != nil {
		logrus.WithError(err).Panic("Unable to get JetStream context")
		return nil, nil, nil
	}

	for _, stream := range streams { // streams are defined in streams.go
		name := cfg.TopicFor(stream.Name)
		info, err := s.StreamInfo(name)
		if err != nil && err != natsclient.ErrStreamNotFound {
			logrus.WithError(err).Fatal("Unable to get stream info")
		}
		if info == nil {
			stream.Subjects = []string{name}

			// If we're trying to keep everything in memory (e.g. unit tests)
			// then overwrite the storage policy.
			if cfg.InMemory {
				stream.Storage = nats.MemoryStorage
			}

			// Namespace the streams without modifying the original streams
			// array, otherwise we end up with namespaces on namespaces.
			namespaced := *stream
			namespaced.Name = name
			if _, err = s.AddStream(&namespaced); err != nil {
				logrus.WithError(err).WithField("stream", name).Fatal("Unable to add stream")
			}
		}
	}

	consumer := saramajs.NewJetStreamConsumer(nc, s, "")
	producer := saramajs.NewJetStreamProducer(nc, s, "")
	return s, consumer, producer
}
<commit_msg>Increase maximum message size to 16MB (#2109)<commit_after>package jetstream

import (
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/matrix-org/dendrite/setup/config"
	"github.com/sirupsen/logrus"

	saramajs "github.com/S7evinK/saramajetstream"
	natsserver "github.com/nats-io/nats-server/v2/server"
	"github.com/nats-io/nats.go"
	natsclient "github.com/nats-io/nats.go"
)

var natsServer *natsserver.Server
var natsServerMutex sync.Mutex

func Prepare(cfg *config.JetStream) (nats.JetStreamContext, sarama.Consumer, sarama.SyncProducer) {
	// check if we need an in-process NATS Server
	if len(cfg.Addresses) != 0 {
		return setupNATS(cfg, nil)
	}
	natsServerMutex.Lock()
	if natsServer == nil {
		var err error
		natsServer, err = natsserver.NewServer(&natsserver.Options{
			ServerName:       "monolith",
			DontListen:       true,
			JetStream:        true,
			StoreDir:         string(cfg.StoragePath),
			NoSystemAccount:  true,
			AllowNewAccounts: false,
			MaxPayload:       16 * 1024 * 1024,
		})
		if err != nil {
			panic(err)
		}
		natsServer.ConfigureLogger()
		go natsServer.Start()
	}
	natsServerMutex.Unlock()
	if !natsServer.ReadyForConnections(time.Second * 10) {
		logrus.Fatalln("NATS did not start in time")
	}
	nc, err := natsclient.Connect("", natsclient.InProcessServer(natsServer))
	if err != nil {
		logrus.Fatalln("Failed to create NATS client")
	}
	return setupNATS(cfg, nc)
}

func setupNATS(cfg *config.JetStream, nc *natsclient.Conn) (nats.JetStreamContext, sarama.Consumer, sarama.SyncProducer) {
	if nc == nil {
		var err error
		nc, err = nats.Connect(strings.Join(cfg.Addresses, ","))
		if err != nil {
			logrus.WithError(err).Panic("Unable to connect to NATS")
			return nil, nil, nil
		}
	}

	s, err := nc.JetStream()
	if err != nil {
		logrus.WithError(err).Panic("Unable to get JetStream context")
		return nil, nil, nil
	}

	for _, stream := range streams { // streams are defined in streams.go
		name := cfg.TopicFor(stream.Name)
		info, err := s.StreamInfo(name)
		if err != nil && err != natsclient.ErrStreamNotFound {
			logrus.WithError(err).Fatal("Unable to get stream info")
		}
		if info == nil {
			stream.Subjects = []string{name}

			// If we're trying to keep everything in memory (e.g. unit tests)
			// then overwrite the storage policy.
			if cfg.InMemory {
				stream.Storage = nats.MemoryStorage
			}

			// Namespace the streams without modifying the original streams
			// array, otherwise we end up with namespaces on namespaces.
			namespaced := *stream
			namespaced.Name = name
			if _, err = s.AddStream(&namespaced); err != nil {
				logrus.WithError(err).WithField("stream", name).Fatal("Unable to add stream")
			}
		}
	}

	consumer := saramajs.NewJetStreamConsumer(nc, s, "")
	producer := saramajs.NewJetStreamProducer(nc, s, "")
	return s, consumer, producer
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"fmt"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

const (
	optionsKey = "opts"
)

var (
	//DefaultIsRequired ...
	DefaultIsRequired = false
	// DefaultIsExpand ...
	DefaultIsExpand = true
	// DefaultIsDontChangeValue ...
	DefaultIsDontChangeValue = false
	// DefaultIsAlwaysRun ...
	DefaultIsAlwaysRun = false
	// DefaultIsRequiresAdminUser ...
	DefaultIsRequiresAdminUser = false
	// DefaultIsSkippable ...
	DefaultIsSkippable = false
)

// -------------------
// --- Struct methods

// Normalize ...
func (env *EnvironmentItemModel) Normalize() error {
	opts, err := env.GetOptions()
	if err != nil {
		return err
	}
	(*env)[optionsKey] = opts
	return nil
}

// Validate ...
func (env EnvironmentItemModel) Validate() error {
	key, _, err := env.GetKeyValuePair()
	if err != nil {
		return err
	}
	if key == "" {
		return errors.New("Invalid environment: empty env_key")
	}

	log.Debugln("-> validate")
	options, err := env.GetOptions()
	if err != nil {
		return err
	}

	if options.Title == nil || *options.Title == "" {
		return errors.New("Invalid environment: missing or empty title")
	}

	return nil
}

// FillMissingDeafults ...
func (env *EnvironmentItemModel) FillMissingDeafults() error {
	defaultString := ""

	options, err := env.GetOptions()
	if err != nil {
		return err
	}

	if options.Description == nil {
		options.Description = &defaultString
	}
	if options.IsRequired == nil {
		options.IsRequired = &DefaultIsRequired
	}
	if options.IsExpand == nil {
		options.IsExpand = &DefaultIsExpand
	}
	if options.IsDontChangeValue == nil {
		options.IsDontChangeValue = &DefaultIsDontChangeValue
	}
	return nil
}

// Normalize ...
func (step StepModel) Normalize() error {
	for _, input := range step.Inputs {
		if err := input.Normalize(); err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		if err := output.Normalize(); err != nil {
			return err
		}
	}
	return nil
}

// Validate ...
func (step StepModel) Validate() error {
	if step.Title == nil || *step.Title == "" {
		return errors.New("Invalid step: missing or empty required 'title' property")
	}
	if step.Summary == nil || *step.Summary == "" {
		return errors.New("Invalid step: missing or empty required 'summary' property")
	}
	if step.Website == nil || *step.Website == "" {
		return errors.New("Invalid step: missing or empty required 'website' property")
	}
	if step.Source.Git == nil || *step.Source.Git == "" {
		return errors.New("Invalid step: missing or empty required 'source' property")
	}
	for _, input := range step.Inputs {
		err := input.Validate()
		if err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		err := output.Validate()
		if err != nil {
			return err
		}
	}
	return nil
}

// FillMissingDeafults ...
func (step *StepModel) FillMissingDeafults() error {
	defaultString := ""

	if step.Description == nil {
		step.Description = &defaultString
	}
	if step.SourceCodeURL == nil {
		step.SourceCodeURL = &defaultString
	}
	if step.SupportURL == nil {
		step.SupportURL = &defaultString
	}
	if step.IsRequiresAdminUser == nil {
		step.IsRequiresAdminUser = &DefaultIsRequiresAdminUser
	}
	if step.IsAlwaysRun == nil {
		step.IsAlwaysRun = &DefaultIsAlwaysRun
	}
	if step.IsSkippable == nil {
		step.IsSkippable = &DefaultIsSkippable
	}
	if step.RunIf == nil {
		step.RunIf = &defaultString
	}

	for _, input := range step.Inputs {
		err := input.FillMissingDeafults()
		if err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		err := output.FillMissingDeafults()
		if err != nil {
			return err
		}
	}
	return nil
}

// GetStep ...
func (collection StepCollectionModel) GetStep(id, version string) (StepModel, bool) {
	stepHash := collection.Steps
	stepVersions, found := stepHash[id]
	if !found {
		return StepModel{}, false
	}
	step, found := stepVersions.Versions[version]
	if !found {
		return StepModel{}, false
	}
	return step, true
}

// GetDownloadLocations ...
func (collection StepCollectionModel) GetDownloadLocations(id, version string) ([]DownloadLocationModel, error) {
	step, found := collection.GetStep(id, version)
	if found == false {
		return []DownloadLocationModel{}, fmt.Errorf("Collection doesn't contains step %s (%s)", id, version)
	}

	locations := []DownloadLocationModel{}
	for _, downloadLocation := range collection.DownloadLocations {
		switch downloadLocation.Type {
		case "zip":
			url := downloadLocation.Src + id + "/" + version + "/step.zip"
			location := DownloadLocationModel{
				Type: downloadLocation.Type,
				Src:  url,
			}
			locations = append(locations, location)
		case "git":
			location := DownloadLocationModel{
				Type: downloadLocation.Type,
				Src:  *step.Source.Git,
			}
			locations = append(locations, location)
		default:
			return []DownloadLocationModel{}, fmt.Errorf("[STEPMAN] - Invalid download location (%#v) for step (%#v)", downloadLocation, id)
		}
	}
	if len(locations) < 1 {
		return []DownloadLocationModel{}, fmt.Errorf("[STEPMAN] - No download location found for step (%#v)", id)
	}
	return locations, nil
}

// CompareVersions ...
// semantic version (X.Y.Z)
// 1 if version 2 is greater then version 1, -1 if not
// -2 & error if can't compare (if not supported component found)
func CompareVersions(version1, version2 string) (int, error) {
	version1Slice := strings.Split(version1, ".")
	version2Slice := strings.Split(version2, ".")

	lenDiff := len(version1Slice) - len(version2Slice)
	if lenDiff != 0 {
		makeDefVerComps := func(compLen int) []string {
			comps := make([]string, compLen, compLen)
			for idx := len(comps) - 1; idx >= 0; idx-- {
				comps[idx] = "0"
			}
			return comps
		}
		if lenDiff > 0 {
			// v1 slice is longer
			version2Slice = append(version2Slice, makeDefVerComps(lenDiff)...)
		} else {
			// v2 slice is longer
			version1Slice = append(version1Slice, makeDefVerComps(-lenDiff)...)
		}
	}

	cnt := len(version1Slice)
	for i, num := range version1Slice {
		num1, err := strconv.ParseInt(num, 0, 64)
		if err != nil {
			log.Error("[STEPMAN] - Failed to parse int:", err)
			return -2, err
		}

		num2, err2 := strconv.ParseInt(version2Slice[i], 0, 64)
		if err2 != nil {
			log.Error("[STEPMAN] - Failed to parse int:", err2)
			return -2, err
		}

		if num2 > num1 {
			return 1, nil
		}
		if i == cnt-1 {
			// last one
			if num2 == num1 {
				return 0, nil
			}
		}
	}
	return -1, nil
}

// GetLatestStepVersion ...
func (collection StepCollectionModel) GetLatestStepVersion(id string) (string, error) {
	stepHash := collection.Steps
	stepGroup, found := stepHash[id]
	if !found {
		return "", fmt.Errorf("Collection doesn't contains step %s", id)
	}

	if stepGroup.LatestVersionNumber == "" {
		return "", fmt.Errorf("Failed to find latest version of step %s", id)
	}

	return stepGroup.LatestVersionNumber, nil
}

// GetKeyValuePair ...
func (env EnvironmentItemModel) GetKeyValuePair() (string, string, error) {
	if len(env) > 2 {
		return "", "", errors.New("Invalid env: more then 2 fields ")
	}

	retKey := ""
	retValue := ""

	for key, value := range env {
		if key != optionsKey {
			if retKey != "" {
				return "", "", errors.New("Invalid env: more then 1 key-value field found!")
			}

			valueStr, ok := value.(string)
			if !ok {
				if value == nil {
					valueStr = ""
				} else {
					return "", "", fmt.Errorf("Invalid value (key:%#v) (value:%#v)", key, value)
				}
			}

			retKey = key
			retValue = valueStr
		}
	}

	if retKey == "" {
		return "", "", errors.New("Invalid env: no envKey specified!")
	}

	return retKey, retValue, nil
}

// ParseFromInterfaceMap ...
func (envSerModel *EnvironmentItemOptionsModel) ParseFromInterfaceMap(input map[interface{}]interface{}) error {
	for key, value := range input {
		keyStr, ok := key.(string)
		if !ok {
			return fmt.Errorf("Invalid key, should be a string: %#v", key)
		}
		log.Debugf("  ** processing (key:%#v) (value:%#v) (envSerModel:%#v)", key, value, envSerModel)
		switch keyStr {
		case "title":
			castedValue, ok := value.(string)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.Title = &castedValue
		case "description":
			castedValue, ok := value.(string)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.Description = &castedValue
		case "value_options":
			castedValue, ok := value.([]string)
			if !ok {
				// try with []interface{} instead and cast the
				//  items to string
				castedValue = []string{}
				interfArr, ok := value.([]interface{})
				if !ok {
					return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
				}
				for _, interfItm := range interfArr {
					castedItm, ok := interfItm.(string)
					if !ok {
						return fmt.Errorf("Invalid value in value_options (%#v), not a string: %#v", interfArr, interfItm)
					}
					castedValue = append(castedValue, castedItm)
				}
			}
			envSerModel.ValueOptions = castedValue
		case "is_required":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsRequired = &castedValue
		case "is_expand":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsExpand = &castedValue
		case "is_dont_change_value":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsDontChangeValue = &castedValue
		default:
			return fmt.Errorf("Not supported key found in options: %#v", key)
		}
	}
	return nil
}

// GetOptions ...
func (env EnvironmentItemModel) GetOptions() (EnvironmentItemOptionsModel, error) {
	value, found := env[optionsKey]
	if !found {
		return EnvironmentItemOptionsModel{}, nil
	}

	envItmCasted, ok := value.(EnvironmentItemOptionsModel)
	if ok {
		return envItmCasted, nil
	}

	log.Debugf(" * processing env:%#v", env)

	// if it's read from a file (YAML/JSON) then it's most likely not the proper type
	//  so cast it from the generic interface-interface map
	optionsInterfaceMap, ok := value.(map[interface{}]interface{})
	if !ok {
		return EnvironmentItemOptionsModel{}, fmt.Errorf("Invalid options (value:%#v) - failed to map-interface cast", value)
	}

	options := EnvironmentItemOptionsModel{}
	err := options.ParseFromInterfaceMap(optionsInterfaceMap)
	if err != nil {
		return EnvironmentItemOptionsModel{}, err
	}

	log.Debugf("Parsed options: %#v\n", options)

	return options, nil
}
<commit_msg>env validate fix<commit_after>package models

import (
	"errors"
	"fmt"
	"strconv"
	"strings"

	log "github.com/Sirupsen/logrus"
)

const (
	optionsKey = "opts"
)

var (
	//DefaultIsRequired ...
	DefaultIsRequired = false
	// DefaultIsExpand ...
	DefaultIsExpand = true
	// DefaultIsDontChangeValue ...
	DefaultIsDontChangeValue = false
	// DefaultIsAlwaysRun ...
	DefaultIsAlwaysRun = false
	// DefaultIsRequiresAdminUser ...
	DefaultIsRequiresAdminUser = false
	// DefaultIsSkippable ...
	DefaultIsSkippable = false
)

// -------------------
// --- Struct methods

// Normalize ...
func (env *EnvironmentItemModel) Normalize() error {
	opts, err := env.GetOptions()
	if err != nil {
		return err
	}
	(*env)[optionsKey] = opts
	return nil
}

// Validate ...
func (env EnvironmentItemModel) Validate() error {
	key, _, err := env.GetKeyValuePair()
	if err != nil {
		return err
	}
	if key == "" {
		return errors.New("Invalid environment: empty env_key")
	}

	log.Debugln("-> validate")
	options, err := env.GetOptions()
	if err != nil {
		return err
	}

	if options.Title == nil || *options.Title == "" {
		return errors.New("Invalid environment: missing or empty title")
	}

	return nil
}

// FillMissingDeafults ...
func (env *EnvironmentItemModel) FillMissingDeafults() error {
	defaultString := ""

	options, err := env.GetOptions()
	if err != nil {
		return err
	}

	if options.Description == nil {
		options.Description = &defaultString
	}
	if options.IsRequired == nil {
		options.IsRequired = &DefaultIsRequired
	}
	if options.IsExpand == nil {
		options.IsExpand = &DefaultIsExpand
	}
	if options.IsDontChangeValue == nil {
		options.IsDontChangeValue = &DefaultIsDontChangeValue
	}
	(*env)[optionsKey] = options
	return nil
}

// Normalize ...
func (step StepModel) Normalize() error {
	for _, input := range step.Inputs {
		if err := input.Normalize(); err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		if err := output.Normalize(); err != nil {
			return err
		}
	}
	return nil
}

// Validate ...
func (step StepModel) Validate() error {
	if step.Title == nil || *step.Title == "" {
		return errors.New("Invalid step: missing or empty required 'title' property")
	}
	if step.Summary == nil || *step.Summary == "" {
		return errors.New("Invalid step: missing or empty required 'summary' property")
	}
	if step.Website == nil || *step.Website == "" {
		return errors.New("Invalid step: missing or empty required 'website' property")
	}
	if step.Source.Git == nil || *step.Source.Git == "" {
		return errors.New("Invalid step: missing or empty required 'source' property")
	}
	for _, input := range step.Inputs {
		err := input.Validate()
		if err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		err := output.Validate()
		if err != nil {
			return err
		}
	}
	return nil
}

// FillMissingDeafults ...
func (step *StepModel) FillMissingDeafults() error {
	defaultString := ""

	if step.Description == nil {
		step.Description = &defaultString
	}
	if step.SourceCodeURL == nil {
		step.SourceCodeURL = &defaultString
	}
	if step.SupportURL == nil {
		step.SupportURL = &defaultString
	}
	if step.IsRequiresAdminUser == nil {
		step.IsRequiresAdminUser = &DefaultIsRequiresAdminUser
	}
	if step.IsAlwaysRun == nil {
		step.IsAlwaysRun = &DefaultIsAlwaysRun
	}
	if step.IsSkippable == nil {
		step.IsSkippable = &DefaultIsSkippable
	}
	if step.RunIf == nil {
		step.RunIf = &defaultString
	}

	for _, input := range step.Inputs {
		err := input.FillMissingDeafults()
		if err != nil {
			return err
		}
	}
	for _, output := range step.Outputs {
		err := output.FillMissingDeafults()
		if err != nil {
			return err
		}
	}
	return nil
}

// GetStep ...
func (collection StepCollectionModel) GetStep(id, version string) (StepModel, bool) {
	stepHash := collection.Steps
	stepVersions, found := stepHash[id]
	if !found {
		return StepModel{}, false
	}
	step, found := stepVersions.Versions[version]
	if !found {
		return StepModel{}, false
	}
	return step, true
}

// GetDownloadLocations ...
func (collection StepCollectionModel) GetDownloadLocations(id, version string) ([]DownloadLocationModel, error) {
	step, found := collection.GetStep(id, version)
	if found == false {
		return []DownloadLocationModel{}, fmt.Errorf("Collection doesn't contains step %s (%s)", id, version)
	}

	locations := []DownloadLocationModel{}
	for _, downloadLocation := range collection.DownloadLocations {
		switch downloadLocation.Type {
		case "zip":
			url := downloadLocation.Src + id + "/" + version + "/step.zip"
			location := DownloadLocationModel{
				Type: downloadLocation.Type,
				Src:  url,
			}
			locations = append(locations, location)
		case "git":
			location := DownloadLocationModel{
				Type: downloadLocation.Type,
				Src:  *step.Source.Git,
			}
			locations = append(locations, location)
		default:
			return []DownloadLocationModel{}, fmt.Errorf("[STEPMAN] - Invalid download location (%#v) for step (%#v)", downloadLocation, id)
		}
	}
	if len(locations) < 1 {
		return []DownloadLocationModel{}, fmt.Errorf("[STEPMAN] - No download location found for step (%#v)", id)
	}
	return locations, nil
}

// CompareVersions ...
// semantic version (X.Y.Z)
// 1 if version 2 is greater then version 1, -1 if not
// -2 & error if can't compare (if not supported component found)
func CompareVersions(version1, version2 string) (int, error) {
	version1Slice := strings.Split(version1, ".")
	version2Slice := strings.Split(version2, ".")

	lenDiff := len(version1Slice) - len(version2Slice)
	if lenDiff != 0 {
		makeDefVerComps := func(compLen int) []string {
			comps := make([]string, compLen, compLen)
			for idx := len(comps) - 1; idx >= 0; idx-- {
				comps[idx] = "0"
			}
			return comps
		}
		if lenDiff > 0 {
			// v1 slice is longer
			version2Slice = append(version2Slice, makeDefVerComps(lenDiff)...)
		} else {
			// v2 slice is longer
			version1Slice = append(version1Slice, makeDefVerComps(-lenDiff)...)
		}
	}

	cnt := len(version1Slice)
	for i, num := range version1Slice {
		num1, err := strconv.ParseInt(num, 0, 64)
		if err != nil {
			log.Error("[STEPMAN] - Failed to parse int:", err)
			return -2, err
		}

		num2, err2 := strconv.ParseInt(version2Slice[i], 0, 64)
		if err2 != nil {
			log.Error("[STEPMAN] - Failed to parse int:", err2)
			return -2, err
		}

		if num2 > num1 {
			return 1, nil
		}
		if i == cnt-1 {
			// last one
			if num2 == num1 {
				return 0, nil
			}
		}
	}
	return -1, nil
}

// GetLatestStepVersion ...
func (collection StepCollectionModel) GetLatestStepVersion(id string) (string, error) {
	stepHash := collection.Steps
	stepGroup, found := stepHash[id]
	if !found {
		return "", fmt.Errorf("Collection doesn't contains step %s", id)
	}

	if stepGroup.LatestVersionNumber == "" {
		return "", fmt.Errorf("Failed to find latest version of step %s", id)
	}

	return stepGroup.LatestVersionNumber, nil
}

// GetKeyValuePair ...
func (env EnvironmentItemModel) GetKeyValuePair() (string, string, error) {
	if len(env) > 2 {
		return "", "", errors.New("Invalid env: more then 2 fields ")
	}

	retKey := ""
	retValue := ""

	for key, value := range env {
		if key != optionsKey {
			if retKey != "" {
				return "", "", errors.New("Invalid env: more then 1 key-value field found!")
			}

			valueStr, ok := value.(string)
			if !ok {
				if value == nil {
					valueStr = ""
				} else {
					return "", "", fmt.Errorf("Invalid value (key:%#v) (value:%#v)", key, value)
				}
			}

			retKey = key
			retValue = valueStr
		}
	}

	if retKey == "" {
		return "", "", errors.New("Invalid env: no envKey specified!")
	}

	return retKey, retValue, nil
}

// ParseFromInterfaceMap ...
func (envSerModel *EnvironmentItemOptionsModel) ParseFromInterfaceMap(input map[interface{}]interface{}) error {
	for key, value := range input {
		keyStr, ok := key.(string)
		if !ok {
			return fmt.Errorf("Invalid key, should be a string: %#v", key)
		}
		log.Debugf("  ** processing (key:%#v) (value:%#v) (envSerModel:%#v)", key, value, envSerModel)
		switch keyStr {
		case "title":
			castedValue, ok := value.(string)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.Title = &castedValue
		case "description":
			castedValue, ok := value.(string)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.Description = &castedValue
		case "value_options":
			castedValue, ok := value.([]string)
			if !ok {
				// try with []interface{} instead and cast the
				//  items to string
				castedValue = []string{}
				interfArr, ok := value.([]interface{})
				if !ok {
					return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
				}
				for _, interfItm := range interfArr {
					castedItm, ok := interfItm.(string)
					if !ok {
						return fmt.Errorf("Invalid value in value_options (%#v), not a string: %#v", interfArr, interfItm)
					}
					castedValue = append(castedValue, castedItm)
				}
			}
			envSerModel.ValueOptions = castedValue
		case "is_required":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsRequired = &castedValue
		case "is_expand":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsExpand = &castedValue
		case "is_dont_change_value":
			castedValue, ok := value.(bool)
			if !ok {
				return fmt.Errorf("Invalid value type (key:%s): %#v", keyStr, value)
			}
			envSerModel.IsDontChangeValue = &castedValue
		default:
			return fmt.Errorf("Not supported key found in options: %#v", key)
		}
	}
	return nil
}

// GetOptions ...
func (env EnvironmentItemModel) GetOptions() (EnvironmentItemOptionsModel, error) {
	value, found := env[optionsKey]
	if !found {
		return EnvironmentItemOptionsModel{}, nil
	}

	envItmCasted, ok := value.(EnvironmentItemOptionsModel)
	if ok {
		return envItmCasted, nil
	}

	log.Debugf(" * processing env:%#v", env)

	// if it's read from a file (YAML/JSON) then it's most likely not the proper type
	//  so cast it from the generic interface-interface map
	optionsInterfaceMap, ok := value.(map[interface{}]interface{})
	if !ok {
		return EnvironmentItemOptionsModel{}, fmt.Errorf("Invalid options (value:%#v) - failed to map-interface cast", value)
	}

	options := EnvironmentItemOptionsModel{}
	err := options.ParseFromInterfaceMap(optionsInterfaceMap)
	if err != nil {
		return EnvironmentItemOptionsModel{}, err
	}

	log.Debugf("Parsed options: %#v\n", options)

	return options, nil
}
<|endoftext|>


<commit_before>package pcf_pipelines_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	yaml "gopkg.in/yaml.v2"

	"github.com/concourse/atc"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var placeholderRegexp = regexp.MustCompile("{{([a-zA-Z0-9-_]+)}}")

var _ = Describe("pcf-pipelines", func() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("failed to get working dir: %s", err)
	}

	root := filepath.Dir(cwd)
	baseDir := filepath.Base(cwd)

	var pipelinePaths []string
	err = filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if filepath.Base(path) == "pipeline.yml" {
			relPipelinePath, err := filepath.Rel(cwd, path)
			if err != nil {
				return err
			}
			pipelinePaths = append(pipelinePaths, relPipelinePath)
		}

		return nil
	})
	if err != nil {
		log.Fatalf("failed to walk: %s", err)
	}

	for _, path := range pipelinePaths {
		pipelinePath := path

		Context(fmt.Sprintf("pipeline at %s", pipelinePath), func() {
			var configBytes []byte

			BeforeEach(func() {
				var err error
				configBytes, err = ioutil.ReadFile(pipelinePath)
				Expect(err).NotTo(HaveOccurred())
			})

			It("specifies all and only the params that the pipeline's tasks expect", func() {
				var config atc.Config
				cleanConfigBytes := placeholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					for _, task := range allTasksInPlan(&job.Plan) {
						failMessage := fmt.Sprintf("Found error in the following pipeline:\n    %s\n\nin the following task's params:\n    %s/%s\n", pipelinePath, job.Name, task.Name())

						var configParams []string
						for k := range task.Params {
							configParams = append(configParams, k)
						}

						if strings.HasPrefix(task.TaskConfigPath, baseDir) {
							taskPath := strings.TrimPrefix(task.TaskConfigPath, baseDir+"/")
							relpath, err := filepath.Rel(cwd, filepath.Join(cwd, taskPath))
							Expect(err).NotTo(HaveOccurred())

							bs, err := ioutil.ReadFile(relpath)
							Expect(err).NotTo(HaveOccurred())

							taskConfig := atc.TaskConfig{}
							err = yaml.Unmarshal(bs, &taskConfig)
							Expect(err).NotTo(HaveOccurred())

							var taskParams []string
							for k := range taskConfig.Params {
								taskParams = append(taskParams, k)
							}

							assertUnorderedEqual(taskParams, configParams, failMessage)
						}
					}
				}
			})

			It("has a params file with all and only the params that the pipeline specifies", func() {
				paramsPath := filepath.Join(filepath.Dir(pipelinePath), "params.yml")
				_, err := os.Lstat(paramsPath)
				Expect(err).NotTo(HaveOccurred())

				bs, err := ioutil.ReadFile(paramsPath)
				Expect(err).NotTo(HaveOccurred())

				paramsMap := map[string]interface{}{}
				err = yaml.Unmarshal(bs, paramsMap)
				Expect(err).NotTo(HaveOccurred())

				var params []string
				for k := range paramsMap {
					params = append(params, k)
				}

				matches := placeholderRegexp.FindAllStringSubmatch(string(configBytes), -1)

				uniqueMatches := map[string]struct{}{}
				for _, match := range matches {
					uniqueMatches[match[1]] = struct{}{}
				}

				var placeholders []string
				for match := range uniqueMatches {
					placeholders = append(placeholders, match)
				}

				failMessage := fmt.Sprintf(`
Found error with the following pipeline:
%s

in the following params template:
%s
`, pipelinePath, paramsPath)

				assertUnorderedEqual(placeholders, params, failMessage)
			})

			It("provides all of the resources that the tasks it defines require", func() {
				var config atc.Config
				cleanConfigBytes := placeholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					tasks := allTasksInPlan(&job.Plan)
					resources := availableResources(&job.Plan)

					for i, task := range tasks {
						var inputs []atc.TaskInputConfig
						if !strings.HasPrefix(task.TaskConfigPath, "pcf-pipelines") {
							continue
						}

						var taskConfig atc.TaskConfig
						bs, err := ioutil.ReadFile(filepath.Join(root, task.TaskConfigPath))
						Expect(err).NotTo(HaveOccurred())

						err = yaml.Unmarshal(bs, &taskConfig)
						Expect(err).NotTo(HaveOccurred())

						inputs = taskConfig.Inputs

						if task.TaskConfig != nil {
							inputs = task.TaskConfig.Inputs
						}

						for k := range task.InputMapping {
							var validInputMapping bool
							for _, input := range inputs {
								if input.Name == k {
									validInputMapping = true
									break
								}
							}
							if !validInputMapping {
								Fail(fmt.Sprintf("could not find input mapping for '%s' in '%s'\n", k, inputs))
							}
						}

						for j := 0; j < i; j++ {
							upstreamTask := tasks[j]

							if upstreamTask.TaskConfig != nil {
								for _, output := range upstreamTask.TaskConfig.Outputs {
									resources = append(resources, output.Name)
								}
							}

							if upstreamTask.TaskConfigPath != "" {
								var upstreamTaskConfig atc.TaskConfig
								bs, err := ioutil.ReadFile(filepath.Join(root, upstreamTask.TaskConfigPath))
								Expect(err).NotTo(HaveOccurred())

								err = yaml.Unmarshal(bs, &upstreamTaskConfig)
								Expect(err).NotTo(HaveOccurred())

								for _, output := range upstreamTaskConfig.Outputs {
									resources = append(resources, output.Name)
								}
							}

							for _, v := range upstreamTask.OutputMapping {
								resources = append(resources, v)
							}
						}

					OUTER:
						for _, input := range inputs {
							for _, actual := range resources {
								if input.Name == actual {
									continue OUTER
								}

								for k, v := range task.InputMapping {
									if k == input.Name && v == actual {
										continue OUTER
									}
								}
							}

							Fail(fmt.Sprintf("did not find matching get, put, output or input_mapping of '%s', which is required by task '%s'", input.Name, task.Name()))
						}
					}
				}
			})
		})
	}
})

func allTasksInPlan(seq *atc.PlanSequence) []atc.PlanConfig {
	var tasks []atc.PlanConfig

	for _, planConfig := range *seq {
		if planConfig.Aggregate != nil {
			tasks = append(tasks, allTasksInPlan(planConfig.Aggregate)...)
		}
		if planConfig.Do != nil {
			tasks = append(tasks, allTasksInPlan(planConfig.Do)...)
		}
		if planConfig.Task != "" {
			tasks = append(tasks, planConfig)
		}
	}

	return tasks
}

func availableResources(seq *atc.PlanSequence) []string {
	var resources []string

	for _, planConfig := range *seq {
		if planConfig.Aggregate != nil {
			resources = append(resources, availableResources(planConfig.Aggregate)...)
		}

		if planConfig.Do != nil {
			resources = append(resources, availableResources(planConfig.Do)...)
		}

		if planConfig.Get != "" {
			resources = append(resources, planConfig.Get)
		}

		if planConfig.Put != "" {
			resources = append(resources, planConfig.Put)
		}
	}

	return resources
}

func assertUnorderedEqual(left, right []string, failMessage string) {
	for _, l := range left {
		Expect(right).To(ContainElement(l), failMessage)
	}

	for _, r := range right {
		var found bool

		for _, l := range left {
			if r == l {
				found = true
				break
			}
		}

		if !found {
			Expect(right).NotTo(ContainElement(r), failMessage)
		}
	}
}
<commit_msg>Remove multiple reads of task config files<commit_after>package pcf_pipelines_test

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	yaml "gopkg.in/yaml.v2"

	"github.com/concourse/atc"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var placeholderRegexp = regexp.MustCompile("{{([a-zA-Z0-9-_]+)}}")

var _ = Describe("pcf-pipelines", func() {
	cwd, err := os.Getwd()
	if err != nil {
		log.Fatalf("failed to get working dir: %s", err)
	}

	root := filepath.Dir(cwd)
	baseDir := filepath.Base(cwd)

	var pipelinePaths []string
	err = filepath.Walk(cwd, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if filepath.Base(path) == "pipeline.yml" {
			relPipelinePath, err := filepath.Rel(cwd, path)
			if err != nil {
				return err
			}
			pipelinePaths = append(pipelinePaths, relPipelinePath)
		}

		return nil
	})
	if err != nil {
		log.Fatalf("failed to walk: %s", err)
	}

	for _, path := range pipelinePaths {
		pipelinePath := path

		Context(fmt.Sprintf("pipeline at %s", pipelinePath), func() {
			var configBytes []byte

			BeforeEach(func() {
				var err error
				configBytes, err = ioutil.ReadFile(pipelinePath)
				Expect(err).NotTo(HaveOccurred())
			})

			It("specifies all and only the params that the pipeline's tasks expect", func() {
				var config atc.Config
				cleanConfigBytes := placeholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					for _, task := range allTasksInPlan(&job.Plan) {
						failMessage := fmt.Sprintf("Found error in the following pipeline:\n    %s\n\nin the following task's params:\n    %s/%s\n", pipelinePath, job.Name, task.Name())

						var configParams []string
						for k := range task.Params {
							configParams = append(configParams, k)
						}

						if strings.HasPrefix(task.TaskConfigPath, baseDir) {
							taskPath := strings.TrimPrefix(task.TaskConfigPath, baseDir+"/")
							relpath, err := filepath.Rel(cwd, filepath.Join(cwd, taskPath))
							Expect(err).NotTo(HaveOccurred())

							bs, err := ioutil.ReadFile(relpath)
							Expect(err).NotTo(HaveOccurred())

							taskConfig := atc.TaskConfig{}
							err = yaml.Unmarshal(bs, &taskConfig)
							Expect(err).NotTo(HaveOccurred())

							var taskParams []string
							for k := range taskConfig.Params {
								taskParams = append(taskParams, k)
							}

							assertUnorderedEqual(taskParams, configParams, failMessage)
						}
					}
				}
			})

			It("has a params file with all and only the params that the pipeline specifies", func() {
				paramsPath := filepath.Join(filepath.Dir(pipelinePath), "params.yml")
				_, err := os.Lstat(paramsPath)
				Expect(err).NotTo(HaveOccurred())

				bs, err := ioutil.ReadFile(paramsPath)
				Expect(err).NotTo(HaveOccurred())

				paramsMap := map[string]interface{}{}
				err = yaml.Unmarshal(bs, paramsMap)
				Expect(err).NotTo(HaveOccurred())

				var params []string
				for k := range paramsMap {
					params = append(params, k)
				}

				matches := placeholderRegexp.FindAllStringSubmatch(string(configBytes), -1)

				uniqueMatches := map[string]struct{}{}
				for _, match := range matches {
					uniqueMatches[match[1]] = struct{}{}
				}

				var placeholders []string
				for match := range uniqueMatches {
					placeholders = append(placeholders, match)
				}

				failMessage := fmt.Sprintf(`
Found error with the following pipeline:
%s

in the following params template:
%s
`, pipelinePath, paramsPath)

				assertUnorderedEqual(placeholders, params, failMessage)
			})

			It("provides all of the resources that the tasks it defines require", func() {
				var config atc.Config
				cleanConfigBytes := placeholderRegexp.ReplaceAll(configBytes, []byte("true"))
				err := yaml.Unmarshal(cleanConfigBytes, &config)
				Expect(err).NotTo(HaveOccurred())

				for _, job := range config.Jobs {
					tasks := allTasksInPlan(&job.Plan)
					resources := availableResources(&job.Plan)

					for i, task := range tasks {
						if !strings.HasPrefix(task.TaskConfigPath, "pcf-pipelines") {
							continue
						}

						var inputs []atc.TaskInputConfig
						if task.TaskConfig != nil {
							inputs = task.TaskConfig.Inputs
						} else {
							inputs = taskInputConfigs(filepath.Join(root, task.TaskConfigPath))
						}

						for k := range task.InputMapping {
							var validInputMapping bool
							for _, input := range inputs {
								if input.Name == k {
									validInputMapping = true
									break
								}
							}
							if !validInputMapping {
								Fail(fmt.Sprintf("could not find input mapping for '%s' in '%s'\n", k, inputs))
							}
						}

						for j := 0; j < i; j++ {
							upstreamTask := tasks[j]

							if upstreamTask.TaskConfig != nil {
								for _, output := range upstreamTask.TaskConfig.Outputs {
									resources = append(resources, output.Name)
								}
							}

							if upstreamTask.TaskConfigPath != "" {
								var upstreamTaskConfig atc.TaskConfig
								bs, err := ioutil.ReadFile(filepath.Join(root, upstreamTask.TaskConfigPath))
								Expect(err).NotTo(HaveOccurred())

								err = yaml.Unmarshal(bs, &upstreamTaskConfig)
								Expect(err).NotTo(HaveOccurred())

								for _, output := range upstreamTaskConfig.Outputs {
									resources = append(resources, output.Name)
								}
							}

							for _, v := range upstreamTask.OutputMapping {
								resources = append(resources, v)
							}
						}

					OUTER:
						for _, input := range inputs {
							for _, actual := range resources {
								if input.Name == actual {
									continue OUTER
								}

								for k, v := range task.InputMapping {
									if k == input.Name && v == actual {
										continue OUTER
									}
								}
							}

							Fail(fmt.Sprintf("did not find matching get, put, output or input_mapping of '%s', which is required by task '%s'", input.Name, task.Name()))
						}
					}
				}
			})
		})
	}
})

func allTasksInPlan(seq *atc.PlanSequence) []atc.PlanConfig {
	var tasks []atc.PlanConfig

	for _, planConfig := range *seq {
		if planConfig.Aggregate != nil {
			tasks = append(tasks, allTasksInPlan(planConfig.Aggregate)...)
		}
		if planConfig.Do != nil {
			tasks = append(tasks, allTasksInPlan(planConfig.Do)...)
		}
		if planConfig.Task != "" {
			tasks = append(tasks, planConfig)
		}
	}

	return tasks
}

func availableResources(seq *atc.PlanSequence) []string {
	var resources []string

	for _, planConfig := range *seq {
		if planConfig.Aggregate != nil {
			resources = append(resources, availableResources(planConfig.Aggregate)...)
		}

		if planConfig.Do != nil {
			resources = append(resources, availableResources(planConfig.Do)...)
		}

		if planConfig.Get != "" {
			resources = append(resources, planConfig.Get)
		}

		if planConfig.Put != "" {
			resources = append(resources, planConfig.Put)
		}
	}

	return resources
}

func assertUnorderedEqual(left, right []string, failMessage string) {
	for _, l := range left {
		Expect(right).To(ContainElement(l), failMessage)
	}

	for _, r := range right {
		var found bool

		for _, l := range left {
			if r == l {
				found = true
				break
			}
		}

		if !found {
			Expect(right).NotTo(ContainElement(r), failMessage)
		}
	}
}

var taskConfigs = map[string]*atc.TaskConfig{}

func taskInputConfigs(path string) []atc.TaskInputConfig {
	taskConfig, ok := taskConfigs[path]

	if !ok {
		bs, err := ioutil.ReadFile(path)
		Expect(err).NotTo(HaveOccurred())

		err = yaml.Unmarshal(bs, &taskConfig)
		Expect(err).NotTo(HaveOccurred())

		taskConfigs[path] = taskConfig
	}

	return taskConfig.Inputs
}
<|endoftext|>


<commit_before>package resource

import (
	"errors"
	"fmt"
	"strings"

	"golang.org/x/net/context"

	"github.com/rs/rest-layer/schema"
)

// Lookup holds filter and sort used to select items in a resource collection
type Lookup struct {
	// The client supplied filter. Filter is a MongoDB inspired query with a more limited
	// set of capabilities. See https://github.com/rs/rest-layer#filtering
	// for more info.
	filter schema.Query
	// The client supplied soft. Sort is a list of resource fields or sub-fields separated
	// by comas (,). To invert the sort, a minus (-) can be prefixed.
	// See https://github.com/rs/rest-layer#sorting for more info.
	sort []string
	// The client supplied selector. Selector is a way for the client to reformat the
	// resource representation at runtime by defining which fields should be included
	// in the document. The REST Layer selector language allows field aliasing, field
	// transformation with parameters and sub-item/collection embedding.
	selector []Field
}

// Field is used with Lookup.selector to reformat the resource representation at runtime
// using a field selection language inspired by GraphQL.
type Field struct {
	// Name is the name of the field as define in the resource's schema.
	Name string
	// Alias is the wanted name in the representation.
	Alias string
	// Params defines a list of params to be sent to the field's param handler if any.
	Params map[string]interface{}
	// Fields holds references to child fields if any
	Fields []Field
}

// NewLookup creates an empty lookup object
func NewLookup() *Lookup {
	return &Lookup{
		filter: schema.Query{},
		sort:   []string{},
	}
}

// NewLookupWithQuery creates an empty lookup object with a given query
func NewLookupWithQuery(q schema.Query) *Lookup {
	return &Lookup{
		filter: q,
		sort:   []string{},
	}
}

// Sort is a list of resource fields or sub-fields separated
// by comas (,). To invert the sort, a minus (-) can be prefixed.
//
// See https://github.com/rs/rest-layer#sorting for more info.
func (l *Lookup) Sort() []string {
	return l.sort
}

// Filter is a MongoDB inspired query with a more limited set of capabilities.
//
// See https://github.com/rs/rest-layer#filtering for more info.
func (l *Lookup) Filter() schema.Query {
	return l.filter
}

// SetSorts set the sort fields with a pre-parsed list of fields to sort on.
// This method doesn't validate sort fields.
func (l *Lookup) SetSorts(sorts []string) {
	l.sort = sorts
}

// SetSort parses and validate a sort parameter and set it as lookup's Sort
func (l *Lookup) SetSort(sort string, validator schema.Validator) error {
	sorts := []string{}
	for _, f := range strings.Split(sort, ",") {
		f = strings.Trim(f, " ")
		if f == "" {
			return errors.New("empty soft field")
		}
		// If the field start with - (to indicate descended sort), shift it before
		// validator lookup
		i := 0
		if f[0] == '-' {
			i = 1
		}
		// Make sure the field exists
		field := validator.GetField(f[i:])
		if field == nil {
			return fmt.Errorf("invalid sort field: %s", f[i:])
		}
		if !field.Sortable {
			return fmt.Errorf("field is not sortable: %s", f[i:])
		}
		sorts = append(sorts, f)
	}
	l.sort = sorts
	return nil
}

// AddFilter parses and validate a filter parameter and add it to lookup's filter
//
// The filter query is validated against the provided validator to ensure all queried
// fields exists and are of the right type.
func (l *Lookup) AddFilter(filter string, validator schema.Validator) error {
	f, err := schema.ParseQuery(filter, validator)
	if err != nil {
		return err
	}
	l.AddQuery(f)
	return nil
}

// AddQuery add an existing schema.Query to the lookup's filters
func (l *Lookup) AddQuery(query schema.Query) {
	if l.filter == nil {
		l.filter = query
		return
	}
	for _, exp := range query {
		l.filter = append(l.filter, exp)
	}
}

// SetSelector parses a selector expression, validates it and assign it to the current Lookup.
func (l *Lookup) SetSelector(s string, r *Resource) error {
	pos := 0
	selector, err := parseSelectorExpression([]byte(s), &pos, len(s), false)
	if err != nil {
		return err
	}
	if err = validateSelector(selector, r.Validator()); err != nil {
		return err
	}
	l.selector = selector
	return nil
}

// ReferenceResolver is a function resolving a reference to another field
type ReferenceResolver func(path string) (*Resource, error)

// ApplySelector applies fields filtering / rename to the payload fields
func (l *Lookup) ApplySelector(ctx context.Context, r *Resource, p map[string]interface{}, resolver ReferenceResolver) (map[string]interface{}, error) {
	if len(l.selector) == 0 {
		return p, nil
	}
	payload, err := applySelector(ctx, l.selector, r.Validator(), p, resolver)
	if err == nil {
		// The resulting payload may contain some asyncSelector, we must execute them
		// concurrently until there's no more
		err = resolveAsyncSelectors(ctx, payload)
	}
	return payload, err
}

func resolveAsyncSelectors(ctx context.Context, p map[string]interface{}) error {
	for {
		sr := getSelectorResolvers(p)
		if len(sr) == 0 {
			break
		}
		done := make(chan error, len(sr))
		// TODO limit the number of // sub requests
		for _, r := range sr {
			go r(ctx, done)
		}
		wait := len(sr)
		cleanup := func() {
			// Make sure we empty the channel of remaining future responses
			// to prevent leaks
			for wait > 0 {
				<-done
				wait--
			}
		}
		for wait > 0 {
			select {
			case err := <-done:
				wait--
				if err != nil {
					if wait > 0 {
						go cleanup()
					}
					return err
				}
			case <-ctx.Done():
				if wait > 0 {
					go cleanup()
				}
				return ctx.Err()
			}
		}
	}
	return nil
}

type asyncSelectorResolver func(ctx context.Context, done chan<- error)

func getSelectorResolvers(p map[string]interface{}) []asyncSelectorResolver {
	return append(getAsyncSelectorResolvers(p), getAsyncGetResolver(p)...)
}

// getAsyncSelectorResolvers parse the payload searching for any unresolved asyncSelector
// and build an asyncSelectorResolver for each ones.
func getAsyncSelectorResolvers(p map[string]interface{}) []asyncSelectorResolver {
	as := []asyncSelectorResolver{}
	for name, val := range p {
		switch val := val.(type) {
		case asyncSelector:
			n := name
			as = append(as, func(ctx context.Context, done chan<- error) {
				res, err := val(ctx)
				if err == nil {
					p[n] = res
				}
				done <- err
			})
		case map[string]interface{}:
			as = append(as, getAsyncSelectorResolvers(val)...)
		case []map[string]interface{}:
			for _, sval := range val {
				as = append(as, getAsyncSelectorResolvers(sval)...)
			}
		}
	}
	return as
}

// getAsyncGetResolver search for any unresolved asyncGet and build on asyncSelectorResolver
// per resource with all requested ids coalesced.
func getAsyncGetResolver(p map[string]interface{}) []asyncSelectorResolver {
	ags := findAsyncGets(p)
	if len(ags) == 0 {
		return nil
	}
	// map of resource -> []asyncGet
	r := map[*Resource][]asyncGet{}
	for _, ag := range ags {
		if _ags, found := r[ag.resource]; found {
			r[ag.resource] = append(_ags, ag)
		} else {
			r[ag.resource] = []asyncGet{ag}
		}
	}
	as := make([]asyncSelectorResolver, 0, len(r))
	// create a resource resolver for each resource
	for rsrc, ags := range r {
		as = append(as, func(ctx context.Context, done chan<- error) {
			// Gater ids for each asyncGet
			ids := make([]interface{}, len(ags))
			for i, ag := range ags {
				ids[i] = ag.id
			}
			// Perform the mget
			items, err := rsrc.MultiGet(ctx, ids)
			if err != nil {
				done <- err
				return
			}
			// Route back the value to corresponding asyncGet handlers
			for i, ag := range ags {
				val, err := ag.handler(ctx, items[i])
				if err != nil {
					done <- err
					return
				}
				// Put the response value in place
				ag.payload[ag.field] = val
			}
			done <- nil
		})
	}
	return as
}

func findAsyncGets(p map[string]interface{}) []asyncGet {
	ag := []asyncGet{}
	for _, val := range p {
		switch val := val.(type) {
		case asyncGet:
			ag = append(ag, val)
		case map[string]interface{}:
			ag = append(ag, findAsyncGets(val)...)
		case []map[string]interface{}:
			for _, sval := range val {
				ag = append(ag, findAsyncGets(sval)...)
			}
		}
	}
	return ag
}
<commit_msg>Always apply selectors, event if none supplied<commit_after>package resource

import (
	"errors"
	"fmt"
	"strings"

	"golang.org/x/net/context"

	"github.com/rs/rest-layer/schema"
)

// Lookup holds filter and sort used to select items in a resource collection
type Lookup struct {
	// The client supplied filter. Filter is a MongoDB inspired query with a more limited
	// set of capabilities. See https://github.com/rs/rest-layer#filtering
	// for more info.
	filter schema.Query
	// The client supplied soft. Sort is a list of resource fields or sub-fields separated
	// by comas (,). To invert the sort, a minus (-) can be prefixed.
	// See https://github.com/rs/rest-layer#sorting for more info.
	sort []string
	// The client supplied selector. Selector is a way for the client to reformat the
	// resource representation at runtime by defining which fields should be included
	// in the document. The REST Layer selector language allows field aliasing, field
	// transformation with parameters and sub-item/collection embedding.
	selector []Field
}

// Field is used with Lookup.selector to reformat the resource representation at runtime
// using a field selection language inspired by GraphQL.
type Field struct {
	// Name is the name of the field as define in the resource's schema.
	Name string
	// Alias is the wanted name in the representation.
	Alias string
	// Params defines a list of params to be sent to the field's param handler if any.
	Params map[string]interface{}
	// Fields holds references to child fields if any
	Fields []Field
}

// NewLookup creates an empty lookup object
func NewLookup() *Lookup {
	return &Lookup{
		filter: schema.Query{},
		sort:   []string{},
	}
}

// NewLookupWithQuery creates an empty lookup object with a given query
func NewLookupWithQuery(q schema.Query) *Lookup {
	return &Lookup{
		filter: q,
		sort:   []string{},
	}
}

// Sort is a list of resource fields or sub-fields separated
// by comas (,). To invert the sort, a minus (-) can be prefixed.
//
// See https://github.com/rs/rest-layer#sorting for more info.
func (l *Lookup) Sort() []string {
	return l.sort
}

// Filter is a MongoDB inspired query with a more limited set of capabilities.
//
// See https://github.com/rs/rest-layer#filtering for more info.
func (l *Lookup) Filter() schema.Query {
	return l.filter
}

// SetSorts set the sort fields with a pre-parsed list of fields to sort on.
// This method doesn't validate sort fields.
func (l *Lookup) SetSorts(sorts []string) {
	l.sort = sorts
}

// SetSort parses and validate a sort parameter and set it as lookup's Sort
func (l *Lookup) SetSort(sort string, validator schema.Validator) error {
	sorts := []string{}
	for _, f := range strings.Split(sort, ",") {
		f = strings.Trim(f, " ")
		if f == "" {
			return errors.New("empty soft field")
		}
		// If the field start with - (to indicate descended sort), shift it before
		// validator lookup
		i := 0
		if f[0] == '-' {
			i = 1
		}
		// Make sure the field exists
		field := validator.GetField(f[i:])
		if field == nil {
			return fmt.Errorf("invalid sort field: %s", f[i:])
		}
		if !field.Sortable {
			return fmt.Errorf("field is not sortable: %s", f[i:])
		}
		sorts = append(sorts, f)
	}
	l.sort = sorts
	return nil
}

// AddFilter parses and validate a filter parameter and add it to lookup's filter
//
// The filter query is validated against the provided validator to ensure all queried
// fields exists and are of the right type.
func (l *Lookup) AddFilter(filter string, validator schema.Validator) error {
	f, err := schema.ParseQuery(filter, validator)
	if err != nil {
		return err
	}
	l.AddQuery(f)
	return nil
}

// AddQuery add an existing schema.Query to the lookup's filters
func (l *Lookup) AddQuery(query schema.Query) {
	if l.filter == nil {
		l.filter = query
		return
	}
	for _, exp := range query {
		l.filter = append(l.filter, exp)
	}
}

// SetSelector parses a selector expression, validates it and assign it to the current Lookup.
func (l *Lookup) SetSelector(s string, r *Resource) error {
	pos := 0
	selector, err := parseSelectorExpression([]byte(s), &pos, len(s), false)
	if err != nil {
		return err
	}
	if err = validateSelector(selector, r.Validator()); err != nil {
		return err
	}
	l.selector = selector
	return nil
}

// ReferenceResolver is a function resolving a reference to another field
type ReferenceResolver func(path string) (*Resource, error)

// ApplySelector applies fields filtering / rename to the payload fields
func (l *Lookup) ApplySelector(ctx context.Context, r *Resource, p map[string]interface{}, resolver ReferenceResolver) (map[string]interface{}, error) {
	payload, err := applySelector(ctx, l.selector, r.Validator(), p, resolver)
	if err == nil {
		// The resulting payload may contain some asyncSelector, we must execute them
		// concurrently until there's no more
		err = resolveAsyncSelectors(ctx, payload)
	}
	return payload, err
}

func resolveAsyncSelectors(ctx context.Context, p map[string]interface{}) error {
	for {
		sr := getSelectorResolvers(p)
		if len(sr) == 0 {
			break
		}
		done := make(chan error, len(sr))
		// TODO limit the number of // sub requests
		for _, r := range sr {
			go r(ctx, done)
		}
		wait := len(sr)
		cleanup := func() {
			// Make sure we empty the channel of remaining future responses
			// to prevent leaks
			for wait > 0 {
				<-done
				wait--
			}
		}
		for wait > 0 {
			select {
			case err := <-done:
				wait--
				if err != nil {
					if wait > 0 {
						go cleanup()
					}
					return err
				}
			case <-ctx.Done():
				if wait > 0 {
					go cleanup()
				}
				return ctx.Err()
			}
		}
	}
	return nil
}

type asyncSelectorResolver func(ctx context.Context, done chan<- error)

func getSelectorResolvers(p map[string]interface{}) []asyncSelectorResolver {
	return append(getAsyncSelectorResolvers(p), getAsyncGetResolver(p)...)
}

// getAsyncSelectorResolvers parse the payload searching for any unresolved asyncSelector
// and build an asyncSelectorResolver for each ones.
func getAsyncSelectorResolvers(p map[string]interface{}) []asyncSelectorResolver {
	as := []asyncSelectorResolver{}
	for name, val := range p {
		switch val := val.(type) {
		case asyncSelector:
			n := name
			as = append(as, func(ctx context.Context, done chan<- error) {
				res, err := val(ctx)
				if err == nil {
					p[n] = res
				}
				done <- err
			})
		case map[string]interface{}:
			as = append(as, getAsyncSelectorResolvers(val)...)
		case []map[string]interface{}:
			for _, sval := range val {
				as = append(as, getAsyncSelectorResolvers(sval)...)
			}
		}
	}
	return as
}

// getAsyncGetResolver search for any unresolved asyncGet and build on asyncSelectorResolver
// per resource with all requested ids coalesced.
func getAsyncGetResolver(p map[string]interface{}) []asyncSelectorResolver {
	ags := findAsyncGets(p)
	if len(ags) == 0 {
		return nil
	}
	// map of resource -> []asyncGet
	r := map[*Resource][]asyncGet{}
	for _, ag := range ags {
		if _ags, found := r[ag.resource]; found {
			r[ag.resource] = append(_ags, ag)
		} else {
			r[ag.resource] = []asyncGet{ag}
		}
	}
	as := make([]asyncSelectorResolver, 0, len(r))
	// create a resource resolver for each resource
	for rsrc, ags := range r {
		as = append(as, func(ctx context.Context, done chan<- error) {
			// Gater ids for each asyncGet
			ids := make([]interface{}, len(ags))
			for i, ag := range ags {
				ids[i] = ag.id
			}
			// Perform the mget
			items, err := rsrc.MultiGet(ctx, ids)
			if err != nil {
				done <- err
				return
			}
			// Route back the value to corresponding asyncGet handlers
			for i, ag := range ags {
				val, err := ag.handler(ctx, items[i])
				if err != nil {
					done <- err
					return
				}
				// Put the response value in place
				ag.payload[ag.field] = val
			}
			done <- nil
		})
	}
	return as
}

func findAsyncGets(p map[string]interface{}) []asyncGet {
	ag := []asyncGet{}
	for _, val := range p {
		switch val := val.(type) {
		case asyncGet:
			ag = append(ag, val)
		case map[string]interface{}:
			ag = append(ag, findAsyncGets(val)...)
		case []map[string]interface{}:
			for _, sval := range val {
				ag = append(ag, findAsyncGets(sval)...)
			}
		}
	}
	return ag
}
<|endoftext|>


<commit_before>package goxp

import (
	"bufio"
	"fmt"
	"net"
	"net/http"
)
<commit_msg>response_writer.go +first commit<commit_after>package goxp

import (
	"bufio"
	"fmt"
	"net"
	"net/http"
)

// ResponseWriter is a wrapper around http.ResponseWriter that provides extra information about
// the response. It is recommended that middleware handlers use this construct to wrap a responsewriter
// if the functionality calls for it.
type ResponseWriter interface {
	http.ResponseWriter
	http.Flusher
	http.Hijacker
	// Status returns the status code of the response or 0 if the response has not been Written
	Status() int
	// Written returns whether or not the ResponseWriter has been written.
	Written() bool
	// Size returns the size of the response body.
	Size() int
	// Before allows for a function to be called before the ResponseWriter has been written to. This is
	// useful for setting headers or any other operations tat must happe before a response has been written.
	Before(BeforeFunc)
}

<|endoftext|>