<commit_before>package scaling_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"lib/models"
	"lib/policy_client"
	"lib/testsupport"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"code.cloudfoundry.org/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/rainmaker"
)

const Timeout_Check = 20 * time.Minute

// 3 * observed agent total poll time with 200 containers per cell
const Policy_Update_Wait = 180 * time.Second

var ports []int

var _ = Describe("how the container network performs at scale", func() {
	Describe("networking policy", func() {
		var (
			appProxy       string
			appRegistry    string
			appsTest       []string
			appInstances   int
			applications   int
			proxyInstances int
			sampleSize     int
		)

		BeforeEach(func() {
			appInstances = pushConfig.AppInstances
			applications = pushConfig.Applications
			proxyInstances = pushConfig.ProxyInstances
			sampleSize = pushConfig.SampleSize
			appProxy = pushConfig.Prefix + "proxy"
			appRegistry = pushConfig.Prefix + "registry"
			for i := 0; i < applications; i++ {
				appsTest = append(appsTest, fmt.Sprintf(pushConfig.Prefix+"tick-%d", i))
			}

			ports = []int{8080}
			for i := 0; i < pushConfig.ExtraListenPorts; i++ {
				ports = append(ports, 7000+i)
			}
		})

		It("allows the user to configure policies", func(done Done) {
			By("checking that all test app instances have registered themselves")
			checkRegistry(appRegistry, 10*time.Second, 500*time.Millisecond, len(appsTest)*appInstances)

			appIPs := getAppIPs(appRegistry)
			sample := sampleIPs(appIPs, sampleSize)

			By(fmt.Sprintf("checking that the connection fails sampling %d out of %d IPs on %d ports", len(sample), len(appIPs), len(ports)))
			runWithTimeout("check connection failures", Timeout_Check, func() {
				assertConnectionFails(appProxy, sample, ports, proxyInstances)
			})

			By("creating policies")
			doAllPolicies("create", appProxy, appsTest, ports)

			By(fmt.Sprintf("waiting %s for policies to be updated on cells", Policy_Update_Wait))
			time.Sleep(Policy_Update_Wait)

			sample = sampleIPs(appIPs, sampleSize)
			By(fmt.Sprintf("checking that the connection succeeds sampling %d out of %d IPs on %d ports", len(sample), len(appIPs), len(ports)))
			runWithTimeout("check connection success", Timeout_Check, func() {
				assertConnectionSucceeds(appProxy, sample, ports, proxyInstances)
			})

			By("dumping stats to commit to stats repo")
			dumpStats(appProxy, config.AppsDomain)

			By("sleeping for 5 minutes while policies exist")
			time.Sleep(5 * time.Minute)

			By("deleting policies")
			doAllPolicies("delete", appProxy, appsTest, ports)

			By(fmt.Sprintf("waiting %s for policies to be updated on cells", Policy_Update_Wait))
			time.Sleep(Policy_Update_Wait)

			sample = sampleIPs(appIPs, sampleSize)
			By(fmt.Sprintf("checking that the connection succeeds sampling %d out of %d IPs on %d ports", len(sample), len(appIPs), len(ports)))
			runWithTimeout("check connection failures, again", Timeout_Check, func() {
				assertConnectionFails(appProxy, sample, ports, proxyInstances)
			})

			close(done)
		}, 30*60 /* <-- overall spec timeout in seconds */)
	})
	Describe("sampleIPs", func() {
		var population []string
		BeforeEach(func() {
			population = []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}
		})

		It("returns a sample of unique choices from the population", func() {
			sample := sampleIPs(population, 9)
			Expect(len(sample)).To(Equal(9))
			for i := 0; i < len(sample); i++ {
				for j := i + 1; j < len(sample); j++ {
					Expect(sample[i]).NotTo(Equal(sample[j]))
				}
			}
		})
		Context("when the sample size is larger than the population", func() {
			It("returns the whole population", func() {
				sample := sampleIPs(population, 999)
				Expect(sample).To(Equal(population))
			})
		})
		Context("when the sample size is equal to the population size", func() {
			It("returns the whole population", func() {
				sample := sampleIPs(population, len(population))
				Expect(sample).To(Equal(population))
			})
		})
		Context("when the sample size is zero", func() {
			It("returns the whole population", func() {
				sample := sampleIPs(population, 0)
				Expect(sample).To(Equal(population))
			})
		})
		Context("when the sample size is negative", func() {
			It("returns the whole population", func() {
				sample := sampleIPs(population, -1)
				Expect(sample).To(Equal(population))
			})
		})
	})
})

func getToken() string {
	cmd := exec.Command("cf", "oauth-token")
	session, err := gexec.Start(cmd, nil, nil)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session.Wait(Timeout_Short)).Should(gexec.Exit(0))
	rawOutput := string(session.Out.Contents())
	return strings.TrimSpace(strings.TrimPrefix(rawOutput, "bearer "))
}

func getGuids(sourceAppName string, dstAppNames []string) (string, []string) {
	dstGuids := []string{}
	sourceGuid := ""
	token := getToken()
	appsClient := rainmaker.NewApplicationsService(rainmaker.Config{Host: "http://" + config.ApiEndpoint})

	appsList, err := appsClient.List(token)
	Expect(err).NotTo(HaveOccurred())

	for {
		for _, app := range appsList.Applications {
			if app.Name == sourceAppName {
				sourceGuid = app.GUID
				continue
			}
			for _, tickAppName := range dstAppNames {
				if app.Name == tickAppName {
					dstGuids = append(dstGuids, app.GUID)
					break
				}
			}
		}
		if appsList.HasNextPage() {
			appsList, err = appsList.Next(token)
			Expect(err).NotTo(HaveOccurred())
		} else {
			break
		}
	}

	Expect(sourceGuid).NotTo(BeEmpty())
	Expect(dstGuids).To(HaveLen(len(dstAppNames)))

	return sourceGuid, dstGuids
}

func doAllPolicies(action string, source string, dstList []string, dstPorts []int) {
	policyClient := policy_client.NewExternal(lagertest.NewTestLogger("test"), &http.Client{}, "http://"+config.ApiEndpoint)
	sourceGuid, dstGuids := getGuids(source, dstList)
	policies := []models.Policy{}
	for _, dstGuid := range dstGuids {
		for _, port := range dstPorts {
			policies = append(policies, models.Policy{
				Source: models.Source{
					ID: sourceGuid,
				},
				Destination: models.Destination{
					ID:       dstGuid,
					Port:     port,
					Protocol: "tcp",
				},
			})
		}
	}
	token := getToken()
	if action == "create" {
		Expect(policyClient.AddPolicies(token, policies)).To(Succeed())
	} else if action == "delete" {
		Expect(policyClient.DeletePolicies(token, policies)).To(Succeed())
	}
}

func runWithTimeout(operation string, timeout time.Duration, work func()) {
	done := make(chan bool)
	go func() {
		defer GinkgoRecover()
		defer fmt.Printf("\nfailure during %s\n", operation)

		By(fmt.Sprintf("starting %s\n", operation))
		work()
		done <- true
	}()

	select {
	case <-done:
		By(fmt.Sprintf("completed %s\n", operation))
		return
	case <-time.After(timeout):
		Fail("timeout on " + operation)
	}
}

func dumpStats(host, domain string) {
	resp, err := httpGetBytes(fmt.Sprintf("http://%s.%s/stats", host, domain))
	Expect(err).NotTo(HaveOccurred())

	netStatsFile := os.Getenv("NETWORK_STATS_FILE")
	if netStatsFile != "" {
		Expect(ioutil.WriteFile(netStatsFile, resp.Body, 0600)).To(Succeed())
	}
}

type RegistryInstancesResponse struct {
	Instances []struct {
		ServiceName string `json:"service_name"`
		Endpoint    struct {
			Value string `json:"value"`
		} `json:"endpoint"`
	} `json:"instances"`
}

func getInstancesFromA8(registry string) (*RegistryInstancesResponse, error) {
	resp, err := httpGetBytes(fmt.Sprintf("http://%s.%s/api/v1/instances", registry, config.AppsDomain))
	if err != nil {
		return nil, err
	}

	Expect(resp.StatusCode).To(Equal(http.StatusOK))

	var instancesResponse RegistryInstancesResponse
	err = json.Unmarshal(resp.Body, &instancesResponse)
	if err != nil {
		return nil, err
	}
	return &instancesResponse, nil
}

func checkRegistry(registry string, timeout, pollingInterval time.Duration, totalInstances int) {
	registeredApps := func() (int, error) {
		instancesResponse, err := getInstancesFromA8(registry)
		if err != nil {
			return 0, err
		}
		return len(instancesResponse.Instances), nil
	}

	Eventually(registeredApps, timeout, pollingInterval).Should(Equal(totalInstances))
}

func getAppIPs(registry string) []string {
	instancesResponse, err := getInstancesFromA8(registry)
	Expect(err).NotTo(HaveOccurred())

	ips := []string{}
	for _, instance := range instancesResponse.Instances {
		ip, _, err := net.SplitHostPort(instance.Endpoint.Value)
		Expect(err).NotTo(HaveOccurred())
		ips = append(ips, ip)
	}
	return ips
}

func sampleIPs(population []string, sampleSize int) []string {
	populationSize := len(population)
	if len(population) <= sampleSize || sampleSize < 1 {
		return population
	}
	var sample = []string{}
	for i := 0; i < sampleSize; i++ {
		j := rand.Intn(populationSize)
		sample = append(sample, population[j])
		population = append(population[:j], population[j+1:]...)
		populationSize--
	}
	return sample
}

func assertConnectionSucceeds(sourceApp string, destApps []string, ports []int, nProxies int) {
	parallelRunner := &testsupport.ParallelRunner{
		NumWorkers: 50 * nProxies,
	}
	parallelRunner.RunOnSliceStrings(destApps, func(appIP string) {
		for _, port := range ports {
			assertSingleConnection(appIP, port, sourceApp, true)
		}
	})
}

func assertConnectionFails(sourceApp string, destApps []string, ports []int, nProxies int) {
	parallelRunner := &testsupport.ParallelRunner{
		NumWorkers: 50 * nProxies,
	}
	parallelRunner.RunOnSliceStrings(destApps, func(appIP string) {
		for _, port := range ports {
			assertSingleConnection(appIP, port, sourceApp, false)
		}
	})
}

func assertSingleConnection(destIP string, port int, sourceAppName string, shouldSucceed bool) {
	if shouldSucceed {
		assertResponseContains(destIP, port, sourceAppName, "application_name")
	} else {
		assertResponseContains(destIP, port, sourceAppName, "request failed")
	}
}

func assertResponseContains(destIP string, port int, sourceAppName string, desiredResponse string) {
	proxyTest := func() (string, error) {
		resp, err := httpGetBytes(fmt.Sprintf("http://%s.%s/proxy/%s:%d", sourceAppName, config.AppsDomain, destIP, port))
		if err != nil {
			return "", err
		}
		return string(resp.Body), nil
	}
	Eventually(proxyTest, 10*time.Second, 500*time.Millisecond).Should(ContainSubstring(desiredResponse))
}

var httpClient = &http.Client{
	Transport: &http.Transport{
		DisableKeepAlives: true,
		Dial: (&net.Dialer{
			Timeout:   4 * time.Second,
			KeepAlive: 0,
		}).Dial,
	},
}

type httpResp struct {
	StatusCode int
	Body       []byte
}

func httpGetBytes(url string) (httpResp, error) {
	resp, err := httpClient.Get(url)
	if err != nil {
		return httpResp{}, err
	}
	defer resp.Body.Close()

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return httpResp{}, err
	}

	return httpResp{resp.StatusCode, respBytes}, nil
}
<commit_msg>scaling test: runWithTimeout correctly fails when work func fails<commit_after>package scaling_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"lib/models"
	"lib/policy_client"
	"lib/testsupport"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/exec"
	"strings"
	"time"

	"code.cloudfoundry.org/lager/lagertest"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-cf-experimental/rainmaker"
)

const Timeout_Check = 20 * time.Minute

// 3 * observed agent total poll time with 200 containers per cell
const Policy_Update_Wait = 180 * time.Second

var ports []int

var _ = Describe("how the container network performs at scale", func() {
	Describe("networking policy", func() {
		var (
			appProxy       string
			appRegistry    string
			appsTest       []string
			appInstances   int
			applications   int
			proxyInstances int
			sampleSize     int
		)

		BeforeEach(func() {
			appInstances = pushConfig.AppInstances
			applications = pushConfig.Applications
			proxyInstances = pushConfig.ProxyInstances
			sampleSize = pushConfig.SampleSize
			appProxy = pushConfig.Prefix + "proxy"
			appRegistry = pushConfig.Prefix + "registry"
			for i := 0; i < applications; i++ {
				appsTest = append(appsTest, fmt.Sprintf(pushConfig.Prefix+"tick-%d", i))
			}

			ports = []int{8080}
			for i := 0; i < pushConfig.ExtraListenPorts; i++ {
				ports = append(ports, 7000+i)
			}
		})

		It("allows the user to configure policies", func(done Done) {
			By("checking that all test app instances have registered themselves")
			checkRegistry(appRegistry, 10*time.Second, 500*time.Millisecond, len(appsTest)*appInstances)

			appIPs := getAppIPs(appRegistry)
			sample := sampleIPs(appIPs, sampleSize)

			By(fmt.Sprintf("checking that the connection fails sampling %d out of %d IPs on %d ports", len(sample), len(appIPs), len(ports)))
			runWithTimeout("check connection failures", Timeout_Check, func() {
				assertConnectionFails(appProxy, sample, ports, proxyInstances)
			})

			By("creating policies")
			doAllPolicies("create", appProxy, appsTest, ports)

			By(fmt.Sprintf("waiting %s for policies to be updated on cells", Policy_Update_Wait))
			time.Sleep(Policy_Update_Wait)

			sample = sampleIPs(appIPs, sampleSize)
			By(fmt.Sprintf("checking that the connection succeeds sampling %d out of %d IPs on %d ports", len(sample), len(appIPs), len(ports)))
			runWithTimeout("check connection success", Timeout_Check, func() {
				assertConnectionSucceeds(appProxy, sample, ports, proxyInstances)
			})

			By("dumping stats to commit to stats repo")
			dumpStats(appProxy, config.AppsDomain)

			By("sleeping for 5 minutes while policies exist")
			time.Sleep(5 * time.Minute)

			By("deleting policies")
			doAllPolicies("delete", appProxy, appsTest, ports)

			By(fmt.Sprintf("waiting %s for policies to be updated on cells", Policy_Update_Wait))
			time.Sleep(Policy_Update_Wait)

			sample = sampleIPs(appIPs, sampleSize)
			By(fmt.Sprintf("checking that the connection succeeds sampling %d out of %d IPs on %d ports", len(sample), len(appIPs), len(ports)))
			runWithTimeout("check connection failures, again", Timeout_Check, func() {
				assertConnectionFails(appProxy, sample, ports, proxyInstances)
			})

			close(done)
		}, 30*60 /* <-- overall spec timeout in seconds */)
	})
	Describe("sampleIPs", func() {
		var population []string
		BeforeEach(func() {
			population = []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j"}
		})

		It("returns a sample of unique choices from the population", func() {
			sample := sampleIPs(population, 9)
			Expect(len(sample)).To(Equal(9))
			for i := 0; i < len(sample); i++ {
				for j := i + 1; j < len(sample); j++ {
					Expect(sample[i]).NotTo(Equal(sample[j]))
				}
			}
		})
		Context("when the sample size is larger than the population", func() {
			It("returns the whole population", func() {
				sample := sampleIPs(population, 999)
				Expect(sample).To(Equal(population))
			})
		})
		Context("when the sample size is equal to the population size", func() {
			It("returns the whole population", func() {
				sample := sampleIPs(population, len(population))
				Expect(sample).To(Equal(population))
			})
		})
		Context("when the sample size is zero", func() {
			It("returns the whole population", func() {
				sample := sampleIPs(population, 0)
				Expect(sample).To(Equal(population))
			})
		})
		Context("when the sample size is negative", func() {
			It("returns the whole population", func() {
				sample := sampleIPs(population, -1)
				Expect(sample).To(Equal(population))
			})
		})
	})
})

func getToken() string {
	cmd := exec.Command("cf", "oauth-token")
	session, err := gexec.Start(cmd, nil, nil)
	Expect(err).NotTo(HaveOccurred())
	Eventually(session.Wait(Timeout_Short)).Should(gexec.Exit(0))
	rawOutput := string(session.Out.Contents())
	return strings.TrimSpace(strings.TrimPrefix(rawOutput, "bearer "))
}

func getGuids(sourceAppName string, dstAppNames []string) (string, []string) {
	dstGuids := []string{}
	sourceGuid := ""
	token := getToken()
	appsClient := rainmaker.NewApplicationsService(rainmaker.Config{Host: "http://" + config.ApiEndpoint})

	appsList, err := appsClient.List(token)
	Expect(err).NotTo(HaveOccurred())

	for {
		for _, app := range appsList.Applications {
			if app.Name == sourceAppName {
				sourceGuid = app.GUID
				continue
			}
			for _, tickAppName := range dstAppNames {
				if app.Name == tickAppName {
					dstGuids = append(dstGuids, app.GUID)
					break
				}
			}
		}
		if appsList.HasNextPage() {
			appsList, err = appsList.Next(token)
			Expect(err).NotTo(HaveOccurred())
		} else {
			break
		}
	}

	Expect(sourceGuid).NotTo(BeEmpty())
	Expect(dstGuids).To(HaveLen(len(dstAppNames)))

	return sourceGuid, dstGuids
}

func doAllPolicies(action string, source string, dstList []string, dstPorts []int) {
	policyClient := policy_client.NewExternal(lagertest.NewTestLogger("test"), &http.Client{}, "http://"+config.ApiEndpoint)
	sourceGuid, dstGuids := getGuids(source, dstList)
	policies := []models.Policy{}
	for _, dstGuid := range dstGuids {
		for _, port := range dstPorts {
			policies = append(policies, models.Policy{
				Source: models.Source{
					ID: sourceGuid,
				},
				Destination: models.Destination{
					ID:       dstGuid,
					Port:     port,
					Protocol: "tcp",
				},
			})
		}
	}
	token := getToken()
	if action == "create" {
		Expect(policyClient.AddPolicies(token, policies)).To(Succeed())
	} else if action == "delete" {
		Expect(policyClient.DeletePolicies(token, policies)).To(Succeed())
	}
}

func runWithTimeout(operation string, timeout time.Duration, work func()) {
	done := make(chan bool)
	go func() {
		defer func() { close(done) }()
		defer GinkgoRecover()
		defer fmt.Printf("\nfailure during %s\n", operation)

		By(fmt.Sprintf("starting %s\n", operation))
		work()
		By(fmt.Sprintf("completed %s\n", operation))
		done <- true
	}()

	select {
	case ok := <-done:
		if !ok {
			Fail("failure during " + operation)
		}
	case <-time.After(timeout):
		Fail("timeout on " + operation)
	}
}

func dumpStats(host, domain string) {
	resp, err := httpGetBytes(fmt.Sprintf("http://%s.%s/stats", host, domain))
	Expect(err).NotTo(HaveOccurred())

	netStatsFile := os.Getenv("NETWORK_STATS_FILE")
	if netStatsFile != "" {
		Expect(ioutil.WriteFile(netStatsFile, resp.Body, 0600)).To(Succeed())
	}
}

type RegistryInstancesResponse struct {
	Instances []struct {
		ServiceName string `json:"service_name"`
		Endpoint    struct {
			Value string `json:"value"`
		} `json:"endpoint"`
	} `json:"instances"`
}

func getInstancesFromA8(registry string) (*RegistryInstancesResponse, error) {
	resp, err := httpGetBytes(fmt.Sprintf("http://%s.%s/api/v1/instances", registry, config.AppsDomain))
	if err != nil {
		return nil, err
	}

	Expect(resp.StatusCode).To(Equal(http.StatusOK))

	var instancesResponse RegistryInstancesResponse
	err = json.Unmarshal(resp.Body, &instancesResponse)
	if err != nil {
		return nil, err
	}
	return &instancesResponse, nil
}

func checkRegistry(registry string, timeout, pollingInterval time.Duration, totalInstances int) {
	registeredApps := func() (int, error) {
		instancesResponse, err := getInstancesFromA8(registry)
		if err != nil {
			return 0, err
		}
		return len(instancesResponse.Instances), nil
	}

	Eventually(registeredApps, timeout, pollingInterval).Should(Equal(totalInstances))
}

func getAppIPs(registry string) []string {
	instancesResponse, err := getInstancesFromA8(registry)
	Expect(err).NotTo(HaveOccurred())

	ips := []string{}
	for _, instance := range instancesResponse.Instances {
		ip, _, err := net.SplitHostPort(instance.Endpoint.Value)
		Expect(err).NotTo(HaveOccurred())
		ips = append(ips, ip)
	}
	return ips
}

func sampleIPs(population []string, sampleSize int) []string {
	populationSize := len(population)
	if len(population) <= sampleSize || sampleSize < 1 {
		return population
	}
	var sample = []string{}
	for i := 0; i < sampleSize; i++ {
		j := rand.Intn(populationSize)
		sample = append(sample, population[j])
		population = append(population[:j], population[j+1:]...)
		populationSize--
	}
	return sample
}

func assertConnectionSucceeds(sourceApp string, destApps []string, ports []int, nProxies int) {
	parallelRunner := &testsupport.ParallelRunner{
		NumWorkers: 50 * nProxies,
	}
	parallelRunner.RunOnSliceStrings(destApps, func(appIP string) {
		for _, port := range ports {
			assertSingleConnection(appIP, port, sourceApp, true)
		}
	})
}

func assertConnectionFails(sourceApp string, destApps []string, ports []int, nProxies int) {
	parallelRunner := &testsupport.ParallelRunner{
		NumWorkers: 50 * nProxies,
	}
	parallelRunner.RunOnSliceStrings(destApps, func(appIP string) {
		for _, port := range ports {
			assertSingleConnection(appIP, port, sourceApp, false)
		}
	})
}

func assertSingleConnection(destIP string, port int, sourceAppName string, shouldSucceed bool) {
	if shouldSucceed {
		assertResponseContains(destIP, port, sourceAppName, "application_name")
	} else {
		assertResponseContains(destIP, port, sourceAppName, "request failed")
	}
}

func assertResponseContains(destIP string, port int, sourceAppName string, desiredResponse string) {
	proxyTest := func() (string, error) {
		resp, err := httpGetBytes(fmt.Sprintf("http://%s.%s/proxy/%s:%d", sourceAppName, config.AppsDomain, destIP, port))
		if err != nil {
			return "", err
		}
		return string(resp.Body), nil
	}
	Eventually(proxyTest, 10*time.Second, 500*time.Millisecond).Should(ContainSubstring(desiredResponse))
}

var httpClient = &http.Client{
	Transport: &http.Transport{
		DisableKeepAlives: true,
		Dial: (&net.Dialer{
			Timeout:   4 * time.Second,
			KeepAlive: 0,
		}).Dial,
	},
}

type httpResp struct {
	StatusCode int
	Body       []byte
}

func httpGetBytes(url string) (httpResp, error) {
	resp, err := httpClient.Get(url)
	if err != nil {
		return httpResp{}, err
	}
	defer resp.Body.Close()

	respBytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return httpResp{}, err
	}

	return httpResp{resp.StatusCode, respBytes}, nil
}
<|endoftext|>


<commit_before>package santander

import (
	"testing"
	"time"

	"github.com/mundipagg/boleto-api/env"
	"github.com/mundipagg/boleto-api/mock"
	"github.com/mundipagg/boleto-api/models"
	"github.com/mundipagg/boleto-api/util"
	. "github.com/smartystreets/goconvey/convey"
)

const baseMockJSON = `
{
	"BankNumber": 33,
	"Agreement": {
		"AgreementNumber": 11111111,
		"Agency":"5555",
		"Account":"55555"
	},
	"Title": {
		"ExpireDate": "2035-08-01",
		"AmountInCents": 200,
		"OurNumber":10000000004
	},
	"Buyer": {
		"Name": "TESTE",
		"Document": {
			"Type": "CPF",
			"Number": "12345678903"
		}
	},
	"Recipient": {
		"Name": "TESTE",
		"Document": {
			"Type": "CNPJ",
			"Number": "55555555555555"
		}
	}
}
`

func TestShouldProcessBoletoSantander(t *testing.T) {
	env.Config(true, true, true)
	input := new(models.BoletoRequest)
	if err := util.FromJSON(baseMockJSON, input); err != nil {
		t.Fail()
	}
	bank := New()
	go mock.Run("9097")
	time.Sleep(2 * time.Second)
	Convey("deve-se processar um boleto santander com sucesso", t, func() {
		output, err := bank.ProcessBoleto(input)
		So(err, ShouldBeNil)
		So(output.BarCodeNumber, ShouldNotBeEmpty)
		So(output.DigitableLine, ShouldNotBeEmpty)
		So(output.Errors, ShouldBeEmpty)
	})

	input.Title.BoletoType = "BP"
	Convey("deve-se mapear corretamente o BoletoType de boleto de proposta", t, func() {
		output := bank.GetBoletoType(input)
		So(output, ShouldEqual, "32")
	})

	input.Title.BoletoType = "Santander"
	Convey("deve-se mapear corretamente o BoletoType quando valor enviado não existir", t, func() {
		output := bank.GetBoletoType(input)
		So(output, ShouldEqual, "02")
	})
}
<commit_msg>:art: Melhora teste BoletoType Santander<commit_after>package santander

import (
	"testing"
	"time"

	"github.com/mundipagg/boleto-api/env"
	"github.com/mundipagg/boleto-api/mock"
	"github.com/mundipagg/boleto-api/models"
	"github.com/mundipagg/boleto-api/util"
	. "github.com/smartystreets/goconvey/convey"
)

const baseMockJSON = `
{
	"BankNumber": 33,
	"Agreement": {
		"AgreementNumber": 11111111,
		"Agency":"5555",
		"Account":"55555"
	},
	"Title": {
		"ExpireDate": "2035-08-01",
		"AmountInCents": 200,
		"OurNumber":10000000004
	},
	"Buyer": {
		"Name": "TESTE",
		"Document": {
			"Type": "CPF",
			"Number": "12345678903"
		}
	},
	"Recipient": {
		"Name": "TESTE",
		"Document": {
			"Type": "CNPJ",
			"Number": "55555555555555"
		}
	}
}
`

func TestShouldProcessBoletoSantander(t *testing.T) {
	env.Config(true, true, true)
	input := new(models.BoletoRequest)
	if err := util.FromJSON(baseMockJSON, input); err != nil {
		t.Fail()
	}
	bank := New()
	go mock.Run("9097")
	time.Sleep(2 * time.Second)
	Convey("deve-se processar um boleto santander com sucesso", t, func() {
		output, err := bank.ProcessBoleto(input)
		So(err, ShouldBeNil)
		So(output.BarCodeNumber, ShouldNotBeEmpty)
		So(output.DigitableLine, ShouldNotBeEmpty)
		So(output.Errors, ShouldBeEmpty)
	})
}

func TestShouldMapSantanderBoletoType(t *testing.T) {
	env.Config(true, true, true)
	input := new(models.BoletoRequest)
	if err := util.FromJSON(baseMockJSON, input); err != nil {
		t.Fail()
	}
	bank := New()
	go mock.Run("9097")
	time.Sleep(2 * time.Second)

	Convey("deve-se mapear corretamente o BoletoType quando informação for vazia", t, func() {
		output := bank.GetBoletoType(input)
		So(input.Title.BoletoType, ShouldEqual, "")
		So(output, ShouldEqual, "02")
	})

	input.Title.BoletoType = "BP"
	Convey("deve-se mapear corretamente o BoletoType de boleto de proposta", t, func() {
		output := bank.GetBoletoType(input)
		So(output, ShouldEqual, "32")
	})

	input.Title.BoletoType = "Santander"
	Convey("deve-se mapear corretamente o BoletoType quando valor enviado não existir", t, func() {
		output := bank.GetBoletoType(input)
		So(output, ShouldEqual, "02")
	})
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/gorilla/mux"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

func RequestCountsHandler(params url.Values, request *http.Request) ([]byte, *ApiError) {
	// for a given request service type and date, return the count
	// of requests for that date, grouped by ward, and the city total
	// The output is a map where keys are ward identifiers, and the value is the count.
	//
	// Sample request and output:
	// $ curl "http://localhost:5000/requests/4fd3b167e750846744000005/counts.json?end_date=2013-06-10&count=1"
	// {
        //   "DayData": [
        //     "2013-06-10"
        //   ],
        //   "CityData": {
        //     "Average": 1.5424658,
        //     "DailyMax": [
        //       114,
        //       106,
        //       104,
        //       102,
        //       102,
        //       94,
        //       93
        //     ],
        //     "Count": 563
        //   },
        //   "WardData": {
        //     "1": {
        //       "Counts": [
        //         33
        //       ],
        //       "Average": 6.917808
        //     },
        //     "10": {
        //       "Counts": [
        //         6
        //       ],
        //       "Average": 2.4958904
        //     },
        //     "11": {
        //       "Counts": [
        //         26
        //       ],
        //       "Average": 8.087671
        //     },
        //     (... truncated ...)
        //   }
        // }

	vars := mux.Vars(request)
	service_code := vars["service_code"]

	// determine date range.
	days, err := strconv.Atoi(params.Get("count"))
	if err != nil || days > 60 || days < 1 {
		return nil, &ApiError{Msg: "invalid count, must be integer, 1..60", Code: 400}
	}

	chi, _ := time.LoadLocation("America/Chicago")
	end, _ := time.ParseInLocation("2006-01-02", params["end_date"][0], chi)
	end = end.AddDate(0, 0, 1) // inc to the following day
	start := end.AddDate(0, 0, -days)

	rows, err := api.Db.Query(`SELECT total,ward,requested_date
		FROM daily_counts
		WHERE service_code = $1
			AND requested_date >= $2
			AND requested_date < $3
		ORDER BY ward DESC, requested_date DESC`,
		string(service_code), start, end)

	if err != nil {
		return backend_error(err)
	}

	data := make(map[int]map[string]int)
	// { 32: { '2013-07-23': 42, '2013-07-24': 41 }, 3: { '2013-07-23': 42, '2013-07-24': 41 } }

	for rows.Next() {
		var ward, count int
		var date time.Time

		if err := rows.Scan(&count, &ward, &date); err != nil {
			return backend_error(err)
		}

		if _, present := data[ward]; !present {
			data[ward] = make(map[string]int)
		}

		data[ward][date.Format("2006-01-02")] = count
	}

	type WardCount struct {
		Ward    int
		Counts  []int
		Average float32
	}

	counts := make(map[int]WardCount)
	var day_data []string

	// generate a list of days returned in the results
	for day := 0; day < days; day++ {
		day_data = append(day_data, start.AddDate(0, 0, day).Format("2006-01-02"))
	}

	// for each ward, and each day, find the count and populate result
	for i := 1; i < 51; i++ {
		for day := 0; day < days; day++ {
			d := start.AddDate(0, 0, day)
			c := 0
			if total_for_day, present := data[i][d.Format("2006-01-02")]; present {
				c = total_for_day
			}

			tmp := counts[i]
			tmp.Counts = append(counts[i].Counts, c)
			counts[i] = tmp
		}
	}

	rows, err = api.Db.Query(`SELECT SUM(total)/365.0, ward
             FROM daily_counts
             WHERE requested_date >= DATE(NOW() - INTERVAL '1 year')
                     AND service_code = $1
             GROUP BY ward;`, service_code)

	if err != nil {
		return backend_error(err)
	}

	for rows.Next() {
		var count float32
		var ward int
		if err := rows.Scan(&count, &ward); err != nil {
			return backend_error(err)
		}

		tmp := counts[ward]
		tmp.Average = count
		counts[ward] = tmp
	}

	type CityCount struct {
		Average  float32
		DailyMax []int
		Count    int
	}

	// find total opened for the entire city for date range
	var city_total CityCount
	err = api.Db.QueryRow(`SELECT SUM(total)
                     FROM daily_counts
                     WHERE service_code = $1
                             AND requested_date >= $2
                             AND requested_date < $3;`,
		string(service_code), start, end).Scan(&city_total.Count)

	if err != nil {
		return backend_error(err)
	}

	city_total.Average = float32(city_total.Count) / 365.0

	// find the seven largest days of all time
	rows, err = api.Db.Query(`SELECT total
                     FROM daily_counts
                     WHERE service_code = $1
                     ORDER BY total DESC
                     LIMIT 7;`,
		string(service_code))

	for rows.Next() {
		var daily_max int
		if err := rows.Scan(&daily_max); err != nil {
			return backend_error(err)
		}

		city_total.DailyMax = append(city_total.DailyMax, daily_max)
	}

	// pluck data to return, ensure we return a number, even zero, for each ward
	type WC struct {
		Counts  []int
		Average float32
	}

	complete_wards := make(map[string]WC)
	for i := 1; i < 51; i++ {
		k := strconv.Itoa(i)
		tmp := complete_wards[k]
		tmp.Counts = counts[i].Counts
		tmp.Average = counts[i].Average
		complete_wards[k] = tmp
	}

	type RespData struct {
		DayData  []string
		CityData CityCount
		WardData map[string]WC
	}

	return dumpJson(RespData{CityData: city_total, WardData: complete_wards, DayData: day_data}), nil
}
<commit_msg>gofmt<commit_after>package main

import (
	"github.com/gorilla/mux"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

func RequestCountsHandler(params url.Values, request *http.Request) ([]byte, *ApiError) {
	// for a given request service type and date, return the count
	// of requests for that date, grouped by ward, and the city total
	// The output is a map where keys are ward identifiers, and the value is the count.
	//
	// Sample request and output:
	// $ curl "http://localhost:5000/requests/4fd3b167e750846744000005/counts.json?end_date=2013-06-10&count=1"
	// {
	//   "DayData": [
	//     "2013-06-10"
	//   ],
	//   "CityData": {
	//     "Average": 1.5424658,
	//     "DailyMax": [
	//       114,
	//       106,
	//       104,
	//       102,
	//       102,
	//       94,
	//       93
	//     ],
	//     "Count": 563
	//   },
	//   "WardData": {
	//     "1": {
	//       "Counts": [
	//         33
	//       ],
	//       "Average": 6.917808
	//     },
	//     "10": {
	//       "Counts": [
	//         6
	//       ],
	//       "Average": 2.4958904
	//     },
	//     "11": {
	//       "Counts": [
	//         26
	//       ],
	//       "Average": 8.087671
	//     },
	//     (... truncated ...)
	//   }
	// }

	vars := mux.Vars(request)
	service_code := vars["service_code"]

	// determine date range.
	days, err := strconv.Atoi(params.Get("count"))
	if err != nil || days > 60 || days < 1 {
		return nil, &ApiError{Msg: "invalid count, must be integer, 1..60", Code: 400}
	}

	chi, _ := time.LoadLocation("America/Chicago")
	end, _ := time.ParseInLocation("2006-01-02", params["end_date"][0], chi)
	end = end.AddDate(0, 0, 1) // inc to the following day
	start := end.AddDate(0, 0, -days)

	rows, err := api.Db.Query(`SELECT total,ward,requested_date
		FROM daily_counts
		WHERE service_code = $1
			AND requested_date >= $2
			AND requested_date < $3
		ORDER BY ward DESC, requested_date DESC`,
		string(service_code), start, end)

	if err != nil {
		return backend_error(err)
	}

	data := make(map[int]map[string]int)
	// { 32: { '2013-07-23': 42, '2013-07-24': 41 }, 3: { '2013-07-23': 42, '2013-07-24': 41 } }

	for rows.Next() {
		var ward, count int
		var date time.Time

		if err := rows.Scan(&count, &ward, &date); err != nil {
			return backend_error(err)
		}

		if _, present := data[ward]; !present {
			data[ward] = make(map[string]int)
		}

		data[ward][date.Format("2006-01-02")] = count
	}

	type WardCount struct {
		Ward    int
		Counts  []int
		Average float32
	}

	counts := make(map[int]WardCount)
	var day_data []string

	// generate a list of days returned in the results
	for day := 0; day < days; day++ {
		day_data = append(day_data, start.AddDate(0, 0, day).Format("2006-01-02"))
	}

	// for each ward, and each day, find the count and populate result
	for i := 1; i < 51; i++ {
		for day := 0; day < days; day++ {
			d := start.AddDate(0, 0, day)
			c := 0
			if total_for_day, present := data[i][d.Format("2006-01-02")]; present {
				c = total_for_day
			}

			tmp := counts[i]
			tmp.Counts = append(counts[i].Counts, c)
			counts[i] = tmp
		}
	}

	rows, err = api.Db.Query(`SELECT SUM(total)/365.0, ward
             FROM daily_counts
             WHERE requested_date >= DATE(NOW() - INTERVAL '1 year')
                     AND service_code = $1
             GROUP BY ward;`, service_code)

	if err != nil {
		return backend_error(err)
	}

	for rows.Next() {
		var count float32
		var ward int
		if err := rows.Scan(&count, &ward); err != nil {
			return backend_error(err)
		}

		tmp := counts[ward]
		tmp.Average = count
		counts[ward] = tmp
	}

	type CityCount struct {
		Average  float32
		DailyMax []int
		Count    int
	}

	// find total opened for the entire city for date range
	var city_total CityCount
	err = api.Db.QueryRow(`SELECT SUM(total)
                     FROM daily_counts
                     WHERE service_code = $1
                             AND requested_date >= $2
                             AND requested_date < $3;`,
		string(service_code), start, end).Scan(&city_total.Count)

	if err != nil {
		return backend_error(err)
	}

	city_total.Average = float32(city_total.Count) / 365.0

	// find the seven largest days of all time
	rows, err = api.Db.Query(`SELECT total
                     FROM daily_counts
                     WHERE service_code = $1
                     ORDER BY total DESC
                     LIMIT 7;`,
		string(service_code))

	for rows.Next() {
		var daily_max int
		if err := rows.Scan(&daily_max); err != nil {
			return backend_error(err)
		}

		city_total.DailyMax = append(city_total.DailyMax, daily_max)
	}

	// pluck data to return, ensure we return a number, even zero, for each ward
	type WC struct {
		Counts  []int
		Average float32
	}

	complete_wards := make(map[string]WC)
	for i := 1; i < 51; i++ {
		k := strconv.Itoa(i)
		tmp := complete_wards[k]
		tmp.Counts = counts[i].Counts
		tmp.Average = counts[i].Average
		complete_wards[k] = tmp
	}

	type RespData struct {
		DayData  []string
		CityData CityCount
		WardData map[string]WC
	}

	return dumpJson(RespData{CityData: city_total, WardData: complete_wards, DayData: day_data}), nil
}
<|endoftext|>


<commit_before>package operators

import (
	"fmt"
	"strings"

	. "github.com/onsi/ginkgo"

	exutil "github.com/openshift/origin/test/extended/util"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
	e2e "k8s.io/kubernetes/test/e2e/framework"
)

var _ = Describe("[Feature:Platform][Smoke] Managed cluster should", func() {
	oc := exutil.NewCLIWithoutNamespace("operators")

	It("ensure control plane operators do not make themselves unevictable", func() {
		// iterate over the references to find valid images
		pods, err := oc.KubeFramework().ClientSet.CoreV1().Pods("").List(metav1.ListOptions{})
		if err != nil {
			e2e.Failf("unable to list pods: %v", err)
		}

		// list of pods that use images not in the release payload
		invalidPodTolerations := sets.NewString()
		// a pod in a namespace that begins with kube-* or openshift-* must come from our release payload
		// TODO components in openshift-operators may not come from our payload, may want to weaken restriction
		namespacePrefixes := sets.NewString("kube-", "openshift-")
		excludedNamespaces := sets.NewString("openshift-kube-apiserver", "openshift-kube-controller-manager", "openshift-kube-scheduler", "openshift-etcd")
		// exclude these pods from checks
		whitelistPods := sets.NewString("network-operator", "dns-operator", "olm-operators", "gcp-routes-controller")
		for _, pod := range pods.Items {
			// exclude non-control plane namespaces
			if !hasPrefixSet(pod.Namespace, namespacePrefixes) {
				continue
			}
			// exclude static pod managed namespaces
			if excludedNamespaces.Has(pod.Namespace) {
				continue
			}
			if hasPrefixSet(pod.Name, whitelistPods) {
				continue
			}
			// exclude pods started by DaemonSets
			if ownedByDaemonSet(pod) {
				continue
			}
			for _, toleration := range pod.Spec.Tolerations {
				if toleration.Operator == v1.TolerationOpExists {
					if toleration.Key == "" {
						invalidPodTolerations.Insert(fmt.Sprintf("%s/%s tolerates all taints", pod.Namespace, pod.Name))
					}
					if toleration.Key == schedulerapi.TaintNodeUnreachable || toleration.Key == schedulerapi.TaintNodeNotReady {
						if toleration.TolerationSeconds == nil {
							invalidPodTolerations.Insert(fmt.Sprintf("%s/%s tolerates %s with no tolerationSeconds", pod.Namespace, pod.Name, toleration.Key))
						}
						/* TODO enable this once we can get tolerationSeconds explicity defined in every component */
						/*if *toleration.TolerationSeconds == 300 {
							invalidPodTolerations.Insert(fmt.Sprintf("%s/%s tolerates %s with default tolerationSeconds", pod.Namespace, pod.Name, toleration.Key))
						}*/
					}
				}
			}
		}
		// log for debugging output before we ultimately fail
		//e2e.Logf("Pods found with invalid tolerations: %s", strings.Join(invalidPodTolerations.List(), "\n"))
		numInvalidPodTolerations := len(invalidPodTolerations)
		if numInvalidPodTolerations > 0 {
			e2e.Failf("\n%d pods found with invalid tolerations:\n%s", numInvalidPodTolerations, strings.Join(invalidPodTolerations.List(), "\n"))
		}
	})
})

func hasPrefixSet(name string, set sets.String) bool {
	for _, prefix := range set.List() {
		if strings.HasPrefix(name, prefix) {
			return true
		}
	}
	return false
}

func ownedByDaemonSet(pod v1.Pod) bool {
	for _, ownerRef := range pod.OwnerReferences {
		if ownerRef.Kind == "DaemonSet" {
			return true
		}
	}
	return false
}
<commit_msg>Only check for tolerationSeconds on NoExecute tolerations<commit_after>package operators

import (
	"fmt"
	"strings"

	. "github.com/onsi/ginkgo"

	exutil "github.com/openshift/origin/test/extended/util"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	schedulerapi "k8s.io/kubernetes/pkg/scheduler/api"
	e2e "k8s.io/kubernetes/test/e2e/framework"
)

var _ = Describe("[Feature:Platform][Smoke] Managed cluster should", func() {
	oc := exutil.NewCLIWithoutNamespace("operators")

	It("ensure control plane operators do not make themselves unevictable", func() {
		// iterate over the references to find valid images
		pods, err := oc.KubeFramework().ClientSet.CoreV1().Pods("").List(metav1.ListOptions{})
		if err != nil {
			e2e.Failf("unable to list pods: %v", err)
		}

		// list of pods that use images not in the release payload
		invalidPodTolerations := sets.NewString()
		// a pod in a namespace that begins with kube-* or openshift-* must come from our release payload
		// TODO components in openshift-operators may not come from our payload, may want to weaken restriction
		namespacePrefixes := sets.NewString("kube-", "openshift-")
		excludedNamespaces := sets.NewString("openshift-kube-apiserver", "openshift-kube-controller-manager", "openshift-kube-scheduler", "openshift-etcd")
		// exclude these pods from checks
		whitelistPods := sets.NewString("network-operator", "dns-operator", "olm-operators", "gcp-routes-controller")
		for _, pod := range pods.Items {
			// exclude non-control plane namespaces
			if !hasPrefixSet(pod.Namespace, namespacePrefixes) {
				continue
			}
			// exclude static pod managed namespaces
			if excludedNamespaces.Has(pod.Namespace) {
				continue
			}
			if hasPrefixSet(pod.Name, whitelistPods) {
				continue
			}
			// exclude pods started by DaemonSets
			if ownedByDaemonSet(pod) {
				continue
			}
			for _, toleration := range pod.Spec.Tolerations {
				if toleration.Operator == v1.TolerationOpExists && toleration.Effect == v1.TaintEffectNoExecute {
					if toleration.Key == "" {
						invalidPodTolerations.Insert(fmt.Sprintf("%s/%s tolerates all taints", pod.Namespace, pod.Name))
					}
					if toleration.Key == schedulerapi.TaintNodeUnreachable || toleration.Key == schedulerapi.TaintNodeNotReady {
						if toleration.TolerationSeconds == nil {
							invalidPodTolerations.Insert(fmt.Sprintf("%s/%s tolerates %s with no tolerationSeconds", pod.Namespace, pod.Name, toleration.Key))
						}
						/* TODO enable this once we can get tolerationSeconds explicity defined in every component */
						/*if *toleration.TolerationSeconds == 300 {
							invalidPodTolerations.Insert(fmt.Sprintf("%s/%s tolerates %s with default tolerationSeconds", pod.Namespace, pod.Name, toleration.Key))
						}*/
					}
				}
			}
		}
		// log for debugging output before we ultimately fail
		//e2e.Logf("Pods found with invalid tolerations: %s", strings.Join(invalidPodTolerations.List(), "\n"))
		numInvalidPodTolerations := len(invalidPodTolerations)
		if numInvalidPodTolerations > 0 {
			e2e.Failf("\n%d pods found with invalid tolerations:\n%s", numInvalidPodTolerations, strings.Join(invalidPodTolerations.List(), "\n"))
		}
	})
})

func hasPrefixSet(name string, set sets.String) bool {
	for _, prefix := range set.List() {
		if strings.HasPrefix(name, prefix) {
			return true
		}
	}
	return false
}

func ownedByDaemonSet(pod v1.Pod) bool {
	for _, ownerRef := range pod.OwnerReferences {
		if ownerRef.Kind == "DaemonSet" {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"strconv"
	"time"

	"github.com/gkarlik/expense-tracker/expense-service/v1/model"
	"github.com/gkarlik/expense-tracker/expense-service/v1/proxy"
	"github.com/gkarlik/expense-tracker/shared/errors"
	"github.com/gkarlik/quark-go"
	"github.com/gkarlik/quark-go/data/access/rdbms"
	"github.com/gkarlik/quark-go/data/access/rdbms/gorm"
	"github.com/gkarlik/quark-go/logger"
	"github.com/gkarlik/quark-go/metrics/noop"
	sd "github.com/gkarlik/quark-go/service/discovery"
	"github.com/gkarlik/quark-go/service/discovery/plain"
	gRPC "github.com/gkarlik/quark-go/service/rpc/grpc"
	nt "github.com/gkarlik/quark-go/service/trace/noop"
	_ "github.com/jinzhu/gorm/dialects/postgres"
	uuid "github.com/satori/go.uuid"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

type ExpenseService struct {
	*quark.ServiceBase
}

func CreateExpenseService() *ExpenseService {
	name := quark.GetEnvVar("EXPENSE_SERVICE_NAME")
	version := quark.GetEnvVar("EXPENSE_SERVICE_VERSION")
	p := quark.GetEnvVar("EXPENSE_SERVICE_PORT")
	discovery := quark.GetEnvVar("DISCOVERY")

	port, err := strconv.Atoi(p)
	if err != nil {
		panic("Incorrect port value!")
	}

	addr, err := quark.GetHostAddress(port)
	if err != nil {
		panic("Cannot resolve host address!")
	}

	return &ExpenseService{
		ServiceBase: quark.NewService(
			quark.Name(name),
			quark.Version(version),
			quark.Address(addr),
			quark.Discovery(plain.NewServiceDiscovery(discovery)),
			quark.Metrics(noop.NewMetricsReporter()),
			quark.Tracer(nt.NewTracer())),
	}
}

var (
	dialect   = quark.GetEnvVar("DATABASE_DIALECT")
	dbConnStr = quark.GetEnvVar("DATABASE")
	service   = CreateExpenseService()
)

func NewDbContext() (rdbms.DbContext, error) {
	context, err := gorm.NewDbContext(dialect, dbConnStr)
	if err != nil {
		return nil, err
	}

	// set database table name format
	context.DB.SingularTable(true)

	return context, nil
}

func UpgradeDatabase(s quark.Service) error {
	if quark.GetEnvVar("UPGRADE_DATABASE") != "1" {
		s.Log().Info("Database upgrade is disabled")

		return nil
	}

	s.Log().Info("Upgrading database ...")

	context, err := NewDbContext()
	if err != nil {
		s.Log().ErrorWithFields(logger.Fields{"error": err}, "Cannot create database context")
		return err
	}

	context.(*gorm.DbContext).DB.DropTable(&model.Category{})
	context.(*gorm.DbContext).DB.DropTable(&model.Expense{})
	context.(*gorm.DbContext).DB.AutoMigrate(&model.Category{})
	context.(*gorm.DbContext).DB.AutoMigrate(&model.Expense{})

	s.Log().Info("Database upgrade completed")

	return nil
}

func (es *ExpenseService) GetExpense(ctx context.Context, in *proxy.ExpenseIDRequest) (*proxy.ExpenseResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_get_expense", ctx)
	defer span.Finish()

	if in.ID == "" {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewExpenseRepository(context)
	expense, err := repo.FindByID(in.ID)
	if err != nil {
		return nil, err
	}

	return &proxy.ExpenseResponse{
		ID:         expense.ID,
		CategoryID: expense.CategoryID,
		Value:      expense.Value,
		Date:       expense.Date.Unix(),
	}, nil
}

func (es *ExpenseService) CreateExpense(ctx context.Context, in *proxy.CreateExpenseRequest) (*proxy.ExpenseResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_create_expense", ctx)
	defer span.Finish()

	if (proxy.CreateExpenseRequest{}) == *in {
		return nil, errors.ErrInvalidExpenseModel
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	expense := &model.Expense{
		ID:         uuid.NewV4().String(),
		Date:       time.Unix(in.Date, 0),
		Value:      in.Value,
		UserID:     in.UserID,
		CategoryID: in.CategoryID,
	}

	if ok := expense.IsValid(); !ok {
		return nil, errors.ErrInvalidExpenseModel
	}

	repo := model.NewExpenseRepository(context)
	if err := repo.Save(expense); err != nil {
		return nil, err
	}

	return &proxy.ExpenseResponse{
		ID:         expense.ID,
		Date:       expense.Date.Unix(),
		Value:      expense.Value,
		CategoryID: expense.CategoryID,
	}, nil
}

func (es *ExpenseService) UpdateExpense(ctx context.Context, in *proxy.UpdateExpenseRequest) (*proxy.ExpenseResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_update_expense", ctx)
	defer span.Finish()

	if (proxy.UpdateExpenseRequest{}) == *in {
		return nil, errors.ErrInvalidExpenseModel
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	expense := &model.Expense{
		ID:         in.ID,
		Date:       time.Unix(in.Date, 0),
		Value:      in.Value,
		UserID:     in.UserID,
		CategoryID: in.CategoryID,
	}

	if ok := expense.IsValid(); !ok {
		return nil, errors.ErrInvalidExpenseModel
	}

	repo := model.NewExpenseRepository(context)
	if err := repo.Save(expense); err != nil {
		return nil, err
	}

	return &proxy.ExpenseResponse{
		ID:         expense.ID,
		Date:       expense.Date.Unix(),
		Value:      expense.Value,
		CategoryID: expense.CategoryID,
	}, nil
}

func (es *ExpenseService) RemoveExpense(ctx context.Context, in *proxy.ExpenseIDRequest) (*proxy.EmptyResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_remove_expense", ctx)
	defer span.Finish()

	if in.ID == "" {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewExpenseRepository(context)
	expense, err := repo.FindByID(in.ID)
	if err != nil {
		return nil, err
	}
	if err := repo.Delete(expense); err != nil {
		return nil, err
	}
	return &proxy.EmptyResponse{}, nil
}

func (es *ExpenseService) GetUserExpenses(ctx context.Context, in *proxy.UserPagingRequest) (*proxy.ExpensesResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_get_user_expenses", ctx)
	defer span.Finish()

	if in.Offset < 0 {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewExpenseRepository(context)
	expenses, err := repo.FindByUserID(in.UserID, in.Offset, in.Limit)
	if err != nil {
		return nil, err
	}

	var exps []*proxy.ExpenseResponse
	for i := 0; i < len(expenses); i++ {
		exp := &proxy.ExpenseResponse{
			ID:         expenses[i].ID,
			Date:       expenses[i].Date.Unix(),
			Value:      expenses[i].Value,
			CategoryID: expenses[i].CategoryID,
		}
		exps = append(exps, exp)
	}

	return &proxy.ExpensesResponse{
		Expenses: exps,
	}, nil
}

func (es *ExpenseService) GetCategory(ctx context.Context, in *proxy.CategoryIDRequest) (*proxy.CategoryResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_get_category", ctx)
	defer span.Finish()

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewCategoryRepository(context)
	category, err := repo.FindByID(in.ID)
	if err != nil {
		return nil, err
	}
	return &proxy.CategoryResponse{
		ID:    category.ID,
		Limit: category.Limit,
		Name:  category.Name,
	}, nil
}

func (es *ExpenseService) CreateCategory(ctx context.Context, in *proxy.CreateCategoryRequest) (*proxy.CategoryResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_create_category", ctx)
	defer span.Finish()

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	category := &model.Category{
		ID:     uuid.NewV4().String(),
		Limit:  in.Limit,
		Name:   in.Name,
		UserID: in.UserID,
	}

	if ok := category.IsValid(); !ok {
		return nil, errors.ErrInvalidCategoryModel
	}

	repo := model.NewExpenseRepository(context)
	if err := repo.Save(category); err != nil {
		return nil, err
	}

	return &proxy.CategoryResponse{
		ID:    category.ID,
		Limit: category.Limit,
		Name:  category.Name,
	}, nil
}

func (es *ExpenseService) UpdateCategory(ctx context.Context, in *proxy.UpdateCategoryRequest) (*proxy.CategoryResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_update_category", ctx)
	defer span.Finish()

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	category := &model.Category{
		ID:     in.ID,
		Limit:  in.Limit,
		Name:   in.Name,
		UserID: in.UserID,
	}

	if ok := category.IsValid(); !ok {
		return nil, errors.ErrInvalidCategoryModel
	}

	repo := model.NewExpenseRepository(context)
	if err := repo.Save(category); err != nil {
		return nil, err
	}

	return &proxy.CategoryResponse{
		ID:    category.ID,
		Limit: category.Limit,
		Name:  category.Name,
	}, nil
}

func (es *ExpenseService) RemoveCategory(ctx context.Context, in *proxy.CategoryIDRequest) (*proxy.EmptyResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_remove_category", ctx)
	defer span.Finish()

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewCategoryRepository(context)
	category, err := repo.FindByID(in.ID)
	if err != nil {
		return nil, errors.ErrCategoryNotFound
	}
	if err := repo.Delete(category); err != nil {
		return nil, err
	}
	return &proxy.EmptyResponse{}, nil
}

func (es *ExpenseService) GetUserCategories(ctx context.Context, in *proxy.UserPagingRequest) (*proxy.CategoriesResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_get_user_categories", ctx)
	defer span.Finish()

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewCategoryRepository(context)
	categories, err := repo.FindByUserID(in.UserID, in.Offset, in.Limit)
	if err != nil {
		return nil, err
	}

	var cs []*proxy.CategoryResponse
	for i := 0; i < len(categories); i++ {
		c := &proxy.CategoryResponse{
			ID:    categories[i].ID,
			Name:  categories[i].Name,
			Limit: categories[i].Limit,
		}
		cs = append(cs, c)
	}

	return &proxy.CategoriesResponse{
		Categories: cs,
	}, nil
}

func (es *ExpenseService) RegisterServiceInstance(server interface{}, serviceInstance interface{}) error {
	proxy.RegisterExpenseServiceServer(server.(*grpc.Server), serviceInstance.(proxy.ExpenseServiceServer))

	return nil
}

func main() {
	if err := UpgradeDatabase(service); err != nil {
		panic("Cannot upgrade database!")
	}

	if err := service.Discovery().RegisterService(sd.WithInfo(service.Info())); err != nil {
		service.Log().ErrorWithFields(logger.Fields{
			"err": err,
		}, "Cannot register service")

		panic("Cannot register service!")
	}

	done := quark.HandleInterrupt(service)
	server := gRPC.NewServer()
	defer func() {
		server.Dispose()
		service.Dispose()
	}()

	go func() {
		server.Start(service)
	}()

	<-done
}
<commit_msg>Category API - request params validation.<commit_after>package main

import (
	"strconv"
	"time"

	"github.com/gkarlik/expense-tracker/expense-service/v1/model"
	"github.com/gkarlik/expense-tracker/expense-service/v1/proxy"
	"github.com/gkarlik/expense-tracker/shared/errors"
	"github.com/gkarlik/quark-go"
	"github.com/gkarlik/quark-go/data/access/rdbms"
	"github.com/gkarlik/quark-go/data/access/rdbms/gorm"
	"github.com/gkarlik/quark-go/logger"
	"github.com/gkarlik/quark-go/metrics/noop"
	sd "github.com/gkarlik/quark-go/service/discovery"
	"github.com/gkarlik/quark-go/service/discovery/plain"
	gRPC "github.com/gkarlik/quark-go/service/rpc/grpc"
	nt "github.com/gkarlik/quark-go/service/trace/noop"
	_ "github.com/jinzhu/gorm/dialects/postgres"
	uuid "github.com/satori/go.uuid"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

type ExpenseService struct {
	*quark.ServiceBase
}

func CreateExpenseService() *ExpenseService {
	name := quark.GetEnvVar("EXPENSE_SERVICE_NAME")
	version := quark.GetEnvVar("EXPENSE_SERVICE_VERSION")
	p := quark.GetEnvVar("EXPENSE_SERVICE_PORT")
	discovery := quark.GetEnvVar("DISCOVERY")

	port, err := strconv.Atoi(p)
	if err != nil {
		panic("Incorrect port value!")
	}

	addr, err := quark.GetHostAddress(port)
	if err != nil {
		panic("Cannot resolve host address!")
	}

	return &ExpenseService{
		ServiceBase: quark.NewService(
			quark.Name(name),
			quark.Version(version),
			quark.Address(addr),
			quark.Discovery(plain.NewServiceDiscovery(discovery)),
			quark.Metrics(noop.NewMetricsReporter()),
			quark.Tracer(nt.NewTracer())),
	}
}

var (
	dialect   = quark.GetEnvVar("DATABASE_DIALECT")
	dbConnStr = quark.GetEnvVar("DATABASE")
	service   = CreateExpenseService()
)

func NewDbContext() (rdbms.DbContext, error) {
	context, err := gorm.NewDbContext(dialect, dbConnStr)
	if err != nil {
		return nil, err
	}

	// set database table name format
	context.DB.SingularTable(true)

	return context, nil
}

func UpgradeDatabase(s quark.Service) error {
	if quark.GetEnvVar("UPGRADE_DATABASE") != "1" {
		s.Log().Info("Database upgrade is disabled")

		return nil
	}

	s.Log().Info("Upgrading database ...")

	context, err := NewDbContext()
	if err != nil {
		s.Log().ErrorWithFields(logger.Fields{"error": err}, "Cannot create database context")
		return err
	}

	context.(*gorm.DbContext).DB.DropTable(&model.Category{})
	context.(*gorm.DbContext).DB.DropTable(&model.Expense{})
	context.(*gorm.DbContext).DB.AutoMigrate(&model.Category{})
	context.(*gorm.DbContext).DB.AutoMigrate(&model.Expense{})

	s.Log().Info("Database upgrade completed")

	return nil
}

func (es *ExpenseService) GetExpense(ctx context.Context, in *proxy.ExpenseIDRequest) (*proxy.ExpenseResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_get_expense", ctx)
	defer span.Finish()

	if in.ID == "" {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewExpenseRepository(context)
	expense, err := repo.FindByID(in.ID)
	if err != nil {
		return nil, err
	}

	return &proxy.ExpenseResponse{
		ID:         expense.ID,
		CategoryID: expense.CategoryID,
		Value:      expense.Value,
		Date:       expense.Date.Unix(),
	}, nil
}

func (es *ExpenseService) CreateExpense(ctx context.Context, in *proxy.CreateExpenseRequest) (*proxy.ExpenseResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_create_expense", ctx)
	defer span.Finish()

	if (proxy.CreateExpenseRequest{}) == *in {
		return nil, errors.ErrInvalidExpenseModel
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	expense := &model.Expense{
		ID:         uuid.NewV4().String(),
		Date:       time.Unix(in.Date, 0),
		Value:      in.Value,
		UserID:     in.UserID,
		CategoryID: in.CategoryID,
	}

	if ok := expense.IsValid(); !ok {
		return nil, errors.ErrInvalidExpenseModel
	}

	repo := model.NewExpenseRepository(context)
	if err := repo.Save(expense); err != nil {
		return nil, err
	}

	return &proxy.ExpenseResponse{
		ID:         expense.ID,
		Date:       expense.Date.Unix(),
		Value:      expense.Value,
		CategoryID: expense.CategoryID,
	}, nil
}

func (es *ExpenseService) UpdateExpense(ctx context.Context, in *proxy.UpdateExpenseRequest) (*proxy.ExpenseResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_update_expense", ctx)
	defer span.Finish()

	if (proxy.UpdateExpenseRequest{}) == *in {
		return nil, errors.ErrInvalidExpenseModel
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	expense := &model.Expense{
		ID:         in.ID,
		Date:       time.Unix(in.Date, 0),
		Value:      in.Value,
		UserID:     in.UserID,
		CategoryID: in.CategoryID,
	}

	if ok := expense.IsValid(); !ok {
		return nil, errors.ErrInvalidExpenseModel
	}

	repo := model.NewExpenseRepository(context)
	if err := repo.Save(expense); err != nil {
		return nil, err
	}

	return &proxy.ExpenseResponse{
		ID:         expense.ID,
		Date:       expense.Date.Unix(),
		Value:      expense.Value,
		CategoryID: expense.CategoryID,
	}, nil
}

func (es *ExpenseService) RemoveExpense(ctx context.Context, in *proxy.ExpenseIDRequest) (*proxy.EmptyResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_remove_expense", ctx)
	defer span.Finish()

	if in.ID == "" {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewExpenseRepository(context)
	expense, err := repo.FindByID(in.ID)
	if err != nil {
		return nil, err
	}
	if err := repo.Delete(expense); err != nil {
		return nil, err
	}
	return &proxy.EmptyResponse{}, nil
}

func (es *ExpenseService) GetUserExpenses(ctx context.Context, in *proxy.UserPagingRequest) (*proxy.ExpensesResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_get_user_expenses", ctx)
	defer span.Finish()

	if in.Offset < 0 {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewExpenseRepository(context)
	expenses, err := repo.FindByUserID(in.UserID, in.Offset, in.Limit)
	if err != nil {
		return nil, err
	}

	var exps []*proxy.ExpenseResponse
	for i := 0; i < len(expenses); i++ {
		exp := &proxy.ExpenseResponse{
			ID:         expenses[i].ID,
			Date:       expenses[i].Date.Unix(),
			Value:      expenses[i].Value,
			CategoryID: expenses[i].CategoryID,
		}
		exps = append(exps, exp)
	}

	return &proxy.ExpensesResponse{
		Expenses: exps,
	}, nil
}

func (es *ExpenseService) GetCategory(ctx context.Context, in *proxy.CategoryIDRequest) (*proxy.CategoryResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_get_category", ctx)
	defer span.Finish()

	if in.ID == "" {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewCategoryRepository(context)
	category, err := repo.FindByID(in.ID)
	if err != nil {
		return nil, err
	}
	return &proxy.CategoryResponse{
		ID:    category.ID,
		Limit: category.Limit,
		Name:  category.Name,
	}, nil
}

func (es *ExpenseService) CreateCategory(ctx context.Context, in *proxy.CreateCategoryRequest) (*proxy.CategoryResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_create_category", ctx)
	defer span.Finish()

	if (proxy.CreateCategoryRequest{}) == *in {
		return nil, errors.ErrInvalidCategoryModel
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	category := &model.Category{
		ID:     uuid.NewV4().String(),
		Limit:  in.Limit,
		Name:   in.Name,
		UserID: in.UserID,
	}

	if ok := category.IsValid(); !ok {
		return nil, errors.ErrInvalidCategoryModel
	}

	repo := model.NewExpenseRepository(context)
	if err := repo.Save(category); err != nil {
		return nil, err
	}

	return &proxy.CategoryResponse{
		ID:    category.ID,
		Limit: category.Limit,
		Name:  category.Name,
	}, nil
}

func (es *ExpenseService) UpdateCategory(ctx context.Context, in *proxy.UpdateCategoryRequest) (*proxy.CategoryResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_update_category", ctx)
	defer span.Finish()

	if (proxy.UpdateCategoryRequest{}) == *in {
		return nil, errors.ErrInvalidCategoryModel
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	category := &model.Category{
		ID:     in.ID,
		Limit:  in.Limit,
		Name:   in.Name,
		UserID: in.UserID,
	}

	if ok := category.IsValid(); !ok {
		return nil, errors.ErrInvalidCategoryModel
	}

	repo := model.NewExpenseRepository(context)
	if err := repo.Save(category); err != nil {
		return nil, err
	}

	return &proxy.CategoryResponse{
		ID:    category.ID,
		Limit: category.Limit,
		Name:  category.Name,
	}, nil
}

func (es *ExpenseService) RemoveCategory(ctx context.Context, in *proxy.CategoryIDRequest) (*proxy.EmptyResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_remove_category", ctx)
	defer span.Finish()

	if in.ID == "" {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewCategoryRepository(context)
	category, err := repo.FindByID(in.ID)
	if err != nil {
		return nil, errors.ErrCategoryNotFound
	}
	if err := repo.Delete(category); err != nil {
		return nil, err
	}
	return &proxy.EmptyResponse{}, nil
}

func (es *ExpenseService) GetUserCategories(ctx context.Context, in *proxy.UserPagingRequest) (*proxy.CategoriesResponse, error) {
	span := quark.StartRPCSpan(service, "expense_service_get_user_categories", ctx)
	defer span.Finish()

	if in.Offset < 0 {
		return nil, errors.ErrInvalidRequestParameters
	}

	context, err := NewDbContext()
	if err != nil {
		return nil, err
	}
	defer context.Dispose()

	repo := model.NewCategoryRepository(context)
	categories, err := repo.FindByUserID(in.UserID, in.Offset, in.Limit)
	if err != nil {
		return nil, err
	}

	var cs []*proxy.CategoryResponse
	for i := 0; i < len(categories); i++ {
		c := &proxy.CategoryResponse{
			ID:    categories[i].ID,
			Name:  categories[i].Name,
			Limit: categories[i].Limit,
		}
		cs = append(cs, c)
	}

	return &proxy.CategoriesResponse{
		Categories: cs,
	}, nil
}

func (es *ExpenseService) RegisterServiceInstance(server interface{}, serviceInstance interface{}) error {
	proxy.RegisterExpenseServiceServer(server.(*grpc.Server), serviceInstance.(proxy.ExpenseServiceServer))

	return nil
}

func main() {
	if err := UpgradeDatabase(service); err != nil {
		panic("Cannot upgrade database!")
	}

	if err := service.Discovery().RegisterService(sd.WithInfo(service.Info())); err != nil {
		service.Log().ErrorWithFields(logger.Fields{
			"err": err,
		}, "Cannot register service")

		panic("Cannot register service!")
	}

	done := quark.HandleInterrupt(service)
	server := gRPC.NewServer()
	defer func() {
		server.Dispose()
		service.Dispose()
	}()

	go func() {
		server.Start(service)
	}()

	<-done
}
<|endoftext|>


<commit_before>package instagram

import (
	"strings"
	"time"

	"sync"

	"strconv"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/metrics"
	"github.com/Seklfreak/Robyul2/models"
	goinstaResponse "github.com/ahmdrz/goinsta/response"
	"github.com/globalsign/mgo/bson"
)

var (
	instagramEntryLocks = make(map[string]*sync.Mutex)
)

const (
	InstagramGraphQlWorkers = 15
)

func (m *Handler) checkInstagramGraphQlFeedLoop() {
	log := cache.GetLogger().WithField("module", "instagram")

	defer helpers.Recover()
	defer func() {
		go func() {
			log.Error("The checkInstagramGraphQlFeedLoop died." +
				"Please investigate! Will be restarted in 60 seconds")
			time.Sleep(60 * time.Second)
			m.checkInstagramGraphQlFeedLoop()
		}()
	}()

	for {
		bundledEntries, entriesCount, err := m.getBundledEntries()
		helpers.Relax(err)

		cache.GetLogger().Infof(
			"checking graphql feed on %d accounts for %d feeds with %d workers",
			len(bundledEntries), entriesCount, InstagramGraphQlWorkers)
		start := time.Now()

		jobs := make(chan map[string][]models.InstagramEntry, 0)
		results := make(chan int, 0)

		workerEntries := make(map[int]map[string][]models.InstagramEntry, 0)
		for w := 1; w <= InstagramGraphQlWorkers; w++ {
			go m.checkInstagramGraphQlFeedWorker(w, jobs, results)
			workerEntries[w] = make(map[string][]models.InstagramEntry)
		}

		lastWorker := 1
		for code, codeEntries := range bundledEntries {
			workerEntries[lastWorker][code] = codeEntries
			lastWorker++
			if lastWorker > InstagramGraphQlWorkers {
				lastWorker = 1
			}
		}

		for _, workerEntry := range workerEntries {
			jobs <- workerEntry
		}
		close(jobs)

		for a := 1; a <= InstagramGraphQlWorkers; a++ {
			<-results
		}
		elapsed := time.Since(start)
		cache.GetLogger().Infof(
			"checked graphql feed on %d accounts for %d feeds with %d workers, took %s",
			len(bundledEntries), entriesCount, InstagramGraphQlWorkers, elapsed)
		metrics.InstagramGraphQlFeedRefreshTime.Set(elapsed.Seconds())

		if entriesCount <= 10 {
			time.Sleep(60 * time.Second)
		}
	}
}

func (m *Handler) checkInstagramGraphQlFeedWorker(id int, jobs <-chan map[string][]models.InstagramEntry, results chan<- int) {
	defer helpers.Recover()

	currentProxy, err := helpers.GetRandomProxy()
	helpers.Relax(err)
	cache.GetLogger().WithField("module", "instagram").Infof("switched to random proxy")

	for job := range jobs {
		//cache.GetLogger().WithField("module", "instagram").WithField("worker", id).Infof(
		//	"worker %d started for %d accounts", id, len(job))
	NextEntry:
		for instagramAccountID, entries := range job {
			//cache.GetLogger().WithField("module", "instagram").WithField("worker", id).Infof(
			//	"checking graphql feed for %d for %d channels", instagramAccountID, len(entries))
		RetryGraphQl:
			receivedPosts, err := m.getPosts(instagramAccountID, currentProxy)
			if err != nil {
				if m.retryOnError(err) {
					cache.GetLogger().WithField("module", "instagram").Infof(
						"proxy error connecting to Instagram Account %d (GraphQL), "+
							"switching proxy and then trying again", instagramAccountID)
					currentProxy, err = helpers.GetRandomProxy()
					helpers.Relax(err)
					goto RetryGraphQl
				}
				helpers.RelaxLog(err)
				continue NextEntry
			}

			for _, receivedPost := range receivedPosts {
				postHasBeenPostedEverywhere := true
				for _, entry := range entries {
					postAlreadyPosted := false
					for _, postedPosts := range entry.PostedPosts {
						if postedPosts.Type == models.InstagramPostTypePost && postedPosts.ID == receivedPost.ID {
							postAlreadyPosted = true
						}
					}
					if !postAlreadyPosted {
						postHasBeenPostedEverywhere = false
					}
				}

				if postHasBeenPostedEverywhere {
					continue
				}

				// download specific post data
			RetryPost:
				post, err := m.getPostInformation(receivedPost.Shortcode, currentProxy)
				if err != nil {
					if m.retryOnError(err) {
						cache.GetLogger().WithField("module", "instagram").Infof(
							"hit rate limit checking Instagram Account %d (GraphQL), "+
								"switching proxy and then trying again", instagramAccountID)
						currentProxy, err = helpers.GetRandomProxy()
						helpers.Relax(err)
						goto RetryPost
					}
					helpers.RelaxLog(err)
					continue NextEntry
				}

				for _, entry := range entries {
					entryID := entry.ID
					m.lockEntry(entryID)

					var entry models.InstagramEntry
					err = helpers.MdbOne(
						helpers.MdbCollection(models.InstagramTable).Find(bson.M{"_id": entryID}),
						&entry,
					)
					if err != nil {
						m.unlockEntry(entryID)
						helpers.RelaxLog(err)
						continue
					}

					changes := false
					postAlreadyPosted := false
					for _, postedPosts := range entry.PostedPosts {
						if postedPosts.Type == models.InstagramPostTypePost && postedPosts.ID == receivedPost.ID {
							postAlreadyPosted = true
						}
					}

					if postAlreadyPosted == false {
						cache.GetLogger().WithField("module", "instagram").Infof("Posting Post (GraphQL): #%s", post.ID)
						entry.PostedPosts = append(entry.PostedPosts,
							models.InstagramPostEntry{
								ID:            post.ID,
								Type:          models.InstagramPostTypePost,
								CreatedAtTime: post.TakentAt,
							})
						changes = true
						go m.postPostToChannel(entry.ChannelID, post, entry.SendPostType)
					}

					if changes {
						err = helpers.MDbUpdate(models.InstagramTable, entry.ID, entry)
						if err != nil {
							m.unlockEntry(entryID)
							helpers.RelaxLog(err)
							continue
						}
					}

					m.unlockEntry(entryID)
				}
			}
		}
	}
	results <- len(jobs)
}

func (m *Handler) checkInstagramStoryLoop() {
	log := cache.GetLogger()

	defer helpers.Recover()
	defer func() {
		go func() {
			log.WithField("module", "instagram").Error("The checkInstagramStoryLoop died." +
				"Please investigate! Will be restarted in 60 seconds")
			time.Sleep(60 * time.Second)
			m.checkInstagramStoryLoop()
		}()
	}()

	for {
		bundledEntries, entriesCount, err := m.getBundledEntries()
		helpers.Relax(err)

		cache.GetLogger().WithField("module", "instagram").Infof(
			"checking story on %d accounts for %d feeds", len(bundledEntries), entriesCount)
		start := time.Now()

		for instagramAccountID, entries := range bundledEntries {
		RetryAccount:
			// log.WithField("module", "instagram").Debug(fmt.Sprintf("checking Instagram Account @%s", instagramUsername))

			var posts goinstaResponse.UserFeedResponse
			userIdInt, err := strconv.Atoi(instagramAccountID)
			helpers.Relax(err)
			story, err := instagramClient.GetUserStories(int64(userIdInt))
			if err != nil || story.Status != "ok" {
				if m.retryOnError(err) {
					cache.GetLogger().WithField("module", "instagram").Infof(
						"hit rate limit checking Instagram Account (Stories) %d, "+
							"sleeping for 20 seconds and then trying again", instagramAccountID)
					time.Sleep(20 * time.Second)
					goto RetryAccount
				}
				log.WithField("module", "instagram").Warnf(
					"updating instagram account %d (Story) failed: %s", instagramAccountID, err)
				continue
			}

			// https://github.com/golang/go/wiki/SliceTricks#reversing
			for i := len(posts.Items)/2 - 1; i >= 0; i-- {
				opp := len(posts.Items) - 1 - i
				posts.Items[i], posts.Items[opp] = posts.Items[opp], posts.Items[i]
			}
			for i := len(story.Reel.Items)/2 - 1; i >= 0; i-- {
				opp := len(story.Reel.Items) - 1 - i
				story.Reel.Items[i], story.Reel.Items[opp] = story.Reel.Items[opp], story.Reel.Items[i]
			}

			for _, entry := range entries {
				changes := false

				entryID := entry.ID
				m.lockEntry(entryID)

				var entry models.InstagramEntry
				err = helpers.MdbOne(
					helpers.MdbCollection(models.InstagramTable).Find(bson.M{"_id": entryID}),
					&entry,
				)
				if err != nil {
					m.unlockEntry(entryID)
					if !strings.Contains(err.Error(), "The result does not contain any more rows") {
						helpers.RelaxLog(err)
					}
					continue
				}

				for n, reelMedia := range story.Reel.Items {
					reelMediaAlreadyPosted := false
					for _, reelMediaPostPosted := range entry.PostedPosts {
						if reelMediaPostPosted.Type == models.InstagramPostTypeReel && reelMediaPostPosted.ID == reelMedia.ID {
							reelMediaAlreadyPosted = true
						}
					}
					if reelMediaAlreadyPosted == false {
						log.WithField("module", "instagram").Infof(
							"Posting Reel Media (Story): #%s", reelMedia.ID)
						entry.PostedPosts = append(entry.PostedPosts,
							models.InstagramPostEntry{
								ID:            reelMedia.ID,
								Type:          models.InstagramPostTypeReel,
								CreatedAtTime: time.Unix(int64(reelMedia.DeviceTimestamp), 0),
							})
						changes = true
						go m.postReelMediaToChannel(entry.ChannelID, story, n, entry.SendPostType)
					}
				}

				if changes == true {
					err = helpers.MDbUpdate(models.InstagramTable, entry.ID, entry)
					if err != nil {
						m.unlockEntry(entryID)
						helpers.RelaxLog(err)
						continue
					}
				}

				m.unlockEntry(entryID)
			}
			time.Sleep(1 * time.Second)
		}

		elapsed := time.Since(start)
		cache.GetLogger().WithField("module", "instagram").Infof(
			"checked story on %d accounts for %d feeds, took %s",
			len(bundledEntries), entriesCount, elapsed)
		metrics.InstagramRefreshTime.Set(elapsed.Seconds())

		if entriesCount <= 10 {
			time.Sleep(30 * time.Second)
		}
	}
}

func (m *Handler) lockEntry(entryID bson.ObjectId) {
	if _, ok := instagramEntryLocks[string(entryID)]; ok {
		instagramEntryLocks[string(entryID)].Lock()
		return
	}
	instagramEntryLocks[string(entryID)] = new(sync.Mutex)
	instagramEntryLocks[string(entryID)].Lock()
}

func (m *Handler) unlockEntry(entryID bson.ObjectId) {
	if _, ok := instagramEntryLocks[string(entryID)]; ok {
		instagramEntryLocks[string(entryID)].Unlock()
	}
}

func (m *Handler) retryOnError(err error) (retry bool) {
	if err != nil {
		if strings.Contains(err.Error(), "expected status 200; got 429") ||
			strings.Contains(err.Error(), "request canceled while waiting for connection") ||
			strings.Contains(err.Error(), "connect: no route to host") ||
			strings.Contains(err.Error(), "Please wait a few minutes before you try again.") {
			return true
		}
	}
	return false
}
<commit_msg>[instagram] even more graphql error handling 🙅<commit_after>package instagram

import (
	"strings"
	"time"

	"sync"

	"strconv"

	"github.com/Seklfreak/Robyul2/cache"
	"github.com/Seklfreak/Robyul2/helpers"
	"github.com/Seklfreak/Robyul2/metrics"
	"github.com/Seklfreak/Robyul2/models"
	goinstaResponse "github.com/ahmdrz/goinsta/response"
	"github.com/globalsign/mgo/bson"
)

var (
	instagramEntryLocks = make(map[string]*sync.Mutex)
)

const (
	InstagramGraphQlWorkers = 15
)

func (m *Handler) checkInstagramGraphQlFeedLoop() {
	log := cache.GetLogger().WithField("module", "instagram")

	defer helpers.Recover()
	defer func() {
		go func() {
			log.Error("The checkInstagramGraphQlFeedLoop died." +
				"Please investigate! Will be restarted in 60 seconds")
			time.Sleep(60 * time.Second)
			m.checkInstagramGraphQlFeedLoop()
		}()
	}()

	for {
		bundledEntries, entriesCount, err := m.getBundledEntries()
		helpers.Relax(err)

		cache.GetLogger().Infof(
			"checking graphql feed on %d accounts for %d feeds with %d workers",
			len(bundledEntries), entriesCount, InstagramGraphQlWorkers)
		start := time.Now()

		jobs := make(chan map[string][]models.InstagramEntry, 0)
		results := make(chan int, 0)

		workerEntries := make(map[int]map[string][]models.InstagramEntry, 0)
		for w := 1; w <= InstagramGraphQlWorkers; w++ {
			go m.checkInstagramGraphQlFeedWorker(w, jobs, results)
			workerEntries[w] = make(map[string][]models.InstagramEntry)
		}

		lastWorker := 1
		for code, codeEntries := range bundledEntries {
			workerEntries[lastWorker][code] = codeEntries
			lastWorker++
			if lastWorker > InstagramGraphQlWorkers {
				lastWorker = 1
			}
		}

		for _, workerEntry := range workerEntries {
			jobs <- workerEntry
		}
		close(jobs)

		for a := 1; a <= InstagramGraphQlWorkers; a++ {
			<-results
		}
		elapsed := time.Since(start)
		cache.GetLogger().Infof(
			"checked graphql feed on %d accounts for %d feeds with %d workers, took %s",
			len(bundledEntries), entriesCount, InstagramGraphQlWorkers, elapsed)
		metrics.InstagramGraphQlFeedRefreshTime.Set(elapsed.Seconds())

		if entriesCount <= 10 {
			time.Sleep(60 * time.Second)
		}
	}
}

func (m *Handler) checkInstagramGraphQlFeedWorker(id int, jobs <-chan map[string][]models.InstagramEntry, results chan<- int) {
	defer helpers.Recover()

	currentProxy, err := helpers.GetRandomProxy()
	helpers.Relax(err)
	cache.GetLogger().WithField("module", "instagram").Infof("switched to random proxy")

	for job := range jobs {
		//cache.GetLogger().WithField("module", "instagram").WithField("worker", id).Infof(
		//	"worker %d started for %d accounts", id, len(job))
	NextEntry:
		for instagramAccountID, entries := range job {
			//cache.GetLogger().WithField("module", "instagram").WithField("worker", id).Infof(
			//	"checking graphql feed for %d for %d channels", instagramAccountID, len(entries))
		RetryGraphQl:
			receivedPosts, err := m.getPosts(instagramAccountID, currentProxy)
			if err != nil {
				if m.retryOnError(err) {
					cache.GetLogger().WithField("module", "instagram").Infof(
						"proxy error connecting to Instagram Account %d (GraphQL), "+
							"switching proxy and then trying again", instagramAccountID)
					currentProxy, err = helpers.GetRandomProxy()
					helpers.Relax(err)
					goto RetryGraphQl
				}
				helpers.RelaxLog(err)
				continue NextEntry
			}

			for _, receivedPost := range receivedPosts {
				postHasBeenPostedEverywhere := true
				for _, entry := range entries {
					postAlreadyPosted := false
					for _, postedPosts := range entry.PostedPosts {
						if postedPosts.Type == models.InstagramPostTypePost && postedPosts.ID == receivedPost.ID {
							postAlreadyPosted = true
						}
					}
					if !postAlreadyPosted {
						postHasBeenPostedEverywhere = false
					}
				}

				if postHasBeenPostedEverywhere {
					continue
				}

				// download specific post data
			RetryPost:
				post, err := m.getPostInformation(receivedPost.Shortcode, currentProxy)
				if err != nil {
					if m.retryOnError(err) {
						cache.GetLogger().WithField("module", "instagram").Infof(
							"hit rate limit checking Instagram Account %s (GraphQL), "+
								"switching proxy and then trying again", instagramAccountID)
						currentProxy, err = helpers.GetRandomProxy()
						helpers.Relax(err)
						goto RetryPost
					}
					helpers.RelaxLog(err)
					continue NextEntry
				}

				for _, entry := range entries {
					entryID := entry.ID
					m.lockEntry(entryID)

					var entry models.InstagramEntry
					err = helpers.MdbOne(
						helpers.MdbCollection(models.InstagramTable).Find(bson.M{"_id": entryID}),
						&entry,
					)
					if err != nil {
						m.unlockEntry(entryID)
						helpers.RelaxLog(err)
						continue
					}

					changes := false
					postAlreadyPosted := false
					for _, postedPosts := range entry.PostedPosts {
						if postedPosts.Type == models.InstagramPostTypePost && postedPosts.ID == receivedPost.ID {
							postAlreadyPosted = true
						}
					}

					if postAlreadyPosted == false {
						cache.GetLogger().WithField("module", "instagram").Infof("Posting Post (GraphQL): #%s", post.ID)
						entry.PostedPosts = append(entry.PostedPosts,
							models.InstagramPostEntry{
								ID:            post.ID,
								Type:          models.InstagramPostTypePost,
								CreatedAtTime: post.TakentAt,
							})
						changes = true
						go m.postPostToChannel(entry.ChannelID, post, entry.SendPostType)
					}

					if changes {
						err = helpers.MDbUpdate(models.InstagramTable, entry.ID, entry)
						if err != nil {
							m.unlockEntry(entryID)
							helpers.RelaxLog(err)
							continue
						}
					}

					m.unlockEntry(entryID)
				}
			}
		}
	}
	results <- len(jobs)
}

func (m *Handler) checkInstagramStoryLoop() {
	log := cache.GetLogger()

	defer helpers.Recover()
	defer func() {
		go func() {
			log.WithField("module", "instagram").Error("The checkInstagramStoryLoop died." +
				"Please investigate! Will be restarted in 60 seconds")
			time.Sleep(60 * time.Second)
			m.checkInstagramStoryLoop()
		}()
	}()

	for {
		bundledEntries, entriesCount, err := m.getBundledEntries()
		helpers.Relax(err)

		cache.GetLogger().WithField("module", "instagram").Infof(
			"checking story on %d accounts for %d feeds", len(bundledEntries), entriesCount)
		start := time.Now()

		for instagramAccountID, entries := range bundledEntries {
		RetryAccount:
			// log.WithField("module", "instagram").Debug(fmt.Sprintf("checking Instagram Account @%s", instagramUsername))

			var posts goinstaResponse.UserFeedResponse
			userIdInt, err := strconv.Atoi(instagramAccountID)
			helpers.Relax(err)
			story, err := instagramClient.GetUserStories(int64(userIdInt))
			if err != nil || story.Status != "ok" {
				if m.retryOnError(err) {
					cache.GetLogger().WithField("module", "instagram").Infof(
						"hit rate limit checking Instagram Account (Stories) %s, "+
							"sleeping for 20 seconds and then trying again", instagramAccountID)
					time.Sleep(20 * time.Second)
					goto RetryAccount
				}
				log.WithField("module", "instagram").Warnf(
					"updating instagram account %d (Story) failed: %s", instagramAccountID, err)
				continue
			}

			// https://github.com/golang/go/wiki/SliceTricks#reversing
			for i := len(posts.Items)/2 - 1; i >= 0; i-- {
				opp := len(posts.Items) - 1 - i
				posts.Items[i], posts.Items[opp] = posts.Items[opp], posts.Items[i]
			}
			for i := len(story.Reel.Items)/2 - 1; i >= 0; i-- {
				opp := len(story.Reel.Items) - 1 - i
				story.Reel.Items[i], story.Reel.Items[opp] = story.Reel.Items[opp], story.Reel.Items[i]
			}

			for _, entry := range entries {
				changes := false

				entryID := entry.ID
				m.lockEntry(entryID)

				var entry models.InstagramEntry
				err = helpers.MdbOne(
					helpers.MdbCollection(models.InstagramTable).Find(bson.M{"_id": entryID}),
					&entry,
				)
				if err != nil {
					m.unlockEntry(entryID)
					if !strings.Contains(err.Error(), "The result does not contain any more rows") {
						helpers.RelaxLog(err)
					}
					continue
				}

				for n, reelMedia := range story.Reel.Items {
					reelMediaAlreadyPosted := false
					for _, reelMediaPostPosted := range entry.PostedPosts {
						if reelMediaPostPosted.Type == models.InstagramPostTypeReel && reelMediaPostPosted.ID == reelMedia.ID {
							reelMediaAlreadyPosted = true
						}
					}
					if reelMediaAlreadyPosted == false {
						log.WithField("module", "instagram").Infof(
							"Posting Reel Media (Story): #%s", reelMedia.ID)
						entry.PostedPosts = append(entry.PostedPosts,
							models.InstagramPostEntry{
								ID:            reelMedia.ID,
								Type:          models.InstagramPostTypeReel,
								CreatedAtTime: time.Unix(int64(reelMedia.DeviceTimestamp), 0),
							})
						changes = true
						go m.postReelMediaToChannel(entry.ChannelID, story, n, entry.SendPostType)
					}
				}

				if changes == true {
					err = helpers.MDbUpdate(models.InstagramTable, entry.ID, entry)
					if err != nil {
						m.unlockEntry(entryID)
						helpers.RelaxLog(err)
						continue
					}
				}

				m.unlockEntry(entryID)
			}
			time.Sleep(1 * time.Second)
		}

		elapsed := time.Since(start)
		cache.GetLogger().WithField("module", "instagram").Infof(
			"checked story on %d accounts for %d feeds, took %s",
			len(bundledEntries), entriesCount, elapsed)
		metrics.InstagramRefreshTime.Set(elapsed.Seconds())

		if entriesCount <= 10 {
			time.Sleep(30 * time.Second)
		}
	}
}

func (m *Handler) lockEntry(entryID bson.ObjectId) {
	if _, ok := instagramEntryLocks[string(entryID)]; ok {
		instagramEntryLocks[string(entryID)].Lock()
		return
	}
	instagramEntryLocks[string(entryID)] = new(sync.Mutex)
	instagramEntryLocks[string(entryID)].Lock()
}

func (m *Handler) unlockEntry(entryID bson.ObjectId) {
	if _, ok := instagramEntryLocks[string(entryID)]; ok {
		instagramEntryLocks[string(entryID)].Unlock()
	}
}

func (m *Handler) retryOnError(err error) (retry bool) {
	if err != nil {
		if strings.Contains(err.Error(), "expected status 200; got 429") ||
			strings.Contains(err.Error(), "request canceled while waiting for connection") ||
			strings.Contains(err.Error(), "connect: no route to host") ||
			strings.Contains(err.Error(), "read: connection reset by peer") ||
			strings.Contains(err.Error(), "Please wait a few minutes before you try again.") {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package codecs

import (
	"testing"
)

func TestH264Payloader_Payload(t *testing.T) {
	pck := H264Payloader{}
	smallpayload := []byte{0x90, 0x90, 0x90}
	multiplepayload := []byte{0x00, 0x00, 0x01, 0x90,
		0x00, 0x00, 0x01, 0x90}

	// Positive MTU, nil payload
	res := pck.Payload(1, nil)
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}

	// Negative MTU, small payload
	res = pck.Payload(0, smallpayload)
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}

	// 0 MTU, small payload
	res = pck.Payload(0, smallpayload)
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}

	// Positive MTU, small payload
	res = pck.Payload(1, smallpayload)
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}

	// Positive MTU, small payload
	res = pck.Payload(5, smallpayload)
	if len(res) != 1 {
		t.Fatal("Generated payload shouldn't be empty")
	}
	if len(res[0]) != len(smallpayload) {
		t.Fatal("Generated payload should be the same size as original payload size")
	}

	// Multiple NALU in a single payload
	res = pck.Payload(5, multiplepayload)
	if len(res) != 2 {
		t.Fatal("2 nal units should be broken out")
	}
	for i := 0; i < 2; i++ {
		if len(res[i]) != 1 {
			t.Fatalf("Payload %d of 2 is packed incorrectly", i+1)
		}
	}

	// Nalu type 9 or 12
	res = pck.Payload(5, []byte{0x09, 0x00, 0x00})
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}
}
<commit_msg>Add H264 FU-A Payloader test<commit_after>package codecs

import (
	"reflect"
	"testing"
)

func TestH264Payloader_Payload(t *testing.T) {
	pck := H264Payloader{}
	smallpayload := []byte{0x90, 0x90, 0x90}
	multiplepayload := []byte{0x00, 0x00, 0x01, 0x90, 0x00, 0x00, 0x01, 0x90}

	largepayload := []byte{0x00, 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15}
	largePayloadPacketized := [][]byte{
		{0x1c, 0x80, 0x01, 0x02, 0x03},
		{0x1c, 0x00, 0x04, 0x05, 0x06},
		{0x1c, 0x00, 0x07, 0x08, 0x09},
		{0x1c, 0x00, 0x10, 0x11, 0x12},
		{0x1c, 0x40, 0x13, 0x14, 0x15},
	}

	// Positive MTU, nil payload
	res := pck.Payload(1, nil)
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}

	// Negative MTU, small payload
	res = pck.Payload(0, smallpayload)
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}

	// 0 MTU, small payload
	res = pck.Payload(0, smallpayload)
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}

	// Positive MTU, small payload
	res = pck.Payload(1, smallpayload)
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}

	// Positive MTU, small payload
	res = pck.Payload(5, smallpayload)
	if len(res) != 1 {
		t.Fatal("Generated payload shouldn't be empty")
	}
	if len(res[0]) != len(smallpayload) {
		t.Fatal("Generated payload should be the same size as original payload size")
	}

	// Multiple NALU in a single payload
	res = pck.Payload(5, multiplepayload)
	if len(res) != 2 {
		t.Fatal("2 nal units should be broken out")
	}
	for i := 0; i < 2; i++ {
		if len(res[i]) != 1 {
			t.Fatalf("Payload %d of 2 is packed incorrectly", i+1)
		}
	}

	// Large Payload split across multiple RTP Packets
	res = pck.Payload(5, largepayload)
	if !reflect.DeepEqual(res, largePayloadPacketized) {
		t.Fatal("FU-A packetization failed")
	}

	// Nalu type 9 or 12
	res = pck.Payload(5, []byte{0x09, 0x00, 0x00})
	if len(res) != 0 {
		t.Fatal("Generated payload should be empty")
	}
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsVpcEndpointRouteTableAssociation() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsVPCEndpointRouteTableAssociationCreate,
		Read:   resourceAwsVPCEndpointRouteTableAssociationRead,
		Delete: resourceAwsVPCEndpointRouteTableAssociationDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"vpc_endpoint_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"route_table_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsVPCEndpointRouteTableAssociationCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	endpointId := d.Get("vpc_endpoint_id").(string)
	rtId := d.Get("route_table_id").(string)

	_, err := findResourceVPCEndpoint(conn, endpointId)
	if err != nil {
		return err
	}

	log.Printf(
		"[INFO] Creating VPC Endpoint/Route Table association: %s => %s",
		endpointId, rtId)

	input := &ec2.ModifyVpcEndpointInput{
		VpcEndpointId:    aws.String(endpointId),
		AddRouteTableIds: aws.StringSlice([]string{rtId}),
	}

	_, err = conn.ModifyVpcEndpoint(input)
	if err != nil {
		return fmt.Errorf("Error creating VPC Endpoint/Route Table association: %s", err.Error())
	}
	id := vpcEndpointIdRouteTableIdHash(endpointId, rtId)
	log.Printf("[DEBUG] VPC Endpoint/Route Table association %q created.", id)

	d.SetId(id)

	return resourceAwsVPCEndpointRouteTableAssociationRead(d, meta)
}

func resourceAwsVPCEndpointRouteTableAssociationRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	endpointId := d.Get("vpc_endpoint_id").(string)
	rtId := d.Get("route_table_id").(string)

	vpce, err := findResourceVPCEndpoint(conn, endpointId)
	if err, ok := err.(awserr.Error); ok && err.Code() == "InvalidVpcEndpointId.NotFound" {
		d.SetId("")
		return nil
	}

	found := false
	for _, id := range vpce.RouteTableIds {
		if id != nil && *id == rtId {
			found = true
			break
		}
	}
	if !found {
		// The association no longer exists.
		d.SetId("")
		return nil
	}

	id := vpcEndpointIdRouteTableIdHash(endpointId, rtId)
	log.Printf("[DEBUG] Computed VPC Endpoint/Route Table ID %s", id)
	d.SetId(id)

	return nil
}

func resourceAwsVPCEndpointRouteTableAssociationDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	endpointId := d.Get("vpc_endpoint_id").(string)
	rtId := d.Get("route_table_id").(string)

	input := &ec2.ModifyVpcEndpointInput{
		VpcEndpointId:       aws.String(endpointId),
		RemoveRouteTableIds: aws.StringSlice([]string{rtId}),
	}

	_, err := conn.ModifyVpcEndpoint(input)
	if err != nil {
		ec2err, ok := err.(awserr.Error)
		if !ok {
			return fmt.Errorf("Error deleting VPC Endpoint/Route Table association: %s", err.Error())
		}

		switch ec2err.Code() {
		case "InvalidVpcEndpointId.NotFound":
			fallthrough
		case "InvalidRouteTableId.NotFound":
			fallthrough
		case "InvalidParameter":
			log.Printf("[DEBUG] VPC Endpoint/Route Table association is already gone")
		default:
			return fmt.Errorf("Error deleting VPC Endpoint/Route Table association: %s", err.Error())
		}
	}

	log.Printf("[DEBUG] VPC Endpoint/Route Table association %q deleted", d.Id())
	d.SetId("")

	return nil
}

func findResourceVPCEndpoint(conn *ec2.EC2, id string) (*ec2.VpcEndpoint, error) {
	input := &ec2.DescribeVpcEndpointsInput{
		VpcEndpointIds: aws.StringSlice([]string{id}),
	}

	log.Printf("[DEBUG] Reading VPC Endpoint: %q", id)
	output, err := conn.DescribeVpcEndpoints(input)
	if err != nil {
		return nil, err
	}

	return output.VpcEndpoints[0], nil
}

func vpcEndpointIdRouteTableIdHash(endpointId, rtId string) string {
	return fmt.Sprintf("a-%s%d", endpointId, hashcode.String(rtId))
}
<commit_msg>provider/aws: Guard against panic in aws_vpc_endpoint_association (#11613)<commit_after>package aws

import (
	"fmt"
	"log"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/hashcode"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsVpcEndpointRouteTableAssociation() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsVPCEndpointRouteTableAssociationCreate,
		Read:   resourceAwsVPCEndpointRouteTableAssociationRead,
		Delete: resourceAwsVPCEndpointRouteTableAssociationDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"vpc_endpoint_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"route_table_id": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
		},
	}
}

func resourceAwsVPCEndpointRouteTableAssociationCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	endpointId := d.Get("vpc_endpoint_id").(string)
	rtId := d.Get("route_table_id").(string)

	_, err := findResourceVPCEndpoint(conn, endpointId)
	if err != nil {
		return err
	}

	log.Printf(
		"[INFO] Creating VPC Endpoint/Route Table association: %s => %s",
		endpointId, rtId)

	input := &ec2.ModifyVpcEndpointInput{
		VpcEndpointId:    aws.String(endpointId),
		AddRouteTableIds: aws.StringSlice([]string{rtId}),
	}

	_, err = conn.ModifyVpcEndpoint(input)
	if err != nil {
		return fmt.Errorf("Error creating VPC Endpoint/Route Table association: %s", err.Error())
	}
	id := vpcEndpointIdRouteTableIdHash(endpointId, rtId)
	log.Printf("[DEBUG] VPC Endpoint/Route Table association %q created.", id)

	d.SetId(id)

	return resourceAwsVPCEndpointRouteTableAssociationRead(d, meta)
}

func resourceAwsVPCEndpointRouteTableAssociationRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	endpointId := d.Get("vpc_endpoint_id").(string)
	rtId := d.Get("route_table_id").(string)

	vpce, err := findResourceVPCEndpoint(conn, endpointId)
	if err != nil {
		if err, ok := err.(awserr.Error); ok && err.Code() == "InvalidVpcEndpointId.NotFound" {
			d.SetId("")
			return nil
		}

		return err
	}

	found := false
	for _, id := range vpce.RouteTableIds {
		if id != nil && *id == rtId {
			found = true
			break
		}
	}
	if !found {
		// The association no longer exists.
		d.SetId("")
		return nil
	}

	id := vpcEndpointIdRouteTableIdHash(endpointId, rtId)
	log.Printf("[DEBUG] Computed VPC Endpoint/Route Table ID %s", id)
	d.SetId(id)

	return nil
}

func resourceAwsVPCEndpointRouteTableAssociationDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn
	endpointId := d.Get("vpc_endpoint_id").(string)
	rtId := d.Get("route_table_id").(string)

	input := &ec2.ModifyVpcEndpointInput{
		VpcEndpointId:       aws.String(endpointId),
		RemoveRouteTableIds: aws.StringSlice([]string{rtId}),
	}

	_, err := conn.ModifyVpcEndpoint(input)
	if err != nil {
		ec2err, ok := err.(awserr.Error)
		if !ok {
			return fmt.Errorf("Error deleting VPC Endpoint/Route Table association: %s", err.Error())
		}

		switch ec2err.Code() {
		case "InvalidVpcEndpointId.NotFound":
			fallthrough
		case "InvalidRouteTableId.NotFound":
			fallthrough
		case "InvalidParameter":
			log.Printf("[DEBUG] VPC Endpoint/Route Table association is already gone")
		default:
			return fmt.Errorf("Error deleting VPC Endpoint/Route Table association: %s", err.Error())
		}
	}

	log.Printf("[DEBUG] VPC Endpoint/Route Table association %q deleted", d.Id())
	d.SetId("")

	return nil
}

func findResourceVPCEndpoint(conn *ec2.EC2, id string) (*ec2.VpcEndpoint, error) {
	input := &ec2.DescribeVpcEndpointsInput{
		VpcEndpointIds: aws.StringSlice([]string{id}),
	}

	log.Printf("[DEBUG] Reading VPC Endpoint: %q", id)
	output, err := conn.DescribeVpcEndpoints(input)
	if err != nil {
		return nil, err
	}

	if output.VpcEndpoints == nil {
		return nil, fmt.Errorf("No VPC Endpoints were found for %q", id)
	}

	return output.VpcEndpoints[0], nil
}

func vpcEndpointIdRouteTableIdHash(endpointId, rtId string) string {
	return fmt.Sprintf("a-%s%d", endpointId, hashcode.String(rtId))
}
<|endoftext|>