<commit_before>package webdriver_test

import (
	"fmt"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sclevine/agouti"
	"github.com/sclevine/agouti/api"
	. "github.com/sclevine/agouti/matchers"

	"github.com/johanbrandhorst/protobuf/test/shared"
)

var _ = Describe("gRPC-Web Unit Tests", func() {
	//browserTest("Firefox", seleniumDriver.NewPage)
	browserTest("ChromeDriver", chromeDriver.NewPage)
})

type pageFunc func(...agouti.Option) (*agouti.Page, error)

func browserTest(browserName string, newPage pageFunc) {
	var page *agouti.Page

	BeforeEach(func() {
		var err error
		page, err = newPage()
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		Expect(page.Destroy()).NotTo(HaveOccurred())
	})

	Context(fmt.Sprintf("when testing %s", browserName), func() {
		It("should not find any errors", func() {
			By("Loading the test page", func() {
				Expect(page.Navigate("https://" + shared.GopherJSServer)).NotTo(HaveOccurred())
			})

			By("Finding the number of failures", func() {
				Eventually(page.FirstByClass("failed")).Should(BeFound())
				Eventually(page.FindByID("qunit-testresult").FindByClass("failed")).Should(BeFound())
				failures, err := page.FindByID("qunit-testresult").FindByClass("failed").Text()
				Expect(err).NotTo(HaveOccurred())
				if failures == "0" {
					return
				}

				logs, err := page.ReadAllLogs("browser")
				Expect(err).NotTo(HaveOccurred())
				fmt.Fprintln(GinkgoWriter, "Console output ------------------------------------")
				for _, log := range logs {
					fmt.Fprintf(GinkgoWriter, "[%s][%s]\t%s\n", log.Time.Format("15:04:05.000"), log.Level, log.Message)
				}
				fmt.Fprintln(GinkgoWriter, "Console output ------------------------------------")

				// We have at least one failure - lets compile an error message
				Eventually(page.FindByID(
					"qunit-tests",
				).AllByClass(
					"fail",
				).AllByClass(
					"fail",
				)).Should(BeFound())
				messages := page.FindByID(
					"qunit-tests",
				).AllByClass(
					"fail",
				).AllByClass(
					"fail",
				)
				elements, err := messages.Elements()
				Expect(err).NotTo(HaveOccurred())
				var errMsgs []string
				for _, element := range elements {
					// Get error summary
					msg, err := element.GetElement(api.Selector{
						Using: "css selector",
						Value: ".test-message",
					})
					Expect(err).NotTo(HaveOccurred())
					errText, err := msg.GetText()
					Expect(err).NotTo(HaveOccurred())

					// Get diff
					expected, err := element.GetElements(api.Selector{
						Using: "css selector",
						Value: "del",
					})
					Expect(err).NotTo(HaveOccurred())
					var expectedText string
					if len(expected) > 0 {
						expectedText, err = expected[0].GetText()
						Expect(err).NotTo(HaveOccurred())
					}
					actual, err := element.GetElements(api.Selector{
						Using: "css selector",
						Value: "ins",
					})
					Expect(err).NotTo(HaveOccurred())
					var actualText string
					if len(actual) > 0 {
						actualText, err = actual[0].GetText()
						Expect(err).NotTo(HaveOccurred())
					}

					errMsg := errText
					if expectedText != "" && actualText != "" {
						errMsg = fmt.Sprintf(
							"%s\n\tExpected: %q\n\tActual: %q",
							errText,
							strings.TrimSuffix(expectedText, " "),
							strings.TrimSuffix(actualText, " "),
						)
					}
					errMsgs = append(errMsgs, errMsg)
				}

				// Prints each error
				Fail(strings.Join(errMsgs, "\n-----------------------------------\n"))
			})
		})
	})
}
<commit_msg>Increase timeout for finding the failed div<commit_after>package webdriver_test

import (
	"fmt"
	"strings"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/sclevine/agouti"
	"github.com/sclevine/agouti/api"
	. "github.com/sclevine/agouti/matchers"

	"github.com/johanbrandhorst/protobuf/test/shared"
)

var _ = Describe("gRPC-Web Unit Tests", func() {
	//browserTest("Firefox", seleniumDriver.NewPage)
	browserTest("ChromeDriver", chromeDriver.NewPage)
})

type pageFunc func(...agouti.Option) (*agouti.Page, error)

func browserTest(browserName string, newPage pageFunc) {
	var page *agouti.Page

	BeforeEach(func() {
		var err error
		page, err = newPage()
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		Expect(page.Destroy()).NotTo(HaveOccurred())
	})

	Context(fmt.Sprintf("when testing %s", browserName), func() {
		It("should not find any errors", func() {
			By("Loading the test page", func() {
				Expect(page.Navigate("https://" + shared.GopherJSServer)).NotTo(HaveOccurred())
			})

			By("Finding the number of failures", func() {
				Eventually(page.FirstByClass("failed"), 2).Should(BeFound())
				Eventually(page.FindByID("qunit-testresult").FindByClass("failed")).Should(BeFound())
				failures, err := page.FindByID("qunit-testresult").FindByClass("failed").Text()
				Expect(err).NotTo(HaveOccurred())
				if failures == "0" {
					return
				}

				logs, err := page.ReadAllLogs("browser")
				Expect(err).NotTo(HaveOccurred())
				fmt.Fprintln(GinkgoWriter, "Console output ------------------------------------")
				for _, log := range logs {
					fmt.Fprintf(GinkgoWriter, "[%s][%s]\t%s\n", log.Time.Format("15:04:05.000"), log.Level, log.Message)
				}
				fmt.Fprintln(GinkgoWriter, "Console output ------------------------------------")

				// We have at least one failure - lets compile an error message
				Eventually(page.FindByID(
					"qunit-tests",
				).AllByClass(
					"fail",
				).AllByClass(
					"fail",
				)).Should(BeFound())
				messages := page.FindByID(
					"qunit-tests",
				).AllByClass(
					"fail",
				).AllByClass(
					"fail",
				)
				elements, err := messages.Elements()
				Expect(err).NotTo(HaveOccurred())
				var errMsgs []string
				for _, element := range elements {
					// Get error summary
					msg, err := element.GetElement(api.Selector{
						Using: "css selector",
						Value: ".test-message",
					})
					Expect(err).NotTo(HaveOccurred())
					errText, err := msg.GetText()
					Expect(err).NotTo(HaveOccurred())

					// Get diff
					expected, err := element.GetElements(api.Selector{
						Using: "css selector",
						Value: "del",
					})
					Expect(err).NotTo(HaveOccurred())
					var expectedText string
					if len(expected) > 0 {
						expectedText, err = expected[0].GetText()
						Expect(err).NotTo(HaveOccurred())
					}
					actual, err := element.GetElements(api.Selector{
						Using: "css selector",
						Value: "ins",
					})
					Expect(err).NotTo(HaveOccurred())
					var actualText string
					if len(actual) > 0 {
						actualText, err = actual[0].GetText()
						Expect(err).NotTo(HaveOccurred())
					}

					errMsg := errText
					if expectedText != "" && actualText != "" {
						errMsg = fmt.Sprintf(
							"%s\n\tExpected: %q\n\tActual: %q",
							errText,
							strings.TrimSuffix(expectedText, " "),
							strings.TrimSuffix(actualText, " "),
						)
					}
					errMsgs = append(errMsgs, errMsg)
				}

				// Prints each error
				Fail(strings.Join(errMsgs, "\n-----------------------------------\n"))
			})
		})
	})
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/albertyw/reaction-pics/tumblr"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	"go.uber.org/zap"
)

type HandlerTestSuite struct {
	suite.Suite
	deps handlerDeps
}

func TestHandlerTestSuite(t *testing.T) {
	suite.Run(t, new(HandlerTestSuite))
}

func (s *HandlerTestSuite) SetupTest() {
	logger := zap.NewNop().Sugar()
	board := tumblr.NewBoard([]tumblr.Post{})
	s.deps = handlerDeps{
		logger:         logger,
		board:          &board,
		appCacheString: appCacheString(logger),
	}
}

func (s *HandlerTestSuite) TestIndexFile() {
	request, err := http.NewRequest("GET", "/", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)

	assert.Contains(s.T(), response.Body.String(), s.deps.appCacheString)
}

func (s *HandlerTestSuite) TestOnlyIndexFile() {
	request, err := http.NewRequest("GET", "/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestReadFile() {
	request, err := http.NewRequest("GET", "/static/favicon/manifest.json", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	staticHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.True(s.T(), len(response.Body.String()) > 100)
}

func (s *HandlerTestSuite) TestNoExactURL() {
	request, err := http.NewRequest("GET", "/static/asdf.js", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	staticHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)

	response = httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestSearchHandler() {
	request, err := http.NewRequest("GET", "/search", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":0,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestSearchHandlerOffset() {
	request, err := http.NewRequest("GET", "/search?offset=1", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":1,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestSearchHandlerMalformedOffset() {
	request, err := http.NewRequest("GET", "/search?offset=asdf", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":0,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestPostHandlerMalformed() {
	request, err := http.NewRequest("GET", "/post/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostHandlerNotFound() {
	request, err := http.NewRequest("GET", "/post/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostHandler() {
	post := tumblr.Post{ID: 1234}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/post/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.NotEqual(s.T(), len(response.Body.String()), 0)
}

func (s *HandlerTestSuite) TestPostDataHandler() {
	post := tumblr.Post{ID: 1234}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.NotEqual(s.T(), len(response.Body.String()), 0)
}

func (s *HandlerTestSuite) TestPostDataPercentHandler() {
	post := tumblr.Post{ID: 1234, Title: `asdf% qwer`}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	var data map[string][]map[string]interface{}
	json.Unmarshal(response.Body.Bytes(), &data)
	title := data["data"][0]["title"].(string)
	assert.Equal(s.T(), `asdf% qwer`, title)
}

func (s *HandlerTestSuite) TestPostDataHandlerMalformed() {
	request, err := http.NewRequest("GET", "/postdata/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostDataHandlerUnknown() {
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestStatsHandler() {
	request, err := http.NewRequest("GET", "/stats.json", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	statsHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"keywords\":[],\"postCount\":\"0\"}")
}
<commit_msg>Add test for sitemapHandler<commit_after>package server

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/albertyw/reaction-pics/tumblr"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	"go.uber.org/zap"
)

type HandlerTestSuite struct {
	suite.Suite
	deps handlerDeps
}

func TestHandlerTestSuite(t *testing.T) {
	suite.Run(t, new(HandlerTestSuite))
}

func (s *HandlerTestSuite) SetupTest() {
	logger := zap.NewNop().Sugar()
	board := tumblr.NewBoard([]tumblr.Post{})
	s.deps = handlerDeps{
		logger:         logger,
		board:          &board,
		appCacheString: appCacheString(logger),
	}
}

func (s *HandlerTestSuite) TestIndexFile() {
	request, err := http.NewRequest("GET", "/", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)

	assert.Contains(s.T(), response.Body.String(), s.deps.appCacheString)
}

func (s *HandlerTestSuite) TestOnlyIndexFile() {
	request, err := http.NewRequest("GET", "/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestReadFile() {
	request, err := http.NewRequest("GET", "/static/favicon/manifest.json", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	staticHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.True(s.T(), len(response.Body.String()) > 100)
}

func (s *HandlerTestSuite) TestNoExactURL() {
	request, err := http.NewRequest("GET", "/static/asdf.js", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	staticHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)

	response = httptest.NewRecorder()
	indexHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestSearchHandler() {
	request, err := http.NewRequest("GET", "/search", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":0,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestSearchHandlerOffset() {
	request, err := http.NewRequest("GET", "/search?offset=1", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":1,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestSearchHandlerMalformedOffset() {
	request, err := http.NewRequest("GET", "/search?offset=asdf", nil)
	assert.NoError(s.T(), err)

	q := request.URL.Query()
	q.Add("query", "searchTerm")
	response := httptest.NewRecorder()
	searchHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"data\":[],\"offset\":0,\"totalResults\":0}")
}

func (s *HandlerTestSuite) TestPostHandlerMalformed() {
	request, err := http.NewRequest("GET", "/post/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostHandlerNotFound() {
	request, err := http.NewRequest("GET", "/post/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostHandler() {
	post := tumblr.Post{ID: 1234}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/post/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.NotEqual(s.T(), len(response.Body.String()), 0)
}

func (s *HandlerTestSuite) TestPostDataHandler() {
	post := tumblr.Post{ID: 1234}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.NotEqual(s.T(), len(response.Body.String()), 0)
}

func (s *HandlerTestSuite) TestPostDataPercentHandler() {
	post := tumblr.Post{ID: 1234, Title: `asdf% qwer`}
	s.deps.board.AddPost(post)
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	var data map[string][]map[string]interface{}
	json.Unmarshal(response.Body.Bytes(), &data)
	title := data["data"][0]["title"].(string)
	assert.Equal(s.T(), `asdf% qwer`, title)
}

func (s *HandlerTestSuite) TestPostDataHandlerMalformed() {
	request, err := http.NewRequest("GET", "/postdata/asdf", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestPostDataHandlerUnknown() {
	request, err := http.NewRequest("GET", "/postdata/1234", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	postDataHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 404)
}

func (s *HandlerTestSuite) TestStatsHandler() {
	request, err := http.NewRequest("GET", "/stats.json", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	statsHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.Equal(s.T(), response.Body.String(), "{\"keywords\":[],\"postCount\":\"0\"}")
}

func (s *HandlerTestSuite) TestSitemapHandler() {
	request, err := http.NewRequest("GET", "/sitemap.xml", nil)
	assert.NoError(s.T(), err)

	response := httptest.NewRecorder()
	sitemapHandler(response, request, s.deps)
	assert.Equal(s.T(), response.Code, 200)
	assert.True(s.T(), len(response.Body.String()) > 100)
}
<|endoftext|>


<commit_before>package service

import (
	"log"
	"strconv"
	"strings"
	"sync"

	"github.com/artemnikitin/devicefarm-ci-tool/config"
	"github.com/artemnikitin/devicefarm-ci-tool/tools"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/devicefarm"
)

func createScheduleRunInput(client *devicefarm.DeviceFarm, conf *config.RunConfig, projectArn string) *devicefarm.ScheduleRunInput {
	var wg sync.WaitGroup
	result := &devicefarm.ScheduleRunInput{
		ProjectArn: aws.String(projectArn),
		Test: &devicefarm.ScheduleRunTest{},
		Configuration: &devicefarm.ScheduleRunConfiguration{
			Radios: &devicefarm.Radios{
				Bluetooth: aws.Bool(true),
				Gps:       aws.Bool(true),
				Nfc:       aws.Bool(true),
				Wifi:      aws.Bool(true),
			},
			Location: &devicefarm.Location{
				Latitude:  aws.Float64(47.6204),
				Longitude: aws.Float64(-122.3491),
			},
		},
	}
	if conf.RunName != "" {
		result.Name = aws.String(conf.RunName)
	}

	processTestBlock(conf, result)
	if conf.Test.TestPackageArn == "" && conf.Test.TestPackagePath != "" {
		wg.Add(1)
		go func() {
			log.Println("Prepare tests for uploading...")
			t := config.GetUploadTypeForTest(conf.Test.Type)
			arn, url := CreateUploadWithType(client, projectArn, conf.Test.TestPackagePath, t)
			httpResponse := tools.UploadFile(conf.Test.TestPackagePath, url)
			if httpResponse != 200 {
				log.Fatal("Can't upload test app")
			}
			WaitForAppProcessed(client, arn)
			result.Test.TestPackageArn = aws.String(arn)
			wg.Done()
		}()
	}

	processConfigurationBlock(conf, result)
	if conf.AdditionalData.ExtraDataPackageArn == "" && conf.AdditionalData.ExtraDataPackagePath != "" {
		wg.Add(1)
		go func() {
			log.Println("Prepare extra data for uploading...")
			arn, url := CreateUploadWithType(client, projectArn, conf.AdditionalData.ExtraDataPackagePath, "EXTERNAL_DATA")
			httpResponse := tools.UploadFile(conf.AdditionalData.ExtraDataPackagePath, url)
			if httpResponse != 200 {
				log.Fatal("Can't upload test app")
			}
			WaitForAppProcessed(client, arn)
			result.Configuration.ExtraDataPackageArn = aws.String(arn)
			wg.Done()
		}()
	}
	wg.Wait()
	return result
}

func processTestBlock(conf *config.RunConfig, sri *devicefarm.ScheduleRunInput) {
	if conf.Test.Type != "" {
		sri.Test.Type = aws.String(conf.Test.Type)
	}
	if conf.Test.TestPackageArn != "" {
		sri.Test.TestPackageArn = aws.String(conf.Test.TestPackageArn)
	}
	if conf.Test.Filter != "" {
		sri.Test.Filter = aws.String(conf.Test.Filter)
	}
	params := conf.Test.Parameters
	if len(params) > 0 {
		temp := make(map[string]*string)
		for k, v := range params {
			temp[k] = aws.String(v)
		}
		sri.Test.Parameters = temp
	}
}

func processConfigurationBlock(conf *config.RunConfig, sri *devicefarm.ScheduleRunInput) {
	if conf.AdditionalData.BillingMethod != "" {
		sri.Configuration.BillingMethod = aws.String(conf.AdditionalData.BillingMethod)
	}
	if conf.AdditionalData.Locale != "" {
		sri.Configuration.Locale = aws.String(conf.AdditionalData.Locale)
	}
	if conf.AdditionalData.NetworkProfileArn != "" {
		sri.Configuration.NetworkProfileArn = aws.String(conf.AdditionalData.NetworkProfileArn)
	}
	if conf.AdditionalData.Location.Latitude != 0 && conf.AdditionalData.Location.Longitude != 0 {
		sri.Configuration.Location.Latitude = aws.Float64(conf.AdditionalData.Location.Latitude)
		sri.Configuration.Location.Longitude = aws.Float64(conf.AdditionalData.Location.Longitude)
	}
	if len(conf.AdditionalData.AuxiliaryApps) != 0 {
		array := conf.AdditionalData.AuxiliaryApps
		sri.Configuration.AuxiliaryApps = aws.StringSlice(array)
	}
	if strings.ToLower(conf.AdditionalData.Radios.Bluetooth) == "false" {
		b, _ := strconv.ParseBool(conf.AdditionalData.Radios.Bluetooth)
		sri.Configuration.Radios.Bluetooth = aws.Bool(b)
	}
	if strings.ToLower(conf.AdditionalData.Radios.Gps) == "false" {
		b, _ := strconv.ParseBool(conf.AdditionalData.Radios.Gps)
		sri.Configuration.Radios.Gps = aws.Bool(b)
	}
	if strings.ToLower(conf.AdditionalData.Radios.Nfc) == "false" {
		b, _ := strconv.ParseBool(conf.AdditionalData.Radios.Nfc)
		sri.Configuration.Radios.Nfc = aws.Bool(b)
	}
	if strings.ToLower(conf.AdditionalData.Radios.Wifi) == "false" {
		b, _ := strconv.ParseBool(conf.AdditionalData.Radios.Wifi)
		sri.Configuration.Radios.Wifi = aws.Bool(b)
	}
	if conf.AdditionalData.ExtraDataPackageArn != "" {
		sri.Configuration.ExtraDataPackageArn = aws.String(conf.AdditionalData.ExtraDataPackageArn)
	}
}
<commit_msg>Apply gofmt<commit_after>package service

import (
	"log"
	"strconv"
	"strings"
	"sync"

	"github.com/artemnikitin/devicefarm-ci-tool/config"
	"github.com/artemnikitin/devicefarm-ci-tool/tools"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/devicefarm"
)

func createScheduleRunInput(client *devicefarm.DeviceFarm, conf *config.RunConfig, projectArn string) *devicefarm.ScheduleRunInput {
	var wg sync.WaitGroup
	result := &devicefarm.ScheduleRunInput{
		ProjectArn: aws.String(projectArn),
		Test:       &devicefarm.ScheduleRunTest{},
		Configuration: &devicefarm.ScheduleRunConfiguration{
			Radios: &devicefarm.Radios{
				Bluetooth: aws.Bool(true),
				Gps:       aws.Bool(true),
				Nfc:       aws.Bool(true),
				Wifi:      aws.Bool(true),
			},
			Location: &devicefarm.Location{
				Latitude:  aws.Float64(47.6204),
				Longitude: aws.Float64(-122.3491),
			},
		},
	}
	if conf.RunName != "" {
		result.Name = aws.String(conf.RunName)
	}

	processTestBlock(conf, result)
	if conf.Test.TestPackageArn == "" && conf.Test.TestPackagePath != "" {
		wg.Add(1)
		go func() {
			log.Println("Prepare tests for uploading...")
			t := config.GetUploadTypeForTest(conf.Test.Type)
			arn, url := CreateUploadWithType(client, projectArn, conf.Test.TestPackagePath, t)
			httpResponse := tools.UploadFile(conf.Test.TestPackagePath, url)
			if httpResponse != 200 {
				log.Fatal("Can't upload test app")
			}
			WaitForAppProcessed(client, arn)
			result.Test.TestPackageArn = aws.String(arn)
			wg.Done()
		}()
	}

	processConfigurationBlock(conf, result)
	if conf.AdditionalData.ExtraDataPackageArn == "" && conf.AdditionalData.ExtraDataPackagePath != "" {
		wg.Add(1)
		go func() {
			log.Println("Prepare extra data for uploading...")
			arn, url := CreateUploadWithType(client, projectArn, conf.AdditionalData.ExtraDataPackagePath, "EXTERNAL_DATA")
			httpResponse := tools.UploadFile(conf.AdditionalData.ExtraDataPackagePath, url)
			if httpResponse != 200 {
				log.Fatal("Can't upload test app")
			}
			WaitForAppProcessed(client, arn)
			result.Configuration.ExtraDataPackageArn = aws.String(arn)
			wg.Done()
		}()
	}
	wg.Wait()
	return result
}

func processTestBlock(conf *config.RunConfig, sri *devicefarm.ScheduleRunInput) {
	if conf.Test.Type != "" {
		sri.Test.Type = aws.String(conf.Test.Type)
	}
	if conf.Test.TestPackageArn != "" {
		sri.Test.TestPackageArn = aws.String(conf.Test.TestPackageArn)
	}
	if conf.Test.Filter != "" {
		sri.Test.Filter = aws.String(conf.Test.Filter)
	}
	params := conf.Test.Parameters
	if len(params) > 0 {
		temp := make(map[string]*string)
		for k, v := range params {
			temp[k] = aws.String(v)
		}
		sri.Test.Parameters = temp
	}
}

func processConfigurationBlock(conf *config.RunConfig, sri *devicefarm.ScheduleRunInput) {
	if conf.AdditionalData.BillingMethod != "" {
		sri.Configuration.BillingMethod = aws.String(conf.AdditionalData.BillingMethod)
	}
	if conf.AdditionalData.Locale != "" {
		sri.Configuration.Locale = aws.String(conf.AdditionalData.Locale)
	}
	if conf.AdditionalData.NetworkProfileArn != "" {
		sri.Configuration.NetworkProfileArn = aws.String(conf.AdditionalData.NetworkProfileArn)
	}
	if conf.AdditionalData.Location.Latitude != 0 && conf.AdditionalData.Location.Longitude != 0 {
		sri.Configuration.Location.Latitude = aws.Float64(conf.AdditionalData.Location.Latitude)
		sri.Configuration.Location.Longitude = aws.Float64(conf.AdditionalData.Location.Longitude)
	}
	if len(conf.AdditionalData.AuxiliaryApps) != 0 {
		array := conf.AdditionalData.AuxiliaryApps
		sri.Configuration.AuxiliaryApps = aws.StringSlice(array)
	}
	if strings.ToLower(conf.AdditionalData.Radios.Bluetooth) == "false" {
		b, _ := strconv.ParseBool(conf.AdditionalData.Radios.Bluetooth)
		sri.Configuration.Radios.Bluetooth = aws.Bool(b)
	}
	if strings.ToLower(conf.AdditionalData.Radios.Gps) == "false" {
		b, _ := strconv.ParseBool(conf.AdditionalData.Radios.Gps)
		sri.Configuration.Radios.Gps = aws.Bool(b)
	}
	if strings.ToLower(conf.AdditionalData.Radios.Nfc) == "false" {
		b, _ := strconv.ParseBool(conf.AdditionalData.Radios.Nfc)
		sri.Configuration.Radios.Nfc = aws.Bool(b)
	}
	if strings.ToLower(conf.AdditionalData.Radios.Wifi) == "false" {
		b, _ := strconv.ParseBool(conf.AdditionalData.Radios.Wifi)
		sri.Configuration.Radios.Wifi = aws.Bool(b)
	}
	if conf.AdditionalData.ExtraDataPackageArn != "" {
		sri.Configuration.ExtraDataPackageArn = aws.String(conf.AdditionalData.ExtraDataPackageArn)
	}
}
<|endoftext|>


<commit_before>package services

import (
	"fmt"
	"github.com/stitchfix/flotilla-os/clients/cluster"
	"github.com/stitchfix/flotilla-os/clients/registry"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/exceptions"
	"github.com/stitchfix/flotilla-os/execution/engine"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
)

//
// ExecutionService interacts with the state manager and queue manager to queue runs, and perform
// CRUD operations on them
// * Acts as an intermediary layer between state and the execution engine
//
type ExecutionService interface {
	Create(definitionID string, clusterName string, env *state.EnvList, ownerID string) (state.Run, error)
	List(
		limit int,
		offset int,
		sortOrder string,
		sortField string,
		filters map[string]string,
		envFilters map[string]string) (state.RunList, error)
	Get(runID string) (state.Run, error)
	UpdateStatus(runID string, status string, exitCode *int64) error
	Terminate(runID string) error
	ReservedVariables() []string
}

type executionService struct {
	sm          state.Manager
	qm          queue.Manager
	cc          cluster.Client
	rc          registry.Client
	ee          engine.Engine
	reservedEnv map[string]func(run state.Run) string
}

//
// NewExecutionService configures and returns an ExecutionService
//
func NewExecutionService(conf config.Config, ee engine.Engine,
	sm state.Manager,
	qm queue.Manager,
	cc cluster.Client,
	rc registry.Client) (ExecutionService, error) {
	es := executionService{
		sm: sm,
		qm: qm,
		cc: cc,
		rc: rc,
		ee: ee,
	}
	//
	// Reserved environment variables dynamically generated
	// per run

	ownerKey := conf.GetString("owner_id_var")
	if len(ownerKey) == 0 {
		ownerKey = "FLOTILLA_RUN_OWNER_ID"
	}
	es.reservedEnv = map[string]func(run state.Run) string{
		"FLOTILLA_SERVER_MODE": func(run state.Run) string {
			return conf.GetString("flotilla_mode")
		},
		"FLOTILLA_RUN_ID": func(run state.Run) string {
			return run.RunID
		},
		ownerKey: func(run state.Run) string {
			return run.User
		},
	}
	return &es, nil
}

//
// ReservedVariables returns the list of reserved run environment variable
// names
//
func (es *executionService) ReservedVariables() []string {
	var keys []string
	for k := range es.reservedEnv {
		keys = append(keys, k)
	}
	return keys
}

//
// Create constructs and queues a new Run on the cluster specified
//
func (es *executionService) Create(
	definitionID string, clusterName string, env *state.EnvList, ownerID string) (state.Run, error) {
	var (
		run state.Run
		err error
	)

	// Ensure definition exists
	definition, err := es.sm.GetDefinition(definitionID)
	if err != nil {
		return run, err
	}

	// Validate that definition can be run (image exists, cluster has resources)
	if err = es.canBeRun(clusterName, definition, env); err != nil {
		return run, err
	}

	// Construct run object with StatusQueued and new UUID4 run id
	run, err = es.constructRun(clusterName, definition, env, ownerID)
	if err != nil {
		return run, err
	}

	// Save run to source of state - it is *CRITICAL* to do this
	// -before- queuing to avoid processing unsaved runs
	if err = es.sm.CreateRun(run); err != nil {
		return run, err
	}

	// Get qurl
	qurl, err := es.qm.QurlFor(run.ClusterName)
	if err != nil {
		return run, err
	}

	// Queue run
	return run, es.qm.Enqueue(qurl, run)
}

func (es *executionService) constructRun(
	clusterName string, definition state.Definition, env *state.EnvList, ownerID string) (state.Run, error) {

	var (
		run state.Run
		err error
	)

	runID, err := state.NewRunID()
	if err != nil {
		return run, err
	}

	run = state.Run{
		RunID:        runID,
		ClusterName:  clusterName,
		GroupName:    definition.GroupName,
		DefinitionID: definition.DefinitionID,
		Status:       state.StatusQueued,
		User:         ownerID,
	}
	runEnv := es.constructEnviron(run, env)
	run.Env = &runEnv
	return run, nil
}

func (es *executionService) constructEnviron(run state.Run, env *state.EnvList) state.EnvList {
	size := len(es.reservedEnv)
	if env != nil {
		size += len(*env)
	}
	runEnv := make([]state.EnvVar, size)
	i := 0
	for k, f := range es.reservedEnv {
		runEnv[i] = state.EnvVar{
			Name:  k,
			Value: f(run),
		}
		i++
	}
	if env != nil {
		for j, e := range *env {
			runEnv[i+j] = e
		}
	}
	return state.EnvList(runEnv)
}

func (es *executionService) canBeRun(clusterName string, definition state.Definition, env *state.EnvList) error {
	if env != nil {
		for _, e := range *env {
			_, usingRestricted := es.reservedEnv[e.Name]
			if usingRestricted {
				return exceptions.ConflictingResource{
					fmt.Sprintf("environment variable %s is reserved", e.Name)}
			}
		}
	}

	ok, err := es.rc.IsImageValid(definition.Image)
	if err != nil {
		return err
	}
	if !ok {
		return exceptions.MissingResource{
			fmt.Sprintf(
				"image [%s] was not found in any of the configured repositories", definition.Image)}
	}

	ok, err = es.cc.CanBeRun(clusterName, definition)
	if err != nil {
		return err
	}
	if !ok {
		return exceptions.MalformedInput{
			fmt.Sprintf(
				"definition [%s] cannot be run on cluster [%s]", definition.DefinitionID, clusterName)}
	}
	return nil
}

//
// List returns a list of Runs
// * validates definition_id and status filters
//
func (es *executionService) List(
	limit int,
	offset int,
	sortOrder string,
	sortField string,
	filters map[string]string,
	envFilters map[string]string) (state.RunList, error) {

	// If definition_id is present in filters, validate its
	// existence first
	definitionID, ok := filters["definition_id"]
	if ok {
		_, err := es.sm.GetDefinition(definitionID)
		if err != nil {
			return state.RunList{}, err
		}
	}

	status, ok := filters["status"]
	if ok && !state.IsValidStatus(status) {
		// Status filter is invalid
		err := exceptions.MalformedInput{
			fmt.Sprintf("invalid status [%s]", status)}
		return state.RunList{}, err
	}
	return es.sm.ListRuns(limit, offset, sortField, sortOrder, filters, envFilters)
}

//
// Get returns the run with the given runID
//
func (es *executionService) Get(runID string) (state.Run, error) {
	return es.sm.GetRun(runID)
}

//
// UpdateStatus is for supporting some legacy runs that still manually update their status
//
func (es *executionService) UpdateStatus(runID string, status string, exitCode *int64) error {
	if !state.IsValidStatus(status) {
		return exceptions.MalformedInput{fmt.Sprintf("status %s is invalid", status)}
	}
	_, err := es.sm.UpdateRun(runID, state.Run{Status: status, ExitCode: exitCode})
	return err
}

//
// Terminate stops the run with the given runID
//
func (es *executionService) Terminate(runID string) error {
	run, err := es.sm.GetRun(runID)
	if err != nil {
		return err
	}

	if run.Status != state.StatusStopped && len(run.TaskArn) > 0 && len(run.ClusterName) > 0 {
		return es.ee.Terminate(run)
	}
	return exceptions.MalformedInput{
		fmt.Sprintf(
			"invalid run, state: %s, arn: %s, clusterName: %s", run.Status, run.TaskArn, run.ClusterName)}
}
<commit_msg>fix case where a QUEUED task cannot be killed<commit_after>package services

import (
	"fmt"
	"github.com/stitchfix/flotilla-os/clients/cluster"
	"github.com/stitchfix/flotilla-os/clients/registry"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/exceptions"
	"github.com/stitchfix/flotilla-os/execution/engine"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
)

//
// ExecutionService interacts with the state manager and queue manager to queue runs, and perform
// CRUD operations on them
// * Acts as an intermediary layer between state and the execution engine
//
type ExecutionService interface {
	Create(definitionID string, clusterName string, env *state.EnvList, ownerID string) (state.Run, error)
	List(
		limit int,
		offset int,
		sortOrder string,
		sortField string,
		filters map[string]string,
		envFilters map[string]string) (state.RunList, error)
	Get(runID string) (state.Run, error)
	UpdateStatus(runID string, status string, exitCode *int64) error
	Terminate(runID string) error
	ReservedVariables() []string
}

type executionService struct {
	sm          state.Manager
	qm          queue.Manager
	cc          cluster.Client
	rc          registry.Client
	ee          engine.Engine
	reservedEnv map[string]func(run state.Run) string
}

//
// NewExecutionService configures and returns an ExecutionService
//
func NewExecutionService(conf config.Config, ee engine.Engine,
	sm state.Manager,
	qm queue.Manager,
	cc cluster.Client,
	rc registry.Client) (ExecutionService, error) {
	es := executionService{
		sm: sm,
		qm: qm,
		cc: cc,
		rc: rc,
		ee: ee,
	}
	//
	// Reserved environment variables dynamically generated
	// per run

	ownerKey := conf.GetString("owner_id_var")
	if len(ownerKey) == 0 {
		ownerKey = "FLOTILLA_RUN_OWNER_ID"
	}
	es.reservedEnv = map[string]func(run state.Run) string{
		"FLOTILLA_SERVER_MODE": func(run state.Run) string {
			return conf.GetString("flotilla_mode")
		},
		"FLOTILLA_RUN_ID": func(run state.Run) string {
			return run.RunID
		},
		ownerKey: func(run state.Run) string {
			return run.User
		},
	}
	return &es, nil
}

//
// ReservedVariables returns the list of reserved run environment variable
// names
//
func (es *executionService) ReservedVariables() []string {
	var keys []string
	for k := range es.reservedEnv {
		keys = append(keys, k)
	}
	return keys
}

//
// Create constructs and queues a new Run on the cluster specified
//
func (es *executionService) Create(
	definitionID string, clusterName string, env *state.EnvList, ownerID string) (state.Run, error) {
	var (
		run state.Run
		err error
	)

	// Ensure definition exists
	definition, err := es.sm.GetDefinition(definitionID)
	if err != nil {
		return run, err
	}

	// Validate that definition can be run (image exists, cluster has resources)
	if err = es.canBeRun(clusterName, definition, env); err != nil {
		return run, err
	}

	// Construct run object with StatusQueued and new UUID4 run id
	run, err = es.constructRun(clusterName, definition, env, ownerID)
	if err != nil {
		return run, err
	}

	// Save run to source of state - it is *CRITICAL* to do this
	// -before- queuing to avoid processing unsaved runs
	if err = es.sm.CreateRun(run); err != nil {
		return run, err
	}

	// Get qurl
	qurl, err := es.qm.QurlFor(run.ClusterName)
	if err != nil {
		return run, err
	}

	// Queue run
	return run, es.qm.Enqueue(qurl, run)
}

func (es *executionService) constructRun(
	clusterName string, definition state.Definition, env *state.EnvList, ownerID string) (state.Run, error) {

	var (
		run state.Run
		err error
	)

	runID, err := state.NewRunID()
	if err != nil {
		return run, err
	}

	run = state.Run{
		RunID:        runID,
		ClusterName:  clusterName,
		GroupName:    definition.GroupName,
		DefinitionID: definition.DefinitionID,
		Status:       state.StatusQueued,
		User:         ownerID,
	}
	runEnv := es.constructEnviron(run, env)
	run.Env = &runEnv
	return run, nil
}

func (es *executionService) constructEnviron(run state.Run, env *state.EnvList) state.EnvList {
	size := len(es.reservedEnv)
	if env != nil {
		size += len(*env)
	}
	runEnv := make([]state.EnvVar, size)
	i := 0
	for k, f := range es.reservedEnv {
		runEnv[i] = state.EnvVar{
			Name:  k,
			Value: f(run),
		}
		i++
	}
	if env != nil {
		for j, e := range *env {
			runEnv[i+j] = e
		}
	}
	return state.EnvList(runEnv)
}

func (es *executionService) canBeRun(clusterName string, definition state.Definition, env *state.EnvList) error {
	if env != nil {
		for _, e := range *env {
			_, usingRestricted := es.reservedEnv[e.Name]
			if usingRestricted {
				return exceptions.ConflictingResource{
					fmt.Sprintf("environment variable %s is reserved", e.Name)}
			}
		}
	}

	ok, err := es.rc.IsImageValid(definition.Image)
	if err != nil {
		return err
	}
	if !ok {
		return exceptions.MissingResource{
			fmt.Sprintf(
				"image [%s] was not found in any of the configured repositories", definition.Image)}
	}

	ok, err = es.cc.CanBeRun(clusterName, definition)
	if err != nil {
		return err
	}
	if !ok {
		return exceptions.MalformedInput{
			fmt.Sprintf(
				"definition [%s] cannot be run on cluster [%s]", definition.DefinitionID, clusterName)}
	}
	return nil
}

//
// List returns a list of Runs
// * validates definition_id and status filters
//
func (es *executionService) List(
	limit int,
	offset int,
	sortOrder string,
	sortField string,
	filters map[string]string,
	envFilters map[string]string) (state.RunList, error) {

	// If definition_id is present in filters, validate its
	// existence first
	definitionID, ok := filters["definition_id"]
	if ok {
		_, err := es.sm.GetDefinition(definitionID)
		if err != nil {
			return state.RunList{}, err
		}
	}

	status, ok := filters["status"]
	if ok && !state.IsValidStatus(status) {
		// Status filter is invalid
		err := exceptions.MalformedInput{
			fmt.Sprintf("invalid status [%s]", status)}
		return state.RunList{}, err
	}
	return es.sm.ListRuns(limit, offset, sortField, sortOrder, filters, envFilters)
}

//
// Get returns the run with the given runID
//
func (es *executionService) Get(runID string) (state.Run, error) {
	return es.sm.GetRun(runID)
}

//
// UpdateStatus is for supporting some legacy runs that still manually update their status
//
func (es *executionService) UpdateStatus(runID string, status string, exitCode *int64) error {
	if !state.IsValidStatus(status) {
		return exceptions.MalformedInput{fmt.Sprintf("status %s is invalid", status)}
	}
	_, err := es.sm.UpdateRun(runID, state.Run{Status: status, ExitCode: exitCode})
	return err
}

//
// Terminate stops the run with the given runID
//
func (es *executionService) Terminate(runID string) error {
	run, err := es.sm.GetRun(runID)
	if err != nil {
		return err
	}

	// If it's been submitted, let the status update workers handle setting it to stopped
	if run.Status != state.StatusStopped && len(run.TaskArn) > 0 && len(run.ClusterName) > 0 {
		return es.ee.Terminate(run)
	}

	// If it's queued and not submitted, set status to stopped (checked by submit worker)
	if run.Status == state.StatusQueued {
		_, err = es.sm.UpdateRun(runID, state.Run{Status: state.StatusStopped})
		return err
	}

	return exceptions.MalformedInput{
		fmt.Sprintf(
			"invalid run, state: %s, arn: %s, clusterName: %s", run.Status, run.TaskArn, run.ClusterName)}
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"golang.org/x/sync/errgroup"

	"github.com/markbates/deplist"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var setupOptions = struct {
	verbose       bool
	updateGoDeps  bool
	dropDatabases bool
}{}

type setupCheck func() error

var setupCmd = &cobra.Command{
	Use:   "setup",
	Short: "Setups a newly created, or recently checked out application.",
	Long: `Setup runs through checklist to make sure dependencies are setup correcly.

Dependencies (if used):
* Runs "dep ensure" to install required Go dependencies.

Asset Pipeline (if used):
* Runs "npm install" or "yarn install" to install asset dependencies.

Database (if used):
* Runs "buffalo db create -a" to create databases.
* Runs "buffalo db migrate" to run database migrations.
* Runs "buffalo task db:seed" to seed the database (if the task exists).

Tests:
* Runs "buffalo test" to confirm the application's tests are running properly.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		for _, check := range []setupCheck{assetCheck, updateGoDepsCheck, databaseCheck, testCheck} {
			err := check()
			if err != nil {
				return errors.WithStack(err)
			}
		}
		return nil
	},
}

func updateGoDepsCheck() error {
	deps, _ := deplist.List()
	if _, err := os.Stat("Gopkg.toml"); err == nil {
		// use github.com/golang/dep
		args := []string{"ensure"}
		if setupOptions.verbose {
			args = append(args, "-v")
		}
		if setupOptions.updateGoDeps {
			args = append(args, "--update")
		}
		err := run(exec.Command("dep", args...))
		if err != nil {
			return errors.WithStack(err)
		}
		return nil
	}

	// go old school with the installation
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	wg, ctx := errgroup.WithContext(ctx)
	deps, err := deplist.List()
	if err != nil {
		return errors.WithStack(err)
	}
	for dep := range deps {
		args := []string{"get"}
		if setupOptions.verbose {
			args = append(args, "-v")
		}
		if setupOptions.updateGoDeps {
			args = append(args, "-u")
		}
		args = append(args, dep)
		c := exec.Command("go", args...)
		f := func() error {
			return run(c)
		}
		wg.Go(f)
	}
	err = wg.Wait()
	if err != nil {
		return errors.Errorf("We encountered the following error trying to install and update the dependencies for this application:\n%s", err)
	}
	return nil
}

func testCheck() error {
	err := run(exec.Command("buffalo", "test"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to run your applications tests:\n%s", err)
	}
	return nil
}

func databaseCheck() error {
	if _, err := os.Stat("./database.yml"); err != nil {
		// no database.yml, so move on
		return nil
	}
	for _, check := range []setupCheck{dbCreateCheck, dbMigrateCheck, dbSeedCheck} {
		err := check()
		if err != nil {
			return err
		}
	}
	return nil
}

func dbCreateCheck() error {
	if setupOptions.dropDatabases {
		err := run(exec.Command("buffalo", "db", "drop", "-a"))
		if err != nil {
			return errors.Errorf("We encountered an error when trying to drop your application's databases. Please check to make sure that your database server is running and that the username and passwords found in the database.yml are properly configured and set up on your database server.\n %s", err)
		}
	}
	err := run(exec.Command("buffalo", "db", "create", "-a"))
	if err != nil {
		return errors.Errorf("We encountered an error when trying to create your application's databases. Please check to make sure that your database server is running and that the username and passwords found in the database.yml are properly configured and set up on your database server.\n %s", err)
	}
	return nil
}

func dbMigrateCheck() error {
	err := run(exec.Command("buffalo", "db", "migrate"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to migrate your database:\n%s", err)
	}
	return nil
}

func dbSeedCheck() error {
	cmd := exec.Command("buffalo", "t", "list")
	out, err := cmd.Output()
	if err != nil {
		// no tasks configured, so return
		return nil
	}
	if bytes.Contains(out, []byte("db:seed")) {
		err := run(exec.Command("buffalo", "task", "db:seed"))
		if err != nil {
			return errors.Errorf("We encountered the following error when trying to seed your database:\n%s", err)
		}
	}
	return nil
}

func assetCheck() error {
	if _, err := os.Stat("./yarn.lock"); err == nil {
		return yarnCheck()
	}
	if _, err := os.Stat("./package.json"); err == nil {
		return npmCheck()
	}
	// no asset pipeline, so move on.
	return nil
}

func npmCheck() error {
	err := nodeCheck()
	if err != nil {
		return errors.WithStack(err)
	}
	err = run(exec.Command("npm", "install", "--no-progress"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to install your asset dependencies using npm:\n%s", err)
	}
	return nil
}

func yarnCheck() error {
	err := nodeCheck()
	if err != nil {
		return errors.WithStack(err)
	}
	if _, err := exec.LookPath("yarn"); err != nil {
		err := run(exec.Command("npm", "install", "yarn"))
		if err != nil {
			return errors.Errorf("This application require yarn, and we could not find it installed on your system. We tried to install it for you, but ran into the following error:\n%s", err)
		}
	}
	err = run(exec.Command("yarn", "install"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to install your asset dependencies using yarn:\n%s", err)
	}
	return nil
}

func nodeCheck() error {
	if _, err := exec.LookPath("node"); err != nil {
		return errors.New("this application requires node, and we could not find it installed on your system please install node and try again")
	}
	if _, err := exec.LookPath("npm"); err != nil {
		return errors.New("this application requires npm, and we could not find it installed on your system please install npm and try again")
	}
	return nil
}

func run(cmd *exec.Cmd) error {
	fmt.Printf("--> %s\n", strings.Join(cmd.Args, " "))
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func init() {
	setupCmd.Flags().BoolVarP(&setupOptions.verbose, "verbose", "v", false, "run with verbose output")
	setupCmd.Flags().BoolVarP(&setupOptions.updateGoDeps, "update", "u", false, "run go get -u against the application's Go dependencies")
	setupCmd.Flags().BoolVarP(&setupOptions.dropDatabases, "drop", "d", false, "drop existing databases")

	decorate("setup", setupCmd)
	RootCmd.AddCommand(setupCmd)
}
<commit_msg>make sure yarn uses --no-progress<commit_after>package cmd

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"os/exec"
	"strings"

	"golang.org/x/sync/errgroup"

	"github.com/markbates/deplist"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var setupOptions = struct {
	verbose       bool
	updateGoDeps  bool
	dropDatabases bool
}{}

type setupCheck func() error

var setupCmd = &cobra.Command{
	Use:   "setup",
	Short: "Setups a newly created, or recently checked out application.",
	Long: `Setup runs through checklist to make sure dependencies are setup correcly.

Dependencies (if used):
* Runs "dep ensure" to install required Go dependencies.

Asset Pipeline (if used):
* Runs "npm install" or "yarn install" to install asset dependencies.

Database (if used):
* Runs "buffalo db create -a" to create databases.
* Runs "buffalo db migrate" to run database migrations.
* Runs "buffalo task db:seed" to seed the database (if the task exists).

Tests:
* Runs "buffalo test" to confirm the application's tests are running properly.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		for _, check := range []setupCheck{assetCheck, updateGoDepsCheck, databaseCheck, testCheck} {
			err := check()
			if err != nil {
				return errors.WithStack(err)
			}
		}
		return nil
	},
}

func updateGoDepsCheck() error {
	deps, _ := deplist.List()
	if _, err := os.Stat("Gopkg.toml"); err == nil {
		// use github.com/golang/dep
		args := []string{"ensure"}
		if setupOptions.verbose {
			args = append(args, "-v")
		}
		if setupOptions.updateGoDeps {
			args = append(args, "--update")
		}
		err := run(exec.Command("dep", args...))
		if err != nil {
			return errors.WithStack(err)
		}
		return nil
	}

	// go old school with the installation
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	wg, ctx := errgroup.WithContext(ctx)
	deps, err := deplist.List()
	if err != nil {
		return errors.WithStack(err)
	}
	for dep := range deps {
		args := []string{"get"}
		if setupOptions.verbose {
			args = append(args, "-v")
		}
		if setupOptions.updateGoDeps {
			args = append(args, "-u")
		}
		args = append(args, dep)
		c := exec.Command("go", args...)
		f := func() error {
			return run(c)
		}
		wg.Go(f)
	}
	err = wg.Wait()
	if err != nil {
		return errors.Errorf("We encountered the following error trying to install and update the dependencies for this application:\n%s", err)
	}
	return nil
}

func testCheck() error {
	err := run(exec.Command("buffalo", "test"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to run your applications tests:\n%s", err)
	}
	return nil
}

func databaseCheck() error {
	if _, err := os.Stat("./database.yml"); err != nil {
		// no database.yml, so move on
		return nil
	}
	for _, check := range []setupCheck{dbCreateCheck, dbMigrateCheck, dbSeedCheck} {
		err := check()
		if err != nil {
			return err
		}
	}
	return nil
}

func dbCreateCheck() error {
	if setupOptions.dropDatabases {
		err := run(exec.Command("buffalo", "db", "drop", "-a"))
		if err != nil {
			return errors.Errorf("We encountered an error when trying to drop your application's databases. Please check to make sure that your database server is running and that the username and passwords found in the database.yml are properly configured and set up on your database server.\n %s", err)
		}
	}
	err := run(exec.Command("buffalo", "db", "create", "-a"))
	if err != nil {
		return errors.Errorf("We encountered an error when trying to create your application's databases. Please check to make sure that your database server is running and that the username and passwords found in the database.yml are properly configured and set up on your database server.\n %s", err)
	}
	return nil
}

func dbMigrateCheck() error {
	err := run(exec.Command("buffalo", "db", "migrate"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to migrate your database:\n%s", err)
	}
	return nil
}

func dbSeedCheck() error {
	cmd := exec.Command("buffalo", "t", "list")
	out, err := cmd.Output()
	if err != nil {
		// no tasks configured, so return
		return nil
	}
	if bytes.Contains(out, []byte("db:seed")) {
		err := run(exec.Command("buffalo", "task", "db:seed"))
		if err != nil {
			return errors.Errorf("We encountered the following error when trying to seed your database:\n%s", err)
		}
	}
	return nil
}

func assetCheck() error {
	if _, err := os.Stat("./yarn.lock"); err == nil {
		return yarnCheck()
	}
	if _, err := os.Stat("./package.json"); err == nil {
		return npmCheck()
	}
	// no asset pipeline, so move on.
	return nil
}

func npmCheck() error {
	err := nodeCheck()
	if err != nil {
		return errors.WithStack(err)
	}
	err = run(exec.Command("npm", "install", "--no-progress"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to install your asset dependencies using npm:\n%s", err)
	}
	return nil
}

func yarnCheck() error {
	err := nodeCheck()
	if err != nil {
		return errors.WithStack(err)
	}
	if _, err := exec.LookPath("yarn"); err != nil {
		err := run(exec.Command("npm", "install", "yarn"))
		if err != nil {
			return errors.Errorf("This application require yarn, and we could not find it installed on your system. We tried to install it for you, but ran into the following error:\n%s", err)
		}
	}
	err = run(exec.Command("yarn", "--no-progress", "install"))
	if err != nil {
		return errors.Errorf("We encountered the following error when trying to install your asset dependencies using yarn:\n%s", err)
	}
	return nil
}

func nodeCheck() error {
	if _, err := exec.LookPath("node"); err != nil {
		return errors.New("this application requires node, and we could not find it installed on your system please install node and try again")
	}
	if _, err := exec.LookPath("npm"); err != nil {
		return errors.New("this application requires npm, and we could not find it installed on your system please install npm and try again")
	}
	return nil
}

func run(cmd *exec.Cmd) error {
	fmt.Printf("--> %s\n", strings.Join(cmd.Args, " "))
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func init() {
	setupCmd.Flags().BoolVarP(&setupOptions.verbose, "verbose", "v", false, "run with verbose output")
	setupCmd.Flags().BoolVarP(&setupOptions.updateGoDeps, "update", "u", false, "run go get -u against the application's Go dependencies")
	setupCmd.Flags().BoolVarP(&setupOptions.dropDatabases, "drop", "d", false, "drop existing databases")

	decorate("setup", setupCmd)
	RootCmd.AddCommand(setupCmd)
}
<|endoftext|>


<commit_before>package e2e

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/client/conditions"
	"k8s.io/kubernetes/test/e2e/framework"
	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)

// getDaemonSetLabelSelector returns labels of daemonset given name and namespace dynamically,
// needed since labels are not same for helm and non-helm deployments.
func getDaemonSetLabelSelector(f *framework.Framework, ns, daemonSetName string) (string, error) {
	ds, err := f.ClientSet.AppsV1().DaemonSets(ns).Get(context.TODO(), daemonSetName, metav1.GetOptions{})
	if err != nil {
		e2elog.Logf("Error getting daemonsets with name %s in namespace %s", daemonSetName, ns)
		return "", err
	}
	s, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector)
	if err != nil {
		e2elog.Logf("Error parsing %s daemonset selector in namespace %s", daemonSetName, ns)
		return "", err
	}
	e2elog.Logf("LabelSelector for %s daemonsets in namespace %s: %s", daemonSetName, ns, s.String())
	return s.String(), nil
}

func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error {
	timeout := time.Duration(t) * time.Minute
	start := time.Now()
	e2elog.Logf("Waiting up to %v for all daemonsets in namespace '%s' to start", timeout, ns)

	return wait.PollImmediate(poll, timeout, func() (bool, error) {
		ds, err := c.AppsV1().DaemonSets(ns).Get(context.TODO(), name, metav1.GetOptions{})
		if err != nil {
			e2elog.Logf("Error getting daemonsets in namespace: '%s': %v", ns, err)
			if strings.Contains(err.Error(), "not found") {
				return false, nil
			}
			if isRetryableAPIError(err) {
				return false, nil
			}
			return false, err
		}
		dNum := ds.Status.DesiredNumberScheduled
		ready := ds.Status.NumberReady
		e2elog.Logf("%d / %d pods ready in namespace '%s' in daemonset '%s' (%d seconds elapsed)", ready, dNum, ns, ds.ObjectMeta.Name, int(time.Since(start).Seconds()))
		if ready != dNum {
			return false, nil
		}

		return true, nil
	})
}

// Waits for the deployment to complete.

func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) error {
	var (
		deployment *appsv1.Deployment
		reason     string
		err        error
	)
	timeout := time.Duration(t) * time.Minute
	err = wait.PollImmediate(poll, timeout, func() (bool, error) {
		deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{})
		if err != nil {
			// a StatusError is not marked as 'retryable', but we want to retry anyway
			if isRetryableAPIError(err) || strings.Contains(err.Error(), "etcdserver: request timed out") {
				// hide API-server timeouts, so that PollImmediate() retries
				e2elog.Logf("deployment error: %v", err)
				return false, nil
			}
			return false, err
		}

		// TODO need to check rolling update

		// When the deployment status and its underlying resources reach the
		// desired state, we're done
		if deployment.Status.Replicas == deployment.Status.ReadyReplicas {
			return true, nil
		}
		e2elog.Logf("deployment status: expected replica count %d running replica count %d", deployment.Status.Replicas, deployment.Status.ReadyReplicas)
		reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String())
		return false, nil
	})

	if errors.Is(err, wait.ErrWaitTimeout) {
		err = fmt.Errorf("%s", reason)
	}
	if err != nil {
		return fmt.Errorf("error waiting for deployment %q status to match expectation: %w", name, err)
	}
	return nil
}

func getCommandInPodOpts(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (framework.ExecOptions, error) {
	cmd := []string{"/bin/sh", "-c", c}
	podList, err := f.PodClientNS(ns).List(context.TODO(), *opt)
	framework.ExpectNoError(err)
	if len(podList.Items) == 0 {
		return framework.ExecOptions{}, errors.New("podlist is empty")
	}
	if err != nil {
		return framework.ExecOptions{}, err
	}
	return framework.ExecOptions{
		Command:            cmd,
		PodName:            podList.Items[0].Name,
		Namespace:          ns,
		ContainerName:      podList.Items[0].Spec.Containers[0].Name,
		Stdin:              nil,
		CaptureStdout:      true,
		CaptureStderr:      true,
		PreserveWhitespace: true,
	}, nil
}

func execCommandInPod(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string, error) {
	podOpt, err := getCommandInPodOpts(f, c, ns, opt)
	if err != nil {
		return "", "", err
	}
	stdOut, stdErr, err := f.ExecWithOptions(podOpt)
	if stdErr != "" {
		e2elog.Logf("stdErr occurred: %v", stdErr)
	}
	return stdOut, stdErr, err
}

func execCommandInToolBoxPod(f *framework.Framework, c, ns string) (string, string, error) {
	opt := &metav1.ListOptions{
		LabelSelector: rookToolBoxPodLabel,
	}
	podOpt, err := getCommandInPodOpts(f, c, ns, opt)
	if err != nil {
		return "", "", err
	}
	stdOut, stdErr, err := f.ExecWithOptions(podOpt)
	if stdErr != "" {
		e2elog.Logf("stdErr occurred: %v", stdErr)
	}
	return stdOut, stdErr, err
}

func execCommandInPodAndAllowFail(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string) {
	podOpt, err := getCommandInPodOpts(f, c, ns, opt)
	if err != nil {
		return "", err.Error()
	}
	stdOut, stdErr, err := f.ExecWithOptions(podOpt)
	if err != nil {
		e2elog.Logf("command %s failed: %v", c, err)
	}
	return stdOut, stdErr
}

func loadApp(path string) (*v1.Pod, error) {
	app := v1.Pod{}
	err := unmarshal(path, &app)
	if err != nil {
		return nil, err
	}
	for i := range app.Spec.Containers {
		app.Spec.Containers[i].ImagePullPolicy = v1.PullIfNotPresent
	}
	return &app, nil
}

func createApp(c kubernetes.Interface, app *v1.Pod, timeout int) error {
	_, err := c.CoreV1().Pods(app.Namespace).Create(context.TODO(), app, metav1.CreateOptions{})
	if err != nil {
		return err
	}
	return waitForPodInRunningState(app.Name, app.Namespace, c, timeout)
}

func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int) error {
	timeout := time.Duration(t) * time.Minute
	start := time.Now()
	e2elog.Logf("Waiting up to %v to be in Running state", name)
	return wait.PollImmediate(poll, timeout, func() (bool, error) {
		pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		switch pod.Status.Phase {
		case v1.PodRunning:
			return true, nil
		case v1.PodFailed, v1.PodSucceeded:
			return false, conditions.ErrPodCompleted
		}
		e2elog.Logf("%s app  is in %s phase expected to be in Running  state (%d seconds elapsed)", name, pod.Status.Phase, int(time.Since(start).Seconds()))
		return false, nil
	})
}

func deletePod(name, ns string, c kubernetes.Interface, t int) error {
	timeout := time.Duration(t) * time.Minute
	err := c.CoreV1().Pods(ns).Delete(context.TODO(), name, metav1.DeleteOptions{})
	if err != nil {
		return err
	}
	start := time.Now()
	e2elog.Logf("Waiting for pod %v to be deleted", name)
	return wait.PollImmediate(poll, timeout, func() (bool, error) {
		_, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})

		if apierrs.IsNotFound(err) {
			return true, nil
		}
		e2elog.Logf("%s app  to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
		if err != nil {
			return false, err
		}
		return false, nil
	})
}

func deletePodWithLabel(label, ns string, skipNotFound bool) error {
	_, err := framework.RunKubectl(ns, "delete", "po", "-l", label, fmt.Sprintf("--ignore-not-found=%t", skipNotFound))
	if err != nil {
		e2elog.Logf("failed to delete pod %v", err)
	}
	return err
}

// calculateSHA512sum returns the sha512sum of a file inside a pod.
func calculateSHA512sum(f *framework.Framework, app *v1.Pod, filePath string, opt *metav1.ListOptions) (string, error) {
	cmd := fmt.Sprintf("sha512sum %s", filePath)
	sha512sumOut, stdErr, err := execCommandInPod(f, cmd, app.Namespace, opt)
	if err != nil {
		return "", err
	}
	if stdErr != "" {
		return "", fmt.Errorf("error: sha512sum could not be calculated %v", stdErr)
	}
	// extract checksum from sha512sum output.
	checkSum := strings.Split(sha512sumOut, "")[0]
	e2elog.Logf("Calculated checksum  %s", checkSum)
	return checkSum, nil
}
<commit_msg>e2e: add listPods()<commit_after>package e2e

import (
	"context"
	"errors"
	"fmt"
	"strings"
	"time"

	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	apierrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/kubernetes/pkg/client/conditions"
	"k8s.io/kubernetes/test/e2e/framework"
	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
)

// getDaemonSetLabelSelector returns labels of daemonset given name and namespace dynamically,
// needed since labels are not same for helm and non-helm deployments.
func getDaemonSetLabelSelector(f *framework.Framework, ns, daemonSetName string) (string, error) {
	ds, err := f.ClientSet.AppsV1().DaemonSets(ns).Get(context.TODO(), daemonSetName, metav1.GetOptions{})
	if err != nil {
		e2elog.Logf("Error getting daemonsets with name %s in namespace %s", daemonSetName, ns)
		return "", err
	}
	s, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector)
	if err != nil {
		e2elog.Logf("Error parsing %s daemonset selector in namespace %s", daemonSetName, ns)
		return "", err
	}
	e2elog.Logf("LabelSelector for %s daemonsets in namespace %s: %s", daemonSetName, ns, s.String())
	return s.String(), nil
}

func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error {
	timeout := time.Duration(t) * time.Minute
	start := time.Now()
	e2elog.Logf("Waiting up to %v for all daemonsets in namespace '%s' to start", timeout, ns)

	return wait.PollImmediate(poll, timeout, func() (bool, error) {
		ds, err := c.AppsV1().DaemonSets(ns).Get(context.TODO(), name, metav1.GetOptions{})
		if err != nil {
			e2elog.Logf("Error getting daemonsets in namespace: '%s': %v", ns, err)
			if strings.Contains(err.Error(), "not found") {
				return false, nil
			}
			if isRetryableAPIError(err) {
				return false, nil
			}
			return false, err
		}
		dNum := ds.Status.DesiredNumberScheduled
		ready := ds.Status.NumberReady
		e2elog.Logf("%d / %d pods ready in namespace '%s' in daemonset '%s' (%d seconds elapsed)", ready, dNum, ns, ds.ObjectMeta.Name, int(time.Since(start).Seconds()))
		if ready != dNum {
			return false, nil
		}

		return true, nil
	})
}

// Waits for the deployment to complete.

func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) error {
	var (
		deployment *appsv1.Deployment
		reason     string
		err        error
	)
	timeout := time.Duration(t) * time.Minute
	err = wait.PollImmediate(poll, timeout, func() (bool, error) {
		deployment, err = c.AppsV1().Deployments(ns).Get(context.TODO(), name, metav1.GetOptions{})
		if err != nil {
			// a StatusError is not marked as 'retryable', but we want to retry anyway
			if isRetryableAPIError(err) || strings.Contains(err.Error(), "etcdserver: request timed out") {
				// hide API-server timeouts, so that PollImmediate() retries
				e2elog.Logf("deployment error: %v", err)
				return false, nil
			}
			return false, err
		}

		// TODO need to check rolling update

		// When the deployment status and its underlying resources reach the
		// desired state, we're done
		if deployment.Status.Replicas == deployment.Status.ReadyReplicas {
			return true, nil
		}
		e2elog.Logf("deployment status: expected replica count %d running replica count %d", deployment.Status.Replicas, deployment.Status.ReadyReplicas)
		reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String())
		return false, nil
	})

	if errors.Is(err, wait.ErrWaitTimeout) {
		err = fmt.Errorf("%s", reason)
	}
	if err != nil {
		return fmt.Errorf("error waiting for deployment %q status to match expectation: %w", name, err)
	}
	return nil
}

func getCommandInPodOpts(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (framework.ExecOptions, error) {
	cmd := []string{"/bin/sh", "-c", c}
	pods, err := listPods(f, ns, opt)
	if err != nil {
		return framework.ExecOptions{}, err
	}
	return framework.ExecOptions{
		Command:            cmd,
		PodName:            pods[0].Name,
		Namespace:          ns,
		ContainerName:      pods[0].Spec.Containers[0].Name,
		Stdin:              nil,
		CaptureStdout:      true,
		CaptureStderr:      true,
		PreserveWhitespace: true,
	}, nil
}

// listPods returns slice of pods matching given ListOptions and namespace.
func listPods(f *framework.Framework, ns string, opt *metav1.ListOptions) ([]v1.Pod, error) {
	podList, err := f.PodClientNS(ns).List(context.TODO(), *opt)
	if len(podList.Items) == 0 {
		return podList.Items, fmt.Errorf("podlist for label '%s' in namespace %s is empty", opt.LabelSelector, ns)
	}
	return podList.Items, err
}

func execCommandInPod(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string, error) {
	podOpt, err := getCommandInPodOpts(f, c, ns, opt)
	if err != nil {
		return "", "", err
	}
	stdOut, stdErr, err := f.ExecWithOptions(podOpt)
	if stdErr != "" {
		e2elog.Logf("stdErr occurred: %v", stdErr)
	}
	return stdOut, stdErr, err
}

func execCommandInToolBoxPod(f *framework.Framework, c, ns string) (string, string, error) {
	opt := &metav1.ListOptions{
		LabelSelector: rookToolBoxPodLabel,
	}
	podOpt, err := getCommandInPodOpts(f, c, ns, opt)
	if err != nil {
		return "", "", err
	}
	stdOut, stdErr, err := f.ExecWithOptions(podOpt)
	if stdErr != "" {
		e2elog.Logf("stdErr occurred: %v", stdErr)
	}
	return stdOut, stdErr, err
}

func execCommandInPodAndAllowFail(f *framework.Framework, c, ns string, opt *metav1.ListOptions) (string, string) {
	podOpt, err := getCommandInPodOpts(f, c, ns, opt)
	if err != nil {
		return "", err.Error()
	}
	stdOut, stdErr, err := f.ExecWithOptions(podOpt)
	if err != nil {
		e2elog.Logf("command %s failed: %v", c, err)
	}
	return stdOut, stdErr
}

func loadApp(path string) (*v1.Pod, error) {
	app := v1.Pod{}
	err := unmarshal(path, &app)
	if err != nil {
		return nil, err
	}
	for i := range app.Spec.Containers {
		app.Spec.Containers[i].ImagePullPolicy = v1.PullIfNotPresent
	}
	return &app, nil
}

func createApp(c kubernetes.Interface, app *v1.Pod, timeout int) error {
	_, err := c.CoreV1().Pods(app.Namespace).Create(context.TODO(), app, metav1.CreateOptions{})
	if err != nil {
		return err
	}
	return waitForPodInRunningState(app.Name, app.Namespace, c, timeout)
}

func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int) error {
	timeout := time.Duration(t) * time.Minute
	start := time.Now()
	e2elog.Logf("Waiting up to %v to be in Running state", name)
	return wait.PollImmediate(poll, timeout, func() (bool, error) {
		pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
		if err != nil {
			return false, err
		}
		switch pod.Status.Phase {
		case v1.PodRunning:
			return true, nil
		case v1.PodFailed, v1.PodSucceeded:
			return false, conditions.ErrPodCompleted
		}
		e2elog.Logf("%s app  is in %s phase expected to be in Running  state (%d seconds elapsed)", name, pod.Status.Phase, int(time.Since(start).Seconds()))
		return false, nil
	})
}

func deletePod(name, ns string, c kubernetes.Interface, t int) error {
	timeout := time.Duration(t) * time.Minute
	err := c.CoreV1().Pods(ns).Delete(context.TODO(), name, metav1.DeleteOptions{})
	if err != nil {
		return err
	}
	start := time.Now()
	e2elog.Logf("Waiting for pod %v to be deleted", name)
	return wait.PollImmediate(poll, timeout, func() (bool, error) {
		_, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})

		if apierrs.IsNotFound(err) {
			return true, nil
		}
		e2elog.Logf("%s app  to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
		if err != nil {
			return false, err
		}
		return false, nil
	})
}

func deletePodWithLabel(label, ns string, skipNotFound bool) error {
	_, err := framework.RunKubectl(ns, "delete", "po", "-l", label, fmt.Sprintf("--ignore-not-found=%t", skipNotFound))
	if err != nil {
		e2elog.Logf("failed to delete pod %v", err)
	}
	return err
}

// calculateSHA512sum returns the sha512sum of a file inside a pod.
func calculateSHA512sum(f *framework.Framework, app *v1.Pod, filePath string, opt *metav1.ListOptions) (string, error) {
	cmd := fmt.Sprintf("sha512sum %s", filePath)
	sha512sumOut, stdErr, err := execCommandInPod(f, cmd, app.Namespace, opt)
	if err != nil {
		return "", err
	}
	if stdErr != "" {
		return "", fmt.Errorf("error: sha512sum could not be calculated %v", stdErr)
	}
	// extract checksum from sha512sum output.
	checkSum := strings.Split(sha512sumOut, "")[0]
	e2elog.Logf("Calculated checksum  %s", checkSum)
	return checkSum, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/abesto/easyssh/discoverers"
	"github.com/abesto/easyssh/executors"
	"github.com/abesto/easyssh/filters"
	"github.com/abesto/easyssh/interfaces"
	"github.com/abesto/easyssh/target"
	"github.com/abesto/easyssh/util"
	"github.com/alexcesaro/log/stdlog"
)

func main() {
	var (
		discovererDefinition string
		discoverer           interfaces.Discoverer
		executorDefinition   string
		executor             interfaces.Executor
		user                 string
		filterDefinition     string
		filter               interfaces.TargetFilter
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s [options] target-definition [command]

Where
  target-definition is the input to the discoverer(s) defined with -d
  command, if provided, will be run on the targets

Ideally a single alias should cover all your use-cases. For example:
  smartssh_executor='(if-command (ssh-exec-parallel) (if-one-target (ssh-login) (tmux-cssh)))'
  smartssh_discoverer='(first-matching (knife) (comma-separated))'
  smartssh_filter='(list (ec2-instance-id us-east-1) (ec2-instance-id us-west-1))'
  alias s="%s -e='$smartssh_executor' -d='$smartssh_discoverer' -f='$smartssh_filter'"

Configuration details:
  open https://github.com/abesto/smartssh/blob/master/README.md#configuration

Options:
`, os.Args[0], os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&user, "l", "",
		"Specifies the user to log in as on the remote machine.")
	flag.StringVar(&discovererDefinition, "d", "(comma-separated)",
		fmt.Sprintf("Discoverer definition. Supported discoverers: %s", strings.Join(discoverers.SupportedDiscovererNames(), ", ")))
	flag.StringVar(&executorDefinition, "e", "(ssh-login)",
		fmt.Sprintf("Executor definition. Supported executors: %s", strings.Join(executors.SupportedExecutorNames(), ", ")))
	flag.StringVar(&filterDefinition, "f", "(id)",
		fmt.Sprintf("Filter definition. Supported filters: %s", strings.Join(filters.SupportedFilterNames(), ", ")))
	verbose := flag.Bool("v", false, "Verbose output (alias of '-log debug')")
	flag.Parse()

	if *verbose {
		flag.Set("log", "debug")
	}
	util.Logger = stdlog.GetFromFlags()
	logger := util.Logger

	if flag.NArg() == 0 {
		logger.Critical("Required argument for target host lookup missing")
		flag.Usage()
		return
	}

	defer func() {
		if err := recover(); err != nil {
			// discoverer, executor and filter are created in this order
			// if at least one of them is nil, then the creation of the first one that is nil has generated the error.
			if discoverer == nil {
				util.Logger.Critical("Failed to create discoverer")
			} else if executor == nil {
				util.Logger.Critical("Failed to create executor")
			} else if filter == nil {
				util.Logger.Critical("Failed to create filter")
			}
			switch err.(type) {
			case string:
				util.Logger.Critical(err.(string))
				os.Exit(1)
			default:
				panic(err)
			}
		}
	}()

	discoverer = discoverers.Make(discovererDefinition)
	executor = executors.Make(executorDefinition)
	filter = filters.Make(filterDefinition)

	targets := []target.Target{}
	for _, host := range discoverer.Discover(flag.Arg(0)) {
		targets = append(targets, target.Target{Host: host, User: user})
	}
	if len(targets) == 0 {
		util.Panicf("No targets found")
	}

	logger.Debugf("Targets before filters: %s", targets)
	targets = filter.Filter(targets)
	logger.Infof("Targets: %s", targets)

	command := flag.Args()[1:]
	executor.Exec(targets, command)
}
<commit_msg>Add flag -version, populated by goxc via -ldflags<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/abesto/easyssh/discoverers"
	"github.com/abesto/easyssh/executors"
	"github.com/abesto/easyssh/filters"
	"github.com/abesto/easyssh/interfaces"
	"github.com/abesto/easyssh/target"
	"github.com/abesto/easyssh/util"
	"github.com/alexcesaro/log/stdlog"
)

var VERSION = "HEAD"
var BUILD_DATE = "LOCAL"

func main() {
	var (
		discovererDefinition string
		discoverer           interfaces.Discoverer
		executorDefinition   string
		executor             interfaces.Executor
		user                 string
		filterDefinition     string
		filter               interfaces.TargetFilter
	)

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `Usage: %s [options] target-definition [command]

Where
  target-definition is the input to the discoverer(s) defined with -d
  command, if provided, will be run on the targets

Ideally a single alias should cover all your use-cases. For example:
  smartssh_executor='(if-command (ssh-exec-parallel) (if-one-target (ssh-login) (tmux-cssh)))'
  smartssh_discoverer='(first-matching (knife) (comma-separated))'
  smartssh_filter='(list (ec2-instance-id us-east-1) (ec2-instance-id us-west-1))'
  alias s="%s -e='$smartssh_executor' -d='$smartssh_discoverer' -f='$smartssh_filter'"

Configuration details:
  open https://github.com/abesto/smartssh/blob/master/README.md#configuration

Options:
`, os.Args[0], os.Args[0])
		flag.PrintDefaults()
	}

	flag.StringVar(&user, "l", "",
		"Specifies the user to log in as on the remote machine.")
	flag.StringVar(&discovererDefinition, "d", "(comma-separated)",
		fmt.Sprintf("Discoverer definition. Supported discoverers: %s", strings.Join(discoverers.SupportedDiscovererNames(), ", ")))
	flag.StringVar(&executorDefinition, "e", "(ssh-login)",
		fmt.Sprintf("Executor definition. Supported executors: %s", strings.Join(executors.SupportedExecutorNames(), ", ")))
	flag.StringVar(&filterDefinition, "f", "(id)",
		fmt.Sprintf("Filter definition. Supported filters: %s", strings.Join(filters.SupportedFilterNames(), ", ")))
	verbose := flag.Bool("v", false, "Verbose output (alias of '-log debug')")
	versionRequested := flag.Bool("version", false, "Display the version number and exit")
	flag.Parse()

	if *versionRequested {
		fmt.Printf("easyssh version %s build %s\n", VERSION, BUILD_DATE)
		return
	}

	if *verbose {
		flag.Set("log", "debug")
	}
	util.Logger = stdlog.GetFromFlags()
	logger := util.Logger

	if flag.NArg() == 0 {
		logger.Critical("Required argument for target host lookup missing")
		flag.Usage()
		return
	}

	defer func() {
		if err := recover(); err != nil {
			// discoverer, executor and filter are created in this order
			// if at least one of them is nil, then the creation of the first one that is nil has generated the error.
			if discoverer == nil {
				util.Logger.Critical("Failed to create discoverer")
			} else if executor == nil {
				util.Logger.Critical("Failed to create executor")
			} else if filter == nil {
				util.Logger.Critical("Failed to create filter")
			}
			switch err.(type) {
			case string:
				util.Logger.Critical(err.(string))
				os.Exit(1)
			default:
				panic(err)
			}
		}
	}()

	discoverer = discoverers.Make(discovererDefinition)
	executor = executors.Make(executorDefinition)
	filter = filters.Make(filterDefinition)

	targets := []target.Target{}
	for _, host := range discoverer.Discover(flag.Arg(0)) {
		targets = append(targets, target.Target{Host: host, User: user})
	}
	if len(targets) == 0 {
		util.Panicf("No targets found")
	}

	logger.Debugf("Targets before filters: %s", targets)
	targets = filter.Filter(targets)
	logger.Infof("Targets: %s", targets)

	command := flag.Args()[1:]
	executor.Exec(targets, command)
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/zeromq/goczmq"
	"github.internal.digitalocean.com/digitalocean/doge.git/log"
)

func main() {

	go func() {
		publisher := goczmq.NewSock(goczmq.PUB)
		publisher.Bind("tcp://*:6665")
		for {
			publisher.SendFrame([]byte("157:6666"), 0)
			time.Sleep(1 * time.Second)
		}
	}()

	router := goczmq.NewSock(goczmq.ROUTER)
	router.Bind("tcp://*:6666")

	for {
		msg, err := router.RecvMessage()
		if err != nil {
			log.Errorf("error: %s\n", err)
		}

		if len(msg) == 2 {
			if string(msg[1]) == "Hello" {
				msg[1] = []byte("hello from taotetek")
			} else {
				msg[1] = []byte("Error")
			}
			err = router.SendMessage(msg)
			if err != nil {
				log.Errorf("error: %s\n", err)
			}
		}
	}
}
<commit_msg>go server tweaks<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/zeromq/goczmq"
	"github.internal.digitalocean.com/digitalocean/doge.git/log"
)

func main() {

	go func() {
		publisher := goczmq.NewSock(goczmq.PUB)
		publisher.Bind("tcp://*:6665")
		for {
			publisher.SendFrame([]byte("157:6666"), 0)
			time.Sleep(1 * time.Second)
		}
	}()

	router := goczmq.NewSock(goczmq.ROUTER)
	router.Bind("tcp://*:6666")

	for {
		msg, err := router.RecvMessage()
		if err != nil {
			log.Errorf("error: %s\n", err)
		}

		if len(msg) == 2 {
			if string(msg[1]) == "Hello" {
				fmt.Println("someone said hello")
				msg[1] = []byte("hello from taotetek")
			} else {
				msg[1] = []byte("Error")
			}
			err = router.SendMessage(msg)
			if err != nil {
				log.Errorf("error: %s\n", err)
			}
		}
	}
}
<|endoftext|>