<commit_before>package restpc

import (
	"runtime"
	"strings"
)

type TracebackRecord interface {
	File() string
	Function() string
	FunctionLocal() string
	Line() int
}

type Traceback interface {
	Callers() []uintptr
	Records() []TracebackRecord
	MapRecords() []map[string]interface{}
}

type tracebackRecordImp struct {
	file     string
	function string
	line     int
}

func (tr *tracebackRecordImp) File() string {
	return tr.file
}
func (tr *tracebackRecordImp) Function() string {
	return tr.function
}
func (tr *tracebackRecordImp) FunctionLocal() string {
	full := tr.function
	if full == "" {
		return ""
	}
	parts := strings.Split(full, ".")
	return parts[len(parts)-1]
}
func (tr *tracebackRecordImp) Line() int {
	return tr.line
}

type tracebackImp struct {
	callers []uintptr
}

func (t *tracebackImp) Callers() []uintptr {
	return t.callers
}

func (t *tracebackImp) Records() []TracebackRecord {
	frames := runtime.CallersFrames(t.callers)
	records := []TracebackRecord{}
	processFrame := func(frame runtime.Frame) bool {
		if frame.Func == nil {
			return true
		}
		records = append(records, &tracebackRecordImp{
			file:     frame.File,
			function: frame.Function,
			line:     frame.Line,
		})
		_, isHandler := handlers[frame.Function]
		if isHandler {
			return false
		}
		return true
	}
	for {
		frame, more := frames.Next()
		if !processFrame(frame) || !more {
			break
		}
	}
	return records
}

func (t *tracebackImp) MapRecords() []map[string]interface{} {
	records := t.Records()
	mapRecords := make([]map[string]interface{}, len(records))
	for index, record := range records {
		mapRecords[index] = map[string]interface{}{
			"file":          record.File(),
			"function":      record.Function(),
			"functionLocal": record.FunctionLocal(),
			"line":          record.Line(),
		}
	}
	return mapRecords
}
<commit_msg>add rpcErr.Traceback().HandlerName()<commit_after>package restpc

import (
	"runtime"
	"strings"
)

type TracebackRecord interface {
	File() string
	Function() string
	FunctionLocal() string
	Line() int
}

type Traceback interface {
	Callers() []uintptr
	Records() []TracebackRecord
	MapRecords() []map[string]interface{}
	HandlerName() string
}

type tracebackRecordImp struct {
	file     string
	function string
	line     int
}

func (tr *tracebackRecordImp) File() string {
	return tr.file
}
func (tr *tracebackRecordImp) Function() string {
	return tr.function
}
func (tr *tracebackRecordImp) FunctionLocal() string {
	full := tr.function
	if full == "" {
		return ""
	}
	parts := strings.Split(full, ".")
	return parts[len(parts)-1]
}
func (tr *tracebackRecordImp) Line() int {
	return tr.line
}

type tracebackImp struct {
	callers     []uintptr         // must set initially
	records     []TracebackRecord // will be set on-demand
	handlerName string            // will be set on-demand
}

func (t *tracebackImp) Callers() []uintptr {
	return t.callers
}

func (t *tracebackImp) Records() []TracebackRecord {
	if t.records != nil {
		return t.records
	}
	frames := runtime.CallersFrames(t.callers)
	records := []TracebackRecord{}
	processFrame := func(frame runtime.Frame) bool {
		if frame.Func == nil {
			return true
		}
		frameRecord := &tracebackRecordImp{
			file:     frame.File,
			function: frame.Function,
			line:     frame.Line,
		}
		records = append(records, frameRecord)
		_, isHandler := handlers[frame.Function]
		if isHandler {
			t.handlerName = frameRecord.FunctionLocal()
			return false
		}
		return true
	}
	for {
		frame, more := frames.Next()
		if !processFrame(frame) || !more {
			break
		}
	}
	t.records = records
	return records
}

func (t *tracebackImp) MapRecords() []map[string]interface{} {
	records := t.Records()
	mapRecords := make([]map[string]interface{}, len(records))
	for index, record := range records {
		mapRecords[index] = map[string]interface{}{
			"file":          record.File(),
			"function":      record.Function(),
			"functionLocal": record.FunctionLocal(),
			"line":          record.Line(),
		}
	}
	return mapRecords
}

func (t *tracebackImp) HandlerName() string {
	if t.handlerName != "" {
		return t.handlerName
	}
	if t.records == nil {
		t.Records()
		return t.handlerName
	}
	return ""
}
<|endoftext|>


<commit_before>package pivnet_test

import (
	"encoding/json"
	"errors"
	"fmt"
	"math/rand"
	"net/http"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
	"github.com/pivotal-cf-experimental/pivnet-resource/pivnet"
)

const (
	apiPrefix = "/api/v2"
)

var _ = Describe("PivnetClient", func() {
	var (
		server     *ghttp.Server
		client     pivnet.Client
		token      string
		apiAddress string
	)

	BeforeEach(func() {
		server = ghttp.NewServer()
		apiAddress = server.URL() + apiPrefix
		token = "my-auth-token"
		client = pivnet.NewClient(apiAddress, token)
	})

	AfterEach(func() {
		server.Close()
	})

	It("has authenticated headers for each request", func() {
		response := fmt.Sprintf(`{"releases": [{"version": "1234"}]}`)

		server.AppendHandlers(
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", apiPrefix+"/products/my-product-id/releases"),
				ghttp.VerifyHeaderKV("Authorization", fmt.Sprintf("Token %s", token)),
				ghttp.RespondWith(http.StatusOK, response),
			),
		)

		_, err := client.ProductVersions("my-product-id")
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("Get Release", func() {
		It("returns the release based on the name and version", func() {
			response := `{"releases": [{"id": 3, "version": "3.2.1", "_links": {"product_files": {"href":"https://banana.org/cookies/download"}}}]}`

			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases"),
					ghttp.RespondWith(http.StatusOK, response),
				),
			)

			release, err := client.GetRelease("banana", "3.2.1")
			Expect(err).NotTo(HaveOccurred())
			Expect(release.Links.ProductFiles["href"]).To(Equal("https://banana.org/cookies/download"))
		})

		Context("when the requested version is not available but the request is successful", func() {
			It("returns an error", func() {
				response := `{"releases": [{"id": 3, "version": "3.2.1"}]}`

				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases"),
						ghttp.RespondWith(http.StatusOK, response),
					),
				)

				_, err := client.GetRelease("banana", "1.0.0")
				Expect(err).To(MatchError(errors.New("The requested version: 1.0.0 - could not be found")))
			})
		})

		Context("when the server responds with a non-2XX status code", func() {
			It("returns an error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases"),
						ghttp.RespondWith(http.StatusTeapot, nil),
					),
				)

				_, err := client.GetRelease("banana", "1.0.0")
				Expect(err).To(MatchError(errors.New(
					"Pivnet returned status code: 418 for the request - expected 200")))
			})
		})
	})

	Describe("Get Product Files", func() {
		It("returns the product files for the given release", func() {
			response, err := json.Marshal(pivnet.ProductFiles{[]pivnet.ProductFile{
				{ID: 3, AWSObjectKey: "anything", Links: pivnet.Links{Download: map[string]string{"href": "/products/banana/releases/666/product_files/6/download"}}},
				{ID: 4, AWSObjectKey: "something", Links: pivnet.Links{Download: map[string]string{"href": "/products/banana/releases/666/product_files/8/download"}}},
			},
			})
			Expect(err).NotTo(HaveOccurred())

			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases/666/product_files"),
					ghttp.RespondWith(http.StatusOK, response),
				),
			)

			release := pivnet.Release{
				Links: pivnet.Links{
					ProductFiles: map[string]string{"href": apiAddress + "/products/banana/releases/666/product_files"},
				},
			}

			product, err := client.GetProductFiles(release)
			Expect(err).NotTo(HaveOccurred())
			Expect(product.ProductFiles).To(HaveLen(2))

			Expect(product.ProductFiles[0].AWSObjectKey).To(Equal("anything"))
			Expect(product.ProductFiles[1].AWSObjectKey).To(Equal("something"))

			Expect(product.ProductFiles[0].Links.Download["href"]).To(Equal("/products/banana/releases/666/product_files/6/download"))
			Expect(product.ProductFiles[1].Links.Download["href"]).To(Equal("/products/banana/releases/666/product_files/8/download"))
		})

		Context("when the server responds with a non-2XX status code", func() {
			It("returns an error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases/666/product_files"),
						ghttp.RespondWith(http.StatusTeapot, nil),
					),
				)
				release := pivnet.Release{
					Links: pivnet.Links{
						ProductFiles: map[string]string{"href": apiAddress + "/products/banana/releases/666/product_files"},
					},
				}

				_, err := client.GetProductFiles(release)
				Expect(err).To(MatchError(errors.New(
					"Pivnet returned status code: 418 for the request - expected 200")))
			})
		})
	})

	Describe("Product Versions", func() {
		Context("when parsing the url fails with error", func() {
			It("forwards the error", func() {
				badURL := "%%%"
				client = pivnet.NewClient(badURL, token)

				_, err := client.ProductVersions("some product")
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("%%%"))
			})
		})

		Context("when making the request fails with error", func() {
			It("forwards the error", func() {
				badURL := "https://not-a-real-url.com"
				client = pivnet.NewClient(badURL, token)

				_, err := client.ProductVersions("some product")
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("no such host"))
			})
		})

		Context("when a non-200 comes back from Pivnet", func() {
			It("returns an error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/my-product-id/releases"),
						ghttp.RespondWith(http.StatusNotFound, nil),
					),
				)

				_, err := client.ProductVersions("my-product-id")
				Expect(err).To(HaveOccurred())
				Expect(err).To(MatchError(
					"Pivnet returned status code: 404 for the request - expected 200"))
			})
		})

		Context("when the json unmarshalling fails with error", func() {
			It("forwards the error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/my-product-id/releases"),
						ghttp.RespondWith(http.StatusOK, "%%%"),
					),
				)

				_, err := client.ProductVersions("my-product-id")
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("invalid character"))
			})
		})

		It("gets versions", func() {
			productVersion := "v" + strconv.Itoa(rand.Int())
			response := fmt.Sprintf(
				`{"releases": [{"version": %q}, {"version": %q}]}`,
				productVersion, productVersion,
			)

			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", apiPrefix+"/products/my-product-id/releases"),
					ghttp.RespondWith(http.StatusOK, response),
				),
			)

			versions, err := client.ProductVersions("my-product-id")
			Expect(err).NotTo(HaveOccurred())
			Expect(server.ReceivedRequests()).Should(HaveLen(1))
			Expect(versions).To(HaveLen(2))
			Expect(versions[0]).Should(Equal(productVersion))
			Expect(versions[1]).Should(Equal(productVersion))
		})
	})

	Describe("Create Release", func() {
		var (
			productVersion      string
			createReleaseConfig pivnet.CreateReleaseConfig
		)

		BeforeEach(func() {
			productVersion = "1.2.3.4"

			createReleaseConfig = pivnet.CreateReleaseConfig{
				EulaSlug:       "some_eula",
				ReleaseType:    "Not a real release",
				ProductVersion: productVersion,
				ProductName:    "some-product-name",
			}
		})

		Context("when the config is valid", func() {
			type requestBody struct {
				Release pivnet.Release `json:"release"`
			}

			var (
				expectedReleaseDate string
				expectedRequestBody requestBody

				validResponse string
			)

			BeforeEach(func() {
				expectedReleaseDate = time.Now().Format("2006-01-02")

				expectedRequestBody = requestBody{
					Release: pivnet.Release{
						Availability: "Admins Only",
						OSSCompliant: "confirm",
						ReleaseDate:  expectedReleaseDate,
						ReleaseType:  createReleaseConfig.ReleaseType,
						Eula: pivnet.Eula{
							Slug: createReleaseConfig.EulaSlug,
						},
						Version: createReleaseConfig.ProductVersion,
					},
				}

				validResponse = `{"release": {"id": 3, "version": "1.2.3.4"}}`
			})

			It("creates the release with the minimum required fields", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", apiPrefix+"/products/some-product-name/releases"),
						ghttp.VerifyJSONRepresenting(&expectedRequestBody),
						ghttp.RespondWith(http.StatusCreated, validResponse),
					),
				)

				release, err := client.CreateRelease(createReleaseConfig)
				Expect(err).NotTo(HaveOccurred())
				Expect(release.Version).To(Equal(productVersion))
			})

			Context("when the optional release date is present", func() {
				var (
					releaseDate string
				)

				BeforeEach(func() {
					releaseDate = "2015-12-24"

					expectedRequestBody.Release.ReleaseDate = releaseDate
				})

				It("creates the release with the release date field", func() {
					server.AppendHandlers(
						ghttp.CombineHandlers(
							ghttp.VerifyRequest("POST", apiPrefix+"/products/some-product-name/releases"),
							ghttp.VerifyJSONRepresenting(&expectedRequestBody),
							ghttp.RespondWith(http.StatusCreated, validResponse),
						),
					)

					release, err := client.CreateRelease(createReleaseConfig)
					Expect(err).NotTo(HaveOccurred())
					Expect(release.Version).To(Equal(productVersion))
				})
			})
		})

		Context("when the server responds with a non-201 status code", func() {
			It("returns an error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", apiPrefix+"/products/some-product-name/releases"),
						ghttp.RespondWith(http.StatusTeapot, nil),
					),
				)

				_, err := client.CreateRelease(createReleaseConfig)
				Expect(err).To(MatchError(errors.New(
					"Pivnet returned status code: 418 for the request - expected 201")))
			})
		})
	})
})
<commit_msg>Fix incorrect test for creating release with specified releaseDate.<commit_after>package pivnet_test

import (
	"encoding/json"
	"errors"
	"fmt"
	"math/rand"
	"net/http"
	"strconv"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/ghttp"
	"github.com/pivotal-cf-experimental/pivnet-resource/pivnet"
)

const (
	apiPrefix = "/api/v2"
)

var _ = Describe("PivnetClient", func() {
	var (
		server     *ghttp.Server
		client     pivnet.Client
		token      string
		apiAddress string
	)

	BeforeEach(func() {
		server = ghttp.NewServer()
		apiAddress = server.URL() + apiPrefix
		token = "my-auth-token"
		client = pivnet.NewClient(apiAddress, token)
	})

	AfterEach(func() {
		server.Close()
	})

	It("has authenticated headers for each request", func() {
		response := fmt.Sprintf(`{"releases": [{"version": "1234"}]}`)

		server.AppendHandlers(
			ghttp.CombineHandlers(
				ghttp.VerifyRequest("GET", apiPrefix+"/products/my-product-id/releases"),
				ghttp.VerifyHeaderKV("Authorization", fmt.Sprintf("Token %s", token)),
				ghttp.RespondWith(http.StatusOK, response),
			),
		)

		_, err := client.ProductVersions("my-product-id")
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("Get Release", func() {
		It("returns the release based on the name and version", func() {
			response := `{"releases": [{"id": 3, "version": "3.2.1", "_links": {"product_files": {"href":"https://banana.org/cookies/download"}}}]}`

			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases"),
					ghttp.RespondWith(http.StatusOK, response),
				),
			)

			release, err := client.GetRelease("banana", "3.2.1")
			Expect(err).NotTo(HaveOccurred())
			Expect(release.Links.ProductFiles["href"]).To(Equal("https://banana.org/cookies/download"))
		})

		Context("when the requested version is not available but the request is successful", func() {
			It("returns an error", func() {
				response := `{"releases": [{"id": 3, "version": "3.2.1"}]}`

				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases"),
						ghttp.RespondWith(http.StatusOK, response),
					),
				)

				_, err := client.GetRelease("banana", "1.0.0")
				Expect(err).To(MatchError(errors.New("The requested version: 1.0.0 - could not be found")))
			})
		})

		Context("when the server responds with a non-2XX status code", func() {
			It("returns an error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases"),
						ghttp.RespondWith(http.StatusTeapot, nil),
					),
				)

				_, err := client.GetRelease("banana", "1.0.0")
				Expect(err).To(MatchError(errors.New(
					"Pivnet returned status code: 418 for the request - expected 200")))
			})
		})
	})

	Describe("Get Product Files", func() {
		It("returns the product files for the given release", func() {
			response, err := json.Marshal(pivnet.ProductFiles{[]pivnet.ProductFile{
				{ID: 3, AWSObjectKey: "anything", Links: pivnet.Links{Download: map[string]string{"href": "/products/banana/releases/666/product_files/6/download"}}},
				{ID: 4, AWSObjectKey: "something", Links: pivnet.Links{Download: map[string]string{"href": "/products/banana/releases/666/product_files/8/download"}}},
			},
			})
			Expect(err).NotTo(HaveOccurred())

			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases/666/product_files"),
					ghttp.RespondWith(http.StatusOK, response),
				),
			)

			release := pivnet.Release{
				Links: pivnet.Links{
					ProductFiles: map[string]string{"href": apiAddress + "/products/banana/releases/666/product_files"},
				},
			}

			product, err := client.GetProductFiles(release)
			Expect(err).NotTo(HaveOccurred())
			Expect(product.ProductFiles).To(HaveLen(2))

			Expect(product.ProductFiles[0].AWSObjectKey).To(Equal("anything"))
			Expect(product.ProductFiles[1].AWSObjectKey).To(Equal("something"))

			Expect(product.ProductFiles[0].Links.Download["href"]).To(Equal("/products/banana/releases/666/product_files/6/download"))
			Expect(product.ProductFiles[1].Links.Download["href"]).To(Equal("/products/banana/releases/666/product_files/8/download"))
		})

		Context("when the server responds with a non-2XX status code", func() {
			It("returns an error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/banana/releases/666/product_files"),
						ghttp.RespondWith(http.StatusTeapot, nil),
					),
				)
				release := pivnet.Release{
					Links: pivnet.Links{
						ProductFiles: map[string]string{"href": apiAddress + "/products/banana/releases/666/product_files"},
					},
				}

				_, err := client.GetProductFiles(release)
				Expect(err).To(MatchError(errors.New(
					"Pivnet returned status code: 418 for the request - expected 200")))
			})
		})
	})

	Describe("Product Versions", func() {
		Context("when parsing the url fails with error", func() {
			It("forwards the error", func() {
				badURL := "%%%"
				client = pivnet.NewClient(badURL, token)

				_, err := client.ProductVersions("some product")
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("%%%"))
			})
		})

		Context("when making the request fails with error", func() {
			It("forwards the error", func() {
				badURL := "https://not-a-real-url.com"
				client = pivnet.NewClient(badURL, token)

				_, err := client.ProductVersions("some product")
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("no such host"))
			})
		})

		Context("when a non-200 comes back from Pivnet", func() {
			It("returns an error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/my-product-id/releases"),
						ghttp.RespondWith(http.StatusNotFound, nil),
					),
				)

				_, err := client.ProductVersions("my-product-id")
				Expect(err).To(HaveOccurred())
				Expect(err).To(MatchError(
					"Pivnet returned status code: 404 for the request - expected 200"))
			})
		})

		Context("when the json unmarshalling fails with error", func() {
			It("forwards the error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("GET", apiPrefix+"/products/my-product-id/releases"),
						ghttp.RespondWith(http.StatusOK, "%%%"),
					),
				)

				_, err := client.ProductVersions("my-product-id")
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("invalid character"))
			})
		})

		It("gets versions", func() {
			productVersion := "v" + strconv.Itoa(rand.Int())
			response := fmt.Sprintf(
				`{"releases": [{"version": %q}, {"version": %q}]}`,
				productVersion, productVersion,
			)

			server.AppendHandlers(
				ghttp.CombineHandlers(
					ghttp.VerifyRequest("GET", apiPrefix+"/products/my-product-id/releases"),
					ghttp.RespondWith(http.StatusOK, response),
				),
			)

			versions, err := client.ProductVersions("my-product-id")
			Expect(err).NotTo(HaveOccurred())
			Expect(server.ReceivedRequests()).Should(HaveLen(1))
			Expect(versions).To(HaveLen(2))
			Expect(versions[0]).Should(Equal(productVersion))
			Expect(versions[1]).Should(Equal(productVersion))
		})
	})

	Describe("Create Release", func() {
		var (
			productVersion      string
			createReleaseConfig pivnet.CreateReleaseConfig
		)

		BeforeEach(func() {
			productVersion = "1.2.3.4"

			createReleaseConfig = pivnet.CreateReleaseConfig{
				EulaSlug:       "some_eula",
				ReleaseType:    "Not a real release",
				ProductVersion: productVersion,
				ProductName:    "some-product-name",
			}
		})

		Context("when the config is valid", func() {
			type requestBody struct {
				Release pivnet.Release `json:"release"`
			}

			var (
				expectedReleaseDate string
				expectedRequestBody requestBody

				validResponse string
			)

			BeforeEach(func() {
				expectedReleaseDate = time.Now().Format("2006-01-02")

				expectedRequestBody = requestBody{
					Release: pivnet.Release{
						Availability: "Admins Only",
						OSSCompliant: "confirm",
						ReleaseDate:  expectedReleaseDate,
						ReleaseType:  createReleaseConfig.ReleaseType,
						Eula: pivnet.Eula{
							Slug: createReleaseConfig.EulaSlug,
						},
						Version: createReleaseConfig.ProductVersion,
					},
				}

				validResponse = `{"release": {"id": 3, "version": "1.2.3.4"}}`
			})

			It("creates the release with the minimum required fields", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", apiPrefix+"/products/some-product-name/releases"),
						ghttp.VerifyJSONRepresenting(&expectedRequestBody),
						ghttp.RespondWith(http.StatusCreated, validResponse),
					),
				)

				release, err := client.CreateRelease(createReleaseConfig)
				Expect(err).NotTo(HaveOccurred())
				Expect(release.Version).To(Equal(productVersion))
			})

			Context("when the optional release date is present", func() {
				var (
					releaseDate string
				)

				BeforeEach(func() {
					releaseDate = "2015-12-24"

					createReleaseConfig.ReleaseDate = releaseDate
					expectedRequestBody.Release.ReleaseDate = releaseDate
				})

				It("creates the release with the release date field", func() {
					server.AppendHandlers(
						ghttp.CombineHandlers(
							ghttp.VerifyRequest("POST", apiPrefix+"/products/some-product-name/releases"),
							ghttp.VerifyJSONRepresenting(&expectedRequestBody),
							ghttp.RespondWith(http.StatusCreated, validResponse),
						),
					)

					release, err := client.CreateRelease(createReleaseConfig)
					Expect(err).NotTo(HaveOccurred())
					Expect(release.Version).To(Equal(productVersion))
				})
			})
		})

		Context("when the server responds with a non-201 status code", func() {
			It("returns an error", func() {
				server.AppendHandlers(
					ghttp.CombineHandlers(
						ghttp.VerifyRequest("POST", apiPrefix+"/products/some-product-name/releases"),
						ghttp.RespondWith(http.StatusTeapot, nil),
					),
				)

				_, err := client.CreateRelease(createReleaseConfig)
				Expect(err).To(MatchError(errors.New(
					"Pivnet returned status code: 418 for the request - expected 201")))
			})
		})
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mtping

import (
	"context"
	"encoding/json"
	"fmt"
	"math/rand"
	"time"

	cloudevents "github.com/cloudevents/sdk-go/v2"
	cecontext "github.com/cloudevents/sdk-go/v2/context"
	"github.com/google/uuid"
	"github.com/robfig/cron/v3"
	"go.uber.org/zap"
	"k8s.io/client-go/kubernetes"
	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/record"

	kncloudevents "knative.dev/eventing/pkg/adapter/v2"
	"knative.dev/eventing/pkg/adapter/v2/util/crstatusevent"
	"knative.dev/eventing/pkg/apis/sources/v1beta2"
)

type CronJobRunner interface {
	Start(stopCh <-chan struct{})
	Stop()
	AddSchedule(source *v1beta2.PingSource) cron.EntryID
	RemoveSchedule(id cron.EntryID)
}

type cronJobsRunner struct {
	// The cron job runner
	cron cron.Cron

	// client sends cloudevents.
	Client cloudevents.Client

	// Where to send logs
	Logger *zap.SugaredLogger

	// kubeClient for sending k8s events
	kubeClient kubernetes.Interface
}

const (
	resourceGroup = "pingsources.sources.knative.dev"
)

func NewCronJobsRunner(ceClient cloudevents.Client, kubeClient kubernetes.Interface, logger *zap.SugaredLogger, opts ...cron.Option) *cronJobsRunner {
	return &cronJobsRunner{
		cron:       *cron.New(opts...),
		Client:     ceClient,
		Logger:     logger,
		kubeClient: kubeClient,
	}
}

func (a *cronJobsRunner) AddSchedule(source *v1beta2.PingSource) cron.EntryID {
	event, err := makeEvent(source)
	if err != nil {
		a.Logger.Error("failed to makeEvent: ", zap.Error(err))
	}

	ctx := context.Background()
	ctx = cloudevents.ContextWithTarget(ctx, source.Status.SinkURI.String())

	var kubeEventSink record.EventSink = &typedcorev1.EventSinkImpl{Interface: a.kubeClient.CoreV1().Events(source.Namespace)}
	ctx = crstatusevent.ContextWithCRStatus(ctx, &kubeEventSink, "ping-source-mt-adapter", source, a.Logger.Infof)

	// Simple retry configuration to be less than 1mn.
	// We might want to retry more times for less-frequent schedule.
	ctx = cloudevents.ContextWithRetriesExponentialBackoff(ctx, 50*time.Millisecond, 5)

	metricTag := &kncloudevents.MetricTag{
		Namespace:     source.Namespace,
		Name:          source.Name,
		ResourceGroup: resourceGroup,
	}

	ctx = kncloudevents.ContextWithMetricTag(ctx, metricTag)
	id, _ := a.cron.AddFunc(source.Spec.Schedule, a.cronTick(ctx, event))
	return id
}

func (a *cronJobsRunner) RemoveSchedule(id cron.EntryID) {
	a.cron.Remove(id)
}

func (a *cronJobsRunner) Start(stopCh <-chan struct{}) {
	a.cron.Start()
	<-stopCh
}

func (a *cronJobsRunner) Stop() {
	ctx := a.cron.Stop() // no more ticks
	if ctx != nil {
		// Wait for all jobs to be done.
		<-ctx.Done()
	}
}

func (a *cronJobsRunner) cronTick(ctx context.Context, event cloudevents.Event) func() {
	return func() {
		event := event.Clone()
		event.SetID(uuid.New().String()) // provide an ID here so we can track it with logging
		defer a.Logger.Debug("Finished sending cloudevent id: ", event.ID())
		target := cecontext.TargetFrom(ctx).String()
		source := event.Context.GetSource()

		// Provide a delay so not all ping fired instantaneously distribute load on resources.
		time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond) //nolint:gosec // Cryptographic randomness not necessary here.

		a.Logger.Debugf("sending cloudevent id: %s, source: %s, target: %s", event.ID(), source, target)

		if result := a.Client.Send(ctx, event); !cloudevents.IsACK(result) {
			// Exhausted number of retries. Event is lost.
			a.Logger.Error("failed to send cloudevent result: ", zap.Any("result", result),
				zap.String("source", source), zap.String("target", target), zap.String("id", event.ID()))
		}
	}
}

func makeEvent(source *v1beta2.PingSource) (cloudevents.Event, error) {
	event := cloudevents.NewEvent()
	event.SetType(v1beta2.PingSourceEventType)
	event.SetSource(v1beta2.PingSourceSource(source.Namespace, source.Name))
	if source.Spec.CloudEventOverrides != nil && source.Spec.CloudEventOverrides.Extensions != nil {
		for key, override := range source.Spec.CloudEventOverrides.Extensions {
			event.SetExtension(key, override)
		}
	}

	// Set event data, at most one of data and dataBase64 exists.
	// 1. If dataBase64 exists, then it's binary data, set event.DataEncoded to []byte(dataBase64)
	// 2. If data exists, then it's not binary data
	//  a. If contentType is not `application/json`, set event.DataEncoded to []byte(data)
	//  b. If contentType is `application/json`, unmarshal it into an interface, event.DataEncoded will be json.Marshal(interface),
	//    this is to be compatible with the existing v1beta1 PingSource -> CloudEvent conversion logic, to make sure
	//    that `data` is populated in the cloudevent json format instead of `data_base64`, and not breaking subscribers
	//    that do not leverage cloudevents sdk.
	var data interface{}
	if source.Spec.DataBase64 != "" {
		data = []byte(source.Spec.DataBase64)
	} else if source.Spec.Data != "" {
		switch source.Spec.ContentType {
		case cloudevents.ApplicationJSON:
			// unmarshal the body into an interface, JSON validation is done in pingsource_validation
			// ignoring the error returned by json.Unmarshal here.
			var objmap map[string]*json.RawMessage
			if err := json.Unmarshal([]byte(source.Spec.Data), &objmap); err != nil {
				return event, fmt.Errorf("error unmarshalling source.Spec.Data: %v, err: %v", source.Spec.Data, err)
			}
			data = objmap
		default:
			data = []byte(source.Spec.Data)
		}
	}

	if data != nil {
		if err := event.SetData(source.Spec.ContentType, data); err != nil {
			return event, fmt.Errorf("error when SetData(%v, %v), err: %v", source.Spec.ContentType, data, err)
		}
	}

	return event, nil
}
<commit_msg>remove json-specific logic useless for binary mode (#4858)<commit_after>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mtping

import (
	"context"
	"fmt"
	"math/rand"
	"time"

	cloudevents "github.com/cloudevents/sdk-go/v2"
	cecontext "github.com/cloudevents/sdk-go/v2/context"
	"github.com/google/uuid"
	"github.com/robfig/cron/v3"
	"go.uber.org/zap"
	"k8s.io/client-go/kubernetes"
	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/record"

	kncloudevents "knative.dev/eventing/pkg/adapter/v2"
	"knative.dev/eventing/pkg/adapter/v2/util/crstatusevent"
	"knative.dev/eventing/pkg/apis/sources/v1beta2"
)

type CronJobRunner interface {
	Start(stopCh <-chan struct{})
	Stop()
	AddSchedule(source *v1beta2.PingSource) cron.EntryID
	RemoveSchedule(id cron.EntryID)
}

type cronJobsRunner struct {
	// The cron job runner
	cron cron.Cron

	// client sends cloudevents.
	Client cloudevents.Client

	// Where to send logs
	Logger *zap.SugaredLogger

	// kubeClient for sending k8s events
	kubeClient kubernetes.Interface
}

const (
	resourceGroup = "pingsources.sources.knative.dev"
)

func NewCronJobsRunner(ceClient cloudevents.Client, kubeClient kubernetes.Interface, logger *zap.SugaredLogger, opts ...cron.Option) *cronJobsRunner {
	return &cronJobsRunner{
		cron:       *cron.New(opts...),
		Client:     ceClient,
		Logger:     logger,
		kubeClient: kubeClient,
	}
}

func (a *cronJobsRunner) AddSchedule(source *v1beta2.PingSource) cron.EntryID {
	event, err := makeEvent(source)
	if err != nil {
		a.Logger.Error("failed to makeEvent: ", zap.Error(err))
	}

	ctx := context.Background()
	ctx = cloudevents.ContextWithTarget(ctx, source.Status.SinkURI.String())

	var kubeEventSink record.EventSink = &typedcorev1.EventSinkImpl{Interface: a.kubeClient.CoreV1().Events(source.Namespace)}
	ctx = crstatusevent.ContextWithCRStatus(ctx, &kubeEventSink, "ping-source-mt-adapter", source, a.Logger.Infof)

	// Simple retry configuration to be less than 1mn.
	// We might want to retry more times for less-frequent schedule.
	ctx = cloudevents.ContextWithRetriesExponentialBackoff(ctx, 50*time.Millisecond, 5)

	metricTag := &kncloudevents.MetricTag{
		Namespace:     source.Namespace,
		Name:          source.Name,
		ResourceGroup: resourceGroup,
	}

	ctx = kncloudevents.ContextWithMetricTag(ctx, metricTag)
	id, _ := a.cron.AddFunc(source.Spec.Schedule, a.cronTick(ctx, event))
	return id
}

func (a *cronJobsRunner) RemoveSchedule(id cron.EntryID) {
	a.cron.Remove(id)
}

func (a *cronJobsRunner) Start(stopCh <-chan struct{}) {
	a.cron.Start()
	<-stopCh
}

func (a *cronJobsRunner) Stop() {
	ctx := a.cron.Stop() // no more ticks
	if ctx != nil {
		// Wait for all jobs to be done.
		<-ctx.Done()
	}
}

func (a *cronJobsRunner) cronTick(ctx context.Context, event cloudevents.Event) func() {
	return func() {
		event := event.Clone()
		event.SetID(uuid.New().String()) // provide an ID here so we can track it with logging
		defer a.Logger.Debug("Finished sending cloudevent id: ", event.ID())
		target := cecontext.TargetFrom(ctx).String()
		source := event.Context.GetSource()

		// Provide a delay so not all ping fired instantaneously distribute load on resources.
		time.Sleep(time.Duration(rand.Intn(500)) * time.Millisecond) //nolint:gosec // Cryptographic randomness not necessary here.

		a.Logger.Debugf("sending cloudevent id: %s, source: %s, target: %s", event.ID(), source, target)

		if result := a.Client.Send(ctx, event); !cloudevents.IsACK(result) {
			// Exhausted number of retries. Event is lost.
			a.Logger.Error("failed to send cloudevent result: ", zap.Any("result", result),
				zap.String("source", source), zap.String("target", target), zap.String("id", event.ID()))
		}
	}
}

func makeEvent(source *v1beta2.PingSource) (cloudevents.Event, error) {
	event := cloudevents.NewEvent()
	event.SetType(v1beta2.PingSourceEventType)
	event.SetSource(v1beta2.PingSourceSource(source.Namespace, source.Name))
	if source.Spec.CloudEventOverrides != nil && source.Spec.CloudEventOverrides.Extensions != nil {
		for key, override := range source.Spec.CloudEventOverrides.Extensions {
			event.SetExtension(key, override)
		}
	}

	var data interface{}
	if source.Spec.DataBase64 != "" {
		data = []byte(source.Spec.DataBase64)
	} else if source.Spec.Data != "" {
		data = []byte(source.Spec.Data)
	}

	if data != nil {
		if err := event.SetData(source.Spec.ContentType, data); err != nil {
			return event, fmt.Errorf("error when SetData(%v, %v), err: %v", source.Spec.ContentType, data, err)
		}
	}

	return event, nil
}
<|endoftext|>


<commit_before>package main

var (
	problemTemplate = `/*

*/
package main

import "github.com/jacobhands/pu"

var (
	answer    = "NA" // Change to correct answer once solved.
	problem   = pu.Problem{{{problem_id}}, solve, answer}
)

func main() {
	problem.Answer()
}

func solve() string {
	return ""
}
`
	problemTestTemplate = `package main

import "testing"

func BenchmarkSolveProblem{{problem_id}}(b *testing.B) {
	problem.Bench(b)
}

func TestSolveProblem{{problem_id}}(t *testing.T) {
	problem.Test(t)
}
`
	problemIDString          = "{{problem_id}}"
	problemDescriptionString = "{{problem_description}}"
)
<commit_msg>Update template<commit_after>package main

var (
	problemTemplate = `/*

 */
package main

import "github.com/jacobhands/pu"

var (
	answer  = "NA" // Change to correct answer once solved.
	problem = pu.Problem{
		ID:            1,
		Solver:        solve,
		CorrectAnswer: answer}
)

func main() {
	problem.Answer()
}

func solve() string {
	return ""
}
`
	problemTestTemplate = `package main

import "testing"

func BenchmarkSolveProblem{{problem_id}}(b *testing.B) {
	problem.Bench(b)
}

func TestSolveProblem{{problem_id}}(t *testing.T) {
	problem.Test(t)
}
`
	problemIDString          = "{{problem_id}}"
	problemDescriptionString = "{{problem_description}}"
)
<|endoftext|>


<commit_before>package core

import (
	"fmt"
	"strings"
	"time"

	"github.com/supergiant/supergiant/pkg/kubernetes"
	"github.com/supergiant/supergiant/pkg/model"
)

var (
	minAgeToExist           = 20 * time.Minute // this is used to prevent adding more nodes while still-pending pods are scheduling to a new node
	maxClusteredPodsPerNode = 2                // prevent putting all nodes of a cluster on one host node
	maxDisksPerNode         = 11
	trackedEventMessages    = [...]string{
		"MatchNodeSelector",
		"PodExceedsMaxPodNumber",
		"PodExceedsFreeMemory",
		"PodExceedsFreeCPU",
		"no nodes available to schedule pods",
		"failed to fit in any node",
	}
)

type CapacityService struct {
	Core            *Core
	WaitBeforeScale time.Duration
}

func (s *CapacityService) Perform() error {
	var kubes []*model.Kube
	if err := s.Core.DB.Preload("CloudAccount").Find(&kubes, "ready = ?", true); err != nil {
		return err
	}

	// TODO
	// 1. Concurrency        -- inParallel? probably shouldn't be on collection
	// 2. "scaling" should be an action on Kube, so we can see the status (actually that may not make sense, just use Nodes ?)

	for _, kube := range kubes {
		if err := newKubeScaler(s, kube).Scale(); err != nil {
			return err
		}
	}

	return nil
}

//------------------------------------------------------------------------------

type KubeScaler struct {
	service         *CapacityService
	kube            *model.Kube
	nodeSizes       []*NodeSize
	largestNodeSize *NodeSize
}

func newKubeScaler(service *CapacityService, kube *model.Kube) *KubeScaler {
	s := &KubeScaler{service: service, kube: kube}
	// We iterate on all nodeSizes here first to preserve the cost order
	for _, it := range service.Core.NodeSizes[kube.CloudAccount.Provider] {
		for _, nodeSizeID := range kube.NodeSizes {
			if it.Name == nodeSizeID {
				s.nodeSizes = append(s.nodeSizes, it)
				break
			}
		}
	}
	s.largestNodeSize = s.nodeSizes[len(s.nodeSizes)-1]
	return s
}

//------------------------------------------------------------------------------

func (s *KubeScaler) Scale() error {
	incomingPods, err := s.incomingPods()
	if err != nil {
		return fmt.Errorf("Capacity service error when fetching incoming pods: %s", err)
	}

	var projectedNodes []*projectedNode
	for _, pod := range incomingPods {
		projectedNodes = append(projectedNodes, &projectedNode{
			false,
			s.largestNodeSize,
			[]*kubernetes.Pod{pod},
		})
	}

	for {
		var (
			pnode1      *projectedNode
			pnode2      *projectedNode
			pnode2Index int
		)

		//==========================================================================
		// find an uncommitted nodeAndPod
		//==========================================================================

		for _, pnode := range projectedNodes {
			if !pnode.Committed {
				pnode1 = pnode
				break
			}
		}

		if pnode1 == nil {
			break
		}

		//==========================================================================
		// find a pnode2 you can merge pnode1 with
		//==========================================================================

		for pnode2IndexCandidate, pnode2Candidate := range projectedNodes {
			if pnode2Candidate == pnode1 { // don't want to merge with self
				continue
			}

			if pnode1.canMergeWith(pnode2Candidate) {
				pnode2 = pnode2Candidate
				pnode2Index = pnode2IndexCandidate
				break
			}
		}

		//==========================================================================
		// merge if found, OR scale down to the smallest instance size it can use and commit it
		//==========================================================================

		if pnode2 != nil {
			// Delete the partner being merged, and merge pods
			i := pnode2Index
			projectedNodes = append(projectedNodes[:i], projectedNodes[i+1:]...)
			pnode1.Pods = append(pnode1.Pods, pnode2.Pods...)
		} else {
			// If we can't merge with anyone, can we scale down to the lowest cost?
			// nodeSizes are asc. by cost, so the first we find is the cheapest.
			for _, nodeSize := range s.nodeSizes {
				if nodeSize.CPUCores >= pnode1.usedCPU() && nodeSize.RAMGIB >= pnode1.usedRAM() {
					pnode1.Size = nodeSize
					pnode1.Committed = true
					break
				}
			}

			if !pnode1.Committed {
				return fmt.Errorf("There is no Node size configured large enough to support %.1f Cores and %1.fGiB RAM", pnode1.usedCPU(), pnode1.usedRAM())
			}
		}
	}

	// Load existing Nodes
	// s.kube.Nodes = make([]*model.Node, 0)
	if err := s.service.Core.DB.Preload("Kube.CloudAccount").Find(&s.kube.Nodes, "kube_name = ?", s.kube.Name); err != nil {
		return err
	}

	for _, node := range s.kube.Nodes {

		// TODO ---- need to label them to prevent disk overflow

		// eventual option to delete nodes when there are pods (w/ or wo/ volumes?) that could move to other nodes (we would have to calculate that)

		hasPods, err := hasPodsWithReservedResources(s.service.Core, node)
		if err != nil {
			return fmt.Errorf("Capacity service error when fetching Pods for Node: %s", err)
		}

		if !hasPods && time.Since(node.ProviderCreationTimestamp) > minAgeToExist {

			s.service.Core.Log.Infof("Terminating node %s", node.Name)

			if err := s.service.Core.Nodes.Delete(node.ID, node).Now(); err != nil {
				return fmt.Errorf("Capacity service error when deleting Node: %s", err)
			}
		}
	}

	//----------------------------------------------------------------------------
	// for _, pnode := range projectedNodes {
	// 	fmt.Println(pnode.Size.Name)
	// 	for _, pod := range pnode.Pods {
	// 		fmt.Println(pod.Metadata.Name)
	// 		for _, container := range pod.Spec.Containers {
	// 			fmt.Println("CPU", container.Resources.Limits.CPU)
	// 			fmt.Println("RAM", container.Resources.Limits.Memory)
	// 		}
	// 	}
	// 	fmt.Println("")
	// }
	//----------------------------------------------------------------------------

	for _, pnode := range projectedNodes {
		node := &model.Node{
			KubeName: s.kube.Name,
			Size:     pnode.Size.Name,
		}

		// If there's an existing node which is spinning up with this type, then
		// don't create.
		// This is a big TODO -- the logic should be much tighter, allowing ongoing
		// projection of pods onto nodes that are still spinning up.

		alreadySpinningUp := false
		for _, existingNode := range s.kube.Nodes {

			if existingNode.Size == node.Size && time.Since(existingNode.ProviderCreationTimestamp) < minAgeToExist {
				// This may be a node that is already being created, or NOTE it could
				// be a broken node that we erroneously identify as spinning up.
				alreadySpinningUp = true
				break
			}
		}
		if alreadySpinningUp {
			s.service.Core.Log.Infof("Capacity service is already waiting on new node with size %s", node.Size)
			continue
		}

		s.service.Core.Log.Infof("Capacity service is creating node with size %s", node.Size)

		if err := s.service.Core.Nodes.Create(node); err != nil {
			return fmt.Errorf("Capacity service error when creating Node: %s", err)
		}
	}
	return nil
}

////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\

func (s *KubeScaler) hasTrackedEvent(pod *kubernetes.Pod) (bool, error) {
	k8s := s.service.Core.K8S(s.kube)

	events, err := k8s.ListEvents("fieldSelector=involvedObject.name=" + pod.Metadata.Name)
	if err != nil {
		return false, err
	}

	for _, event := range events {
		for _, message := range trackedEventMessages {
			if strings.Contains(event.Message, message) {
				return true, nil
			}
		}
	}
	return false, nil
}

func (s *KubeScaler) incomingPods() (incomingPods []*kubernetes.Pod, err error) {
	waitStart := time.Now()

	k8s := s.service.Core.K8S(s.kube)

	for {
		incomingPods = incomingPods[:0] // reset

		pendingPods, err := k8s.ListPods("fieldSelector=status.phase=Pending")
		if err != nil {
			return nil, err
		}

		for _, pod := range pendingPods {
			hasTrackedEvent, err := s.hasTrackedEvent(pod)
			if err != nil {
				return nil, err
			}
			if hasTrackedEvent {
				incomingPods = append(incomingPods, pod)
			}
		}

		elapsed := time.Since(waitStart)
		incomingCount := len(incomingPods)

		if incomingCount > 0 && elapsed < s.service.WaitBeforeScale {
			s.service.Core.Log.Infof("Waiting to add nodes for %d pods; %.1f seconds elapsed", incomingCount, elapsed.Seconds())

			for _, pod := range incomingPods {
				fmt.Println(pod.Metadata.Name)
			}

			time.Sleep(5 * time.Second)
		} else {
			break
		}
	}

	return incomingPods, nil
}

//------------------------------------------------------------------------------

type projectedNode struct {
	Committed bool
	Size      *NodeSize
	Pods      []*kubernetes.Pod
}

func (pnode *projectedNode) usedRAM() (u float64) {
	for _, pod := range pnode.Pods {
		for _, container := range pod.Spec.Containers {

			// NOTE we use limits here, and not requests, because we want to spin up
			// nodes that are at least slightly bigger than the user thinks the pod
			// could utilize. This will ensure that the user's limit CAN BE FILLED AT
			// ALL. This is at the Core of our increased-utilization strategy.

			memStr := container.Resources.Limits.Memory
			if memStr == "" {
				memStr = container.Resources.Requests.Memory
			}

			gib, err := kubernetes.GiBFromMemString(memStr)
			if err != nil {
				panic(err)
			}

			u += gib
		}
	}
	return
}

func (pnode *projectedNode) usedCPU() (u float64) {
	for _, pod := range pnode.Pods {
		for _, container := range pod.Spec.Containers {

			// NOTE above in usedRAM

			cpuStr := container.Resources.Limits.CPU
			if cpuStr == "" {
				cpuStr = container.Resources.Requests.CPU
			}

			Cores, err := kubernetes.CoresFromCPUString(cpuStr)
			if err != nil {
				panic(err)
			}

			u += Cores
		}
	}
	return
}

func (pnode *projectedNode) usedVolumes() (u int) {
	for _, pod := range pnode.Pods {
		for _, vol := range pod.Spec.Volumes {
			if vol.AwsElasticBlockStore != nil || vol.FlexVolume != nil {
				u++
			}
		}
	}
	return
}

func (pnode1 *projectedNode) canMergeWith(pnode2 *projectedNode) bool {
	usedCPU := pnode1.usedCPU() + pnode2.usedCPU()
	usedRAM := pnode1.usedRAM() + pnode2.usedRAM()
	usedVolumes := pnode1.usedVolumes() + pnode2.usedVolumes()
	return pnode1.Size.CPUCores >= usedCPU && pnode1.Size.RAMGIB >= usedRAM && usedVolumes <= maxDisksPerNode
}

//------------------------------------------------------------------------------

func hasPodsWithReservedResources(c *Core, node *model.Node) (bool, error) {
	k8s := c.K8S(node.Kube)
	pods, err := k8s.ListPods("fieldSelector=spec.nodeName=" + node.Name + ",status.phase=Running")
	if err != nil {
		return false, err
	}

	for _, pod := range pods {
		for _, container := range pod.Spec.Containers {

			// TODO
			//
			// These should be moved to json Unmarshal method, so these floats can be parsed once
			//
			gib, err := kubernetes.GiBFromMemString(container.Resources.Requests.Memory)
			if err != nil {
				return false, err
			}
			cores, err := kubernetes.CoresFromCPUString(container.Resources.Requests.CPU)
			if err != nil {
				return false, err
			}

			if gib > 0 || cores > 0 {
				return true, nil
			}
		}
	}

	return false, nil
}
<commit_msg>Add new tracked event messages<commit_after>package core

import (
	"fmt"
	"strings"
	"time"

	"github.com/supergiant/supergiant/pkg/kubernetes"
	"github.com/supergiant/supergiant/pkg/model"
)

var (
	minAgeToExist           = 20 * time.Minute // this is used to prevent adding more nodes while still-pending pods are scheduling to a new node
	maxClusteredPodsPerNode = 2                // prevent putting all nodes of a cluster on one host node
	maxDisksPerNode         = 11
	trackedEventMessages    = [...]string{
		"MatchNodeSelector",
		"PodExceedsMaxPodNumber",
		"PodExceedsFreeMemory",
		"PodExceedsFreeCPU",
		"no nodes available to schedule pods",
		"failed to fit in any node"
		"Insufficient cpu",
		"Insufficient memory",
	}
)

type CapacityService struct {
	Core            *Core
	WaitBeforeScale time.Duration
}

func (s *CapacityService) Perform() error {
	var kubes []*model.Kube
	if err := s.Core.DB.Preload("CloudAccount").Find(&kubes, "ready = ?", true); err != nil {
		return err
	}

	// TODO
	// 1. Concurrency        -- inParallel? probably shouldn't be on collection
	// 2. "scaling" should be an action on Kube, so we can see the status (actually that may not make sense, just use Nodes ?)

	for _, kube := range kubes {
		if err := newKubeScaler(s, kube).Scale(); err != nil {
			return err
		}
	}

	return nil
}

//------------------------------------------------------------------------------

type KubeScaler struct {
	service         *CapacityService
	kube            *model.Kube
	nodeSizes       []*NodeSize
	largestNodeSize *NodeSize
}

func newKubeScaler(service *CapacityService, kube *model.Kube) *KubeScaler {
	s := &KubeScaler{service: service, kube: kube}
	// We iterate on all nodeSizes here first to preserve the cost order
	for _, it := range service.Core.NodeSizes[kube.CloudAccount.Provider] {
		for _, nodeSizeID := range kube.NodeSizes {
			if it.Name == nodeSizeID {
				s.nodeSizes = append(s.nodeSizes, it)
				break
			}
		}
	}
	s.largestNodeSize = s.nodeSizes[len(s.nodeSizes)-1]
	return s
}

//------------------------------------------------------------------------------

func (s *KubeScaler) Scale() error {
	incomingPods, err := s.incomingPods()
	if err != nil {
		return fmt.Errorf("Capacity service error when fetching incoming pods: %s", err)
	}

	var projectedNodes []*projectedNode
	for _, pod := range incomingPods {
		projectedNodes = append(projectedNodes, &projectedNode{
			false,
			s.largestNodeSize,
			[]*kubernetes.Pod{pod},
		})
	}

	for {
		var (
			pnode1      *projectedNode
			pnode2      *projectedNode
			pnode2Index int
		)

		//==========================================================================
		// find an uncommitted nodeAndPod
		//==========================================================================

		for _, pnode := range projectedNodes {
			if !pnode.Committed {
				pnode1 = pnode
				break
			}
		}

		if pnode1 == nil {
			break
		}

		//==========================================================================
		// find a pnode2 you can merge pnode1 with
		//==========================================================================

		for pnode2IndexCandidate, pnode2Candidate := range projectedNodes {
			if pnode2Candidate == pnode1 { // don't want to merge with self
				continue
			}

			if pnode1.canMergeWith(pnode2Candidate) {
				pnode2 = pnode2Candidate
				pnode2Index = pnode2IndexCandidate
				break
			}
		}

		//==========================================================================
		// merge if found, OR scale down to the smallest instance size it can use and commit it
		//==========================================================================

		if pnode2 != nil {
			// Delete the partner being merged, and merge pods
			i := pnode2Index
			projectedNodes = append(projectedNodes[:i], projectedNodes[i+1:]...)
			pnode1.Pods = append(pnode1.Pods, pnode2.Pods...)
		} else {
			// If we can't merge with anyone, can we scale down to the lowest cost?
			// nodeSizes are asc. by cost, so the first we find is the cheapest.
			for _, nodeSize := range s.nodeSizes {
				if nodeSize.CPUCores >= pnode1.usedCPU() && nodeSize.RAMGIB >= pnode1.usedRAM() {
					pnode1.Size = nodeSize
					pnode1.Committed = true
					break
				}
			}

			if !pnode1.Committed {
				return fmt.Errorf("There is no Node size configured large enough to support %.1f Cores and %1.fGiB RAM", pnode1.usedCPU(), pnode1.usedRAM())
			}
		}
	}

	// Load existing Nodes
	// s.kube.Nodes = make([]*model.Node, 0)
	if err := s.service.Core.DB.Preload("Kube.CloudAccount").Find(&s.kube.Nodes, "kube_name = ?", s.kube.Name); err != nil {
		return err
	}

	for _, node := range s.kube.Nodes {

		// TODO ---- need to label them to prevent disk overflow

		// eventual option to delete nodes when there are pods (w/ or wo/ volumes?) that could move to other nodes (we would have to calculate that)

		hasPods, err := hasPodsWithReservedResources(s.service.Core, node)
		if err != nil {
			return fmt.Errorf("Capacity service error when fetching Pods for Node: %s", err)
		}

		if !hasPods && time.Since(node.ProviderCreationTimestamp) > minAgeToExist {

			s.service.Core.Log.Infof("Terminating node %s", node.Name)

			if err := s.service.Core.Nodes.Delete(node.ID, node).Now(); err != nil {
				return fmt.Errorf("Capacity service error when deleting Node: %s", err)
			}
		}
	}

	//----------------------------------------------------------------------------
	// for _, pnode := range projectedNodes {
	// 	fmt.Println(pnode.Size.Name)
	// 	for _, pod := range pnode.Pods {
	// 		fmt.Println(pod.Metadata.Name)
	// 		for _, container := range pod.Spec.Containers {
	// 			fmt.Println("CPU", container.Resources.Limits.CPU)
	// 			fmt.Println("RAM", container.Resources.Limits.Memory)
	// 		}
	// 	}
	// 	fmt.Println("")
	// }
	//----------------------------------------------------------------------------

	for _, pnode := range projectedNodes {
		node := &model.Node{
			KubeName: s.kube.Name,
			Size:     pnode.Size.Name,
		}

		// If there's an existing node which is spinning up with this type, then
		// don't create.
		// This is a big TODO -- the logic should be much tighter, allowing ongoing
		// projection of pods onto nodes that are still spinning up.

		alreadySpinningUp := false
		for _, existingNode := range s.kube.Nodes {

			if existingNode.Size == node.Size && time.Since(existingNode.ProviderCreationTimestamp) < minAgeToExist {
				// This may be a node that is already being created, or NOTE it could
				// be a broken node that we erroneously identify as spinning up.
				alreadySpinningUp = true
				break
			}
		}
		if alreadySpinningUp {
			s.service.Core.Log.Infof("Capacity service is already waiting on new node with size %s", node.Size)
			continue
		}

		s.service.Core.Log.Infof("Capacity service is creating node with size %s", node.Size)

		if err := s.service.Core.Nodes.Create(node); err != nil {
			return fmt.Errorf("Capacity service error when creating Node: %s", err)
		}
	}
	return nil
}

////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
////////////////////////////////////////////////////////////////////////////////
//\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\

func (s *KubeScaler) hasTrackedEvent(pod *kubernetes.Pod) (bool, error) {
	k8s := s.service.Core.K8S(s.kube)

	events, err := k8s.ListEvents("fieldSelector=involvedObject.name=" + pod.Metadata.Name)
	if err != nil {
		return false, err
	}

	for _, event := range events {
		for _, message := range trackedEventMessages {
			if strings.Contains(event.Message, message) {
				return true, nil
			}
		}
	}
	return false, nil
}

func (s *KubeScaler) incomingPods() (incomingPods []*kubernetes.Pod, err error) {
	waitStart := time.Now()

	k8s := s.service.Core.K8S(s.kube)

	for {
		incomingPods = incomingPods[:0] // reset

		pendingPods, err := k8s.ListPods("fieldSelector=status.phase=Pending")
		if err != nil {
			return nil, err
		}

		for _, pod := range pendingPods {
			hasTrackedEvent, err := s.hasTrackedEvent(pod)
			if err != nil {
				return nil, err
			}
			if hasTrackedEvent {
				incomingPods = append(incomingPods, pod)
			}
		}

		elapsed := time.Since(waitStart)
		incomingCount := len(incomingPods)

		if incomingCount > 0 && elapsed < s.service.WaitBeforeScale {
			s.service.Core.Log.Infof("Waiting to add nodes for %d pods; %.1f seconds elapsed", incomingCount, elapsed.Seconds())

			for _, pod := range incomingPods {
				fmt.Println(pod.Metadata.Name)
			}

			time.Sleep(5 * time.Second)
		} else {
			break
		}
	}

	return incomingPods, nil
}

//------------------------------------------------------------------------------

type projectedNode struct {
	Committed bool
	Size      *NodeSize
	Pods      []*kubernetes.Pod
}

func (pnode *projectedNode) usedRAM() (u float64) {
	for _, pod := range pnode.Pods {
		for _, container := range pod.Spec.Containers {

			// NOTE we use limits here, and not requests, because we want to spin up
			// nodes that are at least slightly bigger than the user thinks the pod
			// could utilize. This will ensure that the user's limit CAN BE FILLED AT
			// ALL. This is at the Core of our increased-utilization strategy.

			memStr := container.Resources.Limits.Memory
			if memStr == "" {
				memStr = container.Resources.Requests.Memory
			}

			gib, err := kubernetes.GiBFromMemString(memStr)
			if err != nil {
				panic(err)
			}

			u += gib
		}
	}
	return
}

func (pnode *projectedNode) usedCPU() (u float64) {
	for _, pod := range pnode.Pods {
		for _, container := range pod.Spec.Containers {

			// NOTE above in usedRAM

			cpuStr := container.Resources.Limits.CPU
			if cpuStr == "" {
				cpuStr = container.Resources.Requests.CPU
			}

			Cores, err := kubernetes.CoresFromCPUString(cpuStr)
			if err != nil {
				panic(err)
			}

			u += Cores
		}
	}
	return
}

func (pnode *projectedNode) usedVolumes() (u int) {
	for _, pod := range pnode.Pods {
		for _, vol := range pod.Spec.Volumes {
			if vol.AwsElasticBlockStore != nil || vol.FlexVolume != nil {
				u++
			}
		}
	}
	return
}

func (pnode1 *projectedNode) canMergeWith(pnode2 *projectedNode) bool {
	usedCPU := pnode1.usedCPU() + pnode2.usedCPU()
	usedRAM := pnode1.usedRAM() + pnode2.usedRAM()
	usedVolumes := pnode1.usedVolumes() + pnode2.usedVolumes()
	return pnode1.Size.CPUCores >= usedCPU && pnode1.Size.RAMGIB >= usedRAM && usedVolumes <= maxDisksPerNode
}

//------------------------------------------------------------------------------

func hasPodsWithReservedResources(c *Core, node *model.Node) (bool, error) {
	k8s := c.K8S(node.Kube)
	pods, err := k8s.ListPods("fieldSelector=spec.nodeName=" + node.Name + ",status.phase=Running")
	if err != nil {
		return false, err
	}

	for _, pod := range pods {
		for _, container := range pod.Spec.Containers {

			// TODO
			//
			// These should be moved to json Unmarshal method, so these floats can be parsed once
			//
			gib, err := kubernetes.GiBFromMemString(container.Resources.Requests.Memory)
			if err != nil {
				return false, err
			}
			cores, err := kubernetes.CoresFromCPUString(container.Resources.Requests.CPU)
			if err != nil {
				return false, err
			}

			if gib > 0 || cores > 0 {
				return true, nil
			}
		}
	}

	return false, nil
}
<|endoftext|>


<commit_before>package generator

import (
	"errors"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"

	deployapi "github.com/openshift/origin/pkg/deploy/api"
)

// REST is a RESTStorage implementation for a DeploymentConfigGenerator which supports only
// the Get operation (as the generator has no underlying storage object).
type REST struct {
	generator *DeploymentConfigGenerator
	codec     runtime.Codec
}

func NewREST(generator *DeploymentConfigGenerator, codec runtime.Codec) apiserver.RESTStorage {
	return &REST{generator: generator, codec: codec}
}

func (s *REST) New() runtime.Object {
	return &deployapi.DeploymentConfig{}
}

func (s *REST) List(ctx api.Context, labels, fields labels.Selector) (runtime.Object, error) {
	return nil, errors.New("deploy/generator.REST.List() is not implemented.")
}

func (s *REST) Get(ctx api.Context, id string) (runtime.Object, error) {
	return s.generator.Generate(ctx, id)
}

func (s *REST) Delete(ctx api.Context, id string) (<-chan apiserver.RESTResult, error) {
	return nil, errors.New("deploy/generator.REST.Delete() is not implemented.")
}

func (s *REST) Update(ctx api.Context, obj runtime.Object) (<-chan apiserver.RESTResult, error) {
	return nil, errors.New("deploy/generator.REST.Update() is not implemented.")
}

func (s *REST) Create(ctx api.Context, obj runtime.Object) (<-chan apiserver.RESTResult, error) {
	return nil, errors.New("deploy/generator.REST.Create() is not implemented.")
}
<commit_msg>error strings should not end with punctuation<commit_after>package generator

import (
	"errors"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/apiserver"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/runtime"

	deployapi "github.com/openshift/origin/pkg/deploy/api"
)

// REST is a RESTStorage implementation for a DeploymentConfigGenerator which supports only
// the Get operation (as the generator has no underlying storage object).
type REST struct {
	generator *DeploymentConfigGenerator
	codec     runtime.Codec
}

func NewREST(generator *DeploymentConfigGenerator, codec runtime.Codec) apiserver.RESTStorage {
	return &REST{generator: generator, codec: codec}
}

func (s *REST) New() runtime.Object {
	return &deployapi.DeploymentConfig{}
}

func (s *REST) List(ctx api.Context, labels, fields labels.Selector) (runtime.Object, error) {
	return nil, errors.New("deploy/generator.REST.List() is not implemented")
}

func (s *REST) Get(ctx api.Context, id string) (runtime.Object, error) {
	return s.generator.Generate(ctx, id)
}

func (s *REST) Delete(ctx api.Context, id string) (<-chan apiserver.RESTResult, error) {
	return nil, errors.New("deploy/generator.REST.Delete() is not implemented")
}

func (s *REST) Update(ctx api.Context, obj runtime.Object) (<-chan apiserver.RESTResult, error) {
	return nil, errors.New("deploy/generator.REST.Update() is not implemented")
}

func (s *REST) Create(ctx api.Context, obj runtime.Object) (<-chan apiserver.RESTResult, error) {
	return nil, errors.New("deploy/generator.REST.Create() is not implemented")
}
<|endoftext|>


<commit_before>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eventqueue

import (
	"fmt"
	"reflect"
	"sync"
	"sync/atomic"

	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/spanstat"

	"github.com/sirupsen/logrus"
)

var (
	log = logging.DefaultLogger.WithField(logfields.LogSubsys, "eventqueue")
)

// EventQueue is a structure which is utilized to handle Events in a first-in,
// first-out order. An EventQueue may be closed, in which case all events which
// are queued up, but have not been processed yet, will be cancelled (i.e., not
// ran). It is guaranteed that no events will be scheduled onto an EventQueue
// after it has been closed; if any event is attempted to be scheduled onto an
// EventQueue after it has been closed, it will be cancelled immediately. For
// any event to be processed by the EventQueue, it must implement the
// `EventHandler` interface. This allows for different types of events to be
// processed by anything which chooses to utilize an `EventQueue`.
type EventQueue struct {

	// This should always be a buffered channel.
	events chan *Event

	// close is closed once the EventQueue has been closed.
	close chan struct{}

	// drain is closed when the EventQueue is stopped. Any Event which is
	// Enqueued after this channel is closed will be cancelled / not processed
	// by the queue. If an Event has been Enqueued, but has not been processed
	// before this channel is closed, it will be cancelled and not processed
	// as well.
	drain chan struct{}

	// eventQueueOnce is used to ensure that the EventQueue business logic can
	// only be ran once.
	eventQueueOnce sync.Once

	// closeOnce is used to ensure that the EventQueue can only be closed once.
	closeOnce sync.Once

	// name is used to differentiate this EventQueue from other EventQueues that
	// are also running in logs
	name string

	eventsMu lock.RWMutex

	eventsClosed chan struct{}
}

// NewEventQueue returns an EventQueue with a capacity for only one event at
// a time.
func NewEventQueue() *EventQueue {
	return NewEventQueueBuffered("", 1)

}

func (q *EventQueue) getLogger() *logrus.Entry {
	return log.WithFields(
		logrus.Fields{
			"name": q.name,
		})
}

// NewEventQueueBuffered returns an EventQueue with a capacity of,
// numBufferedEvents at a time, and all other needed fields initialized.
func NewEventQueueBuffered(name string, numBufferedEvents int) *EventQueue {
	log.WithFields(logrus.Fields{
		"name":              name,
		"numBufferedEvents": numBufferedEvents,
	}).Debug("creating new EventQueue")
	return &EventQueue{
		name: name,
		// Up to numBufferedEvents can be Enqueued until Enqueueing blocks.
		events:       make(chan *Event, numBufferedEvents),
		close:        make(chan struct{}),
		drain:        make(chan struct{}),
		eventsClosed: make(chan struct{}),
	}

}

// Event is an event that can be enqueued onto an EventQueue.
type Event struct {
	// Metadata is the information about the event which is sent
	// by its queuer. Metadata must implement the EventHandler interface in
	// order for the Event to be successfully processed by the EventQueue.
	Metadata EventHandler

	// eventResults is a channel on which the results of the event are sent.
	// It is populated by the EventQueue itself, not by the queuer. This channel
	// is closed if the event is cancelled.
	eventResults chan interface{}

	// cancelled signals that the given Event was not ran. This can happen
	// if the EventQueue processing this Event was closed before the Event was
	// Enqueued onto the Event queue, or if the Event was Enqueued onto an
	// EventQueue, and the EventQueue on which the Event was scheduled was
	// closed.
	cancelled chan struct{}

	// stats is a field which contains information about when this event is
	// enqueued, dequeued, etc.
	stats eventStatistics

	// enqueued is an atomic boolean that specifies whether this event has
	// been enqueued on an EventQueue.
	enqueued int32
}

type eventStatistics struct {

	// waitEnqueue shows how long a given event was waiting on the queue before
	// it was actually processed.
	waitEnqueue spanstat.SpanStat

	// durationStat shows how long the actual processing of the event took. This
	// is the time for how long Handle() takes for the event.
	durationStat spanstat.SpanStat

	// waitConsumeOffQueue shows how long it took for the event to be consumed
	// off of the queue.
	waitConsumeOffQueue spanstat.SpanStat
}

// NewEvent returns an Event with all fields initialized.
func NewEvent(meta EventHandler) *Event {
	return &Event{
		Metadata:     meta,
		eventResults: make(chan interface{}, 1),
		cancelled:    make(chan struct{}),
		stats:        eventStatistics{},
	}
}

// WasCancelled returns whether the cancelled channel for the given Event has
// been closed or not. Cancellation occurs if the event was not processed yet
// by an EventQueue onto which this Event was Enqueued, and the queue is closed,
// or if the event was attempted to be scheduled onto an EventQueue which has
// already been closed.
func (ev *Event) WasCancelled() bool {
	select {
	case <-ev.cancelled:
		return true
	default:
		return false
	}
}

// Enqueue pushes the given event onto the EventQueue. If the queue has been
// stopped, the Event will not be enqueued, and its cancel channel will be
// closed, indicating that the Event was not ran. This function may block if
// the queue is at its capacity for events. If a single Event has Enqueue
// called on it multiple times asynchronously, there is no guarantee as to
// which one will return the channel which passes results back to the caller.
// It is up to the caller to check whether the returned channel is nil, as
// waiting to receive on such a channel will block forever. Returns an error
// if the Event has been previously enqueued, if the Event is nil, or the queue
// itself is not initialized properly.
func (q *EventQueue) Enqueue(ev *Event) (<-chan interface{}, error) {
	if q.notSafeToAccess() || ev == nil {
		return nil, fmt.Errorf("unable to Enqueue event")
	}

	// Events can only be enqueued once.
	if atomic.AddInt32(&ev.enqueued, 1) > 1 {
		return nil, fmt.Errorf("unable to Enqueue event; event has already had Enqueue called on it")
	}

	// Multiple Enqueues can occur at the same time. Ensure that events channel
	// is not closed while we are enqueueing events.
	q.eventsMu.RLock()
	defer q.eventsMu.RUnlock()

	select {
	// The event should be drained from the queue (e.g., it should not be
	// processed).
	case <-q.drain:
		// Closed eventResults channel signifies cancellation.
		close(ev.cancelled)
		close(ev.eventResults)

		return ev.eventResults, nil
	default:
		// The events channel may be closed even if an event has been pushed
		// onto the events channel, as events are consumed off of the events
		// channel asynchronously! If the EventQueue is closed before this
		// event is processed, then it will be cancelled.

		ev.stats.waitEnqueue.Start()
		q.events <- ev
		ev.stats.waitEnqueue.End(true)
		ev.stats.waitConsumeOffQueue.Start()
		return ev.eventResults, nil
	}
}

func (ev *Event) printStats(q *EventQueue) {
	if option.Config.Debug {
		q.getLogger().WithFields(logrus.Fields{
			"eventType":                    reflect.TypeOf(ev.Metadata).String(),
			"eventHandlingDuration":        ev.stats.durationStat.Total(),
			"eventEnqueueWaitTime":         ev.stats.waitEnqueue.Total(),
			"eventConsumeOffQueueWaitTime": ev.stats.waitConsumeOffQueue.Total(),
		}).Debug("EventQueue event processing statistics")
	}
}

// Run consumes events that have been queued for this EventQueue. It
// is presumed that the eventQueue is a buffered channel with a length of one
// (i.e., only one event can be processed at a time). All business logic for
// handling queued events is contained within this function. The events in the
// queue must implement the EventHandler interface. If the event queue is
// closed, then all events which were queued up, but not processed, are
// cancelled; any event which is currently being processed will not be
// cancelled.
func (q *EventQueue) Run() {

	if q.notSafeToAccess() {
		return
	}

	go q.eventQueueOnce.Do(func() {
		defer close(q.eventsClosed)
		for ev := range q.events {
			select {
			case <-q.drain:
				ev.stats.waitConsumeOffQueue.End(false)
				close(ev.cancelled)
				close(ev.eventResults)
				ev.printStats(q)
			default:
				ev.stats.waitConsumeOffQueue.End(true)
				ev.stats.durationStat.Start()
				ev.Metadata.Handle(ev.eventResults)
				// Always indicate success for now.
				ev.stats.durationStat.End(true)
				// Ensures that no more results can be sent as the event has
				// already been processed.
				ev.printStats(q)
				close(ev.eventResults)
			}
		}
	})
}

func (q *EventQueue) notSafeToAccess() bool {
	return q == nil || q.close == nil || q.drain == nil || q.events == nil
}

// Stop stops any further events from being processed by the EventQueue. Any
// event which is currently being processed by the EventQueue will continue to
// run. All other events waiting to be processed, and all events that may be
// enqueued will not be processed by the event queue; they will be cancelled.
// If the queue has already been stopped, this is a no-op.
func (q *EventQueue) Stop() {
	if q.notSafeToAccess() {
		return
	}

	q.closeOnce.Do(func() {
		q.getLogger().Debug("stopping EventQueue")
		// Any event that is sent to the queue at this point will be cancelled
		// immediately in Enqueue().
		close(q.drain)

		// Signal that the queue has been drained.
		close(q.close)

		q.eventsMu.Lock()
		close(q.events)
		q.eventsMu.Unlock()
	})
}

// WaitToBeDrained returns the channel which waits for the EventQueue to have been
// stopped. This allows for queuers to ensure that all events in the queue have
// been processed or cancelled. If the queue is nil, returns immediately.
func (q *EventQueue) WaitToBeDrained() {
	if q == nil {
		return
	}
	<-q.close

	// In-flight events may still be running. Wait for them to be completed for
	// the queue to be fully drained.
	<-q.eventsClosed
}

// EventHandler is an interface for allowing an EventQueue to handle events
// in a generic way. To be processed by the EventQueue, all event types must
// implement any function specified in this interface.
type EventHandler interface {
	Handle(chan interface{})
}
<commit_msg>pkg/eventqueue: fix concurrent access of waitConsumeOffQueue<commit_after>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eventqueue

import (
	"fmt"
	"reflect"
	"sync"
	"sync/atomic"

	"github.com/cilium/cilium/pkg/lock"
	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"
	"github.com/cilium/cilium/pkg/option"
	"github.com/cilium/cilium/pkg/spanstat"

	"github.com/sirupsen/logrus"
)

var (
	log = logging.DefaultLogger.WithField(logfields.LogSubsys, "eventqueue")
)

// EventQueue is a structure which is utilized to handle Events in a first-in,
// first-out order. An EventQueue may be closed, in which case all events which
// are queued up, but have not been processed yet, will be cancelled (i.e., not
// ran). It is guaranteed that no events will be scheduled onto an EventQueue
// after it has been closed; if any event is attempted to be scheduled onto an
// EventQueue after it has been closed, it will be cancelled immediately. For
// any event to be processed by the EventQueue, it must implement the
// `EventHandler` interface. This allows for different types of events to be
// processed by anything which chooses to utilize an `EventQueue`.
type EventQueue struct {

	// This should always be a buffered channel.
	events chan *Event

	// close is closed once the EventQueue has been closed.
	close chan struct{}

	// drain is closed when the EventQueue is stopped. Any Event which is
	// Enqueued after this channel is closed will be cancelled / not processed
	// by the queue. If an Event has been Enqueued, but has not been processed
	// before this channel is closed, it will be cancelled and not processed
	// as well.
	drain chan struct{}

	// eventQueueOnce is used to ensure that the EventQueue business logic can
	// only be ran once.
	eventQueueOnce sync.Once

	// closeOnce is used to ensure that the EventQueue can only be closed once.
	closeOnce sync.Once

	// name is used to differentiate this EventQueue from other EventQueues that
	// are also running in logs
	name string

	eventsMu lock.RWMutex

	eventsClosed chan struct{}
}

// NewEventQueue returns an EventQueue with a capacity for only one event at
// a time.
func NewEventQueue() *EventQueue {
	return NewEventQueueBuffered("", 1)

}

func (q *EventQueue) getLogger() *logrus.Entry {
	return log.WithFields(
		logrus.Fields{
			"name": q.name,
		})
}

// NewEventQueueBuffered returns an EventQueue with a capacity of,
// numBufferedEvents at a time, and all other needed fields initialized.
func NewEventQueueBuffered(name string, numBufferedEvents int) *EventQueue {
	log.WithFields(logrus.Fields{
		"name":              name,
		"numBufferedEvents": numBufferedEvents,
	}).Debug("creating new EventQueue")
	return &EventQueue{
		name: name,
		// Up to numBufferedEvents can be Enqueued until Enqueueing blocks.
		events:       make(chan *Event, numBufferedEvents),
		close:        make(chan struct{}),
		drain:        make(chan struct{}),
		eventsClosed: make(chan struct{}),
	}

}

// Event is an event that can be enqueued onto an EventQueue.
type Event struct {
	// Metadata is the information about the event which is sent
	// by its queuer. Metadata must implement the EventHandler interface in
	// order for the Event to be successfully processed by the EventQueue.
	Metadata EventHandler

	// eventResults is a channel on which the results of the event are sent.
	// It is populated by the EventQueue itself, not by the queuer. This channel
	// is closed if the event is cancelled.
	eventResults chan interface{}

	// cancelled signals that the given Event was not ran. This can happen
	// if the EventQueue processing this Event was closed before the Event was
	// Enqueued onto the Event queue, or if the Event was Enqueued onto an
	// EventQueue, and the EventQueue on which the Event was scheduled was
	// closed.
	cancelled chan struct{}

	// stats is a field which contains information about when this event is
	// enqueued, dequeued, etc.
	stats eventStatistics

	// enqueued is an atomic boolean that specifies whether this event has
	// been enqueued on an EventQueue.
	enqueued int32
}

type eventStatistics struct {

	// waitEnqueue shows how long a given event was waiting on the queue before
	// it was actually processed.
	waitEnqueue spanstat.SpanStat

	// durationStat shows how long the actual processing of the event took. This
	// is the time for how long Handle() takes for the event.
	durationStat spanstat.SpanStat

	// waitConsumeOffQueue shows how long it took for the event to be consumed
	// plus the time it the event waited in the queue.
	waitConsumeOffQueue spanstat.SpanStat
}

// NewEvent returns an Event with all fields initialized.
func NewEvent(meta EventHandler) *Event {
	return &Event{
		Metadata:     meta,
		eventResults: make(chan interface{}, 1),
		cancelled:    make(chan struct{}),
		stats:        eventStatistics{},
	}
}

// WasCancelled returns whether the cancelled channel for the given Event has
// been closed or not. Cancellation occurs if the event was not processed yet
// by an EventQueue onto which this Event was Enqueued, and the queue is closed,
// or if the event was attempted to be scheduled onto an EventQueue which has
// already been closed.
func (ev *Event) WasCancelled() bool {
	select {
	case <-ev.cancelled:
		return true
	default:
		return false
	}
}

// Enqueue pushes the given event onto the EventQueue. If the queue has been
// stopped, the Event will not be enqueued, and its cancel channel will be
// closed, indicating that the Event was not ran. This function may block if
// the queue is at its capacity for events. If a single Event has Enqueue
// called on it multiple times asynchronously, there is no guarantee as to
// which one will return the channel which passes results back to the caller.
// It is up to the caller to check whether the returned channel is nil, as
// waiting to receive on such a channel will block forever. Returns an error
// if the Event has been previously enqueued, if the Event is nil, or the queue
// itself is not initialized properly.
func (q *EventQueue) Enqueue(ev *Event) (<-chan interface{}, error) {
	if q.notSafeToAccess() || ev == nil {
		return nil, fmt.Errorf("unable to Enqueue event")
	}

	// Events can only be enqueued once.
	if atomic.AddInt32(&ev.enqueued, 1) > 1 {
		return nil, fmt.Errorf("unable to Enqueue event; event has already had Enqueue called on it")
	}

	// Multiple Enqueues can occur at the same time. Ensure that events channel
	// is not closed while we are enqueueing events.
	q.eventsMu.RLock()
	defer q.eventsMu.RUnlock()

	select {
	// The event should be drained from the queue (e.g., it should not be
	// processed).
	case <-q.drain:
		// Closed eventResults channel signifies cancellation.
		close(ev.cancelled)
		close(ev.eventResults)

		return ev.eventResults, nil
	default:
		// The events channel may be closed even if an event has been pushed
		// onto the events channel, as events are consumed off of the events
		// channel asynchronously! If the EventQueue is closed before this
		// event is processed, then it will be cancelled.

		ev.stats.waitEnqueue.Start()
		ev.stats.waitConsumeOffQueue.Start()
		q.events <- ev
		ev.stats.waitEnqueue.End(true)
		return ev.eventResults, nil
	}
}

func (ev *Event) printStats(q *EventQueue) {
	if option.Config.Debug {
		q.getLogger().WithFields(logrus.Fields{
			"eventType":                    reflect.TypeOf(ev.Metadata).String(),
			"eventHandlingDuration":        ev.stats.durationStat.Total(),
			"eventEnqueueWaitTime":         ev.stats.waitEnqueue.Total(),
			"eventConsumeOffQueueWaitTime": ev.stats.waitConsumeOffQueue.Total(),
		}).Debug("EventQueue event processing statistics")
	}
}

// Run consumes events that have been queued for this EventQueue. It
// is presumed that the eventQueue is a buffered channel with a length of one
// (i.e., only one event can be processed at a time). All business logic for
// handling queued events is contained within this function. The events in the
// queue must implement the EventHandler interface. If the event queue is
// closed, then all events which were queued up, but not processed, are
// cancelled; any event which is currently being processed will not be
// cancelled.
func (q *EventQueue) Run() {

	if q.notSafeToAccess() {
		return
	}

	go q.eventQueueOnce.Do(func() {
		defer close(q.eventsClosed)
		for ev := range q.events {
			select {
			case <-q.drain:
				ev.stats.waitConsumeOffQueue.End(false)
				close(ev.cancelled)
				close(ev.eventResults)
				ev.printStats(q)
			default:
				ev.stats.waitConsumeOffQueue.End(true)
				ev.stats.durationStat.Start()
				ev.Metadata.Handle(ev.eventResults)
				// Always indicate success for now.
				ev.stats.durationStat.End(true)
				// Ensures that no more results can be sent as the event has
				// already been processed.
				ev.printStats(q)
				close(ev.eventResults)
			}
		}
	})
}

func (q *EventQueue) notSafeToAccess() bool {
	return q == nil || q.close == nil || q.drain == nil || q.events == nil
}

// Stop stops any further events from being processed by the EventQueue. Any
// event which is currently being processed by the EventQueue will continue to
// run. All other events waiting to be processed, and all events that may be
// enqueued will not be processed by the event queue; they will be cancelled.
// If the queue has already been stopped, this is a no-op.
func (q *EventQueue) Stop() {
	if q.notSafeToAccess() {
		return
	}

	q.closeOnce.Do(func() {
		q.getLogger().Debug("stopping EventQueue")
		// Any event that is sent to the queue at this point will be cancelled
		// immediately in Enqueue().
		close(q.drain)

		// Signal that the queue has been drained.
		close(q.close)

		q.eventsMu.Lock()
		close(q.events)
		q.eventsMu.Unlock()
	})
}

// WaitToBeDrained returns the channel which waits for the EventQueue to have been
// stopped. This allows for queuers to ensure that all events in the queue have
// been processed or cancelled. If the queue is nil, returns immediately.
func (q *EventQueue) WaitToBeDrained() {
	if q == nil {
		return
	}
	<-q.close

	// In-flight events may still be running. Wait for them to be completed for
	// the queue to be fully drained.
	<-q.eventsClosed
}

// EventHandler is an interface for allowing an EventQueue to handle events
// in a generic way. To be processed by the EventQueue, all event types must
// implement any function specified in this interface.
type EventHandler interface {
	Handle(chan interface{})
}
<|endoftext|>


<commit_before>package git

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"testing"
)

func createLocalGitDirectory(t *testing.T) string {
	dir, err := ioutil.TempDir(os.TempDir(), "s2i-test")
	if err != nil {
		t.Error(err)
	}
	os.Mkdir(filepath.Join(dir, ".git"), 0600)
	return dir
}

func TestParseRepository(t *testing.T) {
	gitLocalDir := createLocalGitDirectory(t)
	defer os.RemoveAll(gitLocalDir)

	tests := map[string]url.URL{
		"git@github.com:user/repo.git": {
			Scheme: "ssh",
			Host:   "github.com",
			User:   url.User("git"),
			Path:   "user/repo.git",
		},
		"git://github.com/user/repo.git": {
			Scheme: "git",
			Host:   "github.com",
			Path:   "/user/repo.git",
		},
		"git://github.com/user/repo": {
			Scheme: "git",
			Host:   "github.com",
			Path:   "/user/repo",
		},
		"http://github.com/user/repo.git": {
			Scheme: "http",
			Host:   "github.com",
			Path:   "/user/repo.git",
		},
		"http://github.com/user/repo": {
			Scheme: "http",
			Host:   "github.com",
			Path:   "/user/repo",
		},
		"https://github.com/user/repo.git": {
			Scheme: "https",
			Host:   "github.com",
			Path:   "/user/repo.git",
		},
		"https://github.com/user/repo": {
			Scheme: "https",
			Host:   "github.com",
			Path:   "/user/repo",
		},
		"file://" + gitLocalDir: {
			Scheme: "file",
			Path:   gitLocalDir,
		},
		gitLocalDir: {
			Scheme: "file",
			Path:   gitLocalDir,
		},
		"git@192.168.122.1:repositories/authooks": {
			Scheme: "ssh",
			Host:   "192.168.122.1",
			User:   url.User("git"),
			Path:   "repositories/authooks",
		},
		"mbalazs@build.ulx.hu:/var/git/eap-ulx.git": {
			Scheme: "ssh",
			Host:   "build.ulx.hu",
			User:   url.User("mbalazs"),
			Path:   "/var/git/eap-ulx.git",
		},
		"ssh://git@[2001:db8::1]/repository.git": {
			Scheme: "ssh",
			Host:   "[2001:db8::1]",
			User:   url.User("git"),
			Path:   "/repository.git",
		},
		"ssh://git@mydomain.com:8080/foo/bar": {
			Scheme: "ssh",
			Host:   "mydomain.com:8080",
			User:   url.User("git"),
			Path:   "/foo/bar",
		},
		"git@[2001:db8::1]:repository.git": {
			Scheme: "ssh",
			Host:   "[2001:db8::1]",
			User:   url.User("git"),
			Path:   "repository.git",
		},
		"git@[2001:db8::1]:/repository.git": {
			Scheme: "ssh",
			Host:   "[2001:db8::1]",
			User:   url.User("git"),
			Path:   "/repository.git",
		},
	}

	for scenario, want := range tests {
		got, err := ParseRepository(scenario)
		if err != nil {
			t.Errorf("ParseRepository returned err: %v", err)
		}

		if !reflect.DeepEqual(*got, want) {
			t.Errorf("%s: got %v, want %v", scenario, got, want)
		}
	}
}
<commit_msg>Print more enlightening string if test fails<commit_after>package git

import (
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"testing"
)

func createLocalGitDirectory(t *testing.T) string {
	dir, err := ioutil.TempDir(os.TempDir(), "s2i-test")
	if err != nil {
		t.Error(err)
	}
	os.Mkdir(filepath.Join(dir, ".git"), 0600)
	return dir
}

func TestParseRepository(t *testing.T) {
	gitLocalDir := createLocalGitDirectory(t)
	defer os.RemoveAll(gitLocalDir)

	tests := map[string]url.URL{
		"git@github.com:user/repo.git": {
			Scheme: "ssh",
			Host:   "github.com",
			User:   url.User("git"),
			Path:   "user/repo.git",
		},
		"git://github.com/user/repo.git": {
			Scheme: "git",
			Host:   "github.com",
			Path:   "/user/repo.git",
		},
		"git://github.com/user/repo": {
			Scheme: "git",
			Host:   "github.com",
			Path:   "/user/repo",
		},
		"http://github.com/user/repo.git": {
			Scheme: "http",
			Host:   "github.com",
			Path:   "/user/repo.git",
		},
		"http://github.com/user/repo": {
			Scheme: "http",
			Host:   "github.com",
			Path:   "/user/repo",
		},
		"https://github.com/user/repo.git": {
			Scheme: "https",
			Host:   "github.com",
			Path:   "/user/repo.git",
		},
		"https://github.com/user/repo": {
			Scheme: "https",
			Host:   "github.com",
			Path:   "/user/repo",
		},
		"file://" + gitLocalDir: {
			Scheme: "file",
			Path:   gitLocalDir,
		},
		gitLocalDir: {
			Scheme: "file",
			Path:   gitLocalDir,
		},
		"git@192.168.122.1:repositories/authooks": {
			Scheme: "ssh",
			Host:   "192.168.122.1",
			User:   url.User("git"),
			Path:   "repositories/authooks",
		},
		"mbalazs@build.ulx.hu:/var/git/eap-ulx.git": {
			Scheme: "ssh",
			Host:   "build.ulx.hu",
			User:   url.User("mbalazs"),
			Path:   "/var/git/eap-ulx.git",
		},
		"ssh://git@[2001:db8::1]/repository.git": {
			Scheme: "ssh",
			Host:   "[2001:db8::1]",
			User:   url.User("git"),
			Path:   "/repository.git",
		},
		"ssh://git@mydomain.com:8080/foo/bar": {
			Scheme: "ssh",
			Host:   "mydomain.com:8080",
			User:   url.User("git"),
			Path:   "/foo/bar",
		},
		"git@[2001:db8::1]:repository.git": {
			Scheme: "ssh",
			Host:   "[2001:db8::1]",
			User:   url.User("git"),
			Path:   "repository.git",
		},
		"git@[2001:db8::1]:/repository.git": {
			Scheme: "ssh",
			Host:   "[2001:db8::1]",
			User:   url.User("git"),
			Path:   "/repository.git",
		},
	}

	for scenario, want := range tests {
		got, err := ParseRepository(scenario)
		if err != nil {
			t.Errorf("ParseRepository returned err: %v", err)
		}

		if !reflect.DeepEqual(*got, want) {
			t.Errorf("%s: got %#v, want %#v", scenario, *got, want)
		}
	}
}
<|endoftext|>


<commit_before>package httphelper

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/julienschmidt/httprouter"
	"github.com/flynn/flynn/Godeps/_workspace/src/golang.org/x/net/context"
	"github.com/flynn/flynn/pkg/cors"
	"github.com/flynn/flynn/pkg/random"
)

type ErrorCode string

const (
	NotFoundError       ErrorCode = "not_found"
	ObjectNotFoundError           = "object_not_found"
	ObjectExistsError             = "object_exists"
	SyntaxError                   = "syntax_error"
	ValidationError               = "validation_error"
	UnknownError                  = "unknown_error"
)

var errorResponseCodes = map[ErrorCode]int{
	NotFoundError:       404,
	ObjectNotFoundError: 404,
	ObjectExistsError:   409,
	SyntaxError:         400,
	ValidationError:     400,
	UnknownError:        500,
}

type JSONError struct {
	Code    ErrorCode       `json:"code"`
	Message string          `json:"message"`
	Detail  json.RawMessage `json:"detail,omitempty"`
}

var CORSAllowAllHandler = cors.Allow(&cors.Options{
	AllowAllOrigins:  true,
	AllowMethods:     []string{"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"},
	AllowHeaders:     []string{"Authorization", "Accept", "Content-Type", "If-Match", "If-None-Match"},
	ExposeHeaders:    []string{"ETag"},
	AllowCredentials: true,
	MaxAge:           time.Hour,
})

type CtxKey string

const (
	CtxKeyComponent CtxKey = "component"
	CtxKeyReqID            = "req_id"
	CtxKeyParams           = "params"
	CtxKeyLogger           = "logger"
)

type Handle func(context.Context, http.ResponseWriter, *http.Request)

func WrapHandler(handler Handle) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
		ctx := contextFromResponseWriter(w)
		ctx = context.WithValue(ctx, CtxKeyParams, params)
		handler(ctx, w, req)
	}
}

func ContextInjector(componentName string, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		reqID := req.Header.Get("X-Request-ID")
		if reqID == "" {
			reqID = random.UUID()
		}
		ctx := context.WithValue(context.Background(), CtxKeyReqID, reqID)
		ctx = context.WithValue(ctx, CtxKeyComponent, componentName)
		rw := NewResponseWriter(w, ctx)
		handler.ServeHTTP(rw, req)
	})
}

func ParamsFromContext(ctx context.Context) httprouter.Params {
	params := ctx.Value(CtxKeyParams).(httprouter.Params)
	return params
}

func contextFromResponseWriter(w http.ResponseWriter) context.Context {
	ctx := w.(*ResponseWriter).Context()
	return ctx
}

func (jsonError JSONError) Error() string {
	return fmt.Sprintf("%s: %s", jsonError.Code, jsonError.Message)
}

func Error(w http.ResponseWriter, err error) {
	var jsonError JSONError
	switch err.(type) {
	case *json.SyntaxError, *json.UnmarshalTypeError:
		jsonError = JSONError{
			Code:    SyntaxError,
			Message: "The provided JSON input is invalid",
		}
	case JSONError:
		jsonError = err.(JSONError)
	case *JSONError:
		jsonError = *err.(*JSONError)
	default:
		log.Println(err)
		jsonError = JSONError{
			Code:    UnknownError,
			Message: "Something went wrong",
		}
	}

	responseCode, ok := errorResponseCodes[jsonError.Code]
	if !ok {
		responseCode = 500
	}
	JSON(w, responseCode, jsonError)
}

func JSON(w http.ResponseWriter, status int, v interface{}) {
	var result []byte
	var err error
	result, err = json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	w.Write(result)
}
<commit_msg>httphelper: Use log15 logger to print error if available<commit_after>package httphelper

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/julienschmidt/httprouter"
	"github.com/flynn/flynn/Godeps/_workspace/src/golang.org/x/net/context"
	log15 "github.com/flynn/flynn/Godeps/_workspace/src/gopkg.in/inconshreveable/log15.v2"
	"github.com/flynn/flynn/pkg/cors"
	"github.com/flynn/flynn/pkg/random"
)

type ErrorCode string

const (
	NotFoundError       ErrorCode = "not_found"
	ObjectNotFoundError           = "object_not_found"
	ObjectExistsError             = "object_exists"
	SyntaxError                   = "syntax_error"
	ValidationError               = "validation_error"
	UnknownError                  = "unknown_error"
)

var errorResponseCodes = map[ErrorCode]int{
	NotFoundError:       404,
	ObjectNotFoundError: 404,
	ObjectExistsError:   409,
	SyntaxError:         400,
	ValidationError:     400,
	UnknownError:        500,
}

type JSONError struct {
	Code    ErrorCode       `json:"code"`
	Message string          `json:"message"`
	Detail  json.RawMessage `json:"detail,omitempty"`
}

var CORSAllowAllHandler = cors.Allow(&cors.Options{
	AllowAllOrigins:  true,
	AllowMethods:     []string{"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"},
	AllowHeaders:     []string{"Authorization", "Accept", "Content-Type", "If-Match", "If-None-Match"},
	ExposeHeaders:    []string{"ETag"},
	AllowCredentials: true,
	MaxAge:           time.Hour,
})

type CtxKey string

const (
	CtxKeyComponent CtxKey = "component"
	CtxKeyReqID            = "req_id"
	CtxKeyParams           = "params"
	CtxKeyLogger           = "logger"
)

type Handle func(context.Context, http.ResponseWriter, *http.Request)

func WrapHandler(handler Handle) httprouter.Handle {
	return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
		ctx := contextFromResponseWriter(w)
		ctx = context.WithValue(ctx, CtxKeyParams, params)
		handler(ctx, w, req)
	}
}

func ContextInjector(componentName string, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		reqID := req.Header.Get("X-Request-ID")
		if reqID == "" {
			reqID = random.UUID()
		}
		ctx := context.WithValue(context.Background(), CtxKeyReqID, reqID)
		ctx = context.WithValue(ctx, CtxKeyComponent, componentName)
		rw := NewResponseWriter(w, ctx)
		handler.ServeHTTP(rw, req)
	})
}

func ParamsFromContext(ctx context.Context) httprouter.Params {
	params := ctx.Value(CtxKeyParams).(httprouter.Params)
	return params
}

func contextFromResponseWriter(w http.ResponseWriter) context.Context {
	ctx := w.(*ResponseWriter).Context()
	return ctx
}

func (jsonError JSONError) Error() string {
	return fmt.Sprintf("%s: %s", jsonError.Code, jsonError.Message)
}

func Error(w http.ResponseWriter, err error) {
	var jsonError JSONError
	switch err.(type) {
	case *json.SyntaxError, *json.UnmarshalTypeError:
		jsonError = JSONError{
			Code:    SyntaxError,
			Message: "The provided JSON input is invalid",
		}
	case JSONError:
		jsonError = err.(JSONError)
	case *JSONError:
		jsonError = *err.(*JSONError)
	default:
		rw, ok := w.(*ResponseWriter)
		if ok {
			rw.Context().Value(CtxKeyLogger).(log15.Logger).Error(err.Error())
		} else {
			log.Println(err)
		}
		jsonError = JSONError{
			Code:    UnknownError,
			Message: "Something went wrong",
		}
	}

	responseCode, ok := errorResponseCodes[jsonError.Code]
	if !ok {
		responseCode = 500
	}
	JSON(w, responseCode, jsonError)
}

func JSON(w http.ResponseWriter, status int, v interface{}) {
	var result []byte
	var err error
	result, err = json.Marshal(v)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(status)
	w.Write(result)
}
<|endoftext|>