<commit_before>package oauth2

import (
	"bytes"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"

	"github.com/hooklift/oauth2/providers/test"
	"github.com/hooklift/oauth2/types"
)

// getTestAuthzCode returns authorization tokens for access tokens issuing tests
func getTestAuthzCode(t *testing.T) (Provider, string) {
	provider := test.NewProvider(true)

	state := "state-test"
	scopes := "read write identity"
	grantType := "code"

	values := url.Values{
		"client_id":     {provider.Client.ID},
		"response_type": {grantType},
		"state":         {state},
		"redirect_uri":  {provider.Client.RedirectURL.String()},
		"scope":         {scopes},
	}

	// http://tools.ietf.org/html/rfc6749#section-4.1.1
	queryStr := values.Encode()
	req, err := http.NewRequest("GET",
		"https://example.com/oauth2/authzs?"+queryStr, nil)
	ok(t, err)

	w := httptest.NewRecorder()
	CreateGrant(w, req, provider)
	equals(t, http.StatusOK, w.Code)

	body := w.Body.String()
	stringz := []string{
		"client_id",
		"redirect_uri",
		"response_type",
		"state",
		"scope",
		"code",
		"read write identity",
		"state-test",
	}

	for _, s := range stringz {
		assert(t, strings.Contains(body, s), "Does not look like we got an authorization form: '%s' was not found in %v", s, body)
	}

	// Sending post to acquire authorization token
	buffer := bytes.NewBufferString(queryStr)
	req, err = http.NewRequest("POST", "https://example.com/oauth2/authzs", buffer)
	ok(t, err)

	req.Header.Set("Content-type", "application/x-www-form-urlencoded")

	w = httptest.NewRecorder()
	CreateGrant(w, req, provider)

	// Tests http://tools.ietf.org/html/rfc6749#section-4.1.2
	equals(t, http.StatusFound, w.Code)

	redirectTo := w.Header().Get("Location")
	u, err := url.Parse(redirectTo)
	ok(t, err)

	authzCode := u.Query().Get("code")
	assert(t, authzCode != "", "It looks like the authorization code came back empty: %s", authzCode)
	equals(t, state, u.Query().Get("state"))

	return provider, authzCode
}

// TestAuthorizationGrant tests a happy web authorization flow in accordance with
// http://tools.ietf.org/html/rfc6749#section-4.1
func TestAuthorizationGrant(t *testing.T) {
	getTestAuthzCode(t)
}

// TestLoginRedirect tests that logging in is required for a resource owner to
// grant any authorization codes to clients.
func TestLoginRedirect(t *testing.T) {
	provider := test.NewProvider(false)

	state := "state-test"
	scopes := "read write identity"
	grantType := "code"
	clientID := provider.Client.ID
	redirectURL := provider.Client.RedirectURL.String()

	values := url.Values{
		"client_id":     {clientID},
		"response_type": {grantType},
		"state":         {state},
		"redirect_uri":  {redirectURL},
		"scope":         {scopes},
	}

	// http://tools.ietf.org/html/rfc6749#section-4.1.1
	queryStr := values.Encode()
	authzURL := "https://example.com/oauth2/authzs?" + queryStr
	req, err := http.NewRequest("GET", authzURL, nil)
	ok(t, err)

	w := httptest.NewRecorder()
	CreateGrant(w, req, provider)
	equals(t, http.StatusFound, w.Code)
	equals(t, provider.LoginURL(authzURL), w.Header().Get("Location"))
}

// TestImplicitGrant tests a happy implicit flow
func TestImplicitGrant(t *testing.T) {
	provider := test.NewProvider(true)

	state := "state-test"
	scopes := "read write identity"
	grantType := "token"
	clientID := provider.Client.ID
	redirectURL := provider.Client.RedirectURL.String()

	values := url.Values{
		"client_id":     {clientID},
		"response_type": {grantType},
		"state":         {state},
		"redirect_uri":  {redirectURL},
		"scope":         {scopes},
	}

	// http://tools.ietf.org/html/rfc6749#section-4.2.1
	queryStr := values.Encode()
	authzURL := "https://example.com/oauth2/authzs?" + queryStr
	req, err := http.NewRequest("GET", authzURL, nil)
	ok(t, err)

	w := httptest.NewRecorder()
	CreateGrant(w, req, provider)
	body := w.Body.String()
	stringz := []string{
		"client_id",
		"redirect_uri",
		"response_type",
		"state",
		"scope",
		"token",
		"read write identity",
		"state-test",
	}

	for _, s := range stringz {
		assert(t, strings.Contains(body, s), "Does not look like we got an authorization form: '%s' was not found in %v", s, body)
	}

	// Sending post to acquire authorization token
	buffer := bytes.NewBufferString(queryStr)
	req, err = http.NewRequest("POST", "https://example.com/oauth2/authzs", buffer)
	ok(t, err)

	req.Header.Set("Content-type", "application/x-www-form-urlencoded")

	w = httptest.NewRecorder()
	CreateGrant(w, req, provider)

	// Tests http://tools.ietf.org/html/rfc6749#section-4.2.2
	equals(t, http.StatusFound, w.Code)

	redirectTo := w.Header().Get("Location")
	u, err := url.Parse(redirectTo)
	ok(t, err)

	fragment, err := url.ParseQuery(strings.TrimPrefix(u.Fragment, "#"))
	ok(t, err)
	accessToken := fragment.Get("access_token")
	assert(t, accessToken != "", "It looks like the authorization code came back empty: ->%s<-", accessToken)
	equals(t, state, fragment.Get("state"))
	equals(t, "600", fragment.Get("expires_in"))
	equals(t, scopes, fragment.Get("scope"))
	equals(t, "bearer", fragment.Get("token_type"))

	// Implict flow should not emit refresh tokens
	refreshToken := fragment.Get("refresh_token")
	equals(t, "", refreshToken)
}

// TestReplayAttackProtection tests that the authorization grant can be used
// only once.
func TestReplayAttackProtection(t *testing.T) {
	provider, authzCode := getTestAuthzCode(t)

	req := AuthzGrantTokenRequestTest(t, "authorization_code", authzCode)
	req.SetBasicAuth("test_client_id", "test_client_id")

	w := httptest.NewRecorder()
	IssueAccessToken(w, req, provider)
	token := types.Token{}
	err := json.Unmarshal(w.Body.Bytes(), &token)
	ok(t, err)
	equals(t, "bearer", token.Type)
	equals(t, "600", token.ExpiresIn)

	w2 := httptest.NewRecorder()
	IssueAccessToken(w2, req, provider)

	// http://tools.ietf.org/html/rfc6749#section-4.1.4
	authzErr := AuthzError{}
	//log.Printf("%s", w2.Body.String())
	err = json.Unmarshal(w2.Body.Bytes(), &authzErr)
	ok(t, err)
	equals(t, "invalid_grant", authzErr.Code)
	equals(t, "Grant code was revoked, expired or already used.", authzErr.Desc)

}

// TestRedirectURLMatch makes sure redirect_uri for requesting an authorization
// grant is the same as the redirect_uri provided to get the correspondent access token.
// This is intended to mitigate the risk of account hijacking by leaking
// authorization codes.
func TestRedirectURLMatch(t *testing.T) {

}

// TestAccessTokenIssuer makes sure a token belongs to the client_id making
// the request with it. This mitigates account hijacking as well.
func TestAccessTokenOwnership(t *testing.T) {

}

// TestAccessTokenExpiration makes sure that access tokens are actually expired.
func TestAccessTokenExpiration(t *testing.T) {

}

// TestScopeIsRequired makes sure it requires clients to provide access scopes when
// getting authorization codes.
func TestScopeIsRequired(t *testing.T) {

}

// TestStateIsRequired makes sure it requires clients to provide a state when
// getting authorization codes.
func TestStateIsRequired(t *testing.T) {

}

// TestSecurityHeaders makes sure security headers are sent along the authorization form.
func TestSecurityHeaders(t *testing.T) {

}

// TestRedirectURIScheme makes sure clients provide redirect URLs that use TLS
func TestRedirectURIScheme(t *testing.T) {

}

// TestRedirectURIUniqueness makes sure there redirect URL is unique across the
// entire system.
func TestRedirectURIUniqueness(t *testing.T) {

}

// TestStateParam makes sure the same state parameter value received to acquire
// the authorization grant is send back when delivering the access token.
func TestStateParam(t *testing.T) {

}
<commit_msg>Adds test for requiring secured redirect URLs to clients<commit_after>package oauth2

import (
	"bytes"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"strings"
	"testing"

	"github.com/hooklift/oauth2/providers/test"
	"github.com/hooklift/oauth2/types"
)

// getTestAuthzCode returns authorization tokens for access tokens issuing tests
func getTestAuthzCode(t *testing.T) (Provider, string) {
	provider := test.NewProvider(true)

	state := "state-test"
	scopes := "read write identity"
	grantType := "code"

	values := url.Values{
		"client_id":     {provider.Client.ID},
		"response_type": {grantType},
		"state":         {state},
		"redirect_uri":  {provider.Client.RedirectURL.String()},
		"scope":         {scopes},
	}

	// http://tools.ietf.org/html/rfc6749#section-4.1.1
	queryStr := values.Encode()
	req, err := http.NewRequest("GET",
		"https://example.com/oauth2/authzs?"+queryStr, nil)
	ok(t, err)

	w := httptest.NewRecorder()
	CreateGrant(w, req, provider)
	equals(t, http.StatusOK, w.Code)

	body := w.Body.String()
	stringz := []string{
		"client_id",
		"redirect_uri",
		"response_type",
		"state",
		"scope",
		"code",
		"read write identity",
		"state-test",
	}

	for _, s := range stringz {
		assert(t, strings.Contains(body, s), "Does not look like we got an authorization form: '%s' was not found in %v", s, body)
	}

	// Sending post to acquire authorization token
	buffer := bytes.NewBufferString(queryStr)
	req, err = http.NewRequest("POST", "https://example.com/oauth2/authzs", buffer)
	ok(t, err)

	req.Header.Set("Content-type", "application/x-www-form-urlencoded")

	w = httptest.NewRecorder()
	CreateGrant(w, req, provider)

	// Tests http://tools.ietf.org/html/rfc6749#section-4.1.2
	equals(t, http.StatusFound, w.Code)

	redirectTo := w.Header().Get("Location")
	u, err := url.Parse(redirectTo)
	ok(t, err)

	authzCode := u.Query().Get("code")
	assert(t, authzCode != "", "It looks like the authorization code came back empty: %s", authzCode)

	// makes sure the same state parameter value received to acquire
	// the authorization grant is send back when delivering the access token.
	equals(t, state, u.Query().Get("state"))

	return provider, authzCode
}

// TestAuthorizationGrant tests a happy web authorization flow in accordance with
// http://tools.ietf.org/html/rfc6749#section-4.1
func TestAuthorizationGrant(t *testing.T) {
	getTestAuthzCode(t)
}

// TestLoginRedirect tests that logging in is required for a resource owner to
// grant any authorization codes to clients.
func TestLoginRedirect(t *testing.T) {
	provider := test.NewProvider(false)

	state := "state-test"
	scopes := "read write identity"
	grantType := "code"
	clientID := provider.Client.ID
	redirectURL := provider.Client.RedirectURL.String()

	values := url.Values{
		"client_id":     {clientID},
		"response_type": {grantType},
		"state":         {state},
		"redirect_uri":  {redirectURL},
		"scope":         {scopes},
	}

	// http://tools.ietf.org/html/rfc6749#section-4.1.1
	queryStr := values.Encode()
	authzURL := "https://example.com/oauth2/authzs?" + queryStr
	req, err := http.NewRequest("GET", authzURL, nil)
	ok(t, err)

	w := httptest.NewRecorder()
	CreateGrant(w, req, provider)
	equals(t, http.StatusFound, w.Code)
	equals(t, provider.LoginURL(authzURL), w.Header().Get("Location"))
}

// TestImplicitGrant tests a happy implicit flow
func TestImplicitGrant(t *testing.T) {
	provider := test.NewProvider(true)

	state := "state-test"
	scopes := "read write identity"
	grantType := "token"
	clientID := provider.Client.ID
	redirectURL := provider.Client.RedirectURL.String()

	values := url.Values{
		"client_id":     {clientID},
		"response_type": {grantType},
		"state":         {state},
		"redirect_uri":  {redirectURL},
		"scope":         {scopes},
	}

	// http://tools.ietf.org/html/rfc6749#section-4.2.1
	queryStr := values.Encode()
	authzURL := "https://example.com/oauth2/authzs?" + queryStr
	req, err := http.NewRequest("GET", authzURL, nil)
	ok(t, err)

	w := httptest.NewRecorder()
	CreateGrant(w, req, provider)
	body := w.Body.String()
	stringz := []string{
		"client_id",
		"redirect_uri",
		"response_type",
		"state",
		"scope",
		"token",
		"read write identity",
		"state-test",
	}

	for _, s := range stringz {
		assert(t, strings.Contains(body, s), "Does not look like we got an authorization form: '%s' was not found in %v", s, body)
	}

	// Sending post to acquire authorization token
	buffer := bytes.NewBufferString(queryStr)
	req, err = http.NewRequest("POST", "https://example.com/oauth2/authzs", buffer)
	ok(t, err)

	req.Header.Set("Content-type", "application/x-www-form-urlencoded")

	w = httptest.NewRecorder()
	CreateGrant(w, req, provider)

	// Tests http://tools.ietf.org/html/rfc6749#section-4.2.2
	equals(t, http.StatusFound, w.Code)

	redirectTo := w.Header().Get("Location")
	u, err := url.Parse(redirectTo)
	ok(t, err)

	fragment, err := url.ParseQuery(strings.TrimPrefix(u.Fragment, "#"))
	ok(t, err)
	accessToken := fragment.Get("access_token")
	assert(t, accessToken != "", "It looks like the authorization code came back empty: ->%s<-", accessToken)
	equals(t, state, fragment.Get("state"))
	equals(t, "600", fragment.Get("expires_in"))
	equals(t, scopes, fragment.Get("scope"))
	equals(t, "bearer", fragment.Get("token_type"))

	// Implict flow should not emit refresh tokens
	refreshToken := fragment.Get("refresh_token")
	equals(t, "", refreshToken)
}

// TestReplayAttackProtection tests that the authorization grant can be used
// only once.
func TestReplayAttackProtection(t *testing.T) {
	provider, authzCode := getTestAuthzCode(t)

	req := AuthzGrantTokenRequestTest(t, "authorization_code", authzCode)
	req.SetBasicAuth("test_client_id", "test_client_id")

	w := httptest.NewRecorder()
	IssueAccessToken(w, req, provider)
	token := types.Token{}
	err := json.Unmarshal(w.Body.Bytes(), &token)
	ok(t, err)
	equals(t, "bearer", token.Type)
	equals(t, "600", token.ExpiresIn)

	w2 := httptest.NewRecorder()
	IssueAccessToken(w2, req, provider)

	// http://tools.ietf.org/html/rfc6749#section-4.1.4
	authzErr := AuthzError{}
	//log.Printf("%s", w2.Body.String())
	err = json.Unmarshal(w2.Body.Bytes(), &authzErr)
	ok(t, err)
	equals(t, "invalid_grant", authzErr.Code)
	equals(t, "Grant code was revoked, expired or already used.", authzErr.Desc)

}

// TestRedirectURLMatch makes sure redirect_uri for requesting an authorization
// grant is the same as the redirect_uri provided to get the correspondent access token.
// This is intended to mitigate the risk of account hijacking by leaking
// authorization codes.
func TestRedirectURLMatch(t *testing.T) {

}

// TestAccessTokenIssuer makes sure a token belongs to the client_id making
// the request with it. This mitigates account hijacking as well.
func TestAccessTokenOwnership(t *testing.T) {

}

// TestAccessTokenExpiration makes sure that access tokens are actually expired.
func TestAccessTokenExpiration(t *testing.T) {

}

// TestScopeIsRequired makes sure it requires clients to provide access scopes when
// getting authorization codes.
func TestScopeIsRequired(t *testing.T) {

}

// TestStateIsRequired makes sure it requires clients to provide a state when
// getting authorization codes.
func TestStateIsRequired(t *testing.T) {

}

// TestSecurityHeaders makes sure security headers are sent along the authorization form.
func TestSecurityHeaders(t *testing.T) {

}

// TestRedirectURIScheme makes sure clients provide redirect URLs that use TLS
func TestRedirectURIScheme(t *testing.T) {
	provider := test.NewProvider(true)

	state := "state-test"
	scopes := "read write identity"
	grantType := "code"

	values := url.Values{
		"client_id":     {provider.Client.ID},
		"response_type": {grantType},
		"state":         {state},
		"redirect_uri":  {"http://unsecured.com/callback"},
		"scope":         {scopes},
	}

	// http://tools.ietf.org/html/rfc6749#section-4.1.1
	queryStr := values.Encode()
	req, err := http.NewRequest("GET",
		"https://example.com/oauth2/authzs?"+queryStr, nil)
	ok(t, err)

	w := httptest.NewRecorder()
	CreateGrant(w, req, provider)
	body := w.Body.String()
	assert(t, strings.Contains(body, "access_denied") == true, "access-denied was not found in response body")
	assert(t, strings.Contains(body, "3rd-party client app provided an invalid redirect_uri. It does not comply with http://tools.ietf.org/html/rfc3986#section-4.3 or does not use HTTPS") == true, "error description does not match.")
}
<|endoftext|>


<commit_before>package paginator

// HandlerParams represents the params needed to handle a pagination
type HandlerParams struct {
	// Page represents the number of the current page
	Page *int `from:"query" json:"page" default:"1"`

	// PerPage represents the maximum number of element we want per page
	// A value of -1 will
	PerPage *int `from:"query" json:"per_page"`
}

// Paginator returns a Paginator from an HandlerParams
func (params *HandlerParams) Paginator(defaultPerPage int) *Paginator {
	// Should never be nil because of the default value
	p := *params.Page

	pp := defaultPerPage
	if params.PerPage != nil {
		pp = *params.PerPage
	}

	return &Paginator{
		CurrentPage: p,
		PerPage:     pp,
		MaxPerPage:  100,
	}
}

// Paginator represents a pagination
type Paginator struct {
	CurrentPage int
	PerPage     int
	MaxPerPage  int
}

// New creates a new Paginator
func New(page int, perPage int) *Paginator {
	return &Paginator{
		CurrentPage: page,
		PerPage:     perPage,
		MaxPerPage:  100,
	}
}

// IsValid checks if the paginator is valid
func (p *Paginator) IsValid() bool {
	return p.CurrentPage > 0 && p.PerPage <= p.MaxPerPage
}

// Offset returns a valid SQL offset value
func (p *Paginator) Offset() int {
	return (p.CurrentPage - 1) * p.PerPage
}

// Limit returns a valid SQL limit value
func (p *Paginator) Limit() int {
	return p.PerPage
}
<commit_msg>fix(paginator): Fix PerPage <= 0<commit_after>package paginator

// HandlerParams represents the params needed to handle a pagination
type HandlerParams struct {
	// Page represents the number of the current page
	Page *int `from:"query" json:"page" default:"1"`

	// PerPage represents the maximum number of element we want per page
	// A value of -1 will
	PerPage *int `from:"query" json:"per_page"`
}

// Paginator returns a Paginator from an HandlerParams
func (params *HandlerParams) Paginator(defaultPerPage int) *Paginator {
	// Should never be nil because of the default value
	p := *params.Page

	pp := defaultPerPage
	if params.PerPage != nil {
		pp = *params.PerPage
	}

	return &Paginator{
		CurrentPage: p,
		PerPage:     pp,
		MaxPerPage:  100,
	}
}

// Paginator represents a pagination
type Paginator struct {
	CurrentPage int
	PerPage     int
	MaxPerPage  int
}

// New creates a new Paginator
func New(page int, perPage int) *Paginator {
	return &Paginator{
		CurrentPage: page,
		PerPage:     perPage,
		MaxPerPage:  100,
	}
}

// IsValid checks if the paginator is valid
func (p *Paginator) IsValid() bool {
	return (p.CurrentPage > 0) && (p.PerPage <= p.MaxPerPage) && (p.PerPage > 0)
}

// Offset returns a valid SQL offset value
func (p *Paginator) Offset() int {
	return (p.CurrentPage - 1) * p.PerPage
}

// Limit returns a valid SQL limit value
func (p *Paginator) Limit() int {
	return p.PerPage
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

const (
	libFolderName   = "lib"
	packageFileName = "package.json"
	baseURL         = "http://search.maven.org/remotecontent?filepath="

	pom     = ".pom"
	jar     = ".jar"
	jarDock = "-javadoc.jar"
	tests   = "-tests.jar"
	source  = "-sources.jar"
)

type dep struct {
	GroupId    string
	ArtifactId string
	Version    string
}

// type deps struct {
// 	dependencies []dep
// }

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func readFile() []dep {
	abspath, _ := os.Getwd()
	filename := abspath + "/" + packageFileName
	// fmt.Println(filename)

	_, err := os.Stat(filename)
	check(err)

	filebyte, err := ioutil.ReadFile(filename)
	check(err)

	var deps []dep

	var depsMap map[string]interface{}
	json.Unmarshal(filebyte, &depsMap)
	// fmt.Printf("Results: %v\n", depsMap)

	for key, value := range depsMap {

		splitName := strings.Split(key, "#")

		str, _ := value.(string)
		deppy := dep{splitName[1], splitName[0], str}

		deps = append(deps, deppy)

		// fmt.Println(deps)
	}

	return deps

}

func downloadFile(depURL string) io.ReadCloser {

	resp, err := http.Get(depURL)
	check(err)
	return resp.Body
}

func runChecks() bool {
	_, err := os.Stat(libFolderName)
	if err != nil {
		return true
	}
	return false
}

func moveFile(file io.ReadCloser, name string) {

	out, err := os.Create(libFolderName + string(os.PathSeparator) + name)
	defer out.Close()

	_, err = io.Copy(out, file)
	check(err)
}

func main() {

	err := runChecks()
	if err {
		fmt.Println("There is not a 'lib' folder in the current directory")
		return //exit
	}

	deps := readFile()

	for _, value := range deps {
		groupIdURL := strings.Replace(value.GroupId, ".", "/", -1)
		file := value.ArtifactId + "-" + value.Version + jar
		finalURL := baseURL + groupIdURL + "/" + value.ArtifactId + "/" + value.Version + "/" + file

		fmt.Println("Going to dowload", finalURL)

		fileDownload := downloadFile(finalURL)
		defer fileDownload.Close()
		moveFile(fileDownload, file)
	}

}
<commit_msg>Update jib.go<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
)

const (
	libFolderName   = "lib"
	packageFileName = "package.json"
	baseURL         = "http://search.maven.org/remotecontent?filepath="

	pom     = ".pom"
	jar     = ".jar"
	jarDock = "-javadoc.jar"
	tests   = "-tests.jar"
	source  = "-sources.jar"
)

type dep struct {
	GroupId    string
	ArtifactId string
	Version    string
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func readFile() []dep {
	abspath, _ := os.Getwd()
	filename := abspath + "/" + packageFileName

	_, err := os.Stat(filename)
	check(err)

	filebyte, err := ioutil.ReadFile(filename)
	check(err)

	var deps []dep

	var depsMap map[string]interface{}
	json.Unmarshal(filebyte, &depsMap)

	for key, value := range depsMap {

		splitName := strings.Split(key, "#")

		str, _ := value.(string)
		deppy := dep{splitName[1], splitName[0], str}

		deps = append(deps, deppy)

		// fmt.Println(deps)
	}

	return deps

}

func downloadFile(depURL string) io.ReadCloser {

	resp, err := http.Get(depURL)
	check(err)
	return resp.Body
}

func runChecks() bool {
	_, err := os.Stat(libFolderName)
	if err != nil {
		return true
	}
	return false
}

func moveFile(file io.ReadCloser, name string) {

	out, err := os.Create(libFolderName + string(os.PathSeparator) + name)
	defer out.Close()

	_, err = io.Copy(out, file)
	check(err)
}

func main() {

	err := runChecks()
	if err {
		fmt.Println("There is not a 'lib' folder in the current directory")
		return //exit
	}

	deps := readFile()

	for _, value := range deps {
		groupIdURL := strings.Replace(value.GroupId, ".", "/", -1)
		file := value.ArtifactId + "-" + value.Version + jar
		finalURL := baseURL + groupIdURL + "/" + value.ArtifactId + "/" + value.Version + "/" + file

		fmt.Println("Going to dowload", finalURL)

		fileDownload := downloadFile(finalURL)
		defer fileDownload.Close()
		moveFile(fileDownload, file)
	}

}
<|endoftext|>


<commit_before><commit_msg>Better error handling for 2-arg `read`.<commit_after><|endoftext|>


<commit_before>// Package mysql parses the mysql slow query log
package mysql

import (
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"

	"github.com/honeycombio/honeytail/event"
	"github.com/percona/go-mysql/query"
)

// 3 sample log entries
//
// # Time: 2016-04-01T00:31:09.817887Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.008393  Lock_time: 0.000154 Rows_sent: 1  Rows_examined: 357
// SET timestamp=1459470669;
// show status like 'Uptime';
//
// # Time: 2016-04-01T00:31:09.853523Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.020424  Lock_time: 0.000147 Rows_sent: 494  Rows_examined: 494
// SET timestamp=1459470669;
// SHOW /*innotop*/ GLOBAL VARIABLES;
//
// # Time: 2016-04-01T00:31:09.856726Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.000021  Lock_time: 0.000000 Rows_sent: 494  Rows_examined: 494
// SET timestamp=1459470669;
// # administrator command: Ping;
//
// We should ignore the administrator command entry; the stats it presents (eg rows_sent)
// are actually for the previous command

var (
	reTime       = myRegexp{regexp.MustCompile("^# Time: (?P<time>[^ ]+)Z *$")}
	reAdminPing  = myRegexp{regexp.MustCompile("^# administrator command: Ping; *$")}
	reUser       = myRegexp{regexp.MustCompile("^# User@Host: (?P<user>[^ ]+) @ (?P<host>[^ ]+).*$")}
	reQueryStats = myRegexp{regexp.MustCompile("^# Query_time: (?P<queryTime>[0-9.]+) *Lock_time: (?P<lockTime>[0-9.]+) *Rows_sent: (?P<rowsSent>[0-9]+) *Rows_examined: (?P<rowsExamined>[0-9]+) *$")}
	reSetTime    = myRegexp{regexp.MustCompile("^SET timestamp=(?P<unixTime>[0-9]+);$")}
	reQuery      = myRegexp{regexp.MustCompile("^(?P<query>[^#]*).*$")}
	reUse        = myRegexp{regexp.MustCompile("^(?i)use ")}
)

const timeFormat = "2006-01-02T15:04:05.000000"

type Options struct {
}

type Parser struct {
	conf  Options
	wg    sync.WaitGroup
	nower Nower
}

type Nower interface {
	Now() time.Time
}

type RealNower struct{}

func (n *RealNower) Now() time.Time {
	return time.Now().UTC()
}

// rawEvent is the unparsed multi-line entry representing a single slow query
type rawEvent struct {
	lines []string
}

// slowQuery represents the structured form of a query from the slow query log
type SlowQuery struct {
	Timestamp       time.Time `json:"time"`
	UnixTime        int       `json:"unixtime"`
	User            string    `json:"user"`
	Host            string    `json:"host"`
	QueryTime       float64   `json:"query_time"`
	LockTime        float64   `json:"lock_time"`
	RowsSent        int       `json:"rows_sent"`
	RowsExamined    int       `json:"rows_examined"`
	Query           string    `json:"query",omitempty`
	NormalizedQuery string    `json:"normalized_query,omitempty"`
	DB              string    `json:"db,omitempty"`
	skipQuery       bool
}

func (p *Parser) Init(_ interface{}) error {
	p.nower = &RealNower{}
	return nil
}

func (p *Parser) ProcessLines(lines <-chan string, send chan<- event.Event) {
	// start up a goroutine to handle grouped sets of lines
	rawEvents := make(chan rawEvent)
	var wg sync.WaitGroup
	p.wg = wg
	defer p.wg.Wait()
	p.wg.Add(1)
	go p.handleEvents(rawEvents, send)

	// flag to indicate when we've got a complete event to send
	var sendEvent bool
	groupedLines := make([]string, 0, 5)
	for line := range lines {
		if strings.HasPrefix(line, "# Time: ") {
			// we've started a new event. Send the previous one.
			sendEvent = true
		}
		if sendEvent {
			sendEvent = false
			rawEvents <- rawEvent{lines: groupedLines}
			groupedLines = make([]string, 0, 5)
		}
		groupedLines = append(groupedLines, line)
	}
	if len(groupedLines) != 0 {
		rawEvents <- rawEvent{lines: groupedLines}
	}
	logrus.Debug("lines channel is closed, ending mysql processor")
	close(rawEvents)
}

func (p *Parser) handleEvents(rawEvents <-chan rawEvent, send chan<- event.Event) {
	defer p.wg.Done()
	for rawE := range rawEvents {
		sq := p.handleEvent(rawE)
		ev, err := p.processSlowQuery(sq)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"err":       err,
				"slowQuery": sq,
			}).Debug("skipping query")
			continue
		}
		send <- ev
	}
	logrus.Debug("done with mysql handleEvents")
}

// handle a single event
func (p *Parser) handleEvent(rawE rawEvent) SlowQuery {
	var err error
	sq := SlowQuery{}
	logrus.WithFields(logrus.Fields{
		"rawE": rawE,
	}).Debug("About to parse event")
	for _, line := range rawE.lines {
		// parse each line and populate the SlowQuery object
		switch {
		case reTime.MatchString(line):
			matchGroups := reTime.FindStringSubmatchMap(line)
			sq.Timestamp, err = time.Parse(timeFormat, matchGroups["time"])
			if err != nil {
				sq.Timestamp = p.nower.Now()
			}
		case reAdminPing.MatchString(line):
			// this evetn is an administrative ping and we should
			// ignore the entire event
			logrus.WithFields(logrus.Fields{
				"line":  line,
				"event": rawE,
			}).Debug("readmin ping detected; skipping this event")
			sq.skipQuery = true
		case reUser.MatchString(line):
			matchGroups := reUser.FindStringSubmatchMap(line)
			sq.User = matchGroups["user"]
			sq.Host = matchGroups["host"]
		case reQueryStats.MatchString(line):
			matchGroups := reQueryStats.FindStringSubmatchMap(line)
			sq.QueryTime, err = strconv.ParseFloat(matchGroups["queryTime"], 64)
			sq.LockTime, err = strconv.ParseFloat(matchGroups["lockTime"], 64)
			sq.RowsSent, err = strconv.Atoi(matchGroups["rowsSent"])
			sq.RowsExamined, err = strconv.Atoi(matchGroups["rowsExamined"])
		case reUse.FindString(line) != "":
			db := strings.TrimPrefix(line, reUse.FindString(line))
			db = strings.TrimRight(db, ";")
			db = strings.Trim(db, "`")
			sq.DB = db
			// Use this line as the query unless, if a real query follows it will be replaced.
			sq.Query = line
		case reSetTime.MatchString(line):
			matchGroups := reSetTime.FindStringSubmatchMap(line)
			sq.UnixTime, err = strconv.Atoi(matchGroups["unixTime"])
		case reQuery.MatchString(line):
			matchGroups := reQuery.FindStringSubmatchMap(line)
			sq.Query = matchGroups["query"]
			sq.NormalizedQuery = query.Fingerprint(sq.Query)
		default:
			// unknown row; log and skip
			logrus.WithFields(logrus.Fields{
				"line": line,
			}).Debug("No regex match for line in the middle of a query. skipping")
		}
	}
	// We always need a timestamp; if we never found the line, use Now()
	if sq.Timestamp.IsZero() {
		sq.Timestamp = p.nower.Now()
	}
	return sq
}

// custom error to indicate empty query
type emptyQueryError struct {
	err string
}

func (e *emptyQueryError) Error() string {
	e.err = "skipped slow query"
	return e.err
}

func (p *Parser) processSlowQuery(sq SlowQuery) (event.Event, error) {
	// if we didn't match any lines at all, skip the query
	if sq == (SlowQuery{}) {
		sq.skipQuery = true
	}
	// OK, we've collected all the lines, send in the event
	if !sq.skipQuery {
		return event.Event{
			Timestamp: sq.Timestamp,
			Data:      sq.mapify(),
		}, nil
	}
	// we're skipping this query
	return event.Event{}, &emptyQueryError{}
}

func (s SlowQuery) mapify() map[string]interface{} {
	return map[string]interface{}{
		"time":             s.Timestamp,
		"unixtime":         s.UnixTime,
		"user":             s.User,
		"host":             s.Host,
		"query_time":       s.QueryTime,
		"lock_time":        s.LockTime,
		"rows_sent":        s.RowsSent,
		"rows_examined":    s.RowsExamined,
		"query":            s.Query,
		"normalized_query": s.NormalizedQuery,
	}
}
<commit_msg>Add support for detection of RDS host<commit_after>// Package mysql parses the mysql slow query log
package mysql

import (
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"database/sql"

	"github.com/Sirupsen/logrus"
	_ "github.com/go-sql-driver/mysql"
	"github.com/honeycombio/honeytail/event"
	"github.com/percona/go-mysql/query"
)

// 3 sample log entries
//
// # Time: 2016-04-01T00:31:09.817887Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.008393  Lock_time: 0.000154 Rows_sent: 1  Rows_examined: 357
// SET timestamp=1459470669;
// show status like 'Uptime';
//
// # Time: 2016-04-01T00:31:09.853523Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.020424  Lock_time: 0.000147 Rows_sent: 494  Rows_examined: 494
// SET timestamp=1459470669;
// SHOW /*innotop*/ GLOBAL VARIABLES;
//
// # Time: 2016-04-01T00:31:09.856726Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.000021  Lock_time: 0.000000 Rows_sent: 494  Rows_examined: 494
// SET timestamp=1459470669;
// # administrator command: Ping;
//
// We should ignore the administrator command entry; the stats it presents (eg rows_sent)
// are actually for the previous command

var (
	reTime       = myRegexp{regexp.MustCompile("^# Time: (?P<time>[^ ]+)Z *$")}
	reAdminPing  = myRegexp{regexp.MustCompile("^# administrator command: Ping; *$")}
	reUser       = myRegexp{regexp.MustCompile("^# User@Host: (?P<user>[^ ]+) @ (?P<host>[^ ]+).*$")}
	reQueryStats = myRegexp{regexp.MustCompile("^# Query_time: (?P<queryTime>[0-9.]+) *Lock_time: (?P<lockTime>[0-9.]+) *Rows_sent: (?P<rowsSent>[0-9]+) *Rows_examined: (?P<rowsExamined>[0-9]+) *$")}
	reSetTime    = myRegexp{regexp.MustCompile("^SET timestamp=(?P<unixTime>[0-9]+);$")}
	reQuery      = myRegexp{regexp.MustCompile("^(?P<query>[^#]*).*$")}
	reUse        = myRegexp{regexp.MustCompile("^(?i)use ")}

	rdsStr  = "rds"
	ec2Str  = "ec2"
	selfStr = "self"
)

const timeFormat = "2006-01-02T15:04:05.000000"

type Options struct {
	Host string `long:"host" description:"MySQL host in the format (address:port)"`
	User string `long:"user" description:"MySQL username"`
	Pass string `long:"pass" description:"MySQL password"`
}

type Parser struct {
	conf     Options
	wg       sync.WaitGroup
	nower    Nower
	hostedOn string
}

type Nower interface {
	Now() time.Time
}

type RealNower struct{}

func (n *RealNower) Now() time.Time {
	return time.Now().UTC()
}

// rawEvent is the unparsed multi-line entry representing a single slow query
type rawEvent struct {
	lines []string
}

// slowQuery represents the structured form of a query from the slow query log
type SlowQuery struct {
	Timestamp       time.Time `json:"time"`
	UnixTime        int       `json:"unixtime"`
	User            string    `json:"user"`
	Host            string    `json:"host"`
	QueryTime       float64   `json:"query_time"`
	LockTime        float64   `json:"lock_time"`
	RowsSent        int       `json:"rows_sent"`
	RowsExamined    int       `json:"rows_examined"`
	Query           string    `json:"query,omitempty"`
	NormalizedQuery string    `json:"normalized_query,omitempty"`
	DB              string    `json:"db,omitempty"`
	HostedOn        string    `json:"hosted_on,omitempty"`
	skipQuery       bool
}

func (p *Parser) Init(options interface{}) error {
	p.conf = *options.(*Options)
	p.nower = &RealNower{}
	if len(p.conf.Host) > 0 {
		p.hostedOn = getHostedOn(p.conf.Host, p.conf.User, p.conf.Pass)
	}
	return nil
}

func getHostedOn(host, user, pass string) string {
	url := fmt.Sprintf("%s:%s@tcp(%s)/", user, pass, host)
	db, err := sql.Open("mysql", url)
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	rows, err := db.Query("SHOW GLOBAL VARIABLES like 'basedir';")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()

	var varName, value string

	for rows.Next() {
		if err := rows.Scan(&varName, &value); err != nil {
			log.Fatal(err)
		}
		if strings.HasPrefix(value, "/rdsdbbin/") {
			return rdsStr
		}
	}

	// TODO: implement ec2 detection

	if err = rows.Err(); err != nil {
		log.Fatal(err)
	}
	return selfStr
}

func (p *Parser) ProcessLines(lines <-chan string, send chan<- event.Event) {
	// start up a goroutine to handle grouped sets of lines
	rawEvents := make(chan rawEvent)
	var wg sync.WaitGroup
	p.wg = wg
	defer p.wg.Wait()
	p.wg.Add(1)
	go p.handleEvents(rawEvents, send)

	// flag to indicate when we've got a complete event to send
	var sendEvent bool
	groupedLines := make([]string, 0, 5)
	for line := range lines {
		if strings.HasPrefix(line, "# Time: ") {
			// we've started a new event. Send the previous one.
			sendEvent = true
		}
		if sendEvent {
			sendEvent = false
			rawEvents <- rawEvent{lines: groupedLines}
			groupedLines = make([]string, 0, 5)
		}
		groupedLines = append(groupedLines, line)
	}
	if len(groupedLines) != 0 {
		rawEvents <- rawEvent{lines: groupedLines}
	}
	logrus.Debug("lines channel is closed, ending mysql processor")
	close(rawEvents)
}

func (p *Parser) handleEvents(rawEvents <-chan rawEvent, send chan<- event.Event) {
	defer p.wg.Done()
	for rawE := range rawEvents {
		sq := p.handleEvent(rawE)
		sq.HostedOn = p.hostedOn
		ev, err := p.processSlowQuery(sq)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"err":       err,
				"slowQuery": sq,
			}).Debug("skipping query")
			continue
		}
		send <- ev
	}
	logrus.Debug("done with mysql handleEvents")
}

// handle a single event
func (p *Parser) handleEvent(rawE rawEvent) SlowQuery {
	var err error
	sq := SlowQuery{}
	logrus.WithFields(logrus.Fields{
		"rawE": rawE,
	}).Debug("About to parse event")
	for _, line := range rawE.lines {
		// parse each line and populate the SlowQuery object
		switch {
		case reTime.MatchString(line):
			matchGroups := reTime.FindStringSubmatchMap(line)
			sq.Timestamp, err = time.Parse(timeFormat, matchGroups["time"])
			if err != nil {
				sq.Timestamp = p.nower.Now()
			}
		case reAdminPing.MatchString(line):
			// this evetn is an administrative ping and we should
			// ignore the entire event
			logrus.WithFields(logrus.Fields{
				"line":  line,
				"event": rawE,
			}).Debug("readmin ping detected; skipping this event")
			sq.skipQuery = true
		case reUser.MatchString(line):
			matchGroups := reUser.FindStringSubmatchMap(line)
			sq.User = matchGroups["user"]
			sq.Host = matchGroups["host"]
		case reQueryStats.MatchString(line):
			matchGroups := reQueryStats.FindStringSubmatchMap(line)
			sq.QueryTime, err = strconv.ParseFloat(matchGroups["queryTime"], 64)
			sq.LockTime, err = strconv.ParseFloat(matchGroups["lockTime"], 64)
			sq.RowsSent, err = strconv.Atoi(matchGroups["rowsSent"])
			sq.RowsExamined, err = strconv.Atoi(matchGroups["rowsExamined"])
		case reUse.FindString(line) != "":
			db := strings.TrimPrefix(line, reUse.FindString(line))
			db = strings.TrimRight(db, ";")
			db = strings.Trim(db, "`")
			sq.DB = db
			// Use this line as the query unless, if a real query follows it will be replaced.
			sq.Query = line
		case reSetTime.MatchString(line):
			matchGroups := reSetTime.FindStringSubmatchMap(line)
			sq.UnixTime, err = strconv.Atoi(matchGroups["unixTime"])
		case reQuery.MatchString(line):
			matchGroups := reQuery.FindStringSubmatchMap(line)
			sq.Query = matchGroups["query"]
			sq.NormalizedQuery = query.Fingerprint(sq.Query)
		default:
			// unknown row; log and skip
			logrus.WithFields(logrus.Fields{
				"line": line,
			}).Debug("No regex match for line in the middle of a query. skipping")
		}
	}
	// We always need a timestamp; if we never found the line, use Now()
	if sq.Timestamp.IsZero() {
		sq.Timestamp = p.nower.Now()
	}
	return sq
}

// custom error to indicate empty query
type emptyQueryError struct {
	err string
}

func (e *emptyQueryError) Error() string {
	e.err = "skipped slow query"
	return e.err
}

func (p *Parser) processSlowQuery(sq SlowQuery) (event.Event, error) {
	// if we didn't match any lines at all, skip the query
	if sq == (SlowQuery{}) {
		sq.skipQuery = true
	}
	// OK, we've collected all the lines, send in the event
	if !sq.skipQuery {
		return event.Event{
			Timestamp: sq.Timestamp,
			Data:      sq.mapify(),
		}, nil
	}
	// we're skipping this query
	return event.Event{}, &emptyQueryError{}
}

func (s SlowQuery) mapify() map[string]interface{} {
	return map[string]interface{}{
		"time":             s.Timestamp,
		"unixtime":         s.UnixTime,
		"user":             s.User,
		"host":             s.Host,
		"query_time":       s.QueryTime,
		"lock_time":        s.LockTime,
		"rows_sent":        s.RowsSent,
		"rows_examined":    s.RowsExamined,
		"query":            s.Query,
		"normalized_query": s.NormalizedQuery,
		"hosted_on":        s.HostedOn,
	}
}
<|endoftext|>


<commit_before>package schedule

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/slok/khronos/job"
)

func TestDummyScheduler(t *testing.T) {

	tests := []struct {
		givenExitStatus int
		givenOut        string
	}{
		{givenExitStatus: job.ResultOK, givenOut: "test1"},
		{givenExitStatus: job.ResultError, givenOut: "test2"},
		{givenExitStatus: job.ResultUnknow, givenOut: "test3"},
	}

	for _, test := range tests {
		u, _ := url.Parse("http://test.org/test")
		j := &job.Job{URL: u}
		r := &job.Result{}

		DummyScheduler(test.givenExitStatus, test.givenOut,
			SchedulerFunc(func(r *job.Result, j *job.Job) {})).
			Run(r, j)

		if test.givenOut != r.Out {
			t.Errorf("result output should be: %s; got: %s", test.givenOut, r.Out)
		}
		if test.givenExitStatus != r.Status {
			t.Errorf("result exit status should be: %d; got: %d", test.givenExitStatus, r.Status)
		}
	}
}

func TestTimingScheduler(t *testing.T) {
	j := &job.Job{}
	r := &job.Result{}

	TimingScheduler(SchedulerFunc(func(r *job.Result, j *job.Job) {})).Run(r, j)

	if !r.Start.Before(r.Finish) || !r.Finish.After(r.Start) {
		t.Errorf("Wrong timing on job execution, start: %v; finish: %v", r.Start, r.Finish)
	}
}

func TestHTTPScheduler(t *testing.T) {
	tests := []struct {
		givenBody  string
		givenCode  int
		wantStatus int
		timeout    time.Duration
	}{
		{
			givenBody:  "Job finished without errors",
			givenCode:  http.StatusOK,
			wantStatus: job.ResultOK,
			timeout:    2 * time.Second,
		},
		{
			givenBody:  "Job finished with errors",
			givenCode:  http.StatusInternalServerError,
			wantStatus: job.ResultError,
			timeout:    2 * time.Second,
		},
		{
			givenBody:  "dunno what do you want...",
			givenCode:  http.StatusBadRequest,
			wantStatus: job.ResultUnknow,
			timeout:    2 * time.Second,
		},
		{
			wantStatus: job.ResultInternalError,
			timeout:    3,
		},
	}

	for _, test := range tests {
		// Create our fake server
		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(test.givenCode)
			fmt.Fprint(w, test.givenBody)
		}))
		defer ts.Close()

		// Create the job
		u, _ := url.Parse(ts.URL)
		j := &job.Job{
			URL: u,
		}
		r := &job.Result{}

		// Use the scheduler
		HTTPScheduler(test.timeout, SchedulerFunc(func(r *job.Result, j *job.Job) {})).Run(r, j)

		// Check result is ok
		// Only check the body of calls that where returned ok
		if test.wantStatus != job.ResultInternalError && test.givenBody != r.Out {
			t.Errorf("result output should be: %s; got: %s", test.givenBody, r.Out)
		}
		// Always check result exit code
		if test.wantStatus != r.Status {
			t.Errorf("result exit status should be: %d; got: %d", test.wantStatus, r.Status)
		}
	}
}
<commit_msg>Set timeout correct for http scheduler timeout case<commit_after>package schedule

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/slok/khronos/job"
)

func TestDummyScheduler(t *testing.T) {

	tests := []struct {
		givenExitStatus int
		givenOut        string
	}{
		{givenExitStatus: job.ResultOK, givenOut: "test1"},
		{givenExitStatus: job.ResultError, givenOut: "test2"},
		{givenExitStatus: job.ResultUnknow, givenOut: "test3"},
	}

	for _, test := range tests {
		u, _ := url.Parse("http://test.org/test")
		j := &job.Job{URL: u}
		r := &job.Result{}

		DummyScheduler(test.givenExitStatus, test.givenOut,
			SchedulerFunc(func(r *job.Result, j *job.Job) {})).
			Run(r, j)

		if test.givenOut != r.Out {
			t.Errorf("result output should be: %s; got: %s", test.givenOut, r.Out)
		}
		if test.givenExitStatus != r.Status {
			t.Errorf("result exit status should be: %d; got: %d", test.givenExitStatus, r.Status)
		}
	}
}

func TestTimingScheduler(t *testing.T) {
	j := &job.Job{}
	r := &job.Result{}

	TimingScheduler(SchedulerFunc(func(r *job.Result, j *job.Job) {})).Run(r, j)

	if !r.Start.Before(r.Finish) || !r.Finish.After(r.Start) {
		t.Errorf("Wrong timing on job execution, start: %v; finish: %v", r.Start, r.Finish)
	}
}

func TestHTTPScheduler(t *testing.T) {
	tests := []struct {
		givenBody  string
		givenCode  int
		wantStatus int
		timeout    time.Duration
	}{
		{
			givenBody:  "Job finished without errors",
			givenCode:  http.StatusOK,
			wantStatus: job.ResultOK,
			timeout:    2 * time.Second,
		},
		{
			givenBody:  "Job finished with errors",
			givenCode:  http.StatusInternalServerError,
			wantStatus: job.ResultError,
			timeout:    2 * time.Second,
		},
		{
			givenBody:  "dunno what do you want...",
			givenCode:  http.StatusBadRequest,
			wantStatus: job.ResultUnknow,
			timeout:    2 * time.Second,
		},
		{
			wantStatus: job.ResultInternalError,
			timeout:    0,
		},
	}

	for _, test := range tests {
		// Create our fake server
		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			w.WriteHeader(test.givenCode)
			fmt.Fprint(w, test.givenBody)
		}))
		defer ts.Close()

		// Create the job
		u, _ := url.Parse(ts.URL)
		j := &job.Job{
			URL: u,
		}
		r := &job.Result{}

		// Use the scheduler
		HTTPScheduler(test.timeout, SchedulerFunc(func(r *job.Result, j *job.Job) {})).Run(r, j)

		// Check result is ok
		// Only check the body of calls that where returned ok
		if test.wantStatus != job.ResultInternalError && test.givenBody != r.Out {
			t.Errorf("result output should be: %s; got: %s", test.givenBody, r.Out)
		}
		// Always check result exit code
		if test.wantStatus != r.Status {
			t.Errorf("result exit status should be: %d; got: %d", test.wantStatus, r.Status)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"testing"
)

var basicHTML = `
	<!DOCTYPE html>

    <title>My Title</title>

This is a header

I want to go toDA CNN

This is a link with some images inside

`

var invalidHTML = `
  <!DOCTYPE html>

    <title>Weird nested links</title>

      What happens if you do this?

`

func TestCanExtractLinksFromPage(t *testing.T) {
	links, warnings, err := ExtractLinksFromPage("http://www.foo.com", strings.NewReader(basicHTML))
	if err != nil {
		t.Fatalf("Expected no error from basicHTML, got %v", err)
	}

	if len(warnings) != 0 {
		t.Errorf("Expected 0 warnings from parsing basicHTML, got %d", len(warnings))
	}

	if len(links) != 2 {
		t.Fatalf("Expected 2 links in basicHTML, found %d", len(links))
	}

	checkLink(t, links[0], "http://www.cnn.com", "DA CNN")
	checkLink(t, links[1], "http://www.complicated.com", "This is a link with some images inside")
}

func TestCanExtractRelativeLinks(t *testing.T) {

}

func TestReturnsErrorOnRandomPage(t *testing.T) {

}

func checkLink(t *testing.T, l Link, expectedHref string, expectedAnchor string) {
	if l.URL != expectedHref || l.anchorText != expectedAnchor {
		t.Errorf("Expected link to be href %s, anchor %s, instead was %s, anchor %s",
			expectedHref, expectedAnchor, l.URL, l.anchorText)
	}
}
<commit_msg>complete basic tests for link extraction<commit_after>package main

import (
	"strings"
	"testing"
)

var basicHTML = `
	<!DOCTYPE html>

    <title>My Title</title>

This is a header

I want to go toDA CNN

This is a link with some images inside

`

var relativeLinkHTML = `
	<!DOCTYPE html>

    <title>My Title</title>

This is a header

I want to go toLeading Slash

Relative

`

var invalidHTML = `
  This is not a
Well-formed PAGE
`

func TestCanExtractLinksFromPage(t *testing.T) {
	links, warnings, err := ExtractLinksFromPage("http://www.foo.com", strings.NewReader(basicHTML))
	if err != nil {
		t.Fatalf("Expected no error from basicHTML, got %v", err)
	}

	if len(warnings) != 0 {
		t.Errorf("Expected 0 warnings from parsing basicHTML, got %d", len(warnings))
	}

	if len(links) != 2 {
		t.Fatalf("Expected 2 links in basicHTML, found %d", len(links))
	}

	checkLink(t, links[0], "http://www.cnn.com", "DA CNN")
	checkLink(t, links[1], "http://www.complicated.com", "This is a link with some images inside")
}

func TestCanExtractRelativeLinks(t *testing.T) {
	links, warnings, err := ExtractLinksFromPage("http://www.bar.com/some_dir/some_subdir/", strings.NewReader(relativeLinkHTML))
	if err != nil {
		t.Fatalf("Expected no error from relativeLinkHTML, got %v", err)
	}

	if len(warnings) != 0 {
		t.Errorf("Expected 0 warnings from parsing relativeLinkHTML, got %d", len(warnings))
	}

	if len(links) != 2 {
		t.Fatalf("Expected 2 links in relativeLinkHTML, found %d", len(links))
	}

	checkLink(t, links[0], "http://www.bar.com/leading-slash", "Leading Slash")
	checkLink(t, links[1], "http://www.bar.com/some_dir/relative", "Relative")

}

func TestReturnsErrorOnRandomBasePage(t *testing.T) {
	_, _, err := ExtractLinksFromPage("1\\/%@!qwertyuiop!", strings.NewReader(invalidHTML))

	if err == nil {
		t.Fatalf("Expected error from errorHTML, got nil")
	}
}

func checkLink(t *testing.T, l Link, expectedHref string, expectedAnchor string) {
	if l.URL != expectedHref || l.anchorText != expectedAnchor {
		t.Errorf("Expected link to be href %s, anchor %s, instead was %s, anchor %s",
			expectedHref, expectedAnchor, l.URL, l.anchorText)
	}
}
<|endoftext|>


<commit_before>// +build linux
package facts

import (
	"bufio"
	"fmt"
	"github.com/woosley/gogate/gate/types"
	"github.com/woosley/gogate/gate/utils"
	"io/ioutil"
	"os"
	"strconv"
	"strings"
)

var osfiles []string = []string{
	"/etc/redhat-release",
	"/etc/os-release",
}

const (
	AllBlocksDir     = "/sys/block"
	VirtualBlocksDir = "/sys/devices/virtual/block"
)

const unknown string = "Unknown"
const zero int64 = 0

func getOsFile(osfs []string) string {
	for _, f := range osfs {
		if exists, err := utils.IsFile(f); err == nil && exists {
			return f
		}
	}
	return ""
}

func GetOs() string {

	switch f := getOsFile(osfiles); f {
	case "/etc/redhat-release":
		if bytes, err := ioutil.ReadFile("/etc/redhat-release"); err == nil {
			return strings.TrimSpace(string(bytes))
		}
	case "/etc/os-release":
		return getFromOsRelease(f)

	default:
		return unknown
	}
	return unknown
}

//getFromOsRelease return distribution name from /etc/os-release
func getFromOsRelease(f string) string {
	fd, err := os.Open(f)
	if err != nil {
		return unknown
	}

	fscanner := bufio.NewScanner(fd)

	for fscanner.Scan() {
		line := fscanner.Text()
		if strings.Contains(line, "=") {
			kv := strings.SplitN(line, "=", 2)
			k, v := strings.TrimSpace(kv[0]), strings.TrimSpace(kv[1])
			if k == "PRETTY_NAME" {
				return strings.Trim(v, "\"")
			}
		}
	}
	return unknown
}

func GetCpu() (types.CpuInfo, error) {
	var cpu types.CpuInfo
	if fd, err := os.Open("/proc/cpuinfo"); err != nil {
		return cpu, err
	} else {
		cpuinfo := make(map[string]string)
		fscanner := bufio.NewScanner(fd)
		for fscanner.Scan() {
			line := fscanner.Text()
			if strings.Contains(line, ":") {
				kv := strings.SplitN(fscanner.Text(), ":", 2)
				cpuinfo[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
			}
		}
		cpu.Count, _ = strconv.Atoi(cpuinfo["physical id"])
		cpu.Count += 1
		cpu.Cores, _ = strconv.Atoi(cpuinfo["cpu cores"])
		return cpu, nil
	}
}

//getMemoryLinux returns both MemTotal and SwapTotal
func GetMemory() (string, string, error) {
	if fd, err := os.Open("/proc/meminfo"); err != nil {
		return "", "", err
	} else {
		mem := make(map[string]string)
		fscanner := bufio.NewScanner(fd)
		for fscanner.Scan() {
			kv := strings.SplitN(fscanner.Text(), ":", 2)
			mem[kv[0]] = strings.Trim(kv[1], " ")
		}
		return mem["MemTotal"], mem["SwapTotal"], nil
	}
}

func GetUptime() int64 {
	fname := "/proc/uptime"
	if bytes, err := ioutil.ReadFile(fname); err == nil {
		up := strings.SplitN(strings.TrimSpace(string(bytes)), " ", 2)[0]
		upt, err := strconv.ParseFloat(up, 64)
		if err != nil {
			return zero
		} else {
			return int64(upt)
		}
	}
	return zero
}

func GetDiskInfo() []types.DiskInfo {
	// lsdir /sys/block
	// lsdir /sys/devices/virtual/block
	// /sys/block/sda/queue/hw_sector_size disk sector size
	// /sys/block/sda/size disk sectors
	pdisks := make([]types.DiskInfo, 0)
	disks, _ := utils.LsDir(AllBlocksDir)
	vdisks, _ := utils.LsDir(VirtualBlocksDir)

	// physical is what listed in AllBlocksDir but not in VirtualBlocksDir
	for _, v := range disks {
		if !utils.ListHasString(vdisks, v) {
			size := GetDiskSize(v)
			pdisks = append(pdisks, types.DiskInfo{Name: v, Size: size})
		}
	}
	return pdisks
}

func GetDiskSize(disk string) string {
	sectors, _ := utils.SlurpFile(fmt.Sprintf("%s/%s/size", AllBlocksDir, disk))
	sectorSize, _ := utils.SlurpFile(fmt.Sprintf("%s/%s/queue/hw_sector_size", AllBlocksDir, disk))
	s1, _ := strconv.Atoi(sectors[0])
	s2, _ := strconv.Atoi(sectorSize[0])
	return fmt.Sprintf("%vMB", s1*s2/1000/1000)

}
<commit_msg>updates<commit_after>// +build linux
package facts

import (
	"bufio"
	"github.com/woosley/gogate/gate/types"
	"github.com/woosley/gogate/gate/utils"
	"io/ioutil"
	"fmt"
	"os"
	"strconv"
	"strings"
)

var osfiles []string = []string{
	"/etc/redhat-release",
	"/etc/os-release",
}

const  (
	AllBlocksDir = "/sys/block"
	VirtualBlocksDir = "/sys/devices/virtual/block"
)

const unknown string = "Unknown"
const zero int64 = 0

func getOsFile(osfs []string) string {
	for _, f := range osfs {
		if exists, err := utils.IsFile(f); err == nil && exists {
			return f
		}
	}
	return ""
}

func GetOs() string {

	switch f := getOsFile(osfiles); f {
	case "/etc/redhat-release":
		if bytes, err := ioutil.ReadFile("/etc/redhat-release"); err == nil {
			return strings.TrimSpace(string(bytes))
		}
	case "/etc/os-release":
		return getFromOsRelease(f)

	default:
		return unknown
	}
	return unknown
}

//getFromOsRelease return distribution name from /etc/os-release
func getFromOsRelease(f string) string {
	fd, err := os.Open(f)
	if err != nil {
		return unknown
	}

	fscanner := bufio.NewScanner(fd)

	for fscanner.Scan() {
		line := fscanner.Text()
		if strings.Contains(line, "=") {
			kv := strings.SplitN(line, "=", 2)
			k, v := strings.TrimSpace(kv[0]), strings.TrimSpace(kv[1])
			if k == "PRETTY_NAME" {
				return strings.Trim(v, "\"")
			}
		}
	}
	return unknown
}

func GetCpu() (types.CpuInfo, error) {
	var cpu types.CpuInfo
	if fd, err := os.Open("/proc/cpuinfo"); err != nil {
		return cpu, err
	} else {
		cpuinfo := make(map[string]string)
		fscanner := bufio.NewScanner(fd)
		for fscanner.Scan() {
			line := fscanner.Text()
			if strings.Contains(line, ":") {
				kv := strings.SplitN(fscanner.Text(), ":", 2)
				cpuinfo[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
			}
		}
		cpu.Count, _ = strconv.Atoi(cpuinfo["physical id"])
		cpu.Count += 1
		cpu.Cores, _ = strconv.Atoi(cpuinfo["cpu cores"])
		return cpu, nil
	}
}

//getMemoryLinux returns both MemTotal and SwapTotal
func GetMemory() (string, string, error) {
	if fd, err := os.Open("/proc/meminfo"); err != nil {
		return "", "", err
	} else {
		mem := make(map[string]string)
		fscanner := bufio.NewScanner(fd)
		for fscanner.Scan() {
			kv := strings.SplitN(fscanner.Text(), ":", 2)
			mem[kv[0]] = strings.Trim(kv[1], " ")
		}
		return mem["MemTotal"], mem["SwapTotal"], nil
	}
}

func GetUptime() int64 {
	fname := "/proc/uptime"
	if bytes, err := ioutil.ReadFile(fname); err == nil {
		up := strings.SplitN(strings.TrimSpace(string(bytes)), " ", 2)[0]
		upt, err := strconv.ParseFloat(up, 64)
		if err != nil {
			return zero
		} else {
			return int64(upt)
		}
	}
	return zero
}

func GetDiskInfo() []types.DiskInfo{
	// lsdir /sys/block
	// lsdir /sys/devices/virtual/block
	// /sys/block/sda/queue/hw_sector_size disk sector size
	// /sys/block/sda/size disk sectors
	pdisks := make([]types.DiskInfo, 0)
	disks, _ := utils.LsDir(AllBlocksDir)
	vdisks, _ := utils.LsDir(VirtualBlocksDir)

	// physical is what listed in AllBlocksDir but not in VirtualBlocksDir
	for _, v := range(disks) {
		if !utils.ListHasString(vdisks, v) {
			size := GetDiskSize(v)
			pdisks = append(pdisks, types.DiskInfo{Name: v, Size: size})
		}
	}
	return pdisks
}

func GetDiskSize(disk string) string {
	sectors, _ := utils.SlurpFile(fmt.Sprintf("%s/%s/size", AllBlocksDir, disk))
	sectorSize, _ := utils.SlurpFile(fmt.Sprintf("%s/%s/queue/hw_sector_size", AllBlocksDir, disk))
	s1, _ := strconv.Atoi(sectors[0])
	s2, _ := strconv.Atoi(sectorSize[0])
	return fmt.Sprintf("%vMB", s1 * s2/1000/1000)

}
<|endoftext|>


<commit_before>package pipelines_test

import (
	"github.com/concourse/testflight/gitserver"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Configuring a resource in a pipeline config", func() {
	var originGitServer *gitserver.Server

	BeforeEach(func() {
		originGitServer = gitserver.Start(gitServerRootfs, gardenClient)
		originGitServer.CommitResource()

		configurePipeline(
			"-c", "fixtures/resource-types.yml",
			"-v", "testflight-helper-image="+guidServerRootfs,
			"-v", "origin-git-server="+originGitServer.URI(),
		)
	})

	AfterEach(func() {
		originGitServer.Stop()
	})

	It("can use custom resource types for 'get'", func() {
		watch := flyWatch("resource-getter")
		Expect(watch).To(gbytes.Say("rootfs/some-file"))
	})

	It("can use custom resource types for 'put'", func() {
		watch := flyWatch("resource-putter")
		Expect(watch).To(gbytes.Say("pushing using custom resource"))
		Expect(watch).To(gbytes.Say("some-output/some-file"))
	})

	It("can use custom resource types for a task's 'image_resource'", func() {
		watch := flyWatch("resource-imgur")
		Expect(watch).To(gbytes.Say("fetched from custom resource"))
		Expect(watch).To(gbytes.Say("SOME_ENV=yep"))
	})
})
<commit_msg>squash resource_type tests into one example<commit_after>package pipelines_test

import (
	"github.com/concourse/testflight/gitserver"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
)

var _ = Describe("Configuring a resource in a pipeline config", func() {
	var originGitServer *gitserver.Server

	BeforeEach(func() {
		originGitServer = gitserver.Start(gitServerRootfs, gardenClient)
		originGitServer.CommitResource()

		configurePipeline(
			"-c", "fixtures/resource-types.yml",
			"-v", "testflight-helper-image="+guidServerRootfs,
			"-v", "origin-git-server="+originGitServer.URI(),
		)
	})

	AfterEach(func() {
		originGitServer.Stop()
	})

	It("can use custom resource types for 'get', 'put', and task 'image_resource's", func() {
		watch := flyWatch("resource-getter")
		Expect(watch).To(gbytes.Say("rootfs/some-file"))

		watch = flyWatch("resource-putter")
		Expect(watch).To(gbytes.Say("pushing using custom resource"))
		Expect(watch).To(gbytes.Say("some-output/some-file"))

		watch = flyWatch("resource-imgur")
		Expect(watch).To(gbytes.Say("fetched from custom resource"))
		Expect(watch).To(gbytes.Say("SOME_ENV=yep"))
	})
})
<|endoftext|>


<commit_before>package pluginproxy

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/api/datasource"
	"github.com/grafana/grafana/pkg/infra/httpclient"
	glog "github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/infra/tracing"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/services/datasources"
	"github.com/grafana/grafana/pkg/services/oauthtoken"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
	"github.com/grafana/grafana/pkg/util/proxyutil"
	"go.opentelemetry.io/otel/attribute"
)

var (
	logger = glog.New("data-proxy-log")
	client = newHTTPClient()
)

type DataSourceProxy struct {
	ds                 *models.DataSource
	ctx                *models.ReqContext
	targetUrl          *url.URL
	proxyPath          string
	matchedRoute       *plugins.Route
	pluginRoutes       []*plugins.Route
	cfg                *setting.Cfg
	clientProvider     httpclient.Provider
	oAuthTokenService  oauthtoken.OAuthTokenService
	dataSourcesService datasources.DataSourceService
	tracer             tracing.Tracer
}

type httpClient interface {
	Do(req *http.Request) (*http.Response, error)
}

// NewDataSourceProxy creates a new Datasource proxy
func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext,
	proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider,
	oAuthTokenService oauthtoken.OAuthTokenService, dsService datasources.DataSourceService,
	tracer tracing.Tracer) (*DataSourceProxy, error) {
	targetURL, err := datasource.ValidateURL(ds.Type, ds.Url)
	if err != nil {
		return nil, err
	}

	return &DataSourceProxy{
		ds:                 ds,
		pluginRoutes:       pluginRoutes,
		ctx:                ctx,
		proxyPath:          proxyPath,
		targetUrl:          targetURL,
		cfg:                cfg,
		clientProvider:     clientProvider,
		oAuthTokenService:  oAuthTokenService,
		dataSourcesService: dsService,
		tracer:             tracer,
	}, nil
}

func newHTTPClient() httpClient {
	return &http.Client{
		Timeout:   30 * time.Second,
		Transport: &http.Transport{Proxy: http.ProxyFromEnvironment},
	}
}

func (proxy *DataSourceProxy) HandleRequest() {
	if err := proxy.validateRequest(); err != nil {
		proxy.ctx.JsonApiErr(403, err.Error(), nil)
		return
	}

	proxyErrorLogger := logger.New(
		"userId", proxy.ctx.UserId,
		"orgId", proxy.ctx.OrgId,
		"uname", proxy.ctx.Login,
		"path", proxy.ctx.Req.URL.Path,
		"remote_addr", proxy.ctx.RemoteAddr(),
		"referer", proxy.ctx.Req.Referer(),
	)

	transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ctx.Req.Context(), proxy.ds, proxy.clientProvider)
	if err != nil {
		proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err)
		return
	}

	modifyResponse := func(resp *http.Response) error {
		if resp.StatusCode == 401 {
			// The data source rejected the request as unauthorized, convert to 400 (bad request)
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("failed to read data source response body: %w", err)
			}
			_ = resp.Body.Close()

			proxyErrorLogger.Info("Authentication to data source failed", "body", string(body), "statusCode",
				resp.StatusCode)
			msg := "Authentication to data source failed"
			*resp = http.Response{
				StatusCode:    400,
				Status:        "Bad Request",
				Body:          ioutil.NopCloser(strings.NewReader(msg)),
				ContentLength: int64(len(msg)),
				Header:        http.Header{},
			}
		}
		return nil
	}

	reverseProxy := proxyutil.NewReverseProxy(
		proxyErrorLogger,
		proxy.director,
		proxyutil.WithTransport(transport),
		proxyutil.WithModifyResponse(modifyResponse),
	)

	proxy.logRequest()
	ctx, span := proxy.tracer.Start(proxy.ctx.Req.Context(), "datasource reverse proxy")
	defer span.End()

	proxy.ctx.Req = proxy.ctx.Req.WithContext(ctx)

	span.SetAttributes("datasource_name", proxy.ds.Name, attribute.Key("datasource_name").String(proxy.ds.Name))
	span.SetAttributes("datasource_type", proxy.ds.Type, attribute.Key("datasource_type").String(proxy.ds.Type))
	span.SetAttributes("user", proxy.ctx.SignedInUser.Login, attribute.Key("user").String(proxy.ctx.SignedInUser.Login))
	span.SetAttributes("org_id", proxy.ctx.SignedInUser.OrgId, attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgId))

	proxy.addTraceFromHeaderValue(span, "X-Panel-Id", "panel_id")
	proxy.addTraceFromHeaderValue(span, "X-Dashboard-Id", "dashboard_id")

	proxy.tracer.Inject(ctx, proxy.ctx.Req.Header, span)

	reverseProxy.ServeHTTP(proxy.ctx.Resp, proxy.ctx.Req)
}

func (proxy *DataSourceProxy) addTraceFromHeaderValue(span tracing.Span, headerName string, tagName string) {
	panelId := proxy.ctx.Req.Header.Get(headerName)
	dashId, err := strconv.Atoi(panelId)
	if err == nil {
		span.SetAttributes(tagName, dashId, attribute.Key(tagName).Int(dashId))
	}
}

func (proxy *DataSourceProxy) director(req *http.Request) {
	req.URL.Scheme = proxy.targetUrl.Scheme
	req.URL.Host = proxy.targetUrl.Host
	req.Host = proxy.targetUrl.Host

	reqQueryVals := req.URL.Query()

	switch proxy.ds.Type {
	case models.DS_INFLUXDB_08:
		password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds)
		if err != nil {
			logger.Error("Error interpolating proxy url", "error", err)
			return
		}

		req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, "db/"+proxy.ds.Database+"/"+proxy.proxyPath)
		reqQueryVals.Add("u", proxy.ds.User)
		reqQueryVals.Add("p", password)
		req.URL.RawQuery = reqQueryVals.Encode()
	case models.DS_INFLUXDB:
		password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds)
		if err != nil {
			logger.Error("Error interpolating proxy url", "error", err)
			return
		}
		req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath)
		req.URL.RawQuery = reqQueryVals.Encode()
		if !proxy.ds.BasicAuth {
			req.Header.Set(
				"Authorization",
				util.GetBasicAuthHeader(proxy.ds.User, password),
			)
		}
	default:
		req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath)
	}

	unescapedPath, err := url.PathUnescape(req.URL.RawPath)
	if err != nil {
		logger.Error("Failed to unescape raw path", "rawPath", req.URL.RawPath, "error", err)
		return
	}

	req.URL.Path = unescapedPath

	if proxy.ds.BasicAuth {
		password, err := proxy.dataSourcesService.DecryptedBasicAuthPassword(req.Context(), proxy.ds)
		if err != nil {
			logger.Error("Error interpolating proxy url", "error", err)
			return
		}
		req.Header.Set("Authorization", util.GetBasicAuthHeader(proxy.ds.BasicAuthUser,
			password))
	}

	dsAuth := req.Header.Get("X-DS-Authorization")
	if len(dsAuth) > 0 {
		req.Header.Del("X-DS-Authorization")
		req.Header.Set("Authorization", dsAuth)
	}

	applyUserHeader(proxy.cfg.SendUserHeader, req, proxy.ctx.SignedInUser)

	keepCookieNames := []string{}
	if proxy.ds.JsonData != nil {
		if keepCookies := proxy.ds.JsonData.Get("keepCookies"); keepCookies != nil {
			keepCookieNames = keepCookies.MustStringArray()
		}
	}

	proxyutil.ClearCookieHeader(req, keepCookieNames)
	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))

	jsonData := make(map[string]interface{})
	if proxy.ds.JsonData != nil {
		jsonData, err = proxy.ds.JsonData.Map()
		if err != nil {
			logger.Error("Failed to get json data as map", "jsonData", proxy.ds.JsonData, "error", err)
			return
		}
	}

	if proxy.matchedRoute != nil {
		decryptedValues, err := proxy.dataSourcesService.DecryptedValues(req.Context(), proxy.ds)
		if err != nil {
			logger.Error("Error interpolating proxy url", "error", err)
			return
		}

		ApplyRoute(req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{
			ID:                      proxy.ds.Id,
			Updated:                 proxy.ds.Updated,
			JSONData:                jsonData,
			DecryptedSecureJSONData: decryptedValues,
		}, proxy.cfg)
	}

	if proxy.oAuthTokenService.IsOAuthPassThruEnabled(proxy.ds) {
		if token := proxy.oAuthTokenService.GetCurrentOAuthToken(req.Context(), proxy.ctx.SignedInUser); token != nil {
			req.Header.Set("Authorization", fmt.Sprintf("%s %s", token.Type(), token.AccessToken))

			idToken, ok := token.Extra("id_token").(string)
			if ok && idToken != "" {
				req.Header.Set("X-ID-Token", idToken)
			}
		}
	}
}

func (proxy *DataSourceProxy) validateRequest() error {
	if !checkWhiteList(proxy.ctx, proxy.targetUrl.Host) {
		return errors.New("target URL is not a valid target")
	}

	if proxy.ds.Type == models.DS_ES {
		if proxy.ctx.Req.Method == "DELETE" {
			return errors.New("deletes not allowed on proxied Elasticsearch datasource")
		}
		if proxy.ctx.Req.Method == "PUT" {
			return errors.New("puts not allowed on proxied Elasticsearch datasource")
		}
		if proxy.ctx.Req.Method == "POST" && proxy.proxyPath != "_msearch" {
			return errors.New("posts not allowed on proxied Elasticsearch datasource except on /_msearch")
		}
	}

	// found route if there are any
	for _, route := range proxy.pluginRoutes {
		// method match
		if route.Method != "" && route.Method != "*" && route.Method != proxy.ctx.Req.Method {
			continue
		}

		// route match
		if !strings.HasPrefix(proxy.proxyPath, route.Path) {
			continue
		}

		if route.ReqRole.IsValid() {
			if !proxy.ctx.HasUserRole(route.ReqRole) {
				return errors.New("plugin proxy route access denied")
			}
		}

		proxy.matchedRoute = route
		return nil
	}

	// Trailing validation below this point for routes that were not matched
	if proxy.ds.Type == models.DS_PROMETHEUS {
		if proxy.ctx.Req.Method == "DELETE" {
			return errors.New("non allow-listed DELETEs not allowed on proxied Prometheus datasource")
		}
		if proxy.ctx.Req.Method == "PUT" {
			return errors.New("non allow-listed PUTs not allowed on proxied Prometheus datasource")
		}
		if proxy.ctx.Req.Method == "POST" {
			return errors.New("non allow-listed POSTs not allowed on proxied Prometheus datasource")
		}
	}

	return nil
}

func (proxy *DataSourceProxy) logRequest() {
	if !proxy.cfg.DataProxyLogging {
		return
	}

	var body string
	if proxy.ctx.Req.Body != nil {
		buffer, err := ioutil.ReadAll(proxy.ctx.Req.Body)
		if err == nil {
			proxy.ctx.Req.Body = ioutil.NopCloser(bytes.NewBuffer(buffer))
			body = string(buffer)
		}
	}

	logger.Info("Proxying incoming request",
		"userid", proxy.ctx.UserId,
		"orgid", proxy.ctx.OrgId,
		"username", proxy.ctx.Login,
		"datasource", proxy.ds.Type,
		"uri", proxy.ctx.Req.RequestURI,
		"method", proxy.ctx.Req.Method,
		"body", body)
}

func checkWhiteList(c *models.ReqContext, host string) bool {
	if host != "" && len(setting.DataProxyWhiteList) > 0 {
		if _, exists := setting.DataProxyWhiteList[host]; !exists {
			c.JsonApiErr(403, "Data proxy hostname and ip are not included in whitelist", nil)
			return false
		}
	}

	return true
}
<commit_msg>Add TraceID to data proxy logs (#49302)<commit_after>package pluginproxy

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/grafana/grafana/pkg/api/datasource"
	"github.com/grafana/grafana/pkg/infra/httpclient"
	glog "github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/infra/tracing"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/plugins"
	"github.com/grafana/grafana/pkg/services/datasources"
	"github.com/grafana/grafana/pkg/services/oauthtoken"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/util"
	"github.com/grafana/grafana/pkg/util/proxyutil"
	"go.opentelemetry.io/otel/attribute"
)

var (
	logger = glog.New("data-proxy-log")
	client = newHTTPClient()
)

type DataSourceProxy struct {
	ds                 *models.DataSource
	ctx                *models.ReqContext
	targetUrl          *url.URL
	proxyPath          string
	matchedRoute       *plugins.Route
	pluginRoutes       []*plugins.Route
	cfg                *setting.Cfg
	clientProvider     httpclient.Provider
	oAuthTokenService  oauthtoken.OAuthTokenService
	dataSourcesService datasources.DataSourceService
	tracer             tracing.Tracer
}

type httpClient interface {
	Do(req *http.Request) (*http.Response, error)
}

// NewDataSourceProxy creates a new Datasource proxy
func NewDataSourceProxy(ds *models.DataSource, pluginRoutes []*plugins.Route, ctx *models.ReqContext,
	proxyPath string, cfg *setting.Cfg, clientProvider httpclient.Provider,
	oAuthTokenService oauthtoken.OAuthTokenService, dsService datasources.DataSourceService,
	tracer tracing.Tracer) (*DataSourceProxy, error) {
	targetURL, err := datasource.ValidateURL(ds.Type, ds.Url)
	if err != nil {
		return nil, err
	}

	return &DataSourceProxy{
		ds:                 ds,
		pluginRoutes:       pluginRoutes,
		ctx:                ctx,
		proxyPath:          proxyPath,
		targetUrl:          targetURL,
		cfg:                cfg,
		clientProvider:     clientProvider,
		oAuthTokenService:  oAuthTokenService,
		dataSourcesService: dsService,
		tracer:             tracer,
	}, nil
}

func newHTTPClient() httpClient {
	return &http.Client{
		Timeout:   30 * time.Second,
		Transport: &http.Transport{Proxy: http.ProxyFromEnvironment},
	}
}

func (proxy *DataSourceProxy) HandleRequest() {
	if err := proxy.validateRequest(); err != nil {
		proxy.ctx.JsonApiErr(403, err.Error(), nil)
		return
	}

	traceID := tracing.TraceIDFromContext(proxy.ctx.Req.Context(), false)
	proxyErrorLogger := logger.New(
		"userId", proxy.ctx.UserId,
		"orgId", proxy.ctx.OrgId,
		"uname", proxy.ctx.Login,
		"path", proxy.ctx.Req.URL.Path,
		"remote_addr", proxy.ctx.RemoteAddr(),
		"referer", proxy.ctx.Req.Referer(),
		"traceID", traceID,
	)

	transport, err := proxy.dataSourcesService.GetHTTPTransport(proxy.ctx.Req.Context(), proxy.ds, proxy.clientProvider)
	if err != nil {
		proxy.ctx.JsonApiErr(400, "Unable to load TLS certificate", err)
		return
	}

	modifyResponse := func(resp *http.Response) error {
		if resp.StatusCode == 401 {
			// The data source rejected the request as unauthorized, convert to 400 (bad request)
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return fmt.Errorf("failed to read data source response body: %w", err)
			}
			_ = resp.Body.Close()

			proxyErrorLogger.Info("Authentication to data source failed", "body", string(body), "statusCode",
				resp.StatusCode)
			msg := "Authentication to data source failed"
			*resp = http.Response{
				StatusCode:    400,
				Status:        "Bad Request",
				Body:          ioutil.NopCloser(strings.NewReader(msg)),
				ContentLength: int64(len(msg)),
				Header:        http.Header{},
			}
		}
		return nil
	}

	reverseProxy := proxyutil.NewReverseProxy(
		proxyErrorLogger,
		proxy.director,
		proxyutil.WithTransport(transport),
		proxyutil.WithModifyResponse(modifyResponse),
	)

	proxy.logRequest()
	ctx, span := proxy.tracer.Start(proxy.ctx.Req.Context(), "datasource reverse proxy")
	defer span.End()

	proxy.ctx.Req = proxy.ctx.Req.WithContext(ctx)

	span.SetAttributes("datasource_name", proxy.ds.Name, attribute.Key("datasource_name").String(proxy.ds.Name))
	span.SetAttributes("datasource_type", proxy.ds.Type, attribute.Key("datasource_type").String(proxy.ds.Type))
	span.SetAttributes("user", proxy.ctx.SignedInUser.Login, attribute.Key("user").String(proxy.ctx.SignedInUser.Login))
	span.SetAttributes("org_id", proxy.ctx.SignedInUser.OrgId, attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgId))

	proxy.addTraceFromHeaderValue(span, "X-Panel-Id", "panel_id")
	proxy.addTraceFromHeaderValue(span, "X-Dashboard-Id", "dashboard_id")

	proxy.tracer.Inject(ctx, proxy.ctx.Req.Header, span)

	reverseProxy.ServeHTTP(proxy.ctx.Resp, proxy.ctx.Req)
}

func (proxy *DataSourceProxy) addTraceFromHeaderValue(span tracing.Span, headerName string, tagName string) {
	panelId := proxy.ctx.Req.Header.Get(headerName)
	dashId, err := strconv.Atoi(panelId)
	if err == nil {
		span.SetAttributes(tagName, dashId, attribute.Key(tagName).Int(dashId))
	}
}

func (proxy *DataSourceProxy) director(req *http.Request) {
	req.URL.Scheme = proxy.targetUrl.Scheme
	req.URL.Host = proxy.targetUrl.Host
	req.Host = proxy.targetUrl.Host

	reqQueryVals := req.URL.Query()

	switch proxy.ds.Type {
	case models.DS_INFLUXDB_08:
		password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds)
		if err != nil {
			logger.Error("Error interpolating proxy url", "error", err)
			return
		}

		req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, "db/"+proxy.ds.Database+"/"+proxy.proxyPath)
		reqQueryVals.Add("u", proxy.ds.User)
		reqQueryVals.Add("p", password)
		req.URL.RawQuery = reqQueryVals.Encode()
	case models.DS_INFLUXDB:
		password, err := proxy.dataSourcesService.DecryptedPassword(req.Context(), proxy.ds)
		if err != nil {
			logger.Error("Error interpolating proxy url", "error", err)
			return
		}
		req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath)
		req.URL.RawQuery = reqQueryVals.Encode()
		if !proxy.ds.BasicAuth {
			req.Header.Set(
				"Authorization",
				util.GetBasicAuthHeader(proxy.ds.User, password),
			)
		}
	default:
		req.URL.RawPath = util.JoinURLFragments(proxy.targetUrl.Path, proxy.proxyPath)
	}

	unescapedPath, err := url.PathUnescape(req.URL.RawPath)
	if err != nil {
		logger.Error("Failed to unescape raw path", "rawPath", req.URL.RawPath, "error", err)
		return
	}

	req.URL.Path = unescapedPath

	if proxy.ds.BasicAuth {
		password, err := proxy.dataSourcesService.DecryptedBasicAuthPassword(req.Context(), proxy.ds)
		if err != nil {
			logger.Error("Error interpolating proxy url", "error", err)
			return
		}
		req.Header.Set("Authorization", util.GetBasicAuthHeader(proxy.ds.BasicAuthUser,
			password))
	}

	dsAuth := req.Header.Get("X-DS-Authorization")
	if len(dsAuth) > 0 {
		req.Header.Del("X-DS-Authorization")
		req.Header.Set("Authorization", dsAuth)
	}

	applyUserHeader(proxy.cfg.SendUserHeader, req, proxy.ctx.SignedInUser)

	keepCookieNames := []string{}
	if proxy.ds.JsonData != nil {
		if keepCookies := proxy.ds.JsonData.Get("keepCookies"); keepCookies != nil {
			keepCookieNames = keepCookies.MustStringArray()
		}
	}

	proxyutil.ClearCookieHeader(req, keepCookieNames)
	req.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))

	jsonData := make(map[string]interface{})
	if proxy.ds.JsonData != nil {
		jsonData, err = proxy.ds.JsonData.Map()
		if err != nil {
			logger.Error("Failed to get json data as map", "jsonData", proxy.ds.JsonData, "error", err)
			return
		}
	}

	if proxy.matchedRoute != nil {
		decryptedValues, err := proxy.dataSourcesService.DecryptedValues(req.Context(), proxy.ds)
		if err != nil {
			logger.Error("Error interpolating proxy url", "error", err)
			return
		}

		ApplyRoute(req.Context(), req, proxy.proxyPath, proxy.matchedRoute, DSInfo{
			ID:                      proxy.ds.Id,
			Updated:                 proxy.ds.Updated,
			JSONData:                jsonData,
			DecryptedSecureJSONData: decryptedValues,
		}, proxy.cfg)
	}

	if proxy.oAuthTokenService.IsOAuthPassThruEnabled(proxy.ds) {
		if token := proxy.oAuthTokenService.GetCurrentOAuthToken(req.Context(), proxy.ctx.SignedInUser); token != nil {
			req.Header.Set("Authorization", fmt.Sprintf("%s %s", token.Type(), token.AccessToken))

			idToken, ok := token.Extra("id_token").(string)
			if ok && idToken != "" {
				req.Header.Set("X-ID-Token", idToken)
			}
		}
	}
}

func (proxy *DataSourceProxy) validateRequest() error {
	if !checkWhiteList(proxy.ctx, proxy.targetUrl.Host) {
		return errors.New("target URL is not a valid target")
	}

	if proxy.ds.Type == models.DS_ES {
		if proxy.ctx.Req.Method == "DELETE" {
			return errors.New("deletes not allowed on proxied Elasticsearch datasource")
		}
		if proxy.ctx.Req.Method == "PUT" {
			return errors.New("puts not allowed on proxied Elasticsearch datasource")
		}
		if proxy.ctx.Req.Method == "POST" && proxy.proxyPath != "_msearch" {
			return errors.New("posts not allowed on proxied Elasticsearch datasource except on /_msearch")
		}
	}

	// found route if there are any
	for _, route := range proxy.pluginRoutes {
		// method match
		if route.Method != "" && route.Method != "*" && route.Method != proxy.ctx.Req.Method {
			continue
		}

		// route match
		if !strings.HasPrefix(proxy.proxyPath, route.Path) {
			continue
		}

		if route.ReqRole.IsValid() {
			if !proxy.ctx.HasUserRole(route.ReqRole) {
				return errors.New("plugin proxy route access denied")
			}
		}

		proxy.matchedRoute = route
		return nil
	}

	// Trailing validation below this point for routes that were not matched
	if proxy.ds.Type == models.DS_PROMETHEUS {
		if proxy.ctx.Req.Method == "DELETE" {
			return errors.New("non allow-listed DELETEs not allowed on proxied Prometheus datasource")
		}
		if proxy.ctx.Req.Method == "PUT" {
			return errors.New("non allow-listed PUTs not allowed on proxied Prometheus datasource")
		}
		if proxy.ctx.Req.Method == "POST" {
			return errors.New("non allow-listed POSTs not allowed on proxied Prometheus datasource")
		}
	}

	return nil
}

func (proxy *DataSourceProxy) logRequest() {
	if !proxy.cfg.DataProxyLogging {
		return
	}

	var body string
	if proxy.ctx.Req.Body != nil {
		buffer, err := ioutil.ReadAll(proxy.ctx.Req.Body)
		if err == nil {
			proxy.ctx.Req.Body = ioutil.NopCloser(bytes.NewBuffer(buffer))
			body = string(buffer)
		}
	}

	logger.Info("Proxying incoming request",
		"userid", proxy.ctx.UserId,
		"orgid", proxy.ctx.OrgId,
		"username", proxy.ctx.Login,
		"datasource", proxy.ds.Type,
		"uri", proxy.ctx.Req.RequestURI,
		"method", proxy.ctx.Req.Method,
		"body", body)
}

func checkWhiteList(c *models.ReqContext, host string) bool {
	if host != "" && len(setting.DataProxyWhiteList) > 0 {
		if _, exists := setting.DataProxyWhiteList[host]; !exists {
			c.JsonApiErr(403, "Data proxy hostname and ip are not included in whitelist", nil)
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"fmt"
	"time"

	cm "knative.dev/pkg/configmap"
	"knative.dev/serving/pkg/apis/autoscaling"
	"knative.dev/serving/pkg/autoscaler/config/autoscalerconfig"

	corev1 "k8s.io/api/core/v1"
)

const (
	// ConfigName is the name of the config map of the autoscaler.
	ConfigName = "config-autoscaler"

	// BucketSize is the size of the buckets of stats we create.
	// NB: if this is more than 1s, we need to average values in the
	// metrics buckets.
	BucketSize = 1 * time.Second

	defaultTargetUtilization = 0.7
)

func defaultConfig() *autoscalerconfig.Config {
	return &autoscalerconfig.Config{
		EnableScaleToZero:                  true,
		ContainerConcurrencyTargetFraction: defaultTargetUtilization,
		ContainerConcurrencyTargetDefault:  100,
		// TODO(#1956): Tune target usage based on empirical data.
		TargetUtilization:             defaultTargetUtilization,
		RPSTargetDefault:              200,
		MaxScaleUpRate:                1000,
		MaxScaleDownRate:              2,
		TargetBurstCapacity:           200,
		PanicWindowPercentage:         10,
		ActivatorCapacity:             100,
		PanicThresholdPercentage:      200,
		StableWindow:                  60 * time.Second,
		ScaleToZeroGracePeriod:        30 * time.Second,
		ScaleToZeroPodRetentionPeriod: 0 * time.Second,
		ScaleDownDelay:                0 * time.Second,
		PodAutoscalerClass:            autoscaling.KPA,
		AllowZeroInitialScale:         false,
		InitialScale:                  1,
		MaxScale:                      0,
		MaxScaleLimit:                 0,
	}
}

// NewConfigFromMap creates a Config from the supplied map
func NewConfigFromMap(data map[string]string) (*autoscalerconfig.Config, error) {
	lc := defaultConfig()

	if err := cm.Parse(data,
		cm.AsString("pod-autoscaler-class", &lc.PodAutoscalerClass),

		cm.AsBool("enable-scale-to-zero", &lc.EnableScaleToZero),
		cm.AsBool("allow-zero-initial-scale", &lc.AllowZeroInitialScale),

		cm.AsFloat64("max-scale-up-rate", &lc.MaxScaleUpRate),
		cm.AsFloat64("max-scale-down-rate", &lc.MaxScaleDownRate),
		cm.AsFloat64("container-concurrency-target-percentage", &lc.ContainerConcurrencyTargetFraction),
		cm.AsFloat64("container-concurrency-target-default", &lc.ContainerConcurrencyTargetDefault),
		cm.AsFloat64("requests-per-second-target-default", &lc.RPSTargetDefault),
		cm.AsFloat64("target-burst-capacity", &lc.TargetBurstCapacity),
		cm.AsFloat64("panic-window-percentage", &lc.PanicWindowPercentage),
		cm.AsFloat64("activator-capacity", &lc.ActivatorCapacity),
		cm.AsFloat64("panic-threshold-percentage", &lc.PanicThresholdPercentage),

		cm.AsInt32("initial-scale", &lc.InitialScale),
		cm.AsInt32("max-scale", &lc.MaxScale),
		cm.AsInt32("max-scale-limit", &lc.MaxScaleLimit),

		cm.AsDuration("stable-window", &lc.StableWindow),
		cm.AsDuration("scale-down-delay", &lc.ScaleDownDelay),
		cm.AsDuration("scale-to-zero-grace-period", &lc.ScaleToZeroGracePeriod),
		cm.AsDuration("scale-to-zero-pod-retention-period", &lc.ScaleToZeroPodRetentionPeriod),
	); err != nil {
		return nil, fmt.Errorf("failed to parse data: %w", err)
	}

	// Adjust % ⇒ fractions: for legacy reasons we allow values in the
	// (0, 1] interval, so minimal percentage must be greater than 1.0.
	// Internally we want to have fractions, since otherwise we'll have
	// to perform division on each computation.
	if lc.ContainerConcurrencyTargetFraction > 1.0 {
		lc.ContainerConcurrencyTargetFraction /= 100.0
	}

	return validate(lc)
}

func validate(lc *autoscalerconfig.Config) (*autoscalerconfig.Config, error) {
	if lc.ScaleToZeroGracePeriod <= 0 {
		return nil, fmt.Errorf("scale-to-zero-grace-period must be positive, was: %v", lc.ScaleToZeroGracePeriod)
	}

	if lc.ScaleDownDelay < 0 {
		return nil, fmt.Errorf("scale-down-delay cannot be negative, was: %v", lc.ScaleDownDelay)
	}

	if lc.ScaleDownDelay.Round(time.Second) != lc.ScaleDownDelay {
		return nil, fmt.Errorf("scale-down-delay = %v, must be specified with at most second precision", lc.ScaleDownDelay)
	}

	if lc.ScaleToZeroPodRetentionPeriod < 0 {
		return nil, fmt.Errorf("scale-to-zero-pod-retention-period cannot be negative, was: %v", lc.ScaleToZeroPodRetentionPeriod)
	}

	if lc.TargetBurstCapacity < 0 && lc.TargetBurstCapacity != -1 {
		return nil, fmt.Errorf("target-burst-capacity must be either non-negative or -1 (for unlimited), was: %f", lc.TargetBurstCapacity)
	}

	if lc.ContainerConcurrencyTargetFraction <= 0 || lc.ContainerConcurrencyTargetFraction > 1 {
		return nil, fmt.Errorf("container-concurrency-target-percentage = %f is outside of valid range of (0, 100]", lc.ContainerConcurrencyTargetFraction)
	}

	if x := lc.ContainerConcurrencyTargetFraction * lc.ContainerConcurrencyTargetDefault; x < autoscaling.TargetMin {
		return nil, fmt.Errorf("container-concurrency-target-percentage and container-concurrency-target-default yield target concurrency of %v, can't be less than %v", x, autoscaling.TargetMin)
	}

	if lc.RPSTargetDefault < autoscaling.TargetMin {
		return nil, fmt.Errorf("requests-per-second-target-default must be at least %v, was: %v", autoscaling.TargetMin, lc.RPSTargetDefault)
	}

	if lc.ActivatorCapacity < 1 {
		return nil, fmt.Errorf("activator-capacity = %v, must be at least 1", lc.ActivatorCapacity)
	}

	if lc.MaxScaleUpRate <= 1.0 {
		return nil, fmt.Errorf("max-scale-up-rate = %v, must be greater than 1.0", lc.MaxScaleUpRate)
	}

	if lc.MaxScaleDownRate <= 1.0 {
		return nil, fmt.Errorf("max-scale-down-rate = %v, must be greater than 1.0", lc.MaxScaleDownRate)
	}

	// We can't permit stable window be less than our aggregation window for correctness.
	// Or too big, so that our decisions are too imprecise.
	if lc.StableWindow < autoscaling.WindowMin || lc.StableWindow > autoscaling.WindowMax {
		return nil, fmt.Errorf("stable-window = %v, must be in [%v; %v] range", lc.StableWindow,
			autoscaling.WindowMin, autoscaling.WindowMax)
	}

	if lc.StableWindow.Round(time.Second) != lc.StableWindow {
		return nil, fmt.Errorf("stable-window = %v, must be specified with at most second precision", lc.StableWindow)
	}

	// We ensure BucketSize in the `MakeMetric`, so just ensure percentage is in the correct region.
	if lc.PanicWindowPercentage < autoscaling.PanicWindowPercentageMin ||
		lc.PanicWindowPercentage > autoscaling.PanicWindowPercentageMax {
		return nil, fmt.Errorf("panic-window-percentage = %v, must be in [%v, %v] interval",
			lc.PanicWindowPercentage, autoscaling.PanicWindowPercentageMin, autoscaling.PanicWindowPercentageMax)

	}

	if lc.InitialScale < 0 || (lc.InitialScale == 0 && !lc.AllowZeroInitialScale) {
		return nil, fmt.Errorf("initial-scale = %v, must be at least 0 (or at least 1 when allow-zero-initial-scale is false)", lc.InitialScale)
	}

	var minMaxScale int32 = 0
	if lc.MaxScaleLimit > 0 {
		// Default maxScale must be set if maxScaleLimit is set.
		minMaxScale = 1
	}

	if lc.MaxScale < minMaxScale || (lc.MaxScaleLimit > 0 && lc.MaxScale > lc.MaxScaleLimit) {
		return nil, fmt.Errorf("max-scale = %d, must be in [%d, max-scale-limit] range", lc.MaxScale, minMaxScale)
	}

	if lc.MaxScaleLimit < 0 {
		return nil, fmt.Errorf("max-scale-limit = %v, must be at least 0", lc.MaxScaleLimit)
	}

	return lc, nil
}

// NewConfigFromConfigMap creates a Config from the supplied ConfigMap
func NewConfigFromConfigMap(configMap *corev1.ConfigMap) (*autoscalerconfig.Config, error) {
	return NewConfigFromMap(configMap.Data)
}
<commit_msg>Afterthough: we should probably report the current value as well (#10932)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"fmt"
	"time"

	cm "knative.dev/pkg/configmap"
	"knative.dev/serving/pkg/apis/autoscaling"
	"knative.dev/serving/pkg/autoscaler/config/autoscalerconfig"

	corev1 "k8s.io/api/core/v1"
)

const (
	// ConfigName is the name of the config map of the autoscaler.
	ConfigName = "config-autoscaler"

	// BucketSize is the size of the buckets of stats we create.
	// NB: if this is more than 1s, we need to average values in the
	// metrics buckets.
	BucketSize = 1 * time.Second

	defaultTargetUtilization = 0.7
)

func defaultConfig() *autoscalerconfig.Config {
	return &autoscalerconfig.Config{
		EnableScaleToZero:                  true,
		ContainerConcurrencyTargetFraction: defaultTargetUtilization,
		ContainerConcurrencyTargetDefault:  100,
		// TODO(#1956): Tune target usage based on empirical data.
		TargetUtilization:             defaultTargetUtilization,
		RPSTargetDefault:              200,
		MaxScaleUpRate:                1000,
		MaxScaleDownRate:              2,
		TargetBurstCapacity:           200,
		PanicWindowPercentage:         10,
		ActivatorCapacity:             100,
		PanicThresholdPercentage:      200,
		StableWindow:                  60 * time.Second,
		ScaleToZeroGracePeriod:        30 * time.Second,
		ScaleToZeroPodRetentionPeriod: 0 * time.Second,
		ScaleDownDelay:                0 * time.Second,
		PodAutoscalerClass:            autoscaling.KPA,
		AllowZeroInitialScale:         false,
		InitialScale:                  1,
		MaxScale:                      0,
		MaxScaleLimit:                 0,
	}
}

// NewConfigFromMap creates a Config from the supplied map
func NewConfigFromMap(data map[string]string) (*autoscalerconfig.Config, error) {
	lc := defaultConfig()

	if err := cm.Parse(data,
		cm.AsString("pod-autoscaler-class", &lc.PodAutoscalerClass),

		cm.AsBool("enable-scale-to-zero", &lc.EnableScaleToZero),
		cm.AsBool("allow-zero-initial-scale", &lc.AllowZeroInitialScale),

		cm.AsFloat64("max-scale-up-rate", &lc.MaxScaleUpRate),
		cm.AsFloat64("max-scale-down-rate", &lc.MaxScaleDownRate),
		cm.AsFloat64("container-concurrency-target-percentage", &lc.ContainerConcurrencyTargetFraction),
		cm.AsFloat64("container-concurrency-target-default", &lc.ContainerConcurrencyTargetDefault),
		cm.AsFloat64("requests-per-second-target-default", &lc.RPSTargetDefault),
		cm.AsFloat64("target-burst-capacity", &lc.TargetBurstCapacity),
		cm.AsFloat64("panic-window-percentage", &lc.PanicWindowPercentage),
		cm.AsFloat64("activator-capacity", &lc.ActivatorCapacity),
		cm.AsFloat64("panic-threshold-percentage", &lc.PanicThresholdPercentage),

		cm.AsInt32("initial-scale", &lc.InitialScale),
		cm.AsInt32("max-scale", &lc.MaxScale),
		cm.AsInt32("max-scale-limit", &lc.MaxScaleLimit),

		cm.AsDuration("stable-window", &lc.StableWindow),
		cm.AsDuration("scale-down-delay", &lc.ScaleDownDelay),
		cm.AsDuration("scale-to-zero-grace-period", &lc.ScaleToZeroGracePeriod),
		cm.AsDuration("scale-to-zero-pod-retention-period", &lc.ScaleToZeroPodRetentionPeriod),
	); err != nil {
		return nil, fmt.Errorf("failed to parse data: %w", err)
	}

	// Adjust % ⇒ fractions: for legacy reasons we allow values in the
	// (0, 1] interval, so minimal percentage must be greater than 1.0.
	// Internally we want to have fractions, since otherwise we'll have
	// to perform division on each computation.
	if lc.ContainerConcurrencyTargetFraction > 1.0 {
		lc.ContainerConcurrencyTargetFraction /= 100.0
	}

	return validate(lc)
}

func validate(lc *autoscalerconfig.Config) (*autoscalerconfig.Config, error) {
	if lc.ScaleToZeroGracePeriod <= 0 {
		return nil, fmt.Errorf("scale-to-zero-grace-period must be positive, was: %v", lc.ScaleToZeroGracePeriod)
	}

	if lc.ScaleDownDelay < 0 {
		return nil, fmt.Errorf("scale-down-delay cannot be negative, was: %v", lc.ScaleDownDelay)
	}

	if lc.ScaleDownDelay.Round(time.Second) != lc.ScaleDownDelay {
		return nil, fmt.Errorf("scale-down-delay = %v, must be specified with at most second precision", lc.ScaleDownDelay)
	}

	if lc.ScaleToZeroPodRetentionPeriod < 0 {
		return nil, fmt.Errorf("scale-to-zero-pod-retention-period cannot be negative, was: %v", lc.ScaleToZeroPodRetentionPeriod)
	}

	if lc.TargetBurstCapacity < 0 && lc.TargetBurstCapacity != -1 {
		return nil, fmt.Errorf("target-burst-capacity must be either non-negative or -1 (for unlimited), was: %f", lc.TargetBurstCapacity)
	}

	if lc.ContainerConcurrencyTargetFraction <= 0 || lc.ContainerConcurrencyTargetFraction > 1 {
		return nil, fmt.Errorf("container-concurrency-target-percentage = %f is outside of valid range of (0, 100]", lc.ContainerConcurrencyTargetFraction)
	}

	if x := lc.ContainerConcurrencyTargetFraction * lc.ContainerConcurrencyTargetDefault; x < autoscaling.TargetMin {
		return nil, fmt.Errorf("container-concurrency-target-percentage and container-concurrency-target-default yield target concurrency of %v, can't be less than %v", x, autoscaling.TargetMin)
	}

	if lc.RPSTargetDefault < autoscaling.TargetMin {
		return nil, fmt.Errorf("requests-per-second-target-default must be at least %v, was: %v", autoscaling.TargetMin, lc.RPSTargetDefault)
	}

	if lc.ActivatorCapacity < 1 {
		return nil, fmt.Errorf("activator-capacity = %v, must be at least 1", lc.ActivatorCapacity)
	}

	if lc.MaxScaleUpRate <= 1.0 {
		return nil, fmt.Errorf("max-scale-up-rate = %v, must be greater than 1.0", lc.MaxScaleUpRate)
	}

	if lc.MaxScaleDownRate <= 1.0 {
		return nil, fmt.Errorf("max-scale-down-rate = %v, must be greater than 1.0", lc.MaxScaleDownRate)
	}

	// We can't permit stable window be less than our aggregation window for correctness.
	// Or too big, so that our decisions are too imprecise.
	if lc.StableWindow < autoscaling.WindowMin || lc.StableWindow > autoscaling.WindowMax {
		return nil, fmt.Errorf("stable-window = %v, must be in [%v; %v] range", lc.StableWindow,
			autoscaling.WindowMin, autoscaling.WindowMax)
	}

	if lc.StableWindow.Round(time.Second) != lc.StableWindow {
		return nil, fmt.Errorf("stable-window = %v, must be specified with at most second precision", lc.StableWindow)
	}

	// We ensure BucketSize in the `MakeMetric`, so just ensure percentage is in the correct region.
	if lc.PanicWindowPercentage < autoscaling.PanicWindowPercentageMin ||
		lc.PanicWindowPercentage > autoscaling.PanicWindowPercentageMax {
		return nil, fmt.Errorf("panic-window-percentage = %v, must be in [%v, %v] interval",
			lc.PanicWindowPercentage, autoscaling.PanicWindowPercentageMin, autoscaling.PanicWindowPercentageMax)

	}

	if lc.InitialScale < 0 || (lc.InitialScale == 0 && !lc.AllowZeroInitialScale) {
		return nil, fmt.Errorf("initial-scale = %v, must be at least 0 (or at least 1 when allow-zero-initial-scale is false)", lc.InitialScale)
	}

	var minMaxScale int32 = 0
	if lc.MaxScaleLimit > 0 {
		// Default maxScale must be set if maxScaleLimit is set.
		minMaxScale = 1
	}

	if lc.MaxScale < minMaxScale || (lc.MaxScaleLimit > 0 && lc.MaxScale > lc.MaxScaleLimit) {
		return nil, fmt.Errorf("max-scale = %d, must be in [%d, max-scale-limit(%d)] range",
			lc.MaxScale, minMaxScale, lc.MaxScaleLimit)
	}

	if lc.MaxScaleLimit < 0 {
		return nil, fmt.Errorf("max-scale-limit = %v, must be at least 0", lc.MaxScaleLimit)
	}

	return lc, nil
}

// NewConfigFromConfigMap creates a Config from the supplied ConfigMap
func NewConfigFromConfigMap(configMap *corev1.ConfigMap) (*autoscalerconfig.Config, error) {
	return NewConfigFromMap(configMap.Data)
}
<|endoftext|>


<commit_before>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package render

import (
	"github.com/quarnster/util/text"
)

const (
	DRAW_EMPTY ViewRegionFlags = (1 << iota)
	HIDE_ON_MINIMAP
	DRAW_EMPTY_AS_OVERWRITE
	DRAW_NO_FILL
	DRAW_NO_OUTLINE
	DRAW_SOLID_UNDERLINE
	DRAW_STIPPLED_UNDERLINE
	DRAW_SQUIGGLY_UNDERLINE
	PERSISTENT
	HIDDEN
	FOREGROUND
	SELECTION
	HIGHLIGHT
	DRAW_TEXT
	DEFAULT ViewRegionFlags = 0
)

type (
	ViewRegionMap   map[string]ViewRegions
	ViewRegionFlags int
	ViewRegions     struct {
		Regions text.RegionSet
		Scope   string
		Icon    string
		Flags   ViewRegionFlags
	}
)

func (vrm *ViewRegionMap) Cull(viewport text.Region) {
	rm := []string{}
	for k, v := range *vrm {
		v.Cull(viewport)
		if v.Regions.Len() == 0 {
			rm = append(rm, k)
		} else {
			(*vrm)[k] = v
		}
	}
	for _, r := range rm {
		delete(*vrm, r)
	}
}

func (vr *ViewRegions) Cull(viewport text.Region) {
	nr := []text.Region{}
	for _, r := range vr.Regions.Regions() {
		if viewport.Intersects(r) {
			nr = append(nr, viewport.Intersection(r))
		}
	}
	vr.Regions.Clear()
	vr.Regions.AddAll(nr)
}

func (vr *ViewRegions) Clone() ViewRegions {
	ret := ViewRegions{Scope: vr.Scope, Icon: vr.Icon, Flags: vr.Flags}
	ret.Regions.AddAll(vr.Regions.Regions())
	return ret
}
<commit_msg>Always exclude 0 size regions in ViewRegions.Cull.<commit_after>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.

package render

import (
	"github.com/quarnster/util/text"
)

const (
	DRAW_EMPTY ViewRegionFlags = (1 << iota)
	HIDE_ON_MINIMAP
	DRAW_EMPTY_AS_OVERWRITE
	DRAW_NO_FILL
	DRAW_NO_OUTLINE
	DRAW_SOLID_UNDERLINE
	DRAW_STIPPLED_UNDERLINE
	DRAW_SQUIGGLY_UNDERLINE
	PERSISTENT
	HIDDEN
	FOREGROUND
	SELECTION
	HIGHLIGHT
	DRAW_TEXT
	DEFAULT ViewRegionFlags = 0
)

type (
	ViewRegionMap   map[string]ViewRegions
	ViewRegionFlags int
	ViewRegions     struct {
		Regions text.RegionSet
		Scope   string
		Icon    string
		Flags   ViewRegionFlags
	}
)

func (vrm *ViewRegionMap) Cull(viewport text.Region) {
	rm := []string{}
	for k, v := range *vrm {
		v.Cull(viewport)
		if v.Regions.Len() == 0 {
			rm = append(rm, k)
		} else {
			(*vrm)[k] = v
		}
	}
	for _, r := range rm {
		delete(*vrm, r)
	}
}

func (vr *ViewRegions) Cull(viewport text.Region) {
	nr := []text.Region{}
	for _, r := range vr.Regions.Regions() {
		if viewport.Intersects(r) {
			in := viewport.Intersection(r)
			if in.Size() != 0 {
				nr = append(nr, in)
			}
		}
	}
	vr.Regions.Clear()
	vr.Regions.AddAll(nr)
}

func (vr *ViewRegions) Clone() ViewRegions {
	ret := ViewRegions{Scope: vr.Scope, Icon: vr.Icon, Flags: vr.Flags}
	ret.Regions.AddAll(vr.Regions.Regions())
	return ret
}
<|endoftext|>