<commit_before>package rest

import (
	"errors"
	"github.com/ant0ine/go-json-rest/rest/trie"
	"net/url"
	"strings"
)

type router struct {
	routes                 []*Route
	disableTrieCompression bool
	index                  map[*Route]int
	trie                   *trie.Trie
}

func escapedPath(urlObj *url.URL) string {
	// the escape method of url.URL should be public
	// that would avoid this split.
	parts := strings.SplitN(urlObj.RequestURI(), "?", 2)
	return parts[0]
}

var preEscape = strings.NewReplacer("*", "__SPLAT_PLACEHOLDER__", "#", "__RELAXED_PLACEHOLDER__")

var postEscape = strings.NewReplacer("__SPLAT_PLACEHOLDER__", "*", "__RELAXED_PLACEHOLDER__", "#")

func escapedPathExp(pathExp string) (string, error) {

	// PathExp validation
	if pathExp == "" {
		return "", errors.New("empty PathExp")
	}
	if pathExp[0] != '/' {
		return "", errors.New("PathExp must start with /")
	}
	if strings.Contains(pathExp, "?") {
		return "", errors.New("PathExp must not contain the query string")
	}

	// Get the right escaping
	// XXX a bit hacky

	pathExp = preEscape.Replace(pathExp)

	urlObj, err := url.Parse(pathExp)
	if err != nil {
		return "", err
	}

	// get the same escaping as find requests
	pathExp = urlObj.RequestURI()

	pathExp = postEscape.Replace(pathExp)

	return pathExp, nil
}

// This validates the Routes and prepares the Trie data structure.
// It must be called once the Routes are defined and before trying to find Routes.
// The order matters, if multiple Routes match, the first defined will be used.
func (rt *router) start() error {

	rt.trie = trie.New()
	rt.index = map[*Route]int{}

	for i, route := range rt.routes {

		// work with the PathExp urlencoded.
		pathExp, err := escapedPathExp(route.PathExp)
		if err != nil {
			return err
		}

		// insert in the Trie
		err = rt.trie.AddRoute(
			strings.ToUpper(route.HttpMethod), // work with the HttpMethod in uppercase
			pathExp,
			route,
		)
		if err != nil {
			return err
		}

		// index
		rt.index[route] = i
	}

	if rt.disableTrieCompression == false {
		rt.trie.Compress()
	}

	return nil
}

// return the result that has the route defined the earliest
func (rt *router) ofFirstDefinedRoute(matches []*trie.Match) *trie.Match {
	minIndex := -1
	matchesByIndex := map[int]*trie.Match{}

	for _, result := range matches {
		route := result.Route.(*Route)
		routeIndex := rt.index[route]
		matchesByIndex[routeIndex] = result
		if minIndex == -1 || routeIndex < minIndex {
			minIndex = routeIndex
		}
	}

	return matchesByIndex[minIndex]
}

// Return the first matching Route and the corresponding parameters for a given URL object.
func (rt *router) findRouteFromURL(httpMethod string, urlObj *url.URL) (*Route, map[string]string, bool) {

	// lookup the routes in the Trie
	matches, pathMatched := rt.trie.FindRoutesAndPathMatched(
		strings.ToUpper(httpMethod), // work with the httpMethod in uppercase
		escapedPath(urlObj),         // work with the path urlencoded
	)

	// short cuts
	if len(matches) == 0 {
		// no route found
		return nil, nil, pathMatched
	}

	if len(matches) == 1 {
		// one route found
		return matches[0].Route.(*Route), matches[0].Params, pathMatched
	}

	// multiple routes found, pick the first defined
	result := rt.ofFirstDefinedRoute(matches)
	return result.Route.(*Route), result.Params, pathMatched
}

// Parse the url string (complete or just the path) and return the first matching Route and the corresponding parameters.
func (rt *router) findRoute(httpMethod, urlStr string) (*Route, map[string]string, bool, error) {

	// parse the url
	urlObj, err := url.Parse(urlStr)
	if err != nil {
		return nil, nil, false, err
	}

	route, params, pathMatched := rt.findRouteFromURL(httpMethod, urlObj)
	return route, params, pathMatched, nil
}
<commit_msg>Replace a useless map by a pointer, 16% perf improvement<commit_after>package rest

import (
	"errors"
	"github.com/ant0ine/go-json-rest/rest/trie"
	"net/url"
	"strings"
)

type router struct {
	routes                 []*Route
	disableTrieCompression bool
	index                  map[*Route]int
	trie                   *trie.Trie
}

// This is run for each new request, perf is important.
func escapedPath(urlObj *url.URL) string {
	// the escape method of url.URL should be public
	// that would avoid this split.
	parts := strings.SplitN(urlObj.RequestURI(), "?", 2)
	return parts[0]
}

var preEscape = strings.NewReplacer("*", "__SPLAT_PLACEHOLDER__", "#", "__RELAXED_PLACEHOLDER__")

var postEscape = strings.NewReplacer("__SPLAT_PLACEHOLDER__", "*", "__RELAXED_PLACEHOLDER__", "#")

// This is run at init time only.
func escapedPathExp(pathExp string) (string, error) {

	// PathExp validation
	if pathExp == "" {
		return "", errors.New("empty PathExp")
	}
	if pathExp[0] != '/' {
		return "", errors.New("PathExp must start with /")
	}
	if strings.Contains(pathExp, "?") {
		return "", errors.New("PathExp must not contain the query string")
	}

	// Get the right escaping
	// XXX a bit hacky

	pathExp = preEscape.Replace(pathExp)

	urlObj, err := url.Parse(pathExp)
	if err != nil {
		return "", err
	}

	// get the same escaping as find requests
	pathExp = urlObj.RequestURI()

	pathExp = postEscape.Replace(pathExp)

	return pathExp, nil
}

// This validates the Routes and prepares the Trie data structure.
// It must be called once the Routes are defined and before trying to find Routes.
// The order matters, if multiple Routes match, the first defined will be used.
func (rt *router) start() error {

	rt.trie = trie.New()
	rt.index = map[*Route]int{}

	for i, route := range rt.routes {

		// work with the PathExp urlencoded.
		pathExp, err := escapedPathExp(route.PathExp)
		if err != nil {
			return err
		}

		// insert in the Trie
		err = rt.trie.AddRoute(
			strings.ToUpper(route.HttpMethod), // work with the HttpMethod in uppercase
			pathExp,
			route,
		)
		if err != nil {
			return err
		}

		// index
		rt.index[route] = i
	}

	if rt.disableTrieCompression == false {
		rt.trie.Compress()
	}

	return nil
}

// return the result that has the route defined the earliest
func (rt *router) ofFirstDefinedRoute(matches []*trie.Match) *trie.Match {
	minIndex := -1
	var bestMatch *trie.Match

	for _, result := range matches {
		route := result.Route.(*Route)
		routeIndex := rt.index[route]
		if minIndex == -1 || routeIndex < minIndex {
			minIndex = routeIndex
			bestMatch = result
		}
	}

	return bestMatch
}

// Return the first matching Route and the corresponding parameters for a given URL object.
func (rt *router) findRouteFromURL(httpMethod string, urlObj *url.URL) (*Route, map[string]string, bool) {

	// lookup the routes in the Trie
	matches, pathMatched := rt.trie.FindRoutesAndPathMatched(
		strings.ToUpper(httpMethod), // work with the httpMethod in uppercase
		escapedPath(urlObj),         // work with the path urlencoded
	)

	// short cuts
	if len(matches) == 0 {
		// no route found
		return nil, nil, pathMatched
	}

	if len(matches) == 1 {
		// one route found
		return matches[0].Route.(*Route), matches[0].Params, pathMatched
	}

	// multiple routes found, pick the first defined
	result := rt.ofFirstDefinedRoute(matches)
	return result.Route.(*Route), result.Params, pathMatched
}

// Parse the url string (complete or just the path) and return the first matching Route and the corresponding parameters.
func (rt *router) findRoute(httpMethod, urlStr string) (*Route, map[string]string, bool, error) {

	// parse the url
	urlObj, err := url.Parse(urlStr)
	if err != nil {
		return nil, nil, false, err
	}

	route, params, pathMatched := rt.findRouteFromURL(httpMethod, urlObj)
	return route, params, pathMatched, nil
}
<|endoftext|>


<commit_before>package retry

import (
	"log"
	"reflect"
	"time"

	"github.com/cenkalti/backoff"
)

type BackOffOpts struct {
	InitialInterval time.Duration
	MaxInterval     time.Duration
	MaxElapsedTime  time.Duration
}

var DefaultBackOffOpts *BackOffOpts = &BackOffOpts{
	InitialInterval: 500 * time.Millisecond,
	MaxInterval:     3 * time.Second,
	MaxElapsedTime:  10 * time.Second}

var RetryOnAnyError = func(error) bool { return true }

type RetryNotifier func(*RetryEvent)

type Retrier struct {
	Name              string
	backOffOpts       *BackOffOpts
	shouldRetryFunc   func(error) bool
	notifyRetryFuncs  []RetryNotifier
	notifyGaveUpFuncs []RetryNotifier
}

func NewRetrier(name string,
	backOffOpts *BackOffOpts, shouldRetryFunc func(error) bool) *Retrier {

	return &Retrier{
		Name: name, backOffOpts: backOffOpts, shouldRetryFunc: shouldRetryFunc,
		notifyRetryFuncs:  []RetryNotifier{logRetry},
		notifyGaveUpFuncs: []RetryNotifier{logGaveUp}}
}

func NewErrorTypeRetrier(name string,
	backOffOpts *BackOffOpts, errorTypes ...interface{}) *Retrier {

	return &Retrier{
		Name:            name,
		backOffOpts:     backOffOpts,
		shouldRetryFunc: RetryWhenErrorTypeMatches(instancesToTypes(errorTypes))}
}

func (r *Retrier) Retry(f func() (interface{}, error)) (interface{}, error) {
	var val interface{}
	var err error
	var next time.Duration

	numTries := 0
	b := r.newBackOff()
	b.Reset()
	for {
		numTries++
		if val, err = f(); err == nil {
			return val, nil
		}

		if !r.shouldRetryFunc(err) {
			return val, err
		}

		if next = b.NextBackOff(); next == backoff.Stop {
			r.notifyGaveUp(err, numTries)
			return val, err
		}

		time.Sleep(next)
		r.notifyRetry(err, numTries)
	}
}

type RetryEvent struct {
	Retrier *Retrier
	Err     error
	NumTries int
}

func (r *Retrier) AddNotifyRetry(f RetryNotifier) {
	r.notifyRetryFuncs = append(r.notifyRetryFuncs, f)
}

func (r *Retrier) AddNotifyGaveUp(f RetryNotifier) {
	r.notifyGaveUpFuncs = append(r.notifyGaveUpFuncs, f)
}

func (r *Retrier) notifyGaveUp(err error, numTries int) {
	retryEvent := &RetryEvent{Retrier: r, Err: err, NumTries: numTries}
	for _, notifyGaveUpFunc := range r.notifyGaveUpFuncs {
		notifyGaveUpFunc(retryEvent)
	}
}

func (r *Retrier) notifyRetry(err error, numTries int) {
	retryEvent := &RetryEvent{Retrier: r, Err: err, NumTries: numTries}
	for _, notifyRetryFunc := range r.notifyRetryFuncs {
		notifyRetryFunc(retryEvent)
	}
}

func logRetry(re *RetryEvent) {
	log.Printf("Retrying after %d tries: error=%s count#retry.%s.retry_count=1\n",
		re.Err.Error(), re.Retrier.Name, re.NumTries)
}

func logGaveUp(re *RetryEvent) {
	log.Printf("Giving up after %d tries: error=%s count#retry.%s.gave_up_count=1\n",
		re.Err.Error(), re.Retrier.Name, re.NumTries)
}

func RetryWhenErrorTypeMatches(errorTypes []reflect.Type) func(error) bool {
	errorTypeSet := make(map[reflect.Type]bool)
	for _, t := range errorTypes {
		errorTypeSet[t] = true
	}
	return func(e error) bool {
		errorType := reflect.TypeOf(e)
		return errorTypeSet[errorType] == true
	}
}

func (r *Retrier) SetBackOffOpts(b *BackOffOpts) {
	r.backOffOpts = b
}

func (r *Retrier) newBackOff() backoff.BackOff {
	b := backoff.NewExponentialBackOff()
	b.InitialInterval = r.backOffOpts.InitialInterval
	b.MaxInterval = r.backOffOpts.MaxInterval
	b.MaxElapsedTime = r.backOffOpts.MaxElapsedTime
	return b
}

func instancesToTypes(instances []interface{}) []reflect.Type {
	types := []reflect.Type{}
	for _, instance := range instances {
		types = append(types, reflect.TypeOf(instance))
	}
	return types
}
<commit_msg>Fix logging.<commit_after>package retry

import (
	"log"
	"reflect"
	"time"

	"github.com/cenkalti/backoff"
)

type BackOffOpts struct {
	InitialInterval time.Duration
	MaxInterval     time.Duration
	MaxElapsedTime  time.Duration
}

var DefaultBackOffOpts *BackOffOpts = &BackOffOpts{
	InitialInterval: 500 * time.Millisecond,
	MaxInterval:     3 * time.Second,
	MaxElapsedTime:  10 * time.Second}

var RetryOnAnyError = func(error) bool { return true }

type RetryNotifier func(*RetryEvent)

type Retrier struct {
	Name              string
	backOffOpts       *BackOffOpts
	shouldRetryFunc   func(error) bool
	notifyRetryFuncs  []RetryNotifier
	notifyGaveUpFuncs []RetryNotifier
}

func NewRetrier(name string,
	backOffOpts *BackOffOpts, shouldRetryFunc func(error) bool) *Retrier {

	return &Retrier{
		Name: name, backOffOpts: backOffOpts, shouldRetryFunc: shouldRetryFunc,
		notifyRetryFuncs:  []RetryNotifier{logRetry},
		notifyGaveUpFuncs: []RetryNotifier{logGaveUp}}
}

func NewErrorTypeRetrier(name string,
	backOffOpts *BackOffOpts, errorTypes ...interface{}) *Retrier {

	return &Retrier{
		Name:            name,
		backOffOpts:     backOffOpts,
		shouldRetryFunc: RetryWhenErrorTypeMatches(instancesToTypes(errorTypes))}
}

func (r *Retrier) Retry(f func() (interface{}, error)) (interface{}, error) {
	var val interface{}
	var err error
	var next time.Duration

	numTries := 0
	b := r.newBackOff()
	b.Reset()
	for {
		numTries++
		if val, err = f(); err == nil {
			return val, nil
		}

		if !r.shouldRetryFunc(err) {
			return val, err
		}

		if next = b.NextBackOff(); next == backoff.Stop {
			r.notifyGaveUp(err, numTries)
			return val, err
		}

		time.Sleep(next)
		r.notifyRetry(err, numTries)
	}
}

type RetryEvent struct {
	Retrier *Retrier
	Err     error
	NumTries int
}

func (r *Retrier) AddNotifyRetry(f RetryNotifier) {
	r.notifyRetryFuncs = append(r.notifyRetryFuncs, f)
}

func (r *Retrier) AddNotifyGaveUp(f RetryNotifier) {
	r.notifyGaveUpFuncs = append(r.notifyGaveUpFuncs, f)
}

func (r *Retrier) notifyGaveUp(err error, numTries int) {
	retryEvent := &RetryEvent{Retrier: r, Err: err, NumTries: numTries}
	for _, notifyGaveUpFunc := range r.notifyGaveUpFuncs {
		notifyGaveUpFunc(retryEvent)
	}
}

func (r *Retrier) notifyRetry(err error, numTries int) {
	retryEvent := &RetryEvent{Retrier: r, Err: err, NumTries: numTries}
	for _, notifyRetryFunc := range r.notifyRetryFuncs {
		notifyRetryFunc(retryEvent)
	}
}

func logRetry(re *RetryEvent) {
	log.Printf("Retrying after %d tries: error=%s count#retry.%s.retry_count=1\n",
		re.NumTries, re.Err.Error(), re.Retrier.Name)
}

func logGaveUp(re *RetryEvent) {
	log.Printf("Giving up after %d tries: error=%s count#retry.%s.gave_up_count=1\n",
		re.NumTries, re.Err.Error(), re.Retrier.Name)
}

func RetryWhenErrorTypeMatches(errorTypes []reflect.Type) func(error) bool {
	errorTypeSet := make(map[reflect.Type]bool)
	for _, t := range errorTypes {
		errorTypeSet[t] = true
	}
	return func(e error) bool {
		errorType := reflect.TypeOf(e)
		return errorTypeSet[errorType] == true
	}
}

func (r *Retrier) SetBackOffOpts(b *BackOffOpts) {
	r.backOffOpts = b
}

func (r *Retrier) newBackOff() backoff.BackOff {
	b := backoff.NewExponentialBackOff()
	b.InitialInterval = r.backOffOpts.InitialInterval
	b.MaxInterval = r.backOffOpts.MaxInterval
	b.MaxElapsedTime = r.backOffOpts.MaxElapsedTime
	return b
}

func instancesToTypes(instances []interface{}) []reflect.Type {
	types := []reflect.Type{}
	for _, instance := range instances {
		types = append(types, reflect.TypeOf(instance))
	}
	return types
}
<|endoftext|>


<commit_before>// Copyright © 2016 See CONTRIBUTORS <ignasi.fosch@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reviewer

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/google/go-github/github"
	"github.com/spf13/viper"
	"golang.org/x/oauth2"
)

// GetString contains the function used to lookup environment variables.
var GetString = viper.GetString

// ChangesServicer is an interface for listing changes.
type ChangesServicer interface {
	List(string, string, *github.PullRequestListOptions) ([]github.PullRequest, *github.Response, error)
	Get(string, string, int) (*github.PullRequest, *github.Response, error)
}

// TicketsServicer is an interface for listing changes.
type TicketsServicer interface {
	ListComments(string, string, int, *github.IssueListCommentsOptions) ([]github.IssueComment, *github.Response, error)
}

// GHClient is the wrapper around github.Client.
type GHClient struct {
	client  *github.Client
	Changes ChangesServicer
	Tickets TicketsServicer
}

// NewGHClient is the constructor for GHClient.
func NewGHClient(httpClient *http.Client) *GHClient {
	client := &GHClient{
		client: github.NewClient(httpClient),
	}
	client.Changes = client.client.PullRequests
	client.Tickets = client.client.Issues
	return client
}

// PullRequestInfo contains the id, title, and CR score of a pull request.
type PullRequestInfo struct {
	Number int // id of the pull request
	Title  string
	Score  int
}

// GetClient returns a github.Client authenticated.
func GetClient() (*GHClient, error) {
	token := GetString("authorization.token")
	if token == "" {
		return nil, errors.New("An error occurred getting REVIEWER_TOKEN environment variable\n")
	}
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)

	return NewGHClient(tc), nil
}

// getCommentSuccesScore returns the score for the Comment.
func getCommentSuccessScore(comment string) int {
	score := 0
	if strings.Contains(comment, "+1") {
		score++
	}
	if strings.Contains(comment, "-1") {
		score--
	}
	return score
}

// GetPullRequestInfos returns the list of pull requests and the CR success score based on comments
func GetPullRequestInfos(client *GHClient, owner string, repo string) ([]PullRequestInfo, error) {
	//TODO: https://github.com/gophergala2016/reviewer/issues/23

	pullRequests, _, err := client.Changes.List(owner, repo, nil)
	if err != nil {
		return nil, err
	}
	pris := make([]PullRequestInfo, len(pullRequests))
	for n, pullRequest := range pullRequests {
		pris[n].Number = *pullRequest.Number
		pris[n].Title = *pullRequest.Title
		comments, _, err := client.Tickets.ListComments(owner, repo, *pullRequest.Number, nil)
		if err != nil {
			return nil, err
		}
		for _, comment := range comments {
			if comment.Body == nil {
				continue
			}
			pris[n].Score += getCommentSuccessScore(*comment.Body)
		}
	}
	return pris, nil
}

// IsMergeable returns true if the PullRequest is mergeable.
func IsMergeable(pullRequest *github.PullRequest) bool {
	return *pullRequest.Mergeable
}

// PassedTests checks if the PR statuses are ok.
func PassedTests(client *GHClient, pullRequest *github.PullRequest, owner string, repo string) (bool, error) {
	head := *pullRequest.Head.SHA
	combinedStatus, _, err := client.client.Repositories.GetCombinedStatus(owner, repo, head, nil)

	if err != nil {
		return false, err
	}
	return (*combinedStatus.State == "success"), nil
}

// Execute checks if the PR defers to be merged.
func Execute(DryRun bool) bool {
	if DryRun {
		fmt.Printf("Working in dry-run mode...\n")
	}
	err := CheckFile()
	if err != nil {
		log.Fatal(err)
	}
	err = CheckRepositories()
	if err != nil {
		log.Fatal(err)
	}
	repositories := NewConfig(viper.Sub("repositories"))
	client, err := GetClient()
	if err != nil {
		log.Fatalf("Error creating GitHub client %v", err)
	}

	//TODO: validate imput parameters (e.g. Required = 0)
	for _, repoName := range repositories.AllKeys() {
		username := repositories.GetString(repoName + ".username")
		status := repositories.GetBool(repoName + ".status")
		required := repositories.GetInt(repoName + ".required")
		if !status {
			fmt.Printf("- %v/%v Discarded (repo disabled)\n", username, repoName)
			continue
		}
		prInfos, err := GetPullRequestInfos(client, username, repoName)
		if err != nil {
			fmt.Printf("Error getting pull request info of repo %v/%v", username, repoName)
			continue
		}
		fmt.Printf("+ %v/%v\n", username, repoName)
		for _, prInfo := range prInfos {
			pullRequest, _, err := client.Changes.Get(username, repoName, prInfo.Number)
			if err != nil {
				fmt.Printf("  - %v NOP   (%v) Failure getting pull request\n", prInfo.Number, prInfo.Title)
				continue
			}
			if !IsMergeable(pullRequest) {
				fmt.Printf("  - %v NOP   (%v) Not mergeable\n", prInfo.Number, prInfo.Title)
				continue
			}
			var passedTests bool
			passedTests, err = PassedTests(client, pullRequest, username, repoName)
			if err != nil {
				fmt.Printf("  - %v NOP   (%v) %s\n", prInfo.Number, prInfo.Title, err)
				continue
			}
			if !passedTests {
				fmt.Printf("  - %v NOP   (%v) Tests not passed\n", prInfo.Number, prInfo.Title)
				continue
			}
			if prInfo.Score < required {
				fmt.Printf("  - %v NOP   (%v) score %v of %v required\n", prInfo.Number, prInfo.Title, prInfo.Score, required)
				continue
			}
			if !DryRun {
				fmt.Printf("  + %v MERGE (%v) score %v of %v required\n", prInfo.Number, prInfo.Title, prInfo.Score, required)
				// merge here
			} else {
				fmt.Printf("  - %v (merge)  (%v) score %v of %v required\n", prInfo.Number, prInfo.Title, prInfo.Score, required)
			}
			continue
		}
	}
	return true
}
<commit_msg>Adds Merge function<commit_after>// Copyright © 2016 See CONTRIBUTORS <ignasi.fosch@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package reviewer

import (
	"errors"
	"fmt"
	"log"
	"net/http"
	"strings"

	"github.com/google/go-github/github"
	"github.com/spf13/viper"
	"golang.org/x/oauth2"
)

// GetString contains the function used to lookup environment variables.
var GetString = viper.GetString

// ChangesServicer is an interface for listing changes.
type ChangesServicer interface {
	List(string, string, *github.PullRequestListOptions) ([]github.PullRequest, *github.Response, error)
	Get(string, string, int) (*github.PullRequest, *github.Response, error)
}

// TicketsServicer is an interface for listing changes.
type TicketsServicer interface {
	ListComments(string, string, int, *github.IssueListCommentsOptions) ([]github.IssueComment, *github.Response, error)
}

// GHClient is the wrapper around github.Client.
type GHClient struct {
	client  *github.Client
	Changes ChangesServicer
	Tickets TicketsServicer
}

// NewGHClient is the constructor for GHClient.
func NewGHClient(httpClient *http.Client) *GHClient {
	client := &GHClient{
		client: github.NewClient(httpClient),
	}
	client.Changes = client.client.PullRequests
	client.Tickets = client.client.Issues
	return client
}

// PullRequestInfo contains the id, title, and CR score of a pull request.
type PullRequestInfo struct {
	Number int // id of the pull request
	Title  string
	Score  int
}

// GetClient returns a github.Client authenticated.
func GetClient() (*GHClient, error) {
	token := GetString("authorization.token")
	if token == "" {
		return nil, errors.New("An error occurred getting REVIEWER_TOKEN environment variable\n")
	}
	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: token},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)

	return NewGHClient(tc), nil
}

// getCommentSuccesScore returns the score for the Comment.
func getCommentSuccessScore(comment string) int {
	score := 0
	if strings.Contains(comment, "+1") {
		score++
	}
	if strings.Contains(comment, "-1") {
		score--
	}
	return score
}

// GetPullRequestInfos returns the list of pull requests and the CR success score based on comments
func GetPullRequestInfos(client *GHClient, owner string, repo string) ([]PullRequestInfo, error) {
	//TODO: https://github.com/gophergala2016/reviewer/issues/23

	pullRequests, _, err := client.Changes.List(owner, repo, nil)
	if err != nil {
		return nil, err
	}
	pris := make([]PullRequestInfo, len(pullRequests))
	for n, pullRequest := range pullRequests {
		pris[n].Number = *pullRequest.Number
		pris[n].Title = *pullRequest.Title
		comments, _, err := client.Tickets.ListComments(owner, repo, *pullRequest.Number, nil)
		if err != nil {
			return nil, err
		}
		for _, comment := range comments {
			if comment.Body == nil {
				continue
			}
			pris[n].Score += getCommentSuccessScore(*comment.Body)
		}
	}
	return pris, nil
}

// IsMergeable returns true if the PullRequest is mergeable.
func IsMergeable(pullRequest *github.PullRequest) bool {
	return *pullRequest.Mergeable
}

// PassedTests checks if the PR statuses are ok.
func PassedTests(client *GHClient, pullRequest *github.PullRequest, owner string, repo string) (bool, error) {
	head := *pullRequest.Head.SHA
	combinedStatus, _, err := client.client.Repositories.GetCombinedStatus(owner, repo, head, nil)

	if err != nil {
		return false, err
	}
	return (*combinedStatus.State == "success"), nil
}

// Merge does the merge.
func Merge(client *GHClient, owner string, repo string, number int) (*github.PullRequestMergeResult, error) {
	result, _, err := client.client.PullRequests.Merge(owner, repo, number, "Merged automatically by Reviewer")
	return result, err
}

// Execute checks if the PR defers to be merged.
func Execute(DryRun bool) bool {
	if DryRun {
		fmt.Printf("Working in dry-run mode...\n")
	}
	err := CheckFile()
	if err != nil {
		log.Fatal(err)
	}
	err = CheckRepositories()
	if err != nil {
		log.Fatal(err)
	}
	repositories := NewConfig(viper.Sub("repositories"))
	client, err := GetClient()
	if err != nil {
		log.Fatalf("Error creating GitHub client %v", err)
	}

	//TODO: validate imput parameters (e.g. Required = 0)
	for _, repoName := range repositories.AllKeys() {
		username := repositories.GetString(repoName + ".username")
		status := repositories.GetBool(repoName + ".status")
		required := repositories.GetInt(repoName + ".required")
		if !status {
			fmt.Printf("- %v/%v Discarded (repo disabled)\n", username, repoName)
			continue
		}
		prInfos, err := GetPullRequestInfos(client, username, repoName)
		if err != nil {
			fmt.Printf("Error getting pull request info of repo %v/%v", username, repoName)
			continue
		}
		fmt.Printf("+ %v/%v\n", username, repoName)
		for _, prInfo := range prInfos {
			pullRequest, _, err := client.Changes.Get(username, repoName, prInfo.Number)
			if err != nil {
				fmt.Printf("  - %v NOP   (%v) Failure getting pull request\n", prInfo.Number, prInfo.Title)
				continue
			}
			if !IsMergeable(pullRequest) {
				fmt.Printf("  - %v NOP   (%v) Not mergeable\n", prInfo.Number, prInfo.Title)
				continue
			}
			var passedTests bool
			passedTests, err = PassedTests(client, pullRequest, username, repoName)
			if err != nil {
				fmt.Printf("  - %v NOP   (%v) %s\n", prInfo.Number, prInfo.Title, err)
				continue
			}
			if !passedTests {
				fmt.Printf("  - %v NOP   (%v) Tests not passed\n", prInfo.Number, prInfo.Title)
				continue
			}
			if prInfo.Score < required {
				fmt.Printf("  - %v NOP   (%v) score %v of %v required\n", prInfo.Number, prInfo.Title, prInfo.Score, required)
				continue
			}
			if !DryRun {
				fmt.Printf("  + %v MERGE (%v) score %v of %v required\n", prInfo.Number, prInfo.Title, prInfo.Score, required)
				// merge here
			} else {
				fmt.Printf("  - %v (merge)  (%v) score %v of %v required\n", prInfo.Number, prInfo.Title, prInfo.Score, required)
			}
			continue
		}
	}
	return true
}
<|endoftext|>


<commit_before>/*
	Package message contains Message structure and implements every method on it.
*/
package message

import (
	"bytes"
	"errors"
	"fmt"
	"time"

	"github.com/ishail/m-mail/common"
)

func NewMessage(subject, body, emailType string, settings ...MessageSetting) *Message {
	if emailType == "html" || emailType == "text/html" {
		emailType = "text/html"
	} else {
		emailType = "text/plain"
	}

	msg := &Message{
		subject:   subject,
		body:      body,
		emailType: emailType,
		header:    make(common.Header),
		charset:   "UTF-8",
		encoding:  common.QuotedPrintable,
	}

	msg.ApplySettings(settings)

	if msg.encoding == common.Base64 {
		msg.hEncoder = common.BEncoding
	} else {
		msg.hEncoder = common.QEncoding
	}

	return msg
}

func (msg *Message) ApplySettings(settings []MessageSetting) {
	for _, setting := range settings {
		setting(msg)
	}
}

// Reset resets the message so it can be reused. The message keeps its previous
// settings so it is in the same state that after a call to NewMessage.
func (msg *Message) Reset() {
	for key := range msg.header {
		delete(msg.header, key)
	}
	msg.parts = nil
	msg.attachments = nil
	msg.embedded = nil
}

func (msg *Message) SetHeader(field string, value ...string) {
	msg.encodeHeader(value)
	msg.header[field] = value
}

func (msg *Message) encodeHeader(values []string) {
	for index, val := range values {
		values[index] = msg.encodeString(val)
	}
}

func (msg *Message) encodeString(value string) string {
	return msg.hEncoder.Encode(msg.charset, value)
}

// SetHeaders sets the message headers.
func (msg *Message) SetHeaders(headers common.Header) {
	for key, val := range headers {
		msg.SetHeader(key, val...)
	}
}

// SetAddressHeader sets an address to the given header field.
func (msg *Message) SetAddressHeader(field, address, name string) {
	msg.header[field] = []string{msg.FormatAddress(address, name)}
}

// FormatAddress formats an address and a name as a valid RFC 5322 address.
func (msg *Message) FormatAddress(address, name string) string {
	if name == "" {
		return address
	}

	enc := msg.encodeString(name)
	if enc == name {
		msg.buff.WriteByte('"')
		for _, character := range name {
			if character == '\\' || character == '"' {
				msg.buff.WriteByte('\\')
			}
			msg.buff.WriteByte(byte(character))
		}
		msg.buff.WriteByte('"')
	} else if common.HasSpecials(name) {
		msg.buff.WriteString(common.BEncoding.Encode(msg.charset, name))
	} else {
		msg.buff.WriteString(enc)
	}

	msg.buff.WriteString(" <")
	msg.buff.WriteString(address)
	msg.buff.WriteByte('>')

	addr := msg.buff.String()
	msg.buff.Reset()
	return addr
}

// SetDateHeader sets a date to the given header field.
func (msg *Message) SetDateHeader(field string, date time.Time) {
	msg.header[field] = []string{common.FormatDate(date)}
}

// GetHeader gets a header field.
func (msg *Message) GetHeader(field string) []string {
	return msg.header[field]
}

//Get From address from Message model
func (msg *Message) GetFrom() (string, error) {
	if from, ok := msg.header["From"]; ok {
		if len(from) > 0 {
			return common.ParseAddress(from[0])
		}
	}
	return "", errors.New("m-mail: invalid message, 'From' field is missing!")
}

//Get list of recipients(To, Cc, Bcc) from Message object
func (msg *Message) GetRecipients() ([]string, error) {
	recipientLength := 0
	addrHeaderList := []string{"To", "Cc", "Bcc"}

	for _, field := range addrHeaderList {
		if addresses, ok := msg.header[field]; ok {
			recipientLength += len(addresses)
		}
	}
	recipients := make([]string, recipientLength)
	index := 0

	for _, field := range addrHeaderList {
		if addresses, ok := msg.header[field]; ok {
			for _, addr := range addresses {
				if addr, err := common.ParseAddress(addr); err != nil {
					return nil, fmt.Errorf(
						"m-mail: Unable to parse address. Address: %s, Error: %v", addr, err)
				} else {
					recipients[index] = addr
					index++
				}
			}
		}
	}

	return recipients, nil
}

//Convert Message object into bytes
func (msg *Message) GetEmailBytes(to string) []byte {
	var msgBytes bytes.Buffer

	msgBytes.WriteString("To: " + to + "\r\n")
	msgBytes.WriteString("Date: " + time.Now().String() + "\r\n")
	msgBytes.WriteString("Subject: " + msg.subject + "\r\n")
	msgBytes.WriteString("Content-Type: multipart/alternative;\r\n")
	msgBytes.WriteString(`    boundary="boundary-type-1234567892-alt"` + "\r\n")
	msgBytes.WriteString("Mime-Version: 1.0\r\n\r\n")
	msgBytes.WriteString("--boundary-type-1234567892-alt\r\n")
	msgBytes.WriteString("Content-Type: " + msg.emailType + `; charset=` + msg.charset + "\r\n")
	msgBytes.WriteString("Content-Transfer-Encoding: quoted-printable\r\n\r\n")
	msgBytes.WriteString(msg.body + "\r\n")

	return msgBytes.Bytes()
}

//Returns headers of message as RFC format
func (msg *Message) getHeadersBytes() []byte {
	var headers bytes.Buffer
	for key, value := range msg.header {
		headers.Write(getHeaderBytes(key, value...))
	}

	return headers.Bytes()
}
<commit_msg>html email fix<commit_after>/*
	Package message contains Message structure and implements every method on it.
*/
package message

import (
	"bytes"
	"errors"
	"fmt"
	"mime/quotedprintable"
	"time"

	"github.com/ishail/m-mail/common"
)

func NewMessage(subject, body, emailType string, settings ...MessageSetting) *Message {
	if emailType == "html" || emailType == "text/html" {
		emailType = "text/html"
	} else {
		emailType = "text/plain"
	}

	msg := &Message{
		subject:   subject,
		body:      body,
		emailType: emailType,
		header:    make(common.Header),
		charset:   "UTF-8",
		encoding:  common.QuotedPrintable,
	}

	msg.ApplySettings(settings)

	if msg.encoding == common.Base64 {
		msg.hEncoder = common.BEncoding
	} else {
		msg.hEncoder = common.QEncoding
	}

	return msg
}

func (msg *Message) ApplySettings(settings []MessageSetting) {
	for _, setting := range settings {
		setting(msg)
	}
}

// Reset resets the message so it can be reused. The message keeps its previous
// settings so it is in the same state that after a call to NewMessage.
func (msg *Message) Reset() {
	for key := range msg.header {
		delete(msg.header, key)
	}
	msg.parts = nil
	msg.attachments = nil
	msg.embedded = nil
}

func (msg *Message) SetHeader(field string, value ...string) {
	msg.encodeHeader(value)
	msg.header[field] = value
}

func (msg *Message) encodeHeader(values []string) {
	for index, val := range values {
		values[index] = msg.encodeString(val)
	}
}

func (msg *Message) encodeString(value string) string {
	return msg.hEncoder.Encode(msg.charset, value)
}

// SetHeaders sets the message headers.
func (msg *Message) SetHeaders(headers common.Header) {
	for key, val := range headers {
		msg.SetHeader(key, val...)
	}
}

// SetAddressHeader sets an address to the given header field.
func (msg *Message) SetAddressHeader(field, address, name string) {
	msg.header[field] = []string{msg.FormatAddress(address, name)}
}

// FormatAddress formats an address and a name as a valid RFC 5322 address.
func (msg *Message) FormatAddress(address, name string) string {
	if name == "" {
		return address
	}

	enc := msg.encodeString(name)
	if enc == name {
		msg.buff.WriteByte('"')
		for _, character := range name {
			if character == '\\' || character == '"' {
				msg.buff.WriteByte('\\')
			}
			msg.buff.WriteByte(byte(character))
		}
		msg.buff.WriteByte('"')
	} else if common.HasSpecials(name) {
		msg.buff.WriteString(common.BEncoding.Encode(msg.charset, name))
	} else {
		msg.buff.WriteString(enc)
	}

	msg.buff.WriteString(" <")
	msg.buff.WriteString(address)
	msg.buff.WriteByte('>')

	addr := msg.buff.String()
	msg.buff.Reset()
	return addr
}

// SetDateHeader sets a date to the given header field.
func (msg *Message) SetDateHeader(field string, date time.Time) {
	msg.header[field] = []string{common.FormatDate(date)}
}

// GetHeader gets a header field.
func (msg *Message) GetHeader(field string) []string {
	return msg.header[field]
}

//Get From address from Message model
func (msg *Message) GetFrom() (string, error) {
	if from, ok := msg.header["From"]; ok {
		if len(from) > 0 {
			return common.ParseAddress(from[0])
		}
	}
	return "", errors.New("m-mail: invalid message, 'From' field is missing!")
}

//Get list of recipients(To, Cc, Bcc) from Message object
func (msg *Message) GetRecipients() ([]string, error) {
	recipientLength := 0
	addrHeaderList := []string{"To", "Cc", "Bcc"}

	for _, field := range addrHeaderList {
		if addresses, ok := msg.header[field]; ok {
			recipientLength += len(addresses)
		}
	}
	recipients := make([]string, recipientLength)
	index := 0

	for _, field := range addrHeaderList {
		if addresses, ok := msg.header[field]; ok {
			for _, addr := range addresses {
				if addr, err := common.ParseAddress(addr); err != nil {
					return nil, fmt.Errorf(
						"m-mail: Unable to parse address. Address: %s, Error: %v", addr, err)
				} else {
					recipients[index] = addr
					index++
				}
			}
		}
	}

	return recipients, nil
}

//Convert Message object into bytes
func (msg *Message) GetEmailBytes(to string) []byte {
	var msgBytes bytes.Buffer

	msgBytes.WriteString("To: " + to + "\r\n")
	msgBytes.WriteString("Date: " + time.Now().String() + "\r\n")
	msgBytes.WriteString("Subject: " + msg.subject + "\r\n")
	// msgBytes.WriteString("Content-Type: multipart/alternative;\r\n")
	// msgBytes.WriteString(`    boundary="boundary-type-1234567892-alt"` + "\r\n")
	// msgBytes.WriteString("Mime-Version: 1.0\r\n\r\n")
	// msgBytes.WriteString("--boundary-type-1234567892-alt\r\n")
	msgBytes.WriteString("Content-Type: " + msg.emailType + `; charset=` + msg.charset + "\r\n")
	msgBytes.WriteString("Content-Transfer-Encoding: quoted-printable\r\n\r\n")

	w := quotedprintable.NewWriter(&msgBytes)
	w.Write([]byte(msg.body))

	// if _, ok := msg.header["Mime-Version"]; !ok {
	// 	msgBytes.WriteString("Mime-Version: 1.0\r\n")
	// }
	// if _, ok := msg.header["Date"]; !ok {
	// 	msgBytes.Write(writeHeader("Date", common.FormatDate(time.Now())))
	// }
	// msgBytes.Write(writeHeaders(msg.header))

	// if msg.hasMixedPart() {
	// 	msgBytes.Write(openMultipart("mixed"))
	// }

	// if msg.hasRelatedPart() {
	// 	msgBytes.Write(openMultipart("related"))
	// }

	// if msg.hasAlternativePart() {
	// 	msgBytes.Write(openMultipart("alternative"))
	// }
	// for _, part := range msg.parts {
	// 	msgBytes.Write(writePart(part, msg.charset))
	// }
	// if msg.hasAlternativePart() {
	// 	msgBytes.closeMultipart()
	// }

	// msgBytes.addFiles(msg.embedded, false)
	// if msg.hasRelatedPart() {
	// 	msgBytes.closeMultipart()
	// }

	// msgBytes.addFiles(msg.attachments, true)
	// if msg.hasMixedPart() {
	// 	msgBytes.closeMultipart()
	// }

	return msgBytes.Bytes()
}

func writeHeaders(header common.Header) []byte {
	var buff bytes.Buffer
	for key, val := range header {
		buff.Write(writeHeader(key, val...))
	}

	return buff.Bytes()
}

func writeHeader(key string, value ...string) []byte {
	var buff bytes.Buffer

	buff.WriteString(key)
	if len(value) == 0 {
		buff.WriteString(":\r\n")
		return buff.Bytes()
	}
	buff.WriteString(": ")

	// Max header line length is 78 characters in RFC 5322 and 76 characters
	// in RFC 2047. So for the sake of simplicity we use the 76 characters
	// limit.
	charsLeft := 76 - len(key) - len(": ")

	for i, val := range value {
		// If the line is already too long, insert a newline right away.
		if charsLeft < 1 {
			if i == 0 {
				buff.WriteString("\r\n ")
			} else {
				buff.WriteString(",\r\n ")
			}
			charsLeft = 75
		} else {
			buff.WriteString(", ")
			charsLeft -= 2
		}
		buff.WriteString(val)

		// While the header content is too long, fold it by inserting a newline.
		// for len(val) > charsLeft {
		// 	n, _ := buff.WriteString(writeLine(val, charsLeft))
		// 	charsLeft = 75
		// }
		// buff.WriteString(val)
		// if i := lastIndexByte(val, '\n'); i != -1 {
		// 	charsLeft = 75 - (len(val) - i - 1)
		// } else {
		// 	charsLeft -= len(val)
		// }
	}
	buff.WriteString("\r\n")

	return buff.Bytes()
}

// func writeLine(s string, charsLeft int) string {
// 	var buff bytes.Buffer
// 	// If there is already a newline before the limit. Write the line.
// 	if i := strings.IndexByte(s, '\n'); i != -1 && i < charsLeft {
// 		buff.WriteString(s[:i+1])
// 		return s[i+1:]
// 	}

// 	for i := charsLeft - 1; i >= 0; i-- {
// 		if s[i] == ' ' {
// 			buff.WriteString(s[:i])
// 			buff.WriteString("\r\n ")
// 			return s[i+1:]
// 		}
// 	}

// 	// We could not insert a newline cleanly so look for a space or a newline
// 	// even if it is after the limit.
// 	for i := 75; i < len(s); i++ {
// 		if s[i] == ' ' {
// 			buff.WriteString(s[:i])
// 			buff.WriteString("\r\n ")
// 			return s[i+1:]
// 		}
// 		if s[i] == '\n' {
// 			buff.WriteString(s[:i+1])
// 			return s[i+1:]
// 		}
// 	}

// 	// Too bad, no space or newline in the whole string. Just write everything.
// 	buff.WriteString(s)
// 	return ""
// }

//Returns headers of message as RFC format
func (msg *Message) getHeadersBytes() []byte {
	var headers bytes.Buffer
	for key, value := range msg.header {
		headers.Write(getHeaderBytes(key, value...))
	}

	return headers.Bytes()
}

func (msg *Message) hasMixedPart() bool {
	return len(msg.attachments) > 1 || (len(msg.parts) > 0 && len(msg.attachments) > 0)
}

func (msg *Message) hasRelatedPart() bool {
	return len(msg.attachments) > 1 || (len(msg.parts) > 0 && len(msg.embedded) > 0)
}

func (msg *Message) hasAlternativePart() bool {
	return len(msg.parts) > 1
}

func openMultipart(mimeType string) []byte {
	var buff bytes.Buffer
	contentType := "multipart/" + mimeType + ";\r\n boundary=boundary-type-1234567892-alt"

	buff.Write(writeHeader("Content-Type", contentType))
	buff.WriteString("\r\n")
	return buff.Bytes()
}

// func writePart(msg, contentType, charset string) []byte {
// 	var buff bytes.Buffer
// 	buff.Write(
// 		writeHeaders(map[string][]string{
// 			"Content-Type":              {contentType + "; charset=" + charset},
// 			"Content-Transfer-Encoding": {string(p.encoding)},
// 		}))
// 	w.writeBody(p.copier, p.encoding)

// 	return buff.Bytes()
// }
<|endoftext|>


<commit_before>package m_etcd

import (
	"fmt"
	"os"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
)

type Watcher func(cordCtx metafora.CoordinatorContext)
type EtcdWatcher struct {
	cordCtx      metafora.CoordinatorContext
	path         string
	responseChan chan *etcd.Response
	stopChan     chan bool
	errorChan    chan error
	client       *etcd.Client
	isClosed     bool
}

func (w *EtcdWatcher) StartWatching() {
	go func() {
		const recursive = true
		const etcdIndex = uint64(0) //0 == latest version

		//TODO, Im guessing that watch only picks up new nodes.
		//   We need to manually do an ls at startup and dump the results to taskWatcherResponses,
		//   after we filter out all non-claimed tasks.
		_, err := w.client.Watch(
			w.path,
			etcdIndex,
			recursive,
			w.responseChan,
			w.stopChan)
		if err == etcd.ErrWatchStoppedByUser {
			// This isn't actually an error, return nil
			err = nil
		}
		w.errorChan <- err
		close(w.errorChan)
	}()
}

// Close stops the watching goroutine. Close will panic if called more than
// once.
func (w *EtcdWatcher) Close() error {
	if w.isClosed {
		close(w.stopChan)
		w.isClosed = true
		return <-w.errorChan
	}

	return nil
}

type EtcdCoordinator struct {
	Client    *etcd.Client
	cordCtx   metafora.CoordinatorContext
	Namespace string
	TaskPath  string

	taskWatcher       *EtcdWatcher
	tasksPrefetchChan chan *etcd.Node

	NodeID         string
	CommandPath    string
	commandWatcher *EtcdWatcher
}

func NewEtcdCoordinator(nodeId, namespace string, client *etcd.Client) metafora.Coordinator {
	namespace = strings.Trim(namespace, "/ ")

	if nodeId == "" {
		hn, _ := os.Hostname()
		//Adding the UUID incase we run two nodes on the same box.
		// TODO lets move this to the Readme as part of the example of calling NewEtcdCoordinator.
		// Then just remove the Autocreated nodeId.
		nodeId = hn + uuid.NewRandom().String()
	}

	nodeId = strings.Trim(nodeId, "/ ")

	etcd := &EtcdCoordinator{
		Client:    client,
		Namespace: namespace,

		TaskPath:    fmt.Sprintf("/%s/%s", namespace, TasksPath), //TODO MAKE A PACKAGE FUNC TO CREATE THIS PATH.
		taskWatcher: nil,

		NodeID:         nodeId,
		CommandPath:    fmt.Sprintf("/%s/%s/%s/%s", namespace, NodesPath, nodeId, CommandsPath),
		commandWatcher: nil,
	}

	return etcd
}

// Init is called once by the consumer to provide a Logger to Coordinator
// implementations.
func (ec *EtcdCoordinator) Init(cordCtx metafora.CoordinatorContext) {
	ec.Client.SetConsistency(etcd.STRONG_CONSISTENCY)

	cordCtx.Log(metafora.LogLevelDebug, "namespace[%s] Etcd-Cluster-Peers[%s]", ec.Namespace,
		strings.Join(ec.Client.GetCluster(), ", "))

	ec.cordCtx = cordCtx

	ec.upsertDir("/"+ec.Namespace, ForeverTTL)
	ec.upsertDir(ec.TaskPath, ForeverTTL)
	//ec.upsertDir(ec.NodePath, ForeverTTL)
	//ec.upsertDir(ec.CommandPath, ForeverTTL)

	ec.tasksPrefetchChan = make(chan *etcd.Node)
	ec.taskWatcher = &EtcdWatcher{
		cordCtx:      cordCtx,
		path:         ec.TaskPath,
		responseChan: make(chan *etcd.Response),
		stopChan:     make(chan bool),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	go func() {
		//First read the and dump the current tasks, then start up the background watcher()
		const sorted = false
		const recursive = true
		resp, err := ec.Client.Get(ec.TaskPath, sorted, recursive)

		if err != nil {
			cordCtx.Log(metafora.LogLevelDebug, "Init error getting the current tasks from the path:[%s] error:[%v]", ec.TaskPath, err)
		} else {

			cordCtx.Log(metafora.LogLevelDebug, "Fetching tasks, response from GET %s: response[action:%v etcdIndex:%v nodeKey:%v]",
				ec.TaskPath, resp.Action, resp.EtcdIndex, resp.Node.Key)

			for _, node := range resp.Node.Nodes {
				cordCtx.Log(metafora.LogLevelDebug, "---"+node.Key)
				ec.tasksPrefetchChan <- node
			}
		}

		//Don't start up the watcher until we've iterated over the existing tasks
		ec.taskWatcher.StartWatching()
	}()

	ec.commandWatcher = &EtcdWatcher{
		cordCtx:      cordCtx,
		path:         ec.CommandPath,
		responseChan: make(chan *etcd.Response),
		stopChan:     make(chan bool),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.commandWatcher.StartWatching()
}

func (ec *EtcdCoordinator) upsertDir(path string, ttl uint64) {

	pathMarker := path + "/" + CreatedMarker //hidden etcd key that isn't visible to directory get commands

	const sorted = false
	const recursive = false
	_, err := ec.Client.Get(pathMarker, sorted, recursive)
	if err != nil && strings.Contains(err.Error(), "Key not found") {
		_, err := ec.Client.CreateDir(path, ttl)
		if err != nil {
			ec.cordCtx.Log(metafora.LogLevelDebug, "Error trying to create directory. path:[%s] error:[ %v ]", path, err)
		}
		host, _ := os.Hostname()
		markerVal := fmt.Sprintf("createdAt=%s&by=%s", time.Now().String(), host)
		ec.Client.Set(pathMarker, markerVal, ttl)
	}
}

// Watch should do a blocking watch on the broker and return a task ID that
// can be claimed.
func (ec *EtcdCoordinator) Watch() (taskID string, err error) {

	for {

		select {
		// before the watcher starts, this channel is filled up with existing tasks during init()
		case node, ok := <-ec.tasksPrefetchChan:
			if !ok {
				return "", nil
			}
			ec.cordCtx.Log(metafora.LogLevelDebug, "New node from the tasksPrefetchChan %s: node[nodeKey:%v]",
				ec.TaskPath, node.Key)

			taskId, skip := ec.parseTaskIDFromNode(node)
			if !skip {
				return taskId, nil
			}
		case resp, ok := <-ec.taskWatcher.responseChan:
			if !ok {
				return "", nil
			}
			ec.cordCtx.Log(metafora.LogLevelDebug, "New response from watching %s: response[action:%v etcdIndex:%v nodeKey:%v]",
				ec.TaskPath, resp.Action, resp.EtcdIndex, resp.Node.Key)

			if resp.Action != "create" { //TODO eventually we'll need to deal with expired "owner" nodes.
				continue
			}

			taskId, skip := ec.parseTaskIDFromNode(resp.Node)
			if !skip {
				return taskId, nil
			}

		case err, ok := <-ec.taskWatcher.errorChan:
			if !ok {
				return "", nil
			}
			//if the watcher sends a nil, its because the etcd watcher was shutdown by Close()
			return "", err
		}
	}
}

func (ec *EtcdCoordinator) parseTaskIDFromNode(node *etcd.Node) (taskID string, skip bool) {
	taskId := ""

	if node == nil {
		//TODO log
		return "", true
	}

	//FIXME There's gotta be a better way to only detect tasks #32
	if strings.Contains(node.Key, OwnerMarker) {
		return "", true
	}

	ec.cordCtx.Log(metafora.LogLevelDebug, "Node key:%s", node.Key)
	taskpath := strings.Split(node.Key, "/")
	if len(taskpath) == 0 {
		//TODO log
		return "", true
	}
	if !node.Dir {
		return "", true
	}

	taskId = taskpath[len(taskpath)-1]

	if taskId == "tasks" {
		return "", true
	}

	return taskId, false
}

// Claim is called by the Consumer when a Balancer has determined that a task
// ID can be claimed. Claim returns false if another consumer has already
// claimed the ID.
func (ec *EtcdCoordinator) Claim(taskID string) bool {
	key := fmt.Sprintf("%s/%s/%s", ec.TaskPath, taskID, OwnerMarker)
	res, err := ec.Client.CreateDir(key, ClaimTTL)
	if err != nil {
		ec.cordCtx.Log(metafora.LogLevelDebug, "Claim failed: err %v", err)
		return false
	}
	ec.cordCtx.Log(metafora.LogLevelDebug, "Claim successful: resp %v", res)
	return true
}

// Release deletes the claim directory.
func (ec *EtcdCoordinator) Release(taskID string) {
	key := fmt.Sprintf("%s/%s/%s", ec.TaskPath, taskID, OwnerMarker)
	//FIXME Conditionally delete only if this node is actually the owner
	_, err := ec.Client.DeleteDir(key)
	if err != nil {
		//TODO Pause and retry?!
		ec.cordCtx.Log(metafora.LogLevelError, "Release failed: %v", err)
	}
}

// Command blocks until a command for this node is received from the broker
// by the coordinator.
func (ec *EtcdCoordinator) Command() (cmd string, err error) {
	<-ec.commandWatcher.responseChan
	return "", nil
	/*  TODO 1) cleanup the log here to match that of Watch
	         2) discuss the schema for the command channel...
	         3) Add code to the close method.
	select {
	case resp := <-ec.commandWatcherResponses:
		taskId := ""
		if resp.Node.Dir {
			taskId = resp.Node.Key
		} else {
			ec.cordCtx.Log(LogLevelWarning, "Command node needs to be a dir, not a key.")
			taskId = resp.Node.Key
		}
		return taskId, nil
	case err := <-ec.commandWatcherErrs:
		return "", err
	}
	*/
}

func (ec *EtcdCoordinator) Close() error {
	ec.taskWatcher.Close()
	ec.commandWatcher.Close()
	return nil
}
<commit_msg>cleaning some comments<commit_after>package m_etcd

import (
	"fmt"
	"os"
	"strings"
	"time"

	"code.google.com/p/go-uuid/uuid"
	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
)

type Watcher func(cordCtx metafora.CoordinatorContext)
type EtcdWatcher struct {
	cordCtx      metafora.CoordinatorContext
	path         string
	responseChan chan *etcd.Response
	stopChan     chan bool
	errorChan    chan error
	client       *etcd.Client
	isClosed     bool
}

func (w *EtcdWatcher) StartWatching() {
	go func() {
		const recursive = true
		const etcdIndex = uint64(0) //0 == latest version

		//TODO, Im guessing that watch only picks up new nodes.
		//   We need to manually do an ls at startup and dump the results to taskWatcherResponses,
		//   after we filter out all non-claimed tasks.
		_, err := w.client.Watch(
			w.path,
			etcdIndex,
			recursive,
			w.responseChan,
			w.stopChan)
		if err == etcd.ErrWatchStoppedByUser {
			// This isn't actually an error, return nil
			err = nil
		}
		w.errorChan <- err
		close(w.errorChan)
	}()
}

// Close stops the watching goroutine. Close will panic if called more than
// once.
func (w *EtcdWatcher) Close() error {
	if w.isClosed {
		close(w.stopChan)
		w.isClosed = true
		return <-w.errorChan
	}

	return nil
}

type EtcdCoordinator struct {
	Client    *etcd.Client
	cordCtx   metafora.CoordinatorContext
	Namespace string
	TaskPath  string

	taskWatcher       *EtcdWatcher
	tasksPrefetchChan chan *etcd.Node

	NodeID         string
	CommandPath    string
	commandWatcher *EtcdWatcher
}

func NewEtcdCoordinator(nodeId, namespace string, client *etcd.Client) metafora.Coordinator {
	namespace = strings.Trim(namespace, "/ ")

	if nodeId == "" {
		hn, _ := os.Hostname()
		//Adding the UUID incase we run two nodes on the same box.
		// TODO lets move this to the Readme as part of the example of calling NewEtcdCoordinator.
		// Then just remove the Autocreated nodeId.
		nodeId = hn + uuid.NewRandom().String()
	}

	nodeId = strings.Trim(nodeId, "/ ")

	etcd := &EtcdCoordinator{
		Client:    client,
		Namespace: namespace,

		TaskPath:    fmt.Sprintf("/%s/%s", namespace, TasksPath), //TODO MAKE A PACKAGE FUNC TO CREATE THIS PATH.
		taskWatcher: nil,

		NodeID:         nodeId,
		CommandPath:    fmt.Sprintf("/%s/%s/%s/%s", namespace, NodesPath, nodeId, CommandsPath),
		commandWatcher: nil,
	}

	return etcd
}

// Init is called once by the consumer to provide a Logger to Coordinator
// implementations.
func (ec *EtcdCoordinator) Init(cordCtx metafora.CoordinatorContext) {
	ec.Client.SetConsistency(etcd.STRONG_CONSISTENCY)

	cordCtx.Log(metafora.LogLevelDebug, "namespace[%s] Etcd-Cluster-Peers[%s]", ec.Namespace,
		strings.Join(ec.Client.GetCluster(), ", "))

	ec.cordCtx = cordCtx

	ec.upsertDir("/"+ec.Namespace, ForeverTTL)
	ec.upsertDir(ec.TaskPath, ForeverTTL)
	//ec.upsertDir(ec.NodePath, ForeverTTL)
	//ec.upsertDir(ec.CommandPath, ForeverTTL)

	ec.tasksPrefetchChan = make(chan *etcd.Node)
	ec.taskWatcher = &EtcdWatcher{
		cordCtx:      cordCtx,
		path:         ec.TaskPath,
		responseChan: make(chan *etcd.Response),
		stopChan:     make(chan bool),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	go func() {
		//Read the tasks
		const sorted = false
		const recursive = true
		resp, err := ec.Client.Get(ec.TaskPath, sorted, recursive)
		if err != nil {
			cordCtx.Log(metafora.LogLevelDebug, "Init error getting the current tasks from the path:[%s] error:[%v]", ec.TaskPath, err)
		} else {
			//Now dump any current tasks
			cordCtx.Log(metafora.LogLevelDebug, "Fetching tasks, response from GET %s: response[action:%v etcdIndex:%v nodeKey:%v]",
				ec.TaskPath, resp.Action, resp.EtcdIndex, resp.Node.Key)

			for _, node := range resp.Node.Nodes {
				ec.tasksPrefetchChan <- node
			}
		}

		//After processing the existing tasks, lets start up the background watcher() to look for new ones.
		ec.taskWatcher.StartWatching()
	}()

	ec.commandWatcher = &EtcdWatcher{
		cordCtx:      cordCtx,
		path:         ec.CommandPath,
		responseChan: make(chan *etcd.Response),
		stopChan:     make(chan bool),
		errorChan:    make(chan error),
		client:       ec.Client,
	}

	ec.commandWatcher.StartWatching()
}

func (ec *EtcdCoordinator) upsertDir(path string, ttl uint64) {

	//hidden etcd key that isn't visible to ls commands on the directory,
	//  you have to know about it to find it :).  I'm using it to add some
	//  info about when the cluster's schema was setup.
	pathMarker := path + "/" + CreatedMarker

	const sorted = false
	const recursive = false
	_, err := ec.Client.Get(path, sorted, recursive)
	if err != nil && strings.Contains(err.Error(), "Key not found") {
		_, err := ec.Client.CreateDir(path, ttl)
		if err != nil {
			ec.cordCtx.Log(metafora.LogLevelDebug, "Error trying to create directory. path:[%s] error:[ %v ]", path, err)
		}
		host, _ := os.Hostname()
		markerVal := fmt.Sprintf("createdAt:[%s] by:[%s] nodeid:[%s]", time.Now().String(), host, ec.NodeID) //debug info only
		ec.Client.Set(pathMarker, markerVal, ttl)
	}
}

// Watch will do a blocking etcd watch() on taskPath until a taskId is returned.
// The return taskId isn't guaranteed to be claimable.
//
func (ec *EtcdCoordinator) Watch() (taskID string, err error) {

	for {
		select {
		// before the watcher starts, this channel is filled up with existing tasks during init()
		case node, ok := <-ec.tasksPrefetchChan:
			if !ok {
				return "", nil
			}
			ec.cordCtx.Log(metafora.LogLevelDebug, "New node from the tasksPrefetchChan %s: node[nodeKey:%v]",
				ec.TaskPath, node.Key)

			taskId, skip := ec.parseTaskIDFromNode(node)
			if !skip {
				return taskId, nil
			}
		case resp, ok := <-ec.taskWatcher.responseChan:
			if !ok {
				return "", nil
			}
			ec.cordCtx.Log(metafora.LogLevelDebug, "New response from watching %s: response[action:%v etcdIndex:%v nodeKey:%v]",
				ec.TaskPath, resp.Action, resp.EtcdIndex, resp.Node.Key)

			if resp.Action != "create" { //TODO eventually we'll need to deal with expired "owner" nodes.
				continue
			}

			taskId, skip := ec.parseTaskIDFromNode(resp.Node)
			if !skip {
				return taskId, nil
			}

		case err, ok := <-ec.taskWatcher.errorChan:
			if !ok {
				return "", nil
			}
			//if the watcher sends a nil, its because the etcd watcher was shutdown by Close()
			return "", err
		}
	}
}

func (ec *EtcdCoordinator) parseTaskIDFromNode(node *etcd.Node) (taskID string, skip bool) {
	taskId := ""

	if node == nil {
		//TODO log
		return "", true
	}

	//FIXME There's gotta be a better way to only detect tasks #32
	//TODO per discussion last night, we are going to prefix tasks in the client,
	//     so we can easily determine tasks from other keys/dirs created in the
	//     tasks path.
	if strings.Contains(node.Key, OwnerMarker) {
		return "", true
	}

	ec.cordCtx.Log(metafora.LogLevelDebug, "Node key:%s", node.Key)
	taskpath := strings.Split(node.Key, "/")
	if len(taskpath) == 0 {
		//TODO log
		return "", true
	}
	if !node.Dir {
		return "", true
	}

	taskId = taskpath[len(taskpath)-1]

	if taskId == "tasks" {
		return "", true
	}

	return taskId, false
}

// Claim is called by the Consumer when a Balancer has determined that a task
// ID can be claimed. Claim returns false if another consumer has already
// claimed the ID.
func (ec *EtcdCoordinator) Claim(taskID string) bool {
	key := fmt.Sprintf("%s/%s/%s", ec.TaskPath, taskID, OwnerMarker)
	res, err := ec.Client.CreateDir(key, ClaimTTL)
	if err != nil {
		ec.cordCtx.Log(metafora.LogLevelDebug, "Claim failed: err %v", err)
		return false
	}
	ec.cordCtx.Log(metafora.LogLevelDebug, "Claim successful: resp %v", res)
	return true
}

// Release deletes the claim directory.
func (ec *EtcdCoordinator) Release(taskID string) {
	key := fmt.Sprintf("%s/%s/%s", ec.TaskPath, taskID, OwnerMarker)
	//FIXME Conditionally delete only if this node is actually the owner
	_, err := ec.Client.DeleteDir(key)
	if err != nil {
		//TODO Pause and retry?!
		ec.cordCtx.Log(metafora.LogLevelError, "Release failed: %v", err)
	}
}

// Command blocks until a command for this node is received from the broker
// by the coordinator.
func (ec *EtcdCoordinator) Command() (cmd string, err error) {
	<-ec.commandWatcher.responseChan
	return "", nil
	/*  TODO 1) cleanup the log here to match that of Watch
	         2) discuss the schema for the command channel...
	         3) Add code to the close method.
	select {
	case resp := <-ec.commandWatcherResponses:
		taskId := ""
		if resp.Node.Dir {
			taskId = resp.Node.Key
		} else {
			ec.cordCtx.Log(LogLevelWarning, "Command node needs to be a dir, not a key.")
			taskId = resp.Node.Key
		}
		return taskId, nil
	case err := <-ec.commandWatcherErrs:
		return "", err
	}
	*/
}

func (ec *EtcdCoordinator) Close() error {
	ec.taskWatcher.Close()
	ec.commandWatcher.Close()
	return nil
}
<|endoftext|>


<commit_before>package helm

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/event"
	"github.com/Aptomi/aptomi/pkg/lang"
	"github.com/Aptomi/aptomi/pkg/util"
	"github.com/pmezard/go-difflib/difflib"
	"gopkg.in/yaml.v2"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/helm/pkg/helm"
	"strings"
)

var helmCodeTypes = []string{"helm", "aptomi/code/kubernetes-helm"}

// GetSupportedCodeTypes returns all code types for which this plugin is registered to
func (plugin *Plugin) GetSupportedCodeTypes() []string {
	return helmCodeTypes
}

// Create implements creation of a new component instance in the cloud by deploying a Helm chart
func (plugin *Plugin) Create(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log) error {
	return plugin.createOrUpdate(cluster, deployName, params, eventLog, true)
}

// Update implements update of an existing component instance in the cloud by updating parameters of a helm chart
func (plugin *Plugin) Update(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log) error {
	return plugin.createOrUpdate(cluster, deployName, params, eventLog, true)
}

func (plugin *Plugin) createOrUpdate(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log, create bool) error {
	cache, err := plugin.getClusterCache(cluster, eventLog)
	if err != nil {
		return err
	}

	kubeClient, err := cache.newKubeClient()
	if err != nil {
		return err
	}

	err = cache.ensureKubeNamespace(kubeClient, cache.namespace)
	if err != nil {
		return err
	}

	releaseName := getHelmReleaseName(deployName)
	chartRepo, chartName, chartVersion, err := getHelmReleaseInfo(params)
	if err != nil {
		return err
	}

	helmClient, err := cache.newHelmClient(eventLog)
	if err != nil {
		return err
	}

	chartPath, err := plugin.fetchChart(chartRepo, chartName, chartVersion)
	if err != nil {
		return err
	}

	helmParams, err := yaml.Marshal(params)
	if err != nil {
		return err
	}

	currRelease, err := helmClient.ReleaseContent(releaseName)
	if err != nil && !strings.Contains(err.Error(), "not found") {
		return fmt.Errorf("error while looking for Helm release %s: %s", releaseName, err)
	}

	if create {
		if currRelease != nil {
			// If a release already exists, let's just go ahead and update it
			eventLog.WithFields(event.Fields{}).Infof("Release '%s' already exists. Updating it", releaseName)
		} else {
			eventLog.WithFields(event.Fields{
				"release": releaseName,
				"chart":   chartName,
				"path":    chartPath,
				"params":  string(helmParams),
			}).Infof("Installing Helm release '%s', chart '%s', cluster: '%s'", releaseName, chartName, cluster.Name)

			_, err = helmClient.InstallRelease(chartPath, cache.namespace, helm.ReleaseName(releaseName), helm.ValueOverrides(helmParams), helm.InstallReuseName(true))

			return err
		}
	}

	eventLog.WithFields(event.Fields{
		"release": releaseName,
		"chart":   chartName,
		"path":    chartPath,
		"params":  string(helmParams),
	}).Infof("Updating Helm release '%s', chart '%s', cluster: '%s'", releaseName, chartName, cluster.Name)

	newRelease, err := helmClient.UpdateRelease(releaseName, chartPath, helm.UpdateValueOverrides(helmParams))
	if err != nil {
		return err
	}

	diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
		A:        difflib.SplitLines(currRelease.Release.Manifest),
		B:        difflib.SplitLines(newRelease.Release.Manifest),
		FromFile: "Previous",
		ToFile:   "Current",
		Context:  3,
	})
	if err != nil {
		return fmt.Errorf("error while calculating diff between chart manifests for Helm release '%s', chart '%s', cluster: '%s'", releaseName, chartName, cluster.Name)
	}

	if len(diff) == 0 {
		diff = "without changes"
	} else {
		diff = "with diff: \n\n" + diff
	}

	eventLog.WithFields(event.Fields{
		"release": releaseName,
		"chart":   chartName,
		"path":    chartPath,
		"params":  string(helmParams),
	}).Debugf("Updated Helm release '%s', chart '%s', cluster: '%s' %s", releaseName, chartName, cluster.Name, diff)

	return err
}

// Destroy implements destruction of an existing component instance in the cloud by running "helm delete" on the corresponding helm chart
func (plugin *Plugin) Destroy(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log) error {
	cache, err := plugin.getClusterCache(cluster, eventLog)
	if err != nil {
		return err
	}

	releaseName := getHelmReleaseName(deployName)

	helmClient, err := cache.newHelmClient(eventLog)
	if err != nil {
		return err
	}

	eventLog.WithFields(event.Fields{
		"release": releaseName,
	}).Infof("Deleting Helm release '%s'", releaseName)

	_, err = helmClient.DeleteRelease(releaseName, helm.DeletePurge(true))
	return err
}

// Cleanup implements cleanup phase for the Helm plugin. It closes all created and cached Tiller tunnels.
func (plugin *Plugin) Cleanup() error {
	var err error
	plugin.cache.Range(func(key, value interface{}) bool {
		if c, ok := value.(*clusterCache); ok {
			c.tillerTunnel.Close()
		} else {
			panic(fmt.Sprintf("clusterCache expected in Plugin cache, but found: %v", c))
		}
		return true
	})
	return err
}

// Endpoints returns map from port type to url for all services of the current chart
// TODO: reduce cyclomatic complexity
func (plugin *Plugin) Endpoints(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log) (map[string]string, error) { // nolint: gocyclo
	cache, err := plugin.getClusterCache(cluster, eventLog)
	if err != nil {
		return nil, err
	}

	kubeClient, err := cache.newKubeClient()
	if err != nil {
		return nil, err
	}

	client := kubeClient.CoreV1()

	releaseName := getHelmReleaseName(deployName)

	selector := labels.Set{"release": releaseName}.AsSelector().String()
	options := meta.ListOptions{LabelSelector: selector}

	endpoints := make(map[string]string)

	// Check all corresponding services
	services, err := client.Services(cache.namespace).List(options)
	if err != nil {
		return nil, err
	}

	kubeHost, err := cache.getKubeExternalAddress()
	if err != nil {
		return nil, err
	}

	for _, service := range services.Items {
		if service.Spec.Type == "NodePort" {
			for _, port := range service.Spec.Ports {
				sURL := fmt.Sprintf("%s:%d", kubeHost, port.NodePort)

				// todo(slukjanov): could we somehow detect real schema? I think no :(
				if util.StringContainsAny(port.Name, "https") {
					sURL = "https://" + sURL
				} else if util.StringContainsAny(port.Name, "ui", "rest", "http", "grafana") {
					sURL = "http://" + sURL
				}

				endpoints[port.Name] = sURL
			}
		}
	}

	// Find Istio Ingress service (how ingress itself exposed)
	service, err := client.Services(cache.namespace).Get("istio-ingress", meta.GetOptions{})
	if err != nil {
		// return if there is no Istio deployed
		if k8serrors.IsNotFound(err) {
			return endpoints, nil
		}
		return nil, err
	}

	istioIngress := "<unresolved>"
	if service.Spec.Type == "NodePort" {
		for _, port := range service.Spec.Ports {
			if port.Name == "http" {
				istioIngress = fmt.Sprintf("%s:%d", kubeHost, port.NodePort)
			}
		}
	}

	// Check all corresponding istio ingresses
	ingresses, err := kubeClient.ExtensionsV1beta1().Ingresses(cache.namespace).List(options)
	if err != nil {
		return nil, err
	}

	// todo(slukjanov): support more then one ingress / rule / path
	for _, ingress := range ingresses.Items {
		if class, ok := ingress.Annotations["kubernetes.io/ingress.class"]; !ok || class != "istio" {
			continue
		}
		for _, rule := range ingress.Spec.Rules {
			for _, path := range rule.HTTP.Paths {
				pathStr := strings.Trim(path.Path, ".*")

				if rule.Host == "" {
					endpoints["ingress"] = "http://" + istioIngress + pathStr
				} else {
					endpoints["ingress"] = "http://" + rule.Host + pathStr
				}
			}
		}
	}

	return endpoints, nil
}
<commit_msg>Handle case when trying to update helm chart with another namespace<commit_after>package helm

import (
	"fmt"
	"github.com/Aptomi/aptomi/pkg/event"
	"github.com/Aptomi/aptomi/pkg/lang"
	"github.com/Aptomi/aptomi/pkg/util"
	"github.com/pmezard/go-difflib/difflib"
	"gopkg.in/yaml.v2"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/helm/pkg/helm"
	"strings"
)

var helmCodeTypes = []string{"helm", "aptomi/code/kubernetes-helm"}

// GetSupportedCodeTypes returns all code types for which this plugin is registered to
func (plugin *Plugin) GetSupportedCodeTypes() []string {
	return helmCodeTypes
}

// Create implements creation of a new component instance in the cloud by deploying a Helm chart
func (plugin *Plugin) Create(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log) error {
	return plugin.createOrUpdate(cluster, deployName, params, eventLog, true)
}

// Update implements update of an existing component instance in the cloud by updating parameters of a helm chart
func (plugin *Plugin) Update(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log) error {
	return plugin.createOrUpdate(cluster, deployName, params, eventLog, true)
}

func (plugin *Plugin) createOrUpdate(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log, create bool) error {
	cache, err := plugin.getClusterCache(cluster, eventLog)
	if err != nil {
		return err
	}

	kubeClient, err := cache.newKubeClient()
	if err != nil {
		return err
	}

	err = cache.ensureKubeNamespace(kubeClient, cache.namespace)
	if err != nil {
		return err
	}

	releaseName := getHelmReleaseName(deployName)
	chartRepo, chartName, chartVersion, err := getHelmReleaseInfo(params)
	if err != nil {
		return err
	}

	helmClient, err := cache.newHelmClient(eventLog)
	if err != nil {
		return err
	}

	chartPath, err := plugin.fetchChart(chartRepo, chartName, chartVersion)
	if err != nil {
		return err
	}

	helmParams, err := yaml.Marshal(params)
	if err != nil {
		return err
	}

	currRelease, err := helmClient.ReleaseContent(releaseName)
	if err != nil && !strings.Contains(err.Error(), "not found") {
		return fmt.Errorf("error while looking for Helm release %s: %s", releaseName, err)
	}

	if create {
		if currRelease != nil {
			// If a release already exists, let's just go ahead and update it
			eventLog.WithFields(event.Fields{}).Infof("Release '%s' already exists. Updating it", releaseName)
		} else {
			eventLog.WithFields(event.Fields{
				"release": releaseName,
				"chart":   chartName,
				"path":    chartPath,
				"params":  string(helmParams),
			}).Infof("Installing Helm release '%s', chart '%s', cluster: '%s'", releaseName, chartName, cluster.Name)

			_, err = helmClient.InstallRelease(chartPath, cache.namespace, helm.ReleaseName(releaseName), helm.ValueOverrides(helmParams), helm.InstallReuseName(true))

			return err
		}
	}

	eventLog.WithFields(event.Fields{
		"release": releaseName,
		"chart":   chartName,
		"path":    chartPath,
		"params":  string(helmParams),
	}).Infof("Updating Helm release '%s', chart '%s', cluster: '%s'", releaseName, chartName, cluster.Name)

	status, err := helmClient.ReleaseStatus(releaseName)
	if err != nil {
		return fmt.Errorf("error while getting status of current release %s: %s", releaseName, err)
	}
	if status.Namespace != cache.namespace {
		return fmt.Errorf("it's not allowed to change namespace of the release %s (was %s, requested %s)", releaseName, status.Namespace, cache.namespace)
	}

	newRelease, err := helmClient.UpdateRelease(releaseName, chartPath, helm.UpdateValueOverrides(helmParams))
	if err != nil {
		return err
	}

	diff, err := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
		A:        difflib.SplitLines(currRelease.Release.Manifest),
		B:        difflib.SplitLines(newRelease.Release.Manifest),
		FromFile: "Previous",
		ToFile:   "Current",
		Context:  3,
	})
	if err != nil {
		return fmt.Errorf("error while calculating diff between chart manifests for Helm release '%s', chart '%s', cluster: '%s'", releaseName, chartName, cluster.Name)
	}

	if len(diff) == 0 {
		diff = "without changes"
	} else {
		diff = "with diff: \n\n" + diff
	}

	eventLog.WithFields(event.Fields{
		"release": releaseName,
		"chart":   chartName,
		"path":    chartPath,
		"params":  string(helmParams),
	}).Debugf("Updated Helm release '%s', chart '%s', cluster: '%s' %s", releaseName, chartName, cluster.Name, diff)

	return err
}

// Destroy implements destruction of an existing component instance in the cloud by running "helm delete" on the corresponding helm chart
func (plugin *Plugin) Destroy(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log) error {
	cache, err := plugin.getClusterCache(cluster, eventLog)
	if err != nil {
		return err
	}

	releaseName := getHelmReleaseName(deployName)

	helmClient, err := cache.newHelmClient(eventLog)
	if err != nil {
		return err
	}

	eventLog.WithFields(event.Fields{
		"release": releaseName,
	}).Infof("Deleting Helm release '%s'", releaseName)

	_, err = helmClient.DeleteRelease(releaseName, helm.DeletePurge(true))
	return err
}

// Cleanup implements cleanup phase for the Helm plugin. It closes all created and cached Tiller tunnels.
func (plugin *Plugin) Cleanup() error {
	var err error
	plugin.cache.Range(func(key, value interface{}) bool {
		if c, ok := value.(*clusterCache); ok {
			c.tillerTunnel.Close()
		} else {
			panic(fmt.Sprintf("clusterCache expected in Plugin cache, but found: %v", c))
		}
		return true
	})
	return err
}

// Endpoints returns map from port type to url for all services of the current chart
// TODO: reduce cyclomatic complexity
func (plugin *Plugin) Endpoints(cluster *lang.Cluster, deployName string, params util.NestedParameterMap, eventLog *event.Log) (map[string]string, error) { // nolint: gocyclo
	cache, err := plugin.getClusterCache(cluster, eventLog)
	if err != nil {
		return nil, err
	}

	kubeClient, err := cache.newKubeClient()
	if err != nil {
		return nil, err
	}

	client := kubeClient.CoreV1()

	releaseName := getHelmReleaseName(deployName)

	selector := labels.Set{"release": releaseName}.AsSelector().String()
	options := meta.ListOptions{LabelSelector: selector}

	endpoints := make(map[string]string)

	// Check all corresponding services
	services, err := client.Services(cache.namespace).List(options)
	if err != nil {
		return nil, err
	}

	kubeHost, err := cache.getKubeExternalAddress()
	if err != nil {
		return nil, err
	}

	for _, service := range services.Items {
		if service.Spec.Type == "NodePort" {
			for _, port := range service.Spec.Ports {
				sURL := fmt.Sprintf("%s:%d", kubeHost, port.NodePort)

				// todo(slukjanov): could we somehow detect real schema? I think no :(
				if util.StringContainsAny(port.Name, "https") {
					sURL = "https://" + sURL
				} else if util.StringContainsAny(port.Name, "ui", "rest", "http", "grafana") {
					sURL = "http://" + sURL
				}

				endpoints[port.Name] = sURL
			}
		}
	}

	// Find Istio Ingress service (how ingress itself exposed)
	service, err := client.Services(cache.namespace).Get("istio-ingress", meta.GetOptions{})
	if err != nil {
		// return if there is no Istio deployed
		if k8serrors.IsNotFound(err) {
			return endpoints, nil
		}
		return nil, err
	}

	istioIngress := "<unresolved>"
	if service.Spec.Type == "NodePort" {
		for _, port := range service.Spec.Ports {
			if port.Name == "http" {
				istioIngress = fmt.Sprintf("%s:%d", kubeHost, port.NodePort)
			}
		}
	}

	// Check all corresponding istio ingresses
	ingresses, err := kubeClient.ExtensionsV1beta1().Ingresses(cache.namespace).List(options)
	if err != nil {
		return nil, err
	}

	// todo(slukjanov): support more then one ingress / rule / path
	for _, ingress := range ingresses.Items {
		if class, ok := ingress.Annotations["kubernetes.io/ingress.class"]; !ok || class != "istio" {
			continue
		}
		for _, rule := range ingress.Spec.Rules {
			for _, path := range rule.HTTP.Paths {
				pathStr := strings.Trim(path.Path, ".*")

				if rule.Host == "" {
					endpoints["ingress"] = "http://" + istioIngress + pathStr
				} else {
					endpoints["ingress"] = "http://" + rule.Host + pathStr
				}
			}
		}
	}

	return endpoints, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"strings"

	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/tools/cache"

	"github.com/ghodss/yaml"
	"github.com/go-kit/kit/log/level"
	"github.com/openshift/prom-label-proxy/injectproxy"
	"github.com/pkg/errors"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/promql"
)

const labelPrometheusName = "prometheus-name"

// The maximum `Data` size of a ConfigMap seems to differ between
// environments. This is probably due to different meta data sizes which count
// into the overall maximum size of a ConfigMap. Thereby lets leave a
// large buffer.
var maxConfigMapDataSize = int(float64(v1.MaxSecretSize) * 0.5)

func (c *Operator) createOrUpdateRuleConfigMaps(p *monitoringv1.Prometheus) ([]string, error) {
	cClient := c.kclient.CoreV1().ConfigMaps(p.Namespace)

	namespaces, err := c.selectRuleNamespaces(p)
	if err != nil {
		return nil, err
	}

	newRules, err := c.selectRules(p, namespaces)
	if err != nil {
		return nil, err
	}

	currentConfigMapList, err := cClient.List(prometheusRulesConfigMapSelector(p.Name))
	if err != nil {
		return nil, err
	}
	currentConfigMaps := currentConfigMapList.Items

	currentRules := map[string]string{}
	for _, cm := range currentConfigMaps {
		for ruleFileName, ruleFile := range cm.Data {
			currentRules[ruleFileName] = ruleFile
		}
	}

	equal := reflect.DeepEqual(newRules, currentRules)
	if equal && len(currentConfigMaps) != 0 {
		level.Debug(c.logger).Log(
			"msg", "no PrometheusRule changes",
			"namespace", p.Namespace,
			"prometheus", p.Name,
		)
		currentConfigMapNames := []string{}
		for _, cm := range currentConfigMaps {
			currentConfigMapNames = append(currentConfigMapNames, cm.Name)
		}
		return currentConfigMapNames, nil
	}

	newConfigMaps, err := makeRulesConfigMaps(p, newRules)
	if err != nil {
		return nil, errors.Wrap(err, "failed to make rules ConfigMaps")
	}

	newConfigMapNames := []string{}
	for _, cm := range newConfigMaps {
		newConfigMapNames = append(newConfigMapNames, cm.Name)
	}

	if len(currentConfigMaps) == 0 {
		level.Debug(c.logger).Log(
			"msg", "no PrometheusRule configmap found, creating new one",
			"namespace", p.Namespace,
			"prometheus", p.Name,
		)
		for _, cm := range newConfigMaps {
			_, err = cClient.Create(&cm)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to create ConfigMap '%v'", cm.Name)
			}
		}
		return newConfigMapNames, nil
	}

	// Simply deleting old ConfigMaps and creating new ones for now. Could be
	// replaced by logic that only deletes obsolete ConfigMaps in the future.
	for _, cm := range currentConfigMaps {
		err := cClient.Delete(cm.Name, &metav1.DeleteOptions{})
		if err != nil {
			return nil, errors.Wrapf(err, "failed to delete current ConfigMap '%v'", cm.Name)
		}
	}

	level.Debug(c.logger).Log(
		"msg", "updating PrometheusRule",
		"namespace", p.Namespace,
		"prometheus", p.Name,
	)
	for _, cm := range newConfigMaps {
		_, err = cClient.Create(&cm)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to create new ConfigMap '%v'", cm.Name)
		}
	}

	return newConfigMapNames, nil
}

func prometheusRulesConfigMapSelector(prometheusName string) metav1.ListOptions {
	return metav1.ListOptions{LabelSelector: fmt.Sprintf("%v=%v", labelPrometheusName, prometheusName)}
}

func (c *Operator) selectRuleNamespaces(p *monitoringv1.Prometheus) ([]string, error) {
	namespaces := []string{}

	// If 'RuleNamespaceSelector' is nil, only check own namespace.
	if p.Spec.RuleNamespaceSelector == nil {
		namespaces = append(namespaces, p.Namespace)
	} else {
		ruleNamespaceSelector, err := metav1.LabelSelectorAsSelector(p.Spec.RuleNamespaceSelector)
		if err != nil {
			return namespaces, errors.Wrap(err, "convert rule namespace label selector to selector")
		}

		namespaces, err = c.listMatchingNamespaces(ruleNamespaceSelector)
		if err != nil {
			return nil, err
		}
	}

	level.Debug(c.logger).Log(
		"msg", "selected RuleNamespaces",
		"namespaces", strings.Join(namespaces, ","),
		"namespace", p.Namespace,
		"prometheus", p.Name,
	)

	return namespaces, nil
}

func (c *Operator) selectRules(p *monitoringv1.Prometheus, namespaces []string) (map[string]string, error) {
	rules := map[string]string{}

	ruleSelector, err := metav1.LabelSelectorAsSelector(p.Spec.RuleSelector)
	if err != nil {
		return rules, errors.Wrap(err, "convert rule label selector to selector")
	}

	for _, ns := range namespaces {
		var marshalErr error
		err := cache.ListAllByNamespace(c.ruleInf.GetIndexer(), ns, ruleSelector, func(obj interface{}) {
			promRule := obj.(*monitoringv1.PrometheusRule)
			content, err := generateContent(promRule.Spec, p.Spec.EnforcedNamespaceLabel, promRule.Namespace)
			if err != nil {
				marshalErr = err
				return
			}
			rules[fmt.Sprintf("%v-%v.yaml", promRule.Namespace, promRule.Name)] = content
		})
		if err != nil {
			return nil, err
		}
		if marshalErr != nil {
			return nil, marshalErr
		}
	}

	ruleNames := []string{}
	for name := range rules {
		ruleNames = append(ruleNames, name)
	}

	level.Debug(c.logger).Log(
		"msg", "selected Rules",
		"rules", strings.Join(ruleNames, ","),
		"namespace", p.Namespace,
		"prometheus", p.Name,
	)

	return rules, nil
}

func generateContent(promRule monitoringv1.PrometheusRuleSpec, enforcedNsLabel, ns string) (string, error) {
	if enforcedNsLabel != "" {
		for gi, group := range promRule.Groups {
			group.PartialResponseStrategy = ""
			for ri, r := range group.Rules {
				if len(promRule.Groups[gi].Rules[ri].Labels) == 0 {
					promRule.Groups[gi].Rules[ri].Labels = map[string]string{}
				}
				promRule.Groups[gi].Rules[ri].Labels[enforcedNsLabel] = ns

				expr := r.Expr.String()
				parsedExpr, err := promql.ParseExpr(expr)
				if err != nil {
					return "", errors.Wrap(err, "failed to parse promql expression")
				}
				err = injectproxy.SetRecursive(parsedExpr, []*labels.Matcher{{
					Name:  enforcedNsLabel,
					Type:  labels.MatchEqual,
					Value: ns,
				}})
				if err != nil {
					return "", errors.Wrap(err, "failed to inject labels to expression")
				}

				promRule.Groups[gi].Rules[ri].Expr = intstr.FromString(parsedExpr.String())
			}
		}
	}
	content, err := yaml.Marshal(promRule)
	if err != nil {
		return "", errors.Wrap(err, "failed to unmarshal content")
	}
	return string(content), nil
}

// makeRulesConfigMaps takes a Prometheus configuration and rule files and
// returns a list of Kubernetes ConfigMaps to be later on mounted into the
// Prometheus instance.
// If the total size of rule files exceeds the Kubernetes ConfigMap limit,
// they are split up via the simple first-fit [1] bin packing algorithm. In the
// future this can be replaced by a more sophisticated algorithm, but for now
// simplicity should be sufficient.
// [1] https://en.wikipedia.org/wiki/Bin_packing_problem#First-fit_algorithm
func makeRulesConfigMaps(p *monitoringv1.Prometheus, ruleFiles map[string]string) ([]v1.ConfigMap, error) {
	//check if none of the rule files is too large for a single ConfigMap
	for filename, file := range ruleFiles {
		if len(file) > maxConfigMapDataSize {
			return nil, errors.Errorf(
				"rule file '%v' is too large for a single Kubernetes ConfigMap",
				filename,
			)
		}
	}

	buckets := []map[string]string{
		{},
	}
	currBucketIndex := 0

	// To make bin packing algorithm deterministic, sort ruleFiles filenames and
	// iterate over filenames instead of ruleFiles map (not deterministic).
	fileNames := []string{}
	for n := range ruleFiles {
		fileNames = append(fileNames, n)
	}
	sort.Strings(fileNames)

	for _, filename := range fileNames {
		// If rule file doesn't fit into current bucket, create new bucket.
		if bucketSize(buckets[currBucketIndex])+len(ruleFiles[filename]) > maxConfigMapDataSize {
			buckets = append(buckets, map[string]string{})
			currBucketIndex++
		}
		buckets[currBucketIndex][filename] = ruleFiles[filename]
	}

	ruleFileConfigMaps := []v1.ConfigMap{}
	for i, bucket := range buckets {
		cm := makeRulesConfigMap(p, bucket)
		cm.Name = cm.Name + "-" + strconv.Itoa(i)
		ruleFileConfigMaps = append(ruleFileConfigMaps, cm)
	}

	return ruleFileConfigMaps, nil
}

func bucketSize(bucket map[string]string) int {
	totalSize := 0
	for _, v := range bucket {
		totalSize += len(v)
	}

	return totalSize
}

func makeRulesConfigMap(p *monitoringv1.Prometheus, ruleFiles map[string]string) v1.ConfigMap {
	boolTrue := true

	labels := map[string]string{labelPrometheusName: p.Name}
	for k, v := range managedByOperatorLabels {
		labels[k] = v
	}

	return v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:   prometheusRuleConfigMapName(p.Name),
			Labels: labels,
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion:         p.APIVersion,
					BlockOwnerDeletion: &boolTrue,
					Controller:         &boolTrue,
					Kind:               p.Kind,
					Name:               p.Name,
					UID:                p.UID,
				},
			},
		},
		Data: ruleFiles,
	}
}

func prometheusRuleConfigMapName(prometheusName string) string {
	return "prometheus-" + prometheusName + "-rulefiles"
}
<commit_msg>pkg/prometheus/rules.go: Do not modify the PrometheusRule cache<commit_after>// Copyright 2016 The prometheus-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"strings"

	monitoringv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/tools/cache"

	"github.com/ghodss/yaml"
	"github.com/go-kit/kit/log/level"
	"github.com/openshift/prom-label-proxy/injectproxy"
	"github.com/pkg/errors"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/promql"
)

const labelPrometheusName = "prometheus-name"

// The maximum `Data` size of a ConfigMap seems to differ between
// environments. This is probably due to different meta data sizes which count
// into the overall maximum size of a ConfigMap. Thereby lets leave a
// large buffer.
var maxConfigMapDataSize = int(float64(v1.MaxSecretSize) * 0.5)

func (c *Operator) createOrUpdateRuleConfigMaps(p *monitoringv1.Prometheus) ([]string, error) {
	cClient := c.kclient.CoreV1().ConfigMaps(p.Namespace)

	namespaces, err := c.selectRuleNamespaces(p)
	if err != nil {
		return nil, err
	}

	newRules, err := c.selectRules(p, namespaces)
	if err != nil {
		return nil, err
	}

	currentConfigMapList, err := cClient.List(prometheusRulesConfigMapSelector(p.Name))
	if err != nil {
		return nil, err
	}
	currentConfigMaps := currentConfigMapList.Items

	currentRules := map[string]string{}
	for _, cm := range currentConfigMaps {
		for ruleFileName, ruleFile := range cm.Data {
			currentRules[ruleFileName] = ruleFile
		}
	}

	equal := reflect.DeepEqual(newRules, currentRules)
	if equal && len(currentConfigMaps) != 0 {
		level.Debug(c.logger).Log(
			"msg", "no PrometheusRule changes",
			"namespace", p.Namespace,
			"prometheus", p.Name,
		)
		currentConfigMapNames := []string{}
		for _, cm := range currentConfigMaps {
			currentConfigMapNames = append(currentConfigMapNames, cm.Name)
		}
		return currentConfigMapNames, nil
	}

	newConfigMaps, err := makeRulesConfigMaps(p, newRules)
	if err != nil {
		return nil, errors.Wrap(err, "failed to make rules ConfigMaps")
	}

	newConfigMapNames := []string{}
	for _, cm := range newConfigMaps {
		newConfigMapNames = append(newConfigMapNames, cm.Name)
	}

	if len(currentConfigMaps) == 0 {
		level.Debug(c.logger).Log(
			"msg", "no PrometheusRule configmap found, creating new one",
			"namespace", p.Namespace,
			"prometheus", p.Name,
		)
		for _, cm := range newConfigMaps {
			_, err = cClient.Create(&cm)
			if err != nil {
				return nil, errors.Wrapf(err, "failed to create ConfigMap '%v'", cm.Name)
			}
		}
		return newConfigMapNames, nil
	}

	// Simply deleting old ConfigMaps and creating new ones for now. Could be
	// replaced by logic that only deletes obsolete ConfigMaps in the future.
	for _, cm := range currentConfigMaps {
		err := cClient.Delete(cm.Name, &metav1.DeleteOptions{})
		if err != nil {
			return nil, errors.Wrapf(err, "failed to delete current ConfigMap '%v'", cm.Name)
		}
	}

	level.Debug(c.logger).Log(
		"msg", "updating PrometheusRule",
		"namespace", p.Namespace,
		"prometheus", p.Name,
	)
	for _, cm := range newConfigMaps {
		_, err = cClient.Create(&cm)
		if err != nil {
			return nil, errors.Wrapf(err, "failed to create new ConfigMap '%v'", cm.Name)
		}
	}

	return newConfigMapNames, nil
}

func prometheusRulesConfigMapSelector(prometheusName string) metav1.ListOptions {
	return metav1.ListOptions{LabelSelector: fmt.Sprintf("%v=%v", labelPrometheusName, prometheusName)}
}

func (c *Operator) selectRuleNamespaces(p *monitoringv1.Prometheus) ([]string, error) {
	namespaces := []string{}

	// If 'RuleNamespaceSelector' is nil, only check own namespace.
	if p.Spec.RuleNamespaceSelector == nil {
		namespaces = append(namespaces, p.Namespace)
	} else {
		ruleNamespaceSelector, err := metav1.LabelSelectorAsSelector(p.Spec.RuleNamespaceSelector)
		if err != nil {
			return namespaces, errors.Wrap(err, "convert rule namespace label selector to selector")
		}

		namespaces, err = c.listMatchingNamespaces(ruleNamespaceSelector)
		if err != nil {
			return nil, err
		}
	}

	level.Debug(c.logger).Log(
		"msg", "selected RuleNamespaces",
		"namespaces", strings.Join(namespaces, ","),
		"namespace", p.Namespace,
		"prometheus", p.Name,
	)

	return namespaces, nil
}

func (c *Operator) selectRules(p *monitoringv1.Prometheus, namespaces []string) (map[string]string, error) {
	rules := map[string]string{}

	ruleSelector, err := metav1.LabelSelectorAsSelector(p.Spec.RuleSelector)
	if err != nil {
		return rules, errors.Wrap(err, "convert rule label selector to selector")
	}

	for _, ns := range namespaces {
		var marshalErr error
		err := cache.ListAllByNamespace(c.ruleInf.GetIndexer(), ns, ruleSelector, func(obj interface{}) {
			promRule := obj.(*monitoringv1.PrometheusRule).DeepCopy()
			content, err := generateContent(promRule.Spec, p.Spec.EnforcedNamespaceLabel, promRule.Namespace)
			if err != nil {
				marshalErr = err
				return
			}
			rules[fmt.Sprintf("%v-%v.yaml", promRule.Namespace, promRule.Name)] = content
		})
		if err != nil {
			return nil, err
		}
		if marshalErr != nil {
			return nil, marshalErr
		}
	}

	ruleNames := []string{}
	for name := range rules {
		ruleNames = append(ruleNames, name)
	}

	level.Debug(c.logger).Log(
		"msg", "selected Rules",
		"rules", strings.Join(ruleNames, ","),
		"namespace", p.Namespace,
		"prometheus", p.Name,
	)

	return rules, nil
}

func generateContent(promRule monitoringv1.PrometheusRuleSpec, enforcedNsLabel, ns string) (string, error) {
	if enforcedNsLabel != "" {
		for gi, group := range promRule.Groups {
			group.PartialResponseStrategy = ""
			for ri, r := range group.Rules {
				if len(promRule.Groups[gi].Rules[ri].Labels) == 0 {
					promRule.Groups[gi].Rules[ri].Labels = map[string]string{}
				}
				promRule.Groups[gi].Rules[ri].Labels[enforcedNsLabel] = ns

				expr := r.Expr.String()
				parsedExpr, err := promql.ParseExpr(expr)
				if err != nil {
					return "", errors.Wrap(err, "failed to parse promql expression")
				}
				err = injectproxy.SetRecursive(parsedExpr, []*labels.Matcher{{
					Name:  enforcedNsLabel,
					Type:  labels.MatchEqual,
					Value: ns,
				}})
				if err != nil {
					return "", errors.Wrap(err, "failed to inject labels to expression")
				}

				promRule.Groups[gi].Rules[ri].Expr = intstr.FromString(parsedExpr.String())
			}
		}
	}
	content, err := yaml.Marshal(promRule)
	if err != nil {
		return "", errors.Wrap(err, "failed to unmarshal content")
	}
	return string(content), nil
}

// makeRulesConfigMaps takes a Prometheus configuration and rule files and
// returns a list of Kubernetes ConfigMaps to be later on mounted into the
// Prometheus instance.
// If the total size of rule files exceeds the Kubernetes ConfigMap limit,
// they are split up via the simple first-fit [1] bin packing algorithm. In the
// future this can be replaced by a more sophisticated algorithm, but for now
// simplicity should be sufficient.
// [1] https://en.wikipedia.org/wiki/Bin_packing_problem#First-fit_algorithm
func makeRulesConfigMaps(p *monitoringv1.Prometheus, ruleFiles map[string]string) ([]v1.ConfigMap, error) {
	//check if none of the rule files is too large for a single ConfigMap
	for filename, file := range ruleFiles {
		if len(file) > maxConfigMapDataSize {
			return nil, errors.Errorf(
				"rule file '%v' is too large for a single Kubernetes ConfigMap",
				filename,
			)
		}
	}

	buckets := []map[string]string{
		{},
	}
	currBucketIndex := 0

	// To make bin packing algorithm deterministic, sort ruleFiles filenames and
	// iterate over filenames instead of ruleFiles map (not deterministic).
	fileNames := []string{}
	for n := range ruleFiles {
		fileNames = append(fileNames, n)
	}
	sort.Strings(fileNames)

	for _, filename := range fileNames {
		// If rule file doesn't fit into current bucket, create new bucket.
		if bucketSize(buckets[currBucketIndex])+len(ruleFiles[filename]) > maxConfigMapDataSize {
			buckets = append(buckets, map[string]string{})
			currBucketIndex++
		}
		buckets[currBucketIndex][filename] = ruleFiles[filename]
	}

	ruleFileConfigMaps := []v1.ConfigMap{}
	for i, bucket := range buckets {
		cm := makeRulesConfigMap(p, bucket)
		cm.Name = cm.Name + "-" + strconv.Itoa(i)
		ruleFileConfigMaps = append(ruleFileConfigMaps, cm)
	}

	return ruleFileConfigMaps, nil
}

func bucketSize(bucket map[string]string) int {
	totalSize := 0
	for _, v := range bucket {
		totalSize += len(v)
	}

	return totalSize
}

func makeRulesConfigMap(p *monitoringv1.Prometheus, ruleFiles map[string]string) v1.ConfigMap {
	boolTrue := true

	labels := map[string]string{labelPrometheusName: p.Name}
	for k, v := range managedByOperatorLabels {
		labels[k] = v
	}

	return v1.ConfigMap{
		ObjectMeta: metav1.ObjectMeta{
			Name:   prometheusRuleConfigMapName(p.Name),
			Labels: labels,
			OwnerReferences: []metav1.OwnerReference{
				{
					APIVersion:         p.APIVersion,
					BlockOwnerDeletion: &boolTrue,
					Controller:         &boolTrue,
					Kind:               p.Kind,
					Name:               p.Name,
					UID:                p.UID,
				},
			},
		},
		Data: ruleFiles,
	}
}

func prometheusRuleConfigMapName(prometheusName string) string {
	return "prometheus-" + prometheusName + "-rulefiles"
}
<|endoftext|>