<commit_before>package sumoCFFirehose

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/eventQueue"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/events"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/logging"
)

type SumoLogicAppender struct {
	url                         string
	connectionTimeout           int //10000
	httpClient                  http.Client
	nozzleQueue                 *eventQueue.Queue
	eventsBatchSize             int
	sumoPostMinimumDelay        time.Duration
	timerBetweenPost            time.Time
	sumoCategory                string
	sumoName                    string
	sumoHost                    string
	verboseLogMessages          bool
	customMetadata              string
	includeOnlyMatchingFilter   string
	excludeAlwaysMatchingFilter string
	nozzleVersion               string
}

type SumoBuffer struct {
	logStringToSend          *bytes.Buffer
	logEventsInCurrentBuffer int
	timerIdlebuffer          time.Time
}

func NewSumoLogicAppender(urlValue string, connectionTimeoutValue int, nozzleQueue *eventQueue.Queue, eventsBatchSize int, sumoPostMinimumDelay time.Duration, sumoCategory string, sumoName string, sumoHost string, verboseLogMessages bool, customMetadata string, includeOnlyMatchingFilter string, excludeAlwaysMatchingFilter string, nozzleVersion string) *SumoLogicAppender {
	return &SumoLogicAppender{
		url:                         urlValue,
		connectionTimeout:           connectionTimeoutValue,
		httpClient:                  http.Client{Timeout: time.Duration(connectionTimeoutValue * int(time.Millisecond))},
		nozzleQueue:                 nozzleQueue,
		eventsBatchSize:             eventsBatchSize,
		sumoPostMinimumDelay:        sumoPostMinimumDelay,
		sumoCategory:                sumoCategory,
		sumoName:                    sumoName,
		sumoHost:                    sumoHost,
		verboseLogMessages:          verboseLogMessages,
		customMetadata:              customMetadata,
		includeOnlyMatchingFilter:   includeOnlyMatchingFilter,
		excludeAlwaysMatchingFilter: excludeAlwaysMatchingFilter,
		nozzleVersion:               nozzleVersion,
	}
}

func newBuffer() SumoBuffer {
	return SumoBuffer{
		logStringToSend:          bytes.NewBufferString(""),
		logEventsInCurrentBuffer: 0,
	}
}

func (s *SumoLogicAppender) Start() {
	s.timerBetweenPost = time.Now()
	Buffer := newBuffer()
	Buffer.timerIdlebuffer = time.Now()
	logging.Info.Println("Starting Appender Worker")
	for {
		logging.Info.Printf("Log queue size: %d", s.nozzleQueue.GetCount())
		if s.nozzleQueue.GetCount() == 0 {
			logging.Trace.Println("Waiting for 300 ms")
			time.Sleep(300 * time.Millisecond)
		}

		if time.Since(Buffer.timerIdlebuffer).Seconds() >= 10 && Buffer.logEventsInCurrentBuffer > 0 {
			logging.Info.Println("Sending current batch of logs after timer exceeded limit")
			go s.SendToSumo(Buffer.logStringToSend.String())
			Buffer = newBuffer()
			Buffer.timerIdlebuffer = time.Now()
			continue
		}

		if s.nozzleQueue.GetCount() != 0 {
			queueCount := s.nozzleQueue.GetCount()
			remainingBufferCount := s.eventsBatchSize - Buffer.logEventsInCurrentBuffer
			if queueCount >= remainingBufferCount {
				logging.Trace.Println("Pushing Logs to Sumo: ")
				logging.Trace.Println(remainingBufferCount)
				for i := 0; i < remainingBufferCount; i++ {
					s.AppendLogs(&Buffer)
					Buffer.timerIdlebuffer = time.Now()
				}

				go s.SendToSumo(Buffer.logStringToSend.String())
				Buffer = newBuffer()
			} else {
				logging.Trace.Println("Pushing Logs to Buffer: ")
				logging.Trace.Println(queueCount)
				for i := 0; i < queueCount; i++ {
					s.AppendLogs(&Buffer)
					Buffer.timerIdlebuffer = time.Now()
				}
			}
		}

	}

}
func WantedEvent(event string, includeOnlyMatchingFilter string, excludeAlwaysMatchingFilter string) bool {
	if includeOnlyMatchingFilter != "" && excludeAlwaysMatchingFilter != "" {
		subsliceInclude := ParseCustomInput(includeOnlyMatchingFilter)
		subsliceExclude := ParseCustomInput(excludeAlwaysMatchingFilter)
		for key, value := range subsliceInclude {
			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
				for key, value := range subsliceExclude {
					if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
						return false
					}
				}
				return true
			}
		}
		for key, value := range subsliceExclude {
			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
				return false
			}
		}
		return true
	} else if includeOnlyMatchingFilter != "" {
		subslice := ParseCustomInput(includeOnlyMatchingFilter)
		for key, value := range subslice {
			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
				return true
			}
		}
		return false

	} else if excludeAlwaysMatchingFilter != "" {
		subslice := ParseCustomInput(excludeAlwaysMatchingFilter)
		for key, value := range subslice {
			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
				return false
			}
		}
		return true
	}
	return true

}
func FormatTimestamp(event *events.Event, timestamp string) {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered in f", r)
		}
	}()

	if reflect.TypeOf(event.Fields[timestamp]).Kind() != reflect.Int64 {
		if reflect.TypeOf(event.Fields[timestamp]).Kind() == reflect.String {
			event.Fields[timestamp] = event.Fields[timestamp].(string)
		} else {
			event.Fields[timestamp] = ""
		}

	}
	if reflect.TypeOf(event.Fields[timestamp]).Kind() == reflect.Int64 {
		if len(strconv.FormatInt(event.Fields[timestamp].(int64), 10)) == 19 {
			event.Fields[timestamp] = time.Unix(0, event.Fields[timestamp].(int64)*int64(time.Nanosecond)).String()
		} else if len(strconv.FormatInt(event.Fields[timestamp].(int64), 10)) < 13 {
			event.Fields[timestamp] = ""
		}
	}

}

func StringBuilder(event *events.Event, verboseLogMessages bool, includeOnlyMatchingFilter string, excludeAlwaysMatchingFilter string, customMetadata string) string {
	if customMetadata != "" {
		customMetadataMap := ParseCustomInput(customMetadata)
		for key, value := range customMetadataMap {
			event.Fields[key] = value
		}
	}
	eventType := event.Type
	var msg []byte
	switch eventType {
	case "HttpStart":
		FormatTimestamp(event, "timestamp")
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "HttpStop":
		FormatTimestamp(event, "timestamp")
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "HttpStartStop":
		FormatTimestamp(event, "start_timestamp")
		FormatTimestamp(event, "stop_timestamp")
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "LogMessage":
		FormatTimestamp(event, "timestamp")
		if verboseLogMessages == true {
			message, err := json.Marshal(event)
			if err == nil {
				msg = message
			}
		} else {
			eventNoVerbose := events.Event{
				Fields: map[string]interface{}{
					"timestamp":   event.Fields["timestamp"],
					"cf_app_guid": event.Fields["cf_app_id"],
				},
				Msg:  event.Msg,
				Type: event.Type,
			}
			if customMetadata != "" {
				customMetadataMap := ParseCustomInput(customMetadata)
				for key, value := range customMetadataMap {
					eventNoVerbose.Fields[key] = value
				}
			}
			message, err := json.Marshal(eventNoVerbose)
			if err == nil {
				msg = message
			}
		}
	case "ValueMetric":
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "CounterEvent":
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "Error":
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "ContainerMetric":
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	}

	buf := new(bytes.Buffer)
	buf.Write(msg)
	if WantedEvent(buf.String(), includeOnlyMatchingFilter, excludeAlwaysMatchingFilter) {
		return buf.String() + "\n"
	} else {
		return ""
	}

}

func (s *SumoLogicAppender) AppendLogs(buffer *SumoBuffer) {
	buffer.logStringToSend.Write([]byte(StringBuilder(s.nozzleQueue.Pop(), s.verboseLogMessages, s.includeOnlyMatchingFilter, s.excludeAlwaysMatchingFilter, s.customMetadata)))
	buffer.logEventsInCurrentBuffer++

}
func ParseCustomInput(customInput string) map[string]string {
	cInputArray := strings.Split(customInput, ",")
	customInputMap := make(map[string]string)
	for i := 0; i < len(cInputArray); i++ {
		customInputMap[strings.Split(cInputArray[i], ":")[0]] = strings.Split(cInputArray[i], ":")[1]
	}
	return customInputMap
}

func (s *SumoLogicAppender) SendToSumo(logStringToSend string) {
	if logStringToSend != "" {
		var buf bytes.Buffer
		g := gzip.NewWriter(&buf)
		g.Write([]byte(logStringToSend))
		g.Close()
		request, err := http.NewRequest("POST", s.url, &buf)
		if err != nil {
			logging.Error.Printf("http.NewRequest() error: %v\n", err)
			return
		}
		request.Header.Add("Content-Encoding", "gzip")
		request.Header.Add("X-Sumo-Client", "cloudfoundry-sumologic-nozzle v"+s.nozzleVersion)

		if s.sumoName != "" {
			request.Header.Add("X-Sumo-Name", s.sumoName)
		}
		if s.sumoHost != "" {
			request.Header.Add("X-Sumo-Host", s.sumoHost)
		}
		if s.sumoCategory != "" {
			request.Header.Add("X-Sumo-Category", s.sumoCategory)
		}
		//checking the timer before first POST intent
		for time.Since(s.timerBetweenPost) < s.sumoPostMinimumDelay {
			logging.Trace.Println("Delaying Post because minimum post timer not expired")
			time.Sleep(100 * time.Millisecond)
		}
		response, err := s.httpClient.Do(request)

		if (err != nil) || (response.StatusCode != 200 && response.StatusCode != 302 && response.StatusCode < 500) {
			logging.Info.Println("Endpoint dropped the post send")
			logging.Info.Println("Waiting for 300 ms to retry")
			time.Sleep(300 * time.Millisecond)
			statusCode := 0
			err := Retry(func(attempt int) (bool, error) {
				var errRetry error
				request, err := http.NewRequest("POST", s.url, &buf)
				if err != nil {
					logging.Error.Printf("http.NewRequest() error: %v\n", err)
				}
				request.Header.Add("Content-Encoding", "gzip")
				request.Header.Add("X-Sumo-Client", "cloudfoundry-sumologic-nozzle v"+s.nozzleVersion)

				if s.sumoName != "" {
					request.Header.Add("X-Sumo-Name", s.sumoName)
				}
				if s.sumoHost != "" {
					request.Header.Add("X-Sumo-Host", s.sumoHost)
				}
				if s.sumoCategory != "" {
					request.Header.Add("X-Sumo-Category", s.sumoCategory)
				}
				//checking the timer before POST (retry intent)
				for time.Since(s.timerBetweenPost) < s.sumoPostMinimumDelay {
					logging.Trace.Println("Delaying Post because minimum post timer not expired")
					time.Sleep(100 * time.Millisecond)
				}
				response, errRetry = s.httpClient.Do(request)
				if errRetry != nil {
					logging.Error.Printf("http.Do() error: %v\n", errRetry)
					logging.Info.Println("Waiting for 300 ms to retry after error")
					time.Sleep(300 * time.Millisecond)
					return attempt < 5, errRetry
				} else if response.StatusCode != 200 && response.StatusCode != 302 && response.StatusCode < 500 {
					logging.Info.Println("Endpoint dropped the post send again")
					logging.Info.Println("Waiting for 300 ms to retry after a retry ...")
					statusCode = response.StatusCode
					time.Sleep(300 * time.Millisecond)
					return attempt < 5, errRetry
				} else if response.StatusCode == 200 {
					logging.Trace.Println("Post of logs successful after retry...")
					s.timerBetweenPost = time.Now()
					statusCode = response.StatusCode
					return true, err
				}
				return attempt < 5, errRetry
			})
			if err != nil {
				logging.Error.Println("Error, Not able to post after retry")
				logging.Error.Printf("http.Do() error: %v\n", err)
				return
			} else if statusCode != 200 {
				logging.Error.Printf("Not able to post after retry, with status code: %d", statusCode)
			}
		} else if response.StatusCode == 200 {
			logging.Trace.Println("Post of logs successful")
			s.timerBetweenPost = time.Now()
		}

		if response != nil {
			defer response.Body.Close()
		}
	}

}

//------------------Retry Logic Code-------------------------------

// MaxRetries is the maximum number of retries before bailing.
var MaxRetries = 10
var errMaxRetriesReached = errors.New("exceeded retry limit")

// Func represents functions that can be retried.
type Func func(attempt int) (retry bool, err error)

// Do keeps trying the function until the second argument
// returns false, or no error is returned.
func Retry(fn Func) error {
	var err error
	var cont bool
	attempt := 1
	for {
		cont, err = fn(attempt)
		if !cont || err == nil {
			break
		}
		attempt++
		if attempt > MaxRetries {
			return errMaxRetriesReached
		}
	}
	return err
}

// IsMaxRetries checks whether the error is due to hitting the
// maximum number of retries or not.
func IsMaxRetries(err error) bool {
	return err == errMaxRetriesReached
}
<commit_msg>commented out line 76, <commit_after>package sumoCFFirehose

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/eventQueue"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/events"
	"github.com/SumoLogic/sumologic-cloudfoundry-nozzle/logging"
)

type SumoLogicAppender struct {
	url                         string
	connectionTimeout           int //10000
	httpClient                  http.Client
	nozzleQueue                 *eventQueue.Queue
	eventsBatchSize             int
	sumoPostMinimumDelay        time.Duration
	timerBetweenPost            time.Time
	sumoCategory                string
	sumoName                    string
	sumoHost                    string
	verboseLogMessages          bool
	customMetadata              string
	includeOnlyMatchingFilter   string
	excludeAlwaysMatchingFilter string
	nozzleVersion               string
}

type SumoBuffer struct {
	logStringToSend          *bytes.Buffer
	logEventsInCurrentBuffer int
	timerIdlebuffer          time.Time
}

func NewSumoLogicAppender(urlValue string, connectionTimeoutValue int, nozzleQueue *eventQueue.Queue, eventsBatchSize int, sumoPostMinimumDelay time.Duration, sumoCategory string, sumoName string, sumoHost string, verboseLogMessages bool, customMetadata string, includeOnlyMatchingFilter string, excludeAlwaysMatchingFilter string, nozzleVersion string) *SumoLogicAppender {
	return &SumoLogicAppender{
		url:                         urlValue,
		connectionTimeout:           connectionTimeoutValue,
		httpClient:                  http.Client{Timeout: time.Duration(connectionTimeoutValue * int(time.Millisecond))},
		nozzleQueue:                 nozzleQueue,
		eventsBatchSize:             eventsBatchSize,
		sumoPostMinimumDelay:        sumoPostMinimumDelay,
		sumoCategory:                sumoCategory,
		sumoName:                    sumoName,
		sumoHost:                    sumoHost,
		verboseLogMessages:          verboseLogMessages,
		customMetadata:              customMetadata,
		includeOnlyMatchingFilter:   includeOnlyMatchingFilter,
		excludeAlwaysMatchingFilter: excludeAlwaysMatchingFilter,
		nozzleVersion:               nozzleVersion,
	}
}

func newBuffer() SumoBuffer {
	return SumoBuffer{
		logStringToSend:          bytes.NewBufferString(""),
		logEventsInCurrentBuffer: 0,
	}
}

func (s *SumoLogicAppender) Start() {
	s.timerBetweenPost = time.Now()
	Buffer := newBuffer()
	Buffer.timerIdlebuffer = time.Now()
	logging.Info.Println("Starting Appender Worker")
	for {
		//logging.Info.Printf("Log queue size: %d", s.nozzleQueue.GetCount())
		if s.nozzleQueue.GetCount() == 0 {
			logging.Trace.Println("Waiting for 300 ms")
			time.Sleep(300 * time.Millisecond)
		}

		if time.Since(Buffer.timerIdlebuffer).Seconds() >= 10 && Buffer.logEventsInCurrentBuffer > 0 {
			logging.Info.Println("Sending current batch of logs after timer exceeded limit")
			go s.SendToSumo(Buffer.logStringToSend.String())
			Buffer = newBuffer()
			Buffer.timerIdlebuffer = time.Now()
			continue
		}

		if s.nozzleQueue.GetCount() != 0 {
			queueCount := s.nozzleQueue.GetCount()
			remainingBufferCount := s.eventsBatchSize - Buffer.logEventsInCurrentBuffer
			if queueCount >= remainingBufferCount {
				logging.Trace.Println("Pushing Logs to Sumo: ")
				logging.Trace.Println(remainingBufferCount)
				for i := 0; i < remainingBufferCount; i++ {
					s.AppendLogs(&Buffer)
					Buffer.timerIdlebuffer = time.Now()
				}

				go s.SendToSumo(Buffer.logStringToSend.String())
				Buffer = newBuffer()
			} else {
				logging.Trace.Println("Pushing Logs to Buffer: ")
				logging.Trace.Println(queueCount)
				for i := 0; i < queueCount; i++ {
					s.AppendLogs(&Buffer)
					Buffer.timerIdlebuffer = time.Now()
				}
			}
		}

	}

}
func WantedEvent(event string, includeOnlyMatchingFilter string, excludeAlwaysMatchingFilter string) bool {
	if includeOnlyMatchingFilter != "" && excludeAlwaysMatchingFilter != "" {
		subsliceInclude := ParseCustomInput(includeOnlyMatchingFilter)
		subsliceExclude := ParseCustomInput(excludeAlwaysMatchingFilter)
		for key, value := range subsliceInclude {
			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
				for key, value := range subsliceExclude {
					if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
						return false
					}
				}
				return true
			}
		}
		for key, value := range subsliceExclude {
			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
				return false
			}
		}
		return true
	} else if includeOnlyMatchingFilter != "" {
		subslice := ParseCustomInput(includeOnlyMatchingFilter)
		for key, value := range subslice {
			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
				return true
			}
		}
		return false

	} else if excludeAlwaysMatchingFilter != "" {
		subslice := ParseCustomInput(excludeAlwaysMatchingFilter)
		for key, value := range subslice {
			if strings.Contains(event, "\""+key+"\":\""+value+"\"") {
				return false
			}
		}
		return true
	}
	return true

}
func FormatTimestamp(event *events.Event, timestamp string) {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered in f", r)
		}
	}()

	if reflect.TypeOf(event.Fields[timestamp]).Kind() != reflect.Int64 {
		if reflect.TypeOf(event.Fields[timestamp]).Kind() == reflect.String {
			event.Fields[timestamp] = event.Fields[timestamp].(string)
		} else {
			event.Fields[timestamp] = ""
		}

	}
	if reflect.TypeOf(event.Fields[timestamp]).Kind() == reflect.Int64 {
		if len(strconv.FormatInt(event.Fields[timestamp].(int64), 10)) == 19 {
			event.Fields[timestamp] = time.Unix(0, event.Fields[timestamp].(int64)*int64(time.Nanosecond)).String()
		} else if len(strconv.FormatInt(event.Fields[timestamp].(int64), 10)) < 13 {
			event.Fields[timestamp] = ""
		}
	}

}

func StringBuilder(event *events.Event, verboseLogMessages bool, includeOnlyMatchingFilter string, excludeAlwaysMatchingFilter string, customMetadata string) string {
	if customMetadata != "" {
		customMetadataMap := ParseCustomInput(customMetadata)
		for key, value := range customMetadataMap {
			event.Fields[key] = value
		}
	}
	eventType := event.Type
	var msg []byte
	switch eventType {
	case "HttpStart":
		FormatTimestamp(event, "timestamp")
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "HttpStop":
		FormatTimestamp(event, "timestamp")
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "HttpStartStop":
		FormatTimestamp(event, "start_timestamp")
		FormatTimestamp(event, "stop_timestamp")
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "LogMessage":
		FormatTimestamp(event, "timestamp")
		if verboseLogMessages == true {
			message, err := json.Marshal(event)
			if err == nil {
				msg = message
			}
		} else {
			eventNoVerbose := events.Event{
				Fields: map[string]interface{}{
					"timestamp":   event.Fields["timestamp"],
					"cf_app_guid": event.Fields["cf_app_id"],
				},
				Msg:  event.Msg,
				Type: event.Type,
			}
			if customMetadata != "" {
				customMetadataMap := ParseCustomInput(customMetadata)
				for key, value := range customMetadataMap {
					eventNoVerbose.Fields[key] = value
				}
			}
			message, err := json.Marshal(eventNoVerbose)
			if err == nil {
				msg = message
			}
		}
	case "ValueMetric":
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "CounterEvent":
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "Error":
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	case "ContainerMetric":
		message, err := json.Marshal(event)
		if err == nil {
			msg = message
		}
	}

	buf := new(bytes.Buffer)
	buf.Write(msg)
	if WantedEvent(buf.String(), includeOnlyMatchingFilter, excludeAlwaysMatchingFilter) {
		return buf.String() + "\n"
	} else {
		return ""
	}

}

func (s *SumoLogicAppender) AppendLogs(buffer *SumoBuffer) {
	buffer.logStringToSend.Write([]byte(StringBuilder(s.nozzleQueue.Pop(), s.verboseLogMessages, s.includeOnlyMatchingFilter, s.excludeAlwaysMatchingFilter, s.customMetadata)))
	buffer.logEventsInCurrentBuffer++

}
func ParseCustomInput(customInput string) map[string]string {
	cInputArray := strings.Split(customInput, ",")
	customInputMap := make(map[string]string)
	for i := 0; i < len(cInputArray); i++ {
		customInputMap[strings.Split(cInputArray[i], ":")[0]] = strings.Split(cInputArray[i], ":")[1]
	}
	return customInputMap
}

func (s *SumoLogicAppender) SendToSumo(logStringToSend string) {
	if logStringToSend != "" {
		var buf bytes.Buffer
		g := gzip.NewWriter(&buf)
		g.Write([]byte(logStringToSend))
		g.Close()
		request, err := http.NewRequest("POST", s.url, &buf)
		if err != nil {
			logging.Error.Printf("http.NewRequest() error: %v\n", err)
			return
		}
		request.Header.Add("Content-Encoding", "gzip")
		request.Header.Add("X-Sumo-Client", "cloudfoundry-sumologic-nozzle v"+s.nozzleVersion)

		if s.sumoName != "" {
			request.Header.Add("X-Sumo-Name", s.sumoName)
		}
		if s.sumoHost != "" {
			request.Header.Add("X-Sumo-Host", s.sumoHost)
		}
		if s.sumoCategory != "" {
			request.Header.Add("X-Sumo-Category", s.sumoCategory)
		}
		//checking the timer before first POST intent
		for time.Since(s.timerBetweenPost) < s.sumoPostMinimumDelay {
			logging.Trace.Println("Delaying Post because minimum post timer not expired")
			time.Sleep(100 * time.Millisecond)
		}
		response, err := s.httpClient.Do(request)

		if (err != nil) || (response.StatusCode != 200 && response.StatusCode != 302 && response.StatusCode < 500) {
			logging.Info.Println("Endpoint dropped the post send")
			logging.Info.Println("Waiting for 300 ms to retry")
			time.Sleep(300 * time.Millisecond)
			statusCode := 0
			err := Retry(func(attempt int) (bool, error) {
				var errRetry error
				request, err := http.NewRequest("POST", s.url, &buf)
				if err != nil {
					logging.Error.Printf("http.NewRequest() error: %v\n", err)
				}
				request.Header.Add("Content-Encoding", "gzip")
				request.Header.Add("X-Sumo-Client", "cloudfoundry-sumologic-nozzle v"+s.nozzleVersion)

				if s.sumoName != "" {
					request.Header.Add("X-Sumo-Name", s.sumoName)
				}
				if s.sumoHost != "" {
					request.Header.Add("X-Sumo-Host", s.sumoHost)
				}
				if s.sumoCategory != "" {
					request.Header.Add("X-Sumo-Category", s.sumoCategory)
				}
				//checking the timer before POST (retry intent)
				for time.Since(s.timerBetweenPost) < s.sumoPostMinimumDelay {
					logging.Trace.Println("Delaying Post because minimum post timer not expired")
					time.Sleep(100 * time.Millisecond)
				}
				response, errRetry = s.httpClient.Do(request)
				if errRetry != nil {
					logging.Error.Printf("http.Do() error: %v\n", errRetry)
					logging.Info.Println("Waiting for 300 ms to retry after error")
					time.Sleep(300 * time.Millisecond)
					return attempt < 5, errRetry
				} else if response.StatusCode != 200 && response.StatusCode != 302 && response.StatusCode < 500 {
					logging.Info.Println("Endpoint dropped the post send again")
					logging.Info.Println("Waiting for 300 ms to retry after a retry ...")
					statusCode = response.StatusCode
					time.Sleep(300 * time.Millisecond)
					return attempt < 5, errRetry
				} else if response.StatusCode == 200 {
					logging.Trace.Println("Post of logs successful after retry...")
					s.timerBetweenPost = time.Now()
					statusCode = response.StatusCode
					return true, err
				}
				return attempt < 5, errRetry
			})
			if err != nil {
				logging.Error.Println("Error, Not able to post after retry")
				logging.Error.Printf("http.Do() error: %v\n", err)
				return
			} else if statusCode != 200 {
				logging.Error.Printf("Not able to post after retry, with status code: %d", statusCode)
			}
		} else if response.StatusCode == 200 {
			logging.Trace.Println("Post of logs successful")
			s.timerBetweenPost = time.Now()
		}

		if response != nil {
			defer response.Body.Close()
		}
	}

}

//------------------Retry Logic Code-------------------------------

// MaxRetries is the maximum number of retries before bailing.
var MaxRetries = 10
var errMaxRetriesReached = errors.New("exceeded retry limit")

// Func represents functions that can be retried.
type Func func(attempt int) (retry bool, err error)

// Do keeps trying the function until the second argument
// returns false, or no error is returned.
func Retry(fn Func) error {
	var err error
	var cont bool
	attempt := 1
	for {
		cont, err = fn(attempt)
		if !cont || err == nil {
			break
		}
		attempt++
		if attempt > MaxRetries {
			return errMaxRetriesReached
		}
	}
	return err
}

// IsMaxRetries checks whether the error is due to hitting the
// maximum number of retries or not.
func IsMaxRetries(err error) bool {
	return err == errMaxRetriesReached
}
<|endoftext|>


<commit_before>package irmaclient

import (
	"encoding/binary"
	"encoding/json"
	"github.com/go-errors/errors"
	"path/filepath"
	"time"

	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/fs"
	"go.etcd.io/bbolt"
)

// This file contains the storage struct and its methods,
// and some general filesystem functions.

// Storage provider for a Client
type storage struct {
	storagePath   string
	db            *bbolt.DB
	Configuration *irma.Configuration
}

type transaction struct {
	*bbolt.Tx
}

// Filenames
const databaseFile = "db"

// Bucketnames bbolt
const (
	userdataBucket = "userdata"    // Key/value: specified below
	skKey          = "sk"          // Value: *secretKey
	preferencesKey = "preferences" // Value: Preferences
	updatesKey     = "updates"     // Value: []update
	kssKey         = "kss"         // Value: map[irma.SchemeManagerIdentifier]*keyshareServer

	attributesBucket = "attrs" // Key: irma.CredentialIdentifier, value: []*irma.AttributeList
	logsBucket       = "logs"  // Key: (auto-increment index), value: *LogEntry
	signaturesBucket = "sigs"  // Key: credential.attrs.Hash, value: *gabi.CLSignature
)

func (s *storage) path(p string) string {
	return filepath.Join(s.storagePath, p)
}

// EnsureStorageExists initializes the credential storage folder,
// ensuring that it is in a usable state.
// Setting it up in a properly protected location (e.g., with automatic
// backups to iCloud/Google disabled) is the responsibility of the user.
func (s *storage) EnsureStorageExists() error {
	var err error
	if err = fs.AssertPathExists(s.storagePath); err != nil {
		return err
	}
	s.db, err = bbolt.Open(s.path(databaseFile), 0600, &bbolt.Options{Timeout: 1 * time.Second})
	return err
}

func (s *storage) Close() error {
	return s.db.Close()
}

func (s *storage) txStore(tx *transaction, bucketName string, key string, value interface{}) error {
	b, err := tx.CreateBucketIfNotExists([]byte(bucketName))
	if err != nil {
		return err
	}
	btsValue, err := json.Marshal(value)
	if err != nil {
		return err
	}

	return b.Put([]byte(key), btsValue)
}

func (s *storage) txDelete(tx *transaction, key string, bucketName string) error {
	b, err := tx.CreateBucketIfNotExists([]byte(bucketName))
	if err != nil {
		return err
	}

	return b.Delete([]byte(key))
}

func (s *storage) txLoad(tx *transaction, key string, dest interface{}, bucketName string) (found bool, err error) {
	b := tx.Bucket([]byte(bucketName))
	if b == nil {
		return false, nil
	}
	bts := b.Get([]byte(key))
	b.Sequence()
	if bts == nil {
		return false, nil
	}
	return true, json.Unmarshal(bts, dest)
}

func (s *storage) load(key string, dest interface{}, bucketName string) (found bool, err error) {
	err = s.db.View(func(tx *bbolt.Tx) error {
		found, err = s.txLoad(&transaction{tx}, key, dest, bucketName)
		return err
	})
	return
}

func (s *storage) DoStoreTransaction(f func(*transaction) error) error {
	return s.db.Update(func(tx *bbolt.Tx) error {
		return f(&transaction{tx})
	})
}

func (s *storage) TxDeleteSignature(tx *transaction, attrs *irma.AttributeList) error {
	return s.txDelete(tx, attrs.Hash(), signaturesBucket)
}

func (s *storage) TxDeleteAllSignatures(tx *transaction) error {
	return tx.DeleteBucket([]byte(signaturesBucket))
}

func (s *storage) TxStoreSignature(tx *transaction, cred *credential) error {
	return s.TxStoreCLSignature(tx, cred.AttributeList().Hash(), cred.Signature)
}

func (s *storage) TxStoreCLSignature(tx *transaction, credHash string, sig *gabi.CLSignature) error {
	// We take the SHA256 hash over all attributes as the bucket key for the signature.
	// This means that of the signatures of two credentials that have identical attributes
	// only one gets stored, one overwriting the other - but that doesn't
	// matter, because either one of the signatures is valid over both attribute lists,
	// so keeping one of them suffices.
	return s.txStore(tx, signaturesBucket, credHash, sig)
}

func (s *storage) StoreSecretKey(sk *secretKey) error {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStoreSecretKey(tx, sk)
	})
}

func (s *storage) TxStoreSecretKey(tx *transaction, sk *secretKey) error {
	return s.txStore(tx, userdataBucket, skKey, sk)
}

func (s *storage) StoreAttributes(credTypeID irma.CredentialTypeIdentifier, attrlistlist []*irma.AttributeList) error {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStoreAttributes(tx, credTypeID, attrlistlist)
	})
}

func (s *storage) TxStoreAttributes(tx *transaction, credTypeID irma.CredentialTypeIdentifier,
	attrlistlist []*irma.AttributeList) error {

	// If no credentials are left of a certain type, the full entry can be deleted.
	if len(attrlistlist) == 0 {
		return s.txDelete(tx, credTypeID.String(), attributesBucket)
	}
	return s.txStore(tx, attributesBucket, credTypeID.String(), attrlistlist)
}

func (s *storage) TxDeleteAllAttributes(tx *transaction) error {
	return tx.DeleteBucket([]byte(attributesBucket))
}

func (s *storage) StoreKeyshareServers(keyshareServers map[irma.SchemeManagerIdentifier]*keyshareServer) error {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStoreKeyshareServers(tx, keyshareServers)
	})
}

func (s *storage) TxStoreKeyshareServers(tx *transaction, keyshareServers map[irma.SchemeManagerIdentifier]*keyshareServer) error {
	return s.txStore(tx, userdataBucket, kssKey, keyshareServers)
}

func (s *storage) AddLogEntry(entry *LogEntry) error {
	return s.db.Update(func(tx *bbolt.Tx) error {
		return s.TxAddLogEntry(&transaction{tx}, entry)
	})
}

func (s *storage) TxAddLogEntry(tx *transaction, entry *LogEntry) error {
	b, err := tx.CreateBucketIfNotExists([]byte(logsBucket))
	if err != nil {
		return err
	}

	entry.ID, err = b.NextSequence()
	if err != nil {
		return err
	}
	k := s.logEntryKeyToBytes(entry.ID)
	v, err := json.Marshal(entry)

	return b.Put(k, v)
}

func (s *storage) logEntryKeyToBytes(id uint64) []byte {
	k := make([]byte, 8)
	binary.BigEndian.PutUint64(k, id)
	return k
}

func (s *storage) StorePreferences(prefs Preferences) error {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStorePreferences(tx, prefs)
	})
}

func (s *storage) TxStorePreferences(tx *transaction, prefs Preferences) error {
	return s.txStore(tx, userdataBucket, preferencesKey, prefs)
}

func (s *storage) StoreUpdates(updates []update) (err error) {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStoreUpdates(tx, updates)
	})
}

func (s *storage) TxStoreUpdates(tx *transaction, updates []update) error {
	return s.txStore(tx, userdataBucket, updatesKey, updates)
}

func (s *storage) LoadSignature(attrs *irma.AttributeList) (signature *gabi.CLSignature, err error) {
	signature = new(gabi.CLSignature)
	found, err := s.load(attrs.Hash(), signature, signaturesBucket)
	if err != nil {
		return nil, err
	} else if !found {
		return nil, errors.Errorf("Signature of credential with hash %s cannot be found", attrs.Hash())
	}
	return
}

// LoadSecretKey retrieves and returns the secret key from bbolt storage, or if no secret key
// was found in storage, it generates, saves, and returns a new secret key.
func (s *storage) LoadSecretKey() (*secretKey, error) {
	sk := &secretKey{}
	found, err := s.load(skKey, sk, userdataBucket)
	if err != nil {
		return nil, err
	}
	if found {
		return sk, nil
	}

	if sk, err = generateSecretKey(); err != nil {
		return nil, err
	}
	if err = s.StoreSecretKey(sk); err != nil {
		return nil, err
	}
	return sk, nil
}

func (s *storage) LoadAttributes() (list map[irma.CredentialTypeIdentifier][]*irma.AttributeList, err error) {
	list = make(map[irma.CredentialTypeIdentifier][]*irma.AttributeList)
	return list, s.db.View(func(tx *bbolt.Tx) error {
		b := tx.Bucket([]byte(attributesBucket))
		if b == nil {
			return nil
		}
		return b.ForEach(func(key, value []byte) error {
			credTypeID := irma.NewCredentialTypeIdentifier(string(key))

			var attrlistlist []*irma.AttributeList
			err = json.Unmarshal(value, &attrlistlist)
			if err != nil {
				return err
			}

			// Initialize metadata attributes
			for _, attrlist := range attrlistlist {
				attrlist.MetadataAttribute = irma.MetadataFromInt(attrlist.Ints[0], s.Configuration)
			}

			list[credTypeID] = attrlistlist
			return nil
		})
	})
}

func (s *storage) LoadKeyshareServers() (ksses map[irma.SchemeManagerIdentifier]*keyshareServer, err error) {
	ksses = make(map[irma.SchemeManagerIdentifier]*keyshareServer)
	_, err = s.load(kssKey, &ksses, userdataBucket)
	return
}

// Returns all logs stored before log with ID 'index' sorted from new to old with
// a maximum result length of 'max'.
func (s *storage) LoadLogsBefore(index uint64, max int) ([]*LogEntry, error) {
	return s.loadLogs(max, func(c *bbolt.Cursor) (key, value []byte) {
		c.Seek(s.logEntryKeyToBytes(index))
		return c.Prev()
	})
}

// Returns the latest logs stored sorted from new to old with a maximum result length of 'max'
func (s *storage) LoadNewestLogs(max int) ([]*LogEntry, error) {
	return s.loadLogs(max, func(c *bbolt.Cursor) (key, value []byte) {
		return c.Last()
	})
}

// Returns the logs stored sorted from new to old with a maximum result length of 'max' where the starting position
// of the bbolt cursor can be manipulated by the anonymous function 'startAt'. 'startAt' should return
// the key and the value of the first element from the bbolt database that should be loaded.
func (s *storage) loadLogs(max int, startAt func(*bbolt.Cursor) (key, value []byte)) ([]*LogEntry, error) {
	logs := make([]*LogEntry, 0, max)
	return logs, s.db.View(func(tx *bbolt.Tx) error {
		bucket := tx.Bucket([]byte(logsBucket))
		if bucket == nil {
			return nil
		}
		c := bucket.Cursor()

		for k, v := startAt(c); k != nil && len(logs) < max; k, v = c.Prev() {
			var log LogEntry
			if err := json.Unmarshal(v, &log); err != nil {
				return err
			}

			logs = append(logs, &log)
		}
		return nil
	})
}

func (s *storage) LoadUpdates() (updates []update, err error) {
	updates = []update{}
	_, err = s.load(updatesKey, &updates, userdataBucket)
	return
}

func (s *storage) LoadPreferences() (Preferences, error) {
	config := defaultPreferences
	_, err := s.load(preferencesKey, &config, userdataBucket)
	return config, err
}
<commit_msg>Place bucketName parameter at beginning of function signature<commit_after>package irmaclient

import (
	"encoding/binary"
	"encoding/json"
	"github.com/go-errors/errors"
	"path/filepath"
	"time"

	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/fs"
	"go.etcd.io/bbolt"
)

// This file contains the storage struct and its methods,
// and some general filesystem functions.

// Storage provider for a Client
type storage struct {
	storagePath   string
	db            *bbolt.DB
	Configuration *irma.Configuration
}

type transaction struct {
	*bbolt.Tx
}

// Filenames
const databaseFile = "db"

// Bucketnames bbolt
const (
	userdataBucket = "userdata"    // Key/value: specified below
	skKey          = "sk"          // Value: *secretKey
	preferencesKey = "preferences" // Value: Preferences
	updatesKey     = "updates"     // Value: []update
	kssKey         = "kss"         // Value: map[irma.SchemeManagerIdentifier]*keyshareServer

	attributesBucket = "attrs" // Key: irma.CredentialIdentifier, value: []*irma.AttributeList
	logsBucket       = "logs"  // Key: (auto-increment index), value: *LogEntry
	signaturesBucket = "sigs"  // Key: credential.attrs.Hash, value: *gabi.CLSignature
)

func (s *storage) path(p string) string {
	return filepath.Join(s.storagePath, p)
}

// EnsureStorageExists initializes the credential storage folder,
// ensuring that it is in a usable state.
// Setting it up in a properly protected location (e.g., with automatic
// backups to iCloud/Google disabled) is the responsibility of the user.
func (s *storage) EnsureStorageExists() error {
	var err error
	if err = fs.AssertPathExists(s.storagePath); err != nil {
		return err
	}
	s.db, err = bbolt.Open(s.path(databaseFile), 0600, &bbolt.Options{Timeout: 1 * time.Second})
	return err
}

func (s *storage) Close() error {
	return s.db.Close()
}

func (s *storage) txStore(tx *transaction, bucketName string, key string, value interface{}) error {
	b, err := tx.CreateBucketIfNotExists([]byte(bucketName))
	if err != nil {
		return err
	}
	btsValue, err := json.Marshal(value)
	if err != nil {
		return err
	}

	return b.Put([]byte(key), btsValue)
}

func (s *storage) txDelete(tx *transaction, bucketName string, key string) error {
	b, err := tx.CreateBucketIfNotExists([]byte(bucketName))
	if err != nil {
		return err
	}

	return b.Delete([]byte(key))
}

func (s *storage) txLoad(tx *transaction, bucketName string, key string, dest interface{}) (found bool, err error) {
	b := tx.Bucket([]byte(bucketName))
	if b == nil {
		return false, nil
	}
	bts := b.Get([]byte(key))
	b.Sequence()
	if bts == nil {
		return false, nil
	}
	return true, json.Unmarshal(bts, dest)
}

func (s *storage) load(bucketName string, key string, dest interface{}) (found bool, err error) {
	err = s.db.View(func(tx *bbolt.Tx) error {
		found, err = s.txLoad(&transaction{tx}, bucketName, key, dest)
		return err
	})
	return
}

func (s *storage) DoStoreTransaction(f func(*transaction) error) error {
	return s.db.Update(func(tx *bbolt.Tx) error {
		return f(&transaction{tx})
	})
}

func (s *storage) TxDeleteSignature(tx *transaction, attrs *irma.AttributeList) error {
	return s.txDelete(tx, signaturesBucket, attrs.Hash())
}

func (s *storage) TxDeleteAllSignatures(tx *transaction) error {
	return tx.DeleteBucket([]byte(signaturesBucket))
}

func (s *storage) TxStoreSignature(tx *transaction, cred *credential) error {
	return s.TxStoreCLSignature(tx, cred.AttributeList().Hash(), cred.Signature)
}

func (s *storage) TxStoreCLSignature(tx *transaction, credHash string, sig *gabi.CLSignature) error {
	// We take the SHA256 hash over all attributes as the bucket key for the signature.
	// This means that of the signatures of two credentials that have identical attributes
	// only one gets stored, one overwriting the other - but that doesn't
	// matter, because either one of the signatures is valid over both attribute lists,
	// so keeping one of them suffices.
	return s.txStore(tx, signaturesBucket, credHash, sig)
}

func (s *storage) StoreSecretKey(sk *secretKey) error {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStoreSecretKey(tx, sk)
	})
}

func (s *storage) TxStoreSecretKey(tx *transaction, sk *secretKey) error {
	return s.txStore(tx, userdataBucket, skKey, sk)
}

func (s *storage) StoreAttributes(credTypeID irma.CredentialTypeIdentifier, attrlistlist []*irma.AttributeList) error {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStoreAttributes(tx, credTypeID, attrlistlist)
	})
}

func (s *storage) TxStoreAttributes(tx *transaction, credTypeID irma.CredentialTypeIdentifier,
	attrlistlist []*irma.AttributeList) error {

	// If no credentials are left of a certain type, the full entry can be deleted.
	if len(attrlistlist) == 0 {
		return s.txDelete(tx, attributesBucket, credTypeID.String())
	}
	return s.txStore(tx, attributesBucket, credTypeID.String(), attrlistlist)
}

func (s *storage) TxDeleteAllAttributes(tx *transaction) error {
	return tx.DeleteBucket([]byte(attributesBucket))
}

func (s *storage) StoreKeyshareServers(keyshareServers map[irma.SchemeManagerIdentifier]*keyshareServer) error {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStoreKeyshareServers(tx, keyshareServers)
	})
}

func (s *storage) TxStoreKeyshareServers(tx *transaction, keyshareServers map[irma.SchemeManagerIdentifier]*keyshareServer) error {
	return s.txStore(tx, userdataBucket, kssKey, keyshareServers)
}

func (s *storage) AddLogEntry(entry *LogEntry) error {
	return s.db.Update(func(tx *bbolt.Tx) error {
		return s.TxAddLogEntry(&transaction{tx}, entry)
	})
}

func (s *storage) TxAddLogEntry(tx *transaction, entry *LogEntry) error {
	b, err := tx.CreateBucketIfNotExists([]byte(logsBucket))
	if err != nil {
		return err
	}

	entry.ID, err = b.NextSequence()
	if err != nil {
		return err
	}
	k := s.logEntryKeyToBytes(entry.ID)
	v, err := json.Marshal(entry)

	return b.Put(k, v)
}

func (s *storage) logEntryKeyToBytes(id uint64) []byte {
	k := make([]byte, 8)
	binary.BigEndian.PutUint64(k, id)
	return k
}

func (s *storage) StorePreferences(prefs Preferences) error {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStorePreferences(tx, prefs)
	})
}

func (s *storage) TxStorePreferences(tx *transaction, prefs Preferences) error {
	return s.txStore(tx, userdataBucket, preferencesKey, prefs)
}

func (s *storage) StoreUpdates(updates []update) (err error) {
	return s.DoStoreTransaction(func(tx *transaction) error {
		return s.TxStoreUpdates(tx, updates)
	})
}

func (s *storage) TxStoreUpdates(tx *transaction, updates []update) error {
	return s.txStore(tx, userdataBucket, updatesKey, updates)
}

func (s *storage) LoadSignature(attrs *irma.AttributeList) (signature *gabi.CLSignature, err error) {
	signature = new(gabi.CLSignature)
	found, err := s.load(signaturesBucket, attrs.Hash(), signature)
	if err != nil {
		return nil, err
	} else if !found {
		return nil, errors.Errorf("Signature of credential with hash %s cannot be found", attrs.Hash())
	}
	return
}

// LoadSecretKey retrieves and returns the secret key from bbolt storage, or if no secret key
// was found in storage, it generates, saves, and returns a new secret key.
func (s *storage) LoadSecretKey() (*secretKey, error) {
	sk := &secretKey{}
	found, err := s.load(userdataBucket, skKey, sk)
	if err != nil {
		return nil, err
	}
	if found {
		return sk, nil
	}

	if sk, err = generateSecretKey(); err != nil {
		return nil, err
	}
	if err = s.StoreSecretKey(sk); err != nil {
		return nil, err
	}
	return sk, nil
}

func (s *storage) LoadAttributes() (list map[irma.CredentialTypeIdentifier][]*irma.AttributeList, err error) {
	list = make(map[irma.CredentialTypeIdentifier][]*irma.AttributeList)
	return list, s.db.View(func(tx *bbolt.Tx) error {
		b := tx.Bucket([]byte(attributesBucket))
		if b == nil {
			return nil
		}
		return b.ForEach(func(key, value []byte) error {
			credTypeID := irma.NewCredentialTypeIdentifier(string(key))

			var attrlistlist []*irma.AttributeList
			err = json.Unmarshal(value, &attrlistlist)
			if err != nil {
				return err
			}

			// Initialize metadata attributes
			for _, attrlist := range attrlistlist {
				attrlist.MetadataAttribute = irma.MetadataFromInt(attrlist.Ints[0], s.Configuration)
			}

			list[credTypeID] = attrlistlist
			return nil
		})
	})
}

func (s *storage) LoadKeyshareServers() (ksses map[irma.SchemeManagerIdentifier]*keyshareServer, err error) {
	ksses = make(map[irma.SchemeManagerIdentifier]*keyshareServer)
	_, err = s.load(userdataBucket, kssKey, &ksses)
	return
}

// Returns all logs stored before log with ID 'index' sorted from new to old with
// a maximum result length of 'max'.
func (s *storage) LoadLogsBefore(index uint64, max int) ([]*LogEntry, error) {
	return s.loadLogs(max, func(c *bbolt.Cursor) (key, value []byte) {
		c.Seek(s.logEntryKeyToBytes(index))
		return c.Prev()
	})
}

// Returns the latest logs stored sorted from new to old with a maximum result length of 'max'
func (s *storage) LoadNewestLogs(max int) ([]*LogEntry, error) {
	return s.loadLogs(max, func(c *bbolt.Cursor) (key, value []byte) {
		return c.Last()
	})
}

// Returns the logs stored sorted from new to old with a maximum result length of 'max' where the starting position
// of the bbolt cursor can be manipulated by the anonymous function 'startAt'. 'startAt' should return
// the key and the value of the first element from the bbolt database that should be loaded.
func (s *storage) loadLogs(max int, startAt func(*bbolt.Cursor) (key, value []byte)) ([]*LogEntry, error) {
	logs := make([]*LogEntry, 0, max)
	return logs, s.db.View(func(tx *bbolt.Tx) error {
		bucket := tx.Bucket([]byte(logsBucket))
		if bucket == nil {
			return nil
		}
		c := bucket.Cursor()

		for k, v := startAt(c); k != nil && len(logs) < max; k, v = c.Prev() {
			var log LogEntry
			if err := json.Unmarshal(v, &log); err != nil {
				return err
			}

			logs = append(logs, &log)
		}
		return nil
	})
}

func (s *storage) LoadUpdates() (updates []update, err error) {
	updates = []update{}
	_, err = s.load(userdataBucket, updatesKey, &updates)
	return
}

func (s *storage) LoadPreferences() (Preferences, error) {
	config := defaultPreferences
	_, err := s.load(userdataBucket, preferencesKey, &config)
	return config, err
}
<|endoftext|>


<commit_before>//Package atomicTypes provides object locking / unlocking for setting and getting.
package atomicTypes

import "sync"

//AtomicString provides a string object that is lock safe.
type AtomicString struct {
	valueSync sync.RWMutex
	value     string
}

//Get returns the string value
func (obj *AtomicString) Get() (value string) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the string value
func (obj *AtomicString) Set(value string) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicInt provides an int object that is lock safe.
type AtomicInt struct {
	valueSync sync.RWMutex
	value     int
}

//Get returns the int value
func (obj *AtomicInt) Get() (value int) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the int value
func (obj *AtomicInt) Set(value int) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicBool provides an bool object that is lock safe.
type AtomicBool struct {
	valueSync sync.RWMutex
	value     bool
}

//Get returns the bool value
func (obj *AtomicBool) Get() (value bool) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the bool value
func (obj *AtomicBool) Set(value bool) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}
<commit_msg>Added more atomic types.<commit_after>//Package atomicTypes provides object locking / unlocking for setting and getting.
package atomicTypes

import (
	"sync"
	"time"
)

//AtomicString provides a string object that is lock safe.
type AtomicString struct {
	valueSync sync.RWMutex
	value     string
}

//Get returns the string value
func (obj *AtomicString) Get() (value string) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the string value
func (obj *AtomicString) Set(value string) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicInt provides an int object that is lock safe.
type AtomicInt struct {
	valueSync sync.RWMutex
	value     int
}

//Get returns the int value
func (obj *AtomicInt) Get() (value int) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the int value
func (obj *AtomicInt) Set(value int) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicBool provides an bool object that is lock safe.
type AtomicBool struct {
	valueSync sync.RWMutex
	value     bool
}

//Get returns the bool value
func (obj *AtomicBool) Get() (value bool) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the bool value
func (obj *AtomicBool) Set(value bool) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicTime provides a time.Time object that is lock safe.
type AtomicTime struct {
	valueSync sync.RWMutex
	value     time.Time
}

//Get returns the time.Time value
func (obj *AtomicTime) Get() (value time.Time) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the time.Time value
func (obj *AtomicTime) Set(value time.Time) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}

//AtomicByteArray provides a []byte object that is lock safe.
type AtomicByteArray struct {
	valueSync sync.RWMutex
	value     []byte
}

//Get returns the []byte value
func (obj *AtomicByteArray) Get() (value []byte) {
	obj.valueSync.RLock()
	value = obj.value
	obj.valueSync.RUnlock()
	return
}

//Set sets the []byte value
func (obj *AtomicByteArray) Set(value []byte) {
	obj.valueSync.Lock()
	obj.value = value
	obj.valueSync.Unlock()
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http

import (
	"crypto/tls"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"k8s.io/kubernetes/pkg/probe"

	"github.com/golang/glog"
)

func New() HTTPProber {
	tlsConfig := &tls.Config{InsecureSkipVerify: true}
	transport := &http.Transport{TLSClientConfig: tlsConfig}
	return httpProber{transport}
}

type HTTPProber interface {
	Probe(url *url.URL, timeout time.Duration) (probe.Result, string, error)
}

type httpProber struct {
	transport *http.Transport
}

// Probe returns a ProbeRunner capable of running an http check.
func (pr httpProber) Probe(url *url.URL, timeout time.Duration) (probe.Result, string, error) {
	return DoHTTPProbe(url, &http.Client{Timeout: timeout, Transport: pr.transport})
}

type HTTPGetInterface interface {
	Get(u string) (*http.Response, error)
}

// DoHTTPProbe checks if a GET request to the url succeeds.
// If the HTTP response code is successful (i.e. 400 > code >= 200), it returns Success.
// If the HTTP response code is unsuccessful or HTTP communication fails, it returns Failure.
// This is exported because some other packages may want to do direct HTTP probes.
func DoHTTPProbe(url *url.URL, client HTTPGetInterface) (probe.Result, string, error) {
	res, err := client.Get(url.String())
	if err != nil {
		// Convert errors into failures to catch timeouts.
		return probe.Failure, err.Error(), nil
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return probe.Failure, "", err
	}
	body := string(b)
	if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
		glog.V(4).Infof("Probe succeeded for %s, Response: %v", url.String(), *res)
		return probe.Success, body, nil
	}
	glog.V(4).Infof("Probe failed for %s, Response: %v", url.String(), *res)
	return probe.Failure, body, nil
}
<commit_msg>UPSTREAM: 15733: Disable keepalive on liveness probes<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package http

import (
	"crypto/tls"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"k8s.io/kubernetes/pkg/probe"

	"github.com/golang/glog"
)

func New() HTTPProber {
	tlsConfig := &tls.Config{InsecureSkipVerify: true}
	transport := &http.Transport{TLSClientConfig: tlsConfig, DisableKeepAlives: true}
	return httpProber{transport}
}

type HTTPProber interface {
	Probe(url *url.URL, timeout time.Duration) (probe.Result, string, error)
}

type httpProber struct {
	transport *http.Transport
}

// Probe returns a ProbeRunner capable of running an http check.
func (pr httpProber) Probe(url *url.URL, timeout time.Duration) (probe.Result, string, error) {
	return DoHTTPProbe(url, &http.Client{Timeout: timeout, Transport: pr.transport})
}

type HTTPGetInterface interface {
	Get(u string) (*http.Response, error)
}

// DoHTTPProbe checks if a GET request to the url succeeds.
// If the HTTP response code is successful (i.e. 400 > code >= 200), it returns Success.
// If the HTTP response code is unsuccessful or HTTP communication fails, it returns Failure.
// This is exported because some other packages may want to do direct HTTP probes.
func DoHTTPProbe(url *url.URL, client HTTPGetInterface) (probe.Result, string, error) {
	res, err := client.Get(url.String())
	if err != nil {
		// Convert errors into failures to catch timeouts.
		return probe.Failure, err.Error(), nil
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return probe.Failure, "", err
	}
	body := string(b)
	if res.StatusCode >= http.StatusOK && res.StatusCode < http.StatusBadRequest {
		glog.V(4).Infof("Probe succeeded for %s, Response: %v", url.String(), *res)
		return probe.Success, body, nil
	}
	glog.V(4).Infof("Probe failed for %s, Response: %v", url.String(), *res)
	return probe.Failure, body, nil
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package provisioner

import (
	"fmt"

	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/names"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver/common"
	"launchpad.net/juju-core/state/watcher"
)

// ProvisionerAPI provides access to the Provisioner API facade.
type ProvisionerAPI struct {
	*common.Remover
	*common.StatusSetter
	*common.DeadEnsurer
	*common.PasswordChanger
	*common.LifeGetter

	st          *state.State
	resources   *common.Resources
	authorizer  common.Authorizer
	getAuthFunc common.GetAuthFunc
}

// NewProvisionerAPI creates a new server-side ProvisionerAPI facade.
func NewProvisionerAPI(
	st *state.State,
	resources *common.Resources,
	authorizer common.Authorizer,
) (*ProvisionerAPI, error) {
	if !authorizer.AuthMachineAgent() && !authorizer.AuthEnvironManager() {
		return nil, common.ErrPerm
	}
	getAuthFunc := func() (common.AuthFunc, error) {
		isEnvironManager := authorizer.AuthEnvironManager()
		isMachineAgent := authorizer.AuthMachineAgent()
		authEntityTag := authorizer.GetAuthTag()

		return func(tag string) bool {
			if isMachineAgent && tag == authEntityTag {
				// A machine agent can always access its own machine.
				return true
			}
			_, id, err := names.ParseTag(tag, names.MachineTagKind)
			if err != nil {
				return false
			}
			parentId := state.ParentId(id)
			if parentId == "" {
				// All top-level machines are accessible by the
				// environment manager.
				return isEnvironManager
			}
			// All containers with the authenticated machine as a
			// parent are accessible by it.
			return isMachineAgent && names.MachineTag(parentId) == authEntityTag
		}, nil
	}
	return &ProvisionerAPI{
		Remover:         common.NewRemover(st, false, getAuthFunc),
		StatusSetter:    common.NewStatusSetter(st, getAuthFunc),
		DeadEnsurer:     common.NewDeadEnsurer(st, getAuthFunc),
		PasswordChanger: common.NewPasswordChanger(st, getAuthFunc),
		LifeGetter:      common.NewLifeGetter(st, getAuthFunc),
		st:              st,
		resources:       resources,
		authorizer:      authorizer,
		getAuthFunc:     getAuthFunc,
	}, nil
}

func (p *ProvisionerAPI) getMachine(canAccess common.AuthFunc, tag string) (*state.Machine, error) {
	if !canAccess(tag) {
		return nil, common.ErrPerm
	}
	entity, err := p.st.FindEntity(tag)
	if err != nil {
		return nil, err
	}
	// The authorization function guarantees that the tag represents a
	// machine.
	return entity.(*state.Machine), nil
}

func (p *ProvisionerAPI) watchOneMachineContainers(arg params.WatchContainer) (params.StringsWatchResult, error) {
	nothing := params.StringsWatchResult{}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return nothing, err
	}
	if !canAccess(arg.MachineTag) {
		return nothing, common.ErrPerm
	}
	_, id, err := names.ParseTag(arg.MachineTag, names.MachineTagKind)
	if err != nil {
		return nothing, err
	}
	machine, err := p.st.Machine(id)
	if err != nil {
		return nothing, err
	}
	watch := machine.WatchContainers(instance.ContainerType(arg.ContainerType))
	// Consume the initial event and forward it to the result.
	if changes, ok := <-watch.Changes(); ok {
		return params.StringsWatchResult{
			StringsWatcherId: p.resources.Register(watch),
			Changes:          changes,
		}, nil
	}
	return nothing, watcher.MustErr(watch)
}

// WatchContainers starts a StringsWatcher to watch all containers deployed to
// any machine passed in args.
func (p *ProvisionerAPI) WatchContainers(args params.WatchContainers) (params.StringsWatchResults, error) {
	result := params.StringsWatchResults{
		Results: make([]params.StringsWatchResult, len(args.Params)),
	}
	for i, arg := range args.Params {
		watcherResult, err := p.watchOneMachineContainers(arg)
		result.Results[i] = watcherResult
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// WatchForEnvironConfigChanges returns a NotifyWatcher to observe
// changes to the environment configuration.
func (p *ProvisionerAPI) WatchForEnvironConfigChanges() (params.NotifyWatchResult, error) {
	result := params.NotifyWatchResult{}
	watch := p.st.WatchForEnvironConfigChanges()
	// Consume the initial event. Technically, API
	// calls to Watch 'transmit' the initial event
	// in the Watch response. But NotifyWatchers
	// have no state to transmit.
	if _, ok := <-watch.Changes(); ok {
		result.NotifyWatcherId = p.resources.Register(watch)
	} else {
		return result, watcher.MustErr(watch)
	}
	return result, nil
}

// EnvironConfig returns the current environment's configuration.
func (p *ProvisionerAPI) EnvironConfig() (params.ConfigResult, error) {
	result := params.ConfigResult{}
	config, err := p.st.EnvironConfig()
	if err != nil {
		return result, err
	}
	result.Config = config.AllAttrs()
	return result, nil
}

// Status returns the status of each given machine entity.
func (p *ProvisionerAPI) Status(args params.Entities) (params.StatusResults, error) {
	result := params.StatusResults{
		Results: make([]params.StatusResult, len(args.Entities)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, entity := range args.Entities {
		machine, err := p.getMachine(canAccess, entity.Tag)
		if err == nil {
			var status params.Status
			var info string
			status, info, err = machine.Status()
			if err == nil {
				result.Results[i].Status = status
				result.Results[i].Info = info
			}
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// Series returns the deployed series for each given machine entity.
func (p *ProvisionerAPI) Series(args params.Entities) (params.StringResults, error) {
	result := params.StringResults{
		Results: make([]params.StringResult, len(args.Entities)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, entity := range args.Entities {
		machine, err := p.getMachine(canAccess, entity.Tag)
		if err == nil {
			result.Results[i].Result = machine.Series()
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// Constraints returns the constraints for each given machine entity.
func (p *ProvisionerAPI) Constraints(args params.Entities) (params.ConstraintsResults, error) {
	result := params.ConstraintsResults{
		Results: make([]params.ConstraintsResult, len(args.Entities)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, entity := range args.Entities {
		machine, err := p.getMachine(canAccess, entity.Tag)
		if err == nil {
			var cons constraints.Value
			cons, err = machine.Constraints()
			if err == nil {
				result.Results[i].Constraints = cons
			}
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// SetProvisioned sets the provider specific machine id, nonce and
// metadata for each given machine. Once set, the instance id cannot
// be changed.
func (p *ProvisionerAPI) SetProvisioned(args params.SetProvisioned) (params.ErrorResults, error) {
	result := params.ErrorResults{
		Results: make([]params.ErrorResult, len(args.Machines)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, arg := range args.Machines {
		machine, err := p.getMachine(canAccess, arg.Tag)
		if err == nil {
			err = machine.SetProvisioned(arg.InstanceId, arg.Nonce, arg.Characteristics)
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// InstanceId returns the provider specific instance id for each given
// machine or an CodeNotProvisioned error, if not set.
func (p *ProvisionerAPI) InstanceId(args params.Entities) (params.StringResults, error) {
	result := params.StringResults{
		Results: make([]params.StringResult, len(args.Entities)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, entity := range args.Entities {
		machine, err := p.getMachine(canAccess, entity.Tag)
		if err == nil {
			var instanceId instance.Id
			instanceId, err = machine.InstanceId()
			if err == nil {
				result.Results[i].Result = string(instanceId)
			}
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// WatchEnvironMachines returns a StringsWatcher that notifies of
// changes to the lifecycles of the machines (but not containers) in
// the current environment.
func (p *ProvisionerAPI) WatchEnvironMachines() (params.StringsWatchResult, error) {
	result := params.StringsWatchResult{}
	if !p.authorizer.AuthEnvironManager() {
		return result, common.ErrPerm
	}
	watch := p.st.WatchEnvironMachines()
	// Consume the initial event and forward it to the result.
	if changes, ok := <-watch.Changes(); ok {
		result.StringsWatcherId = p.resources.Register(watch)
		result.Changes = changes
	} else {
		err := watcher.MustErr(watch)
		return result, fmt.Errorf("cannot obtain initial environment machines: %v", err)
	}
	return result, nil
}
<commit_msg>Changes after review<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package provisioner

import (
	"fmt"

	"launchpad.net/juju-core/constraints"
	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/names"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/params"
	"launchpad.net/juju-core/state/apiserver/common"
	"launchpad.net/juju-core/state/watcher"
)

// ProvisionerAPI provides access to the Provisioner API facade.
type ProvisionerAPI struct {
	*common.Remover
	*common.StatusSetter
	*common.DeadEnsurer
	*common.PasswordChanger
	*common.LifeGetter

	st          *state.State
	resources   *common.Resources
	authorizer  common.Authorizer
	getAuthFunc common.GetAuthFunc
}

// NewProvisionerAPI creates a new server-side ProvisionerAPI facade.
func NewProvisionerAPI(
	st *state.State,
	resources *common.Resources,
	authorizer common.Authorizer,
) (*ProvisionerAPI, error) {
	if !authorizer.AuthMachineAgent() && !authorizer.AuthEnvironManager() {
		return nil, common.ErrPerm
	}
	getAuthFunc := func() (common.AuthFunc, error) {
		isEnvironManager := authorizer.AuthEnvironManager()
		isMachineAgent := authorizer.AuthMachineAgent()
		authEntityTag := authorizer.GetAuthTag()

		return func(tag string) bool {
			if isMachineAgent && tag == authEntityTag {
				// A machine agent can always access its own machine.
				return true
			}
			_, id, err := names.ParseTag(tag, names.MachineTagKind)
			if err != nil {
				return false
			}
			parentId := state.ParentId(id)
			if parentId == "" {
				// All top-level machines are accessible by the
				// environment manager.
				return isEnvironManager
			}
			// All containers with the authenticated machine as a
			// parent are accessible by it.
			return isMachineAgent && names.MachineTag(parentId) == authEntityTag
		}, nil
	}
	return &ProvisionerAPI{
		Remover:         common.NewRemover(st, false, getAuthFunc),
		StatusSetter:    common.NewStatusSetter(st, getAuthFunc),
		DeadEnsurer:     common.NewDeadEnsurer(st, getAuthFunc),
		PasswordChanger: common.NewPasswordChanger(st, getAuthFunc),
		LifeGetter:      common.NewLifeGetter(st, getAuthFunc),
		st:              st,
		resources:       resources,
		authorizer:      authorizer,
		getAuthFunc:     getAuthFunc,
	}, nil
}

func (p *ProvisionerAPI) getMachine(canAccess common.AuthFunc, tag string) (*state.Machine, error) {
	if !canAccess(tag) {
		return nil, common.ErrPerm
	}
	entity, err := p.st.FindEntity(tag)
	if err != nil {
		return nil, err
	}
	// The authorization function guarantees that the tag represents a
	// machine.
	return entity.(*state.Machine), nil
}

func (p *ProvisionerAPI) watchOneMachineContainers(arg params.WatchContainer) (params.StringsWatchResult, error) {
	nothing := params.StringsWatchResult{}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return nothing, err
	}
	if !canAccess(arg.MachineTag) {
		return nothing, common.ErrPerm
	}
	_, id, err := names.ParseTag(arg.MachineTag, names.MachineTagKind)
	if err != nil {
		return nothing, err
	}
	machine, err := p.st.Machine(id)
	if err != nil {
		return nothing, err
	}
	watch := machine.WatchContainers(instance.ContainerType(arg.ContainerType))
	// Consume the initial event and forward it to the result.
	if changes, ok := <-watch.Changes(); ok {
		return params.StringsWatchResult{
			StringsWatcherId: p.resources.Register(watch),
			Changes:          changes,
		}, nil
	}
	return nothing, watcher.MustErr(watch)
}

// WatchContainers starts a StringsWatcher to watch all containers deployed to
// any machine passed in args.
func (p *ProvisionerAPI) WatchContainers(args params.WatchContainers) (params.StringsWatchResults, error) {
	result := params.StringsWatchResults{
		Results: make([]params.StringsWatchResult, len(args.Params)),
	}
	for i, arg := range args.Params {
		watcherResult, err := p.watchOneMachineContainers(arg)
		result.Results[i] = watcherResult
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// WatchForEnvironConfigChanges returns a NotifyWatcher to observe
// changes to the environment configuration.
func (p *ProvisionerAPI) WatchForEnvironConfigChanges() (params.NotifyWatchResult, error) {
	result := params.NotifyWatchResult{}
	watch := p.st.WatchForEnvironConfigChanges()
	// Consume the initial event. Technically, API
	// calls to Watch 'transmit' the initial event
	// in the Watch response. But NotifyWatchers
	// have no state to transmit.
	if _, ok := <-watch.Changes(); ok {
		result.NotifyWatcherId = p.resources.Register(watch)
	} else {
		return result, watcher.MustErr(watch)
	}
	return result, nil
}

// EnvironConfig returns the current environment's configuration.
func (p *ProvisionerAPI) EnvironConfig() (params.ConfigResult, error) {
	result := params.ConfigResult{}
	config, err := p.st.EnvironConfig()
	if err != nil {
		return result, err
	}
	result.Config = config.AllAttrs()
	return result, nil
}

// Status returns the status of each given machine entity.
func (p *ProvisionerAPI) Status(args params.Entities) (params.StatusResults, error) {
	result := params.StatusResults{
		Results: make([]params.StatusResult, len(args.Entities)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, entity := range args.Entities {
		machine, err := p.getMachine(canAccess, entity.Tag)
		if err == nil {
			r := &result.Results[i]
			r.Status, r.Info, err = machine.Status()
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// Series returns the deployed series for each given machine entity.
func (p *ProvisionerAPI) Series(args params.Entities) (params.StringResults, error) {
	result := params.StringResults{
		Results: make([]params.StringResult, len(args.Entities)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, entity := range args.Entities {
		machine, err := p.getMachine(canAccess, entity.Tag)
		if err == nil {
			result.Results[i].Result = machine.Series()
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// Constraints returns the constraints for each given machine entity.
func (p *ProvisionerAPI) Constraints(args params.Entities) (params.ConstraintsResults, error) {
	result := params.ConstraintsResults{
		Results: make([]params.ConstraintsResult, len(args.Entities)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, entity := range args.Entities {
		machine, err := p.getMachine(canAccess, entity.Tag)
		if err == nil {
			var cons constraints.Value
			cons, err = machine.Constraints()
			if err == nil {
				result.Results[i].Constraints = cons
			}
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// SetProvisioned sets the provider specific machine id, nonce and
// metadata for each given machine. Once set, the instance id cannot
// be changed.
func (p *ProvisionerAPI) SetProvisioned(args params.SetProvisioned) (params.ErrorResults, error) {
	result := params.ErrorResults{
		Results: make([]params.ErrorResult, len(args.Machines)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, arg := range args.Machines {
		machine, err := p.getMachine(canAccess, arg.Tag)
		if err == nil {
			err = machine.SetProvisioned(arg.InstanceId, arg.Nonce, arg.Characteristics)
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// InstanceId returns the provider specific instance id for each given
// machine or an CodeNotProvisioned error, if not set.
func (p *ProvisionerAPI) InstanceId(args params.Entities) (params.StringResults, error) {
	result := params.StringResults{
		Results: make([]params.StringResult, len(args.Entities)),
	}
	canAccess, err := p.getAuthFunc()
	if err != nil {
		return result, err
	}
	for i, entity := range args.Entities {
		machine, err := p.getMachine(canAccess, entity.Tag)
		if err == nil {
			var instanceId instance.Id
			instanceId, err = machine.InstanceId()
			if err == nil {
				result.Results[i].Result = string(instanceId)
			}
		}
		result.Results[i].Error = common.ServerError(err)
	}
	return result, nil
}

// WatchEnvironMachines returns a StringsWatcher that notifies of
// changes to the lifecycles of the machines (but not containers) in
// the current environment.
func (p *ProvisionerAPI) WatchEnvironMachines() (params.StringsWatchResult, error) {
	result := params.StringsWatchResult{}
	if !p.authorizer.AuthEnvironManager() {
		return result, common.ErrPerm
	}
	watch := p.st.WatchEnvironMachines()
	// Consume the initial event and forward it to the result.
	if changes, ok := <-watch.Changes(); ok {
		result.StringsWatcherId = p.resources.Register(watch)
		result.Changes = changes
	} else {
		err := watcher.MustErr(watch)
		return result, fmt.Errorf("cannot obtain initial environment machines: %v", err)
	}
	return result, nil
}
<|endoftext|>


<commit_before>package service

import (
	"context"
	"fmt"
	"net"
	"strconv"
	"time"

	"github.com/onsi/ginkgo"
	configv1 "github.com/openshift/api/config/v1"
	configclient "github.com/openshift/client-go/config/clientset/versioned"
	"github.com/openshift/origin/pkg/monitor/backenddisruption"
	"github.com/openshift/origin/test/extended/util/disruption"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	"k8s.io/kubernetes/test/e2e/framework"
	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
	"k8s.io/kubernetes/test/e2e/framework/service"
	"k8s.io/kubernetes/test/e2e/upgrades"
)

// serviceLoadBalancerUpgradeTest tests that a service is available before, during, and
// after a cluster upgrade.
type serviceLoadBalancerUpgradeTest struct {
	// filled in by pre-setup
	jig                 *service.TestJig
	tcpService          *v1.Service
	unsupportedPlatform bool
	hostGetter          *backenddisruption.SimpleHostGetter

	backendDisruptionTest disruption.BackendDisruptionUpgradeTest
}

func NewServiceLoadBalancerWithNewConnectionsTest() upgrades.Test {
	serviceLBTest := &serviceLoadBalancerUpgradeTest{
		hostGetter: backenddisruption.NewSimpleHostGetter(""), // late binding host
	}
	serviceLBTest.backendDisruptionTest =
		disruption.NewBackendDisruptionTest(
			"[sig-network-edge] Application behind service load balancer with PDB remains available using new connections",
			backenddisruption.NewBackend(
				serviceLBTest.hostGetter,
				"service-load-balancer-with-pdb",
				"/echo?msg=Hello",
				backenddisruption.NewConnectionType).
				WithExpectedBody("Hello"),
		).
			WithPreSetup(serviceLBTest.loadBalancerSetup)

	return serviceLBTest
}

func NewServiceLoadBalancerWithReusedConnectionsTest() upgrades.Test {
	serviceLBTest := &serviceLoadBalancerUpgradeTest{
		hostGetter: backenddisruption.NewSimpleHostGetter(""), // late binding host
	}
	serviceLBTest.backendDisruptionTest =
		disruption.NewBackendDisruptionTest(
			"[sig-network-edge] Application behind service load balancer with PDB remains available using reused connections",
			backenddisruption.NewBackend(
				serviceLBTest.hostGetter,
				"service-load-balancer-with-pdb",
				"/echo?msg=Hello",
				backenddisruption.ReusedConnectionType).
				WithExpectedBody("Hello"),
		).
			WithPreSetup(serviceLBTest.loadBalancerSetup)

	return serviceLBTest
}

func (t *serviceLoadBalancerUpgradeTest) Name() string { return t.backendDisruptionTest.Name() }
func (t *serviceLoadBalancerUpgradeTest) DisplayName() string {
	return t.backendDisruptionTest.DisplayName()
}

// RequiresKubeNamespace indicates we get an e2e-k8s- namespace so we can bind low ports.
func (t *serviceLoadBalancerUpgradeTest) RequiresKubeNamespace() bool {
	return true
}

func shouldTestPDBs() bool { return true }

func (t *serviceLoadBalancerUpgradeTest) loadBalancerSetup(f *framework.Framework, backendSampler disruption.BackendSampler) error {
	// we must update our namespace to bypass SCC so that we can avoid default mutation of our pod and SCC evaluation.
	// technically we could also choose to bind an SCC, but I don't see a lot of value in doing that and we have to wait
	// for a secondary cache to fill to reflect that.  If we miss that cache filling, we'll get assigned a restricted on
	// and fail.
	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		ns, err := f.ClientSet.CoreV1().Namespaces().Get(context.Background(), f.Namespace.Name, metav1.GetOptions{})
		if err != nil {
			return err
		}
		if ns.Labels == nil {
			ns.Labels = map[string]string{}
		}
		ns.Labels["security.openshift.io/disable-securitycontextconstraints"] = "true"
		ns, err = f.ClientSet.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{})
		if err != nil {
			return err
		}
		return nil
	})
	framework.ExpectNoError(err)

	configClient, err := configclient.NewForConfig(f.ClientConfig())
	framework.ExpectNoError(err)
	infra, err := configClient.ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
	framework.ExpectNoError(err)
	// ovirt does not support service type loadbalancer because it doesn't program a cloud.
	if infra.Status.PlatformStatus.Type == configv1.OvirtPlatformType || infra.Status.PlatformStatus.Type == configv1.KubevirtPlatformType || infra.Status.PlatformStatus.Type == configv1.LibvirtPlatformType || infra.Status.PlatformStatus.Type == configv1.VSpherePlatformType || infra.Status.PlatformStatus.Type == configv1.BareMetalPlatformType {
		t.unsupportedPlatform = true
	}
	// single node clusters are not supported because the replication controller has 2 replicas with anti-affinity for running on the same node.
	if infra.Status.ControlPlaneTopology == configv1.SingleReplicaTopologyMode {
		t.unsupportedPlatform = true
	}
	if t.unsupportedPlatform {
		return nil
	}

	serviceName := "service-test"
	jig := service.NewTestJig(f.ClientSet, f.Namespace.Name, serviceName)

	ns := f.Namespace
	cs := f.ClientSet

	ginkgo.By("creating a TCP service " + serviceName + " with type=LoadBalancer in namespace " + ns.Name)
	tcpService, err := jig.CreateTCPService(func(s *v1.Service) {
		s.Spec.Type = v1.ServiceTypeLoadBalancer
		// ServiceExternalTrafficPolicyTypeCluster performs during disruption, Local does not
		s.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster
		if s.Annotations == nil {
			s.Annotations = make(map[string]string)
		}
		// We tune the LB checks to match the longest intervals available so that interactions between
		// upgrading components and the service are more obvious.
		// - AWS allows configuration, default is 70s (6 failed with 10s interval in 1.17) set to match GCP
		s.Annotations["service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval"] = "8"
		s.Annotations["service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold"] = "3"
		s.Annotations["service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold"] = "2"
		// - Azure is hardcoded to 15s (2 failed with 5s interval in 1.17) and is sufficient
		// - GCP has a non-configurable interval of 32s (3 failed health checks with 8s interval in 1.17)
		//   - thus pods need to stay up for > 32s, so pod shutdown period will will be 45s
	})
	framework.ExpectNoError(err)
	tcpService, err = jig.WaitForLoadBalancer(service.GetServiceLoadBalancerCreationTimeout(cs))
	framework.ExpectNoError(err)

	// Get info to hit it with
	tcpIngressIP := service.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
	svcPort := int(tcpService.Spec.Ports[0].Port)

	ginkgo.By("creating RC to be part of service " + serviceName)
	rc, err := jig.Run(func(rc *v1.ReplicationController) {
		// ensure the pod waits long enough during update for the LB to see the newly ready pod, which
		// must be longer than the worst load balancer above (GCP at 32s)
		rc.Spec.MinReadySeconds = 33
		// ensure the pod waits long enough for most LBs to take it out of rotation, which has to be
		// longer than the LB failed health check duration + 1 cycle
		rc.Spec.Template.Spec.Containers[0].Lifecycle = &v1.Lifecycle{
			PreStop: &v1.LifecycleHandler{
				Exec: &v1.ExecAction{Command: []string{"sleep", "45"}},
			},
		}
		// ensure the pod is not forcibly deleted at 30s, but waits longer than the graceful sleep
		minute := int64(60)
		rc.Spec.Template.Spec.TerminationGracePeriodSeconds = &minute

		jig.AddRCAntiAffinity(rc)
	})
	framework.ExpectNoError(err)

	if shouldTestPDBs() {
		ginkgo.By("creating a PodDisruptionBudget to cover the ReplicationController")
		_, err = jig.CreatePDB(rc)
		framework.ExpectNoError(err)
	}

	// Hit it once before considering ourselves ready
	ginkgo.By("hitting pods through the service's LoadBalancer")
	timeout := 10 * time.Minute
	// require thirty seconds of passing requests to continue (in case the SLB becomes available and then degrades)
	// TODO this seems weird to @deads2k, why is status not trustworthy
	TestReachableHTTPWithMinSuccessCount(tcpIngressIP, svcPort, 30, timeout)

	t.hostGetter.SetHost(fmt.Sprintf("http://%s", net.JoinHostPort(tcpIngressIP, strconv.Itoa(svcPort))))

	t.jig = jig
	t.tcpService = tcpService
	return nil
}

// Test runs a connectivity check to the service.
func (t *serviceLoadBalancerUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	if t.unsupportedPlatform {
		return
	}

	t.backendDisruptionTest.Test(f, done, upgrade)

	// verify finalizer behavior
	defer func() {
		ginkgo.By("Check that service can be deleted with finalizer")
		service.WaitForServiceDeletedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name)
	}()
	ginkgo.By("Check that finalizer is present on loadBalancer type service")
	service.WaitForServiceUpdatedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name, true)
}

func (t *serviceLoadBalancerUpgradeTest) Teardown(f *framework.Framework) {
	t.backendDisruptionTest.Teardown(f)
}

func (t *serviceLoadBalancerUpgradeTest) Setup(f *framework.Framework) {
	t.backendDisruptionTest.Setup(f)
}

// TestReachableHTTPWithMinSuccessCount tests that the given host serves HTTP on the given port for a minimum of successCount number of
// counts at a given interval. If the service reachability fails, the counter gets reset
func TestReachableHTTPWithMinSuccessCount(host string, port int, successCount int, timeout time.Duration) {
	consecutiveSuccessCnt := 0
	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
		result := e2enetwork.PokeHTTP(host, port, "/echo?msg=hello",
			&e2enetwork.HTTPPokeParams{
				BodyContains:   "hello",
				RetriableCodes: []int{},
			})
		if result.Status == e2enetwork.HTTPSuccess {
			consecutiveSuccessCnt++
			return consecutiveSuccessCnt >= successCount, nil
		}
		consecutiveSuccessCnt = 0
		return false, nil // caller can retry
	})
	framework.ExpectNoError(err)
}
<commit_msg>update the service load balancer test to work with readiness on pod delete<commit_after>package service

import (
	"context"
	"fmt"
	"net"
	"strconv"
	"time"

	"github.com/onsi/ginkgo"
	configv1 "github.com/openshift/api/config/v1"
	configclient "github.com/openshift/client-go/config/clientset/versioned"
	"github.com/openshift/origin/pkg/monitor/backenddisruption"
	"github.com/openshift/origin/test/extended/util/disruption"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/util/retry"
	"k8s.io/kubernetes/test/e2e/framework"
	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
	"k8s.io/kubernetes/test/e2e/framework/service"
	"k8s.io/kubernetes/test/e2e/upgrades"
)

// serviceLoadBalancerUpgradeTest tests that a service is available before, during, and
// after a cluster upgrade.
type serviceLoadBalancerUpgradeTest struct {
	// filled in by pre-setup
	jig                 *service.TestJig
	tcpService          *v1.Service
	unsupportedPlatform bool
	hostGetter          *backenddisruption.SimpleHostGetter

	backendDisruptionTest disruption.BackendDisruptionUpgradeTest
}

func NewServiceLoadBalancerWithNewConnectionsTest() upgrades.Test {
	serviceLBTest := &serviceLoadBalancerUpgradeTest{
		hostGetter: backenddisruption.NewSimpleHostGetter(""), // late binding host
	}
	serviceLBTest.backendDisruptionTest =
		disruption.NewBackendDisruptionTest(
			"[sig-network-edge] Application behind service load balancer with PDB remains available using new connections",
			backenddisruption.NewBackend(
				serviceLBTest.hostGetter,
				"service-load-balancer-with-pdb",
				"/echo?msg=Hello",
				backenddisruption.NewConnectionType).
				WithExpectedBody("Hello"),
		).
			WithPreSetup(serviceLBTest.loadBalancerSetup)

	return serviceLBTest
}

func NewServiceLoadBalancerWithReusedConnectionsTest() upgrades.Test {
	serviceLBTest := &serviceLoadBalancerUpgradeTest{
		hostGetter: backenddisruption.NewSimpleHostGetter(""), // late binding host
	}
	serviceLBTest.backendDisruptionTest =
		disruption.NewBackendDisruptionTest(
			"[sig-network-edge] Application behind service load balancer with PDB remains available using reused connections",
			backenddisruption.NewBackend(
				serviceLBTest.hostGetter,
				"service-load-balancer-with-pdb",
				"/echo?msg=Hello",
				backenddisruption.ReusedConnectionType).
				WithExpectedBody("Hello"),
		).
			WithPreSetup(serviceLBTest.loadBalancerSetup)

	return serviceLBTest
}

func (t *serviceLoadBalancerUpgradeTest) Name() string { return t.backendDisruptionTest.Name() }
func (t *serviceLoadBalancerUpgradeTest) DisplayName() string {
	return t.backendDisruptionTest.DisplayName()
}

// RequiresKubeNamespace indicates we get an e2e-k8s- namespace so we can bind low ports.
func (t *serviceLoadBalancerUpgradeTest) RequiresKubeNamespace() bool {
	return true
}

func shouldTestPDBs() bool { return true }

func (t *serviceLoadBalancerUpgradeTest) loadBalancerSetup(f *framework.Framework, backendSampler disruption.BackendSampler) error {
	// we must update our namespace to bypass SCC so that we can avoid default mutation of our pod and SCC evaluation.
	// technically we could also choose to bind an SCC, but I don't see a lot of value in doing that and we have to wait
	// for a secondary cache to fill to reflect that.  If we miss that cache filling, we'll get assigned a restricted on
	// and fail.
	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		ns, err := f.ClientSet.CoreV1().Namespaces().Get(context.Background(), f.Namespace.Name, metav1.GetOptions{})
		if err != nil {
			return err
		}
		if ns.Labels == nil {
			ns.Labels = map[string]string{}
		}
		ns.Labels["security.openshift.io/disable-securitycontextconstraints"] = "true"
		ns, err = f.ClientSet.CoreV1().Namespaces().Update(context.Background(), ns, metav1.UpdateOptions{})
		if err != nil {
			return err
		}
		return nil
	})
	framework.ExpectNoError(err)

	configClient, err := configclient.NewForConfig(f.ClientConfig())
	framework.ExpectNoError(err)
	infra, err := configClient.ConfigV1().Infrastructures().Get(context.Background(), "cluster", metav1.GetOptions{})
	framework.ExpectNoError(err)
	// ovirt does not support service type loadbalancer because it doesn't program a cloud.
	if infra.Status.PlatformStatus.Type == configv1.OvirtPlatformType || infra.Status.PlatformStatus.Type == configv1.KubevirtPlatformType || infra.Status.PlatformStatus.Type == configv1.LibvirtPlatformType || infra.Status.PlatformStatus.Type == configv1.VSpherePlatformType || infra.Status.PlatformStatus.Type == configv1.BareMetalPlatformType {
		t.unsupportedPlatform = true
	}
	// single node clusters are not supported because the replication controller has 2 replicas with anti-affinity for running on the same node.
	if infra.Status.ControlPlaneTopology == configv1.SingleReplicaTopologyMode {
		t.unsupportedPlatform = true
	}
	if t.unsupportedPlatform {
		return nil
	}

	serviceName := "service-test"
	jig := service.NewTestJig(f.ClientSet, f.Namespace.Name, serviceName)

	ns := f.Namespace
	cs := f.ClientSet

	ginkgo.By("creating a TCP service " + serviceName + " with type=LoadBalancer in namespace " + ns.Name)
	tcpService, err := jig.CreateTCPService(func(s *v1.Service) {
		s.Spec.Type = v1.ServiceTypeLoadBalancer
		// ServiceExternalTrafficPolicyTypeCluster performs during disruption, Local does not
		s.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster
		if s.Annotations == nil {
			s.Annotations = make(map[string]string)
		}
		// We tune the LB checks to match the longest intervals available so that interactions between
		// upgrading components and the service are more obvious.
		// - AWS allows configuration, default is 70s (6 failed with 10s interval in 1.17) set to match GCP
		s.Annotations["service.beta.kubernetes.io/aws-load-balancer-healthcheck-interval"] = "8"
		s.Annotations["service.beta.kubernetes.io/aws-load-balancer-healthcheck-unhealthy-threshold"] = "3"
		s.Annotations["service.beta.kubernetes.io/aws-load-balancer-healthcheck-healthy-threshold"] = "2"
		// - Azure is hardcoded to 15s (2 failed with 5s interval in 1.17) and is sufficient
		// - GCP has a non-configurable interval of 32s (3 failed health checks with 8s interval in 1.17)
		//   - thus pods need to stay up for > 32s, so pod shutdown period will will be 45s
	})
	framework.ExpectNoError(err)
	tcpService, err = jig.WaitForLoadBalancer(service.GetServiceLoadBalancerCreationTimeout(cs))
	framework.ExpectNoError(err)

	// Get info to hit it with
	tcpIngressIP := service.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
	svcPort := int(tcpService.Spec.Ports[0].Port)

	ginkgo.By("creating RC to be part of service " + serviceName)
	rc, err := jig.Run(func(rc *v1.ReplicationController) {
		// ensure the pod waits long enough during update for the LB to see the newly ready pod, which
		// must be longer than the worst load balancer above (GCP at 32s)
		rc.Spec.MinReadySeconds = 33

		// use a readiness endpoint that will go not ready before the pod terminates.
		// the probe will go false when the sig-term is sent.
		rc.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Path = "/readyz"

		// delay shutdown long enough to go readyz=false before the process exits when the pod is deleted.
		rc.Spec.Template.Spec.Containers[0].Args = append(rc.Spec.Template.Spec.Containers[0].Args, "--delay-shutdown=45")

		// ensure the pod is not forcibly deleted at 30s, but waits longer than the graceful sleep
		minute := int64(60)
		rc.Spec.Template.Spec.TerminationGracePeriodSeconds = &minute

		jig.AddRCAntiAffinity(rc)
	})
	framework.ExpectNoError(err)

	if shouldTestPDBs() {
		ginkgo.By("creating a PodDisruptionBudget to cover the ReplicationController")
		_, err = jig.CreatePDB(rc)
		framework.ExpectNoError(err)
	}

	// Hit it once before considering ourselves ready
	ginkgo.By("hitting pods through the service's LoadBalancer")
	timeout := 10 * time.Minute
	// require thirty seconds of passing requests to continue (in case the SLB becomes available and then degrades)
	// TODO this seems weird to @deads2k, why is status not trustworthy
	TestReachableHTTPWithMinSuccessCount(tcpIngressIP, svcPort, 30, timeout)

	t.hostGetter.SetHost(fmt.Sprintf("http://%s", net.JoinHostPort(tcpIngressIP, strconv.Itoa(svcPort))))

	t.jig = jig
	t.tcpService = tcpService
	return nil
}

// Test runs a connectivity check to the service.
func (t *serviceLoadBalancerUpgradeTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	if t.unsupportedPlatform {
		return
	}

	t.backendDisruptionTest.Test(f, done, upgrade)

	// verify finalizer behavior
	defer func() {
		ginkgo.By("Check that service can be deleted with finalizer")
		service.WaitForServiceDeletedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name)
	}()
	ginkgo.By("Check that finalizer is present on loadBalancer type service")
	service.WaitForServiceUpdatedWithFinalizer(t.jig.Client, t.tcpService.Namespace, t.tcpService.Name, true)
}

func (t *serviceLoadBalancerUpgradeTest) Teardown(f *framework.Framework) {
	t.backendDisruptionTest.Teardown(f)
}

func (t *serviceLoadBalancerUpgradeTest) Setup(f *framework.Framework) {
	t.backendDisruptionTest.Setup(f)
}

// TestReachableHTTPWithMinSuccessCount tests that the given host serves HTTP on the given port for a minimum of successCount number of
// counts at a given interval. If the service reachability fails, the counter gets reset
func TestReachableHTTPWithMinSuccessCount(host string, port int, successCount int, timeout time.Duration) {
	consecutiveSuccessCnt := 0
	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
		result := e2enetwork.PokeHTTP(host, port, "/echo?msg=hello",
			&e2enetwork.HTTPPokeParams{
				BodyContains:   "hello",
				RetriableCodes: []int{},
			})
		if result.Status == e2enetwork.HTTPSuccess {
			consecutiveSuccessCnt++
			return consecutiveSuccessCnt >= successCount, nil
		}
		consecutiveSuccessCnt = 0
		return false, nil // caller can retry
	})
	framework.ExpectNoError(err)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/zipfs"
	"io"
	"log"
	"os"
	"os/exec"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

var _ = log.Printf

func main() {
	// Scans the arg list and sets up flags
	debug := flag.Bool("debug", false, "print debugging messages.")
	latencies := flag.Bool("latencies", false, "record operation latencies.")
	profile := flag.String("profile", "", "record cpu profile.")
	mem_profile := flag.String("mem-profile", "", "record memory profile.")
	command := flag.String("run", "", "run this command after mounting.")
	ttl := flag.Float64("ttl", 1.0, "attribute/entry cache TTL.")
	flag.Parse()
	if flag.NArg() < 2 {
		fmt.Fprintf(os.Stderr, "usage: %s MOUNTPOINT ZIP-FILE\n", os.Args[0])
		os.Exit(2)
	}

	var profFile, memProfFile io.Writer
	var err error
	if *profile != "" {
		profFile, err = os.Create(*profile)
		if err != nil {
			log.Fatalf("os.Create: %v", err)
		}
	}
	if *mem_profile != "" {
		memProfFile, err = os.Create(*mem_profile)
		if err != nil {
			log.Fatalf("os.Create: %v", err)
		}
	}

	var fs fuse.NodeFileSystem
	fs, err = zipfs.NewArchiveFileSystem(flag.Arg(1))
	if err != nil {
		fmt.Fprintf(os.Stderr, "NewArchiveFileSystem failed: %v\n", err)
		os.Exit(1)
	}

	opts := &fuse.FileSystemOptions{
		AttrTimeout: time.Duration(*ttl * float64(time.Second)),
		EntryTimeout: time.Duration(*ttl * float64(time.Second)),
	}
	state, _, err := fuse.MountNodeFileSystem(flag.Arg(0), fs, opts)
	if err != nil {
		fmt.Printf("Mount fail: %v\n", err)
		os.Exit(1)
	}

	state.SetRecordStatistics(*latencies)
	state.Debug = *debug
	runtime.GC()
	if profFile != nil {
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if *command != "" {
		args := strings.Split(*command, " ")
		cmd := exec.Command(args[0], args[1:]...)
		cmd.Start()
	}

	state.Loop()
	if memProfFile != nil {
		pprof.WriteHeapProfile(memProfFile)
	}
}
<commit_msg>Pass Stdout to zipfs subcommand.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/zipfs"
	"io"
	"log"
	"os"
	"os/exec"
	"runtime"
	"runtime/pprof"
	"strings"
	"time"
)

var _ = log.Printf

func main() {
	// Scans the arg list and sets up flags
	debug := flag.Bool("debug", false, "print debugging messages.")
	latencies := flag.Bool("latencies", false, "record operation latencies.")
	profile := flag.String("profile", "", "record cpu profile.")
	mem_profile := flag.String("mem-profile", "", "record memory profile.")
	command := flag.String("run", "", "run this command after mounting.")
	ttl := flag.Float64("ttl", 1.0, "attribute/entry cache TTL.")
	flag.Parse()
	if flag.NArg() < 2 {
		fmt.Fprintf(os.Stderr, "usage: %s MOUNTPOINT ZIP-FILE\n", os.Args[0])
		os.Exit(2)
	}

	var profFile, memProfFile io.Writer
	var err error
	if *profile != "" {
		profFile, err = os.Create(*profile)
		if err != nil {
			log.Fatalf("os.Create: %v", err)
		}
	}
	if *mem_profile != "" {
		memProfFile, err = os.Create(*mem_profile)
		if err != nil {
			log.Fatalf("os.Create: %v", err)
		}
	}

	var fs fuse.NodeFileSystem
	fs, err = zipfs.NewArchiveFileSystem(flag.Arg(1))
	if err != nil {
		fmt.Fprintf(os.Stderr, "NewArchiveFileSystem failed: %v\n", err)
		os.Exit(1)
	}

	opts := &fuse.FileSystemOptions{
		AttrTimeout: time.Duration(*ttl * float64(time.Second)),
		EntryTimeout: time.Duration(*ttl * float64(time.Second)),
	}
	state, _, err := fuse.MountNodeFileSystem(flag.Arg(0), fs, opts)
	if err != nil {
		fmt.Printf("Mount fail: %v\n", err)
		os.Exit(1)
	}

	state.SetRecordStatistics(*latencies)
	state.Debug = *debug
	runtime.GC()
	if profFile != nil {
		pprof.StartCPUProfile(profFile)
		defer pprof.StopCPUProfile()
	}

	if *command != "" {
		args := strings.Split(*command, " ")
		cmd := exec.Command(args[0], args[1:]...)
		cmd.Stdout = os.Stdout
		cmd.Start()
	}

	state.Loop()
	if memProfFile != nil {
		pprof.WriteHeapProfile(memProfFile)
	}
}
<|endoftext|>