<commit_before>package main

import (
	"strings"
	"time"

	"github.com/docker/docker/pkg/integration/checker"
	"github.com/go-check/check"
)

func (s *DockerSuite) TestUpdateRestartPolicy(c *check.C) {
	out, _ := dockerCmd(c, "run", "-d", "--restart=on-failure:3", "busybox", "sh", "-c", "sleep 1 && false")
	timeout := 60 * time.Second
	if daemonPlatform == "windows" {
		timeout = 150 * time.Second
	}

	id := strings.TrimSpace(string(out))

	// update restart policy to on-failure:5
	dockerCmd(c, "update", "--restart=on-failure:5", id)

	err := waitExited(id, timeout)
	c.Assert(err, checker.IsNil)

	count := inspectField(c, id, "RestartCount")
	c.Assert(count, checker.Equals, "5")

	maximumRetryCount := inspectField(c, id, "HostConfig.RestartPolicy.MaximumRetryCount")
	c.Assert(maximumRetryCount, checker.Equals, "5")
}
<commit_msg>Fix flaky TestUpdateRestartPolicy on Windows<commit_after>package main

import (
	"strings"
	"time"

	"github.com/docker/docker/pkg/integration/checker"
	"github.com/go-check/check"
)

func (s *DockerSuite) TestUpdateRestartPolicy(c *check.C) {
	out, _ := dockerCmd(c, "run", "-d", "--restart=on-failure:3", "busybox", "sh", "-c", "sleep 1 && false")
	timeout := 60 * time.Second
	if daemonPlatform == "windows" {
		timeout = 180 * time.Second
	}

	id := strings.TrimSpace(string(out))

	// update restart policy to on-failure:5
	dockerCmd(c, "update", "--restart=on-failure:5", id)

	err := waitExited(id, timeout)
	c.Assert(err, checker.IsNil)

	count := inspectField(c, id, "RestartCount")
	c.Assert(count, checker.Equals, "5")

	maximumRetryCount := inspectField(c, id, "HostConfig.RestartPolicy.MaximumRetryCount")
	c.Assert(maximumRetryCount, checker.Equals, "5")
}
<|endoftext|>


<commit_before>package externalservices

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"poliskarta/structs"
	"sync"
)

func CallPoliceScraping(policeEvent *structs.PoliceEvent, wg *sync.WaitGroup) {
	scrapeURL := "https://api.import.io/store/data/3c3e1355-d3c9-4047-bd2e-f86d36af29dc/_query?input/webpage/url="
	apikey := "&_user=***REMOVED***&_apikey=***REMOVED***"

	httpResponse, httperr := http.Get(scrapeURL + policeEvent.PoliceEventURL + apikey)

	if httperr != nil {
		fmt.Println("Importio http-error: " + httperr.Error())
		policeEvent.DescriptionLong = "<N/A>"
	} else {
		defer httpResponse.Body.Close()
		body, ioerr := ioutil.ReadAll(httpResponse.Body)

		if ioerr != nil {
			fmt.Println("Ioutilreadallerror: ", ioerr.Error())
			policeEvent.DescriptionLong = "<N/A>"
		} else {
			var scrapedEvents ScrapedEvents
			unmarshErr := json.Unmarshal(body, &scrapedEvents)

			//For unknown reasons, unmarshal fails some times, might be that the response from
			//police scraping is wrong (200OK instead of a real http-error)
			if unmarshErr != nil {
				fmt.Println("Unmarshal error after police scraping (import.io): " + unmarshErr.Error())
				policeEvent.DescriptionLong = "<N/A>"
			} else {
				//Everything was fine, set description
				policeEvent.DescriptionLong = scrapedEvents.Results[0].Result
			}
		}
	}

	defer wg.Done()
}

type ScrapedEvents struct {
	Results []ScrapedEvent `json:"results"`
}
type ScrapedEvent struct {
	Result string `json:"result"`
}
<commit_msg>Improved reliability of policescraping<commit_after>package externalservices

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"poliskarta/structs"
	"sync"
)

func CallPoliceScraping(policeEvent *structs.PoliceEvent, wg *sync.WaitGroup) {
	scrapeURL := "https://api.import.io/store/data/3c3e1355-d3c9-4047-bd2e-f86d36af29dc/_query?input/webpage/url="
	apikey := "&_user=***REMOVED***&_apikey=***REMOVED***"

	httpResponse, httperr := http.Get(scrapeURL + policeEvent.PoliceEventURL + apikey)

	if httperr != nil {
		fmt.Println("Importio http-error: " + httperr.Error())
		policeEvent.DescriptionLong = "<N/A>"
	} else {
		defer httpResponse.Body.Close()
		body, ioerr := ioutil.ReadAll(httpResponse.Body)

		if ioerr != nil {
			fmt.Println("Ioutilreadallerror: ", ioerr.Error())
			policeEvent.DescriptionLong = "<N/A>"
		} else {
			var scrapedEvents ScrapedEvents
			unmarshErr := json.Unmarshal(body, &scrapedEvents)

			//For unknown reasons, unmarshal fails some times, might be that the response from
			//police scraping is wrong (200OK instead of a real http-error)
			if unmarshErr != nil {
				fmt.Println("Unmarshal error after police scraping (import.io): " + unmarshErr.Error())
				policeEvent.DescriptionLong = "<N/A>"
			} else {
				//We dont know why, but even though everything seems fine here,
				//policeEvent.DescriptionLong = scrapedEvents.Results[0].Result,
				//sometimes crashes the server. Below is a safety measure.
				for _, result := range scrapedEvents.Results {
					policeEvent.DescriptionLong = result.Result
					break
				}

			}
		}
	}

	defer wg.Done()
}

type ScrapedEvents struct {
	Results []ScrapedEvent `json:"results"`
}
type ScrapedEvent struct {
	Result string `json:"result"`
}
<|endoftext|>


<commit_before>package radius

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"strconv"
)

//
//定义radius中各结构的方法
//

//methods of Radius_Code
func (i R_Code) String() (s string) {
	switch i {
	case CodeAccessRequest:
		return "CodeAccessRequest(1)"
	case CodeAccessAccept:
		return "CodeAccessAccept(2)"
	case CodeAccessReject:
		return "CodeAccessReject(3)"
	case CodeAccountingRequest:
		return "CodeAccountingRequest(4)"
	case CodeAccountingRespons:
		return "CodeAccountingRespons(5)"
	case CodeAccessChallenge:
		return "CodeAccessChallenge(11)"
	case CodeStatusServer:
		return "CodeStatusServer(12)"
	case CodeStatusClient:
		return "CodeStatusClient(13)"
	case CodeReserved:
		return "CodeReserved(255)"
	}
	return ERR_CODE_WRONG.Error() + ":(" + strconv.Itoa(int(i)) + ")"
}

//
func (r *R_Code) readFromBuff(buf *bytes.Buffer) error {
	b, err := buf.ReadByte()
	if err != nil {
		return ERR_RADIUS_FMT
	}
	i := R_Code(b)
	if i < 6 || (i >= 11 && i <= 13) || i == 255 {
		*r = i
		return nil
	}
	return ERR_CODE_WRONG
}

//
func (r R_Code) Judge(judge bool) (R_Code, error) {
	switch r {
	case CodeAccessRequest:
		if judge {
			return CodeAccessAccept, nil
		}
		return CodeAccessReject, nil
	case CodeAccountingRequest:
		return CodeAccountingRespons, nil
	}
	return CodeAccessReject, ERR_NOTSUPPORT
}

//
func (r R_Code) IsSupported() bool {
	if r == CodeAccessRequest || r == CodeAccessAccept || r == CodeAccessReject || r == CodeAccountingRequest || r == CodeAccountingRespons {
		return true
	}
	return false
}

//
func (r R_Code) IsRequest() bool {
	if r == CodeAccessRequest || r == CodeAccountingRequest {
		return true
	}
	return false
}

//
func (r R_Code) IsRespons() bool {
	if r == CodeAccessAccept || r == CodeAccessReject || r == CodeAccountingRespons {
		return true
	}
	return false
}

//methods of R_Id
func (i R_Id) String() string {
	return fmt.Sprintf("Id(%d)", i)
}

//
func (r *R_Id) readFromBuff(buf *bytes.Buffer) error {
	b, err := buf.ReadByte()
	if err != nil {
		return ERR_RADIUS_FMT
	}
	i := R_Id(b)
	*r = i
	return nil
}

//methods of R_Length
func (l R_Length) String() string {
	return fmt.Sprintf("Length(%d)", l)
}

//
func (r R_Length) isValidLenth() bool {
	if r >= radiusLength_MIN || r <= radiusLength_MAX {
		return true
	}
	return false
}

//
func (r *Radius) checkLengthWithBuff(buf *bytes.Buffer) bool {
	l := R_Length(buf.Len())
	if r.R_Length == l {
		return true
	}
	return false
}

//
func (r *R_Length) readFromBuff(buf *bytes.Buffer) error {
	var b1, b2 byte
	var err1, err2 error
	b1, err1 = buf.ReadByte()
	b2, err2 = buf.ReadByte()
	if err1 != nil || err2 != nil {
		return ERR_LEN_INVALID
	}
	l := R_Length(b1<<8) + R_Length(b2)
	if l.isValidLenth() {
		*r = l
		return nil
	}
	return ERR_LEN_INVALID
}

//methods of R_Authenticator
func (a R_Authenticator) String() string {
	return fmt.Sprintf("Authenticator %v", []byte(a))
}

//
func (r *R_Authenticator) readFromBuff(buf *bytes.Buffer) error {
	b := buf.Next(Radius_Authenticator_LEN)
	*r = b
	return nil
}

//

//methosd of Attributes maps of Id or Name

//methods of Radus
func (r *Radius) String() string {
	return r.R_Code.String() + "\n" +
		r.R_Id.String() + "\n" +
		r.R_Length.String() + "\n" +
		r.R_Authenticator.String() + "\n" +
		r.AttributeList.String()
}

//
func (r *Radius) ReadFromBuffer(buf *bytes.Buffer) error {
	err := r.R_Code.readFromBuff(buf)
	if err != nil {
		return errors.New("Format wrong on Code")
	}
	err = r.R_Id.readFromBuff(buf)
	if err != nil {
		return errors.New("Format wrong on Id")
	}

	err = r.R_Length.readFromBuff(buf)
	if err != nil {
		return errors.New("Format wrong on Length")
	}

	err = r.R_Authenticator.readFromBuff(buf)
	if err != nil {
		return errors.New("Format wrong on Authenticator")
	}
	for {
		v, err := readAttribute(buf)
		if isEOF(err) {
			break
		}
		if err != nil {
			return err
		}
		r.AttributeList.AddAttr(v)
	}
	return nil
}

//
func (r *Radius) WriteToBuff(buf *bytes.Buffer) {
	buf.WriteByte(byte(r.R_Code))
	buf.WriteByte(byte(r.R_Id))
	binary.Write(buf, binary.BigEndian, r.R_Length)
	buf.Write([]byte(r.R_Authenticator))
	for _, v := range r.AttributeList.attributes {
		v.writeBuffer(buf)
	}
}

//
func (r *Radius) GetLength() R_Length {
	var l R_Length
	l = 20
	for _, v := range r.AttributeList.attributes {
		switch v.AttributeId.(type) {
		case AttId:
			l += R_Length(v.AttributeValue.ValueLen() + 2)
		case AttIdV:
			if v.AttributeId.(AttIdV).VendorTypestring() == "IETF" {
				l += R_Length(v.AttributeValue.ValueLen() + 8)
			}
			if v.AttributeId.(AttIdV).VendorTypestring() == "TYPE4" {
				l += R_Length(v.AttributeValue.ValueLen() + 10)
			}
		}
	}
	return l
}

//
func (r *Radius) setLength() {
	r.R_Length = r.GetLength()
}

//
func (r *Radius) setAuthenticator() {

}

//
func (r *Radius) Finished() {
	r.setLength()
	r.setAuthenticator()
}
<commit_msg>调整一些函数和方法<commit_after>package radius

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"strconv"
)

//
//定义radius中各结构的方法
//

//methods of Radius_Code
func (i R_Code) String() (s string) {
	switch i {
	case CodeAccessRequest:
		return "CodeAccessRequest(1)"
	case CodeAccessAccept:
		return "CodeAccessAccept(2)"
	case CodeAccessReject:
		return "CodeAccessReject(3)"
	case CodeAccountingRequest:
		return "CodeAccountingRequest(4)"
	case CodeAccountingRespons:
		return "CodeAccountingRespons(5)"
	case CodeAccessChallenge:
		return "CodeAccessChallenge(11)"
	case CodeStatusServer:
		return "CodeStatusServer(12)"
	case CodeStatusClient:
		return "CodeStatusClient(13)"
	case CodeReserved:
		return "CodeReserved(255)"
	}
	return ERR_CODE_WRONG.Error() + ":(" + strconv.Itoa(int(i)) + ")"
}

//从buf填充Code
func (r *R_Code) readFromBuff(buf *bytes.Buffer) error {
	b, err := buf.ReadByte()
	if err != nil {
		return ERR_RADIUS_FMT
	}
	i := R_Code(b)
	if i.IsSupported() {
		*r = i
		return nil
	}
	return ERR_CODE_WRONG
}

//Judge判断响应报文的Code
func (r R_Code) Judge(judge bool) (R_Code, error) {
	switch r {
	case CodeAccessRequest:
		if judge {
			return CodeAccessAccept, nil
		}
		return CodeAccessReject, nil
	case CodeAccountingRequest:
		return CodeAccountingRespons, nil
	}
	return CodeAccessReject, ERR_NOTSUPPORT
}

//判断是否是支持的Code
func (r R_Code) IsSupported() bool {
	if r == CodeAccessRequest || r == CodeAccessAccept || r == CodeAccessReject || r == CodeAccountingRequest || r == CodeAccountingRespons {
		return true
	}
	return false
}

//判断是否是请求报文
func (r R_Code) IsRequest() bool {
	if r == CodeAccessRequest || r == CodeAccountingRequest {
		return true
	}
	return false
}

//判断是否是响应报文
func (r R_Code) IsRespons() bool {
	if r == CodeAccessAccept || r == CodeAccessReject || r == CodeAccountingRespons {
		return true
	}
	return false
}

//methods of R_Id
func (i R_Id) String() string {
	return fmt.Sprintf("Id(%d)", i)
}

//从buf填充Id
func (r *R_Id) readFromBuff(buf *bytes.Buffer) error {
	b, err := buf.ReadByte()
	if err != nil {
		return ERR_RADIUS_FMT
	}
	i := R_Id(b)
	*r = i
	return nil
}

//methods of R_Length
func (l R_Length) String() string {
	return fmt.Sprintf("Length(%d)", l)
}

//判断是否是有效的radius长度
func (r R_Length) IsValidLenth() bool {
	if r >= R_Length_MIN || r <= R_Length_MAX {
		return true
	}
	return false
}

//checkLengthWithBuff判断buf长度和radius Length是否相等
func (r *Radius) checkLengthWithBuff(buf *bytes.Buffer) bool {
	l := R_Length(buf.Len())
	if r.R_Length == l {
		return true
	}
	return false
}

//从buf填充radius Length
func (r *R_Length) readFromBuff(buf *bytes.Buffer) error {
	var b1, b2 byte
	var err1, err2 error
	b1, err1 = buf.ReadByte()
	b2, err2 = buf.ReadByte()
	if err1 != nil || err2 != nil {
		return ERR_LEN_INVALID
	}
	l := R_Length(b1<<8) + R_Length(b2)
	if l.IsValidLenth() && buf.Len() >= int(l) { //不允许buf长度小于radius长度，但是大于可以
		*r = l
		return nil
	}
	return ERR_LEN_INVALID
}

//methods of R_Authenticator
func (a R_Authenticator) String() string {
	return fmt.Sprintf("Authenticator %v", []byte(a))
}

//从buf填充Authenticator
func (r *R_Authenticator) readFromBuff(buf *bytes.Buffer) error {
	b := buf.Next(R_Authenticator_LEN)
	*r = b
	return nil
}

//methods of Radus
func (r *Radius) String() string {
	return r.R_Code.String() + "\n" +
		r.R_Id.String() + "\n" +
		r.R_Length.String() + "\n" +
		r.R_Authenticator.String() + "\n" +
		r.AttributeList.String()
}

//ReadFromBuffer从buf填充radius结构
func (r *Radius) ReadFromBuffer(buf *bytes.Buffer) error {
	err := r.R_Code.readFromBuff(buf)
	if err != nil {
		return errors.New("Format wrong on Code")
	}

	err = r.R_Id.readFromBuff(buf)
	if err != nil {
		return errors.New("Format wrong on Id")
	}

	err = r.R_Length.readFromBuff(buf)
	if err != nil {
		return errors.New("Format wrong on Length")
	}

	err = r.R_Authenticator.readFromBuff(buf)
	if err != nil {
		return errors.New("Format wrong on Authenticator")
	}
	for {
		v, err := readAttribute(buf)
		if isEOF(err) {
			break
		}
		if err != nil {
			return err
		}
		r.AttributeList.AddAttr(v)
	}
	if r.GetLength() != r.R_Length {
		return ERR_OTHER
	}
	return nil
}

//WriteToBuff将radius结构字节化写入buf
func (r *Radius) WriteToBuff(buf *bytes.Buffer) {
	buf.WriteByte(byte(r.R_Code))
	buf.WriteByte(byte(r.R_Id))
	binary.Write(buf, binary.BigEndian, r.R_Length)
	buf.Write([]byte(r.R_Authenticator))
	for _, v := range r.AttributeList.attributes {
		v.writeBuffer(buf)
	}
}

//GetLength获取radius结构字节化后的长度
func (r *Radius) GetLength() R_Length {
	var l R_Length
	l = 20
	for _, v := range r.AttributeList.attributes {
		switch v.AttributeId.(type) {
		case AttId:
			l += R_Length(v.AttributeValue.ValueLen() + 2)
		case AttIdV:
			if v.AttributeId.(AttIdV).VendorTypestring() == "IETF" {
				l += R_Length(v.AttributeValue.ValueLen() + 8)
			}
			if v.AttributeId.(AttIdV).VendorTypestring() == "TYPE4" {
				l += R_Length(v.AttributeValue.ValueLen() + 10)
			}
		}
	}
	return l
}
<|endoftext|>


<commit_before>//go:build ceph_preview
// +build ceph_preview

package rados

/*
#cgo LDFLAGS: -lrados
#include <stdlib.h>
#include <rados/librados.h>
extern void watchNotifyCb(void*, uint64_t, uint64_t, uint64_t, void*, size_t);
extern void watchErrorCb(void*, uint64_t, int);
*/
import "C"

import (
	"encoding/binary"
	"fmt"
	"math"
	"sync"
	"time"
	"unsafe"
)

type (
	// WatcherID is the unique id of a Watcher.
	WatcherID uint64
	// NotifyID is the unique id of a NotifyEvent.
	NotifyID uint64
	// NotifierID is the unique id of a notifying client.
	NotifierID uint64
)

// NotifyEvent is received by a watcher for each notification.
type NotifyEvent struct {
	ID         NotifyID
	WatcherID  WatcherID
	NotifierID NotifierID
	Data       []byte
}

// NotifyAck represents an acknowleged notification.
type NotifyAck struct {
	WatcherID  WatcherID
	NotifierID NotifierID
	Response   []byte
}

// NotifyTimeout represents an unacknowleged notification.
type NotifyTimeout struct {
	WatcherID  WatcherID
	NotifierID NotifierID
}

// Watcher receives all notifications for certain object.
type Watcher struct {
	id     WatcherID
	oid    string
	ioctx  *IOContext
	events chan NotifyEvent
	errors chan error
	done   chan struct{}
}

var (
	watchers    = map[WatcherID]*Watcher{}
	watchersMtx sync.RWMutex
)

// Watch creates a Watcher for the specified object.
//  PREVIEW
//
// A Watcher receives all notifications that are sent to the object on which it
// has been created. It exposes two read-only channels: Events() receives all
// the NotifyEvents and Errors() receives all occuring errors. A typical code
// creating a Watcher could look like this:
//
//  watcher, err := ioctx.Watch(oid)
//  go func() { // event handler
//    for ne := range watcher.Events() {
//      ...
//      ne.Ack([]byte("response data..."))
//      ...
//    }
//  }()
//  go func() { // error handler
//    for err := range watcher.Errors() {
//      ... handle err ...
//    }
//  }()
//
// CAUTION: the Watcher references the IOContext in which it has been created.
// Therefore all watchers must be deleted with the Delete() method before the
// IOContext is being destroyed.
//
// Implements:
//  int rados_watch2(rados_ioctx_t io, const char* o, uint64_t* cookie,
//    rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, void* arg)
func (ioctx *IOContext) Watch(obj string) (*Watcher, error) {
	return ioctx.WatchWithTimeout(obj, 0)
}

// WatchWithTimeout creates a watcher on an object. Same as Watcher(), but
// different timeout than the default can be specified.
//  PREVIEW
//
// Implements:
//  int rados_watch3(rados_ioctx_t io, const char *o, uint64_t *cookie,
// 	  rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, uint32_t timeout,
// 	  void *arg);
func (ioctx *IOContext) WatchWithTimeout(oid string, timeout time.Duration) (*Watcher, error) {
	cObj := C.CString(oid)
	defer C.free(unsafe.Pointer(cObj))
	var id C.uint64_t
	watchersMtx.Lock()
	defer watchersMtx.Unlock()
	ret := C.rados_watch3(
		ioctx.ioctx,
		cObj,
		&id,
		(C.rados_watchcb2_t)(C.watchNotifyCb),
		(C.rados_watcherrcb_t)(C.watchErrorCb),
		C.uint32_t(timeout.Milliseconds()/1000),
		nil,
	)
	if err := getError(ret); err != nil {
		return nil, err
	}
	evCh := make(chan NotifyEvent)
	errCh := make(chan error)
	w := &Watcher{
		id:     WatcherID(id),
		ioctx:  ioctx,
		oid:    oid,
		events: evCh,
		errors: errCh,
		done:   make(chan struct{}),
	}
	watchers[WatcherID(id)] = w
	return w, nil
}

// ID returns the WatcherId of the Watcher
//  PREVIEW
func (w *Watcher) ID() WatcherID {
	return w.id
}

// Events returns a read-only channel, that receives all notifications that are
// sent to the object of the Watcher.
//  PREVIEW
func (w *Watcher) Events() <-chan NotifyEvent {
	return w.events
}

// Errors returns a read-only channel, that receives all errors for the Watcher.
//  PREVIEW
func (w *Watcher) Errors() <-chan error {
	return w.errors
}

// Check on the status of a Watcher.
//  PREVIEW
//
// Returns the time since it was last confirmed. If there is an error, the
// Watcher is no longer valid, and should be destroyed with the Delete() method.
//
// Implements:
//  int rados_watch_check(rados_ioctx_t io, uint64_t cookie)
func (w *Watcher) Check() (time.Duration, error) {
	ret := C.rados_watch_check(w.ioctx.ioctx, C.uint64_t(w.id))
	if ret < 0 {
		return 0, getError(ret)
	}
	return time.Millisecond * time.Duration(ret), nil
}

// Delete the watcher. This closes both the event and error channel.
//  PREVIEW
//
// Implements:
//  int rados_unwatch2(rados_ioctx_t io, uint64_t cookie)
func (w *Watcher) Delete() error {
	watchersMtx.Lock()
	_, ok := watchers[w.id]
	if ok {
		delete(watchers, w.id)
	}
	watchersMtx.Unlock()
	if !ok {
		return nil
	}
	ret := C.rados_unwatch2(w.ioctx.ioctx, C.uint64_t(w.id))
	if ret != 0 {
		return getError(ret)
	}
	close(w.done) // unblock blocked callbacks
	close(w.events)
	close(w.errors)
	return nil
}

// Notify sends a notification with the provided data to all Watchers of the
// specified object.
//  PREVIEW
//
// CAUTION: even if the error is not nil. the returned slices
// might still contain data.
func (ioctx *IOContext) Notify(obj string, data []byte) ([]NotifyAck, []NotifyTimeout, error) {
	return ioctx.NotifyWithTimeout(obj, data, 0)
}

// NotifyWithTimeout is like Notify() but with a different timeout than the
// default.
//  PREVIEW
//
// Implements:
//  int rados_notify2(rados_ioctx_t io, const char* o, const char* buf, int buf_len,
//    uint64_t timeout_ms, char** reply_buffer, size_t* reply_buffer_len)
func (ioctx *IOContext) NotifyWithTimeout(obj string, data []byte, timeout time.Duration) ([]NotifyAck,
	[]NotifyTimeout, error) {
	cObj := C.CString(obj)
	defer C.free(unsafe.Pointer(cObj))
	var cResponse *C.char
	defer C.rados_buffer_free(cResponse)
	var responseLen C.size_t
	var dataPtr *C.char
	if len(data) > 0 {
		dataPtr = (*C.char)(unsafe.Pointer(&data[0]))
	}
	ret := C.rados_notify2(
		ioctx.ioctx,
		cObj,
		dataPtr,
		C.int(len(data)),
		C.uint64_t(timeout.Milliseconds()),
		&cResponse,
		&responseLen,
	)
	// cResponse has been set even if an error is returned, so we decode it anyway
	acks, timeouts := decodeNotifyResponse(cResponse, responseLen)
	return acks, timeouts, getError(ret)
}

// Ack sends an acknowledgement with the specified response data to the notfier
// of the NotifyEvent. If a notify is not ack'ed, the originating Notify() call
// blocks and eventiually times out.
//  PREVIEW
//
// Implements:
//  int rados_notify_ack(rados_ioctx_t io, const char *o, uint64_t notify_id,
//    uint64_t cookie, const char *buf, int buf_len)
func (ne *NotifyEvent) Ack(response []byte) error {
	watchersMtx.RLock()
	w, ok := watchers[ne.WatcherID]
	watchersMtx.RUnlock()
	if !ok {
		return fmt.Errorf("can't ack on deleted watcher %v", ne.WatcherID)
	}
	cOID := C.CString(w.oid)
	defer C.free(unsafe.Pointer(cOID))
	var respPtr *C.char
	if len(response) > 0 {
		respPtr = (*C.char)(unsafe.Pointer(&response[0]))
	}
	ret := C.rados_notify_ack(
		w.ioctx.ioctx,
		cOID,
		C.uint64_t(ne.ID),
		C.uint64_t(ne.WatcherID),
		respPtr,
		C.int(len(response)),
	)
	return getError(ret)
}

// WatcherFlush flushes all pending notifications of the cluster.
//  PREVIEW
//
// Implements:
//  int rados_watch_flush(rados_t cluster)
func (c *Conn) WatcherFlush() error {
	if !c.connected {
		return ErrNotConnected
	}
	ret := C.rados_watch_flush(c.cluster)
	return getError(ret)
}

// decoder for this notify response format:
//    le32 num_acks
//    {
//      le64 gid     global id for the client (for client.1234 that's 1234)
//      le64 cookie  cookie for the client
//      le32 buflen  length of reply message buffer
//      u8 buflen  payload
//    } num_acks
//    le32 num_timeouts
//    {
//      le64 gid     global id for the client
//      le64 cookie  cookie for the client
//    } num_timeouts
//
// NOTE: starting with pacific this is implemented as a C function and this can
// be replaced later
func decodeNotifyResponse(response *C.char, len C.size_t) ([]NotifyAck, []NotifyTimeout) {
	if len == 0 || response == nil {
		return nil, nil
	}
	b := (*[math.MaxInt32]byte)(unsafe.Pointer(response))[:len:len]
	pos := 0

	num := binary.LittleEndian.Uint32(b[pos:])
	pos += 4
	acks := make([]NotifyAck, num)
	for i := range acks {
		acks[i].NotifierID = NotifierID(binary.LittleEndian.Uint64(b[pos:]))
		pos += 8
		acks[i].WatcherID = WatcherID(binary.LittleEndian.Uint64(b[pos:]))
		pos += 8
		dataLen := binary.LittleEndian.Uint32(b[pos:])
		pos += 4
		if dataLen > 0 {
			acks[i].Response = C.GoBytes(unsafe.Pointer(&b[pos]), C.int(dataLen))
			pos += int(dataLen)
		}
	}

	num = binary.LittleEndian.Uint32(b[pos:])
	pos += 4
	timeouts := make([]NotifyTimeout, num)
	for i := range timeouts {
		timeouts[i].NotifierID = NotifierID(binary.LittleEndian.Uint64(b[pos:]))
		pos += 8
		timeouts[i].WatcherID = WatcherID(binary.LittleEndian.Uint64(b[pos:]))
		pos += 8
	}
	return acks, timeouts
}

//export watchNotifyCb
func watchNotifyCb(_ unsafe.Pointer, notifyID C.uint64_t, id C.uint64_t,
	notifierID C.uint64_t, cData unsafe.Pointer, dataLen C.size_t) {
	watchersMtx.RLock()
	w, ok := watchers[WatcherID(id)]
	watchersMtx.RUnlock()
	if !ok {
		// usually this should not happen, but who knows
		// TODO: some log message (once we have logging)
		return
	}
	ev := NotifyEvent{
		ID:         NotifyID(notifyID),
		WatcherID:  WatcherID(id),
		NotifierID: NotifierID(notifierID),
	}
	if dataLen > 0 {
		ev.Data = C.GoBytes(cData, C.int(dataLen))
	}
	select {
	case <-w.done: // unblock when deleted
	case w.events <- ev:
	}
}

//export watchErrorCb
func watchErrorCb(_ unsafe.Pointer, id C.uint64_t, err C.int) {
	watchersMtx.RLock()
	w, ok := watchers[WatcherID(id)]
	watchersMtx.RUnlock()
	if !ok {
		// usually this should not happen, but who knows
		// TODO: some log message (once we have logging)
		return
	}
	select {
	case <-w.done: // unblock when deleted
	case w.errors <- getError(err):
	}
}
<commit_msg>rados: add warning logs to watcher callbacks<commit_after>//go:build ceph_preview
// +build ceph_preview

package rados

/*
#cgo LDFLAGS: -lrados
#include <stdlib.h>
#include <rados/librados.h>
extern void watchNotifyCb(void*, uint64_t, uint64_t, uint64_t, void*, size_t);
extern void watchErrorCb(void*, uint64_t, int);
*/
import "C"

import (
	"encoding/binary"
	"fmt"
	"math"
	"sync"
	"time"
	"unsafe"

	"github.com/ceph/go-ceph/internal/log"
)

type (
	// WatcherID is the unique id of a Watcher.
	WatcherID uint64
	// NotifyID is the unique id of a NotifyEvent.
	NotifyID uint64
	// NotifierID is the unique id of a notifying client.
	NotifierID uint64
)

// NotifyEvent is received by a watcher for each notification.
type NotifyEvent struct {
	ID         NotifyID
	WatcherID  WatcherID
	NotifierID NotifierID
	Data       []byte
}

// NotifyAck represents an acknowleged notification.
type NotifyAck struct {
	WatcherID  WatcherID
	NotifierID NotifierID
	Response   []byte
}

// NotifyTimeout represents an unacknowleged notification.
type NotifyTimeout struct {
	WatcherID  WatcherID
	NotifierID NotifierID
}

// Watcher receives all notifications for certain object.
type Watcher struct {
	id     WatcherID
	oid    string
	ioctx  *IOContext
	events chan NotifyEvent
	errors chan error
	done   chan struct{}
}

var (
	watchers    = map[WatcherID]*Watcher{}
	watchersMtx sync.RWMutex
)

// Watch creates a Watcher for the specified object.
//  PREVIEW
//
// A Watcher receives all notifications that are sent to the object on which it
// has been created. It exposes two read-only channels: Events() receives all
// the NotifyEvents and Errors() receives all occuring errors. A typical code
// creating a Watcher could look like this:
//
//  watcher, err := ioctx.Watch(oid)
//  go func() { // event handler
//    for ne := range watcher.Events() {
//      ...
//      ne.Ack([]byte("response data..."))
//      ...
//    }
//  }()
//  go func() { // error handler
//    for err := range watcher.Errors() {
//      ... handle err ...
//    }
//  }()
//
// CAUTION: the Watcher references the IOContext in which it has been created.
// Therefore all watchers must be deleted with the Delete() method before the
// IOContext is being destroyed.
//
// Implements:
//  int rados_watch2(rados_ioctx_t io, const char* o, uint64_t* cookie,
//    rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, void* arg)
func (ioctx *IOContext) Watch(obj string) (*Watcher, error) {
	return ioctx.WatchWithTimeout(obj, 0)
}

// WatchWithTimeout creates a watcher on an object. Same as Watcher(), but
// different timeout than the default can be specified.
//  PREVIEW
//
// Implements:
//  int rados_watch3(rados_ioctx_t io, const char *o, uint64_t *cookie,
// 	  rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, uint32_t timeout,
// 	  void *arg);
func (ioctx *IOContext) WatchWithTimeout(oid string, timeout time.Duration) (*Watcher, error) {
	cObj := C.CString(oid)
	defer C.free(unsafe.Pointer(cObj))
	var id C.uint64_t
	watchersMtx.Lock()
	defer watchersMtx.Unlock()
	ret := C.rados_watch3(
		ioctx.ioctx,
		cObj,
		&id,
		(C.rados_watchcb2_t)(C.watchNotifyCb),
		(C.rados_watcherrcb_t)(C.watchErrorCb),
		C.uint32_t(timeout.Milliseconds()/1000),
		nil,
	)
	if err := getError(ret); err != nil {
		return nil, err
	}
	evCh := make(chan NotifyEvent)
	errCh := make(chan error)
	w := &Watcher{
		id:     WatcherID(id),
		ioctx:  ioctx,
		oid:    oid,
		events: evCh,
		errors: errCh,
		done:   make(chan struct{}),
	}
	watchers[WatcherID(id)] = w
	return w, nil
}

// ID returns the WatcherId of the Watcher
//  PREVIEW
func (w *Watcher) ID() WatcherID {
	return w.id
}

// Events returns a read-only channel, that receives all notifications that are
// sent to the object of the Watcher.
//  PREVIEW
func (w *Watcher) Events() <-chan NotifyEvent {
	return w.events
}

// Errors returns a read-only channel, that receives all errors for the Watcher.
//  PREVIEW
func (w *Watcher) Errors() <-chan error {
	return w.errors
}

// Check on the status of a Watcher.
//  PREVIEW
//
// Returns the time since it was last confirmed. If there is an error, the
// Watcher is no longer valid, and should be destroyed with the Delete() method.
//
// Implements:
//  int rados_watch_check(rados_ioctx_t io, uint64_t cookie)
func (w *Watcher) Check() (time.Duration, error) {
	ret := C.rados_watch_check(w.ioctx.ioctx, C.uint64_t(w.id))
	if ret < 0 {
		return 0, getError(ret)
	}
	return time.Millisecond * time.Duration(ret), nil
}

// Delete the watcher. This closes both the event and error channel.
//  PREVIEW
//
// Implements:
//  int rados_unwatch2(rados_ioctx_t io, uint64_t cookie)
func (w *Watcher) Delete() error {
	watchersMtx.Lock()
	_, ok := watchers[w.id]
	if ok {
		delete(watchers, w.id)
	}
	watchersMtx.Unlock()
	if !ok {
		return nil
	}
	ret := C.rados_unwatch2(w.ioctx.ioctx, C.uint64_t(w.id))
	if ret != 0 {
		return getError(ret)
	}
	close(w.done) // unblock blocked callbacks
	close(w.events)
	close(w.errors)
	return nil
}

// Notify sends a notification with the provided data to all Watchers of the
// specified object.
//  PREVIEW
//
// CAUTION: even if the error is not nil. the returned slices
// might still contain data.
func (ioctx *IOContext) Notify(obj string, data []byte) ([]NotifyAck, []NotifyTimeout, error) {
	return ioctx.NotifyWithTimeout(obj, data, 0)
}

// NotifyWithTimeout is like Notify() but with a different timeout than the
// default.
//  PREVIEW
//
// Implements:
//  int rados_notify2(rados_ioctx_t io, const char* o, const char* buf, int buf_len,
//    uint64_t timeout_ms, char** reply_buffer, size_t* reply_buffer_len)
func (ioctx *IOContext) NotifyWithTimeout(obj string, data []byte, timeout time.Duration) ([]NotifyAck,
	[]NotifyTimeout, error) {
	cObj := C.CString(obj)
	defer C.free(unsafe.Pointer(cObj))
	var cResponse *C.char
	defer C.rados_buffer_free(cResponse)
	var responseLen C.size_t
	var dataPtr *C.char
	if len(data) > 0 {
		dataPtr = (*C.char)(unsafe.Pointer(&data[0]))
	}
	ret := C.rados_notify2(
		ioctx.ioctx,
		cObj,
		dataPtr,
		C.int(len(data)),
		C.uint64_t(timeout.Milliseconds()),
		&cResponse,
		&responseLen,
	)
	// cResponse has been set even if an error is returned, so we decode it anyway
	acks, timeouts := decodeNotifyResponse(cResponse, responseLen)
	return acks, timeouts, getError(ret)
}

// Ack sends an acknowledgement with the specified response data to the notfier
// of the NotifyEvent. If a notify is not ack'ed, the originating Notify() call
// blocks and eventiually times out.
//  PREVIEW
//
// Implements:
//  int rados_notify_ack(rados_ioctx_t io, const char *o, uint64_t notify_id,
//    uint64_t cookie, const char *buf, int buf_len)
func (ne *NotifyEvent) Ack(response []byte) error {
	watchersMtx.RLock()
	w, ok := watchers[ne.WatcherID]
	watchersMtx.RUnlock()
	if !ok {
		return fmt.Errorf("can't ack on deleted watcher %v", ne.WatcherID)
	}
	cOID := C.CString(w.oid)
	defer C.free(unsafe.Pointer(cOID))
	var respPtr *C.char
	if len(response) > 0 {
		respPtr = (*C.char)(unsafe.Pointer(&response[0]))
	}
	ret := C.rados_notify_ack(
		w.ioctx.ioctx,
		cOID,
		C.uint64_t(ne.ID),
		C.uint64_t(ne.WatcherID),
		respPtr,
		C.int(len(response)),
	)
	return getError(ret)
}

// WatcherFlush flushes all pending notifications of the cluster.
//  PREVIEW
//
// Implements:
//  int rados_watch_flush(rados_t cluster)
func (c *Conn) WatcherFlush() error {
	if !c.connected {
		return ErrNotConnected
	}
	ret := C.rados_watch_flush(c.cluster)
	return getError(ret)
}

// decoder for this notify response format:
//    le32 num_acks
//    {
//      le64 gid     global id for the client (for client.1234 that's 1234)
//      le64 cookie  cookie for the client
//      le32 buflen  length of reply message buffer
//      u8 buflen  payload
//    } num_acks
//    le32 num_timeouts
//    {
//      le64 gid     global id for the client
//      le64 cookie  cookie for the client
//    } num_timeouts
//
// NOTE: starting with pacific this is implemented as a C function and this can
// be replaced later
func decodeNotifyResponse(response *C.char, len C.size_t) ([]NotifyAck, []NotifyTimeout) {
	if len == 0 || response == nil {
		return nil, nil
	}
	b := (*[math.MaxInt32]byte)(unsafe.Pointer(response))[:len:len]
	pos := 0

	num := binary.LittleEndian.Uint32(b[pos:])
	pos += 4
	acks := make([]NotifyAck, num)
	for i := range acks {
		acks[i].NotifierID = NotifierID(binary.LittleEndian.Uint64(b[pos:]))
		pos += 8
		acks[i].WatcherID = WatcherID(binary.LittleEndian.Uint64(b[pos:]))
		pos += 8
		dataLen := binary.LittleEndian.Uint32(b[pos:])
		pos += 4
		if dataLen > 0 {
			acks[i].Response = C.GoBytes(unsafe.Pointer(&b[pos]), C.int(dataLen))
			pos += int(dataLen)
		}
	}

	num = binary.LittleEndian.Uint32(b[pos:])
	pos += 4
	timeouts := make([]NotifyTimeout, num)
	for i := range timeouts {
		timeouts[i].NotifierID = NotifierID(binary.LittleEndian.Uint64(b[pos:]))
		pos += 8
		timeouts[i].WatcherID = WatcherID(binary.LittleEndian.Uint64(b[pos:]))
		pos += 8
	}
	return acks, timeouts
}

//export watchNotifyCb
func watchNotifyCb(_ unsafe.Pointer, notifyID C.uint64_t, id C.uint64_t,
	notifierID C.uint64_t, cData unsafe.Pointer, dataLen C.size_t) {
	ev := NotifyEvent{
		ID:         NotifyID(notifyID),
		WatcherID:  WatcherID(id),
		NotifierID: NotifierID(notifierID),
	}
	if dataLen > 0 {
		ev.Data = C.GoBytes(cData, C.int(dataLen))
	}
	watchersMtx.RLock()
	w, ok := watchers[WatcherID(id)]
	watchersMtx.RUnlock()
	if !ok {
		// usually this should not happen, but who knows
		log.Warnf("received notification for unknown watcher ID: %#v", ev)
		return
	}
	select {
	case <-w.done: // unblock when deleted
	case w.events <- ev:
	}
}

//export watchErrorCb
func watchErrorCb(_ unsafe.Pointer, id C.uint64_t, err C.int) {
	watchersMtx.RLock()
	w, ok := watchers[WatcherID(id)]
	watchersMtx.RUnlock()
	if !ok {
		// usually this should not happen, but who knows
		log.Warnf("received error for unknown watcher ID: id=%d err=%#v", id, err)
		return
	}
	select {
	case <-w.done: // unblock when deleted
	case w.errors <- getError(err):
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 kubeflow.org.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package istio

import (
	"encoding/json"
	"fmt"

	"github.com/kubeflow/kfserving/pkg/apis/serving/v1alpha2"
	"github.com/kubeflow/kfserving/pkg/constants"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	duckv1beta1 "knative.dev/pkg/apis/duck/v1beta1"
	istiov1alpha1 "knative.dev/pkg/apis/istio/common/v1alpha1"
	istiov1alpha3 "knative.dev/pkg/apis/istio/v1alpha3"
)

const (
	IngressConfigKeyName = "ingress"
)

// Status Constants
var (
	PredictorSpecMissing       = "predictorSpecMissing"
	PredictorStatusUnknown     = "predictorStatusUnknown"
	PredictorHostnameUnknown   = "predictorHostnameUnknown"
	TransformerSpecMissing     = "transformerSpecMissing"
	TransformerStatusUnknown   = "transformerStatusUnknown"
	TransformerHostnameUnknown = "transformerHostnameUnknown"
	ExplainerSpecMissing       = "explainerSpecMissing"
	ExplainerStatusUnknown     = "explainerStatusUnknown"
	ExplainerHostnameUnknown   = "explainerHostnameUnknown"
)

type IngressConfig struct {
	IngressGateway     string `json:"ingressGateway,omitempty"`
	IngressServiceName string `json:"ingressService,omitempty"`
}

type VirtualServiceBuilder struct {
	ingressConfig *IngressConfig
}

func NewVirtualServiceBuilder(config *corev1.ConfigMap) *VirtualServiceBuilder {
	ingressConfig := &IngressConfig{}
	if ingress, ok := config.Data[IngressConfigKeyName]; ok {
		err := json.Unmarshal([]byte(ingress), &ingressConfig)
		if err != nil {
			panic(fmt.Errorf("Unable to parse ingress config json: %v", err))
		}

		if ingressConfig.IngressGateway == "" || ingressConfig.IngressServiceName == "" {
			panic(fmt.Errorf("Invalid ingress config, ingressGateway and ingressService are required."))
		}
	}

	return &VirtualServiceBuilder{ingressConfig: ingressConfig}
}

func createFailedStatus(reason string, message string) *v1alpha2.VirtualServiceStatus {
	return &v1alpha2.VirtualServiceStatus{
		Status: duckv1beta1.Status{
			Conditions: duckv1beta1.Conditions{{
				Type:    v1alpha2.RoutesReady,
				Status:  corev1.ConditionFalse,
				Reason:  reason,
				Message: message,
			}},
		},
	}
}

func (r *VirtualServiceBuilder) CreateVirtualService(isvc *v1alpha2.InferenceService) (*istiov1alpha3.VirtualService, *v1alpha2.VirtualServiceStatus) {

	httpRoutes := []istiov1alpha3.HTTPRoute{}

	// destination for the default predict is required
	predictDefaultSpec, reason := getPredictStatusConfigurationSpec(&isvc.Spec.Default, isvc.Status.Default)
	if predictDefaultSpec == nil {
		return nil, createFailedStatus(reason, "Failed to reconcile default predictor")
	}

	// use transformer instead (if one is configured)
	if isvc.Spec.Default.Transformer != nil {
		predictDefaultSpec, reason = getTransformerStatusConfigurationSpec(&isvc.Spec.Default, isvc.Status.Default)
		if predictDefaultSpec == nil {
			return nil, createFailedStatus(reason, "Failed to reconcile default transformer")
		}
	}

	// extract the virtual service hostname from the predictor hostname
	serviceHostname := constants.VirtualServiceHostname(isvc.Name, predictDefaultSpec.Hostname)
	serviceURL := constants.ServiceURL(isvc.Name, serviceHostname)

	// add the default route
	defaultWeight := 100 - isvc.Spec.CanaryTrafficPercent
	canaryWeight := isvc.Spec.CanaryTrafficPercent
	predictRouteDestinations := []istiov1alpha3.HTTPRouteDestination{
		createHTTPRouteDestination(predictDefaultSpec.Hostname, defaultWeight, r.ingressConfig.IngressServiceName),
	}

	// optionally get a destination for canary predict
	if isvc.Spec.Canary != nil {
		predictCanarySpec, reason := getPredictStatusConfigurationSpec(isvc.Spec.Canary, isvc.Status.Canary)
		if predictCanarySpec == nil {
			return nil, createFailedStatus(reason, "Failed to reconcile canary predictor")
		}

		// attempt use transformer instead if *Default* had one, see discussion: https://github.com/kubeflow/kfserving/issues/324
		if isvc.Spec.Default.Transformer != nil {
			predictCanarySpec, reason = getTransformerStatusConfigurationSpec(isvc.Spec.Canary, isvc.Status.Canary)
			if predictCanarySpec == nil {
				return nil, createFailedStatus(reason, "Failed to reconcile canary transformer")
			}
		}

		canaryRouteDestination := createHTTPRouteDestination(predictCanarySpec.Hostname, canaryWeight, r.ingressConfig.IngressServiceName)
		predictRouteDestinations = append(predictRouteDestinations, canaryRouteDestination)
	}

	// prepare the predict route
	predictRoute := istiov1alpha3.HTTPRoute{
		Match: []istiov1alpha3.HTTPMatchRequest{
			istiov1alpha3.HTTPMatchRequest{
				URI: &istiov1alpha1.StringMatch{
					Prefix: constants.PredictPrefix(isvc.Name),
				},
			},
		},
		Route: predictRouteDestinations,
	}
	httpRoutes = append(httpRoutes, predictRoute)

	// optionally add the explain route
	explainRouteDestinations := []istiov1alpha3.HTTPRouteDestination{}
	if isvc.Spec.Default.Explainer != nil {
		explainDefaultSpec, defaultExplainerReason := getExplainStatusConfigurationSpec(&isvc.Spec.Default, isvc.Status.Default)
		if explainDefaultSpec != nil {
			routeDefaultDestination := createHTTPRouteDestination(explainDefaultSpec.Hostname, defaultWeight, r.ingressConfig.IngressServiceName)
			explainRouteDestinations = append(explainRouteDestinations, routeDefaultDestination)

			explainCanarySpec, canaryExplainerReason := getExplainStatusConfigurationSpec(isvc.Spec.Canary, isvc.Status.Canary)
			if explainCanarySpec != nil {
				routeCanaryDestination := createHTTPRouteDestination(explainCanarySpec.Hostname, canaryWeight, r.ingressConfig.IngressServiceName)
				explainRouteDestinations = append(explainRouteDestinations, routeCanaryDestination)
			} else {
				return nil, createFailedStatus(canaryExplainerReason, "Failed to reconcile canary explainer")
			}
		} else {
			return nil, createFailedStatus(defaultExplainerReason, "Failed to reconcile default explainer")
		}

		explainRoute := istiov1alpha3.HTTPRoute{
			Match: []istiov1alpha3.HTTPMatchRequest{
				istiov1alpha3.HTTPMatchRequest{
					URI: &istiov1alpha1.StringMatch{
						Prefix: constants.ExplainPrefix(isvc.Name),
					},
				},
			},
			Route: predictRouteDestinations,
		}
		httpRoutes = append(httpRoutes, explainRoute)
	}

	vs := istiov1alpha3.VirtualService{
		ObjectMeta: metav1.ObjectMeta{
			Name:        isvc.Name,
			Namespace:   isvc.Namespace,
			Labels:      isvc.Labels,
			Annotations: isvc.Annotations,
		},
		Spec: istiov1alpha3.VirtualServiceSpec{
			Hosts: []string{
				serviceHostname,
			},
			Gateways: []string{
				r.ingressConfig.IngressGateway,
			},
			HTTP: httpRoutes,
		},
	}

	status := v1alpha2.VirtualServiceStatus{
		URL:           serviceURL,
		CanaryWeight:  canaryWeight,
		DefaultWeight: defaultWeight,
		Status: duckv1beta1.Status{
			Conditions: duckv1beta1.Conditions{{
				Type:   v1alpha2.RoutesReady,
				Status: corev1.ConditionTrue,
			}},
		},
	}

	return &vs, &status
}

func getPredictStatusConfigurationSpec(endpointSpec *v1alpha2.EndpointSpec, endpointStatusMap *v1alpha2.EndpointStatusMap) (*v1alpha2.StatusConfigurationSpec, string) {
	if endpointSpec == nil {
		return nil, PredictorSpecMissing
	}

	if predictorStatus, ok := (*endpointStatusMap)[constants.Predictor]; !ok {
		return nil, PredictorStatusUnknown
	} else if len(predictorStatus.Hostname) == 0 {
		return nil, PredictorHostnameUnknown
	} else {
		return predictorStatus, ""
	}
}

func getTransformerStatusConfigurationSpec(endpointSpec *v1alpha2.EndpointSpec, endpointStatusMap *v1alpha2.EndpointStatusMap) (*v1alpha2.StatusConfigurationSpec, string) {
	if endpointSpec.Transformer == nil {
		return nil, TransformerSpecMissing
	}

	if transformerStatus, ok := (*endpointStatusMap)[constants.Transformer]; !ok {
		return nil, TransformerStatusUnknown
	} else if len(transformerStatus.Hostname) == 0 {
		return nil, TransformerHostnameUnknown
	} else {
		return transformerStatus, ""
	}
}
func getExplainStatusConfigurationSpec(endpointSpec *v1alpha2.EndpointSpec, endpointStatusMap *v1alpha2.EndpointStatusMap) (*v1alpha2.StatusConfigurationSpec, string) {
	if endpointSpec.Explainer == nil {
		return nil, ExplainerSpecMissing
	}

	explainerStatus, ok := (*endpointStatusMap)[constants.Explainer]
	if !ok {
		return nil, ExplainerStatusUnknown
	} else if len(explainerStatus.Hostname) == 0 {
		return nil, ExplainerHostnameUnknown
	}

	return explainerStatus, ""
}

func createHTTPRouteDestination(targetHost string, weight int, gatewayService string) istiov1alpha3.HTTPRouteDestination {
	httpRouteDestination := istiov1alpha3.HTTPRouteDestination{
		Weight: weight,
		Headers: &istiov1alpha3.Headers{
			Request: &istiov1alpha3.HeaderOperations{
				Set: map[string]string{
					"Host": targetHost,
				},
			},
		},
		Destination: istiov1alpha3.Destination{
			Host: gatewayService,
		},
	}

	return httpRouteDestination
}
<commit_msg>Explainer virtual service cannot work (#446)<commit_after>/*
Copyright 2019 kubeflow.org.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package istio

import (
	"encoding/json"
	"fmt"

	"github.com/kubeflow/kfserving/pkg/apis/serving/v1alpha2"
	"github.com/kubeflow/kfserving/pkg/constants"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	duckv1beta1 "knative.dev/pkg/apis/duck/v1beta1"
	istiov1alpha1 "knative.dev/pkg/apis/istio/common/v1alpha1"
	istiov1alpha3 "knative.dev/pkg/apis/istio/v1alpha3"
)

const (
	IngressConfigKeyName = "ingress"
)

// Status Constants
var (
	PredictorSpecMissing       = "predictorSpecMissing"
	PredictorStatusUnknown     = "predictorStatusUnknown"
	PredictorHostnameUnknown   = "predictorHostnameUnknown"
	TransformerSpecMissing     = "transformerSpecMissing"
	TransformerStatusUnknown   = "transformerStatusUnknown"
	TransformerHostnameUnknown = "transformerHostnameUnknown"
	ExplainerSpecMissing       = "explainerSpecMissing"
	ExplainerStatusUnknown     = "explainerStatusUnknown"
	ExplainerHostnameUnknown   = "explainerHostnameUnknown"
)

type IngressConfig struct {
	IngressGateway     string `json:"ingressGateway,omitempty"`
	IngressServiceName string `json:"ingressService,omitempty"`
}

type VirtualServiceBuilder struct {
	ingressConfig *IngressConfig
}

func NewVirtualServiceBuilder(config *corev1.ConfigMap) *VirtualServiceBuilder {
	ingressConfig := &IngressConfig{}
	if ingress, ok := config.Data[IngressConfigKeyName]; ok {
		err := json.Unmarshal([]byte(ingress), &ingressConfig)
		if err != nil {
			panic(fmt.Errorf("Unable to parse ingress config json: %v", err))
		}

		if ingressConfig.IngressGateway == "" || ingressConfig.IngressServiceName == "" {
			panic(fmt.Errorf("Invalid ingress config, ingressGateway and ingressService are required."))
		}
	}

	return &VirtualServiceBuilder{ingressConfig: ingressConfig}
}

func createFailedStatus(reason string, message string) *v1alpha2.VirtualServiceStatus {
	return &v1alpha2.VirtualServiceStatus{
		Status: duckv1beta1.Status{
			Conditions: duckv1beta1.Conditions{{
				Type:    v1alpha2.RoutesReady,
				Status:  corev1.ConditionFalse,
				Reason:  reason,
				Message: message,
			}},
		},
	}
}

func (r *VirtualServiceBuilder) getPredictRouteDestination(
	endpointSpec *v1alpha2.EndpointSpec, endpointStatusMap *v1alpha2.EndpointStatusMap, weight int) (*istiov1alpha3.HTTPRouteDestination, *v1alpha2.VirtualServiceStatus) {
	if endpointSpec == nil {
		return nil, nil
	}
	// destination for the predict is required
	predictSpec, reason := getPredictStatusConfigurationSpec(endpointSpec, endpointStatusMap)
	if predictSpec == nil {
		return nil, createFailedStatus(reason, "Failed to reconcile predictor")
	}

	// use transformer instead (if one is configured)
	if endpointSpec.Transformer != nil {
		predictSpec, reason = getTransformerStatusConfigurationSpec(endpointSpec, endpointStatusMap)
		if predictSpec == nil {
			return nil, createFailedStatus(reason, "Failed to reconcile transformer")
		}
	}

	httpRouteDestination := createHTTPRouteDestination(predictSpec.Hostname, weight, r.ingressConfig.IngressServiceName)
	return &httpRouteDestination, nil
}

func (r *VirtualServiceBuilder) getExplainerRouteDestination(
	endpointSpec *v1alpha2.EndpointSpec, endpointStatusMap *v1alpha2.EndpointStatusMap, weight int) (*istiov1alpha3.HTTPRouteDestination, *v1alpha2.VirtualServiceStatus) {
	if endpointSpec == nil {
		return nil, nil
	}
	if endpointSpec.Explainer != nil {
		explainSpec, explainerReason := getExplainStatusConfigurationSpec(endpointSpec, endpointStatusMap)
		if explainSpec != nil {
			httpRouteDestination := createHTTPRouteDestination(explainSpec.Hostname, weight, r.ingressConfig.IngressServiceName)
			return &httpRouteDestination, nil
		} else {
			return nil, createFailedStatus(explainerReason, "Failed to reconcile default explainer")
		}
	}
	return nil, nil
}

func (r *VirtualServiceBuilder) CreateVirtualService(isvc *v1alpha2.InferenceService) (*istiov1alpha3.VirtualService, *v1alpha2.VirtualServiceStatus) {

	httpRoutes := []istiov1alpha3.HTTPRoute{}
	predictRouteDestinations := []istiov1alpha3.HTTPRouteDestination{}

	defaultWeight := 100 - isvc.Spec.CanaryTrafficPercent
	canaryWeight := isvc.Spec.CanaryTrafficPercent

	if defaultPredictRouteDestination, err := r.getPredictRouteDestination(&isvc.Spec.Default, isvc.Status.Default, defaultWeight); err != nil {
		return nil, err
	} else {
		predictRouteDestinations = append(predictRouteDestinations, *defaultPredictRouteDestination)
	}
	if canaryPredictRouteDestination, err := r.getPredictRouteDestination(isvc.Spec.Canary, isvc.Status.Canary, canaryWeight); err != nil {
		return nil, err
	} else {
		if canaryPredictRouteDestination != nil {
			predictRouteDestinations = append(predictRouteDestinations, *canaryPredictRouteDestination)
		}
	}
	// prepare the predict route
	predictRoute := istiov1alpha3.HTTPRoute{
		Match: []istiov1alpha3.HTTPMatchRequest{
			istiov1alpha3.HTTPMatchRequest{
				URI: &istiov1alpha1.StringMatch{
					Prefix: constants.PredictPrefix(isvc.Name),
				},
			},
		},
		Route: predictRouteDestinations,
	}
	httpRoutes = append(httpRoutes, predictRoute)

	// optionally add the explain route
	explainRouteDestinations := []istiov1alpha3.HTTPRouteDestination{}
	if defaultExplainRouteDestination, err := r.getExplainerRouteDestination(&isvc.Spec.Default, isvc.Status.Default, defaultWeight); err != nil {
		return nil, err
	} else {
		if defaultExplainRouteDestination != nil {
			explainRouteDestinations = append(explainRouteDestinations, *defaultExplainRouteDestination)
		}
	}
	if canaryExplainRouteDestination, err := r.getExplainerRouteDestination(isvc.Spec.Canary, isvc.Status.Canary, canaryWeight); err != nil {
		return nil, err
	} else {
		if canaryExplainRouteDestination != nil {
			explainRouteDestinations = append(explainRouteDestinations, *canaryExplainRouteDestination)
		}
	}

	if len(explainRouteDestinations) > 0 {
		explainRoute := istiov1alpha3.HTTPRoute{
			Match: []istiov1alpha3.HTTPMatchRequest{
				istiov1alpha3.HTTPMatchRequest{
					URI: &istiov1alpha1.StringMatch{
						Prefix: constants.ExplainPrefix(isvc.Name),
					},
				},
			},
			Route: explainRouteDestinations,
		}
		httpRoutes = append(httpRoutes, explainRoute)
	}
	// extract the virtual service hostname from the predictor hostname
	serviceHostname, _ := getServiceHostname(isvc)
	serviceURL := constants.ServiceURL(isvc.Name, serviceHostname)

	vs := istiov1alpha3.VirtualService{
		ObjectMeta: metav1.ObjectMeta{
			Name:        isvc.Name,
			Namespace:   isvc.Namespace,
			Labels:      isvc.Labels,
			Annotations: isvc.Annotations,
		},
		Spec: istiov1alpha3.VirtualServiceSpec{
			Hosts: []string{
				serviceHostname,
			},
			Gateways: []string{
				r.ingressConfig.IngressGateway,
			},
			HTTP: httpRoutes,
		},
	}

	status := v1alpha2.VirtualServiceStatus{
		URL:           serviceURL,
		CanaryWeight:  canaryWeight,
		DefaultWeight: defaultWeight,
		Status: duckv1beta1.Status{
			Conditions: duckv1beta1.Conditions{{
				Type:   v1alpha2.RoutesReady,
				Status: corev1.ConditionTrue,
			}},
		},
	}

	return &vs, &status
}

func getServiceHostname(isvc *v1alpha2.InferenceService) (string, error) {
	predictSpec, reason := getPredictStatusConfigurationSpec(&isvc.Spec.Default, isvc.Status.Default)
	if predictSpec == nil {
		return "", fmt.Errorf("Fail to get service hostname: %s.", reason)
	}
	return constants.VirtualServiceHostname(isvc.Name, predictSpec.Hostname), nil
}

func getPredictStatusConfigurationSpec(endpointSpec *v1alpha2.EndpointSpec, endpointStatusMap *v1alpha2.EndpointStatusMap) (*v1alpha2.StatusConfigurationSpec, string) {
	if endpointSpec == nil {
		return nil, PredictorSpecMissing
	}

	if predictorStatus, ok := (*endpointStatusMap)[constants.Predictor]; !ok {
		return nil, PredictorStatusUnknown
	} else if len(predictorStatus.Hostname) == 0 {
		return nil, PredictorHostnameUnknown
	} else {
		return predictorStatus, ""
	}
}

func getTransformerStatusConfigurationSpec(endpointSpec *v1alpha2.EndpointSpec, endpointStatusMap *v1alpha2.EndpointStatusMap) (*v1alpha2.StatusConfigurationSpec, string) {
	if endpointSpec.Transformer == nil {
		return nil, TransformerSpecMissing
	}

	if transformerStatus, ok := (*endpointStatusMap)[constants.Transformer]; !ok {
		return nil, TransformerStatusUnknown
	} else if len(transformerStatus.Hostname) == 0 {
		return nil, TransformerHostnameUnknown
	} else {
		return transformerStatus, ""
	}
}
func getExplainStatusConfigurationSpec(endpointSpec *v1alpha2.EndpointSpec, endpointStatusMap *v1alpha2.EndpointStatusMap) (*v1alpha2.StatusConfigurationSpec, string) {
	if endpointSpec.Explainer == nil {
		return nil, ExplainerSpecMissing
	}

	explainerStatus, ok := (*endpointStatusMap)[constants.Explainer]
	if !ok {
		return nil, ExplainerStatusUnknown
	} else if len(explainerStatus.Hostname) == 0 {
		return nil, ExplainerHostnameUnknown
	}

	return explainerStatus, ""
}

func createHTTPRouteDestination(targetHost string, weight int, gatewayService string) istiov1alpha3.HTTPRouteDestination {
	httpRouteDestination := istiov1alpha3.HTTPRouteDestination{
		Weight: weight,
		Headers: &istiov1alpha3.Headers{
			Request: &istiov1alpha3.HeaderOperations{
				Set: map[string]string{
					"Host": targetHost,
				},
			},
		},
		Destination: istiov1alpha3.Destination{
			Host: gatewayService,
		},
	}

	return httpRouteDestination
}
<|endoftext|>


<commit_before>package redisq

import (
	"strings"
	"time"
)

// Killmail from a kill
type Killmail struct {
	ID            int        `json:"killmail_id"`
	Hash          string     `json:"killmail_hash"`
	SolarSystemID int        `json:"solar_system_id"`
	Timestamp     KillTime   `json:"killmail_time"`
	Victim        Victim     `json:"victim"`
	Attackers     []Attacker `json:"attackers"`
	Zkb           Zkb        `json:"zkb"`
}

// KillTime embeds time.Time and implements the UnmarshalJSON interface to
// handle CREST's non RFC 3339 timestamp.
type KillTime struct {
	time.Time
}

// UnmarshalJSON parses the timestamp from CREST in to Go's time.Time type.
func (t *KillTime) UnmarshalJSON(b []byte) (err error) {
	t.Time, err = time.Parse("2006.01.02 15:04:05", strings.Replace(string(b), "\"", "", 2))
	return err
}

// Attacker in a killmail
type Attacker struct {
	CharacterID    int     `json:"character_id"`
	CorporationID  int     `json:"corporation_id"`
	AllianceID     int     `json:"alliance_id"`
	ShipTypeID     int     `json:"ship_type_id"`
	WeaponTypeID   int     `json:"weapon_type_id"`
	DamageDone     float32 `json:"damage_done"`
	FinalBlow      bool    `json:"final_blow"`
	SecurityStatus float32 `json:"security_status"`
}

// Victim in a killmail
type Victim struct {
	CharacterID   int    `json:"character_id"`
	CorporationID int    `json:"corporation_id"`
	AllianceID    int    `json:"alliance_id"`
	ShipTypeID    int    `json:"ship_type_id"`
	DamageTaken   int    `json:"damage_taken"`
	Items         []Item `json:"items"`
	Position      struct {
		X float64 `json:"x"`
		Y float64 `json:"y"`
		Z float64 `json:"z"`
	} `json:"position"`
}

// Item dropped/destroyed in a killmail
type Item struct {
	ItemTypeID        int `json:"item_type_id"`
	Flag              int `json:"flag"`
	Singleton         int `json:"singleton"`
	QuantityDropped   int `json:"quantity_dropped"`
	QuantityDestroyed int `json:"quantity_destroyed"`
}

// Zkb is the meta data returned from ZKillboard.
type Zkb struct {
	Hash        string  `json:"hash"`
	FittedValue float32 `json:"fittedValue"`
	TotalValue  float32 `json:"totalValue"`
	Points      int     `json:"points"`
	NPC         bool    `json:"npc"`
	Solo        bool    `json:"solo"`
	AWOX        bool    `json:"aox"`
	Href        string  `json:"href"`
}
<commit_msg>fixes json struct tag typo<commit_after>package redisq

import (
	"strings"
	"time"
)

// Killmail from a kill
type Killmail struct {
	ID            int        `json:"killmail_id"`
	Hash          string     `json:"killmail_hash"`
	SolarSystemID int        `json:"solar_system_id"`
	Timestamp     KillTime   `json:"killmail_time"`
	Victim        Victim     `json:"victim"`
	Attackers     []Attacker `json:"attackers"`
	Zkb           Zkb        `json:"zkb"`
}

// KillTime embeds time.Time and implements the UnmarshalJSON interface to
// handle CREST's non RFC 3339 timestamp.
type KillTime struct {
	time.Time
}

// UnmarshalJSON parses the timestamp from CREST in to Go's time.Time type.
func (t *KillTime) UnmarshalJSON(b []byte) (err error) {
	t.Time, err = time.Parse("2006.01.02 15:04:05", strings.Replace(string(b), "\"", "", 2))
	return err
}

// Attacker in a killmail
type Attacker struct {
	CharacterID    int     `json:"character_id"`
	CorporationID  int     `json:"corporation_id"`
	AllianceID     int     `json:"alliance_id"`
	ShipTypeID     int     `json:"ship_type_id"`
	WeaponTypeID   int     `json:"weapon_type_id"`
	DamageDone     float32 `json:"damage_done"`
	FinalBlow      bool    `json:"final_blow"`
	SecurityStatus float32 `json:"security_status"`
}

// Victim in a killmail
type Victim struct {
	CharacterID   int    `json:"character_id"`
	CorporationID int    `json:"corporation_id"`
	AllianceID    int    `json:"alliance_id"`
	ShipTypeID    int    `json:"ship_type_id"`
	DamageTaken   int    `json:"damage_taken"`
	Items         []Item `json:"items"`
	Position      struct {
		X float64 `json:"x"`
		Y float64 `json:"y"`
		Z float64 `json:"z"`
	} `json:"position"`
}

// Item dropped/destroyed in a killmail
type Item struct {
	ItemTypeID        int `json:"item_type_id"`
	Flag              int `json:"flag"`
	Singleton         int `json:"singleton"`
	QuantityDropped   int `json:"quantity_dropped"`
	QuantityDestroyed int `json:"quantity_destroyed"`
}

// Zkb is the meta data returned from ZKillboard.
type Zkb struct {
	Hash        string  `json:"hash"`
	FittedValue float32 `json:"fittedValue"`
	TotalValue  float32 `json:"totalValue"`
	Points      int     `json:"points"`
	NPC         bool    `json:"npc"`
	Solo        bool    `json:"solo"`
	AWOX        bool    `json:"awox"`
	Href        string  `json:"href"`
}
<|endoftext|>


<commit_before>package v1alpha1

import (
	"fmt"
	"strconv"
	"strings"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

const (
	// BackupLocationResourceName is name for "backuplocation" resource
	BackupLocationResourceName = "backuplocation"
	// BackupLocationResourcePlural is plural for "backuplocation" resource
	BackupLocationResourcePlural = "backuplocations"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// BackupLocation represents a backuplocation object
type BackupLocation struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`
	Location          BackupLocationItem `json:"location"`
}

// BackupLocationItem is the spec used to store a backup location
// Only one of S3Config, AzureConfig or GoogleConfig should be specified and
// should match the Type field. Members of the config can be specified inline or
// through the SecretConfig
type BackupLocationItem struct {
	Type BackupLocationType `json:"type"`
	// Path is either the bucket or any other path for the backup location
	Path          string        `json:"path"`
	EncryptionKey string        `json:"encryptionKey"`
	S3Config      *S3Config     `json:"s3Config,omitempty"`
	AzureConfig   *AzureConfig  `json:"azureConfig,omitempty"`
	GoogleConfig  *GoogleConfig `json:"googleConfig,omitempty"`
	SecretConfig  string        `json:"secretConfig"`
	Sync          bool          `json:"sync"`
}

// BackupLocationType is the type of the backup location
type BackupLocationType string

const (
	// BackupLocationS3 stores the backup in an S3-compliant objectstore
	BackupLocationS3 BackupLocationType = "s3"
	// BackupLocationAzure stores the backup in Azure Blob Storage
	BackupLocationAzure BackupLocationType = "azure"
	// BackupLocationGoogle stores the backup in Google Cloud Storage
	BackupLocationGoogle BackupLocationType = "google"
)

// S3Config speficies the config required to connect to an S3-compliant
// objectstore
type S3Config struct {
	// Endpoint will be defaulted to s3.amazonaws.com by the controller if not provided
	Endpoint        string `json:"endpoint"`
	AccessKeyID     string `json:"accessKeyID"`
	SecretAccessKey string `json:"secretAccessKey"`
	// Region will be defaulted to us-east-1 by the controller if not provided
	Region string `json:"region"`
	// Disable SSL option if using with a non-AWS S3 objectstore which doesn't
	// have SSL enabled
	DisableSSL bool `json:"disableSSL"`
}

// AzureConfig specifies the config required to connect to Azure Blob Storage
type AzureConfig struct {
	StorageAccountName string `json:"storageAccountName"`
	StorageAccountKey  string `json:"storageAccountKey"`
}

// GoogleConfig specifies the config required to connect to Google Cloud Storage
type GoogleConfig struct {
	ProjectID  string `json:"projectID"`
	AccountKey string `json:"accountKey"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// BackupLocationList is a list of ApplicationBackups
type BackupLocationList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitempty"`

	Items []BackupLocation `json:"items"`
}

// UpdateFromSecret updated the config information from the secret if not provided inline
func (bl *BackupLocation) UpdateFromSecret(client kubernetes.Interface) error {
	if bl.Location.SecretConfig != "" {
		secretConfig, err := client.CoreV1().Secrets(bl.Namespace).Get(bl.Location.SecretConfig, metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("error getting secretConfig for backupLocation: %v", err)
		}
		if val, ok := secretConfig.Data["encryptionKey"]; ok && val != nil {
			bl.Location.EncryptionKey = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["path"]; ok && val != nil {
			bl.Location.Path = strings.TrimSuffix(string(val), "\n")
		}
	}
	switch bl.Location.Type {
	case BackupLocationS3:
		return bl.getMergedS3Config(client)
	case BackupLocationAzure:
		return bl.getMergedAzureConfig(client)
	case BackupLocationGoogle:
		return bl.getMergedGoogleConfig(client)
	default:
		return fmt.Errorf("Invalid BackupLocation type %v", bl.Location.Type)
	}
}

func (bl *BackupLocation) getMergedS3Config(client kubernetes.Interface) error {
	if bl.Location.S3Config == nil {
		bl.Location.S3Config = &S3Config{}
		bl.Location.S3Config.Endpoint = "s3.amazonaws.com"
		bl.Location.S3Config.Region = "us-east-1"
		bl.Location.S3Config.DisableSSL = false
	}
	if bl.Location.SecretConfig != "" {
		secretConfig, err := client.CoreV1().Secrets(bl.Namespace).Get(bl.Location.SecretConfig, metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("error getting secretConfig for backupLocation: %v", err)
		}
		if val, ok := secretConfig.Data["endpoint"]; ok && val != nil {
			bl.Location.S3Config.Endpoint = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["accessKeyID"]; ok && val != nil {
			bl.Location.S3Config.AccessKeyID = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["secretAccessKey"]; ok && val != nil {
			bl.Location.S3Config.SecretAccessKey = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["region"]; ok && val != nil {
			bl.Location.S3Config.Region = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["disableSSL"]; ok && val != nil {
			bl.Location.S3Config.DisableSSL, err = strconv.ParseBool(strings.TrimSuffix(string(val), "\n"))
			if err != nil {
				return fmt.Errorf("error parding disableSSL from Secret: %v", err)
			}
		}
	}
	return nil
}

func (bl *BackupLocation) getMergedAzureConfig(client kubernetes.Interface) error {
	if bl.Location.AzureConfig == nil {
		bl.Location.AzureConfig = &AzureConfig{}
	}
	if bl.Location.SecretConfig != "" {
		secretConfig, err := client.CoreV1().Secrets(bl.Namespace).Get(bl.Location.SecretConfig, metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("error getting secretConfig for backupLocation: %v", err)
		}
		if val, ok := secretConfig.Data["storageAccountName"]; ok && val != nil {
			bl.Location.AzureConfig.StorageAccountName = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["storageAccountKey"]; ok && val != nil {
			bl.Location.AzureConfig.StorageAccountKey = strings.TrimSuffix(string(val), "\n")
		}
	}
	return nil
}

func (bl *BackupLocation) getMergedGoogleConfig(client kubernetes.Interface) error {
	if bl.Location.GoogleConfig == nil {
		bl.Location.GoogleConfig = &GoogleConfig{}
	}
	if bl.Location.SecretConfig != "" {
		secretConfig, err := client.CoreV1().Secrets(bl.Namespace).Get(bl.Location.SecretConfig, metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("error getting secretConfig for backupLocation: %v", err)
		}
		if val, ok := secretConfig.Data["projectID"]; ok && val != nil {
			bl.Location.GoogleConfig.ProjectID = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["accountKey"]; ok && val != nil {
			bl.Location.GoogleConfig.AccountKey = strings.TrimSuffix(string(val), "\n")
		}
	}
	return nil
}
<commit_msg>Add storageclass to s3 config in BackupLocation<commit_after>package v1alpha1

import (
	"fmt"
	"strconv"
	"strings"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

const (
	// BackupLocationResourceName is name for "backuplocation" resource
	BackupLocationResourceName = "backuplocation"
	// BackupLocationResourcePlural is plural for "backuplocation" resource
	BackupLocationResourcePlural = "backuplocations"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// BackupLocation represents a backuplocation object
type BackupLocation struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`
	Location          BackupLocationItem `json:"location"`
}

// BackupLocationItem is the spec used to store a backup location
// Only one of S3Config, AzureConfig or GoogleConfig should be specified and
// should match the Type field. Members of the config can be specified inline or
// through the SecretConfig
type BackupLocationItem struct {
	Type BackupLocationType `json:"type"`
	// Path is either the bucket or any other path for the backup location
	Path          string        `json:"path"`
	EncryptionKey string        `json:"encryptionKey"`
	S3Config      *S3Config     `json:"s3Config,omitempty"`
	AzureConfig   *AzureConfig  `json:"azureConfig,omitempty"`
	GoogleConfig  *GoogleConfig `json:"googleConfig,omitempty"`
	SecretConfig  string        `json:"secretConfig"`
	Sync          bool          `json:"sync"`
}

// BackupLocationType is the type of the backup location
type BackupLocationType string

const (
	// BackupLocationS3 stores the backup in an S3-compliant objectstore
	BackupLocationS3 BackupLocationType = "s3"
	// BackupLocationAzure stores the backup in Azure Blob Storage
	BackupLocationAzure BackupLocationType = "azure"
	// BackupLocationGoogle stores the backup in Google Cloud Storage
	BackupLocationGoogle BackupLocationType = "google"
)

// S3Config speficies the config required to connect to an S3-compliant
// objectstore
type S3Config struct {
	// Endpoint will be defaulted to s3.amazonaws.com by the controller if not provided
	Endpoint        string `json:"endpoint"`
	AccessKeyID     string `json:"accessKeyID"`
	SecretAccessKey string `json:"secretAccessKey"`
	// Region will be defaulted to us-east-1 by the controller if not provided
	Region string `json:"region"`
	// Disable SSL option if using with a non-AWS S3 objectstore which doesn't
	// have SSL enabled
	DisableSSL bool `json:"disableSSL"`
	// The S3 Storage Class to use when uploading objects. Glacier storage
	// classes are not supported
	StorageClass string `json:"storageClass"`
}

// AzureConfig specifies the config required to connect to Azure Blob Storage
type AzureConfig struct {
	StorageAccountName string `json:"storageAccountName"`
	StorageAccountKey  string `json:"storageAccountKey"`
}

// GoogleConfig specifies the config required to connect to Google Cloud Storage
type GoogleConfig struct {
	ProjectID  string `json:"projectID"`
	AccountKey string `json:"accountKey"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// BackupLocationList is a list of ApplicationBackups
type BackupLocationList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata,omitempty"`

	Items []BackupLocation `json:"items"`
}

// UpdateFromSecret updated the config information from the secret if not provided inline
func (bl *BackupLocation) UpdateFromSecret(client kubernetes.Interface) error {
	if bl.Location.SecretConfig != "" {
		secretConfig, err := client.CoreV1().Secrets(bl.Namespace).Get(bl.Location.SecretConfig, metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("error getting secretConfig for backupLocation: %v", err)
		}
		if val, ok := secretConfig.Data["encryptionKey"]; ok && val != nil {
			bl.Location.EncryptionKey = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["path"]; ok && val != nil {
			bl.Location.Path = strings.TrimSuffix(string(val), "\n")
		}
	}
	switch bl.Location.Type {
	case BackupLocationS3:
		return bl.getMergedS3Config(client)
	case BackupLocationAzure:
		return bl.getMergedAzureConfig(client)
	case BackupLocationGoogle:
		return bl.getMergedGoogleConfig(client)
	default:
		return fmt.Errorf("Invalid BackupLocation type %v", bl.Location.Type)
	}
}

func (bl *BackupLocation) getMergedS3Config(client kubernetes.Interface) error {
	if bl.Location.S3Config == nil {
		bl.Location.S3Config = &S3Config{}
		bl.Location.S3Config.Endpoint = "s3.amazonaws.com"
		bl.Location.S3Config.Region = "us-east-1"
		bl.Location.S3Config.DisableSSL = false
	}
	if bl.Location.SecretConfig != "" {
		secretConfig, err := client.CoreV1().Secrets(bl.Namespace).Get(bl.Location.SecretConfig, metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("error getting secretConfig for backupLocation: %v", err)
		}
		if val, ok := secretConfig.Data["endpoint"]; ok && val != nil {
			bl.Location.S3Config.Endpoint = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["accessKeyID"]; ok && val != nil {
			bl.Location.S3Config.AccessKeyID = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["secretAccessKey"]; ok && val != nil {
			bl.Location.S3Config.SecretAccessKey = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["region"]; ok && val != nil {
			bl.Location.S3Config.Region = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["disableSSL"]; ok && val != nil {
			bl.Location.S3Config.DisableSSL, err = strconv.ParseBool(strings.TrimSuffix(string(val), "\n"))
			if err != nil {
				return fmt.Errorf("error parding disableSSL from Secret: %v", err)
			}
		}
		if val, ok := secretConfig.Data["storageClass"]; ok && val != nil {
			bl.Location.S3Config.StorageClass = strings.TrimSuffix(string(val), "\n")
		}
	}
	return nil
}

func (bl *BackupLocation) getMergedAzureConfig(client kubernetes.Interface) error {
	if bl.Location.AzureConfig == nil {
		bl.Location.AzureConfig = &AzureConfig{}
	}
	if bl.Location.SecretConfig != "" {
		secretConfig, err := client.CoreV1().Secrets(bl.Namespace).Get(bl.Location.SecretConfig, metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("error getting secretConfig for backupLocation: %v", err)
		}
		if val, ok := secretConfig.Data["storageAccountName"]; ok && val != nil {
			bl.Location.AzureConfig.StorageAccountName = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["storageAccountKey"]; ok && val != nil {
			bl.Location.AzureConfig.StorageAccountKey = strings.TrimSuffix(string(val), "\n")
		}
	}
	return nil
}

func (bl *BackupLocation) getMergedGoogleConfig(client kubernetes.Interface) error {
	if bl.Location.GoogleConfig == nil {
		bl.Location.GoogleConfig = &GoogleConfig{}
	}
	if bl.Location.SecretConfig != "" {
		secretConfig, err := client.CoreV1().Secrets(bl.Namespace).Get(bl.Location.SecretConfig, metav1.GetOptions{})
		if err != nil {
			return fmt.Errorf("error getting secretConfig for backupLocation: %v", err)
		}
		if val, ok := secretConfig.Data["projectID"]; ok && val != nil {
			bl.Location.GoogleConfig.ProjectID = strings.TrimSuffix(string(val), "\n")
		}
		if val, ok := secretConfig.Data["accountKey"]; ok && val != nil {
			bl.Location.GoogleConfig.AccountKey = strings.TrimSuffix(string(val), "\n")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package finalizer

import (
	"fmt"
	"reflect"
	"time"

	"k8s.io/klog"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	client "k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion"
	informers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion"
	listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion"
)

// OverlappingBuiltInResources returns the set of built-in group/resources that are persisted
// in storage paths that overlap with CRD storage paths, and should not be deleted
// by this controller if an associated CRD is deleted.
func OverlappingBuiltInResources() map[schema.GroupResource]bool {
	return map[schema.GroupResource]bool{
		{Group: "apiregistration.k8s.io", Resource: "apiservices"}:             true,
		{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}: true,
	}
}

// CRDFinalizer is a controller that finalizes the CRD by deleting all the CRs associated with it.
type CRDFinalizer struct {
	crdClient      client.CustomResourceDefinitionsGetter
	crClientGetter CRClientGetter

	crdLister listers.CustomResourceDefinitionLister
	crdSynced cache.InformerSynced

	// To allow injection for testing.
	syncFn func(key string) error

	queue workqueue.RateLimitingInterface
}

// ListerCollectionDeleter combines rest.Lister and rest.CollectionDeleter.
type ListerCollectionDeleter interface {
	rest.Lister
	rest.CollectionDeleter
}

// CRClientGetter knows how to get a ListerCollectionDeleter for a given CRD UID.
type CRClientGetter interface {
	// GetCustomResourceListerCollectionDeleter gets the ListerCollectionDeleter for the given CRD
	// UID.
	GetCustomResourceListerCollectionDeleter(crd *apiextensions.CustomResourceDefinition) (ListerCollectionDeleter, error)
}

// NewCRDFinalizer creates a new CRDFinalizer.
func NewCRDFinalizer(
	crdInformer informers.CustomResourceDefinitionInformer,
	crdClient client.CustomResourceDefinitionsGetter,
	crClientGetter CRClientGetter,
) *CRDFinalizer {
	c := &CRDFinalizer{
		crdClient:      crdClient,
		crdLister:      crdInformer.Lister(),
		crdSynced:      crdInformer.Informer().HasSynced,
		crClientGetter: crClientGetter,
		queue:          workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_finalizer"),
	}

	crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addCustomResourceDefinition,
		UpdateFunc: c.updateCustomResourceDefinition,
	})

	c.syncFn = c.sync

	return c
}

func (c *CRDFinalizer) sync(key string) error {
	cachedCRD, err := c.crdLister.Get(key)
	if apierrors.IsNotFound(err) {
		return nil
	}
	if err != nil {
		return err
	}

	// no work to do
	if cachedCRD.DeletionTimestamp.IsZero() || !apiextensions.CRDHasFinalizer(cachedCRD, apiextensions.CustomResourceCleanupFinalizer) {
		return nil
	}

	crd := cachedCRD.DeepCopy()

	// update the status condition.  This cleanup could take a while.
	apiextensions.SetCRDCondition(crd, apiextensions.CustomResourceDefinitionCondition{
		Type:    apiextensions.Terminating,
		Status:  apiextensions.ConditionTrue,
		Reason:  "InstanceDeletionInProgress",
		Message: "CustomResource deletion is in progress",
	})
	crd, err = c.crdClient.CustomResourceDefinitions().UpdateStatus(crd)
	if apierrors.IsNotFound(err) || apierrors.IsConflict(err) {
		// deleted or changed in the meantime, we'll get called again
		return nil
	}
	if err != nil {
		return err
	}

	// Now we can start deleting items.  We should use the REST API to ensure that all normal admission runs.
	// Since we control the endpoints, we know that delete collection works. No need to delete if not established.
	if OverlappingBuiltInResources()[schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}] {
		// Skip deletion, explain why, and proceed to remove the finalizer and delete the CRD
		apiextensions.SetCRDCondition(crd, apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionFalse,
			Reason:  "OverlappingBuiltInResource",
			Message: "instances overlap with built-in resources in storage",
		})
	} else if apiextensions.IsCRDConditionTrue(crd, apiextensions.Established) {
		cond, deleteErr := c.deleteInstances(crd)
		apiextensions.SetCRDCondition(crd, cond)
		if deleteErr != nil {
			if _, err = c.crdClient.CustomResourceDefinitions().UpdateStatus(crd); err != nil {
				utilruntime.HandleError(err)
			}
			return deleteErr
		}
	} else {
		apiextensions.SetCRDCondition(crd, apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionFalse,
			Reason:  "NeverEstablished",
			Message: "resource was never established",
		})
	}

	apiextensions.CRDRemoveFinalizer(crd, apiextensions.CustomResourceCleanupFinalizer)
	crd, err = c.crdClient.CustomResourceDefinitions().UpdateStatus(crd)
	if apierrors.IsNotFound(err) || apierrors.IsConflict(err) {
		// deleted or changed in the meantime, we'll get called again
		return nil
	}
	return err
}

func (c *CRDFinalizer) deleteInstances(crd *apiextensions.CustomResourceDefinition) (apiextensions.CustomResourceDefinitionCondition, error) {
	// Now we can start deleting items. While it would be ideal to use a REST API client, doing so
	// could incorrectly delete a ThirdPartyResource with the same URL as the CustomResource, so we go
	// directly to the storage instead. Since we control the storage, we know that delete collection works.
	crClient, err := c.crClientGetter.GetCustomResourceListerCollectionDeleter(crd)
	if err != nil {
		err = fmt.Errorf("unable to find a custom resource client for %s.%s: %v", crd.Status.AcceptedNames.Plural, crd.Spec.Group, err)
		return apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionTrue,
			Reason:  "InstanceDeletionFailed",
			Message: fmt.Sprintf("could not list instances: %v", err),
		}, err
	}

	ctx := genericapirequest.NewContext()
	allResources, err := crClient.List(ctx, nil)
	if err != nil {
		return apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionTrue,
			Reason:  "InstanceDeletionFailed",
			Message: fmt.Sprintf("could not list instances: %v", err),
		}, err
	}

	deletedNamespaces := sets.String{}
	deleteErrors := []error{}
	for _, item := range allResources.(*unstructured.UnstructuredList).Items {
		metadata, err := meta.Accessor(&item)
		if err != nil {
			utilruntime.HandleError(err)
			continue
		}
		if deletedNamespaces.Has(metadata.GetNamespace()) {
			continue
		}
		// don't retry deleting the same namespace
		deletedNamespaces.Insert(metadata.GetNamespace())
		nsCtx := genericapirequest.WithNamespace(ctx, metadata.GetNamespace())
		if _, err := crClient.DeleteCollection(nsCtx, rest.ValidateAllObjectFunc, nil, nil); err != nil {
			deleteErrors = append(deleteErrors, err)
			continue
		}
	}
	if deleteError := utilerrors.NewAggregate(deleteErrors); deleteError != nil {
		return apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionTrue,
			Reason:  "InstanceDeletionFailed",
			Message: fmt.Sprintf("could not issue all deletes: %v", deleteError),
		}, deleteError
	}

	// now we need to wait until all the resources are deleted.  Start with a simple poll before we do anything fancy.
	// TODO not all servers are synchronized on caches.  It is possible for a stale one to still be creating things.
	// Once we have a mechanism for servers to indicate their states, we should check that for concurrence.
	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
		listObj, err := crClient.List(ctx, nil)
		if err != nil {
			return false, err
		}
		if len(listObj.(*unstructured.UnstructuredList).Items) == 0 {
			return true, nil
		}
		klog.V(2).Infof("%s.%s waiting for %d items to be removed", crd.Status.AcceptedNames.Plural, crd.Spec.Group, len(listObj.(*unstructured.UnstructuredList).Items))
		return false, nil
	})
	if err != nil {
		return apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionTrue,
			Reason:  "InstanceDeletionCheck",
			Message: fmt.Sprintf("could not confirm zero CustomResources remaining: %v", err),
		}, err
	}
	return apiextensions.CustomResourceDefinitionCondition{
		Type:    apiextensions.Terminating,
		Status:  apiextensions.ConditionFalse,
		Reason:  "InstanceDeletionCompleted",
		Message: "removed all instances",
	}, nil
}

func (c *CRDFinalizer) Run(workers int, stopCh <-chan struct{}) {
	defer utilruntime.HandleCrash()
	defer c.queue.ShutDown()

	klog.Infof("Starting CRDFinalizer")
	defer klog.Infof("Shutting down CRDFinalizer")

	if !cache.WaitForCacheSync(stopCh, c.crdSynced) {
		return
	}

	for i := 0; i < workers; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	<-stopCh
}

func (c *CRDFinalizer) runWorker() {
	for c.processNextWorkItem() {
	}
}

// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
func (c *CRDFinalizer) processNextWorkItem() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)

	err := c.syncFn(key.(string))
	if err == nil {
		c.queue.Forget(key)
		return true
	}

	utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
	c.queue.AddRateLimited(key)

	return true
}

func (c *CRDFinalizer) enqueue(obj *apiextensions.CustomResourceDefinition) {
	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", obj, err))
		return
	}

	c.queue.Add(key)
}

func (c *CRDFinalizer) addCustomResourceDefinition(obj interface{}) {
	castObj := obj.(*apiextensions.CustomResourceDefinition)
	// only queue deleted things
	if !castObj.DeletionTimestamp.IsZero() && apiextensions.CRDHasFinalizer(castObj, apiextensions.CustomResourceCleanupFinalizer) {
		c.enqueue(castObj)
	}
}

func (c *CRDFinalizer) updateCustomResourceDefinition(oldObj, newObj interface{}) {
	oldCRD := oldObj.(*apiextensions.CustomResourceDefinition)
	newCRD := newObj.(*apiextensions.CustomResourceDefinition)
	// only queue deleted things that haven't been finalized by us
	if newCRD.DeletionTimestamp.IsZero() || !apiextensions.CRDHasFinalizer(newCRD, apiextensions.CustomResourceCleanupFinalizer) {
		return
	}

	// always requeue resyncs just in case
	if oldCRD.ResourceVersion == newCRD.ResourceVersion {
		c.enqueue(newCRD)
		return
	}

	// If the only difference is in the terminating condition, then there's no reason to requeue here.  This controller
	// is likely to be the originator, so requeuing would hot-loop us.  Failures are requeued by the workqueue directly.
	// This is a low traffic and scale resource, so the copy is terrible.  It's not good, so better ideas
	// are welcome.
	oldCopy := oldCRD.DeepCopy()
	newCopy := newCRD.DeepCopy()
	oldCopy.ResourceVersion = ""
	newCopy.ResourceVersion = ""
	apiextensions.RemoveCRDCondition(oldCopy, apiextensions.Terminating)
	apiextensions.RemoveCRDCondition(newCopy, apiextensions.Terminating)

	if !reflect.DeepEqual(oldCopy, newCopy) {
		c.enqueue(newCRD)
	}
}
<commit_msg>Fix a new staticcheck issue. vendor/k8s.io/apiextensions-apiserver/pkg/controller/finalizer/crd_finalizer.go:167:2: this value of crd is never used (SA4006)<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package finalizer

import (
	"fmt"
	"reflect"
	"time"

	"k8s.io/klog"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/wait"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"

	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
	client "k8s.io/apiextensions-apiserver/pkg/client/clientset/internalclientset/typed/apiextensions/internalversion"
	informers "k8s.io/apiextensions-apiserver/pkg/client/informers/internalversion/apiextensions/internalversion"
	listers "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/internalversion"
)

// OverlappingBuiltInResources returns the set of built-in group/resources that are persisted
// in storage paths that overlap with CRD storage paths, and should not be deleted
// by this controller if an associated CRD is deleted.
func OverlappingBuiltInResources() map[schema.GroupResource]bool {
	return map[schema.GroupResource]bool{
		{Group: "apiregistration.k8s.io", Resource: "apiservices"}:             true,
		{Group: "apiextensions.k8s.io", Resource: "customresourcedefinitions"}: true,
	}
}

// CRDFinalizer is a controller that finalizes the CRD by deleting all the CRs associated with it.
type CRDFinalizer struct {
	crdClient      client.CustomResourceDefinitionsGetter
	crClientGetter CRClientGetter

	crdLister listers.CustomResourceDefinitionLister
	crdSynced cache.InformerSynced

	// To allow injection for testing.
	syncFn func(key string) error

	queue workqueue.RateLimitingInterface
}

// ListerCollectionDeleter combines rest.Lister and rest.CollectionDeleter.
type ListerCollectionDeleter interface {
	rest.Lister
	rest.CollectionDeleter
}

// CRClientGetter knows how to get a ListerCollectionDeleter for a given CRD UID.
type CRClientGetter interface {
	// GetCustomResourceListerCollectionDeleter gets the ListerCollectionDeleter for the given CRD
	// UID.
	GetCustomResourceListerCollectionDeleter(crd *apiextensions.CustomResourceDefinition) (ListerCollectionDeleter, error)
}

// NewCRDFinalizer creates a new CRDFinalizer.
func NewCRDFinalizer(
	crdInformer informers.CustomResourceDefinitionInformer,
	crdClient client.CustomResourceDefinitionsGetter,
	crClientGetter CRClientGetter,
) *CRDFinalizer {
	c := &CRDFinalizer{
		crdClient:      crdClient,
		crdLister:      crdInformer.Lister(),
		crdSynced:      crdInformer.Informer().HasSynced,
		crClientGetter: crClientGetter,
		queue:          workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "crd_finalizer"),
	}

	crdInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    c.addCustomResourceDefinition,
		UpdateFunc: c.updateCustomResourceDefinition,
	})

	c.syncFn = c.sync

	return c
}

func (c *CRDFinalizer) sync(key string) error {
	cachedCRD, err := c.crdLister.Get(key)
	if apierrors.IsNotFound(err) {
		return nil
	}
	if err != nil {
		return err
	}

	// no work to do
	if cachedCRD.DeletionTimestamp.IsZero() || !apiextensions.CRDHasFinalizer(cachedCRD, apiextensions.CustomResourceCleanupFinalizer) {
		return nil
	}

	crd := cachedCRD.DeepCopy()

	// update the status condition.  This cleanup could take a while.
	apiextensions.SetCRDCondition(crd, apiextensions.CustomResourceDefinitionCondition{
		Type:    apiextensions.Terminating,
		Status:  apiextensions.ConditionTrue,
		Reason:  "InstanceDeletionInProgress",
		Message: "CustomResource deletion is in progress",
	})
	crd, err = c.crdClient.CustomResourceDefinitions().UpdateStatus(crd)
	if apierrors.IsNotFound(err) || apierrors.IsConflict(err) {
		// deleted or changed in the meantime, we'll get called again
		return nil
	}
	if err != nil {
		return err
	}

	// Now we can start deleting items.  We should use the REST API to ensure that all normal admission runs.
	// Since we control the endpoints, we know that delete collection works. No need to delete if not established.
	if OverlappingBuiltInResources()[schema.GroupResource{Group: crd.Spec.Group, Resource: crd.Spec.Names.Plural}] {
		// Skip deletion, explain why, and proceed to remove the finalizer and delete the CRD
		apiextensions.SetCRDCondition(crd, apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionFalse,
			Reason:  "OverlappingBuiltInResource",
			Message: "instances overlap with built-in resources in storage",
		})
	} else if apiextensions.IsCRDConditionTrue(crd, apiextensions.Established) {
		cond, deleteErr := c.deleteInstances(crd)
		apiextensions.SetCRDCondition(crd, cond)
		if deleteErr != nil {
			if _, err = c.crdClient.CustomResourceDefinitions().UpdateStatus(crd); err != nil {
				utilruntime.HandleError(err)
			}
			return deleteErr
		}
	} else {
		apiextensions.SetCRDCondition(crd, apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionFalse,
			Reason:  "NeverEstablished",
			Message: "resource was never established",
		})
	}

	apiextensions.CRDRemoveFinalizer(crd, apiextensions.CustomResourceCleanupFinalizer)
	_, err = c.crdClient.CustomResourceDefinitions().UpdateStatus(crd)
	if apierrors.IsNotFound(err) || apierrors.IsConflict(err) {
		// deleted or changed in the meantime, we'll get called again
		return nil
	}
	return err
}

func (c *CRDFinalizer) deleteInstances(crd *apiextensions.CustomResourceDefinition) (apiextensions.CustomResourceDefinitionCondition, error) {
	// Now we can start deleting items. While it would be ideal to use a REST API client, doing so
	// could incorrectly delete a ThirdPartyResource with the same URL as the CustomResource, so we go
	// directly to the storage instead. Since we control the storage, we know that delete collection works.
	crClient, err := c.crClientGetter.GetCustomResourceListerCollectionDeleter(crd)
	if err != nil {
		err = fmt.Errorf("unable to find a custom resource client for %s.%s: %v", crd.Status.AcceptedNames.Plural, crd.Spec.Group, err)
		return apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionTrue,
			Reason:  "InstanceDeletionFailed",
			Message: fmt.Sprintf("could not list instances: %v", err),
		}, err
	}

	ctx := genericapirequest.NewContext()
	allResources, err := crClient.List(ctx, nil)
	if err != nil {
		return apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionTrue,
			Reason:  "InstanceDeletionFailed",
			Message: fmt.Sprintf("could not list instances: %v", err),
		}, err
	}

	deletedNamespaces := sets.String{}
	deleteErrors := []error{}
	for _, item := range allResources.(*unstructured.UnstructuredList).Items {
		metadata, err := meta.Accessor(&item)
		if err != nil {
			utilruntime.HandleError(err)
			continue
		}
		if deletedNamespaces.Has(metadata.GetNamespace()) {
			continue
		}
		// don't retry deleting the same namespace
		deletedNamespaces.Insert(metadata.GetNamespace())
		nsCtx := genericapirequest.WithNamespace(ctx, metadata.GetNamespace())
		if _, err := crClient.DeleteCollection(nsCtx, rest.ValidateAllObjectFunc, nil, nil); err != nil {
			deleteErrors = append(deleteErrors, err)
			continue
		}
	}
	if deleteError := utilerrors.NewAggregate(deleteErrors); deleteError != nil {
		return apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionTrue,
			Reason:  "InstanceDeletionFailed",
			Message: fmt.Sprintf("could not issue all deletes: %v", deleteError),
		}, deleteError
	}

	// now we need to wait until all the resources are deleted.  Start with a simple poll before we do anything fancy.
	// TODO not all servers are synchronized on caches.  It is possible for a stale one to still be creating things.
	// Once we have a mechanism for servers to indicate their states, we should check that for concurrence.
	err = wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) {
		listObj, err := crClient.List(ctx, nil)
		if err != nil {
			return false, err
		}
		if len(listObj.(*unstructured.UnstructuredList).Items) == 0 {
			return true, nil
		}
		klog.V(2).Infof("%s.%s waiting for %d items to be removed", crd.Status.AcceptedNames.Plural, crd.Spec.Group, len(listObj.(*unstructured.UnstructuredList).Items))
		return false, nil
	})
	if err != nil {
		return apiextensions.CustomResourceDefinitionCondition{
			Type:    apiextensions.Terminating,
			Status:  apiextensions.ConditionTrue,
			Reason:  "InstanceDeletionCheck",
			Message: fmt.Sprintf("could not confirm zero CustomResources remaining: %v", err),
		}, err
	}
	return apiextensions.CustomResourceDefinitionCondition{
		Type:    apiextensions.Terminating,
		Status:  apiextensions.ConditionFalse,
		Reason:  "InstanceDeletionCompleted",
		Message: "removed all instances",
	}, nil
}

func (c *CRDFinalizer) Run(workers int, stopCh <-chan struct{}) {
	defer utilruntime.HandleCrash()
	defer c.queue.ShutDown()

	klog.Infof("Starting CRDFinalizer")
	defer klog.Infof("Shutting down CRDFinalizer")

	if !cache.WaitForCacheSync(stopCh, c.crdSynced) {
		return
	}

	for i := 0; i < workers; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	<-stopCh
}

func (c *CRDFinalizer) runWorker() {
	for c.processNextWorkItem() {
	}
}

// processNextWorkItem deals with one key off the queue.  It returns false when it's time to quit.
func (c *CRDFinalizer) processNextWorkItem() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)

	err := c.syncFn(key.(string))
	if err == nil {
		c.queue.Forget(key)
		return true
	}

	utilruntime.HandleError(fmt.Errorf("%v failed with: %v", key, err))
	c.queue.AddRateLimited(key)

	return true
}

func (c *CRDFinalizer) enqueue(obj *apiextensions.CustomResourceDefinition) {
	key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %v", obj, err))
		return
	}

	c.queue.Add(key)
}

func (c *CRDFinalizer) addCustomResourceDefinition(obj interface{}) {
	castObj := obj.(*apiextensions.CustomResourceDefinition)
	// only queue deleted things
	if !castObj.DeletionTimestamp.IsZero() && apiextensions.CRDHasFinalizer(castObj, apiextensions.CustomResourceCleanupFinalizer) {
		c.enqueue(castObj)
	}
}

func (c *CRDFinalizer) updateCustomResourceDefinition(oldObj, newObj interface{}) {
	oldCRD := oldObj.(*apiextensions.CustomResourceDefinition)
	newCRD := newObj.(*apiextensions.CustomResourceDefinition)
	// only queue deleted things that haven't been finalized by us
	if newCRD.DeletionTimestamp.IsZero() || !apiextensions.CRDHasFinalizer(newCRD, apiextensions.CustomResourceCleanupFinalizer) {
		return
	}

	// always requeue resyncs just in case
	if oldCRD.ResourceVersion == newCRD.ResourceVersion {
		c.enqueue(newCRD)
		return
	}

	// If the only difference is in the terminating condition, then there's no reason to requeue here.  This controller
	// is likely to be the originator, so requeuing would hot-loop us.  Failures are requeued by the workqueue directly.
	// This is a low traffic and scale resource, so the copy is terrible.  It's not good, so better ideas
	// are welcome.
	oldCopy := oldCRD.DeepCopy()
	newCopy := newCRD.DeepCopy()
	oldCopy.ResourceVersion = ""
	newCopy.ResourceVersion = ""
	apiextensions.RemoveCRDCondition(oldCopy, apiextensions.Terminating)
	apiextensions.RemoveCRDCondition(newCopy, apiextensions.Terminating)

	if !reflect.DeepEqual(oldCopy, newCopy) {
		c.enqueue(newCRD)
	}
}
<|endoftext|>