<commit_before>package secio

import (
	"crypto/cipher"
	"errors"
	"fmt"
	"io"

	"crypto/hmac"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"
	mpool "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio/mpool"
)

// ErrMACInvalid signals that a MAC verification failed
var ErrMACInvalid = errors.New("MAC verification failed")

// bufPool is a ByteSlicePool for messages. we need buffers because (sadly)
// we cannot encrypt in place-- the user needs their buffer back.
var bufPool = mpool.ByteSlicePool

type etmWriter struct {
	// params
	pool mpool.Pool        // for the buffers with encrypted data
	msg  msgio.WriteCloser // msgio for knowing where boundaries lie
	str  cipher.Stream     // the stream cipher to encrypt with
	mac  HMAC              // the mac to authenticate data with
}

// NewETMWriter Encrypt-Then-MAC
func NewETMWriter(w io.Writer, s cipher.Stream, mac HMAC) msgio.WriteCloser {
	return &etmWriter{msg: msgio.NewWriter(w), str: s, mac: mac, pool: bufPool}
}

// Write writes passed in buffer as a single message.
func (w *etmWriter) Write(b []byte) (int, error) {
	if err := w.WriteMsg(b); err != nil {
		return 0, err
	}
	return len(b), nil
}

// WriteMsg writes the msg in the passed in buffer.
func (w *etmWriter) WriteMsg(b []byte) error {

	// encrypt.
	data := w.pool.Get(uint32(len(b))).([]byte)
	data = data[:len(b)] // the pool's buffer may be larger
	w.str.XORKeyStream(data, b)

	// log.Debugf("ENC plaintext (%d): %s %v", len(b), b, b)
	// log.Debugf("ENC ciphertext (%d): %s %v", len(data), data, data)

	// then, mac.
	if _, err := w.mac.Write(data); err != nil {
		return err
	}

	// Sum appends.
	data = w.mac.Sum(data)
	w.mac.Reset()
	// it's sad to append here. our buffers are -- hopefully -- coming from
	// a shared buffer pool, so the append may not actually cause allocation
	// one can only hope. i guess we'll see.

	return w.msg.WriteMsg(data)
}

func (w *etmWriter) Close() error {
	return w.msg.Close()
}

type etmReader struct {
	msgio.Reader
	io.Closer

	// params
	msg msgio.ReadCloser // msgio for knowing where boundaries lie
	str cipher.Stream    // the stream cipher to encrypt with
	mac HMAC             // the mac to authenticate data with
}

// NewETMReader Encrypt-Then-MAC
func NewETMReader(r io.Reader, s cipher.Stream, mac HMAC) msgio.ReadCloser {
	return &etmReader{msg: msgio.NewReader(r), str: s, mac: mac}
}

func (r *etmReader) NextMsgLen() (int, error) {
	return r.msg.NextMsgLen()
}

func (r *etmReader) Read(buf []byte) (int, error) {
	// first, check the buffer has enough space.
	fullLen, err := r.msg.NextMsgLen()
	if err != nil {
		return 0, err
	}

	dataLen := fullLen - r.mac.size
	if cap(buf) < dataLen {
		return 0, io.ErrShortBuffer
	}

	buf2 := buf
	changed := false
	if cap(buf) < fullLen {
		buf2 = make([]byte, fullLen)
		changed = true
	}
	buf2 = buf2[:fullLen]

	n, err := io.ReadFull(r.msg, buf2)
	if err != nil {
		return n, err
	}

	m, err := r.macCheckThenDecrypt(buf2)
	if err != nil {
		return 0, err
	}
	buf2 = buf2[:m]
	if changed {
		return copy(buf, buf2), nil
	}
	return m, nil
}

func (r *etmReader) ReadMsg() ([]byte, error) {
	msg, err := r.msg.ReadMsg()
	if err != nil {
		return nil, err
	}

	n, err := r.macCheckThenDecrypt(msg)
	if err != nil {
		return nil, err
	}
	return msg[:n], nil
}

func (r *etmReader) macCheckThenDecrypt(m []byte) (int, error) {
	l := len(m)
	if l < r.mac.size {
		return 0, fmt.Errorf("buffer (%d) shorter than MAC size (%d)", l, r.mac.size)
	}

	mark := l - r.mac.size
	data := m[:mark]
	macd := m[mark:]

	r.mac.Write(data)
	expected := r.mac.Sum(nil)
	r.mac.Reset()

	// check mac. if failed, return error.
	if !hmac.Equal(macd, expected) {
		log.Error("MAC Invalid:", expected, "!=", macd)
		return 0, ErrMACInvalid
	}

	// ok seems good. decrypt. (can decrypt in place, yay!)
	// log.Debugf("DEC ciphertext (%d): %s %v", len(data), data, data)
	r.str.XORKeyStream(data, data)
	// log.Debugf("DEC plaintext (%d): %s %v", len(data), data, data)

	return mark, nil
}

func (w *etmReader) Close() error {
	return w.msg.Close()
}

// ReleaseMsg signals a buffer can be reused.
func (r *etmReader) ReleaseMsg(b []byte) {
	r.msg.ReleaseMsg(b)
}

// writeMsgCtx is used by the
func writeMsgCtx(ctx context.Context, w msgio.Writer, msg proto.Message) ([]byte, error) {
	enc, err := proto.Marshal(msg)
	if err != nil {
		return nil, err
	}

	// write in a goroutine so we can exit when our context is cancelled.
	done := make(chan error)
	go func(m []byte) {
		err := w.WriteMsg(m)
		done <- err
	}(enc)

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case e := <-done:
		return enc, e
	}
}

func readMsgCtx(ctx context.Context, r msgio.Reader, p proto.Message) ([]byte, error) {
	var msg []byte

	// read in a goroutine so we can exit when our context is cancelled.
	done := make(chan error)
	go func() {
		var err error
		msg, err = r.ReadMsg()
		done <- err
	}()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case e := <-done:
		if e != nil {
			return nil, e
		}
	}

	return msg, proto.Unmarshal(msg, p)
}
<commit_msg>secio: buffer remainders in calls to Read()<commit_after>package secio

import (
	"crypto/cipher"
	"errors"
	"fmt"
	"io"

	"crypto/hmac"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
	msgio "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio"
	mpool "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-msgio/mpool"
)

// ErrMACInvalid signals that a MAC verification failed
var ErrMACInvalid = errors.New("MAC verification failed")

// bufPool is a ByteSlicePool for messages. we need buffers because (sadly)
// we cannot encrypt in place-- the user needs their buffer back.
var bufPool = mpool.ByteSlicePool

type etmWriter struct {
	// params
	pool mpool.Pool        // for the buffers with encrypted data
	msg  msgio.WriteCloser // msgio for knowing where boundaries lie
	str  cipher.Stream     // the stream cipher to encrypt with
	mac  HMAC              // the mac to authenticate data with
}

// NewETMWriter Encrypt-Then-MAC
func NewETMWriter(w io.Writer, s cipher.Stream, mac HMAC) msgio.WriteCloser {
	return &etmWriter{msg: msgio.NewWriter(w), str: s, mac: mac, pool: bufPool}
}

// Write writes passed in buffer as a single message.
func (w *etmWriter) Write(b []byte) (int, error) {
	if err := w.WriteMsg(b); err != nil {
		return 0, err
	}
	return len(b), nil
}

// WriteMsg writes the msg in the passed in buffer.
func (w *etmWriter) WriteMsg(b []byte) error {

	// encrypt.
	data := w.pool.Get(uint32(len(b))).([]byte)
	data = data[:len(b)] // the pool's buffer may be larger
	w.str.XORKeyStream(data, b)

	// log.Debugf("ENC plaintext (%d): %s %v", len(b), b, b)
	// log.Debugf("ENC ciphertext (%d): %s %v", len(data), data, data)

	// then, mac.
	if _, err := w.mac.Write(data); err != nil {
		return err
	}

	// Sum appends.
	data = w.mac.Sum(data)
	w.mac.Reset()
	// it's sad to append here. our buffers are -- hopefully -- coming from
	// a shared buffer pool, so the append may not actually cause allocation
	// one can only hope. i guess we'll see.

	return w.msg.WriteMsg(data)
}

func (w *etmWriter) Close() error {
	return w.msg.Close()
}

type etmReader struct {
	msgio.Reader
	io.Closer

	// buffer
	buf []byte

	// params
	msg msgio.ReadCloser // msgio for knowing where boundaries lie
	str cipher.Stream    // the stream cipher to encrypt with
	mac HMAC             // the mac to authenticate data with
}

// NewETMReader Encrypt-Then-MAC
func NewETMReader(r io.Reader, s cipher.Stream, mac HMAC) msgio.ReadCloser {
	return &etmReader{msg: msgio.NewReader(r), str: s, mac: mac}
}

func (r *etmReader) NextMsgLen() (int, error) {
	return r.msg.NextMsgLen()
}

func (r *etmReader) drainBuf(buf []byte) int {
	if r.buf == nil {
		return 0
	}

	n := copy(buf, r.buf)
	r.buf = r.buf[n:]
	return n
}

func (r *etmReader) Read(buf []byte) (int, error) {
	// first, check if we have anything in the buffer
	copied := r.drainBuf(buf)
	buf = buf[copied:]
	if copied > 0 {
		return copied, nil
		// return here to avoid complicating the rest...
		// user can call io.ReadFull.
	}

	// check the buffer has enough space for the next msg
	fullLen, err := r.msg.NextMsgLen()
	if err != nil {
		return 0, err
	}

	buf2 := buf
	changed := false
	// if not enough space, allocate a new buffer.
	if cap(buf) < fullLen {
		buf2 = make([]byte, fullLen)
		changed = true
	}
	buf2 = buf2[:fullLen]

	n, err := io.ReadFull(r.msg, buf2)
	if err != nil {
		return n, err
	}

	m, err := r.macCheckThenDecrypt(buf2)
	if err != nil {
		return 0, err
	}
	buf2 = buf2[:m]
	if !changed {
		return m, nil
	}

	n = copy(buf, buf2)
	if len(buf2) > len(buf) {
		r.buf = buf2[len(buf):] // had some left over? save it.
	}
	return n, nil
}

func (r *etmReader) ReadMsg() ([]byte, error) {
	msg, err := r.msg.ReadMsg()
	if err != nil {
		return nil, err
	}

	n, err := r.macCheckThenDecrypt(msg)
	if err != nil {
		return nil, err
	}
	return msg[:n], nil
}

func (r *etmReader) macCheckThenDecrypt(m []byte) (int, error) {
	l := len(m)
	if l < r.mac.size {
		return 0, fmt.Errorf("buffer (%d) shorter than MAC size (%d)", l, r.mac.size)
	}

	mark := l - r.mac.size
	data := m[:mark]
	macd := m[mark:]

	r.mac.Write(data)
	expected := r.mac.Sum(nil)
	r.mac.Reset()

	// check mac. if failed, return error.
	if !hmac.Equal(macd, expected) {
		log.Error("MAC Invalid:", expected, "!=", macd)
		return 0, ErrMACInvalid
	}

	// ok seems good. decrypt. (can decrypt in place, yay!)
	// log.Debugf("DEC ciphertext (%d): %s %v", len(data), data, data)
	r.str.XORKeyStream(data, data)
	// log.Debugf("DEC plaintext (%d): %s %v", len(data), data, data)

	return mark, nil
}

func (w *etmReader) Close() error {
	return w.msg.Close()
}

// ReleaseMsg signals a buffer can be reused.
func (r *etmReader) ReleaseMsg(b []byte) {
	r.msg.ReleaseMsg(b)
}

// writeMsgCtx is used by the
func writeMsgCtx(ctx context.Context, w msgio.Writer, msg proto.Message) ([]byte, error) {
	enc, err := proto.Marshal(msg)
	if err != nil {
		return nil, err
	}

	// write in a goroutine so we can exit when our context is cancelled.
	done := make(chan error)
	go func(m []byte) {
		err := w.WriteMsg(m)
		done <- err
	}(enc)

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case e := <-done:
		return enc, e
	}
}

func readMsgCtx(ctx context.Context, r msgio.Reader, p proto.Message) ([]byte, error) {
	var msg []byte

	// read in a goroutine so we can exit when our context is cancelled.
	done := make(chan error)
	go func() {
		var err error
		msg, err = r.ReadMsg()
		done <- err
	}()

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case e := <-done:
		if e != nil {
			return nil, e
		}
	}

	return msg, proto.Unmarshal(msg, p)
}
<|endoftext|>


<commit_before>package susigo

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"log"
	"net"
	"regexp"
	"strconv"
	"time"
)

type Callback func(*Event)

type Susi struct {
	cert             tls.Certificate
	addr             string
	connected        bool
	conn             net.Conn
	encoder          *json.Encoder
	decoder          *json.Decoder
	callbacks        map[string]Callback
	consumers        map[string]map[int64]Callback
	processors       map[string]map[int64]Callback
	publishProcesses map[string][]Callback
}

type Event struct {
	Topic     string              `json:"topic"`
	Payload   interface{}         `json:"payload"`
	Headers   []map[string]string `json:"headers"`
	ID        string              `json:"id"`
	SessionID string              `json:"sessionid"`
}

type Message struct {
	Type string `json:"type"`
	Data *Event `json:"data"`
}

func NewSusi(addr, certFile, keyFile string) (*Susi, error) {
	susi := new(Susi)
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return nil, err
	}
	susi.cert = cert
	susi.callbacks = make(map[string]Callback)
	susi.addr = addr
	susi.consumers = make(map[string]map[int64]Callback)
	susi.processors = make(map[string]map[int64]Callback)
	susi.publishProcesses = make(map[string][]Callback)
	susi.connected = false
	go susi.backend()
	return susi, nil
}

func (susi *Susi) Publish(event Event, callback Callback) error {
	if !susi.connected {
		return errors.New("susi not connected")
	}
	var id = event.ID
	if event.ID == "" {
		id = strconv.FormatInt(time.Now().UnixNano(), 10)
		event.ID = id
	}
	event.ID = id
	packet := map[string]interface{}{
		"type": "publish",
		"data": event,
	}
	susi.callbacks[id] = callback
	return susi.encoder.Encode(packet)
}

func (susi *Susi) RegisterConsumer(topic string, callback Callback) (int64, error) {
	if susi.consumers[topic] == nil {
		susi.consumers[topic] = make(map[int64]Callback)
	}
	consumers := susi.consumers[topic]
	id := time.Now().UnixNano()
	consumers[id] = callback
	if len(consumers) == 1 {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": topic,
			},
		}
		if susi.connected {
			return id, susi.encoder.Encode(packet)
		}
		return id, nil
	}
	return -1, nil
}

func (susi *Susi) RegisterProcessor(topic string, callback Callback) (int64, error) {
	if susi.processors[topic] == nil {
		susi.processors[topic] = make(map[int64]Callback)
	}
	processors := susi.processors[topic]
	id := time.Now().UnixNano()
	processors[id] = callback
	if len(processors) == 1 {
		packet := map[string]interface{}{
			"type": "registerProcessor",
			"data": map[string]interface{}{
				"topic": topic,
			},
		}
		if susi.connected {
			return id, susi.encoder.Encode(packet)
		}
		return id, nil
	}
	return -1, nil
}

func (susi *Susi) UnregisterConsumer(id int64) error {
	for _, consumers := range susi.consumers {
		if topic, ok := consumers[id]; ok {
			delete(consumers, id)
			if len(consumers) == 0 {
				packet := map[string]interface{}{
					"type": "unregisterConsumer",
					"data": map[string]interface{}{
						"topic": topic,
					},
				}
				if susi.connected {
					return susi.encoder.Encode(packet)
				}
				return nil
			}
			return nil
		}
	}
	return errors.New("no such consumer")
}

func (susi *Susi) UnregisterProcessor(id int64) error {
	for _, processors := range susi.processors {
		if topic, ok := processors[id]; ok {
			delete(processors, id)
			if len(processors) == 0 {
				packet := map[string]interface{}{
					"type": "unregisterProcessor",
					"data": map[string]interface{}{
						"topic": topic,
					},
				}
				if susi.connected {
					return susi.encoder.Encode(packet)
				}
				return nil
			}
			return nil
		}
	}
	return errors.New("no such processor")
}

func (susi *Susi) connect() error {
	conn, err := tls.Dial("tcp", susi.addr, &tls.Config{
		Certificates:       []tls.Certificate{susi.cert},
		InsecureSkipVerify: true,
	})
	if err != nil {
		log.Printf("failed connecting susi-core (%v), retry...", err)
		return err
	}
	susi.conn = conn
	susi.connected = true
	susi.encoder = json.NewEncoder(susi.conn)
	susi.decoder = json.NewDecoder(susi.conn)
	for consumerTopic := range susi.consumers {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": consumerTopic,
			},
		}
		susi.encoder.Encode(packet)
	}
	for processorTopic := range susi.processors {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": processorTopic,
			},
		}
		susi.encoder.Encode(packet)
	}
	return nil
}

func (susi *Susi) backend() {
	packet := Message{}
	for {
		if !susi.connected {
			err := susi.connect()
			if err != nil {
				time.Sleep(1 * time.Second)
				continue
			}
		}
		err := susi.decoder.Decode(&packet)
		if err != nil {
			log.Println("Error reading from susi json decoder: ", err)
			susi.connected = false
			continue
		}
		switch packet.Type {
		case "ack":
			{
				event := packet.Data
				id := event.ID
				callback := susi.callbacks[id]
				callback(event)
				delete(susi.callbacks, id)
			}
		case "consumerEvent":
			{
				event := packet.Data
				topic := event.Topic
				var matchingConsumers []Callback
				for pattern, consumers := range susi.consumers {
					if matched, err := regexp.MatchString(pattern, topic); err == nil && matched {
						for _, consumer := range consumers {
							matchingConsumers = append(matchingConsumers, consumer)
						}
					}
				}
				for _, consumer := range matchingConsumers {
					consumer(event)
				}
			}
		case "processorEvent":
			{
				event := packet.Data
				topic := event.Topic
				var matchingProcessors []Callback
				for pattern, processors := range susi.processors {
					if matched, err := regexp.MatchString(pattern, topic); err == nil && matched {
						for _, processor := range processors {
							matchingProcessors = append(matchingProcessors, processor)
						}
					}
				}
				susi.publishProcesses[event.ID] = matchingProcessors
				susi.Ack(event)
			}
		}
	}
}

func (susi *Susi) Ack(event *Event) error {
	if process, ok := susi.publishProcesses[event.ID]; ok {
		if len(process) == 0 {
			packet := map[string]interface{}{
				"type": "ack",
				"data": event,
			}
			delete(susi.publishProcesses, event.ID)
			if susi.connected {
				return susi.encoder.Encode(packet)
			}
			return nil
		}
		cb := process[0]
		process = process[1:]
		cb(event)
		return nil
	}
	return errors.New("no publish process found")
}

func (susi *Susi) Dismiss(event *Event) error {
	if _, ok := susi.publishProcesses[event.ID]; ok {
		packet := map[string]interface{}{
			"type": "dismiss",
			"data": event,
		}
		delete(susi.publishProcesses, event.ID)
		if susi.connected {
			return susi.encoder.Encode(packet)
		}
		return nil
	}
	return errors.New("no publish process found")
}
<commit_msg>fixed wrong reregister message type;<commit_after>package susigo

import (
	"crypto/tls"
	"encoding/json"
	"errors"
	"log"
	"net"
	"regexp"
	"strconv"
	"time"
)

type Callback func(*Event)

type Susi struct {
	cert             tls.Certificate
	addr             string
	connected        bool
	conn             net.Conn
	encoder          *json.Encoder
	decoder          *json.Decoder
	callbacks        map[string]Callback
	consumers        map[string]map[int64]Callback
	processors       map[string]map[int64]Callback
	publishProcesses map[string][]Callback
}

type Event struct {
	Topic     string              `json:"topic"`
	Payload   interface{}         `json:"payload"`
	Headers   []map[string]string `json:"headers"`
	ID        string              `json:"id"`
	SessionID string              `json:"sessionid"`
}

type Message struct {
	Type string `json:"type"`
	Data *Event `json:"data"`
}

func NewSusi(addr, certFile, keyFile string) (*Susi, error) {
	susi := new(Susi)
	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
	if err != nil {
		return nil, err
	}
	susi.cert = cert
	susi.callbacks = make(map[string]Callback)
	susi.addr = addr
	susi.consumers = make(map[string]map[int64]Callback)
	susi.processors = make(map[string]map[int64]Callback)
	susi.publishProcesses = make(map[string][]Callback)
	susi.connected = false
	go susi.backend()
	return susi, nil
}

func (susi *Susi) Publish(event Event, callback Callback) error {
	if !susi.connected {
		return errors.New("susi not connected")
	}
	var id = event.ID
	if event.ID == "" {
		id = strconv.FormatInt(time.Now().UnixNano(), 10)
		event.ID = id
	}
	event.ID = id
	packet := map[string]interface{}{
		"type": "publish",
		"data": event,
	}
	susi.callbacks[id] = callback
	return susi.encoder.Encode(packet)
}

func (susi *Susi) RegisterConsumer(topic string, callback Callback) (int64, error) {
	if susi.consumers[topic] == nil {
		susi.consumers[topic] = make(map[int64]Callback)
	}
	consumers := susi.consumers[topic]
	id := time.Now().UnixNano()
	consumers[id] = callback
	if len(consumers) == 1 {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": topic,
			},
		}
		if susi.connected {
			return id, susi.encoder.Encode(packet)
		}
		return id, nil
	}
	return -1, nil
}

func (susi *Susi) RegisterProcessor(topic string, callback Callback) (int64, error) {
	if susi.processors[topic] == nil {
		susi.processors[topic] = make(map[int64]Callback)
	}
	processors := susi.processors[topic]
	id := time.Now().UnixNano()
	processors[id] = callback
	if len(processors) == 1 {
		packet := map[string]interface{}{
			"type": "registerProcessor",
			"data": map[string]interface{}{
				"topic": topic,
			},
		}
		if susi.connected {
			return id, susi.encoder.Encode(packet)
		}
		return id, nil
	}
	return -1, nil
}

func (susi *Susi) UnregisterConsumer(id int64) error {
	for _, consumers := range susi.consumers {
		if topic, ok := consumers[id]; ok {
			delete(consumers, id)
			if len(consumers) == 0 {
				packet := map[string]interface{}{
					"type": "unregisterConsumer",
					"data": map[string]interface{}{
						"topic": topic,
					},
				}
				if susi.connected {
					return susi.encoder.Encode(packet)
				}
				return nil
			}
			return nil
		}
	}
	return errors.New("no such consumer")
}

func (susi *Susi) UnregisterProcessor(id int64) error {
	for _, processors := range susi.processors {
		if topic, ok := processors[id]; ok {
			delete(processors, id)
			if len(processors) == 0 {
				packet := map[string]interface{}{
					"type": "unregisterProcessor",
					"data": map[string]interface{}{
						"topic": topic,
					},
				}
				if susi.connected {
					return susi.encoder.Encode(packet)
				}
				return nil
			}
			return nil
		}
	}
	return errors.New("no such processor")
}

func (susi *Susi) connect() error {
	conn, err := tls.Dial("tcp", susi.addr, &tls.Config{
		Certificates:       []tls.Certificate{susi.cert},
		InsecureSkipVerify: true,
	})
	if err != nil {
		log.Printf("failed connecting susi-core (%v), retry...", err)
		return err
	}
	susi.conn = conn
	susi.connected = true
	susi.encoder = json.NewEncoder(susi.conn)
	susi.decoder = json.NewDecoder(susi.conn)
	for consumerTopic := range susi.consumers {
		packet := map[string]interface{}{
			"type": "registerConsumer",
			"data": map[string]interface{}{
				"topic": consumerTopic,
			},
		}
		susi.encoder.Encode(packet)
	}
	for processorTopic := range susi.processors {
		packet := map[string]interface{}{
			"type": "registerProcessor",
			"data": map[string]interface{}{
				"topic": processorTopic,
			},
		}
		susi.encoder.Encode(packet)
	}
	return nil
}

func (susi *Susi) backend() {
	packet := Message{}
	for {
		if !susi.connected {
			err := susi.connect()
			if err != nil {
				time.Sleep(1 * time.Second)
				continue
			}
		}
		err := susi.decoder.Decode(&packet)
		if err != nil {
			log.Println("Error reading from susi json decoder: ", err)
			susi.connected = false
			continue
		}
		switch packet.Type {
		case "ack":
			{
				event := packet.Data
				id := event.ID
				callback := susi.callbacks[id]
				callback(event)
				delete(susi.callbacks, id)
			}
		case "consumerEvent":
			{
				event := packet.Data
				topic := event.Topic
				var matchingConsumers []Callback
				for pattern, consumers := range susi.consumers {
					if matched, err := regexp.MatchString(pattern, topic); err == nil && matched {
						for _, consumer := range consumers {
							matchingConsumers = append(matchingConsumers, consumer)
						}
					}
				}
				for _, consumer := range matchingConsumers {
					consumer(event)
				}
			}
		case "processorEvent":
			{
				event := packet.Data
				topic := event.Topic
				var matchingProcessors []Callback
				for pattern, processors := range susi.processors {
					if matched, err := regexp.MatchString(pattern, topic); err == nil && matched {
						for _, processor := range processors {
							matchingProcessors = append(matchingProcessors, processor)
						}
					}
				}
				susi.publishProcesses[event.ID] = matchingProcessors
				susi.Ack(event)
			}
		}
	}
}

func (susi *Susi) Ack(event *Event) error {
	if process, ok := susi.publishProcesses[event.ID]; ok {
		if len(process) == 0 {
			packet := map[string]interface{}{
				"type": "ack",
				"data": event,
			}
			delete(susi.publishProcesses, event.ID)
			if susi.connected {
				return susi.encoder.Encode(packet)
			}
			return nil
		}
		cb := process[0]
		process = process[1:]
		cb(event)
		return nil
	}
	return errors.New("no publish process found")
}

func (susi *Susi) Dismiss(event *Event) error {
	if _, ok := susi.publishProcesses[event.ID]; ok {
		packet := map[string]interface{}{
			"type": "dismiss",
			"data": event,
		}
		delete(susi.publishProcesses, event.ID)
		if susi.connected {
			return susi.encoder.Encode(packet)
		}
		return nil
	}
	return errors.New("no publish process found")
}
<|endoftext|>


<commit_before>package server

import (
	"context"
	"fmt"
	"net"
	"strings"

	cnitypes "github.com/containernetworking/cni/pkg/types"
	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
	"github.com/cri-o/cri-o/internal/lib/sandbox"
	"github.com/cri-o/cri-o/internal/pkg/log"
	"k8s.io/kubernetes/pkg/kubelet/dockershim/network/hostport"
)

// networkStart sets up the sandbox's network and returns the pod IP on success
// or an error
func (s *Server) networkStart(ctx context.Context, sb *sandbox.Sandbox) (podIPs []string, result cnitypes.Result, err error) {
	if sb.HostNetwork() {
		return nil, nil, nil
	}

	podNetwork, err := s.newPodNetwork(sb)
	if err != nil {
		return
	}

	// Ensure network resources are cleaned up if the plugin succeeded
	// but an error happened between plugin success and the end of networkStart()
	defer func() {
		if err != nil {
			s.networkStop(ctx, sb)
		}
	}()

	_, err = s.netPlugin.SetUpPod(podNetwork)
	if err != nil {
		err = fmt.Errorf("failed to create pod network sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
		return
	}

	tmp, err := s.netPlugin.GetPodNetworkStatus(podNetwork)
	if err != nil {
		err = fmt.Errorf("failed to get network status for pod sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
		return
	}

	// only one cnitypes.Result is returned since newPodNetwork sets Networks list empty
	result = tmp[0]
	log.Debugf(ctx, "CNI setup result: %v", result)

	network, err := cnicurrent.GetResult(result)
	if err != nil {
		err = fmt.Errorf("failed to get network JSON for pod sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
		return
	}

	for idx, podIPConfig := range network.IPs {
		podIP := strings.Split(podIPConfig.Address.String(), "/")[0]

		// Apply the hostport mappings only for the first IP to avoid allocating
		// the same host port twice
		if idx == 0 && len(sb.PortMappings()) > 0 {
			ip := net.ParseIP(podIP)
			if ip == nil {
				err = fmt.Errorf("failed to get valid ip address for sandbox %s(%s)", sb.Name(), sb.ID())
				return
			}

			err = s.hostportManager.Add(sb.ID(), &hostport.PodPortMapping{
				Name:         sb.Name(),
				PortMappings: sb.PortMappings(),
				IP:           ip,
				HostNetwork:  false,
			}, "lo")
			if err != nil {
				err = fmt.Errorf("failed to add hostport mapping for sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
				return
			}
		}

		podIPs = append(podIPs, podIP)
	}

	log.Debugf(ctx, "found POD IPs: %v", podIPs)
	return podIPs, result, err
}

// getSandboxIP retrieves the IP address for the sandbox
func (s *Server) getSandboxIPs(sb *sandbox.Sandbox) (podIPs []string, err error) {
	if sb.HostNetwork() {
		return nil, nil
	}

	podNetwork, err := s.newPodNetwork(sb)
	if err != nil {
		return nil, err
	}
	result, err := s.netPlugin.GetPodNetworkStatus(podNetwork)
	if err != nil {
		return nil, fmt.Errorf("failed to get network status for pod sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
	}

	res, err := cnicurrent.GetResult(result[0])
	if err != nil {
		return nil, fmt.Errorf("failed to get network JSON for pod sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
	}

	for _, podIPConfig := range res.IPs {
		podIPs = append(podIPs, strings.Split(podIPConfig.Address.String(), "/")[0])
	}

	return podIPs, nil
}

// networkStop cleans up and removes a pod's network.  It is best-effort and
// must call the network plugin even if the network namespace is already gone
func (s *Server) networkStop(ctx context.Context, sb *sandbox.Sandbox) {
	if sb.HostNetwork() {
		return
	}

	if err := s.hostportManager.Remove(sb.ID(), &hostport.PodPortMapping{
		Name:         sb.Name(),
		PortMappings: sb.PortMappings(),
		HostNetwork:  false,
	}); err != nil {
		log.Warnf(ctx, "failed to remove hostport for pod sandbox %s(%s): %v",
			sb.Name(), sb.ID(), err)
	}

	podNetwork, err := s.newPodNetwork(sb)
	if err != nil {
		log.Warnf(ctx, err.Error())
		return
	}
	if err := s.netPlugin.TearDownPod(podNetwork); err != nil {
		log.Warnf(ctx, "failed to destroy network for pod sandbox %s(%s): %v",
			sb.Name(), sb.ID(), err)
	}
}
<commit_msg>Add network setup metrics<commit_after>package server

import (
	"context"
	"fmt"
	"net"
	"strings"
	"time"

	cnitypes "github.com/containernetworking/cni/pkg/types"
	cnicurrent "github.com/containernetworking/cni/pkg/types/current"
	"github.com/cri-o/cri-o/internal/lib/sandbox"
	"github.com/cri-o/cri-o/internal/pkg/log"
	"github.com/cri-o/cri-o/server/metrics"
	"k8s.io/kubernetes/pkg/kubelet/dockershim/network/hostport"
)

// networkStart sets up the sandbox's network and returns the pod IP on success
// or an error
func (s *Server) networkStart(ctx context.Context, sb *sandbox.Sandbox) (podIPs []string, result cnitypes.Result, err error) {
	overallStart := time.Now()

	if sb.HostNetwork() {
		return nil, nil, nil
	}

	podNetwork, err := s.newPodNetwork(sb)
	if err != nil {
		return
	}

	// Ensure network resources are cleaned up if the plugin succeeded
	// but an error happened between plugin success and the end of networkStart()
	defer func() {
		if err != nil {
			s.networkStop(ctx, sb)
		}
	}()

	podSetUpStart := time.Now()
	_, err = s.netPlugin.SetUpPod(podNetwork)
	if err != nil {
		err = fmt.Errorf("failed to create pod network sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
		return
	}
	// metric about the CNI network setup operation
	metrics.CRIOOperationsLatency.WithLabelValues("network_setup_pod").
		Observe(metrics.SinceInMicroseconds(podSetUpStart))

	tmp, err := s.netPlugin.GetPodNetworkStatus(podNetwork)
	if err != nil {
		err = fmt.Errorf("failed to get network status for pod sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
		return
	}

	// only one cnitypes.Result is returned since newPodNetwork sets Networks list empty
	result = tmp[0]
	log.Debugf(ctx, "CNI setup result: %v", result)

	network, err := cnicurrent.GetResult(result)
	if err != nil {
		err = fmt.Errorf("failed to get network JSON for pod sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
		return
	}

	for idx, podIPConfig := range network.IPs {
		podIP := strings.Split(podIPConfig.Address.String(), "/")[0]

		// Apply the hostport mappings only for the first IP to avoid allocating
		// the same host port twice
		if idx == 0 && len(sb.PortMappings()) > 0 {
			ip := net.ParseIP(podIP)
			if ip == nil {
				err = fmt.Errorf("failed to get valid ip address for sandbox %s(%s)", sb.Name(), sb.ID())
				return
			}

			err = s.hostportManager.Add(sb.ID(), &hostport.PodPortMapping{
				Name:         sb.Name(),
				PortMappings: sb.PortMappings(),
				IP:           ip,
				HostNetwork:  false,
			}, "lo")
			if err != nil {
				err = fmt.Errorf("failed to add hostport mapping for sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
				return
			}
		}

		podIPs = append(podIPs, podIP)
	}

	log.Debugf(ctx, "found POD IPs: %v", podIPs)

	// metric about the whole network setup operation
	metrics.CRIOOperationsLatency.WithLabelValues("network_setup_overall").
		Observe(metrics.SinceInMicroseconds(overallStart))
	return podIPs, result, err
}

// getSandboxIP retrieves the IP address for the sandbox
func (s *Server) getSandboxIPs(sb *sandbox.Sandbox) (podIPs []string, err error) {
	if sb.HostNetwork() {
		return nil, nil
	}

	podNetwork, err := s.newPodNetwork(sb)
	if err != nil {
		return nil, err
	}
	result, err := s.netPlugin.GetPodNetworkStatus(podNetwork)
	if err != nil {
		return nil, fmt.Errorf("failed to get network status for pod sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
	}

	res, err := cnicurrent.GetResult(result[0])
	if err != nil {
		return nil, fmt.Errorf("failed to get network JSON for pod sandbox %s(%s): %v", sb.Name(), sb.ID(), err)
	}

	for _, podIPConfig := range res.IPs {
		podIPs = append(podIPs, strings.Split(podIPConfig.Address.String(), "/")[0])
	}

	return podIPs, nil
}

// networkStop cleans up and removes a pod's network.  It is best-effort and
// must call the network plugin even if the network namespace is already gone
func (s *Server) networkStop(ctx context.Context, sb *sandbox.Sandbox) {
	if sb.HostNetwork() {
		return
	}

	if err := s.hostportManager.Remove(sb.ID(), &hostport.PodPortMapping{
		Name:         sb.Name(),
		PortMappings: sb.PortMappings(),
		HostNetwork:  false,
	}); err != nil {
		log.Warnf(ctx, "failed to remove hostport for pod sandbox %s(%s): %v",
			sb.Name(), sb.ID(), err)
	}

	podNetwork, err := s.newPodNetwork(sb)
	if err != nil {
		log.Warnf(ctx, err.Error())
		return
	}
	if err := s.netPlugin.TearDownPod(podNetwork); err != nil {
		log.Warnf(ctx, "failed to destroy network for pod sandbox %s(%s): %v",
			sb.Name(), sb.ID(), err)
	}
}
<|endoftext|>


<commit_before>package pongo2

import (
	"fmt"
	"math"
)

type Expression struct {
	expr1    IEvaluator
	expr2    IEvaluator
	op_token *Token
}

type relationalExpression struct {
	expr1    IEvaluator
	expr2    IEvaluator
	op_token *Token
}

type simpleExpression struct {
	location_token *Token
	negate         bool
	negative_sign  bool
	term1          IEvaluator
	term2          IEvaluator
	op_token       *Token
}

type term struct {
	factor1  IEvaluator
	factor2  IEvaluator
	op_token *Token
}

type power struct {
	power1 IEvaluator
	power2 IEvaluator
}

func (expr *Expression) Execute(ctx *ExecutionContext) (string, error) {
	value, err := expr.Evaluate(ctx)
	if err != nil {
		return "", err
	}
	return value.String(), nil
}

func (expr *Expression) Evaluate(ctx *ExecutionContext) (*Value, error) {
	v1, err := expr.expr1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	if expr.expr2 != nil {
		v2, err := expr.expr2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		switch expr.op_token.Val {
		case "and", "&&":
			return AsValue(v1.IsTrue() && v2.IsTrue()), nil
		case "or", "||":
			return AsValue(v1.IsTrue() || v2.IsTrue()), nil
		default:
			panic(fmt.Sprintf("unimplemented: %s", expr.op_token.Val))
		}
	} else {
		return v1, nil
	}
}

func (expr *relationalExpression) Evaluate(ctx *ExecutionContext) (*Value, error) {
	v1, err := expr.expr1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	if expr.expr2 != nil {
		v2, err := expr.expr2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		switch expr.op_token.Val {
		case "<=":
			if v1.IsFloat() || v2.IsFloat() {
				// Result will be a float
				return AsValue(v1.Float() <= v2.Float()), nil
			} else {
				// Result will be an integer
				return AsValue(v1.Integer() <= v2.Integer()), nil
			}
		case ">=":
			if v1.IsFloat() || v2.IsFloat() {
				// Result will be a float
				return AsValue(v1.Float() >= v2.Float()), nil
			} else {
				// Result will be an integer
				return AsValue(v1.Integer() >= v2.Integer()), nil
			}
		case "==":
			return AsValue(v1.EqualValueTo(v2)), nil
		case ">":
			if v1.IsFloat() || v2.IsFloat() {
				// Result will be a float
				return AsValue(v1.Float() > v2.Float()), nil
			} else {
				// Result will be an integer
				return AsValue(v1.Integer() > v2.Integer()), nil
			}
		case "<":
			if v1.IsFloat() || v2.IsFloat() {
				// Result will be a float
				return AsValue(v1.Float() < v2.Float()), nil
			} else {
				// Result will be an integer
				return AsValue(v1.Integer() < v2.Integer()), nil
			}
		case "!=", "<>":
			return AsValue(!v1.EqualValueTo(v2)), nil
		case "in":
			return AsValue(v2.Contains(v1)), nil
		default:
			panic(fmt.Sprintf("unimplemented: %s", expr.op_token.Val))
		}
	} else {
		return v1, nil
	}
}

func (expr *simpleExpression) Evaluate(ctx *ExecutionContext) (*Value, error) {
	t1, err := expr.term1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	result := t1

	if expr.negate {
		result = result.Negate()
	}

	if expr.negative_sign {
		if result.IsNumber() {
			switch {
			case result.IsFloat():
				result = AsValue(-1 * result.Float())
			case result.IsInteger():
				result = AsValue(-1 * result.Integer())
			default:
				panic("not possible")
			}
		} else {
			return nil, ctx.Error("Negative sign on a non-number expression", expr.location_token)
		}
	}

	if expr.term2 != nil {
		t2, err := expr.term2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		switch expr.op_token.Val {
		case "+":
			if result.IsFloat() || t2.IsFloat() {
				// Result will be a float
				return AsValue(result.Float() + t2.Float()), nil
			} else {
				// Result will be an integer
				return AsValue(result.Integer() + t2.Integer()), nil
			}
		case "-":
			if result.IsFloat() || t2.IsFloat() {
				// Result will be a float
				return AsValue(result.Float() - t2.Float()), nil
			} else {
				// Result will be an integer
				return AsValue(result.Integer() - t2.Integer()), nil
			}
		default:
			panic("unimplemented")
		}
	}

	return result, nil
}

func (t *term) Evaluate(ctx *ExecutionContext) (*Value, error) {
	f1, err := t.factor1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	if t.factor2 != nil {
		f2, err := t.factor2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		switch t.op_token.Val {
		case "*":
			return AsValue(f1.Integer() * f2.Integer()), nil
		case "/":
			panic("unimplemented")
		default:
			panic("unimplemented")
		}
	} else {
		return f1, nil
	}
}

func (pw *power) Evaluate(ctx *ExecutionContext) (*Value, error) {
	p1, err := pw.power1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	if pw.power2 != nil {
		p2, err := pw.power2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		return AsValue(math.Pow(p1.Float(), p2.Float())), nil
	} else {
		return p1, nil
	}
}

func (p *Parser) parseFactor() (IEvaluator, error) {
	if p.Match(TokenSymbol, "(") != nil {
		expr, err := p.ParseExpression()
		if err != nil {
			return nil, err
		}
		if p.Match(TokenSymbol, ")") == nil {
			return nil, p.Error("Closing bracket expected after expression", nil)
		}
		return expr, nil
	}

	return p.parseVariableOrLiteralWithFilter()
}

func (p *Parser) parseTerm() (IEvaluator, error) {
	term := new(term)

	factor1, err := p.parsePower()
	if err != nil {
		return nil, err
	}
	term.factor1 = factor1

	if p.PeekOne(TokenSymbol, "*", "/") != nil {
		op := p.Current()
		p.Consume()
		factor2, err := p.parseTerm()
		if err != nil {
			return nil, err
		}
		term.factor2 = factor2
		term.op_token = op
	}

	return term, nil
}

func (p *Parser) parsePower() (IEvaluator, error) {
	pw := new(power)

	power1, err := p.parseFactor()
	if err != nil {
		return nil, err
	}
	pw.power1 = power1

	if p.Match(TokenSymbol, "^") != nil {
		power2, err := p.parsePower()
		if err != nil {
			return nil, err
		}
		pw.power2 = power2
	}

	return pw, nil
}

func (p *Parser) parseSimpleExpression() (IEvaluator, error) {
	expr := new(simpleExpression)
	expr.location_token = p.Current()

	if sign := p.MatchOne(TokenSymbol, "+", "-"); sign != nil {
		if sign.Val == "-" {
			expr.negative_sign = true
		}
	}

	if p.Match(TokenSymbol, "!") != nil || p.Match(TokenKeyword, "not") != nil {
		expr.negate = true
	}

	term1, err := p.parseTerm()
	if err != nil {
		return nil, err
	}
	expr.term1 = term1

	if p.PeekOne(TokenSymbol, "+", "-") != nil {
		op := p.Current()
		p.Consume()
		term2, err := p.parseSimpleExpression()
		if err != nil {
			return nil, err
		}
		expr.term2 = term2
		expr.op_token = op
	}

	return expr, nil
}

func (p *Parser) parseRelationalExpression() (IEvaluator, error) {
	expr1, err := p.parseSimpleExpression()
	if err != nil {
		return nil, err
	}

	expr := &relationalExpression{
		expr1: expr1,
	}

	if t := p.MatchOne(TokenSymbol, "==", "<=", ">=", "!=", "<>", ">", "<"); t != nil {
		expr2, err := p.parseRelationalExpression()
		if err != nil {
			return nil, err
		}
		expr.op_token = t
		expr.expr2 = expr2
	} else if t := p.MatchOne(TokenKeyword, "in"); t != nil {
		expr2, err := p.parseSimpleExpression()
		if err != nil {
			return nil, err
		}
		expr.op_token = t
		expr.expr2 = expr2
	}

	return expr, nil
}

func (p *Parser) ParseExpression() (INodeEvaluator, error) {
	rexpr1, err := p.parseRelationalExpression()
	if err != nil {
		return nil, err
	}

	exp := &Expression{
		expr1: rexpr1,
	}

	if p.PeekOne(TokenSymbol, "&&", "||") != nil || p.PeekOne(TokenKeyword, "and", "or") != nil {
		op := p.Current()
		p.Consume()
		expr2, err := p.ParseExpression()
		if err != nil {
			return nil, err
		}
		exp.expr2 = expr2
		exp.op_token = op
	}

	return exp, nil
}
<commit_msg>Removed wrong comments.<commit_after>package pongo2

import (
	"fmt"
	"math"
)

type Expression struct {
	expr1    IEvaluator
	expr2    IEvaluator
	op_token *Token
}

type relationalExpression struct {
	expr1    IEvaluator
	expr2    IEvaluator
	op_token *Token
}

type simpleExpression struct {
	location_token *Token
	negate         bool
	negative_sign  bool
	term1          IEvaluator
	term2          IEvaluator
	op_token       *Token
}

type term struct {
	factor1  IEvaluator
	factor2  IEvaluator
	op_token *Token
}

type power struct {
	power1 IEvaluator
	power2 IEvaluator
}

func (expr *Expression) Execute(ctx *ExecutionContext) (string, error) {
	value, err := expr.Evaluate(ctx)
	if err != nil {
		return "", err
	}
	return value.String(), nil
}

func (expr *Expression) Evaluate(ctx *ExecutionContext) (*Value, error) {
	v1, err := expr.expr1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	if expr.expr2 != nil {
		v2, err := expr.expr2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		switch expr.op_token.Val {
		case "and", "&&":
			return AsValue(v1.IsTrue() && v2.IsTrue()), nil
		case "or", "||":
			return AsValue(v1.IsTrue() || v2.IsTrue()), nil
		default:
			panic(fmt.Sprintf("unimplemented: %s", expr.op_token.Val))
		}
	} else {
		return v1, nil
	}
}

func (expr *relationalExpression) Evaluate(ctx *ExecutionContext) (*Value, error) {
	v1, err := expr.expr1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	if expr.expr2 != nil {
		v2, err := expr.expr2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		switch expr.op_token.Val {
		case "<=":
			if v1.IsFloat() || v2.IsFloat() {
				return AsValue(v1.Float() <= v2.Float()), nil
			} else {
				return AsValue(v1.Integer() <= v2.Integer()), nil
			}
		case ">=":
			if v1.IsFloat() || v2.IsFloat() {
				return AsValue(v1.Float() >= v2.Float()), nil
			} else {
				return AsValue(v1.Integer() >= v2.Integer()), nil
			}
		case "==":
			return AsValue(v1.EqualValueTo(v2)), nil
		case ">":
			if v1.IsFloat() || v2.IsFloat() {
				return AsValue(v1.Float() > v2.Float()), nil
			} else {
				return AsValue(v1.Integer() > v2.Integer()), nil
			}
		case "<":
			if v1.IsFloat() || v2.IsFloat() {
				return AsValue(v1.Float() < v2.Float()), nil
			} else {
				return AsValue(v1.Integer() < v2.Integer()), nil
			}
		case "!=", "<>":
			return AsValue(!v1.EqualValueTo(v2)), nil
		case "in":
			return AsValue(v2.Contains(v1)), nil
		default:
			panic(fmt.Sprintf("unimplemented: %s", expr.op_token.Val))
		}
	} else {
		return v1, nil
	}
}

func (expr *simpleExpression) Evaluate(ctx *ExecutionContext) (*Value, error) {
	t1, err := expr.term1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	result := t1

	if expr.negate {
		result = result.Negate()
	}

	if expr.negative_sign {
		if result.IsNumber() {
			switch {
			case result.IsFloat():
				result = AsValue(-1 * result.Float())
			case result.IsInteger():
				result = AsValue(-1 * result.Integer())
			default:
				panic("not possible")
			}
		} else {
			return nil, ctx.Error("Negative sign on a non-number expression", expr.location_token)
		}
	}

	if expr.term2 != nil {
		t2, err := expr.term2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		switch expr.op_token.Val {
		case "+":
			if result.IsFloat() || t2.IsFloat() {
				// Result will be a float
				return AsValue(result.Float() + t2.Float()), nil
			} else {
				// Result will be an integer
				return AsValue(result.Integer() + t2.Integer()), nil
			}
		case "-":
			if result.IsFloat() || t2.IsFloat() {
				// Result will be a float
				return AsValue(result.Float() - t2.Float()), nil
			} else {
				// Result will be an integer
				return AsValue(result.Integer() - t2.Integer()), nil
			}
		default:
			panic("unimplemented")
		}
	}

	return result, nil
}

func (t *term) Evaluate(ctx *ExecutionContext) (*Value, error) {
	f1, err := t.factor1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	if t.factor2 != nil {
		f2, err := t.factor2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		switch t.op_token.Val {
		case "*":
			return AsValue(f1.Integer() * f2.Integer()), nil
		case "/":
			panic("unimplemented")
		default:
			panic("unimplemented")
		}
	} else {
		return f1, nil
	}
}

func (pw *power) Evaluate(ctx *ExecutionContext) (*Value, error) {
	p1, err := pw.power1.Evaluate(ctx)
	if err != nil {
		return nil, err
	}
	if pw.power2 != nil {
		p2, err := pw.power2.Evaluate(ctx)
		if err != nil {
			return nil, err
		}
		return AsValue(math.Pow(p1.Float(), p2.Float())), nil
	} else {
		return p1, nil
	}
}

func (p *Parser) parseFactor() (IEvaluator, error) {
	if p.Match(TokenSymbol, "(") != nil {
		expr, err := p.ParseExpression()
		if err != nil {
			return nil, err
		}
		if p.Match(TokenSymbol, ")") == nil {
			return nil, p.Error("Closing bracket expected after expression", nil)
		}
		return expr, nil
	}

	return p.parseVariableOrLiteralWithFilter()
}

func (p *Parser) parseTerm() (IEvaluator, error) {
	term := new(term)

	factor1, err := p.parsePower()
	if err != nil {
		return nil, err
	}
	term.factor1 = factor1

	if p.PeekOne(TokenSymbol, "*", "/") != nil {
		op := p.Current()
		p.Consume()
		factor2, err := p.parseTerm()
		if err != nil {
			return nil, err
		}
		term.factor2 = factor2
		term.op_token = op
	}

	return term, nil
}

func (p *Parser) parsePower() (IEvaluator, error) {
	pw := new(power)

	power1, err := p.parseFactor()
	if err != nil {
		return nil, err
	}
	pw.power1 = power1

	if p.Match(TokenSymbol, "^") != nil {
		power2, err := p.parsePower()
		if err != nil {
			return nil, err
		}
		pw.power2 = power2
	}

	return pw, nil
}

func (p *Parser) parseSimpleExpression() (IEvaluator, error) {
	expr := new(simpleExpression)
	expr.location_token = p.Current()

	if sign := p.MatchOne(TokenSymbol, "+", "-"); sign != nil {
		if sign.Val == "-" {
			expr.negative_sign = true
		}
	}

	if p.Match(TokenSymbol, "!") != nil || p.Match(TokenKeyword, "not") != nil {
		expr.negate = true
	}

	term1, err := p.parseTerm()
	if err != nil {
		return nil, err
	}
	expr.term1 = term1

	if p.PeekOne(TokenSymbol, "+", "-") != nil {
		op := p.Current()
		p.Consume()
		term2, err := p.parseSimpleExpression()
		if err != nil {
			return nil, err
		}
		expr.term2 = term2
		expr.op_token = op
	}

	return expr, nil
}

func (p *Parser) parseRelationalExpression() (IEvaluator, error) {
	expr1, err := p.parseSimpleExpression()
	if err != nil {
		return nil, err
	}

	expr := &relationalExpression{
		expr1: expr1,
	}

	if t := p.MatchOne(TokenSymbol, "==", "<=", ">=", "!=", "<>", ">", "<"); t != nil {
		expr2, err := p.parseRelationalExpression()
		if err != nil {
			return nil, err
		}
		expr.op_token = t
		expr.expr2 = expr2
	} else if t := p.MatchOne(TokenKeyword, "in"); t != nil {
		expr2, err := p.parseSimpleExpression()
		if err != nil {
			return nil, err
		}
		expr.op_token = t
		expr.expr2 = expr2
	}

	return expr, nil
}

func (p *Parser) ParseExpression() (INodeEvaluator, error) {
	rexpr1, err := p.parseRelationalExpression()
	if err != nil {
		return nil, err
	}

	exp := &Expression{
		expr1: rexpr1,
	}

	if p.PeekOne(TokenSymbol, "&&", "||") != nil || p.PeekOne(TokenKeyword, "and", "or") != nil {
		op := p.Current()
		p.Consume()
		expr2, err := p.ParseExpression()
		if err != nil {
			return nil, err
		}
		exp.expr2 = expr2
		exp.op_token = op
	}

	return exp, nil
}
<|endoftext|>


<commit_before>package gcs

import (
	"fmt"
	"github.com/astaxie/beego/config"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	storage "google.golang.org/api/storage/v1"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

var (
	jsonKeyFile string
	bucketName  string
	projectID   string
	driverType  string
)

/*
//Should Open this const config to replace below one when use testgcs.go
const (
	jsonfile   = "./gcs/key.json"
	bucketName = "dockyad-example-bucket"
	projectID  = "dockyad-test"
)
*/
func init() {

	//Reading config file named conf/runtime.conf for backend
	conf, err := config.NewConfig("ini", "./runtime.conf")
	if err != nil {
		log.Fatalf("GCS reading conf/runtime.conf err %v", err)
	}

	driverType = conf.String("backenddriver")
	if driverType == "" {
		log.Fatalf("GCS reading conf/runtime.conf, get driverType is nil")
	}
	//Get config var for jsonKeyFile, bucketName, projectID, which should be used later in oauth and get obj
	if jsonKeyFile = conf.String(driverType + "::jsonkeyfile"); jsonKeyFile == "" {
		log.Fatalf("GCS reading conf/runtime.conf, GCS get jsonKeyFile err, is nil")
	}

	if bucketName = conf.String(driverType + "::bucketname"); bucketName == "" {
		log.Fatalf("GCS reading conf/runtime.conf, GCS get bucketName err, is nil")
	}

	if projectID := conf.String(driverType + "::projectid"); projectID == "" {
		log.Fatalf("GCS reading conf/runtime.conf, GCS get projectID err, is nil")
	}
}

func Gcssave(file string) (url string, err error) {

	//read json key(key.json) to do oauth according JWT
	data, err := ioutil.ReadFile(jsonKeyFile)
	if err != nil {
		log.Fatal(err)
	}
	conf, err := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/devstorage.full_control")
	if err != nil {
		log.Fatal(err)
	}

	//new storage service and token, we dont need context here
	client := conf.Client(oauth2.NoContext)
	gcsToken, err := conf.TokenSource(oauth2.NoContext).Token()
	service, err := storage.New(client)
	if err != nil {
		log.Fatalf("GCS unable to create storage service: %v", err)
	}

	// If the bucket already exists and the user has access,  don't try to create it.
	if _, err := service.Buckets.Get(bucketName).Do(); err != nil {
		// If bucket is not exist, Create a bucket.
		if _, err := service.Buckets.Insert(projectID, &storage.Bucket{Name: bucketName}).Do(); err != nil {
			log.Fatalf("GCS failed creating bucket %s: %v", bucketName, err)
		}
	}

	//Split filename as a objectName
	var objectName string
	for _, objectName = range strings.Split(file, "/") {
	}
	object := &storage.Object{Name: objectName}

	// Insert an object into a bucket.
	fileDes, err := os.Open(file)
	if err != nil {
		log.Fatalf("Error opening %q: %v", file, err)
	}
	objs, err := service.Objects.Insert(bucketName, object).Media(fileDes).Do()
	if err != nil {
		log.Fatalf("GCS Objects.Insert failed: %v", err)
	}
	retUrl := objs.MediaLink + "&access_token=" + gcsToken.AccessToken
	fmt.Println(fmt.Sprintf("GCS tmpUrl=%s", retUrl))

	if err != nil {
		return "", err
	} else {
		return retUrl, nil
	}
}
<commit_msg>Modify gcs.go to delete the not neccessary output.<commit_after>package gcs

import (
	"fmt"
	"github.com/astaxie/beego/config"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	storage "google.golang.org/api/storage/v1"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

var (
	jsonKeyFile string
	bucketName  string
	projectID   string
	driverType  string
)

/*
//Should Open this const config to replace below one when use testgcs.go
const (
	jsonfile   = "./gcs/key.json"
	bucketName = "dockyad-example-bucket"
	projectID  = "dockyad-test"
)
*/
func init() {

	//Reading config file named conf/runtime.conf for backend
	conf, err := config.NewConfig("ini", "./runtime.conf")
	if err != nil {
		log.Fatalf("GCS reading conf/runtime.conf err %v", err)
	}

	driverType = conf.String("backenddriver")
	if driverType == "" {
		log.Fatalf("GCS reading conf/runtime.conf, get driverType is nil")
	}
	//Get config var for jsonKeyFile, bucketName, projectID, which should be used later in oauth and get obj
	if jsonKeyFile = conf.String(driverType + "::jsonkeyfile"); jsonKeyFile == "" {
		log.Fatalf("GCS reading conf/runtime.conf, GCS get jsonKeyFile err, is nil")
	}

	if bucketName = conf.String(driverType + "::bucketname"); bucketName == "" {
		log.Fatalf("GCS reading conf/runtime.conf, GCS get bucketName err, is nil")
	}

	if projectID := conf.String(driverType + "::projectid"); projectID == "" {
		log.Fatalf("GCS reading conf/runtime.conf, GCS get projectID err, is nil")
	}
}

func Gcssave(file string) (url string, err error) {

	//read json key(key.json) to do oauth according JWT
	data, err := ioutil.ReadFile(jsonKeyFile)
	if err != nil {
		log.Fatal(err)
	}
	conf, err := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/devstorage.full_control")
	if err != nil {
		log.Fatal(err)
	}

	//new storage service and token, we dont need context here
	client := conf.Client(oauth2.NoContext)
	gcsToken, err := conf.TokenSource(oauth2.NoContext).Token()
	service, err := storage.New(client)
	if err != nil {
		log.Fatalf("GCS unable to create storage service: %v", err)
	}

	// If the bucket already exists and the user has access,  don't try to create it.
	if _, err := service.Buckets.Get(bucketName).Do(); err != nil {
		// If bucket is not exist, Create a bucket.
		if _, err := service.Buckets.Insert(projectID, &storage.Bucket{Name: bucketName}).Do(); err != nil {
			log.Fatalf("GCS failed creating bucket %s: %v", bucketName, err)
		}
	}

	//Split filename as a objectName
	var objectName string
	for _, objectName = range strings.Split(file, "/") {
	}
	object := &storage.Object{Name: objectName}

	// Insert an object into a bucket.
	fileDes, err := os.Open(file)
	if err != nil {
		log.Fatalf("Error opening %q: %v", file, err)
	}
	objs, err := service.Objects.Insert(bucketName, object).Media(fileDes).Do()
	if err != nil {
		log.Fatalf("GCS Objects.Insert failed: %v", err)
	}
	retUrl := objs.MediaLink + "&access_token=" + gcsToken.AccessToken
	//fmt.Println(fmt.Sprintf("GCS tmpUrl=%s", retUrl))

	if err != nil {
		return "", err
	} else {
		return retUrl, nil
	}
}
<|endoftext|>


<commit_before>package encoder

import (
	"bytes"
	"encoding/json"
	"io"
	"unicode/utf8"
)

var hex = "0123456789abcdef"

// WriteString writes a JSON string to the writer. JSON encoding used
// from the encoding/json package.
func WriteString(w io.Writer, v string) error {
	var buf bytes.Buffer
	return WriteStringWithBuffer(w, v, &buf)
}

// WriteString writes a JSON string to the writer. JSON encoding used
// from the encoding/json package.
func WriteStringWithBuffer(w io.Writer, v string, buf *bytes.Buffer) error {
	buf.Reset()
	buf.WriteByte('"')
	prev := 0
	for i := 0; i < len(v); {
		if b := v[i]; b < utf8.RuneSelf {
			if 0x20 <= b && b != '\\' && b != '"' && b != '<' && b != '>' {
				i++
				continue
			}
			if prev < i {
				buf.WriteString(v[prev:i])
			}
			switch b {
			case '\\', '"':
				buf.WriteByte('\\')
				buf.WriteByte(b)
			case '\n':
				buf.WriteByte('\\')
				buf.WriteByte('n')
			case '\r':
				buf.WriteByte('\\')
				buf.WriteByte('r')
			default:
				// This encodes bytes < 0x20 except for \n and \r,
				// as well as < and >. The latter are escaped because they
				// can lead to security holes when user-controlled strings
				// are rendered into JSON and served to some browsers.
				buf.WriteString(`\u00`)
				buf.WriteByte(hex[b>>4])
				buf.WriteByte(hex[b&0xF])
			}
			i++
			prev = i
			continue
		}
		c, size := utf8.DecodeRuneInString(v[i:])
		if c == utf8.RuneError && size == 1 {
			return &json.InvalidUTF8Error{S: v}
		}
		i += size
	}
	if prev < len(v) {
		buf.WriteString(v[prev:])
	}
	buf.WriteByte('"')
	w.Write(buf.Bytes())
	return nil
}
<commit_msg>Remove old string writer.<commit_after><|endoftext|>


<commit_before><commit_msg>Fix up delete function<commit_after><|endoftext|>


<commit_before>package creds

import (
	"sync"
	"time"

	"encoding/json"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
)

//go:generate counterfeiter . VarSourcePool

type VarSourcePool interface {
	FindOrCreate(lager.Logger, map[string]interface{}, ManagerFactory) (Secrets, error)
	Size() int
	Close()
}

type inPoolManager struct {
	manager        Manager
	secretsFactory SecretsFactory
	lastUseTime    time.Time
	clock          clock.Clock
}

func (m *inPoolManager) Close(logger lager.Logger) {
	m.manager.Close(logger)
}

func (m *inPoolManager) NewSecrets() Secrets {
	m.lastUseTime = m.clock.Now()
	return m.secretsFactory.NewSecrets()
}

type varSourcePool struct {
	pool  map[string]*inPoolManager
	lock  sync.Mutex
	ttl   time.Duration
	clock clock.Clock

	closeOnce sync.Once
	closed    chan struct{}
}

func NewVarSourcePool(
	logger lager.Logger,
	ttl time.Duration,
	collectInterval time.Duration,
	clock clock.Clock,
) VarSourcePool {
	pool := &varSourcePool{
		pool:  map[string]*inPoolManager{},
		lock:  sync.Mutex{},
		ttl:   ttl,
		clock: clock,

		closeOnce: sync.Once{},
		closed:    make(chan struct{}),
	}

	go pool.collectLoop(
		logger.Session("collect"),
		collectInterval,
	)

	return pool
}

func (pool *varSourcePool) Size() int {
	return len(pool.pool)
}

func (pool *varSourcePool) FindOrCreate(logger lager.Logger, config map[string]interface{}, factory ManagerFactory) (Secrets, error) {
	b, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	key := string(b)

	pool.lock.Lock()
	defer pool.lock.Unlock()

	if _, ok := pool.pool[key]; !ok {
		manager, err := factory.NewInstance(config)
		if err != nil {
			return nil, err
		}
		err = manager.Init(logger)
		if err != nil {
			return nil, err
		}
		secretsFactory, err := manager.NewSecretsFactory(logger)
		if err != nil {
			return nil, err
		}

		pool.pool[key] = &inPoolManager{
			clock:          pool.clock,
			manager:        manager,
			secretsFactory: secretsFactory,
		}
	} else {
		logger.Debug("found-existing-credential-manager")
	}

	return pool.pool[key].NewSecrets(), nil
}

func (pool *varSourcePool) Close() {
	pool.closeOnce.Do(func() {
		close(pool.closed)
	})
}

func (pool *varSourcePool) collectLoop(logger lager.Logger, interval time.Duration) {
	ticker := pool.clock.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-pool.closed:
			pool.collect(logger.Session("close"), true)
			return
		case <-ticker.C():
			pool.collect(logger.Session("tick"), false)
		}
	}
}

func (pool *varSourcePool) collect(logger lager.Logger, all bool) error {
	pool.lock.Lock()
	defer pool.lock.Unlock()

	logger.Debug("before", lager.Data{"size": len(pool.pool)})

	toDeleteKeys := []string{}
	for key, manager := range pool.pool {
		if all || manager.lastUseTime.Add(pool.ttl).Before(pool.clock.Now()) {
			toDeleteKeys = append(toDeleteKeys, key)
			manager.Close(logger)
		}
	}

	for _, key := range toDeleteKeys {
		delete(pool.pool, key)
	}

	logger.Debug("after", lager.Data{"size": len(pool.pool)})

	return nil
}
<commit_msg>atc: structure: fix data race in var source pool<commit_after>package creds

import (
	"sync"
	"time"

	"encoding/json"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
)

//go:generate counterfeiter . VarSourcePool

type VarSourcePool interface {
	FindOrCreate(lager.Logger, map[string]interface{}, ManagerFactory) (Secrets, error)
	Size() int
	Close()
}

type inPoolManager struct {
	manager        Manager
	secretsFactory SecretsFactory
	lastUseTime    time.Time
	clock          clock.Clock
}

func (m *inPoolManager) Close(logger lager.Logger) {
	m.manager.Close(logger)
}

func (m *inPoolManager) NewSecrets() Secrets {
	m.lastUseTime = m.clock.Now()
	return m.secretsFactory.NewSecrets()
}

type varSourcePool struct {
	pool  map[string]*inPoolManager
	lock  sync.Mutex
	ttl   time.Duration
	clock clock.Clock

	closeOnce sync.Once
	closed    chan struct{}
}

func NewVarSourcePool(
	logger lager.Logger,
	ttl time.Duration,
	collectInterval time.Duration,
	clock clock.Clock,
) VarSourcePool {
	pool := &varSourcePool{
		pool:  map[string]*inPoolManager{},
		lock:  sync.Mutex{},
		ttl:   ttl,
		clock: clock,

		closeOnce: sync.Once{},
		closed:    make(chan struct{}),
	}

	go pool.collectLoop(
		logger.Session("collect"),
		collectInterval,
	)

	return pool
}

func (pool *varSourcePool) Size() int {
	pool.lock.Lock()
	defer pool.lock.Unlock()
	return len(pool.pool)
}

func (pool *varSourcePool) FindOrCreate(logger lager.Logger, config map[string]interface{}, factory ManagerFactory) (Secrets, error) {
	b, err := json.Marshal(config)
	if err != nil {
		return nil, err
	}

	key := string(b)

	pool.lock.Lock()
	defer pool.lock.Unlock()

	if _, ok := pool.pool[key]; !ok {
		manager, err := factory.NewInstance(config)
		if err != nil {
			return nil, err
		}
		err = manager.Init(logger)
		if err != nil {
			return nil, err
		}
		secretsFactory, err := manager.NewSecretsFactory(logger)
		if err != nil {
			return nil, err
		}

		pool.pool[key] = &inPoolManager{
			clock:          pool.clock,
			manager:        manager,
			secretsFactory: secretsFactory,
		}
	} else {
		logger.Debug("found-existing-credential-manager")
	}

	return pool.pool[key].NewSecrets(), nil
}

func (pool *varSourcePool) Close() {
	pool.closeOnce.Do(func() {
		close(pool.closed)
	})
}

func (pool *varSourcePool) collectLoop(logger lager.Logger, interval time.Duration) {
	ticker := pool.clock.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-pool.closed:
			pool.collect(logger.Session("close"), true)
			return
		case <-ticker.C():
			pool.collect(logger.Session("tick"), false)
		}
	}
}

func (pool *varSourcePool) collect(logger lager.Logger, all bool) error {
	pool.lock.Lock()
	defer pool.lock.Unlock()

	logger.Debug("before", lager.Data{"size": len(pool.pool)})

	toDeleteKeys := []string{}
	for key, manager := range pool.pool {
		if all || manager.lastUseTime.Add(pool.ttl).Before(pool.clock.Now()) {
			toDeleteKeys = append(toDeleteKeys, key)
			manager.Close(logger)
		}
	}

	for _, key := range toDeleteKeys {
		delete(pool.pool, key)
	}

	logger.Debug("after", lager.Data{"size": len(pool.pool)})

	return nil
}
<|endoftext|>


<commit_before>//Copyright 2017 Huawei Technologies Co., Ltd
//
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
package notification

import (
	"encoding/json"
	"errors"
	"fmt"
	apt "github.com/ServiceComb/service-center/server/core"
	pb "github.com/ServiceComb/service-center/server/core/proto"
	ms "github.com/ServiceComb/service-center/server/service/microservice"
	"github.com/ServiceComb/service-center/util"
	"github.com/gorilla/websocket"
	"golang.org/x/net/context"
	"time"
)

func HandleWatchJob(watcher *ListWatcher, stream pb.ServiceInstanceCtrl_WatchServer, timeout time.Duration) (err error) {
	for {
		select {
		case <-time.After(timeout):
		// TODO grpc 长连接心跳？
		case job := <-watcher.Job:
			if job == nil {
				err = errors.New("channel is closed")
				util.LOGGER.Errorf(err, "watcher %s %s caught an exception",
					watcher.Subject(), watcher.Id())
				return
			}
			resp := job.(*WatchJob).Response
			util.LOGGER.Infof("event is coming in, watcher %s %s",
				watcher.Subject(), watcher.Id())

			err = stream.Send(resp)
			if err != nil {
				util.LOGGER.Errorf(err, "send message error, watcher %s %s",
					watcher.Subject(), watcher.Id())
				watcher.SetError(err)
				return
			}
		}
	}
}

type WebSocketHandler struct {
	ctx             context.Context
	conn            *websocket.Conn
	watcher         *ListWatcher
	needPingWatcher bool
}

func (wh *WebSocketHandler) Init() error {
	remoteAddr := wh.conn.RemoteAddr().String()
	if err := GetNotifyService().AddSubscriber(wh.watcher); err != nil {
		err = fmt.Errorf("establish[%s] websocket watch failed: notify service error, %s.",
			remoteAddr, err.Error())
		util.LOGGER.Errorf(nil, err.Error())

		err = wh.conn.WriteMessage(websocket.TextMessage, util.StringToBytesWithNoCopy(err.Error()))
		if err != nil {
			util.LOGGER.Errorf(err, "establish[%s] websocket watch failed: write message failed.", remoteAddr)
		}
		return err
	}
	util.LOGGER.Debugf("start watching instance status, watcher[%s] %s %s",
		remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
	return nil
}

func (wh *WebSocketHandler) Timeout() time.Duration {
	return GetNotifyService().Config.NotifyTimeout
}

func (wh *WebSocketHandler) websocketHeartbeat(messageType int) error {
	err := wh.conn.WriteControl(messageType, []byte{}, time.Now().Add(wh.Timeout()))
	if err != nil {
		messageTypeName := "Ping"
		if messageType == websocket.PongMessage {
			messageTypeName = "Pong"
		}
		util.LOGGER.Errorf(err, "fail to send '%s' to watcher[%s] %s %s", messageTypeName,
			wh.conn.RemoteAddr(), wh.watcher.Subject(), wh.watcher.Id())
		//wh.watcher.SetError(err)
		return err
	}
	return nil
}

func (wh *WebSocketHandler) HandleWatchWebSocketControlMessage() {
	remoteAddr := wh.conn.RemoteAddr().String()
	// PING
	wh.conn.SetPingHandler(func(message string) error {
		if wh.needPingWatcher {
			util.LOGGER.Infof("received 'Ping' message '%s' from watcher[%s] %s %s, no longer send 'Ping' to it",
				message, remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
		}
		wh.needPingWatcher = false
		return wh.websocketHeartbeat(websocket.PongMessage)
	})
	// PONG
	wh.conn.SetPongHandler(func(message string) error {
		util.LOGGER.Debugf("received 'Pong' message %s from watcher[%s] %s %s",
			message, remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
		return nil
	})
	// CLOSE
	wh.conn.SetCloseHandler(func(code int, text string) error {
		util.LOGGER.Warnf(nil, "watcher[%s] %s %s active closed", remoteAddr,
			wh.watcher.Subject(), wh.watcher.Id())
		return wh.Close(code, text)
	})

	for {
		_, _, err := wh.conn.ReadMessage()
		if err != nil {
			wh.watcher.SetError(err)
			return
		}
	}
}

func (wh *WebSocketHandler) HandleWatchWebSocketJob() {
	remoteAddr := wh.conn.RemoteAddr().String()

	for {
		select {
		case <-wh.ctx.Done():
			util.LOGGER.Warnf(nil, "handle timed out, watcher[%s] %s %s", remoteAddr,
				wh.watcher.Subject(), wh.watcher.Id())
			return
		case <-time.After(wh.Timeout()):
			if wh.watcher.Err() != nil {
				return
			}

			tenant := util.ParseTenantProject(wh.ctx)
			if !ms.ServiceExist(context.Background(), tenant, wh.watcher.Id()) {
				err := fmt.Errorf("Service does not exit.")
				util.LOGGER.Warnf(err, "watcher[%s] %s %s exit", remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				err = wh.conn.WriteMessage(websocket.TextMessage, util.StringToBytesWithNoCopy(err.Error()))
				if err != nil {
					util.LOGGER.Errorf(err, "watch catch a err: write message error, watcher[%s] %s %s",
						remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				}
				return
			}

			if !wh.needPingWatcher {
				continue
			}

			util.LOGGER.Debugf("send heartbeat to watcher[%s] %s %s", remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
			err := wh.websocketHeartbeat(websocket.PingMessage)
			if err != nil {
				return
			}
		case job := <-wh.watcher.Job:
			if wh.watcher.Err() != nil {
				return
			}

			if job == nil {
				err := wh.conn.WriteMessage(websocket.TextMessage,
					util.StringToBytesWithNoCopy("watch catch a err: watcher quit for server shutdown"))
				if err != nil {
					util.LOGGER.Errorf(err, "watch catch a err: write message error, watcher[%s] %s %s",
						remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
					return
				}
				util.LOGGER.Warnf(nil, "watch catch a err: server shutdown, watcher[%s] %s %s",
					remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				return
			}

			resp := job.(*WatchJob).Response

			util.LOGGER.Warnf(nil, "event[%s] is coming in, watcher[%s] %s %s, providers' info %s %s",
				resp.Action, remoteAddr, wh.watcher.Subject(), wh.watcher.Id(), resp.Instance.ServiceId, resp.Instance.InstanceId)

			resp.Response = nil
			data, err := json.Marshal(resp)
			if err != nil {
				util.LOGGER.Errorf(err, "watch catch a err: marshal output file error, watcher[%s] %s %s",
					remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				message := fmt.Sprintf("marshal output file error, %s", err.Error())
				err = wh.conn.WriteMessage(websocket.TextMessage, util.StringToBytesWithNoCopy(message))
				if err != nil {
					util.LOGGER.Errorf(err, "watch catch a err: write message error, watcher[%s] %s %s",
						remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				}
				return
			}
			err = wh.conn.WriteMessage(websocket.TextMessage, data)
			if err != nil {
				util.LOGGER.Errorf(err, "watch catch a err: write message error, watcher[%s] %s %s",
					remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				return
			}
		}
	}
}

func (wh *WebSocketHandler) Close(code int, text string) error {
	remoteAddr := wh.conn.RemoteAddr().String()
	message := []byte{}
	if code != websocket.CloseNoStatusReceived {
		message = websocket.FormatCloseMessage(code, text)
	}
	err := wh.conn.WriteControl(websocket.CloseMessage, message, time.Now().Add(wh.Timeout()))
	if err != nil {
		util.LOGGER.Errorf(err, "watch catch a err: write 'Close' message error, watcher[%s] %s %s",
			remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
		return err
	}
	return nil
}

func DoWebSocketWatch(ctx context.Context, serviceId string, conn *websocket.Conn) {
	tenant := util.ParseTenantProject(ctx)
	handler := &WebSocketHandler{
		ctx:             ctx,
		conn:            conn,
		watcher:         NewInstanceWatcher(serviceId, apt.GetInstanceRootKey(tenant)+"/"),
		needPingWatcher: true,
	}
	processHandler(handler)
}

func DoWebSocketListAndWatch(ctx context.Context, serviceId string, f func() ([]*pb.WatchInstanceResponse, int64), conn *websocket.Conn) {
	tenant := util.ParseTenantProject(ctx)
	handler := &WebSocketHandler{
		ctx:             ctx,
		conn:            conn,
		watcher:         NewInstanceListWatcher(serviceId, apt.GetInstanceRootKey(tenant)+"/", f),
		needPingWatcher: true,
	}
	processHandler(handler)
}

func processHandler(handler *WebSocketHandler) {
	if err := handler.Init(); err != nil {
		return
	}
	go handler.HandleWatchWebSocketControlMessage()
	handler.HandleWatchWebSocketJob()
}

func EstablishWebSocketError(conn *websocket.Conn, err error) {
	remoteAddr := conn.RemoteAddr().String()
	util.LOGGER.Errorf(err, "establish[%s] websocket watch failed.", remoteAddr)
	if err := conn.WriteMessage(websocket.TextMessage, util.StringToBytesWithNoCopy(err.Error())); err != nil {
		util.LOGGER.Errorf(err, "establish[%s] websocket watch failed: write message failed.", remoteAddr)
	}
}

func PublishInstanceEvent(tenant string, action pb.EventType, serviceKey *pb.MicroServiceKey, instance *pb.MicroServiceInstance, rev int64, subscribers []string) {
	response := &pb.WatchInstanceResponse{
		Response: pb.CreateResponse(pb.Response_SUCCESS, "Watch instance successfully."),
		Action:   string(action),
		Key:      serviceKey,
		Instance: instance,
	}
	for _, consumerId := range subscribers {
		job := NewWatchJob(INSTANCE, consumerId, apt.GetInstanceRootKey(tenant)+"/", rev, response)
		util.LOGGER.Debugf("publish event to notify service, %v", job)

		// TODO add超时怎么处理？
		GetNotifyService().AddJob(job)
	}
}

func NewInstanceWatcher(selfServiceId, instanceRoot string) *ListWatcher {
	return NewWatcher(INSTANCE, selfServiceId, instanceRoot)
}

func NewInstanceListWatcher(selfServiceId, instanceRoot string, listFunc func() (results []*pb.WatchInstanceResponse, rev int64)) *ListWatcher {
	return NewListWatcher(INSTANCE, selfServiceId, instanceRoot, listFunc)
}
<commit_msg>Bug fixes: list & watch do not exit when connection was closed.<commit_after>//Copyright 2017 Huawei Technologies Co., Ltd
//
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
package notification

import (
	"encoding/json"
	"errors"
	"fmt"
	apt "github.com/ServiceComb/service-center/server/core"
	pb "github.com/ServiceComb/service-center/server/core/proto"
	ms "github.com/ServiceComb/service-center/server/service/microservice"
	"github.com/ServiceComb/service-center/util"
	"github.com/gorilla/websocket"
	"golang.org/x/net/context"
	"time"
)

func HandleWatchJob(watcher *ListWatcher, stream pb.ServiceInstanceCtrl_WatchServer, timeout time.Duration) (err error) {
	for {
		select {
		case <-time.After(timeout):
		// TODO grpc 长连接心跳？
		case job := <-watcher.Job:
			if job == nil {
				err = errors.New("channel is closed")
				util.LOGGER.Errorf(err, "watcher %s %s caught an exception",
					watcher.Subject(), watcher.Id())
				return
			}
			resp := job.(*WatchJob).Response
			util.LOGGER.Infof("event is coming in, watcher %s %s",
				watcher.Subject(), watcher.Id())

			err = stream.Send(resp)
			if err != nil {
				util.LOGGER.Errorf(err, "send message error, watcher %s %s",
					watcher.Subject(), watcher.Id())
				watcher.SetError(err)
				return
			}
		}
	}
}

type WebSocketHandler struct {
	ctx             context.Context
	conn            *websocket.Conn
	watcher         *ListWatcher
	needPingWatcher bool
	closed          chan struct{}
}

func (wh *WebSocketHandler) Init() error {
	remoteAddr := wh.conn.RemoteAddr().String()
	if err := GetNotifyService().AddSubscriber(wh.watcher); err != nil {
		err = fmt.Errorf("establish[%s] websocket watch failed: notify service error, %s.",
			remoteAddr, err.Error())
		util.LOGGER.Errorf(nil, err.Error())

		err = wh.conn.WriteMessage(websocket.TextMessage, util.StringToBytesWithNoCopy(err.Error()))
		if err != nil {
			util.LOGGER.Errorf(err, "establish[%s] websocket watch failed: write message failed.", remoteAddr)
		}
		return err
	}
	util.LOGGER.Debugf("start watching instance status, watcher[%s] %s %s",
		remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
	return nil
}

func (wh *WebSocketHandler) Timeout() time.Duration {
	return GetNotifyService().Config.NotifyTimeout
}

func (wh *WebSocketHandler) websocketHeartbeat(messageType int) error {
	err := wh.conn.WriteControl(messageType, []byte{}, time.Now().Add(wh.Timeout()))
	if err != nil {
		messageTypeName := "Ping"
		if messageType == websocket.PongMessage {
			messageTypeName = "Pong"
		}
		util.LOGGER.Errorf(err, "fail to send '%s' to watcher[%s] %s %s", messageTypeName,
			wh.conn.RemoteAddr(), wh.watcher.Subject(), wh.watcher.Id())
		//wh.watcher.SetError(err)
		return err
	}
	return nil
}

func (wh *WebSocketHandler) HandleWatchWebSocketControlMessage() {
	defer close(wh.closed)

	remoteAddr := wh.conn.RemoteAddr().String()
	// PING
	wh.conn.SetPingHandler(func(message string) error {
		if wh.needPingWatcher {
			util.LOGGER.Infof("received 'Ping' message '%s' from watcher[%s] %s %s, no longer send 'Ping' to it",
				message, remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
		}
		wh.needPingWatcher = false
		return wh.websocketHeartbeat(websocket.PongMessage)
	})
	// PONG
	wh.conn.SetPongHandler(func(message string) error {
		util.LOGGER.Debugf("received 'Pong' message %s from watcher[%s] %s %s",
			message, remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
		return nil
	})
	// CLOSE
	wh.conn.SetCloseHandler(func(code int, text string) error {
		util.LOGGER.Warnf(nil, "watcher[%s] %s %s active closed", remoteAddr,
			wh.watcher.Subject(), wh.watcher.Id())
		return wh.Close(code, text)
	})

	for {
		_, _, err := wh.conn.ReadMessage()
		if err != nil {
			wh.watcher.SetError(err)
			return
		}
	}
}

func (wh *WebSocketHandler) HandleWatchWebSocketJob() {
	remoteAddr := wh.conn.RemoteAddr().String()

	for {
		select {
		case <-wh.closed:
			return
		case <-wh.ctx.Done():
			util.LOGGER.Warnf(nil, "handle timed out, watcher[%s] %s %s", remoteAddr,
				wh.watcher.Subject(), wh.watcher.Id())
			return
		case <-time.After(wh.Timeout()):
			if wh.watcher.Err() != nil {
				return
			}

			tenant := util.ParseTenantProject(wh.ctx)
			if !ms.ServiceExist(context.Background(), tenant, wh.watcher.Id()) {
				err := fmt.Errorf("Service does not exit.")
				util.LOGGER.Warnf(err, "watcher[%s] %s %s exit", remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				err = wh.conn.WriteMessage(websocket.TextMessage, util.StringToBytesWithNoCopy(err.Error()))
				if err != nil {
					util.LOGGER.Errorf(err, "watcher[%s] %s %s catch an err: write message error",
						remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				}
				return
			}

			if !wh.needPingWatcher {
				continue
			}

			util.LOGGER.Debugf("send 'Ping' message to watcher[%s] %s %s", remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
			err := wh.websocketHeartbeat(websocket.PingMessage)
			if err != nil {
				return
			}
		case job := <-wh.watcher.Job:
			if wh.watcher.Err() != nil {
				return
			}

			if job == nil {
				err := wh.conn.WriteMessage(websocket.TextMessage,
					util.StringToBytesWithNoCopy("watcher catch an err: server shutdown"))
				if err != nil {
					util.LOGGER.Errorf(err, "watcher[%s] %s %s catch an err: write message error",
						remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
					return
				}
				util.LOGGER.Warnf(nil, "watcher[%s] %s %s catch an err: server shutdown",
					remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				return
			}

			resp := job.(*WatchJob).Response

			util.LOGGER.Warnf(nil, "event[%s] is coming in, watcher[%s] %s %s, providers' info %s %s",
				resp.Action, remoteAddr, wh.watcher.Subject(), wh.watcher.Id(), resp.Instance.ServiceId, resp.Instance.InstanceId)

			resp.Response = nil
			data, err := json.Marshal(resp)
			if err != nil {
				util.LOGGER.Errorf(err, "watcher[%s] %s %s catch an err: marshal output file error",
					remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				message := fmt.Sprintf("marshal output file error, %s", err.Error())
				err = wh.conn.WriteMessage(websocket.TextMessage, util.StringToBytesWithNoCopy(message))
				if err != nil {
					util.LOGGER.Errorf(err, "watcher[%s] %s %s catch an err: write message error",
						remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				}
				return
			}
			err = wh.conn.WriteMessage(websocket.TextMessage, data)
			if err != nil {
				util.LOGGER.Errorf(err, "watcher[%s] %s %s catch an err: write message error",
					remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
				return
			}
		}
	}
}

func (wh *WebSocketHandler) Close(code int, text string) error {
	remoteAddr := wh.conn.RemoteAddr().String()
	message := []byte{}
	if code != websocket.CloseNoStatusReceived {
		message = websocket.FormatCloseMessage(code, text)
	}
	err := wh.conn.WriteControl(websocket.CloseMessage, message, time.Now().Add(wh.Timeout()))
	if err != nil {
		util.LOGGER.Errorf(err, "watcher[%s] %s %s catch an err: write 'Close' message error",
			remoteAddr, wh.watcher.Subject(), wh.watcher.Id())
		return err
	}
	return nil
}

func DoWebSocketWatch(ctx context.Context, serviceId string, conn *websocket.Conn) {
	tenant := util.ParseTenantProject(ctx)
	handler := &WebSocketHandler{
		ctx:             ctx,
		conn:            conn,
		watcher:         NewInstanceWatcher(serviceId, apt.GetInstanceRootKey(tenant)+"/"),
		needPingWatcher: true,
		closed:          make(chan struct{}),
	}
	processHandler(handler)
}

func DoWebSocketListAndWatch(ctx context.Context, serviceId string, f func() ([]*pb.WatchInstanceResponse, int64), conn *websocket.Conn) {
	tenant := util.ParseTenantProject(ctx)
	handler := &WebSocketHandler{
		ctx:             ctx,
		conn:            conn,
		watcher:         NewInstanceListWatcher(serviceId, apt.GetInstanceRootKey(tenant)+"/", f),
		needPingWatcher: true,
		closed:          make(chan struct{}),
	}
	processHandler(handler)
}

func processHandler(handler *WebSocketHandler) {
	if err := handler.Init(); err != nil {
		return
	}
	go handler.HandleWatchWebSocketControlMessage()
	handler.HandleWatchWebSocketJob()
}

func EstablishWebSocketError(conn *websocket.Conn, err error) {
	remoteAddr := conn.RemoteAddr().String()
	util.LOGGER.Errorf(err, "establish[%s] websocket watch failed.", remoteAddr)
	if err := conn.WriteMessage(websocket.TextMessage, util.StringToBytesWithNoCopy(err.Error())); err != nil {
		util.LOGGER.Errorf(err, "establish[%s] websocket watch failed: write message failed.", remoteAddr)
	}
}

func PublishInstanceEvent(tenant string, action pb.EventType, serviceKey *pb.MicroServiceKey, instance *pb.MicroServiceInstance, rev int64, subscribers []string) {
	response := &pb.WatchInstanceResponse{
		Response: pb.CreateResponse(pb.Response_SUCCESS, "Watch instance successfully."),
		Action:   string(action),
		Key:      serviceKey,
		Instance: instance,
	}
	for _, consumerId := range subscribers {
		job := NewWatchJob(INSTANCE, consumerId, apt.GetInstanceRootKey(tenant)+"/", rev, response)
		util.LOGGER.Debugf("publish event to notify service, %v", job)

		// TODO add超时怎么处理？
		GetNotifyService().AddJob(job)
	}
}

func NewInstanceWatcher(selfServiceId, instanceRoot string) *ListWatcher {
	return NewWatcher(INSTANCE, selfServiceId, instanceRoot)
}

func NewInstanceListWatcher(selfServiceId, instanceRoot string, listFunc func() (results []*pb.WatchInstanceResponse, rev int64)) *ListWatcher {
	return NewListWatcher(INSTANCE, selfServiceId, instanceRoot, listFunc)
}
<|endoftext|>


<commit_before>package controlplane

import (
	"context"
	"fmt"
	"time"

	"k8s.io/client-go/rest"

	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/upgrades"

	"github.com/openshift/origin/pkg/monitor"
	"github.com/openshift/origin/test/extended/util/disruption"
)

// NewKubeAvailableWithNewConnectionsTest tests that the Kubernetes control plane remains available during and after a cluster upgrade.
func NewKubeAvailableWithNewConnectionsTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] Kubernetes APIs remain available for new connections",
		name:            "kubernetes-api-available-new-connections",
		startMonitoring: monitor.StartKubeAPIMonitoringWithNewConnections,
	}
}

// NewOpenShiftAvailableNewConnectionsTest tests that the OpenShift APIs remains available during and after a cluster upgrade.
func NewOpenShiftAvailableNewConnectionsTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] OpenShift APIs remain available for new connections",
		name:            "openshift-api-available-new-connections",
		startMonitoring: monitor.StartOpenShiftAPIMonitoringWithNewConnections,
	}
}

// NewOAuthAvailableNewConnectionsTest tests that the OAuth APIs remains available during and after a cluster upgrade.
func NewOAuthAvailableNewConnectionsTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] OAuth APIs remain available for new connections",
		name:            "oauth-api-available-new-connections",
		startMonitoring: monitor.StartOAuthAPIMonitoringWithNewConnections,
	}
}

// NewKubeAvailableWithConnectionReuseTest tests that the Kubernetes control plane remains available during and after a cluster upgrade.
func NewKubeAvailableWithConnectionReuseTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] Kubernetes APIs remain available with reused connections",
		name:            "kubernetes-api-available-reused-connections",
		startMonitoring: monitor.StartKubeAPIMonitoringWithConnectionReuse,
	}
}

// NewOpenShiftAvailableTest tests that the OpenShift APIs remains available during and after a cluster upgrade.
func NewOpenShiftAvailableWithConnectionReuseTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] OpenShift APIs remain available with reused connections",
		name:            "openshift-api-available-reused-connections",
		startMonitoring: monitor.StartOpenShiftAPIMonitoringWithConnectionReuse,
	}
}

// NewOauthAvailableTest tests that the OAuth APIs remains available during and after a cluster upgrade.
func NewOAuthAvailableWithConnectionReuseTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] OAuth APIs remain available with reused connections",
		name:            "oauth-api-available-reused-connections",
		startMonitoring: monitor.StartOAuthAPIMonitoringWithConnectionReuse,
	}
}

type availableTest struct {
	// testName is the name to show in unit
	testName string
	// name helps distinguish which API server in particular is unavailable.
	name            string
	startMonitoring starter
}

type starter func(ctx context.Context, m *monitor.Monitor, clusterConfig *rest.Config, timeout time.Duration) error

func (t availableTest) Name() string { return t.name }
func (t availableTest) DisplayName() string {
	return t.testName
}

// Setup does nothing
func (t *availableTest) Setup(f *framework.Framework) {
}

// Test runs a connectivity check to the core APIs.
func (t *availableTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	config, err := framework.LoadConfig()
	framework.ExpectNoError(err)

	ctx, cancel := context.WithCancel(context.Background())
	m := monitor.NewMonitorWithInterval(time.Second)
	err = t.startMonitoring(ctx, m, config, 15*time.Second)
	framework.ExpectNoError(err, "unable to monitor API")

	start := time.Now()
	m.StartSampling(ctx)

	// wait to ensure API is still up after the test ends
	<-done
	time.Sleep(15 * time.Second)
	cancel()
	end := time.Now()

	// AWS and Azure in 4.7 reached the point of having no disruption and thus
	// now lock in that requirement. GCP has a known issue with health checking
	// https://bugzilla.redhat.com/show_bug.cgi?id=1925698 that when fixed will
	// relax. Having two providers lock in the requirement ensures we block
	// regression and ratchet
	toleratedDisruption := 0.08
	if framework.ProviderIs("aws", "azure") {
		toleratedDisruption = 0
	}
	disruption.ExpectNoDisruption(f, toleratedDisruption, end.Sub(start), m.EventIntervals(time.Time{}, time.Time{}), fmt.Sprintf("API %q was unreachable during disruption", t.name))
}

// Teardown cleans up any remaining resources.
func (t *availableTest) Teardown(f *framework.Framework) {
}
<commit_msg>test: Mark GCP as fixed and AWS as flaky in upgrade availability<commit_after>package controlplane

import (
	"context"
	"fmt"
	"time"

	"k8s.io/client-go/rest"

	"k8s.io/kubernetes/test/e2e/framework"
	"k8s.io/kubernetes/test/e2e/upgrades"

	"github.com/openshift/origin/pkg/monitor"
	"github.com/openshift/origin/test/extended/util/disruption"
)

// NewKubeAvailableWithNewConnectionsTest tests that the Kubernetes control plane remains available during and after a cluster upgrade.
func NewKubeAvailableWithNewConnectionsTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] Kubernetes APIs remain available for new connections",
		name:            "kubernetes-api-available-new-connections",
		startMonitoring: monitor.StartKubeAPIMonitoringWithNewConnections,
	}
}

// NewOpenShiftAvailableNewConnectionsTest tests that the OpenShift APIs remains available during and after a cluster upgrade.
func NewOpenShiftAvailableNewConnectionsTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] OpenShift APIs remain available for new connections",
		name:            "openshift-api-available-new-connections",
		startMonitoring: monitor.StartOpenShiftAPIMonitoringWithNewConnections,
	}
}

// NewOAuthAvailableNewConnectionsTest tests that the OAuth APIs remains available during and after a cluster upgrade.
func NewOAuthAvailableNewConnectionsTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] OAuth APIs remain available for new connections",
		name:            "oauth-api-available-new-connections",
		startMonitoring: monitor.StartOAuthAPIMonitoringWithNewConnections,
	}
}

// NewKubeAvailableWithConnectionReuseTest tests that the Kubernetes control plane remains available during and after a cluster upgrade.
func NewKubeAvailableWithConnectionReuseTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] Kubernetes APIs remain available with reused connections",
		name:            "kubernetes-api-available-reused-connections",
		startMonitoring: monitor.StartKubeAPIMonitoringWithConnectionReuse,
	}
}

// NewOpenShiftAvailableTest tests that the OpenShift APIs remains available during and after a cluster upgrade.
func NewOpenShiftAvailableWithConnectionReuseTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] OpenShift APIs remain available with reused connections",
		name:            "openshift-api-available-reused-connections",
		startMonitoring: monitor.StartOpenShiftAPIMonitoringWithConnectionReuse,
	}
}

// NewOauthAvailableTest tests that the OAuth APIs remains available during and after a cluster upgrade.
func NewOAuthAvailableWithConnectionReuseTest() upgrades.Test {
	return &availableTest{
		testName:        "[sig-api-machinery] OAuth APIs remain available with reused connections",
		name:            "oauth-api-available-reused-connections",
		startMonitoring: monitor.StartOAuthAPIMonitoringWithConnectionReuse,
	}
}

type availableTest struct {
	// testName is the name to show in unit
	testName string
	// name helps distinguish which API server in particular is unavailable.
	name            string
	startMonitoring starter
}

type starter func(ctx context.Context, m *monitor.Monitor, clusterConfig *rest.Config, timeout time.Duration) error

func (t availableTest) Name() string { return t.name }
func (t availableTest) DisplayName() string {
	return t.testName
}

// Setup does nothing
func (t *availableTest) Setup(f *framework.Framework) {
}

// Test runs a connectivity check to the core APIs.
func (t *availableTest) Test(f *framework.Framework, done <-chan struct{}, upgrade upgrades.UpgradeType) {
	config, err := framework.LoadConfig()
	framework.ExpectNoError(err)

	ctx, cancel := context.WithCancel(context.Background())
	m := monitor.NewMonitorWithInterval(time.Second)
	err = t.startMonitoring(ctx, m, config, 15*time.Second)
	framework.ExpectNoError(err, "unable to monitor API")

	start := time.Now()
	m.StartSampling(ctx)

	// wait to ensure API is still up after the test ends
	<-done
	time.Sleep(15 * time.Second)
	cancel()
	end := time.Now()

	toleratedDisruption := 0.08
	if framework.ProviderIs("azure", "gcp") {
		toleratedDisruption = 0
	}
	disruption.ExpectNoDisruption(f, toleratedDisruption, end.Sub(start), m.EventIntervals(time.Time{}, time.Time{}), fmt.Sprintf("API %q was unreachable during disruption (AWS has a known issue: https://bugzilla.redhat.com/show_bug.cgi?id=1943804)", t.name))
}

// Teardown cleans up any remaining resources.
func (t *availableTest) Teardown(f *framework.Framework) {
}
<|endoftext|>


<commit_before>package main

import "fmt"

func setZeroes(matrix [][]int) {
	lengthM := len(matrix)
	colLen := len(matrix[0])

	var zeroRow = make([]int, colLen)
	var colZeroIndex []int

	for i := 0; i < lengthM; i++ {
		testI := 0
		for colIndex, ele := range matrix[i] {
			if ele == 0 {
				colZeroIndex = append(colZeroIndex, colIndex)
				testI += 1
			}
		}
		if testI != 0 {
			matrix[i] = zeroRow
		} else {
			for _, ii := range colZeroIndex {
				matrix[i][ii] = 0
			}
		}

	}
	fmt.Print(matrix)
}

func main() {
	matrix := [][]int{
		[]int{1, 2, 0, 3, 4, 5},
		[]int{0, 4, 5, 6, 7, 8},
		[]int{1, 2, 3, 4, 5, 6},
	}
	setZeroes(matrix)
}
<commit_msg>finish bug<commit_after>package main

import "fmt"

func setZeroes(matrix [][]int) {
	lengthM := len(matrix)
	colLen := len(matrix[0])

	var zeroRow = make([]int, colLen)
	var colZeroIndex []int

	for i := 0; i < lengthM; i++ {
		testI := 0
		for colIndex, ele := range matrix[i] {
			if ele == 0 {
				colZeroIndex = append(colZeroIndex, colIndex)
				testI += 1
			}
		}
		if testI != 0 {
			for rei := 0; rei < i; rei++ {
				for _, ii := range colZeroIndex {
					matrix[rei][ii] = 0
				}
			}
			matrix[i] = zeroRow
		} else {
			for _, ii := range colZeroIndex {
				matrix[i][ii] = 0
			}
		}

	}
	fmt.Print(matrix)
}

func main() {
	matrix := [][]int{
		[]int{1, 2, 0, 3, 4, 5},
		[]int{0, 4, 5, 6, 7, 8},
		[]int{1, 2, 3, 4, 5, 6},
	}
	setZeroes(matrix)

	matrix2 := [][]int{
		[]int{1, 2, 3, 3, 4, 5},
		[]int{0, 4, 5, 6, 7, 8},
		[]int{1, 2, 3, 4, 5, 6},
	}
	setZeroes(matrix2)
}
<|endoftext|>


<commit_before>// +build !linux,!darwin

package networkallocator

func getInitializers() []initializer {
	return nil
}
<commit_msg>Fixing build break for winodws<commit_after>// +build !linux,!darwin,!windows

package networkallocator

func getInitializers() []initializer {
	return nil
}
<|endoftext|>


<commit_before>package bbhw

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
)

// SysFS managed ADCs ------------------------------------

type SysfsADC struct {
	Number uint
	fd     *os.File
	err    error
}

func LoadOverlayForSysfsADC() error {
	err := AddDeviceTreeOverlayIfNotAlreadyLoaded("BB-ADC")
	if err == ERROR_DTO_ALREADY_LOADED {
		return nil
	} else {
		return err
	}
}

// Instantinate a new ADC to read through sysfs. Takes ADC AIN numer (same as in sysfs)
func NewSysfsADC(number uint) (adc *SysfsADC, err error) {
	adc = new(SysfsADC)
	adc.Number = number
	ain := fmt.Sprintf("in_voltage%d_raw", number)

	var adc_dir string
	adc_dir, err = findTSCADCDir()
	if err != nil {
		return
	}

	//check if file really exists and open
	adc.fd, err = os.OpenFile(filepath.Join(adc_dir, ain), os.O_RDONLY|os.O_SYNC, 0666)
	if err != nil {
		return nil, err
	}
	return adc, nil
}

// Wrapper around NewSysfsGPIO. Does not return an error but panics instead. Useful to avoid multiple return values.
// This is the function with the same signature as all the other New*GPIO*s
func NewSysfsADCOrPanic(number uint) (adc *SysfsADC) {
	adc, err := NewSysfsADC(number)
	if err != nil {
		panic(err)
	}
	return adc
}

//returns raw SysFs Value.
// In case of new kernel 4.4 that means raw value which we convert to actual mV
func (adc *SysfsADC) ReadValue() (value uint16) {
	if adc == nil {
		panic("adc == nil")
	}
	if adc.fd == nil {
		panic("adc.fd == nil")
	}
	_, adc.err = adc.fd.Seek(0, 0)
	if adc.err != nil {
		return
	}

	var numread int
	buf := make([]byte, 16, 16)
	numread, adc.err = adc.fd.Read(buf)
	if adc.err != nil {
		return
	}
	var value64 uint64
	value64, adc.err = strconv.ParseUint(string(buf[0:numread-1]), 10, 16)

	return uint16(value64 * 1800 / 4096) //4096 means 1.8V means 1800mV
}

func (adc *SysfsADC) CheckErrorOccurred() error {
	if adc == nil {
		panic("adc == nil")
	}
	return adc.err
}

func (adc *SysfsADC) ReadValueCheckError() (value uint16, err error) {
	value = adc.ReadValue()
	err = adc.CheckErrorOccurred()
	return
}

func findTSCADCDir() (adcdir string, err error) {
	var ocp_dir string
	if ocp_dir, err = findOCPDir(); err != nil {
		return
	}
	adcdir = filepath.Join(ocp_dir, "44e0d000.tscadc/TI-am335x-adc/iio:device0/")
	return
}

func findPyADCDir(ain string) (tdir string, err error) {
	var ocp_dir string
	if ocp_dir, err = findOCPDir(); err != nil {
		return
	}
	re1 := regexp.MustCompile(filepath.Join(ocp_dir, `.*`+ain+`\.\d+`+"$"))
	err = filepath.Walk(ocp_dir, makeFindDirHelperFunc(&tdir, re1, 5))
	if err == foundit_error_ {
		err = nil
	} else if err == nil {
		err = fmt.Errorf("ADC Directory for %s Not Found", ain)
	}
	return
}
<commit_msg>function to wait until adc shows up in sysfs<commit_after>package bbhw

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"time"
)

// SysFS managed ADCs ------------------------------------

type SysfsADC struct {
	Number uint
	fd     *os.File
	err    error
}

func LoadOverlayForSysfsADC() error {
	err := AddDeviceTreeOverlayIfNotAlreadyLoaded("BB-ADC")
	if err == ERROR_DTO_ALREADY_LOADED {
		return nil
	} else {
		return err
	}
}

func WaitUntilSysFSADCRunning() error {
	adc_dir, err := findTSCADCDir()
	if err != nil {
		return err
	}
	adcpath := filepath.Join(adc_dir, "in_voltage0_raw")
	for wait := 200; wait > 0; wait-- {
		if doesPathExist(adcpath) {
			break
		} else {
			time.Sleep(50 * time.Millisecond)
		}
	}
	return nil
}

// Instantinate a new ADC to read through sysfs. Takes ADC AIN numer (same as in sysfs)
func NewSysfsADC(number uint) (adc *SysfsADC, err error) {
	adc = new(SysfsADC)
	adc.Number = number
	ain := fmt.Sprintf("in_voltage%d_raw", number)

	var adc_dir string
	adc_dir, err = findTSCADCDir()
	if err != nil {
		return
	}

	//check if file really exists and open
	adc.fd, err = os.OpenFile(filepath.Join(adc_dir, ain), os.O_RDONLY|os.O_SYNC, 0666)
	if err != nil {
		return nil, err
	}
	return adc, nil
}

// Wrapper around NewSysfsGPIO. Does not return an error but panics instead. Useful to avoid multiple return values.
// This is the function with the same signature as all the other New*GPIO*s
func NewSysfsADCOrPanic(number uint) (adc *SysfsADC) {
	adc, err := NewSysfsADC(number)
	if err != nil {
		panic(err)
	}
	return adc
}

//returns raw SysFs Value.
// In case of new kernel 4.4 that means raw value which we convert to actual mV
func (adc *SysfsADC) ReadValue() (value uint16) {
	if adc == nil {
		panic("adc == nil")
	}
	if adc.fd == nil {
		panic("adc.fd == nil")
	}
	_, adc.err = adc.fd.Seek(0, 0)
	if adc.err != nil {
		return
	}

	var numread int
	buf := make([]byte, 16, 16)
	numread, adc.err = adc.fd.Read(buf)
	if adc.err != nil {
		return
	}
	var value64 uint64
	value64, adc.err = strconv.ParseUint(string(buf[0:numread-1]), 10, 16)

	return uint16(value64 * 1800 / 4096) //4096 means 1.8V means 1800mV
}

func (adc *SysfsADC) CheckErrorOccurred() error {
	if adc == nil {
		panic("adc == nil")
	}
	return adc.err
}

func (adc *SysfsADC) ReadValueCheckError() (value uint16, err error) {
	value = adc.ReadValue()
	err = adc.CheckErrorOccurred()
	return
}

func findTSCADCDir() (adcdir string, err error) {
	var ocp_dir string
	if ocp_dir, err = findOCPDir(); err != nil {
		return
	}
	adcdir = filepath.Join(ocp_dir, "44e0d000.tscadc/TI-am335x-adc/iio:device0/")
	return
}

func findPyADCDir(ain string) (tdir string, err error) {
	var ocp_dir string
	if ocp_dir, err = findOCPDir(); err != nil {
		return
	}
	re1 := regexp.MustCompile(filepath.Join(ocp_dir, `.*`+ain+`\.\d+`+"$"))
	err = filepath.Walk(ocp_dir, makeFindDirHelperFunc(&tdir, re1, 5))
	if err == foundit_error_ {
		err = nil
	} else if err == nil {
		err = fmt.Errorf("ADC Directory for %s Not Found", ain)
	}
	return
}
<|endoftext|>