<commit_before>package store

import (
	"appengine"
	"appengine/datastore"
	"errors"
	"github.com/RangelReale/osin"
	"time"
)

type OsinAppEngineStore struct {
	c appengine.Context
}

type oClient struct {
	Id          string `datastore:"Id"`
	Secret      string `datastore:"Secret,noindex"`
	RedirectUri string `datastore:"RedirectUri,noindex"`
}

// Authorization data
type oAuthorizeData struct {
	Client      *oClient  `datastore:"-,noindex"`
	Code        string    `datastore:"Code"`
	ExpiresIn   int32     `datastore:"ExpiresIn"`
	Scope       string    `datastore:"Scope"`
	RedirectUri string    `datastore:"RedirectUri"`
	State       string    `datastore:"State"`
	CreatedAt   time.Time `datastore:"CreatedAt"`
}

// AccessData
type oAccessData struct {
	Client        *oClient        `datastore:"-,noindex"`
	AuthorizeData *oAuthorizeData `datastore:"-,noindex"`
	AccessData    *oAccessData    `datastore:"-,noindex"`
	AccessToken   string          `datastore:"accessToken"`
	RefreshToken  string          `datastore:"refreshToken"`
	ExpiresIn     int32           `datastore:"expiresIn,noindex"`
	Scope         string          `datastore:"scope,noindex"`
	RedirectUri   string          `datastore:"redirectUri,noindex"`
	CreatedAt     time.Time       `datastore:"createdAt,noindex"`
}

func NewOsinAppEngineStore(c appengine.Context) *OsinAppEngineStore {
	r := &OsinAppEngineStore{c}

	err := r.addClient(&osin.Client{
		Id:          "***REMOVED***",
		Secret:      "***REMOVED***",
		RedirectUri: "urn:ietf:wg:oauth:2.0:oob",
	})

	if err != nil {
		c.Warningf("Failed to initialize oauth server: %v", err)
	}
	return r
}

func (s *OsinAppEngineStore) addClient(c *osin.Client) error {
	s.c.Debugf("AddClient: %s\n", c.Id)
	key := datastore.NewKey(s.c, "osin.client", c.Id, 0, nil)
	_, err := datastore.Put(s.c, key, newInternalClient(c))

	if err != nil {
		return err
	}

	return nil
}

func newInternalClient(c *osin.Client) *oClient {
	if c == nil {
		return nil
	}
	return &oClient{c.Id, c.Secret, c.RedirectUri}
}

func newOsinClient(c *oClient) *osin.Client {
	if c == nil {
		return nil
	}
	return &osin.Client{c.Id, c.Secret, c.RedirectUri, nil}
}

// TODO: implement saving and loading of the structs by their ids
func (d *oAuthorizeData) Load(c <-chan datastore.Property) error {
	if err := datastore.LoadStruct(d, c); err != nil {
		return err
	}

	return nil
}

func (d *oAuthorizeData) Save(c chan<- datastore.Property) error {
	return datastore.SaveStruct(d, c)
}

func (s *OsinAppEngineStore) GetClient(id string) (*osin.Client, error) {
	s.c.Debugf("GetClient: %s\n", id)
	key := datastore.NewKey(s.c, "osin.client", id, 0, nil)
	client := new(oClient)
	err := datastore.Get(s.c, key, client)

	if err != nil {
		s.c.Warningf("Error looking up client by id [%s]: [%v]", id, err)
		return nil, errors.New("Client not found")
	}
	osinClient := newOsinClient(client)
	return osinClient, nil
}

func newInternalAuthorizeData(d *osin.AuthorizeData) *oAuthorizeData {
	if d == nil {
		return nil
	}
	return &oAuthorizeData{newInternalClient(d.Client), d.Code, d.ExpiresIn, d.Scope, d.RedirectUri, d.State, d.CreatedAt}
}

func newOsinAuthorizeData(d *oAuthorizeData) *osin.AuthorizeData {
	if d == nil {
		return nil
	}
	return &osin.AuthorizeData{newOsinClient(d.Client), d.Code, d.ExpiresIn, d.Scope, d.RedirectUri, d.State, d.CreatedAt, nil}
}

func (s *OsinAppEngineStore) SaveAuthorize(data *osin.AuthorizeData) error {
	s.c.Debugf("SaveAuthorize: %s\n", data.Code)
	key := datastore.NewKey(s.c, "authorize.data", data.Code, 0, nil)
	_, err := datastore.Put(s.c, key, newInternalAuthorizeData(data))
	if err != nil {
		s.c.Warningf("Error saving authorize data [%s]: [%v]", data.Code, err)
		return err
	}

	return nil
}

func (s *OsinAppEngineStore) LoadAuthorize(code string) (*osin.AuthorizeData, error) {
	s.c.Debugf("LoadAuthorize: %s\n", code)
	key := datastore.NewKey(s.c, "authorize.data", code, 0, nil)
	authorizeData := new(oAuthorizeData)
	err := datastore.Get(s.c, key, authorizeData)
	if err != nil {
		s.c.Infof("Authorization data not found for code [%s]: %v", code, err)
		return nil, errors.New("Authorize not found")
	}
	return newOsinAuthorizeData(authorizeData), nil
}

func (s *OsinAppEngineStore) RemoveAuthorize(code string) error {
	s.c.Debugf("RemoveAuthorize: %s\n", code)
	key := datastore.NewKey(s.c, "authorize.data", code, 0, nil)
	err := datastore.Delete(s.c, key)
	if err != nil {
		return err
	}

	return nil
}

func newInternalAccessData(d *osin.AccessData) *oAccessData {
	if d == nil {
		return nil
	}
	return &oAccessData{newInternalClient(d.Client), newInternalAuthorizeData(d.AuthorizeData), newInternalAccessData(d.AccessData), d.AccessToken, d.RefreshToken, d.ExpiresIn, d.Scope, d.RedirectUri, d.CreatedAt}
}

func newOsinAccessData(d *oAccessData) *osin.AccessData {
	if d == nil {
		return nil
	}
	return &osin.AccessData{newOsinClient(d.Client), newOsinAuthorizeData(d.AuthorizeData), newOsinAccessData(d.AccessData), d.AccessToken, d.RefreshToken, d.ExpiresIn, d.Scope, d.RedirectUri, d.CreatedAt, nil}
}

func (s *OsinAppEngineStore) SaveAccess(data *osin.AccessData) error {
	s.c.Debugf("SaveAccess: %s\n", data.AccessToken)
	key := datastore.NewKey(s.c, "access.data", data.AccessToken, 0, nil)
	internalAccessData := newInternalAccessData(data)
	_, err := datastore.Put(s.c, key, internalAccessData)
	if err != nil {
		return err
	}

	if data.RefreshToken != "" {
		key = datastore.NewKey(s.c, "access.refresh", data.RefreshToken, 0, nil)
		_, err := datastore.Put(s.c, key, internalAccessData)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *OsinAppEngineStore) LoadAccess(code string) (*osin.AccessData, error) {
	s.c.Debugf("LoadAccess: %s\n", code)
	key := datastore.NewKey(s.c, "access.data", code, 0, nil)
	accessData := new(oAccessData)
	err := datastore.Get(s.c, key, accessData)
	if err != nil {
		s.c.Infof("Access data not found for code [%s]: %v", code, err)
		return nil, errors.New("Access data not found")
	}

	return newOsinAccessData(accessData), nil
}

func (s *OsinAppEngineStore) RemoveAccess(code string) error {
	s.c.Debugf("RemoveAccess: %s\n", code)
	key := datastore.NewKey(s.c, "access.data", code, 0, nil)
	err := datastore.Delete(s.c, key)
	if err != nil {
		return err
	}

	return nil
}

func (s *OsinAppEngineStore) LoadRefresh(code string) (*osin.AccessData, error) {
	s.c.Debugf("LoadRefresh: %s\n", code)
	key := datastore.NewKey(s.c, "access.refresh", code, 0, nil)
	accessData := new(oAccessData)
	_, err := datastore.Put(s.c, key, accessData)
	if err != nil {
		s.c.Infof("Refresh data not found for code [%s]: %v", code, err)
		errors.New("Refresh not found")
	}
	return newOsinAccessData(accessData), nil
}

func (s *OsinAppEngineStore) RemoveRefresh(code string) error {
	s.c.Debugf("RemoveRefresh: %s\n", code)
	key := datastore.NewKey(s.c, "access.refresh", code, 0, nil)
	err := datastore.Delete(s.c, key)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Store ClientIds in AuthorizeData and AccessData only with Id.<commit_after>package store

import (
	"appengine"
	"appengine/datastore"
	"errors"
	"github.com/RangelReale/osin"
	"time"
)

type OsinAppEngineStore struct {
	c appengine.Context
}

type oClient struct {
	Id          string `datastore:"Id"`
	Secret      string `datastore:"Secret,noindex"`
	RedirectUri string `datastore:"RedirectUri,noindex"`
}

// Authorization data
type oAuthorizeData struct {
	ClientId    string    `datastore:"ClientId,noindex"`
	Code        string    `datastore:"Code"`
	ExpiresIn   int32     `datastore:"ExpiresIn"`
	Scope       string    `datastore:"Scope"`
	RedirectUri string    `datastore:"RedirectUri"`
	State       string    `datastore:"State"`
	CreatedAt   time.Time `datastore:"CreatedAt"`
}

// AccessData
type oAccessData struct {
	ClientId          string    `datastore:"ClientId,noindex"`
	AuthorizeDataCode string    `datastore:"AuthorizeDataCode,noindex"`
	AccessDataToken   string    `datastore:"AccessDataToken,noindex"`
	AccessToken       string    `datastore:"AccessToken"`
	RefreshToken      string    `datastore:"RefreshToken"`
	ExpiresIn         int32     `datastore:"ExpiresIn,noindex"`
	Scope             string    `datastore:"Scope,noindex"`
	RedirectUri       string    `datastore:"RedirectUri,noindex"`
	CreatedAt         time.Time `datastore:"CreatedAt,noindex"`
}

func NewOsinAppEngineStore(c appengine.Context) *OsinAppEngineStore {
	r := &OsinAppEngineStore{c}

	err := r.addClient(&osin.Client{
		Id:          "***REMOVED***",
		Secret:      "***REMOVED***",
		RedirectUri: "urn:ietf:wg:oauth:2.0:oob",
	})

	if err != nil {
		c.Warningf("Failed to initialize oauth server: %v", err)
	}
	return r
}

func (s *OsinAppEngineStore) addClient(c *osin.Client) error {
	s.c.Debugf("AddClient: %s\n", c.Id)
	key := datastore.NewKey(s.c, "osin.client", c.Id, 0, nil)
	_, err := datastore.Put(s.c, key, newInternalClient(c))

	if err != nil {
		return err
	}

	return nil
}

func newInternalClient(c *osin.Client) *oClient {
	if c == nil {
		return nil
	}
	return &oClient{c.Id, c.Secret, c.RedirectUri}
}

func newOsinClient(c *oClient) *osin.Client {
	if c == nil {
		return nil
	}
	return &osin.Client{c.Id, c.Secret, c.RedirectUri, nil}
}

func (s *OsinAppEngineStore) GetClient(id string) (*osin.Client, error) {
	s.c.Debugf("GetClient: %s\n", id)
	key := datastore.NewKey(s.c, "osin.client", id, 0, nil)
	client := new(oClient)
	err := datastore.Get(s.c, key, client)

	if err != nil {
		s.c.Warningf("Error looking up client by id [%s]: [%v]", id, err)
		return nil, errors.New("Client not found")
	}
	osinClient := newOsinClient(client)
	return osinClient, nil
}

func newInternalAuthorizeData(d *osin.AuthorizeData) *oAuthorizeData {
	if d == nil {
		return nil
	}

	clientId := ""
	if client := d.Client; client != nil {
		clientId = client.Id
	}

	return &oAuthorizeData{clientId, d.Code, d.ExpiresIn, d.Scope, d.RedirectUri, d.State, d.CreatedAt}
}

func newOsinAuthorizeData(d *oAuthorizeData, c *osin.Client) *osin.AuthorizeData {
	if d == nil {
		return nil
	}
	return &osin.AuthorizeData{c, d.Code, d.ExpiresIn, d.Scope, d.RedirectUri, d.State, d.CreatedAt, nil}
}

func (s *OsinAppEngineStore) SaveAuthorize(data *osin.AuthorizeData) error {
	s.c.Debugf("SaveAuthorize: %s\n", data.Code)
	key := datastore.NewKey(s.c, "authorize.data", data.Code, 0, nil)
	_, err := datastore.Put(s.c, key, newInternalAuthorizeData(data))
	if err != nil {
		s.c.Warningf("Error saving authorize data [%s]: [%v]", data.Code, err)
		return err
	}

	return nil
}

func (s *OsinAppEngineStore) LoadAuthorize(code string) (*osin.AuthorizeData, error) {
	s.c.Debugf("LoadAuthorize: %s\n", code)
	key := datastore.NewKey(s.c, "authorize.data", code, 0, nil)
	authorizeData := new(oAuthorizeData)
	err := datastore.Get(s.c, key, authorizeData)
	if err != nil {
		s.c.Infof("Authorization data not found for code [%s]: %v", code, err)
		return nil, errors.New("Authorize not found")
	}

	var c *osin.Client
	if authorizeData.ClientId != "" {
		c, err = s.GetClient(authorizeData.ClientId)
		if err != nil {
			s.c.Infof("Authorization data can't load client with id [%s]: %v", authorizeData.ClientId, err)
			return nil, errors.New("Client for AuthorizeData not found")
		}
	}

	return newOsinAuthorizeData(authorizeData, c), nil
}

func (s *OsinAppEngineStore) RemoveAuthorize(code string) error {
	s.c.Debugf("RemoveAuthorize: %s\n", code)
	key := datastore.NewKey(s.c, "authorize.data", code, 0, nil)
	err := datastore.Delete(s.c, key)
	if err != nil {
		return err
	}

	return nil
}

func newInternalAccessData(d *osin.AccessData) *oAccessData {
	if d == nil {
		return nil
	}

	clientId := ""
	if client := d.Client; client != nil {
		clientId = client.Id
	}

	authCode := ""
	if authorizeData := d.AuthorizeData; authorizeData != nil {
		authCode = authorizeData.Code
	}

	accessToken := ""
	if accessData := d.AccessData; accessData != nil {
		accessToken = accessData.AccessToken
	}
	return &oAccessData{clientId, authCode, accessToken, d.AccessToken, d.RefreshToken, d.ExpiresIn, d.Scope, d.RedirectUri, d.CreatedAt}
}

func newOsinAccessData(d *oAccessData, c *osin.Client, authData *osin.AuthorizeData, accessData *osin.AccessData) *osin.AccessData {
	if d == nil {
		return nil
	}
	return &osin.AccessData{c, authData, accessData, d.AccessToken, d.RefreshToken, d.ExpiresIn, d.Scope, d.RedirectUri, d.CreatedAt, nil}
}

func (s *OsinAppEngineStore) SaveAccess(data *osin.AccessData) error {
	s.c.Debugf("SaveAccess: %s\n", data.AccessToken)
	key := datastore.NewKey(s.c, "access.data", data.AccessToken, 0, nil)
	internalAccessData := newInternalAccessData(data)
	_, err := datastore.Put(s.c, key, internalAccessData)
	if err != nil {
		return err
	}

	if data.RefreshToken != "" {
		key = datastore.NewKey(s.c, "access.refresh", data.RefreshToken, 0, nil)
		_, err := datastore.Put(s.c, key, internalAccessData)
		if err != nil {
			return err
		}
	}
	return nil
}

func (s *OsinAppEngineStore) LoadAccess(code string) (*osin.AccessData, error) {
	s.c.Debugf("LoadAccess: %s\n", code)
	key := datastore.NewKey(s.c, "access.data", code, 0, nil)
	accessData := new(oAccessData)
	err := datastore.Get(s.c, key, accessData)
	if err != nil {
		s.c.Infof("Access data not found for code [%s]: %v", code, err)
		return nil, errors.New("Access data not found")
	}

	var c *osin.Client
	if accessData.ClientId != "" {
		c, err = s.GetClient(accessData.ClientId)
		if err != nil {
			s.c.Infof("Access data can't load client with id [%s]: %v", accessData.ClientId, err)
			return nil, errors.New("Client for AccessData not found")
		}
	}

	// TODO Load inner authorize and access data
	return newOsinAccessData(accessData, c, nil, nil), nil
}

func (s *OsinAppEngineStore) RemoveAccess(code string) error {
	s.c.Debugf("RemoveAccess: %s\n", code)
	key := datastore.NewKey(s.c, "access.data", code, 0, nil)
	err := datastore.Delete(s.c, key)
	if err != nil {
		return err
	}

	return nil
}

func (s *OsinAppEngineStore) LoadRefresh(code string) (*osin.AccessData, error) {
	s.c.Debugf("LoadRefresh: %s\n", code)
	key := datastore.NewKey(s.c, "access.refresh", code, 0, nil)
	accessData := new(oAccessData)
	_, err := datastore.Put(s.c, key, accessData)
	if err != nil {
		s.c.Infof("Refresh data not found for code [%s]: %v", code, err)
		errors.New("Refresh not found")
	}

	// TODO Gracefully handle no client by setting to nil
	var c *osin.Client
	if accessData.ClientId != "" {
		c, err = s.GetClient(accessData.ClientId)
		if err != nil {
			s.c.Infof("Access data can't load client with id [%s]: %v", accessData.ClientId, err)
			return nil, errors.New("Client for AccessData not found")
		}
	}
	// TODO Load inner authorize and access data
	return newOsinAccessData(accessData, c, nil, nil), nil
}

func (s *OsinAppEngineStore) RemoveRefresh(code string) error {
	s.c.Debugf("RemoveRefresh: %s\n", code)
	key := datastore.NewKey(s.c, "access.refresh", code, 0, nil)
	err := datastore.Delete(s.c, key)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gtcp

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"io"
	"net"
	"time"
)

// TCP connection object.
type Conn struct {
	net.Conn                     // Underlying TCP connection object.
	reader         *bufio.Reader // Buffer reader for connection.
	recvDeadline   time.Time     // Timeout point for reading.
	sendDeadline   time.Time     // Timeout point for writing.
	recvBufferWait time.Duration // Interval duration for reading buffer.
}

const (
	// Default interval for reading buffer.
	gRECV_ALL_WAIT_TIMEOUT = time.Millisecond
)

// NewConn creates and returns a new connection with given address.
func NewConn(addr string, timeout ...time.Duration) (*Conn, error) {
	if conn, err := NewNetConn(addr, timeout...); err == nil {
		return NewConnByNetConn(conn), nil
	} else {
		return nil, err
	}
}

// NewConnTLS creates and returns a new TLS connection
// with given address and TLS configuration.
func NewConnTLS(addr string, tlsConfig *tls.Config) (*Conn, error) {
	if conn, err := NewNetConnTLS(addr, tlsConfig); err == nil {
		return NewConnByNetConn(conn), nil
	} else {
		return nil, err
	}
}

// NewConnKeyCrt creates and returns a new TLS connection
// with given address and TLS certificate and key files.
func NewConnKeyCrt(addr, crtFile, keyFile string) (*Conn, error) {
	if conn, err := NewNetConnKeyCrt(addr, crtFile, keyFile); err == nil {
		return NewConnByNetConn(conn), nil
	} else {
		return nil, err
	}
}

// NewConnByNetConn creates and returns a TCP connection object with given net.Conn object.
func NewConnByNetConn(conn net.Conn) *Conn {
	return &Conn{
		Conn:           conn,
		reader:         bufio.NewReader(conn),
		recvDeadline:   time.Time{},
		sendDeadline:   time.Time{},
		recvBufferWait: gRECV_ALL_WAIT_TIMEOUT,
	}
}

// Send writes data to remote address.
func (c *Conn) Send(data []byte, retry ...Retry) error {
	for {
		if _, err := c.Write(data); err != nil {
			// Connection closed.
			if err == io.EOF {
				return err
			}
			// Still failed even after retrying.
			if len(retry) == 0 || retry[0].Count == 0 {
				return err
			}
			if len(retry) > 0 {
				retry[0].Count--
				if retry[0].Interval == 0 {
					retry[0].Interval = gDEFAULT_RETRY_INTERVAL
				}
				time.Sleep(time.Duration(retry[0].Interval) * time.Millisecond)
			}
		} else {
			return nil
		}
	}
}

// Recv receives data from the connection.
//
// Note that,
// 1. If length = 0, it means it receives the data from current buffer and returns immediately.
// 2. If length < 0, it means it receives all data from buffer and returns if it waits til no data from connection.
//    Developers should notice the package parsing yourself if you decide receiving all data from buffer.
// 3. If length > 0, it means it blocks reading data from connection until length size was received.
func (c *Conn) Recv(length int, retry ...Retry) ([]byte, error) {
	var err error       // Reading error.
	var size int        // Reading size.
	var index int       // Received size.
	var buffer []byte   // Buffer object.
	var bufferWait bool // Whether buffer reading timeout set.

	if length > 0 {
		buffer = make([]byte, length)
	} else {
		buffer = make([]byte, gDEFAULT_READ_BUFFER_SIZE)
	}

	for {
		if length < 0 && index > 0 {
			bufferWait = true
			if err = c.SetReadDeadline(time.Now().Add(c.recvBufferWait)); err != nil {
				return nil, err
			}
		}
		size, err = c.reader.Read(buffer[index:])
		if size > 0 {
			index += size
			if length > 0 {
				// It reads til <length> size if <length> is specified.
				if index == length {
					break
				}
			} else {
				if index >= gDEFAULT_READ_BUFFER_SIZE {
					// If it exceeds the buffer size, it then automatically increases its buffer size.
					buffer = append(buffer, make([]byte, gDEFAULT_READ_BUFFER_SIZE)...)
				} else {
					// It returns immediately if received size is lesser than buffer size.
					if !bufferWait {
						break
					}
				}
			}
		}
		if err != nil {
			// Connection closed.
			if err == io.EOF {
				break
			}
			// Re-set the timeout when reading data.
			if bufferWait && isTimeout(err) {
				if err = c.SetReadDeadline(c.recvDeadline); err != nil {
					return nil, err
				}
				err = nil
				break
			}
			if len(retry) > 0 {
				// It fails even it retried.
				if retry[0].Count == 0 {
					break
				}
				retry[0].Count--
				if retry[0].Interval == 0 {
					retry[0].Interval = gDEFAULT_RETRY_INTERVAL
				}
				time.Sleep(time.Duration(retry[0].Interval) * time.Millisecond)
				continue
			}
			break
		}
		// Just read once from buffer.
		if length == 0 {
			break
		}
	}
	return buffer[:index], err
}

// RecvLine reads data from the connection until reads char '\n'.
// Note that the returned result does not contain the last char '\n'.
func (c *Conn) RecvLine(retry ...Retry) ([]byte, error) {
	var err error
	var buffer []byte
	data := make([]byte, 0)
	for {
		buffer, err = c.Recv(1, retry...)
		if len(buffer) > 0 {
			if buffer[0] == '\n' {
				data = append(data, buffer[:len(buffer)-1]...)
				break
			} else {
				data = append(data, buffer...)
			}
		}
		if err != nil {
			break
		}
	}
	return data, err
}

// RecvTil reads data from the connection until reads bytes <til>.
// Note that the returned result contains the last bytes <til>.
func (c *Conn) RecvTil(til []byte, retry ...Retry) ([]byte, error) {
	var err error
	var buffer []byte
	data := make([]byte, 0)
	length := len(til)
	for {
		buffer, err = c.Recv(1, retry...)
		if len(buffer) > 0 {
			if length > 0 &&
				len(data) >= length-1 &&
				buffer[0] == til[length-1] &&
				bytes.EqualFold(data[len(data)-length+1:], til[:length-1]) {
				data = append(data, buffer...)
				break
			} else {
				data = append(data, buffer...)
			}
		}
		if err != nil {
			break
		}
	}
	return data, err
}

// RecvWithTimeout reads data from the connection with timeout.
func (c *Conn) RecvWithTimeout(length int, timeout time.Duration, retry ...Retry) (data []byte, err error) {
	if err := c.SetRecvDeadline(time.Now().Add(timeout)); err != nil {
		return nil, err
	}
	defer c.SetRecvDeadline(time.Time{})
	data, err = c.Recv(length, retry...)
	return
}

// SendWithTimeout writes data to the connection with timeout.
func (c *Conn) SendWithTimeout(data []byte, timeout time.Duration, retry ...Retry) (err error) {
	if err := c.SetSendDeadline(time.Now().Add(timeout)); err != nil {
		return err
	}
	defer c.SetSendDeadline(time.Time{})
	err = c.Send(data, retry...)
	return
}

// SendRecv writes data to the connection and blocks reading response.
func (c *Conn) SendRecv(data []byte, length int, retry ...Retry) ([]byte, error) {
	if err := c.Send(data, retry...); err == nil {
		return c.Recv(length, retry...)
	} else {
		return nil, err
	}
}

// SendRecvWithTimeout writes data to the connection and reads response with timeout.
func (c *Conn) SendRecvWithTimeout(data []byte, length int, timeout time.Duration, retry ...Retry) ([]byte, error) {
	if err := c.Send(data, retry...); err == nil {
		return c.RecvWithTimeout(length, timeout, retry...)
	} else {
		return nil, err
	}
}

func (c *Conn) SetDeadline(t time.Time) error {
	err := c.Conn.SetDeadline(t)
	if err == nil {
		c.recvDeadline = t
		c.sendDeadline = t
	}
	return err
}

func (c *Conn) SetRecvDeadline(t time.Time) error {
	err := c.SetReadDeadline(t)
	if err == nil {
		c.recvDeadline = t
	}
	return err
}

func (c *Conn) SetSendDeadline(t time.Time) error {
	err := c.SetWriteDeadline(t)
	if err == nil {
		c.sendDeadline = t
	}
	return err
}

// SetRecvBufferWait sets the buffer waiting timeout when reading all data from connection.
// The waiting duration cannot be too long which might delay receiving data from remote address.
func (c *Conn) SetRecvBufferWait(bufferWaitDuration time.Duration) {
	c.recvBufferWait = bufferWaitDuration
}
<commit_msg>fix issue in Retry.Interval type changed for gtcp<commit_after>// Copyright 2018 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gtcp

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"io"
	"net"
	"time"
)

// TCP connection object.
type Conn struct {
	net.Conn                     // Underlying TCP connection object.
	reader         *bufio.Reader // Buffer reader for connection.
	recvDeadline   time.Time     // Timeout point for reading.
	sendDeadline   time.Time     // Timeout point for writing.
	recvBufferWait time.Duration // Interval duration for reading buffer.
}

const (
	// Default interval for reading buffer.
	gRECV_ALL_WAIT_TIMEOUT = time.Millisecond
)

// NewConn creates and returns a new connection with given address.
func NewConn(addr string, timeout ...time.Duration) (*Conn, error) {
	if conn, err := NewNetConn(addr, timeout...); err == nil {
		return NewConnByNetConn(conn), nil
	} else {
		return nil, err
	}
}

// NewConnTLS creates and returns a new TLS connection
// with given address and TLS configuration.
func NewConnTLS(addr string, tlsConfig *tls.Config) (*Conn, error) {
	if conn, err := NewNetConnTLS(addr, tlsConfig); err == nil {
		return NewConnByNetConn(conn), nil
	} else {
		return nil, err
	}
}

// NewConnKeyCrt creates and returns a new TLS connection
// with given address and TLS certificate and key files.
func NewConnKeyCrt(addr, crtFile, keyFile string) (*Conn, error) {
	if conn, err := NewNetConnKeyCrt(addr, crtFile, keyFile); err == nil {
		return NewConnByNetConn(conn), nil
	} else {
		return nil, err
	}
}

// NewConnByNetConn creates and returns a TCP connection object with given net.Conn object.
func NewConnByNetConn(conn net.Conn) *Conn {
	return &Conn{
		Conn:           conn,
		reader:         bufio.NewReader(conn),
		recvDeadline:   time.Time{},
		sendDeadline:   time.Time{},
		recvBufferWait: gRECV_ALL_WAIT_TIMEOUT,
	}
}

// Send writes data to remote address.
func (c *Conn) Send(data []byte, retry ...Retry) error {
	for {
		if _, err := c.Write(data); err != nil {
			// Connection closed.
			if err == io.EOF {
				return err
			}
			// Still failed even after retrying.
			if len(retry) == 0 || retry[0].Count == 0 {
				return err
			}
			if len(retry) > 0 {
				retry[0].Count--
				if retry[0].Interval == 0 {
					retry[0].Interval = gDEFAULT_RETRY_INTERVAL
				}
				time.Sleep(retry[0].Interval)
			}
		} else {
			return nil
		}
	}
}

// Recv receives data from the connection.
//
// Note that,
// 1. If length = 0, which means it receives the data from current buffer and returns immediately.
// 2. If length < 0, which means it receives all data from buffer and returns if it waits til no data from connection.
//    Developers should notice the package parsing yourself if you decide receiving all data from buffer.
// 3. If length > 0, which means it blocks reading data from connection until length size was received.
//    It is the most commonly used length value for data receiving.
func (c *Conn) Recv(length int, retry ...Retry) ([]byte, error) {
	var err error       // Reading error.
	var size int        // Reading size.
	var index int       // Received size.
	var buffer []byte   // Buffer object.
	var bufferWait bool // Whether buffer reading timeout set.

	if length > 0 {
		buffer = make([]byte, length)
	} else {
		buffer = make([]byte, gDEFAULT_READ_BUFFER_SIZE)
	}

	for {
		if length < 0 && index > 0 {
			bufferWait = true
			if err = c.SetReadDeadline(time.Now().Add(c.recvBufferWait)); err != nil {
				return nil, err
			}
		}
		size, err = c.reader.Read(buffer[index:])
		if size > 0 {
			index += size
			if length > 0 {
				// It reads til <length> size if <length> is specified.
				if index == length {
					break
				}
			} else {
				if index >= gDEFAULT_READ_BUFFER_SIZE {
					// If it exceeds the buffer size, it then automatically increases its buffer size.
					buffer = append(buffer, make([]byte, gDEFAULT_READ_BUFFER_SIZE)...)
				} else {
					// It returns immediately if received size is lesser than buffer size.
					if !bufferWait {
						break
					}
				}
			}
		}
		if err != nil {
			// Connection closed.
			if err == io.EOF {
				break
			}
			// Re-set the timeout when reading data.
			if bufferWait && isTimeout(err) {
				if err = c.SetReadDeadline(c.recvDeadline); err != nil {
					return nil, err
				}
				err = nil
				break
			}
			if len(retry) > 0 {
				// It fails even it retried.
				if retry[0].Count == 0 {
					break
				}
				retry[0].Count--
				if retry[0].Interval == 0 {
					retry[0].Interval = gDEFAULT_RETRY_INTERVAL
				}
				time.Sleep(retry[0].Interval)
				continue
			}
			break
		}
		// Just read once from buffer.
		if length == 0 {
			break
		}
	}
	return buffer[:index], err
}

// RecvLine reads data from the connection until reads char '\n'.
// Note that the returned result does not contain the last char '\n'.
func (c *Conn) RecvLine(retry ...Retry) ([]byte, error) {
	var err error
	var buffer []byte
	data := make([]byte, 0)
	for {
		buffer, err = c.Recv(1, retry...)
		if len(buffer) > 0 {
			if buffer[0] == '\n' {
				data = append(data, buffer[:len(buffer)-1]...)
				break
			} else {
				data = append(data, buffer...)
			}
		}
		if err != nil {
			break
		}
	}
	return data, err
}

// RecvTil reads data from the connection until reads bytes <til>.
// Note that the returned result contains the last bytes <til>.
func (c *Conn) RecvTil(til []byte, retry ...Retry) ([]byte, error) {
	var err error
	var buffer []byte
	data := make([]byte, 0)
	length := len(til)
	for {
		buffer, err = c.Recv(1, retry...)
		if len(buffer) > 0 {
			if length > 0 &&
				len(data) >= length-1 &&
				buffer[0] == til[length-1] &&
				bytes.EqualFold(data[len(data)-length+1:], til[:length-1]) {
				data = append(data, buffer...)
				break
			} else {
				data = append(data, buffer...)
			}
		}
		if err != nil {
			break
		}
	}
	return data, err
}

// RecvWithTimeout reads data from the connection with timeout.
func (c *Conn) RecvWithTimeout(length int, timeout time.Duration, retry ...Retry) (data []byte, err error) {
	if err := c.SetRecvDeadline(time.Now().Add(timeout)); err != nil {
		return nil, err
	}
	defer c.SetRecvDeadline(time.Time{})
	data, err = c.Recv(length, retry...)
	return
}

// SendWithTimeout writes data to the connection with timeout.
func (c *Conn) SendWithTimeout(data []byte, timeout time.Duration, retry ...Retry) (err error) {
	if err := c.SetSendDeadline(time.Now().Add(timeout)); err != nil {
		return err
	}
	defer c.SetSendDeadline(time.Time{})
	err = c.Send(data, retry...)
	return
}

// SendRecv writes data to the connection and blocks reading response.
func (c *Conn) SendRecv(data []byte, length int, retry ...Retry) ([]byte, error) {
	if err := c.Send(data, retry...); err == nil {
		return c.Recv(length, retry...)
	} else {
		return nil, err
	}
}

// SendRecvWithTimeout writes data to the connection and reads response with timeout.
func (c *Conn) SendRecvWithTimeout(data []byte, length int, timeout time.Duration, retry ...Retry) ([]byte, error) {
	if err := c.Send(data, retry...); err == nil {
		return c.RecvWithTimeout(length, timeout, retry...)
	} else {
		return nil, err
	}
}

func (c *Conn) SetDeadline(t time.Time) error {
	err := c.Conn.SetDeadline(t)
	if err == nil {
		c.recvDeadline = t
		c.sendDeadline = t
	}
	return err
}

func (c *Conn) SetRecvDeadline(t time.Time) error {
	err := c.SetReadDeadline(t)
	if err == nil {
		c.recvDeadline = t
	}
	return err
}

func (c *Conn) SetSendDeadline(t time.Time) error {
	err := c.SetWriteDeadline(t)
	if err == nil {
		c.sendDeadline = t
	}
	return err
}

// SetRecvBufferWait sets the buffer waiting timeout when reading all data from connection.
// The waiting duration cannot be too long which might delay receiving data from remote address.
func (c *Conn) SetRecvBufferWait(bufferWaitDuration time.Duration) {
	c.recvBufferWait = bufferWaitDuration
}
<|endoftext|>


<commit_before>package operator

import (
	"fmt"
	"log"
	"time"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/pkg/api/unversioned"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
	"k8s.io/client-go/pkg/runtime"
	"k8s.io/client-go/pkg/runtime/serializer"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

var (
	etcdHostOutside string

	VENDOR       = "acid.zalan.do"
	VERSION      = "0.0.1.dev"
	resyncPeriod = 5 * time.Minute

	etcdKeyTemplate = "/service/%s"
)

type Options struct {
	KubeConfig string
}

type Pgconf struct {
	Parameter string `json:"param"`
	Value     string `json:"value"`
}

type SpiloSpec struct {
	EtcdHost              string   `json:"etcd_host"`
	VolumeSize            int      `json:"volume_size"`
	NumberOfInstances     int32    `json:"number_of_instances"`
	DockerImage           string   `json:"docker_image"`
	PostgresConfiguration []Pgconf `json:"postgres_configuration"`
	ResourceCPU           string   `json:"resource_cpu"`
	ResourceMemory        string   `json:"resource_memory"`
}

type Spilo struct {
	unversioned.TypeMeta `json:",inline"`
	Metadata             api.ObjectMeta `json:"metadata"`
	Spec                 SpiloSpec      `json:"spec"`
}

type SpiloList struct {
	unversioned.TypeMeta `json:",inline"`
	Metadata             unversioned.ListMeta `json:"metadata"`
	Items                []Spilo              `json:"items"`
}

func KubernetesConfig(options Options) *rest.Config {
	rules := clientcmd.NewDefaultClientConfigLoadingRules()
	overrides := &clientcmd.ConfigOverrides{}

	if options.KubeConfig != "" {
		rules.ExplicitPath = options.KubeConfig
	}

	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig()

	etcdHostOutside = config.Host

	if err != nil {
		log.Fatalf("Couldn't get Kubernetes default config: %s", err)
	}

	return config
}

func newKubernetesSpiloClient(c *rest.Config) (*rest.RESTClient, error) {
	c.APIPath = "/apis"
	c.GroupVersion = &unversioned.GroupVersion{
		Group:   VENDOR,
		Version: "v1",
	}
	c.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs}

	schemeBuilder := runtime.NewSchemeBuilder(
		func(scheme *runtime.Scheme) error {
			scheme.AddKnownTypes(
				*c.GroupVersion,
				&Spilo{},
				&SpiloList{},
				&api.ListOptions{},
				&api.DeleteOptions{},
			)
			return nil
		})
	schemeBuilder.AddToScheme(api.Scheme)

	return rest.RESTClientFor(c)
}

func EnsureSpiloThirdPartyResource(client *kubernetes.Clientset) error {
	_, err := client.ExtensionsV1beta1().ThirdPartyResources().Get(fmt.Sprintf("spilo.%s", VENDOR))
	if err == nil {
		return err
	}

	// The resource doesn't exist, so we create it.
	tpr := v1beta1.ThirdPartyResource{
		ObjectMeta: v1.ObjectMeta{
			Name: fmt.Sprintf("spilo.%s", VENDOR),
		},
		Description: "A specification of Spilo StatefulSets",
		Versions: []v1beta1.APIVersion{
			{Name: "v1"},
		},
	}

	_, err = client.ExtensionsV1beta1().ThirdPartyResources().Create(&tpr)

	return err
}
<commit_msg>get rid of race condition<commit_after>package operator

import (
	"fmt"
	"log"
	"time"

    "net/http"

	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
    apierrors "k8s.io/client-go/pkg/api/errors"
	"k8s.io/client-go/pkg/api/unversioned"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
	"k8s.io/client-go/pkg/runtime"
	"k8s.io/client-go/pkg/runtime/serializer"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

var (
	etcdHostOutside string

	VENDOR       = "acid.zalan.do"
	VERSION      = "0.0.1.dev"
	resyncPeriod = 5 * time.Minute

	etcdKeyTemplate = "/service/%s"
)

type Options struct {
	KubeConfig string
}

type Pgconf struct {
	Parameter string `json:"param"`
	Value     string `json:"value"`
}

type SpiloSpec struct {
	EtcdHost              string   `json:"etcd_host"`
	VolumeSize            int      `json:"volume_size"`
	NumberOfInstances     int32    `json:"number_of_instances"`
	DockerImage           string   `json:"docker_image"`
	PostgresConfiguration []Pgconf `json:"postgres_configuration"`
	ResourceCPU           string   `json:"resource_cpu"`
	ResourceMemory        string   `json:"resource_memory"`
}

type Spilo struct {
	unversioned.TypeMeta `json:",inline"`
	Metadata             api.ObjectMeta `json:"metadata"`
	Spec                 SpiloSpec      `json:"spec"`
}

type SpiloList struct {
	unversioned.TypeMeta `json:",inline"`
	Metadata             unversioned.ListMeta `json:"metadata"`
	Items                []Spilo              `json:"items"`
}

func KubernetesConfig(options Options) *rest.Config {
	rules := clientcmd.NewDefaultClientConfigLoadingRules()
	overrides := &clientcmd.ConfigOverrides{}

	if options.KubeConfig != "" {
		rules.ExplicitPath = options.KubeConfig
	}

	config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig()

	etcdHostOutside = config.Host

	if err != nil {
		log.Fatalf("Couldn't get Kubernetes default config: %s", err)
	}

	return config
}

func newKubernetesSpiloClient(c *rest.Config) (*rest.RESTClient, error) {
	c.APIPath = "/apis"
	c.GroupVersion = &unversioned.GroupVersion{
		Group:   VENDOR,
		Version: "v1",
	}
	c.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: api.Codecs}

	schemeBuilder := runtime.NewSchemeBuilder(
		func(scheme *runtime.Scheme) error {
			scheme.AddKnownTypes(
				*c.GroupVersion,
				&Spilo{},
				&SpiloList{},
				&api.ListOptions{},
				&api.DeleteOptions{},
			)
			return nil
		})
	schemeBuilder.AddToScheme(api.Scheme)

	return rest.RESTClientFor(c)
}

//TODO: Move to separate package
func IsKubernetesResourceNotFoundError(err error) bool {
    se, ok := err.(*apierrors.StatusError)
    if !ok {
        return false
    }
    if se.Status().Code == http.StatusNotFound && se.Status().Reason == unversioned.StatusReasonNotFound {
        return true
    }
    return false
}

func EnsureSpiloThirdPartyResource(client *kubernetes.Clientset) error {
	// The resource doesn't exist, so we create it.
	tpr := v1beta1.ThirdPartyResource{
		ObjectMeta: v1.ObjectMeta{
			Name: fmt.Sprintf("spilo.%s", VENDOR),
		},
		Description: "A specification of Spilo StatefulSets",
		Versions: []v1beta1.APIVersion{
			{Name: "v1"},
		},
	}

	_, err := client.ExtensionsV1beta1().ThirdPartyResources().Create(&tpr)

    if IsKubernetesResourceNotFoundError(err) {
        return err
    }

    return nil
}
<|endoftext|>


<commit_before>package rabbitmq

import (
	"github.com/streadway/amqp"
)

type Consumer struct {
	// Base struct for Producer
	*RabbitMQ

	// All deliveries from server will send to this channel
	deliveries <-chan amqp.Delivery

	// This handler will be called when a
	handler func(amqp.Delivery)

	// A notifiyng channel for publishings
	// will be used for sync. between close channel and consume handler
	done chan error

	// Current producer connection settings
	session Session
}

var (
	Q  amqp.Queue
	CO ConsumerOptions
)

func (c *Consumer) Deliveries() <-chan amqp.Delivery {
	return c.deliveries
}

// NewConsumer is a constructor for consumer creation
// Accepts Exchange, Queue, BindingOptions and ConsumerOptions
func NewConsumer(e Exchange, q Queue, bo BindingOptions, co ConsumerOptions) (*Consumer, error) {
	rmq, err := newRabbitMQConnection(co.Tag)
	if err != nil {
		return nil, err
	}

	c := &Consumer{
		RabbitMQ: rmq,
		done:     make(chan error),
		session: Session{
			Exchange:        e,
			Queue:           q,
			ConsumerOptions: co,
			BindingOptions:  bo,
		},
	}

	err = c.connect()
	if err != nil {
		return nil, err
	}

	return c, nil
}

// connect internally declares the exchanges and queues
// and starts to stream from given queue
func (c *Consumer) connect() error {
	e := c.session.Exchange
	q := c.session.Queue
	bo := c.session.BindingOptions
	CO = c.session.ConsumerOptions

	var err error

	// declaring Exchange
	if err = c.channel.ExchangeDeclare(
		e.Name,       // name of the exchange
		e.Type,       // type
		e.Durable,    // durable
		e.AutoDelete, // delete when complete
		e.Internal,   // internal
		e.NoWait,     // noWait
		e.Args,       // arguments
	); err != nil {
		return err
	}

	// declaring Queue
	Q, err = c.channel.QueueDeclare(
		q.Name,       // name of the queue
		q.Durable,    // durable
		q.AutoDelete, // delete when usused
		q.Exclusive,  // exclusive
		q.NoWait,     // noWait
		q.Args,       // arguments
	)
	if err != nil {
		return err
	}

	// binding Exchange to Queue
	if err = c.channel.QueueBind(
		// bind to real queue
		Q.Name,        // name of the queue
		bo.RoutingKey, // bindingKey
		e.Name,        // sourceExchange
		bo.NoWait,     // noWait
		bo.Args,       // arguments
	); err != nil {
		return err
	}

	return nil
}

// Consume accepts a handler function for every message streamed from RabbitMq
// will be called within this handler func
func (c *Consumer) Consume(handler func(delivery amqp.Delivery)) error {
	// Exchange bound to Queue, starting Consume
	deliveries, err := c.channel.Consume(
		// consume from real queue
		Q.Name,       // name
		CO.Tag,       // consumerTag,
		CO.AutoAck,   // autoAck
		CO.Exclusive, // exclusive
		CO.NoLocal,   // noLocal
		CO.NoWait,    // noWait
		CO.Args,      // arguments
	)
	if err != nil {
		return err
	}

	// should we stop streaming, in order not to consume from server?
	c.deliveries = deliveries
	c.handler = handler

	// handle all consumer errors, if required re-connect
	// there are problems with reconnection logic for now
	for delivery := range c.deliveries {
		handler(delivery)
	}

	log.Info("handle: deliveries channel closed")
	c.done <- nil
	return nil
}

// ConsumeMessage accepts a handler function and only consumes one message
// stream from RabbitMq and then closes connection
func (c *Consumer) Get(handler func(delivery amqp.Delivery)) error {
	message, ok, err := c.channel.Get(
		Q.Name,
		CO.AutoAck)
	if err != nil {
		return err
	}

	c.handler = handler

	if ok {
		log.Info("Message received")
		handler(message)
	} else {
		log.Info("No message received")
	}

	return shutdown(c.conn, c.channel, c.tag)
}

// Shutdown gracefully closes all connections and waits
// for handler to finish its messages
func (c *Consumer) Shutdown() error {
	// to-do
	// first stop streaming then close connections
	err := shutdown(c.conn, c.channel, c.tag)
	if err != nil {
		return err
	}

	defer log.Info("Consumer shutdown OK")
	log.Info("Waiting for handler to exit")

	// this channel is here for finishing the consumer's ranges of
	// delivery chans.  We need every delivery to be processed, here make
	// sure to wait for all consumers goroutines to finish before exiting our
	// process.
	return <-c.done
}

// RegisterSignalHandler watchs for interrupt signals
// and gracefully closes consumer
func (c *Consumer) RegisterSignalHandler() {
	registerSignalHandler(c)
}
<commit_msg>Moderation: removed global variables from consumer.go<commit_after>package rabbitmq

import (
	"github.com/streadway/amqp"
)

type Consumer struct {
	// Base struct for Producer
	*RabbitMQ

	// All deliveries from server will send to this channel
	deliveries <-chan amqp.Delivery

	// This handler will be called when a
	handler func(amqp.Delivery)

	// A notifiyng channel for publishings
	// will be used for sync. between close channel and consume handler
	done chan error

	// Current producer connection settings
	session Session
}

func (c *Consumer) Deliveries() <-chan amqp.Delivery {
	return c.deliveries
}

// NewConsumer is a constructor for consumer creation
// Accepts Exchange, Queue, BindingOptions and ConsumerOptions
func NewConsumer(e Exchange, q Queue, bo BindingOptions, co ConsumerOptions) (*Consumer, error) {
	rmq, err := newRabbitMQConnection(co.Tag)
	if err != nil {
		return nil, err
	}

	c := &Consumer{
		RabbitMQ: rmq,
		done:     make(chan error),
		session: Session{
			Exchange:        e,
			Queue:           q,
			ConsumerOptions: co,
			BindingOptions:  bo,
		},
	}

	err = c.connect()
	if err != nil {
		return nil, err
	}

	return c, nil
}

// connect internally declares the exchanges and queues
// and starts to stream from given queue
func (c *Consumer) connect() error {
	e := c.session.Exchange
	q := c.session.Queue
	bo := c.session.BindingOptions

	var err error

	// declaring Exchange
	if err = c.channel.ExchangeDeclare(
		e.Name,       // name of the exchange
		e.Type,       // type
		e.Durable,    // durable
		e.AutoDelete, // delete when complete
		e.Internal,   // internal
		e.NoWait,     // noWait
		e.Args,       // arguments
	); err != nil {
		return err
	}

	// declaring Queue
	_, err = c.channel.QueueDeclare(
		q.Name,       // name of the queue
		q.Durable,    // durable
		q.AutoDelete, // delete when usused
		q.Exclusive,  // exclusive
		q.NoWait,     // noWait
		q.Args,       // arguments
	)
	if err != nil {
		return err
	}

	// binding Exchange to Queue
	if err = c.channel.QueueBind(
		// bind to real queue
		q.Name,        // name of the queue
		bo.RoutingKey, // bindingKey
		e.Name,        // sourceExchange
		bo.NoWait,     // noWait
		bo.Args,       // arguments
	); err != nil {
		return err
	}

	return nil
}

// Consume accepts a handler function for every message streamed from RabbitMq
// will be called within this handler func
func (c *Consumer) Consume(handler func(delivery amqp.Delivery)) error {
	co := c.session.ConsumerOptions
	q := c.session.Queue
	// Exchange bound to Queue, starting Consume
	deliveries, err := c.channel.Consume(
		// consume from real queue
		q.Name,       // name
		co.Tag,       // consumerTag,
		co.AutoAck,   // autoAck
		co.Exclusive, // exclusive
		co.NoLocal,   // noLocal
		co.NoWait,    // noWait
		co.Args,      // arguments
	)
	if err != nil {
		return err
	}

	// should we stop streaming, in order not to consume from server?
	c.deliveries = deliveries
	c.handler = handler

	// handle all consumer errors, if required re-connect
	// there are problems with reconnection logic for now
	for delivery := range c.deliveries {
		handler(delivery)
	}

	log.Info("handle: deliveries channel closed")
	c.done <- nil
	return nil
}

// ConsumeMessage accepts a handler function and only consumes one message
// stream from RabbitMq and then closes connection
func (c *Consumer) Get(handler func(delivery amqp.Delivery)) error {
	co := c.session.ConsumerOptions
	q := c.session.Queue
	message, ok, err := c.channel.Get(q.Name, co.AutoAck)
	if err != nil {
		return err
	}

	c.handler = handler

	if ok {
		log.Info("Message received")
		handler(message)
	} else {
		log.Info("No message received")
	}

	return shutdown(c.conn, c.channel, c.tag)
}

// Shutdown gracefully closes all connections and waits
// for handler to finish its messages
func (c *Consumer) Shutdown() error {
	// to-do
	// first stop streaming then close connections
	err := shutdown(c.conn, c.channel, c.tag)
	if err != nil {
		return err
	}

	defer log.Info("Consumer shutdown OK")
	log.Info("Waiting for handler to exit")

	// this channel is here for finishing the consumer's ranges of
	// delivery chans.  We need every delivery to be processed, here make
	// sure to wait for all consumers goroutines to finish before exiting our
	// process.
	return <-c.done
}

// RegisterSignalHandler watchs for interrupt signals
// and gracefully closes consumer
func (c *Consumer) RegisterSignalHandler() {
	registerSignalHandler(c)
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestDeleteUser(t *testing.T) {

}
<commit_msg>janitor: added test case for DeleteUser operation<commit_after>package main

import (
	"encoding/json"
	"koding/db/models"
	"koding/db/mongodb/modelhelper"
	"net"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestDeleteUser(t *testing.T) {
	testHelper := func() {}

	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		testHelper()
		var ur usernameReq

		if err := json.NewDecoder(req.Body).Decode(&ur); err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		var c *http.Cookie
		for _, cookie := range req.Cookies() {
			if cookie.Name == "clientId" {
				c = cookie
				break
			}
		}

		if c == nil {
			http.Error(w, "client id cookie is nil", http.StatusBadRequest)
			return
		}

		ses, err := modelhelper.GetSessionById(c.Value)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}

		if ses == nil {
			http.Error(w, "couldnt find session", http.StatusBadRequest)
			return
		}

		if ses.Username != ur.Username {
			http.Error(w, "usernames did not match", http.StatusBadRequest)
			return
		}

	}))
	defer server.Close()

	deleterFn := newDeleteUserFunc(server.URL)

	Convey("Given a fake delete endpoint", t, func() {

		Convey("Should return OK when everything is fine", func() {
			err := deleterFn(&models.User{Name: "my-test-username"}, "")
			So(err, ShouldBeNil)

			err = deleterFn(&models.User{Name: ""}, "")
			So(err, ShouldNotBeNil)
		})

		Convey("Should return timeout error, if execeeds timeout value", func() {
			testHelperCache := testHelper
			timeoutCache := defClient.Timeout
			defClient.Timeout = time.Millisecond * 100

			testHelper = func() {
				time.Sleep(defClient.Timeout * 2)
			}

			defer func() {
				testHelper = testHelperCache
				defClient.Timeout = timeoutCache
			}()

			err := deleterFn(&models.User{Name: "my-test-username"}, "")
			So(err, ShouldNotBeNil)
		})
	})

	deleterFn = newDeleteUserFunc("http://localhost:" + freePort())

	Convey("Given an invalid fake delete endpoint", t, func() {
		Convey("Should return error immediately", func() {
			err := deleterFn(&models.User{Name: "my-test-username"}, "")
			So(err, ShouldNotBeNil)
		})
	})

}

// Ask the kernel for a free open port that is ready to use
func freePort() string {
	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	if err != nil {
		panic(err)
	}

	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		panic(err)
	}
	defer l.Close()

	return strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
}
<|endoftext|>


<commit_before>package helper

// TODO use loghelper under koding/worker package instead of this

import (
	"os"

	"github.com/koding/logging"
)

func CreateLogger(name string, debug bool) logging.Logger {
	log := logging.NewLogger(name)
	logHandler := logging.NewWriterHandler(os.Stderr)
	logHandler.Colorize = true
	log.SetHandler(logHandler)

	if debug {
		log.SetLevel(logging.DEBUG)
		logHandler.SetLevel(logging.DEBUG)
	}

	return log
}
<commit_msg>Social: update logger with a custom fromattter<commit_after>package helper

// TODO use loghelper under koding/worker package instead of this

import (
	"fmt"
	"os"
	"time"

	"github.com/koding/logging"
)

type Formatter struct{}

func (f *Formatter) Format(rec *logging.Record) string {
	return fmt.Sprintf("%s %-8s [%s] %s",
		time.Now().UTC().Format("2006-01-02T15:04:05.999Z"),
		logging.LevelNames[rec.Level],
		rec.LoggerName,
		fmt.Sprintf(rec.Format, rec.Args...),
	)
}

func CreateLogger(name string, debug bool) logging.Logger {
	log := logging.NewLogger(name)
	logHandler := logging.NewWriterHandler(os.Stderr)
	logHandler.Formatter = &Formatter{}
	logHandler.Colorize = true
	log.SetHandler(logHandler)

	if debug {
		log.SetLevel(logging.DEBUG)
		logHandler.SetLevel(logging.DEBUG)
	}

	return log
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"

	"vitess.io/vitess/go/sqltypes"

	"vitess.io/vitess/go/mysql"

	"github.com/stretchr/testify/require"
)

func TestMain(m *testing.M) {
	exitCode := func() int {
		err := makeVttestserverDockerImages()
		if err != nil {
			return 1
		}
		return m.Run()
	}()
	os.Exit(exitCode)
}

func TestUnsharded(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"unsharded_ks"}, []int{1}, 1000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "show databases", `[[VARCHAR("unsharded_ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`)
			_, err = execute(t, conn, "create table unsharded_ks.t1(id int)")
			require.NoError(t, err)
			_, err = execute(t, conn, "insert into unsharded_ks.t1(id) values (10),(20),(30)")
			require.NoError(t, err)
			assertMatches(t, conn, "select * from unsharded_ks.t1", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
		})
	}
}

func TestSharded(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"ks"}, []int{2}, 1000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "show databases", `[[VARCHAR("ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`)
			_, err = execute(t, conn, "create table ks.t1(id int)")
			require.NoError(t, err)
			_, err = execute(t, conn, "alter vschema on ks.t1 add vindex `binary_md5`(id) using `binary_md5`")
			require.NoError(t, err)
			_, err = execute(t, conn, "insert into ks.t1(id) values (10),(20),(30)")
			require.NoError(t, err)
			assertMatches(t, conn, "select id from ks.t1 order by id", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
		})
	}
}

func TestMysqlMaxCons(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"ks"}, []int{2}, 100000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "select @@max_connections", `[[UINT64(100000)]]`)
		})
	}
}

func TestLargeNumberOfKeyspaces(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			var keyspaces []string
			var numShards []int
			for i := 0; i < 100; i++ {
				keyspaces = append(keyspaces, fmt.Sprintf("unsharded_ks%d", i))
				numShards = append(numShards, 1)
			}

			vtest := newVttestserver(image, keyspaces, numShards, 100000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(15 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()

			// assert that all the keyspaces are correctly setup
			for _, keyspace := range keyspaces {
				_, err = execute(t, conn, "create table "+keyspace+".t1(id int)")
				require.NoError(t, err)
				_, err = execute(t, conn, "insert into "+keyspace+".t1(id) values (10),(20),(30)")
				require.NoError(t, err)
				assertMatches(t, conn, "select * from "+keyspace+".t1", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
			}
		})
	}
}

func execute(t *testing.T, conn *mysql.Conn, query string) (*sqltypes.Result, error) {
	t.Helper()
	return conn.ExecuteFetch(query, 1000, true)
}

func checkedExec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
	t.Helper()
	qr, err := conn.ExecuteFetch(query, 1000, true)
	require.NoError(t, err)
	return qr
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
	t.Helper()
	qr := checkedExec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	diff := cmp.Diff(expected, got)
	if diff != "" {
		t.Errorf("Query: %s (-want +got):\n%s", query, diff)
	}
}
<commit_msg>skipped the large keyspace count test<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/google/go-cmp/cmp"

	"vitess.io/vitess/go/sqltypes"

	"vitess.io/vitess/go/mysql"

	"github.com/stretchr/testify/require"
)

func TestMain(m *testing.M) {
	exitCode := func() int {
		err := makeVttestserverDockerImages()
		if err != nil {
			return 1
		}
		return m.Run()
	}()
	os.Exit(exitCode)
}

func TestUnsharded(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"unsharded_ks"}, []int{1}, 1000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "show databases", `[[VARCHAR("unsharded_ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`)
			_, err = execute(t, conn, "create table unsharded_ks.t1(id int)")
			require.NoError(t, err)
			_, err = execute(t, conn, "insert into unsharded_ks.t1(id) values (10),(20),(30)")
			require.NoError(t, err)
			assertMatches(t, conn, "select * from unsharded_ks.t1", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
		})
	}
}

func TestSharded(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"ks"}, []int{2}, 1000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "show databases", `[[VARCHAR("ks")] [VARCHAR("information_schema")] [VARCHAR("mysql")] [VARCHAR("sys")] [VARCHAR("performance_schema")]]`)
			_, err = execute(t, conn, "create table ks.t1(id int)")
			require.NoError(t, err)
			_, err = execute(t, conn, "alter vschema on ks.t1 add vindex `binary_md5`(id) using `binary_md5`")
			require.NoError(t, err)
			_, err = execute(t, conn, "insert into ks.t1(id) values (10),(20),(30)")
			require.NoError(t, err)
			assertMatches(t, conn, "select id from ks.t1 order by id", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
		})
	}
}

func TestMysqlMaxCons(t *testing.T) {
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			vtest := newVttestserver(image, []string{"ks"}, []int{2}, 100000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(10 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()
			assertMatches(t, conn, "select @@max_connections", `[[UINT64(100000)]]`)
		})
	}
}

func TestLargeNumberOfKeyspaces(t *testing.T) {
	t.Skip("Skip test since it takes huge amount of space and github workflow exits with the error `No space left on device`")
	dockerImages := []string{vttestserverMysql57image, vttestserverMysql80image}
	for _, image := range dockerImages {
		t.Run(image, func(t *testing.T) {
			var keyspaces []string
			var numShards []int
			for i := 0; i < 100; i++ {
				keyspaces = append(keyspaces, fmt.Sprintf("unsharded_ks%d", i))
				numShards = append(numShards, 1)
			}

			vtest := newVttestserver(image, keyspaces, numShards, 100000, 33577)
			err := vtest.startDockerImage()
			require.NoError(t, err)
			defer vtest.teardown()

			// wait for the docker to be setup
			time.Sleep(15 * time.Second)

			ctx := context.Background()
			vttestParams := mysql.ConnParams{
				Host: "localhost",
				Port: vtest.port,
			}
			conn, err := mysql.Connect(ctx, &vttestParams)
			require.NoError(t, err)
			defer conn.Close()

			// assert that all the keyspaces are correctly setup
			for _, keyspace := range keyspaces {
				_, err = execute(t, conn, "create table "+keyspace+".t1(id int)")
				require.NoError(t, err)
				_, err = execute(t, conn, "insert into "+keyspace+".t1(id) values (10),(20),(30)")
				require.NoError(t, err)
				assertMatches(t, conn, "select * from "+keyspace+".t1", `[[INT32(10)] [INT32(20)] [INT32(30)]]`)
			}
		})
	}
}

func execute(t *testing.T, conn *mysql.Conn, query string) (*sqltypes.Result, error) {
	t.Helper()
	return conn.ExecuteFetch(query, 1000, true)
}

func checkedExec(t *testing.T, conn *mysql.Conn, query string) *sqltypes.Result {
	t.Helper()
	qr, err := conn.ExecuteFetch(query, 1000, true)
	require.NoError(t, err)
	return qr
}

func assertMatches(t *testing.T, conn *mysql.Conn, query, expected string) {
	t.Helper()
	qr := checkedExec(t, conn, query)
	got := fmt.Sprintf("%v", qr.Rows)
	diff := cmp.Diff(expected, got)
	if diff != "" {
		t.Errorf("Query: %s (-want +got):\n%s", query, diff)
	}
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package renderer

import (
	"reflect"
	"strings"
)

type Log interface {
	ExistsInGoodWords(word string) bool
	ExistsInWarnWords(word string) bool
	ExistsInGoodLines(word string) bool
	ExistsInBadLines(word string) bool
}

type HTTP struct {
	GoodWords []string `json:"good_words"`
	GoodLines []string `json:"good_lines"`
	WarnWords []string `json:"warn_words"`
	BadLines  []string `json:"bad_lines"`
}

type FTP struct {
	GoodWords []string `json:"good_words"`
	GoodLines []string `json:"good_lines"`
	WarnWords []string `json:"warn_words"`
	BadLines  []string `json:"bad_lines"`
}

func (h *HTTP) ExistsInGoodWords(word string) bool {
	for _, i := range h.GoodWords {
		if strings.Contains(word, i) {
			return true
		}
	}
	return false
}

func (h *HTTP) ExistsInWarnWords(word string) bool {
	for _, i := range h.WarnWords {
		if strings.Contains(word, i) {
			return true
		}
	}
	return false
}

func (h *HTTP) ExistsInGoodWords() bool {
	for _, i := range h.GoodWords {
		if strings.Contains(word, i) {
			return true
		}
	}
	return false
}

func (h *HTTP) ExistsInBadLines() bool {
	for _, i := range h.BadLines {
		if strings.Contains(word, i) {
			return true
		}
	}
	return false
}

func FieldCount(i interface{}) int {
	return reflect.TypeOf(i).NumField()
}
<commit_msg>comment<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package renderer

import (
	"reflect"
	"strings"
)

type Log interface {
	ExistsInGoodWords(word string) bool
	ExistsInWarnWords(word string) bool
	ExistsInGoodLines(word string) bool
	ExistsInBadLines(word string) bool
}

type HTTP struct {
	GoodWords []string `json:"good_words"`
	GoodLines []string `json:"good_lines"`
	WarnWords []string `json:"warn_words"`
	BadLines  []string `json:"bad_lines"`
}

type FTP struct {
	GoodWords []string `json:"good_words"`
	GoodLines []string `json:"good_lines"`
	WarnWords []string `json:"warn_words"`
	BadLines  []string `json:"bad_lines"`
}

func (h *HTTP) ExistsInGoodWords(word string) bool {
	for _, i := range h.GoodWords {
		if strings.Contains(word, i) {
			return true
		}
	}
	return false
}

func (h *HTTP) ExistsInWarnWords(word string) bool {
	for _, i := range h.WarnWords {
		if strings.Contains(word, i) {
			return true
		}
	}
	return false
}

func (h *HTTP) ExistsInGoodWords() bool {
	for _, i := range h.GoodWords {
		if strings.Contains(word, i) {
			return true
		}
	}
	return false
}

func (h *HTTP) ExistsInBadLines() bool {
	for _, i := range h.BadLines {
		if strings.Contains(word, i) {
			return true
		}
	}
	return false
}

// FieldCount will return the number of fields on a given struct
func FieldCount(i interface{}) int {
	return reflect.TypeOf(i).NumField()
}
<|endoftext|>


<commit_before>package modules

import (
	"github.com/Seklfreak/Robyul2/modules/plugins"
	"github.com/Seklfreak/Robyul2/modules/plugins/biasgame"
	"github.com/Seklfreak/Robyul2/modules/plugins/eventlog"
	"github.com/Seklfreak/Robyul2/modules/plugins/idols"
	"github.com/Seklfreak/Robyul2/modules/plugins/levels"
	"github.com/Seklfreak/Robyul2/modules/plugins/mod"
	"github.com/Seklfreak/Robyul2/modules/plugins/notifications"
	"github.com/Seklfreak/Robyul2/modules/plugins/nugugame"
	"github.com/Seklfreak/Robyul2/modules/plugins/youtube"
)

var (
	pluginCache         map[string]*Plugin
	extendedPluginCache map[string]*ExtendedPlugin

	PluginList = []Plugin{
		&notifications.Handler{},
		&plugins.About{},
		&plugins.Stats{},
		&plugins.Uptime{},
		&plugins.Translator{},
		&plugins.UrbanDict{},
		&plugins.Weather{},
		&plugins.VLive{},
		// &instagram.Handler{},
		// &plugins.Facebook{},
		&plugins.WolframAlpha{},
		&plugins.LastFm{},
		&plugins.Twitch{},
		&plugins.Charts{},
		&plugins.Choice{},
		&plugins.Osu{},
		&plugins.Reminders{},
		&plugins.Ratelimit{},
		&plugins.Gfycat{},
		&plugins.RandomPictures{},
		&youtube.Handler{},
		&plugins.Spoiler{},
		&plugins.RandomCat{},
		&plugins.RPS{},
		&plugins.Nuke{},
		&plugins.Dig{},
		&plugins.Streamable{},
		&plugins.Lyrics{},
		&plugins.Friend{},
		&plugins.Names{},
		&plugins.Reddit{},
		&plugins.Color{},
		&plugins.Dog{},
		&plugins.Debug{},
		&plugins.Donators{},
		&plugins.Ping{},
		//&google.Handler{},
		&plugins.BotStatus{},
		&plugins.VanityInvite{},
		&plugins.DiscordMoney{},
		&plugins.Whois{},
		&plugins.Isup{},
		&plugins.ModulePermissions{},
		&plugins.M8ball{},
		&plugins.Feedback{},
		&plugins.DM{},
		&plugins.EmbedPost{},
		&plugins.Useruploads{},
		&plugins.Move{},
		&plugins.Crypto{},
		&plugins.Imgur{},
		&plugins.Steam{},
		&plugins.Config{},
		&plugins.Storage{},
		&plugins.Mirror{},
	}

	PluginExtendedList = []ExtendedPlugin{
		&plugins.Bias{},
		&plugins.GuildAnnouncements{},
		&levels.Levels{},
		&plugins.Gallery{},
		&plugins.CustomCommands{},
		&plugins.ReactionPolls{},
		&mod.Mod{},
		&plugins.AutoRoles{},
		&plugins.Starboard{},
		&plugins.Autoleaver{},
		&plugins.Persistency{},
		&plugins.Twitter{},
		&eventlog.Handler{},
		&plugins.Perspective{},
		&biasgame.Module{},
		&nugugame.Module{},
		&idols.Module{},
	}
)
<commit_msg>[perspective] disables plugin<commit_after>package modules

import (
	"github.com/Seklfreak/Robyul2/modules/plugins"
	"github.com/Seklfreak/Robyul2/modules/plugins/biasgame"
	"github.com/Seklfreak/Robyul2/modules/plugins/eventlog"
	"github.com/Seklfreak/Robyul2/modules/plugins/idols"
	"github.com/Seklfreak/Robyul2/modules/plugins/levels"
	"github.com/Seklfreak/Robyul2/modules/plugins/mod"
	"github.com/Seklfreak/Robyul2/modules/plugins/notifications"
	"github.com/Seklfreak/Robyul2/modules/plugins/nugugame"
	"github.com/Seklfreak/Robyul2/modules/plugins/youtube"
)

var (
	pluginCache         map[string]*Plugin
	extendedPluginCache map[string]*ExtendedPlugin

	PluginList = []Plugin{
		&notifications.Handler{},
		&plugins.About{},
		&plugins.Stats{},
		&plugins.Uptime{},
		&plugins.Translator{},
		&plugins.UrbanDict{},
		&plugins.Weather{},
		&plugins.VLive{},
		&plugins.WolframAlpha{},
		&plugins.LastFm{},
		&plugins.Twitch{},
		&plugins.Charts{},
		&plugins.Choice{},
		&plugins.Osu{},
		&plugins.Reminders{},
		&plugins.Ratelimit{},
		&plugins.Gfycat{},
		&plugins.RandomPictures{},
		&youtube.Handler{},
		&plugins.Spoiler{},
		&plugins.RandomCat{},
		&plugins.RPS{},
		&plugins.Nuke{},
		&plugins.Dig{},
		&plugins.Streamable{},
		&plugins.Lyrics{},
		&plugins.Friend{},
		&plugins.Names{},
		&plugins.Reddit{},
		&plugins.Color{},
		&plugins.Dog{},
		&plugins.Debug{},
		&plugins.Donators{},
		&plugins.Ping{},
		&plugins.BotStatus{},
		&plugins.VanityInvite{},
		&plugins.DiscordMoney{},
		&plugins.Whois{},
		&plugins.Isup{},
		&plugins.ModulePermissions{},
		&plugins.M8ball{},
		&plugins.Feedback{},
		&plugins.DM{},
		&plugins.EmbedPost{},
		&plugins.Useruploads{},
		&plugins.Move{},
		&plugins.Crypto{},
		&plugins.Imgur{},
		&plugins.Steam{},
		&plugins.Config{},
		&plugins.Storage{},
		&plugins.Mirror{},

		// &instagram.Handler{},
		// &plugins.Facebook{},
		// &google.Handler{},
	}

	PluginExtendedList = []ExtendedPlugin{
		&plugins.Bias{},
		&plugins.GuildAnnouncements{},
		&levels.Levels{},
		&plugins.Gallery{},
		&plugins.CustomCommands{},
		&plugins.ReactionPolls{},
		&mod.Mod{},
		&plugins.AutoRoles{},
		&plugins.Starboard{},
		&plugins.Autoleaver{},
		&plugins.Persistency{},
		&plugins.Twitter{},
		&eventlog.Handler{},
		&biasgame.Module{},
		&nugugame.Module{},
		&idols.Module{},

		// &plugins.Perspective{},
	}
)
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package main provides a tool that scans kubernetes e2e test source code
// looking for conformance test declarations, which it emits on stdout.  It
// also looks for legacy, manually added "[Conformance]" tags and reports an
// error if it finds any.
//
// This approach is not air tight, but it will serve our purpose as a
// pre-submit check.
package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

var (
	baseURL                                           = flag.String("url", "https://github.com/kubernetes/kubernetes/tree/master/", "location of the current source")
	confDoc                                           = flag.Bool("conformance", false, "write a conformance document")
	totalConfTests, totalLegacyTests, missingComments int
)

const regexDescribe = "Describe|KubeDescribe|SIGDescribe"
const regexContext = "Context"

type visitor struct {
	FileSet      *token.FileSet
	lastDescribe describe
	cMap         ast.CommentMap
	//list of all the conformance tests in the path
	tests []conformanceData
}

//describe contains text associated with ginkgo describe container
type describe struct {
	text        string
	lastContext context
}

//context contain the text associated with the Context clause
type context struct {
	text string
}

type conformanceData struct {
	// A URL to the line of code in the kube src repo for the test
	URL string
	// Extracted from the "Testname:" comment before the test
	TestName string
	// Extracted from the "Description:" comment before the test
	Description string
}

func (v *visitor) convertToConformanceData(at *ast.BasicLit) {
	cd := conformanceData{}

	comment := v.comment(at)
	pos := v.FileSet.Position(at.Pos())
	cd.URL = fmt.Sprintf("%s%s#L%d", *baseURL, pos.Filename, pos.Line)

	lines := strings.Split(comment, "\n")
	cd.Description = ""
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "Testname:") {
			line = strings.TrimSpace(line[9:])
			cd.TestName = line
			continue
		}
		if strings.HasPrefix(line, "Description:") {
			line = strings.TrimSpace(line[12:])
		}
		cd.Description += line + "\n"
	}

	if cd.TestName == "" {
		testName := v.getDescription(at.Value)
		i := strings.Index(testName, "[Conformance]")
		if i > 0 {
			cd.TestName = strings.TrimSpace(testName[:i])
		} else {
			cd.TestName = testName
		}
	}

	v.tests = append(v.tests, cd)
}

func newVisitor() *visitor {
	return &visitor{
		FileSet: token.NewFileSet(),
	}
}

func (v *visitor) isConformanceCall(call *ast.CallExpr) bool {
	switch fun := call.Fun.(type) {
	case *ast.SelectorExpr:
		if fun.Sel != nil {
			return fun.Sel.Name == "ConformanceIt"
		}
	}
	return false
}

func (v *visitor) isLegacyItCall(call *ast.CallExpr) bool {
	switch fun := call.Fun.(type) {
	case *ast.Ident:
		if fun.Name != "It" {
			return false
		}
		if len(call.Args) < 1 {
			v.failf(call, "Not enough arguments to It()")
		}
	default:
		return false
	}

	switch arg := call.Args[0].(type) {
	case *ast.BasicLit:
		if arg.Kind != token.STRING {
			v.failf(arg, "Unexpected non-string argument to It()")
		}
		if strings.Contains(arg.Value, "[Conformance]") {
			return true
		}
	default:
		// non-literal argument to It()... we just ignore these even though they could be a way to "sneak in" a conformance test
	}

	return false
}

func (v *visitor) failf(expr ast.Expr, format string, a ...interface{}) {
	msg := fmt.Sprintf(format, a...)
	fmt.Fprintf(os.Stderr, "ERROR at %v: %s\n", v.FileSet.Position(expr.Pos()), msg)
}

func (v *visitor) comment(x *ast.BasicLit) string {
	for _, comm := range v.cMap.Comments() {
		testOffset := int(x.Pos()-comm.End()) - len("framework.ConformanceIt(\"")
		if 0 < testOffset && testOffset < 3 {
			return comm.Text()
		}
	}
	return ""
}

func (v *visitor) emit(arg ast.Expr) {
	switch at := arg.(type) {
	case *ast.BasicLit:
		if at.Kind != token.STRING {
			v.failf(at, "framework.ConformanceIt() called with non-string argument")
			return
		}

		at.Value = normalizeTestName(at.Value)
		if *confDoc {
			v.convertToConformanceData(at)
		} else {
			fmt.Printf("%s: %q\n", v.FileSet.Position(at.Pos()).Filename, at.Value)
		}
	default:
		v.failf(at, "framework.ConformanceIt() called with non-literal argument")
		fmt.Fprintf(os.Stderr, "ERROR: non-literal argument %v at %v\n", arg, v.FileSet.Position(arg.Pos()))
	}
}

func (v *visitor) getDescription(value string) string {
	if len(v.lastDescribe.lastContext.text) > 0 {
		return strings.Trim(v.lastDescribe.text, "\"") +
			" " + strings.Trim(v.lastDescribe.lastContext.text, "\"") +
			" " + strings.Trim(value, "\"")
	}
	return strings.Trim(v.lastDescribe.text, "\"") +
		" " + strings.Trim(value, "\"")
}

var (
	regexTag = regexp.MustCompile(`(\[[a-zA-Z0-9:-]+\])`)
)

// normalizeTestName removes tags (e.g., [Feature:Foo]), double quotes and trim
// the spaces to normalize the test name.
func normalizeTestName(s string) string {
	r := regexTag.ReplaceAllString(s, "")
	r = strings.Trim(r, "\"")
	return strings.TrimSpace(r)
}

// funcName converts a selectorExpr with two idents into a string,
// x.y -> "x.y"
func funcName(n ast.Expr) string {
	if sel, ok := n.(*ast.SelectorExpr); ok {
		if x, ok := sel.X.(*ast.Ident); ok {
			return x.String() + "." + sel.Sel.String()
		}
	}
	return ""
}

// isSprintf returns whether the given node is a call to fmt.Sprintf
func isSprintf(n ast.Expr) bool {
	call, ok := n.(*ast.CallExpr)
	return ok && funcName(call.Fun) == "fmt.Sprintf" && len(call.Args) != 0
}

// firstArg attempts to statically determine the value of the first
// argument. It only handles strings, and converts any unknown values
// (fmt.Sprintf interpolations) into *.
func (v *visitor) firstArg(n *ast.CallExpr) string {
	if len(n.Args) == 0 {
		return ""
	}
	var lit *ast.BasicLit
	if isSprintf(n.Args[0]) {
		return v.firstArg(n.Args[0].(*ast.CallExpr))
	}
	lit, ok := n.Args[0].(*ast.BasicLit)
	if ok && lit.Kind == token.STRING {
		val, err := strconv.Unquote(lit.Value)
		if err != nil {
			panic(err)
		}
		if strings.Contains(val, "%") {
			val = strings.Replace(val, "%d", "*", -1)
			val = strings.Replace(val, "%v", "*", -1)
			val = strings.Replace(val, "%s", "*", -1)
		}
		return val
	}
	if ident, ok := n.Args[0].(*ast.Ident); ok {
		return ident.String()
	}
	return "*"
}

// matchFuncName returns the first argument of a function if it's
// a Ginkgo-relevant function (Describe/KubeDescribe/Context),
// and the empty string otherwise.
func (v *visitor) matchFuncName(n *ast.CallExpr, pattern string) string {
	switch x := n.Fun.(type) {
	case *ast.SelectorExpr:
		if match, err := regexp.MatchString(pattern, x.Sel.Name); err == nil && match {
			return v.firstArg(n)
		}
	case *ast.Ident:
		if match, err := regexp.MatchString(pattern, x.Name); err == nil && match {
			return v.firstArg(n)
		}
	default:
		return ""
	}
	return ""
}

// Visit visits each node looking for either calls to framework.ConformanceIt,
// which it will emit in its list of conformance tests, or legacy calls to
// It() with a manually embedded [Conformance] tag, which it will complain
// about.
func (v *visitor) Visit(node ast.Node) (w ast.Visitor) {
	switch t := node.(type) {
	case *ast.CallExpr:
		if name := v.matchFuncName(t, regexDescribe); name != "" && len(t.Args) >= 2 {
			v.lastDescribe = describe{text: name}
		} else if name := v.matchFuncName(t, regexContext); name != "" && len(t.Args) >= 2 {
			v.lastDescribe.lastContext = context{text: name}
		} else if v.isConformanceCall(t) {
			totalConfTests++
			v.emit(t.Args[0])
			return nil
		} else if v.isLegacyItCall(t) {
			totalLegacyTests++
			v.failf(t, "Using It() with manual [Conformance] tag is no longer allowed.  Use framework.ConformanceIt() instead.")
			return nil
		}
	}
	return v
}

func scanfile(path string, src interface{}) []conformanceData {
	v := newVisitor()
	file, err := parser.ParseFile(v.FileSet, path, src, parser.ParseComments)
	if err != nil {
		panic(err)
	}

	v.cMap = ast.NewCommentMap(v.FileSet, file, file.Comments)

	ast.Walk(v, file)
	return v.tests
}

func main() {
	flag.Parse()

	if len(flag.Args()) < 1 {
		fmt.Fprintf(os.Stderr, "USAGE: %s <DIR or FILE> [...]\n", os.Args[0])
		os.Exit(64)
	}

	if *confDoc {
		// Note: this assumes that you're running from the root of the kube src repo
		header, err := ioutil.ReadFile("test/conformance/cf_header.md")
		if err == nil {
			fmt.Printf("%s\n\n", header)
		}
	}

	totalConfTests = 0
	totalLegacyTests = 0
	missingComments = 0
	for _, arg := range flag.Args() {
		filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if strings.HasSuffix(path, ".go") {
				tests := scanfile(path, nil)
				for _, cd := range tests {
					fmt.Printf("## [%s](%s)\n\n", cd.TestName, cd.URL)
					fmt.Printf("%s\n\n", cd.Description)
					if len(cd.Description) < 10 {
						missingComments++
					}
				}
			}
			return nil
		})
	}
	if *confDoc {
		fmt.Println("\n## **Summary**")
		fmt.Printf("\nTotal Conformance Tests: %d, total legacy tests that need conversion: %d, while total tests that need comment sections: %d\n\n", totalConfTests, totalLegacyTests, missingComments)
	}
}
<commit_msg>Add Release information to each of the conformance tests.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package main provides a tool that scans kubernetes e2e test source code
// looking for conformance test declarations, which it emits on stdout.  It
// also looks for legacy, manually added "[Conformance]" tags and reports an
// error if it finds any.
//
// This approach is not air tight, but it will serve our purpose as a
// pre-submit check.
package main

import (
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
)

var (
	baseURL                                           = flag.String("url", "https://github.com/kubernetes/kubernetes/tree/master/", "location of the current source")
	confDoc                                           = flag.Bool("conformance", false, "write a conformance document")
	totalConfTests, totalLegacyTests, missingComments int
)

const regexDescribe = "Describe|KubeDescribe|SIGDescribe"
const regexContext = "Context"

type visitor struct {
	FileSet      *token.FileSet
	lastDescribe describe
	cMap         ast.CommentMap
	//list of all the conformance tests in the path
	tests []conformanceData
}

//describe contains text associated with ginkgo describe container
type describe struct {
	text        string
	lastContext context
}

//context contain the text associated with the Context clause
type context struct {
	text string
}

type conformanceData struct {
	// A URL to the line of code in the kube src repo for the test
	URL string
	// Extracted from the "Testname:" comment before the test
	TestName string
	// Extracted from the "Description:" comment before the test
	Description string
	// Version when this test is added or modified ex: v1.12, v1.13
	Release string
}

func (v *visitor) convertToConformanceData(at *ast.BasicLit) {
	cd := conformanceData{}

	comment := v.comment(at)
	pos := v.FileSet.Position(at.Pos())
	cd.URL = fmt.Sprintf("%s%s#L%d", *baseURL, pos.Filename, pos.Line)

	lines := strings.Split(comment, "\n")
	cd.Description = ""
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if sline := regexp.MustCompile("^Testname\\s*:\\s*").Split(line, -1); len(sline) == 2 {
			cd.TestName = sline[1]
			continue
		}
		if sline := regexp.MustCompile("^Release\\s*:\\s*").Split(line, -1); len(sline) == 2 {
			cd.Release = sline[1]
			continue
		}
		if sline := regexp.MustCompile("^Description\\s*:\\s*").Split(line, -1); len(sline) == 2 {
			line = sline[1]
		}
		cd.Description += line + "\n"
	}

	if cd.TestName == "" {
		testName := v.getDescription(at.Value)
		i := strings.Index(testName, "[Conformance]")
		if i > 0 {
			cd.TestName = strings.TrimSpace(testName[:i])
		} else {
			cd.TestName = testName
		}
	}

	v.tests = append(v.tests, cd)
}

func newVisitor() *visitor {
	return &visitor{
		FileSet: token.NewFileSet(),
	}
}

func (v *visitor) isConformanceCall(call *ast.CallExpr) bool {
	switch fun := call.Fun.(type) {
	case *ast.SelectorExpr:
		if fun.Sel != nil {
			return fun.Sel.Name == "ConformanceIt"
		}
	}
	return false
}

func (v *visitor) isLegacyItCall(call *ast.CallExpr) bool {
	switch fun := call.Fun.(type) {
	case *ast.Ident:
		if fun.Name != "It" {
			return false
		}
		if len(call.Args) < 1 {
			v.failf(call, "Not enough arguments to It()")
		}
	default:
		return false
	}

	switch arg := call.Args[0].(type) {
	case *ast.BasicLit:
		if arg.Kind != token.STRING {
			v.failf(arg, "Unexpected non-string argument to It()")
		}
		if strings.Contains(arg.Value, "[Conformance]") {
			return true
		}
	default:
		// non-literal argument to It()... we just ignore these even though they could be a way to "sneak in" a conformance test
	}

	return false
}

func (v *visitor) failf(expr ast.Expr, format string, a ...interface{}) {
	msg := fmt.Sprintf(format, a...)
	fmt.Fprintf(os.Stderr, "ERROR at %v: %s\n", v.FileSet.Position(expr.Pos()), msg)
}

func (v *visitor) comment(x *ast.BasicLit) string {
	for _, comm := range v.cMap.Comments() {
		testOffset := int(x.Pos()-comm.End()) - len("framework.ConformanceIt(\"")
		if 0 < testOffset && testOffset < 3 {
			return comm.Text()
		}
	}
	return ""
}

func (v *visitor) emit(arg ast.Expr) {
	switch at := arg.(type) {
	case *ast.BasicLit:
		if at.Kind != token.STRING {
			v.failf(at, "framework.ConformanceIt() called with non-string argument")
			return
		}

		at.Value = normalizeTestName(at.Value)
		if *confDoc {
			v.convertToConformanceData(at)
		} else {
			fmt.Printf("%s: %q\n", v.FileSet.Position(at.Pos()).Filename, at.Value)
		}
	default:
		v.failf(at, "framework.ConformanceIt() called with non-literal argument")
		fmt.Fprintf(os.Stderr, "ERROR: non-literal argument %v at %v\n", arg, v.FileSet.Position(arg.Pos()))
	}
}

func (v *visitor) getDescription(value string) string {
	if len(v.lastDescribe.lastContext.text) > 0 {
		return strings.Trim(v.lastDescribe.text, "\"") +
			" " + strings.Trim(v.lastDescribe.lastContext.text, "\"") +
			" " + strings.Trim(value, "\"")
	}
	return strings.Trim(v.lastDescribe.text, "\"") +
		" " + strings.Trim(value, "\"")
}

var (
	regexTag = regexp.MustCompile(`(\[[a-zA-Z0-9:-]+\])`)
)

// normalizeTestName removes tags (e.g., [Feature:Foo]), double quotes and trim
// the spaces to normalize the test name.
func normalizeTestName(s string) string {
	r := regexTag.ReplaceAllString(s, "")
	r = strings.Trim(r, "\"")
	return strings.TrimSpace(r)
}

// funcName converts a selectorExpr with two idents into a string,
// x.y -> "x.y"
func funcName(n ast.Expr) string {
	if sel, ok := n.(*ast.SelectorExpr); ok {
		if x, ok := sel.X.(*ast.Ident); ok {
			return x.String() + "." + sel.Sel.String()
		}
	}
	return ""
}

// isSprintf returns whether the given node is a call to fmt.Sprintf
func isSprintf(n ast.Expr) bool {
	call, ok := n.(*ast.CallExpr)
	return ok && funcName(call.Fun) == "fmt.Sprintf" && len(call.Args) != 0
}

// firstArg attempts to statically determine the value of the first
// argument. It only handles strings, and converts any unknown values
// (fmt.Sprintf interpolations) into *.
func (v *visitor) firstArg(n *ast.CallExpr) string {
	if len(n.Args) == 0 {
		return ""
	}
	var lit *ast.BasicLit
	if isSprintf(n.Args[0]) {
		return v.firstArg(n.Args[0].(*ast.CallExpr))
	}
	lit, ok := n.Args[0].(*ast.BasicLit)
	if ok && lit.Kind == token.STRING {
		val, err := strconv.Unquote(lit.Value)
		if err != nil {
			panic(err)
		}
		if strings.Contains(val, "%") {
			val = strings.Replace(val, "%d", "*", -1)
			val = strings.Replace(val, "%v", "*", -1)
			val = strings.Replace(val, "%s", "*", -1)
		}
		return val
	}
	if ident, ok := n.Args[0].(*ast.Ident); ok {
		return ident.String()
	}
	return "*"
}

// matchFuncName returns the first argument of a function if it's
// a Ginkgo-relevant function (Describe/KubeDescribe/Context),
// and the empty string otherwise.
func (v *visitor) matchFuncName(n *ast.CallExpr, pattern string) string {
	switch x := n.Fun.(type) {
	case *ast.SelectorExpr:
		if match, err := regexp.MatchString(pattern, x.Sel.Name); err == nil && match {
			return v.firstArg(n)
		}
	case *ast.Ident:
		if match, err := regexp.MatchString(pattern, x.Name); err == nil && match {
			return v.firstArg(n)
		}
	default:
		return ""
	}
	return ""
}

// Visit visits each node looking for either calls to framework.ConformanceIt,
// which it will emit in its list of conformance tests, or legacy calls to
// It() with a manually embedded [Conformance] tag, which it will complain
// about.
func (v *visitor) Visit(node ast.Node) (w ast.Visitor) {
	switch t := node.(type) {
	case *ast.CallExpr:
		if name := v.matchFuncName(t, regexDescribe); name != "" && len(t.Args) >= 2 {
			v.lastDescribe = describe{text: name}
		} else if name := v.matchFuncName(t, regexContext); name != "" && len(t.Args) >= 2 {
			v.lastDescribe.lastContext = context{text: name}
		} else if v.isConformanceCall(t) {
			totalConfTests++
			v.emit(t.Args[0])
			return nil
		} else if v.isLegacyItCall(t) {
			totalLegacyTests++
			v.failf(t, "Using It() with manual [Conformance] tag is no longer allowed.  Use framework.ConformanceIt() instead.")
			return nil
		}
	}
	return v
}

func scanfile(path string, src interface{}) []conformanceData {
	v := newVisitor()
	file, err := parser.ParseFile(v.FileSet, path, src, parser.ParseComments)
	if err != nil {
		panic(err)
	}

	v.cMap = ast.NewCommentMap(v.FileSet, file, file.Comments)

	ast.Walk(v, file)
	return v.tests
}

func main() {
	flag.Parse()

	if len(flag.Args()) < 1 {
		fmt.Fprintf(os.Stderr, "USAGE: %s <DIR or FILE> [...]\n", os.Args[0])
		os.Exit(64)
	}

	if *confDoc {
		// Note: this assumes that you're running from the root of the kube src repo
		header, err := ioutil.ReadFile("test/conformance/cf_header.md")
		if err == nil {
			fmt.Printf("%s\n\n", header)
		}
	}

	totalConfTests = 0
	totalLegacyTests = 0
	missingComments = 0
	for _, arg := range flag.Args() {
		filepath.Walk(arg, func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if strings.HasSuffix(path, ".go") {
				tests := scanfile(path, nil)
				for _, cd := range tests {
					fmt.Printf("## [%s](%s)\n\n", cd.TestName, cd.URL)
					fmt.Printf("### Release %s\n", cd.Release)
					fmt.Printf("%s\n\n", cd.Description)
					if len(cd.Description) < 10 {
						missingComments++
					}
				}
			}
			return nil
		})
	}
	if *confDoc {
		fmt.Println("\n## **Summary**")
		fmt.Printf("\nTotal Conformance Tests: %d, total legacy tests that need conversion: %d, while total tests that need comment sections: %d\n\n", totalConfTests, totalLegacyTests, missingComments)
	}
}
<|endoftext|>


<commit_before>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package url

import (
	"errors"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"html"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

var extractError = errors.New("couldn't extract title")

type Module struct {
	re      *regexp.Regexp
	Regex   string   `json:"regex"`
	Exclude []string `json:"exclude"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "url"
}

func (m *Module) Help() string {
	return "Displays HTML titles for HTTP links."
}

func (m *Module) Defaults() {
	m.Regex = `(http|https)\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(:[a-zA-Z0-9]*)?/?([a-zA-Z0-9\-\._\?\,\'/\\\+&%\$#\=~])*`
}

func (m *Module) Load(client *irc.Client) error {
	re, err := regexp.Compile(m.Regex)
	if err != nil {
		return err
	}

	m.re = re
	client.CmdHook("privmsg", m.urlCmd)

	return nil
}

func (m *Module) urlCmd(client *irc.Client, msg irc.Message) error {
	link := m.re.FindString(msg.Data)
	if len(link) <= 0 {
		return nil
	}

	purl, err := url.Parse(link)
	if err != nil || m.isExcluded(purl.Host) {
		return nil
	}

	resp, err := http.Get(purl.String())
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	info := m.infoString(resp)
	if len(info) <= 0 {
		return nil
	}

	return client.Write("NOTICE %s :%s", msg.Receiver, info)
}

func (m *Module) infoString(resp *http.Response) string {
	var mtype string
	var infos []string

	ctype := resp.Header.Get("Content-Type")
	if len(ctype) > 0 {
		m, _, err := mime.ParseMediaType(ctype)
		if err == nil {
			mtype = m
			infos = append(infos, fmt.Sprintf("Type: %s", mtype))
		}
	}

	csize := resp.Header.Get("Content-Length")
	if len(csize) > 0 {
		infos = append(infos, fmt.Sprintf("Size: %s bytes", csize))
	}

	if mtype == "text/html" {
		title, err := m.extractTitle(resp.Body)
		if err == nil {
			infos = append(infos, fmt.Sprintf("Title: %s", title))
		}
	}

	info := strings.Join(infos, ". ")
	if len(info) > 0 {
		info = fmt.Sprintf("%s -- %s", strings.ToUpper(m.Name()), info)
	}

	return info
}

func (m *Module) extractTitle(body io.ReadCloser) (title string, err error) {
	data, err := ioutil.ReadAll(body)
	if err != nil {
		return
	}

	regex := regexp.MustCompile("(?is)<title>(.+)</title>")
	match := regex.Find(data)
	if len(match) <= 0 {
		return "", extractError
	}

	title = string(match)
	title = title[len("<title>"):strings.Index(title, "</title>")]

	title = m.sanitize(html.UnescapeString(title))
	if len(title) <= 0 {
		return "", extractError
	}

	return
}

func (m *Module) sanitize(title string) string {
	normalized := strings.Replace(title, "\n", " ", -1)
	for strings.Contains(normalized, "  ") {
		normalized = strings.Replace(normalized, "  ", " ", -1)
	}

	return strings.TrimSpace(normalized)
}

func (m *Module) isExcluded(host string) bool {
	for _, h := range m.Exclude {
		if host == h {
			return true
		}
	}

	return false
}
<commit_msg>url: Seperate infos with a pipe<commit_after>// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful, but
// WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public
// License along with this program. If not, see <http://www.gnu.org/licenses/>.

package url

import (
	"errors"
	"fmt"
	"github.com/nmeum/marvin/irc"
	"github.com/nmeum/marvin/modules"
	"html"
	"io"
	"io/ioutil"
	"mime"
	"net/http"
	"net/url"
	"regexp"
	"strings"
)

var extractError = errors.New("couldn't extract title")

type Module struct {
	re      *regexp.Regexp
	Regex   string   `json:"regex"`
	Exclude []string `json:"exclude"`
}

func Init(moduleSet *modules.ModuleSet) {
	moduleSet.Register(new(Module))
}

func (m *Module) Name() string {
	return "url"
}

func (m *Module) Help() string {
	return "Displays HTML titles for HTTP links."
}

func (m *Module) Defaults() {
	m.Regex = `(http|https)\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,3}(:[a-zA-Z0-9]*)?/?([a-zA-Z0-9\-\._\?\,\'/\\\+&%\$#\=~])*`
}

func (m *Module) Load(client *irc.Client) error {
	re, err := regexp.Compile(m.Regex)
	if err != nil {
		return err
	}

	m.re = re
	client.CmdHook("privmsg", m.urlCmd)

	return nil
}

func (m *Module) urlCmd(client *irc.Client, msg irc.Message) error {
	link := m.re.FindString(msg.Data)
	if len(link) <= 0 {
		return nil
	}

	purl, err := url.Parse(link)
	if err != nil || m.isExcluded(purl.Host) {
		return nil
	}

	resp, err := http.Get(purl.String())
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	info := m.infoString(resp)
	if len(info) <= 0 {
		return nil
	}

	return client.Write("NOTICE %s :%s", msg.Receiver, info)
}

func (m *Module) infoString(resp *http.Response) string {
	var mtype string
	var infos []string

	ctype := resp.Header.Get("Content-Type")
	if len(ctype) > 0 {
		m, _, err := mime.ParseMediaType(ctype)
		if err == nil {
			mtype = m
			infos = append(infos, fmt.Sprintf("Type: %s", mtype))
		}
	}

	csize := resp.Header.Get("Content-Length")
	if len(csize) > 0 {
		infos = append(infos, fmt.Sprintf("Size: %s bytes", csize))
	}

	if mtype == "text/html" {
		title, err := m.extractTitle(resp.Body)
		if err == nil {
			infos = append(infos, fmt.Sprintf("Title: %s", title))
		}
	}

	info := strings.Join(infos, " | ")
	if len(info) > 0 {
		info = fmt.Sprintf("%s -- %s", strings.ToUpper(m.Name()), info)
	}

	return info
}

func (m *Module) extractTitle(body io.ReadCloser) (title string, err error) {
	data, err := ioutil.ReadAll(body)
	if err != nil {
		return
	}

	regex := regexp.MustCompile("(?is)<title>(.+)</title>")
	match := regex.Find(data)
	if len(match) <= 0 {
		return "", extractError
	}

	title = string(match)
	title = title[len("<title>"):strings.Index(title, "</title>")]

	title = m.sanitize(html.UnescapeString(title))
	if len(title) <= 0 {
		return "", extractError
	}

	return
}

func (m *Module) sanitize(title string) string {
	normalized := strings.Replace(title, "\n", " ", -1)
	for strings.Contains(normalized, "  ") {
		normalized = strings.Replace(normalized, "  ", " ", -1)
	}

	return strings.TrimSpace(normalized)
}

func (m *Module) isExcluded(host string) bool {
	for _, h := range m.Exclude {
		if host == h {
			return true
		}
	}

	return false
}
<|endoftext|>