<commit_before>package irc

import (
	"fmt"
	"log"
	"net"
	"time"
)

type Client struct {
	atime       time.Time
	away        bool
	awayMessage string
	channels    ChannelSet
	ctime       time.Time
	destroyed   bool
	hostname    string
	idleTimer   *time.Timer
	invisible   bool
	loginTimer  *time.Timer
	nick        string
	operator    bool
	quitTimer   *time.Timer
	realname    string
	registered  bool
	replies     chan Reply
	server      *Server
	socket      *Socket
	authorized  bool
	username    string
}

func NewClient(server *Server, conn net.Conn) *Client {
	now := time.Now()
	client := &Client{
		atime:    now,
		channels: make(ChannelSet),
		ctime:    now,
		hostname: AddrLookupHostname(conn.RemoteAddr()),
		replies:  make(chan Reply),
		server:   server,
		socket:   NewSocket(conn),
	}
	client.loginTimer = time.AfterFunc(LOGIN_TIMEOUT, client.Destroy)

	go client.readCommands()
	go client.writeReplies()

	return client
}

func (client *Client) Touch() {
	if client.destroyed {
		return
	}

	client.atime = time.Now()

	if client.quitTimer != nil {
		client.quitTimer.Stop()
	}

	if client.idleTimer == nil {
		client.idleTimer = time.AfterFunc(IDLE_TIMEOUT, client.Idle)
	} else {
		client.idleTimer.Reset(IDLE_TIMEOUT)
	}
}

func (client *Client) Idle() {
	if client.quitTimer == nil {
		client.quitTimer = time.AfterFunc(QUIT_TIMEOUT, client.ConnectionTimeout)
	} else {
		client.quitTimer.Reset(QUIT_TIMEOUT)
	}

	client.Reply(RplPing(client.server, client))
}

func (client *Client) ConnectionTimeout() {
	msg := &QuitCommand{
		message: "connection timeout",
	}
	msg.SetClient(client)
	client.server.Command(msg)
}

func (client *Client) ConnectionClosed() {
	if client.destroyed {
		return
	}

	msg := &QuitCommand{
		message: "connection closed",
	}
	msg.SetClient(client)
	client.server.Command(msg)
}

func (c *Client) readCommands() {
	for line := range c.socket.Read() {
		m, err := ParseCommand(line)
		if err != nil {
			switch err {
			case NotEnoughArgsError:
				c.Reply(ErrNeedMoreParams(c.server, line))
			default:
				c.Reply(ErrUnknownCommand(c.server, line))
			}
			continue
		}

		m.SetClient(c)
		c.server.Command(m)
	}

	if c.registered {
		c.ConnectionClosed()
	} else {
		c.Destroy()
	}
}

func (client *Client) writeReplies() {
	for reply := range client.replies {
		if DEBUG_CLIENT {
			log.Printf("%s ← %s", client, reply)
		}

		if client.socket.Write(reply.Format(client)) != nil {
			close(client.replies)
		}
	}
	client.replies = nil
}

func (client *Client) Destroy() {
	if client.destroyed {
		return
	}

	if DEBUG_CLIENT {
		log.Printf("%s destroying", client)
	}

	client.destroyed = true

	if client.replies != nil {
		close(client.replies)
	}

	client.socket.Close()

	if client.idleTimer != nil {
		client.idleTimer.Stop()
	}

	if client.quitTimer != nil {
		client.quitTimer.Stop()
	}

	// clear channel list
	client.channels = make(ChannelSet)

	client.server.clients.Remove(client)

	if DEBUG_CLIENT {
		log.Printf("%s destroyed", client)
	}
}

func (client *Client) Reply(replies ...Reply) {
	for _, reply := range replies {
		if client.replies == nil {
			if DEBUG_CLIENT {
				log.Printf("%s dropped %s", client, reply)
			}
			continue
		}
		client.replies <- reply
	}
}

func (client *Client) HasNick() bool {
	return client.nick != ""
}

func (client *Client) HasUsername() bool {
	return client.username != ""
}

func (client *Client) InterestedClients() ClientSet {
	clients := make(ClientSet)
	for channel := range client.channels {
		for member := range channel.members {
			clients.Add(member)
		}
	}
	return clients
}

// <mode>
func (c *Client) ModeString() (str string) {
	if c.invisible {
		str += Invisible.String()
	}
	if c.operator {
		str += Operator.String()
	}

	if len(str) > 0 {
		str = "+" + str
	}
	return
}

func (c *Client) UserHost() string {
	username := "*"
	if c.HasUsername() {
		username = c.username
	}
	return fmt.Sprintf("%s!%s@%s", c.Nick(), username, c.hostname)
}

func (c *Client) Nick() string {
	if c.HasNick() {
		return c.nick
	}
	return "*"
}

func (c *Client) Id() string {
	return c.UserHost()
}

func (c *Client) String() string {
	return c.UserHost()
}
<commit_msg>don't close replies twice<commit_after>package irc

import (
	"fmt"
	"log"
	"net"
	"time"
)

type Client struct {
	atime       time.Time
	away        bool
	awayMessage string
	channels    ChannelSet
	ctime       time.Time
	destroyed   bool
	hostname    string
	idleTimer   *time.Timer
	invisible   bool
	loginTimer  *time.Timer
	nick        string
	operator    bool
	quitTimer   *time.Timer
	realname    string
	registered  bool
	replies     chan Reply
	server      *Server
	socket      *Socket
	authorized  bool
	username    string
}

func NewClient(server *Server, conn net.Conn) *Client {
	now := time.Now()
	client := &Client{
		atime:    now,
		channels: make(ChannelSet),
		ctime:    now,
		hostname: AddrLookupHostname(conn.RemoteAddr()),
		replies:  make(chan Reply),
		server:   server,
		socket:   NewSocket(conn),
	}
	client.loginTimer = time.AfterFunc(LOGIN_TIMEOUT, client.Destroy)

	go client.readCommands()
	go client.writeReplies()

	return client
}

func (client *Client) Touch() {
	if client.destroyed {
		return
	}

	client.atime = time.Now()

	if client.quitTimer != nil {
		client.quitTimer.Stop()
	}

	if client.idleTimer == nil {
		client.idleTimer = time.AfterFunc(IDLE_TIMEOUT, client.Idle)
	} else {
		client.idleTimer.Reset(IDLE_TIMEOUT)
	}
}

func (client *Client) Idle() {
	if client.quitTimer == nil {
		client.quitTimer = time.AfterFunc(QUIT_TIMEOUT, client.ConnectionTimeout)
	} else {
		client.quitTimer.Reset(QUIT_TIMEOUT)
	}

	client.Reply(RplPing(client.server, client))
}

func (client *Client) ConnectionTimeout() {
	msg := &QuitCommand{
		message: "connection timeout",
	}
	msg.SetClient(client)
	client.server.Command(msg)
}

func (client *Client) ConnectionClosed() {
	if client.destroyed {
		return
	}

	msg := &QuitCommand{
		message: "connection closed",
	}
	msg.SetClient(client)
	client.server.Command(msg)
}

func (c *Client) readCommands() {
	for line := range c.socket.Read() {
		m, err := ParseCommand(line)
		if err != nil {
			switch err {
			case NotEnoughArgsError:
				c.Reply(ErrNeedMoreParams(c.server, line))
			default:
				c.Reply(ErrUnknownCommand(c.server, line))
			}
			continue
		}

		m.SetClient(c)
		c.server.Command(m)
	}

	if c.registered {
		c.ConnectionClosed()
	} else {
		c.Destroy()
	}
}

func (client *Client) writeReplies() {
	for reply := range client.replies {
		if DEBUG_CLIENT {
			log.Printf("%s ← %s", client, reply)
		}

		if client.socket.Write(reply.Format(client)) != nil {
			break
		}
	}

	if client.replies != nil {
		close(client.replies)
		client.replies = nil
	}
}

func (client *Client) Destroy() {
	if client.destroyed {
		return
	}

	if DEBUG_CLIENT {
		log.Printf("%s destroying", client)
	}

	client.destroyed = true

	if client.replies != nil {
		close(client.replies)
		client.replies = nil
	}

	client.socket.Close()

	if client.idleTimer != nil {
		client.idleTimer.Stop()
	}

	if client.quitTimer != nil {
		client.quitTimer.Stop()
	}

	// clear channel list
	client.channels = make(ChannelSet)

	client.server.clients.Remove(client)

	if DEBUG_CLIENT {
		log.Printf("%s destroyed", client)
	}
}

func (client *Client) Reply(replies ...Reply) {
	for _, reply := range replies {
		if client.replies == nil {
			if DEBUG_CLIENT {
				log.Printf("%s dropped %s", client, reply)
			}
			continue
		}
		client.replies <- reply
	}
}

func (client *Client) HasNick() bool {
	return client.nick != ""
}

func (client *Client) HasUsername() bool {
	return client.username != ""
}

func (client *Client) InterestedClients() ClientSet {
	clients := make(ClientSet)
	for channel := range client.channels {
		for member := range channel.members {
			clients.Add(member)
		}
	}
	return clients
}

// <mode>
func (c *Client) ModeString() (str string) {
	if c.invisible {
		str += Invisible.String()
	}
	if c.operator {
		str += Operator.String()
	}

	if len(str) > 0 {
		str = "+" + str
	}
	return
}

func (c *Client) UserHost() string {
	username := "*"
	if c.HasUsername() {
		username = c.username
	}
	return fmt.Sprintf("%s!%s@%s", c.Nick(), username, c.hostname)
}

func (c *Client) Nick() string {
	if c.HasNick() {
		return c.nick
	}
	return "*"
}

func (c *Client) Id() string {
	return c.UserHost()
}

func (c *Client) String() string {
	return c.UserHost()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"strings"
	"sync/atomic"
	"time"

	r "github.com/dancannon/gorethink"
	"github.com/jaracil/ei"
	"github.com/jaracil/smartio"
	"golang.org/x/net/context"
)

type JsonRpcErr struct {
	Code    int         `json:"code"`
	Message string      `json:"message"`
	Data    interface{} `json:"data,omitempty"`
}

type JsonRpcReq struct {
	Jsonrpc string      `json:"jsonrpc"`
	Id      interface{} `json:"id,omitempty"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params"`
	nc      *NexusConn
}

type JsonRpcRes struct {
	Jsonrpc string      `json:"jsonrpc"`
	Id      interface{} `json:"id,omitempty"`
	Result  interface{} `json:"result,omitempty"`
	Error   *JsonRpcErr `json:"error,omitempty"`
}

type NexusConn struct {
	conn      net.Conn
	proto     string
	connRx    *smartio.SmartReader
	connTx    *smartio.SmartWriter
	connId    string
	user      *UserData
	chRes     chan *JsonRpcRes
	chReq     chan *JsonRpcReq
	context   context.Context
	cancelFun context.CancelFunc
	wdog      int64
	closed    int32 //Atomic bool
}

func NewNexusConn(conn net.Conn) *NexusConn {
	nc := &NexusConn{
		conn:   conn,
		proto:  "unknown",
		connRx: smartio.NewSmartReader(conn),
		connTx: smartio.NewSmartWriter(conn),
		connId: nodeId + safeId(4),
		user:   Nobody,
		chRes:  make(chan *JsonRpcRes, 64),
		chReq:  make(chan *JsonRpcReq, 64),
		wdog:   90,
	}
	nc.context, nc.cancelFun = context.WithCancel(mainContext)
	return nc
}

func (req *JsonRpcReq) Error(code int, message string, data interface{}) {
	if code < 0 {
		if message != "" {
			message = fmt.Sprintf("%s:[%s]", ErrStr[code], message)
		} else {
			message = ErrStr[code]
		}
	}
	req.nc.pushRes(
		&JsonRpcRes{
			Id:    req.Id,
			Error: &JsonRpcErr{Code: code, Message: message, Data: data},
		},
	)
}

func (req *JsonRpcReq) Result(result interface{}) {
	req.nc.pushRes(
		&JsonRpcRes{
			Id:     req.Id,
			Result: result,
		},
	)
}

func (nc *NexusConn) pushRes(res *JsonRpcRes) (err error) {
	select {
	case nc.chRes <- res:
	case <-nc.context.Done():
		err = errors.New("Context cancelled")
	}
	return
}

func (nc *NexusConn) pullRes() (res *JsonRpcRes, err error) {
	select {
	case res = <-nc.chRes:
	case <-nc.context.Done():
		err = errors.New("Context cancelled")
	}
	return
}

func (nc *NexusConn) pushReq(req *JsonRpcReq) (err error) {
	select {
	case nc.chReq <- req:
	case <-nc.context.Done():
		err = errors.New("Context cancelled")
	}
	return
}

func (nc *NexusConn) pullReq() (req *JsonRpcReq, err error) {
	select {
	case req = <-nc.chReq:
	case <-nc.context.Done():
		err = errors.New("Context cancelled")
	}
	return
}

func (nc *NexusConn) getTags(prefix string) (tags map[string]interface{}) {
	tags = map[string]interface{}{}
	if nc.user == nil || nc.user.Tags == nil {
		return
	}
	pfs := prefixes(prefix)
	for _, pf := range pfs {
		if tm, ok := nc.user.Tags[pf]; ok {
			for k, v := range tm {
				tags[k] = v
			}
		}
	}
	return
}

func (nc *NexusConn) handleReq(req *JsonRpcReq) {
	if req.Id == nil {
		return
	}
	switch {
	case strings.HasPrefix(req.Method, "sys."):
		nc.handleSysReq(req)
	case strings.HasPrefix(req.Method, "task."):
		nc.handleTaskReq(req)
	case strings.HasPrefix(req.Method, "pipe."):
		nc.handlePipeReq(req)
	case strings.HasPrefix(req.Method, "topic."):
		nc.handleTopicReq(req)
	case strings.HasPrefix(req.Method, "user."):
		nc.handleUserReq(req)
	case strings.HasPrefix(req.Method, "sync."):
		nc.handleSyncReq(req)
	default:
		req.Error(ErrMethodNotFound, "", nil)
	}
}

func (nc *NexusConn) respWorker() {
	defer nc.close()
	trackCh, err := sesNotify.Register(nc.connId, make(chan interface{}, 1024))
	if err != nil { // Duplicated session ???
		return
	}
	defer sesNotify.Unregister(nc.connId)
	for {
		select {
		case d := <-trackCh:
			resTask := d.(*Task)
			if resTask.ErrCode != nil {
				nc.pushRes(
					&JsonRpcRes{
						Id:    resTask.LocalId,
						Error: &JsonRpcErr{Code: *resTask.ErrCode, Message: resTask.ErrStr, Data: resTask.ErrObj},
					},
				)
			} else {
				nc.pushRes(
					&JsonRpcRes{
						Id:     resTask.LocalId,
						Result: resTask.Result,
					},
				)
			}
		case <-nc.context.Done():
			return
		}
	}
}

func (nc *NexusConn) sendWorker() {
	defer nc.close()
	var null *int
	for {
		res, err := nc.pullRes()
		if err != nil {
			log.Print("error on sendWorker:", err)
			break
		}
		if res.Id == nil {
			if res.Error == nil {
				continue //Skip notification responses
			}
			if res.Error.Code == ErrInvalidRequest || res.Error.Code == ErrParse {
				res.Id = null
			} else {
				continue
			}
		}
		res.Jsonrpc = "2.0"
		if res.Result == nil && res.Error == nil {
			res.Result = null
		}
		buf, err := json.Marshal(res)
		if err != nil {
			log.Print("Marshal error: ", err)
			break
		}
		buf = append(buf, byte('\r'), byte('\n'))
		n, err := nc.connTx.Write(buf)
		if err != nil || n != len(buf) {
			log.Print("Connection write error: ", err)
			break
		}
	}
	log.Print("exit from sendWorker")
}

func (nc *NexusConn) recvWorker() {
	defer nc.close()
	dec := json.NewDecoder(nc.connRx)
	for dec.More() {
		req := &JsonRpcReq{nc: nc}
		nc.connRx.SetLimit(1024 * 1024 * 32)
		err := dec.Decode(req)
		if err != nil {
			if _, ok := err.(*json.SyntaxError); ok {
				req.Error(ErrParse, "", nil)
				dec = json.NewDecoder(nc.connRx) // Refresh decoder
				continue
			}
			if _, ok := err.(*json.UnmarshalTypeError); ok {
				req.Error(ErrInvalidRequest, "", nil)
				continue
			}
			break
		}
		err = nc.pushReq(req)
		if err != nil {
			log.Print("error on recvWorker:", err)
			break
		}
	}
	log.Print("exit from recvWorker")
}

func (nc *NexusConn) watchdog() {
	defer nc.close()
	tick := time.NewTicker(time.Second * 10)
	exit := false
	for !exit {
		select {
		case <-tick.C:
			now := time.Now().Unix()
			max := atomic.LoadInt64(&nc.wdog)
			if (now-nc.connRx.GetLast() > max) &&
				(now-nc.connTx.GetLast() > max) {
				exit = true
				log.Printf("Connection [%s] watch dog expired!", nc.connId)
			}

			nc.updateSession()

		case <-nc.context.Done():
			exit = true
		}
	}
	tick.Stop()
}

func (nc *NexusConn) close() {
	if atomic.CompareAndSwapInt32(&nc.closed, 0, 1) {
		nc.cancelFun()
		nc.conn.Close()
		if mainContext.Err() == nil {
			log.Printf("Close %s session\r\n", nc.connId)
			dbClean(nc.connId)
		}
	}
}

func (nc *NexusConn) updateSession() {
	_, err := r.Table("sessions").
		Get(nc.connId).
		Replace(ei.M{
			"id":            nc.connId,
			"nodeId":        nodeId,
			"creationTime":  r.Row.Field("creationTime").Default(r.Now()),
			"lastSeen":      r.Now(),
			"remoteAddress": nc.conn.RemoteAddr().String(),
			"protocol":      nc.proto,
			"user":          nc.user.User,
		}).
		RunWrite(db)

	if err != nil {
		log.Println("Error updating session", nc.connId, ":", err)
		nc.close()
	}
}

func (nc *NexusConn) deleteSession() {
	res, err := r.Table("sessions").
		Get(nc.connId).
		Delete().
		RunWrite(db)

	if err != nil || res.Deleted != 1 {
		log.Println("Error deregistering session", nc.connId, ":", err)
		nc.close()
	}
}

var numconn int64

func (nc *NexusConn) handle() {

	atomic.AddInt64(&numconn, 1)
	defer func() { atomic.AddInt64(&numconn, -1) }()

	defer nc.close()
	go nc.respWorker()
	go nc.sendWorker()
	go nc.recvWorker()
	go nc.watchdog()

	nc.updateSession()
	defer nc.deleteSession()

	for {
		req, err := nc.pullReq()
		if err != nil {
			log.Print("error on handle:", err)
			break
		}
		log.Printf("Recibida instruccion jsonrpc: %+v", req)
		if (req.Jsonrpc != "2.0" && req.Jsonrpc != "") || req.Method == "" { //"jsonrpc":"2.0" is optional
			req.Error(ErrInvalidRequest, "", nil)
			continue
		}
		go nc.handleReq(req)
	}
}
<commit_msg>Sessions are already being deleted when closing<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"strings"
	"sync/atomic"
	"time"

	r "github.com/dancannon/gorethink"
	"github.com/jaracil/ei"
	"github.com/jaracil/smartio"
	"golang.org/x/net/context"
)

type JsonRpcErr struct {
	Code    int         `json:"code"`
	Message string      `json:"message"`
	Data    interface{} `json:"data,omitempty"`
}

type JsonRpcReq struct {
	Jsonrpc string      `json:"jsonrpc"`
	Id      interface{} `json:"id,omitempty"`
	Method  string      `json:"method"`
	Params  interface{} `json:"params"`
	nc      *NexusConn
}

type JsonRpcRes struct {
	Jsonrpc string      `json:"jsonrpc"`
	Id      interface{} `json:"id,omitempty"`
	Result  interface{} `json:"result,omitempty"`
	Error   *JsonRpcErr `json:"error,omitempty"`
}

type NexusConn struct {
	conn      net.Conn
	proto     string
	connRx    *smartio.SmartReader
	connTx    *smartio.SmartWriter
	connId    string
	user      *UserData
	chRes     chan *JsonRpcRes
	chReq     chan *JsonRpcReq
	context   context.Context
	cancelFun context.CancelFunc
	wdog      int64
	closed    int32 //Atomic bool
}

func NewNexusConn(conn net.Conn) *NexusConn {
	nc := &NexusConn{
		conn:   conn,
		proto:  "unknown",
		connRx: smartio.NewSmartReader(conn),
		connTx: smartio.NewSmartWriter(conn),
		connId: nodeId + safeId(4),
		user:   Nobody,
		chRes:  make(chan *JsonRpcRes, 64),
		chReq:  make(chan *JsonRpcReq, 64),
		wdog:   90,
	}
	nc.context, nc.cancelFun = context.WithCancel(mainContext)
	return nc
}

func (req *JsonRpcReq) Error(code int, message string, data interface{}) {
	if code < 0 {
		if message != "" {
			message = fmt.Sprintf("%s:[%s]", ErrStr[code], message)
		} else {
			message = ErrStr[code]
		}
	}
	req.nc.pushRes(
		&JsonRpcRes{
			Id:    req.Id,
			Error: &JsonRpcErr{Code: code, Message: message, Data: data},
		},
	)
}

func (req *JsonRpcReq) Result(result interface{}) {
	req.nc.pushRes(
		&JsonRpcRes{
			Id:     req.Id,
			Result: result,
		},
	)
}

func (nc *NexusConn) pushRes(res *JsonRpcRes) (err error) {
	select {
	case nc.chRes <- res:
	case <-nc.context.Done():
		err = errors.New("Context cancelled")
	}
	return
}

func (nc *NexusConn) pullRes() (res *JsonRpcRes, err error) {
	select {
	case res = <-nc.chRes:
	case <-nc.context.Done():
		err = errors.New("Context cancelled")
	}
	return
}

func (nc *NexusConn) pushReq(req *JsonRpcReq) (err error) {
	select {
	case nc.chReq <- req:
	case <-nc.context.Done():
		err = errors.New("Context cancelled")
	}
	return
}

func (nc *NexusConn) pullReq() (req *JsonRpcReq, err error) {
	select {
	case req = <-nc.chReq:
	case <-nc.context.Done():
		err = errors.New("Context cancelled")
	}
	return
}

func (nc *NexusConn) getTags(prefix string) (tags map[string]interface{}) {
	tags = map[string]interface{}{}
	if nc.user == nil || nc.user.Tags == nil {
		return
	}
	pfs := prefixes(prefix)
	for _, pf := range pfs {
		if tm, ok := nc.user.Tags[pf]; ok {
			for k, v := range tm {
				tags[k] = v
			}
		}
	}
	return
}

func (nc *NexusConn) handleReq(req *JsonRpcReq) {
	if req.Id == nil {
		return
	}
	switch {
	case strings.HasPrefix(req.Method, "sys."):
		nc.handleSysReq(req)
	case strings.HasPrefix(req.Method, "task."):
		nc.handleTaskReq(req)
	case strings.HasPrefix(req.Method, "pipe."):
		nc.handlePipeReq(req)
	case strings.HasPrefix(req.Method, "topic."):
		nc.handleTopicReq(req)
	case strings.HasPrefix(req.Method, "user."):
		nc.handleUserReq(req)
	case strings.HasPrefix(req.Method, "sync."):
		nc.handleSyncReq(req)
	default:
		req.Error(ErrMethodNotFound, "", nil)
	}
}

func (nc *NexusConn) respWorker() {
	defer nc.close()
	trackCh, err := sesNotify.Register(nc.connId, make(chan interface{}, 1024))
	if err != nil { // Duplicated session ???
		return
	}
	defer sesNotify.Unregister(nc.connId)
	for {
		select {
		case d := <-trackCh:
			resTask := d.(*Task)
			if resTask.ErrCode != nil {
				nc.pushRes(
					&JsonRpcRes{
						Id:    resTask.LocalId,
						Error: &JsonRpcErr{Code: *resTask.ErrCode, Message: resTask.ErrStr, Data: resTask.ErrObj},
					},
				)
			} else {
				nc.pushRes(
					&JsonRpcRes{
						Id:     resTask.LocalId,
						Result: resTask.Result,
					},
				)
			}
		case <-nc.context.Done():
			return
		}
	}
}

func (nc *NexusConn) sendWorker() {
	defer nc.close()
	var null *int
	for {
		res, err := nc.pullRes()
		if err != nil {
			log.Print("error on sendWorker:", err)
			break
		}
		if res.Id == nil {
			if res.Error == nil {
				continue //Skip notification responses
			}
			if res.Error.Code == ErrInvalidRequest || res.Error.Code == ErrParse {
				res.Id = null
			} else {
				continue
			}
		}
		res.Jsonrpc = "2.0"
		if res.Result == nil && res.Error == nil {
			res.Result = null
		}
		buf, err := json.Marshal(res)
		if err != nil {
			log.Print("Marshal error: ", err)
			break
		}
		buf = append(buf, byte('\r'), byte('\n'))
		n, err := nc.connTx.Write(buf)
		if err != nil || n != len(buf) {
			log.Print("Connection write error: ", err)
			break
		}
	}
	log.Print("exit from sendWorker")
}

func (nc *NexusConn) recvWorker() {
	defer nc.close()
	dec := json.NewDecoder(nc.connRx)
	for dec.More() {
		req := &JsonRpcReq{nc: nc}
		nc.connRx.SetLimit(1024 * 1024 * 32)
		err := dec.Decode(req)
		if err != nil {
			if _, ok := err.(*json.SyntaxError); ok {
				req.Error(ErrParse, "", nil)
				dec = json.NewDecoder(nc.connRx) // Refresh decoder
				continue
			}
			if _, ok := err.(*json.UnmarshalTypeError); ok {
				req.Error(ErrInvalidRequest, "", nil)
				continue
			}
			break
		}
		err = nc.pushReq(req)
		if err != nil {
			log.Print("error on recvWorker:", err)
			break
		}
	}
	log.Print("exit from recvWorker")
}

func (nc *NexusConn) watchdog() {
	defer nc.close()
	tick := time.NewTicker(time.Second * 10)
	exit := false
	for !exit {
		select {
		case <-tick.C:
			now := time.Now().Unix()
			max := atomic.LoadInt64(&nc.wdog)
			if (now-nc.connRx.GetLast() > max) &&
				(now-nc.connTx.GetLast() > max) {
				exit = true
				log.Printf("Connection [%s] watch dog expired!", nc.connId)
			}

			nc.updateSession()

		case <-nc.context.Done():
			exit = true
		}
	}
	tick.Stop()
}

func (nc *NexusConn) close() {
	if atomic.CompareAndSwapInt32(&nc.closed, 0, 1) {
		nc.cancelFun()
		nc.conn.Close()
		if mainContext.Err() == nil {
			log.Printf("Close %s session\r\n", nc.connId)
			dbClean(nc.connId)
		}
	}
}

func (nc *NexusConn) updateSession() {
	_, err := r.Table("sessions").
		Get(nc.connId).
		Replace(ei.M{
			"id":            nc.connId,
			"nodeId":        nodeId,
			"creationTime":  r.Row.Field("creationTime").Default(r.Now()),
			"lastSeen":      r.Now(),
			"remoteAddress": nc.conn.RemoteAddr().String(),
			"protocol":      nc.proto,
			"user":          nc.user.User,
		}).
		RunWrite(db)

	if err != nil {
		log.Println("Error updating session", nc.connId, ":", err)
		nc.close()
	}
}

var numconn int64

func (nc *NexusConn) handle() {

	atomic.AddInt64(&numconn, 1)
	defer func() { atomic.AddInt64(&numconn, -1) }()

	defer nc.close()
	go nc.respWorker()
	go nc.sendWorker()
	go nc.recvWorker()
	go nc.watchdog()

	nc.updateSession()

	for {
		req, err := nc.pullReq()
		if err != nil {
			log.Print("error on handle:", err)
			break
		}
		log.Printf("Recibida instruccion jsonrpc: %+v", req)
		if (req.Jsonrpc != "2.0" && req.Jsonrpc != "") || req.Method == "" { //"jsonrpc":"2.0" is optional
			req.Error(ErrInvalidRequest, "", nil)
			continue
		}
		go nc.handleReq(req)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package declarative

import (
	"github.com/lxn/walk"
)

type SplitButton struct {
	AssignTo           **walk.SplitButton
	Name               string
	Enabled            Property
	Visible            Property
	Font               Font
	ToolTipText        Property
	MinSize            Size
	MaxSize            Size
	StretchFactor      int
	Row                int
	RowSpan            int
	Column             int
	ColumnSpan         int
	AlwaysConsumeSpace bool
	ContextMenuItems   []MenuItem
	OnKeyDown          walk.KeyEventHandler
	OnKeyPress         walk.KeyEventHandler
	OnKeyUp            walk.KeyEventHandler
	OnMouseDown        walk.MouseEventHandler
	OnMouseMove        walk.MouseEventHandler
	OnMouseUp          walk.MouseEventHandler
	OnSizeChanged      walk.EventHandler
	Text               Property
	Image              interface{}
	ImageAboveText     bool
	MenuItems          []MenuItem
	OnClicked          walk.EventHandler
}

func (sb SplitButton) Create(builder *Builder) error {
	w, err := walk.NewSplitButton(builder.Parent())
	if err != nil {
		return err
	}

	builder.deferBuildMenuActions(w.Menu(), sb.MenuItems)

	return builder.InitWidget(sb, w, func() error {
		img := sb.Image
		if s, ok := img.(string); ok {
			var err error
			if img, err = imageFromFile(s); err != nil {
				return err
			}
		}
		if img != nil {
			if err := w.SetImage(img.(walk.Image)); err != nil {
				return err
			}
		}

		if err := w.SetImageAboveText(sb.ImageAboveText); err != nil {
			return err
		}

		if sb.OnClicked != nil {
			w.Clicked().Attach(sb.OnClicked)
		}

		if sb.AssignTo != nil {
			*sb.AssignTo = w
		}

		return nil
	})
}

func (w SplitButton) WidgetInfo() (name string, disabled, hidden bool, font *Font, toolTipText string, minSize, maxSize Size, stretchFactor, row, rowSpan, column, columnSpan int, alwaysConsumeSpace bool, contextMenuItems []MenuItem, OnKeyDown walk.KeyEventHandler, OnKeyPress walk.KeyEventHandler, OnKeyUp walk.KeyEventHandler, OnMouseDown walk.MouseEventHandler, OnMouseMove walk.MouseEventHandler, OnMouseUp walk.MouseEventHandler, OnSizeChanged walk.EventHandler) {
	return w.Name, false, false, &w.Font, "", w.MinSize, w.MaxSize, w.StretchFactor, w.Row, w.RowSpan, w.Column, w.ColumnSpan, w.AlwaysConsumeSpace, w.ContextMenuItems, w.OnKeyDown, w.OnKeyPress, w.OnKeyUp, w.OnMouseDown, w.OnMouseMove, w.OnMouseUp, w.OnSizeChanged
}
<commit_msg>declarative/SplitButton: Fix copyright date<commit_after>// Copyright 2016 The Walk Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package declarative

import (
	"github.com/lxn/walk"
)

type SplitButton struct {
	AssignTo           **walk.SplitButton
	Name               string
	Enabled            Property
	Visible            Property
	Font               Font
	ToolTipText        Property
	MinSize            Size
	MaxSize            Size
	StretchFactor      int
	Row                int
	RowSpan            int
	Column             int
	ColumnSpan         int
	AlwaysConsumeSpace bool
	ContextMenuItems   []MenuItem
	OnKeyDown          walk.KeyEventHandler
	OnKeyPress         walk.KeyEventHandler
	OnKeyUp            walk.KeyEventHandler
	OnMouseDown        walk.MouseEventHandler
	OnMouseMove        walk.MouseEventHandler
	OnMouseUp          walk.MouseEventHandler
	OnSizeChanged      walk.EventHandler
	Text               Property
	Image              interface{}
	ImageAboveText     bool
	MenuItems          []MenuItem
	OnClicked          walk.EventHandler
}

func (sb SplitButton) Create(builder *Builder) error {
	w, err := walk.NewSplitButton(builder.Parent())
	if err != nil {
		return err
	}

	builder.deferBuildMenuActions(w.Menu(), sb.MenuItems)

	return builder.InitWidget(sb, w, func() error {
		img := sb.Image
		if s, ok := img.(string); ok {
			var err error
			if img, err = imageFromFile(s); err != nil {
				return err
			}
		}
		if img != nil {
			if err := w.SetImage(img.(walk.Image)); err != nil {
				return err
			}
		}

		if err := w.SetImageAboveText(sb.ImageAboveText); err != nil {
			return err
		}

		if sb.OnClicked != nil {
			w.Clicked().Attach(sb.OnClicked)
		}

		if sb.AssignTo != nil {
			*sb.AssignTo = w
		}

		return nil
	})
}

func (w SplitButton) WidgetInfo() (name string, disabled, hidden bool, font *Font, toolTipText string, minSize, maxSize Size, stretchFactor, row, rowSpan, column, columnSpan int, alwaysConsumeSpace bool, contextMenuItems []MenuItem, OnKeyDown walk.KeyEventHandler, OnKeyPress walk.KeyEventHandler, OnKeyUp walk.KeyEventHandler, OnMouseDown walk.MouseEventHandler, OnMouseMove walk.MouseEventHandler, OnMouseUp walk.MouseEventHandler, OnSizeChanged walk.EventHandler) {
	return w.Name, false, false, &w.Font, "", w.MinSize, w.MaxSize, w.StretchFactor, w.Row, w.RowSpan, w.Column, w.ColumnSpan, w.AlwaysConsumeSpace, w.ContextMenuItems, w.OnKeyDown, w.OnKeyPress, w.OnKeyUp, w.OnMouseDown, w.OnMouseMove, w.OnMouseUp, w.OnSizeChanged
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/binary"
	"fmt"
	"github.com/golang/glog"
	k8sapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"net"
	"strconv"
)

type Cluster struct {
	unversioned.TypeMeta `json:",inline"`
	k8sapi.ObjectMeta    `json:"metadata,omitempty"`

	Spec ClusterSpec `json:"spec,omitempty"`
}

type ClusterSpec struct {
	// The CloudProvider to use (aws or gce)
	CloudProvider string `json:"cloudProvider,omitempty"`

	// The version of kubernetes to install (optional, and can be a "spec" like stable)
	KubernetesVersion string `json:"kubernetesVersion,omitempty"`

	//
	//// The Node initializer technique to use: cloudinit or nodeup
	//NodeInit                      string `json:",omitempty"`

	// Configuration of zones we are targeting
	Zones []*ClusterZoneSpec `json:"zones,omitempty"`
	//Region                        string        `json:",omitempty"`

	// Project is the cloud project we should use, required on GCE
	Project string `json:"project,omitempty"`

	// MasterPermissions contains the IAM permissions for the masters
	MasterPermissions *CloudPermissions `json:"masterPermissions,omitempty"`
	// NodePermissions contains the IAM permissions for the nodes
	NodePermissions *CloudPermissions `json:"nodePermissions,omitempty"`

	// MasterPublicName is the external DNS name for the master nodes
	MasterPublicName string `json:"masterPublicName,omitempty"`
	// MasterInternalName is the internal DNS name for the master nodes
	MasterInternalName string `json:"masterInternalName,omitempty"`

	// The CIDR used for the AWS VPC / GCE Network, or otherwise allocated to k8s
	// This is a real CIDR, not the internal k8s network
	NetworkCIDR string `json:"networkCIDR,omitempty"`

	// NetworkID is an identifier of a network, if we want to reuse/share an existing network (e.g. an AWS VPC)
	NetworkID string `json:"networkID,omitempty"`

	// SecretStore is the VFS path to where secrets are stored
	SecretStore string `json:"secretStore,omitempty"`
	// KeyStore is the VFS path to where SSL keys and certificates are stored
	KeyStore string `json:"keyStore,omitempty"`
	// ConfigStore is the VFS path to where the configuration (CloudConfig, NodeSetConfig etc) is stored
	ConfigStore string `json:"configStore,omitempty"`

	// DNSZone is the DNS zone we should use when configuring DNS
	DNSZone string `json:"dnsZone,omitempty"`

	//InstancePrefix                string `json:",omitempty"`

	// ClusterName is a unique identifier for the cluster, and currently must be a DNS name
	//ClusterName       string `json:",omitempty"`

	//AllocateNodeCIDRs *bool `json:"allocateNodeCIDRs,omitempty"`

	Multizone *bool `json:"mutlizone,omitempty"`

	//ClusterIPRange                string `json:",omitempty"`

	// ServiceClusterIPRange is the CIDR, from the internal network, where we allocate IPs for services
	ServiceClusterIPRange string `json:"serviceClusterIPRange,omitempty"`
	//MasterIPRange                 string `json:",omitempty"`
	//NonMasqueradeCidr             string `json:",omitempty"`
	//
	//NetworkProvider               string `json:",omitempty"`
	//
	//HairpinMode                   string `json:",omitempty"`
	//
	//OpencontrailTag               string `json:",omitempty"`
	//OpencontrailKubernetesTag     string `json:",omitempty"`
	//OpencontrailPublicSubnet      string `json:",omitempty"`
	//
	//EnableClusterMonitoring       string `json:",omitempty"`
	//EnableL7LoadBalancing         string `json:",omitempty"`
	//EnableClusterUI               *bool  `json:",omitempty"`
	//
	//EnableClusterDNS              *bool  `json:",omitempty"`
	//DNSReplicas                   int    `json:",omitempty"`
	//DNSServerIP                   string `json:",omitempty"`

	// DNSDomain is the suffix we use for internal DNS names (normally cluster.local)
	DNSDomain string `json:"dnsDomain,omitempty"`

	//EnableClusterLogging          *bool  `json:",omitempty"`
	//EnableNodeLogging             *bool  `json:",omitempty"`
	//LoggingDestination            string `json:",omitempty"`
	//ElasticsearchLoggingReplicas  int    `json:",omitempty"`
	//
	//EnableClusterRegistry         *bool  `json:",omitempty"`
	//ClusterRegistryDisk           string `json:",omitempty"`
	//ClusterRegistryDiskSize       int    `json:",omitempty"`
	//
	//EnableCustomMetrics           *bool `json:",omitempty"`
	//
	//RegisterMasterKubelet         *bool  `json:",omitempty"`

	//// Image is the default image spec to use for the cluster
	//Image                     string `json:",omitempty"`

	//KubeUser                      string `json:",omitempty"`
	//
	//// These are moved to CAStore / SecretStore
	////KubePassword			string
	////KubeletToken                  string
	////KubeProxyToken                string
	////BearerToken                   string
	////CACert                        []byte
	////CAKey                         []byte
	////KubeletCert                   []byte
	////KubeletKey                    []byte
	////MasterCert                    []byte
	////MasterKey                     []byte
	////KubecfgCert                   []byte
	////KubecfgKey                    []byte
	//
	//AdmissionControl              string `json:",omitempty"`
	//RuntimeConfig                 string `json:",omitempty"`
	//
	//KubeImageTag                  string `json:",omitempty"`
	//KubeDockerRegistry            string `json:",omitempty"`
	//KubeAddonRegistry             string `json:",omitempty"`
	//
	//KubeletPort                   int `json:",omitempty"`
	//
	//KubeApiserverRequestTimeout   int `json:",omitempty"`
	//
	//TerminatedPodGcThreshold      string `json:",omitempty"`
	//
	//EnableManifestURL             *bool  `json:",omitempty"`
	//ManifestURL                   string `json:",omitempty"`
	//ManifestURLHeader             string `json:",omitempty"`
	//
	//TestCluster                   string `json:",omitempty"`
	//
	//DockerOptions                 string `json:",omitempty"`
	//DockerStorage                 string `json:",omitempty"`
	//ExtraDockerOpts               string `json:",omitempty"`
	//
	//E2EStorageTestEnvironment     string `json:",omitempty"`
	//KubeletTestArgs               string `json:",omitempty"`
	//KubeletTestLogLevel           string `json:",omitempty"`
	//DockerTestArgs                string `json:",omitempty"`
	//DockerTestLogLevel            string `json:",omitempty"`
	//ApiserverTestArgs             string `json:",omitempty"`
	//ApiserverTestLogLevel         string `json:",omitempty"`
	//ControllerManagerTestArgs     string `json:",omitempty"`
	//ControllerManagerTestLogLevel string `json:",omitempty"`
	//SchedulerTestArgs             string `json:",omitempty"`
	//SchedulerTestLogLevel         string `json:",omitempty"`
	//KubeProxyTestArgs             string `json:",omitempty"`
	//KubeProxyTestLogLevel         string `json:",omitempty"`

	//NodeUp                        *NodeUpConfig `json:",omitempty"`

	// nodeSets is a list of all the NodeSets in the cluster.
	// It is not exported: we populate it from other files
	//nodeSets                      []*NodeSetConfig `json:",omitempty"`

	//// Masters is the configuration for each master in the cluster
	//Masters []*MasterConfig `json:",omitempty"`

	// EtcdClusters stores the configuration for each cluster
	EtcdClusters []*EtcdClusterSpec `json:"etcdClusters,omitempty"`

	// Component configurations
	Docker                *DockerConfig                `json:"docker,omitempty"`
	KubeDNS               *KubeDNSConfig               `json:"kubeDNS,omitempty"`
	KubeAPIServer         *KubeAPIServerConfig         `json:"kubeAPIServer,omitempty"`
	KubeControllerManager *KubeControllerManagerConfig `json:"kubeControllerManager,omitempty"`
	KubeScheduler         *KubeSchedulerConfig         `json:"kubeScheduler,omitempty"`
	KubeProxy             *KubeProxyConfig             `json:"kubeProxy,omitempty"`
	Kubelet               *KubeletConfig               `json:"kubelet,omitempty"`
	MasterKubelet         *KubeletConfig               `json:"masterKubelet,omitempty"`
}

type KubeDNSConfig struct {
	Replicas int    `json:"replicas,omitempty"`
	Domain   string `json:"domain,omitempty"`
	ServerIP string `json:"serverIP,omitempty"`
}

//
//type MasterConfig struct {
//	Name string `json:",omitempty"`
//
//	Image       string `json:",omitempty"`
//	Zone        string `json:",omitempty"`
//	MachineType string `json:",omitempty"`
//}
//

type EtcdClusterSpec struct {
	// Name is the name of the etcd cluster (main, events etc)
	Name string `json:"name,omitempty"`

	// EtcdMember stores the configurations for each member of the cluster (including the data volume)
	Members []*EtcdMemberSpec `json:"etcdMembers,omitempty"`
}

type EtcdMemberSpec struct {
	// Name is the name of the member within the etcd cluster
	Name string `json:"name,omitempty"`
	Zone string `json:"zone,omitempty"`

	VolumeType string `json:"volumeType,omitempty"`
	VolumeSize int    `json:"volumeSize,omitempty"`
}

type ClusterZoneSpec struct {
	Name string `json:"name,omitempty"`
	CIDR string `json:"cidr,omitempty"`
}

//type NodeUpConfig struct {
//	Source     string `json:",omitempty"`
//	SourceHash string `json:",omitempty"`
//
//	Tags       []string `json:",omitempty"`
//
//	// Assets that NodeUp should use.  This is a "search-path" for resolving dependencies.
//	Assets     []string `json:",omitempty"`
//}

// PerformAssignments populates values that are required and immutable
// For example, it assigns stable Keys to NodeSets & Masters, and
// it assigns CIDRs to subnets
func (c *Cluster) PerformAssignments() error {
	if c.Spec.NetworkCIDR == "" {
		// TODO: Choose non-overlapping networking CIDRs for VPCs?
		c.Spec.NetworkCIDR = "172.20.0.0/16"
	}

	for _, zone := range c.Spec.Zones {
		err := zone.performAssignments(c)
		if err != nil {
			return err
		}
	}

	return nil
}

func (z *ClusterZoneSpec) performAssignments(c *Cluster) error {
	if z.CIDR == "" {
		cidr, err := z.assignCIDR(c)
		if err != nil {
			return err
		}
		glog.Infof("Assigned CIDR %s to zone %s", cidr, z.Name)
		z.CIDR = cidr
	}

	return nil
}

func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) {
	// TODO: We probably could query for the existing subnets & allocate appropriately
	// for now we'll require users to set CIDRs themselves

	lastCharMap := make(map[byte]bool)
	for _, nodeZone := range c.Spec.Zones {
		lastChar := nodeZone.Name[len(nodeZone.Name)-1]
		lastCharMap[lastChar] = true
	}

	index := -1

	if len(lastCharMap) == len(c.Spec.Zones) {
		// Last char of zones are unique (GCE, AWS)
		// At least on AWS, we also want 'a' to be 1, so that we don't collide with the lowest range,
		// because kube-up uses that range
		index = int(z.Name[len(z.Name)-1])
	} else {
		glog.Warningf("Last char of zone names not unique")

		for i, nodeZone := range c.Spec.Zones {
			if nodeZone.Name == z.Name {
				index = i
				break
			}
		}
		if index == -1 {
			return "", fmt.Errorf("zone not configured: %q", z.Name)
		}
	}

	_, cidr, err := net.ParseCIDR(c.Spec.NetworkCIDR)
	if err != nil {
		return "", fmt.Errorf("Invalid NetworkCIDR: %q", c.Spec.NetworkCIDR)
	}
	networkLength, _ := cidr.Mask.Size()

	// We assume a maximum of 8 subnets per network
	// TODO: Does this make sense on GCE?
	// TODO: Should we limit this to say 1000 IPs per subnet? (any reason to?)
	index = index % 8
	networkLength += 3

	ip4 := cidr.IP.To4()
	if ip4 != nil {
		n := binary.BigEndian.Uint32(ip4)
		n += uint32(index) << uint(32-networkLength)
		subnetIP := make(net.IP, len(ip4))
		binary.BigEndian.PutUint32(subnetIP, n)
		subnetCIDR := subnetIP.String() + "/" + strconv.Itoa(networkLength)
		glog.V(2).Infof("Computed CIDR for subnet in zone %q as %q", z.Name, subnetCIDR)
		return subnetCIDR, nil
	}

	return "", fmt.Errorf("Unexpected IP address type for NetworkCIDR: %s", c.Spec.NetworkCIDR)
}

// SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
func (c *Cluster) SharedVPC() bool {
	return c.Spec.NetworkID != ""
}

// CloudPermissions holds IAM-style permissions
type CloudPermissions struct {
	S3Buckets []string `json:"s3Buckets,omitempty"`
}

// AddS3Bucket adds a bucket if it does not already exist
func (p *CloudPermissions) AddS3Bucket(bucket string) {
	for _, b := range p.S3Buckets {
		if b == bucket {
			return
		}
	}

	p.S3Buckets = append(p.S3Buckets, bucket)
}

//
//// findImage finds the default image
//func (c*NodeSetConfig) resolveImage() error {
//	cloud.(*awsup.AWSCloud).ResolveImage()
//
//	if n.Image == "" {
//		if defaultImage == "" {
//			image, err := c.determineImage()
//			if err != nil {
//				return err
//			}
//			defaultImage = image
//		}
//		n.Image = defaultImage
//	}
//
//
//	return nil
//}
<commit_msg>upup: group DNS properties<commit_after>package api

import (
	"encoding/binary"
	"fmt"
	"github.com/golang/glog"
	k8sapi "k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"net"
	"strconv"
)

type Cluster struct {
	unversioned.TypeMeta `json:",inline"`
	k8sapi.ObjectMeta    `json:"metadata,omitempty"`

	Spec ClusterSpec `json:"spec,omitempty"`
}

type ClusterSpec struct {
	// The CloudProvider to use (aws or gce)
	CloudProvider string `json:"cloudProvider,omitempty"`

	// The version of kubernetes to install (optional, and can be a "spec" like stable)
	KubernetesVersion string `json:"kubernetesVersion,omitempty"`

	//
	//// The Node initializer technique to use: cloudinit or nodeup
	//NodeInit                      string `json:",omitempty"`

	// Configuration of zones we are targeting
	Zones []*ClusterZoneSpec `json:"zones,omitempty"`
	//Region                        string        `json:",omitempty"`

	// Project is the cloud project we should use, required on GCE
	Project string `json:"project,omitempty"`

	// MasterPermissions contains the IAM permissions for the masters
	MasterPermissions *CloudPermissions `json:"masterPermissions,omitempty"`
	// NodePermissions contains the IAM permissions for the nodes
	NodePermissions *CloudPermissions `json:"nodePermissions,omitempty"`

	// MasterPublicName is the external DNS name for the master nodes
	MasterPublicName string `json:"masterPublicName,omitempty"`
	// MasterInternalName is the internal DNS name for the master nodes
	MasterInternalName string `json:"masterInternalName,omitempty"`

	// The CIDR used for the AWS VPC / GCE Network, or otherwise allocated to k8s
	// This is a real CIDR, not the internal k8s network
	NetworkCIDR string `json:"networkCIDR,omitempty"`

	// NetworkID is an identifier of a network, if we want to reuse/share an existing network (e.g. an AWS VPC)
	NetworkID string `json:"networkID,omitempty"`

	// SecretStore is the VFS path to where secrets are stored
	SecretStore string `json:"secretStore,omitempty"`
	// KeyStore is the VFS path to where SSL keys and certificates are stored
	KeyStore string `json:"keyStore,omitempty"`
	// ConfigStore is the VFS path to where the configuration (CloudConfig, NodeSetConfig etc) is stored
	ConfigStore string `json:"configStore,omitempty"`

	// DNSZone is the DNS zone we should use when configuring DNS
	// This is because some clouds let us define a managed zone foo.bar, and then have
	// kubernetes.dev.foo.bar, without needing to define dev.foo.bar as a hosted zone.
	// DNSZone will probably be a suffix of DNSDomain
	DNSZone string `json:"dnsZone,omitempty"`

	// DNSDomain is the suffix we use for internal DNS names (normally cluster.local)
	DNSDomain string `json:"dnsDomain,omitempty"`

	//InstancePrefix                string `json:",omitempty"`

	// ClusterName is a unique identifier for the cluster, and currently must be a DNS name
	//ClusterName       string `json:",omitempty"`

	//AllocateNodeCIDRs *bool `json:"allocateNodeCIDRs,omitempty"`

	Multizone *bool `json:"mutlizone,omitempty"`

	//ClusterIPRange                string `json:",omitempty"`

	// ServiceClusterIPRange is the CIDR, from the internal network, where we allocate IPs for services
	ServiceClusterIPRange string `json:"serviceClusterIPRange,omitempty"`
	//MasterIPRange                 string `json:",omitempty"`
	//NonMasqueradeCidr             string `json:",omitempty"`
	//
	//NetworkProvider               string `json:",omitempty"`
	//
	//HairpinMode                   string `json:",omitempty"`
	//
	//OpencontrailTag               string `json:",omitempty"`
	//OpencontrailKubernetesTag     string `json:",omitempty"`
	//OpencontrailPublicSubnet      string `json:",omitempty"`
	//
	//EnableClusterMonitoring       string `json:",omitempty"`
	//EnableL7LoadBalancing         string `json:",omitempty"`
	//EnableClusterUI               *bool  `json:",omitempty"`
	//
	//EnableClusterDNS              *bool  `json:",omitempty"`
	//DNSReplicas                   int    `json:",omitempty"`
	//DNSServerIP                   string `json:",omitempty"`

	//EnableClusterLogging          *bool  `json:",omitempty"`
	//EnableNodeLogging             *bool  `json:",omitempty"`
	//LoggingDestination            string `json:",omitempty"`
	//ElasticsearchLoggingReplicas  int    `json:",omitempty"`
	//
	//EnableClusterRegistry         *bool  `json:",omitempty"`
	//ClusterRegistryDisk           string `json:",omitempty"`
	//ClusterRegistryDiskSize       int    `json:",omitempty"`
	//
	//EnableCustomMetrics           *bool `json:",omitempty"`
	//
	//RegisterMasterKubelet         *bool  `json:",omitempty"`

	//// Image is the default image spec to use for the cluster
	//Image                     string `json:",omitempty"`

	//KubeUser                      string `json:",omitempty"`
	//
	//// These are moved to CAStore / SecretStore
	////KubePassword			string
	////KubeletToken                  string
	////KubeProxyToken                string
	////BearerToken                   string
	////CACert                        []byte
	////CAKey                         []byte
	////KubeletCert                   []byte
	////KubeletKey                    []byte
	////MasterCert                    []byte
	////MasterKey                     []byte
	////KubecfgCert                   []byte
	////KubecfgKey                    []byte
	//
	//AdmissionControl              string `json:",omitempty"`
	//RuntimeConfig                 string `json:",omitempty"`
	//
	//KubeImageTag                  string `json:",omitempty"`
	//KubeDockerRegistry            string `json:",omitempty"`
	//KubeAddonRegistry             string `json:",omitempty"`
	//
	//KubeletPort                   int `json:",omitempty"`
	//
	//KubeApiserverRequestTimeout   int `json:",omitempty"`
	//
	//TerminatedPodGcThreshold      string `json:",omitempty"`
	//
	//EnableManifestURL             *bool  `json:",omitempty"`
	//ManifestURL                   string `json:",omitempty"`
	//ManifestURLHeader             string `json:",omitempty"`
	//
	//TestCluster                   string `json:",omitempty"`
	//
	//DockerOptions                 string `json:",omitempty"`
	//DockerStorage                 string `json:",omitempty"`
	//ExtraDockerOpts               string `json:",omitempty"`
	//
	//E2EStorageTestEnvironment     string `json:",omitempty"`
	//KubeletTestArgs               string `json:",omitempty"`
	//KubeletTestLogLevel           string `json:",omitempty"`
	//DockerTestArgs                string `json:",omitempty"`
	//DockerTestLogLevel            string `json:",omitempty"`
	//ApiserverTestArgs             string `json:",omitempty"`
	//ApiserverTestLogLevel         string `json:",omitempty"`
	//ControllerManagerTestArgs     string `json:",omitempty"`
	//ControllerManagerTestLogLevel string `json:",omitempty"`
	//SchedulerTestArgs             string `json:",omitempty"`
	//SchedulerTestLogLevel         string `json:",omitempty"`
	//KubeProxyTestArgs             string `json:",omitempty"`
	//KubeProxyTestLogLevel         string `json:",omitempty"`

	//NodeUp                        *NodeUpConfig `json:",omitempty"`

	// nodeSets is a list of all the NodeSets in the cluster.
	// It is not exported: we populate it from other files
	//nodeSets                      []*NodeSetConfig `json:",omitempty"`

	//// Masters is the configuration for each master in the cluster
	//Masters []*MasterConfig `json:",omitempty"`

	// EtcdClusters stores the configuration for each cluster
	EtcdClusters []*EtcdClusterSpec `json:"etcdClusters,omitempty"`

	// Component configurations
	Docker                *DockerConfig                `json:"docker,omitempty"`
	KubeDNS               *KubeDNSConfig               `json:"kubeDNS,omitempty"`
	KubeAPIServer         *KubeAPIServerConfig         `json:"kubeAPIServer,omitempty"`
	KubeControllerManager *KubeControllerManagerConfig `json:"kubeControllerManager,omitempty"`
	KubeScheduler         *KubeSchedulerConfig         `json:"kubeScheduler,omitempty"`
	KubeProxy             *KubeProxyConfig             `json:"kubeProxy,omitempty"`
	Kubelet               *KubeletConfig               `json:"kubelet,omitempty"`
	MasterKubelet         *KubeletConfig               `json:"masterKubelet,omitempty"`
}

type KubeDNSConfig struct {
	Replicas int    `json:"replicas,omitempty"`
	Domain   string `json:"domain,omitempty"`
	ServerIP string `json:"serverIP,omitempty"`
}

//
//type MasterConfig struct {
//	Name string `json:",omitempty"`
//
//	Image       string `json:",omitempty"`
//	Zone        string `json:",omitempty"`
//	MachineType string `json:",omitempty"`
//}
//

type EtcdClusterSpec struct {
	// Name is the name of the etcd cluster (main, events etc)
	Name string `json:"name,omitempty"`

	// EtcdMember stores the configurations for each member of the cluster (including the data volume)
	Members []*EtcdMemberSpec `json:"etcdMembers,omitempty"`
}

type EtcdMemberSpec struct {
	// Name is the name of the member within the etcd cluster
	Name string `json:"name,omitempty"`
	Zone string `json:"zone,omitempty"`

	VolumeType string `json:"volumeType,omitempty"`
	VolumeSize int    `json:"volumeSize,omitempty"`
}

type ClusterZoneSpec struct {
	Name string `json:"name,omitempty"`
	CIDR string `json:"cidr,omitempty"`
}

//type NodeUpConfig struct {
//	Source     string `json:",omitempty"`
//	SourceHash string `json:",omitempty"`
//
//	Tags       []string `json:",omitempty"`
//
//	// Assets that NodeUp should use.  This is a "search-path" for resolving dependencies.
//	Assets     []string `json:",omitempty"`
//}

// PerformAssignments populates values that are required and immutable
// For example, it assigns stable Keys to NodeSets & Masters, and
// it assigns CIDRs to subnets
func (c *Cluster) PerformAssignments() error {
	if c.Spec.NetworkCIDR == "" {
		// TODO: Choose non-overlapping networking CIDRs for VPCs?
		c.Spec.NetworkCIDR = "172.20.0.0/16"
	}

	for _, zone := range c.Spec.Zones {
		err := zone.performAssignments(c)
		if err != nil {
			return err
		}
	}

	return nil
}

func (z *ClusterZoneSpec) performAssignments(c *Cluster) error {
	if z.CIDR == "" {
		cidr, err := z.assignCIDR(c)
		if err != nil {
			return err
		}
		glog.Infof("Assigned CIDR %s to zone %s", cidr, z.Name)
		z.CIDR = cidr
	}

	return nil
}

func (z *ClusterZoneSpec) assignCIDR(c *Cluster) (string, error) {
	// TODO: We probably could query for the existing subnets & allocate appropriately
	// for now we'll require users to set CIDRs themselves

	lastCharMap := make(map[byte]bool)
	for _, nodeZone := range c.Spec.Zones {
		lastChar := nodeZone.Name[len(nodeZone.Name)-1]
		lastCharMap[lastChar] = true
	}

	index := -1

	if len(lastCharMap) == len(c.Spec.Zones) {
		// Last char of zones are unique (GCE, AWS)
		// At least on AWS, we also want 'a' to be 1, so that we don't collide with the lowest range,
		// because kube-up uses that range
		index = int(z.Name[len(z.Name)-1])
	} else {
		glog.Warningf("Last char of zone names not unique")

		for i, nodeZone := range c.Spec.Zones {
			if nodeZone.Name == z.Name {
				index = i
				break
			}
		}
		if index == -1 {
			return "", fmt.Errorf("zone not configured: %q", z.Name)
		}
	}

	_, cidr, err := net.ParseCIDR(c.Spec.NetworkCIDR)
	if err != nil {
		return "", fmt.Errorf("Invalid NetworkCIDR: %q", c.Spec.NetworkCIDR)
	}
	networkLength, _ := cidr.Mask.Size()

	// We assume a maximum of 8 subnets per network
	// TODO: Does this make sense on GCE?
	// TODO: Should we limit this to say 1000 IPs per subnet? (any reason to?)
	index = index % 8
	networkLength += 3

	ip4 := cidr.IP.To4()
	if ip4 != nil {
		n := binary.BigEndian.Uint32(ip4)
		n += uint32(index) << uint(32-networkLength)
		subnetIP := make(net.IP, len(ip4))
		binary.BigEndian.PutUint32(subnetIP, n)
		subnetCIDR := subnetIP.String() + "/" + strconv.Itoa(networkLength)
		glog.V(2).Infof("Computed CIDR for subnet in zone %q as %q", z.Name, subnetCIDR)
		return subnetCIDR, nil
	}

	return "", fmt.Errorf("Unexpected IP address type for NetworkCIDR: %s", c.Spec.NetworkCIDR)
}

// SharedVPC is a simple helper function which makes the templates for a shared VPC clearer
func (c *Cluster) SharedVPC() bool {
	return c.Spec.NetworkID != ""
}

// CloudPermissions holds IAM-style permissions
type CloudPermissions struct {
	S3Buckets []string `json:"s3Buckets,omitempty"`
}

// AddS3Bucket adds a bucket if it does not already exist
func (p *CloudPermissions) AddS3Bucket(bucket string) {
	for _, b := range p.S3Buckets {
		if b == bucket {
			return
		}
	}

	p.S3Buckets = append(p.S3Buckets, bucket)
}

//
//// findImage finds the default image
//func (c*NodeSetConfig) resolveImage() error {
//	cloud.(*awsup.AWSCloud).ResolveImage()
//
//	if n.Image == "" {
//		if defaultImage == "" {
//			image, err := c.determineImage()
//			if err != nil {
//				return err
//			}
//			defaultImage = image
//		}
//		n.Image = defaultImage
//	}
//
//
//	return nil
//}
<|endoftext|>


<commit_before>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"testing"
	"time"
)

func TestIssuesService_List_all(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/issues", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testFormValues(t, r, values{
			"filter":    "all",
			"state":     "closed",
			"labels":    "a,b",
			"sort":      "updated",
			"direction": "asc",
			"since":     "2002-02-10T15:30:00Z",
			"page":      "1",
			"per_page":  "2",
		})
		fmt.Fprint(w, `[{"number":1}]`)
	})

	opt := &IssueListOptions{
		"all", "closed", []string{"a", "b"}, "updated", "asc",
		time.Date(2002, time.February, 10, 15, 30, 0, 0, time.UTC),
		ListOptions{Page: 1, PerPage: 2},
	}
	issues, _, err := client.Issues.List(true, opt)

	if err != nil {
		t.Errorf("Issues.List returned error: %v", err)
	}

	want := []Issue{{Number: Int(1)}}
	if !reflect.DeepEqual(issues, want) {
		t.Errorf("Issues.List returned %+v, want %+v", issues, want)
	}
}

func TestIssuesService_List_owned(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/user/issues", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `[{"number":1}]`)
	})

	issues, _, err := client.Issues.List(false, nil)
	if err != nil {
		t.Errorf("Issues.List returned error: %v", err)
	}

	want := []Issue{{Number: Int(1)}}
	if !reflect.DeepEqual(issues, want) {
		t.Errorf("Issues.List returned %+v, want %+v", issues, want)
	}
}

func TestIssuesService_ListByOrg(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/orgs/o/issues", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `[{"number":1}]`)
	})

	issues, _, err := client.Issues.ListByOrg("o", nil)
	if err != nil {
		t.Errorf("Issues.ListByOrg returned error: %v", err)
	}

	want := []Issue{{Number: Int(1)}}
	if !reflect.DeepEqual(issues, want) {
		t.Errorf("Issues.List returned %+v, want %+v", issues, want)
	}
}

func TestIssuesService_ListByOrg_invalidOrg(t *testing.T) {
	_, _, err := client.Issues.ListByOrg("%", nil)
	testURLParseError(t, err)
}

func TestIssuesService_ListByRepo(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/issues", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testFormValues(t, r, values{
			"milestone": "*",
			"state":     "closed",
			"assignee":  "a",
			"creator":   "c",
			"mentioned": "m",
			"labels":    "a,b",
			"sort":      "updated",
			"direction": "asc",
			"since":     "2002-02-10T15:30:00Z",
		})
		fmt.Fprint(w, `[{"number":1}]`)
	})

	opt := &IssueListByRepoOptions{
		"*", "closed", "a", "c", "m", []string{"a", "b"}, "updated", "asc",
		time.Date(2002, time.February, 10, 15, 30, 0, 0, time.UTC),
		ListOptions{0, 0},
	}
	issues, _, err := client.Issues.ListByRepo("o", "r", opt)
	if err != nil {
		t.Errorf("Issues.ListByOrg returned error: %v", err)
	}

	want := []Issue{{Number: Int(1)}}
	if !reflect.DeepEqual(issues, want) {
		t.Errorf("Issues.List returned %+v, want %+v", issues, want)
	}
}

func TestIssuesService_ListByRepo_invalidOwner(t *testing.T) {
	_, _, err := client.Issues.ListByRepo("%", "r", nil)
	testURLParseError(t, err)
}

func TestIssuesService_Get(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/issues/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"number":1, "labels": [{"url": "u", "name": "n", "color": "c"}]}`)
	})

	issue, _, err := client.Issues.Get("o", "r", 1)
	if err != nil {
		t.Errorf("Issues.Get returned error: %v", err)
	}

	want := &Issue{
		Number: Int(1),
		Labels: []Label{{
			URL:   String("u"),
			Name:  String("n"),
			Color: String("c"),
		}},
	}
	if !reflect.DeepEqual(issue, want) {
		t.Errorf("Issues.Get returned %+v, want %+v", issue, want)
	}
}

func TestIssuesService_Get_invalidOwner(t *testing.T) {
	_, _, err := client.Issues.Get("%", "r", 1)
	testURLParseError(t, err)
}

func TestIssuesService_Create(t *testing.T) {
	setup()
	defer teardown()

	input := &IssueRequest{
		Title:    String("t"),
		Body:     String("b"),
		Assignee: String("a"),
		Labels:   []string{"l1", "l2"},
	}

	mux.HandleFunc("/repos/o/r/issues", func(w http.ResponseWriter, r *http.Request) {
		v := new(IssueRequest)
		json.NewDecoder(r.Body).Decode(v)

		testMethod(t, r, "POST")
		if !reflect.DeepEqual(v, input) {
			t.Errorf("Request body = %+v, want %+v", v, input)
		}

		fmt.Fprint(w, `{"number":1}`)
	})

	issue, _, err := client.Issues.Create("o", "r", input)
	if err != nil {
		t.Errorf("Issues.Create returned error: %v", err)
	}

	want := &Issue{Number: Int(1)}
	if !reflect.DeepEqual(issue, want) {
		t.Errorf("Issues.Create returned %+v, want %+v", issue, want)
	}
}

func TestIssuesService_Create_invalidOwner(t *testing.T) {
	_, _, err := client.Issues.Create("%", "r", nil)
	testURLParseError(t, err)
}

func TestIssuesService_Edit(t *testing.T) {
	setup()
	defer teardown()

	input := &IssueRequest{Title: String("t")}

	mux.HandleFunc("/repos/o/r/issues/1", func(w http.ResponseWriter, r *http.Request) {
		v := new(IssueRequest)
		json.NewDecoder(r.Body).Decode(v)

		testMethod(t, r, "PATCH")
		if !reflect.DeepEqual(v, input) {
			t.Errorf("Request body = %+v, want %+v", v, input)
		}

		fmt.Fprint(w, `{"number":1}`)
	})

	issue, _, err := client.Issues.Edit("o", "r", 1, input)
	if err != nil {
		t.Errorf("Issues.Edit returned error: %v", err)
	}

	want := &Issue{Number: Int(1)}
	if !reflect.DeepEqual(issue, want) {
		t.Errorf("Issues.Edit returned %+v, want %+v", issue, want)
	}
}

func TestIssuesService_Edit_invalidOwner(t *testing.T) {
	_, _, err := client.Issues.Edit("%", "r", 1, nil)
	testURLParseError(t, err)
}
<commit_msg>fix broken test when switching IssuesRequest.Labels to slice pointer<commit_after>// Copyright 2013 The go-github AUTHORS. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package github

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"testing"
	"time"
)

func TestIssuesService_List_all(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/issues", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testFormValues(t, r, values{
			"filter":    "all",
			"state":     "closed",
			"labels":    "a,b",
			"sort":      "updated",
			"direction": "asc",
			"since":     "2002-02-10T15:30:00Z",
			"page":      "1",
			"per_page":  "2",
		})
		fmt.Fprint(w, `[{"number":1}]`)
	})

	opt := &IssueListOptions{
		"all", "closed", []string{"a", "b"}, "updated", "asc",
		time.Date(2002, time.February, 10, 15, 30, 0, 0, time.UTC),
		ListOptions{Page: 1, PerPage: 2},
	}
	issues, _, err := client.Issues.List(true, opt)

	if err != nil {
		t.Errorf("Issues.List returned error: %v", err)
	}

	want := []Issue{{Number: Int(1)}}
	if !reflect.DeepEqual(issues, want) {
		t.Errorf("Issues.List returned %+v, want %+v", issues, want)
	}
}

func TestIssuesService_List_owned(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/user/issues", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `[{"number":1}]`)
	})

	issues, _, err := client.Issues.List(false, nil)
	if err != nil {
		t.Errorf("Issues.List returned error: %v", err)
	}

	want := []Issue{{Number: Int(1)}}
	if !reflect.DeepEqual(issues, want) {
		t.Errorf("Issues.List returned %+v, want %+v", issues, want)
	}
}

func TestIssuesService_ListByOrg(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/orgs/o/issues", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `[{"number":1}]`)
	})

	issues, _, err := client.Issues.ListByOrg("o", nil)
	if err != nil {
		t.Errorf("Issues.ListByOrg returned error: %v", err)
	}

	want := []Issue{{Number: Int(1)}}
	if !reflect.DeepEqual(issues, want) {
		t.Errorf("Issues.List returned %+v, want %+v", issues, want)
	}
}

func TestIssuesService_ListByOrg_invalidOrg(t *testing.T) {
	_, _, err := client.Issues.ListByOrg("%", nil)
	testURLParseError(t, err)
}

func TestIssuesService_ListByRepo(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/issues", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		testFormValues(t, r, values{
			"milestone": "*",
			"state":     "closed",
			"assignee":  "a",
			"creator":   "c",
			"mentioned": "m",
			"labels":    "a,b",
			"sort":      "updated",
			"direction": "asc",
			"since":     "2002-02-10T15:30:00Z",
		})
		fmt.Fprint(w, `[{"number":1}]`)
	})

	opt := &IssueListByRepoOptions{
		"*", "closed", "a", "c", "m", []string{"a", "b"}, "updated", "asc",
		time.Date(2002, time.February, 10, 15, 30, 0, 0, time.UTC),
		ListOptions{0, 0},
	}
	issues, _, err := client.Issues.ListByRepo("o", "r", opt)
	if err != nil {
		t.Errorf("Issues.ListByOrg returned error: %v", err)
	}

	want := []Issue{{Number: Int(1)}}
	if !reflect.DeepEqual(issues, want) {
		t.Errorf("Issues.List returned %+v, want %+v", issues, want)
	}
}

func TestIssuesService_ListByRepo_invalidOwner(t *testing.T) {
	_, _, err := client.Issues.ListByRepo("%", "r", nil)
	testURLParseError(t, err)
}

func TestIssuesService_Get(t *testing.T) {
	setup()
	defer teardown()

	mux.HandleFunc("/repos/o/r/issues/1", func(w http.ResponseWriter, r *http.Request) {
		testMethod(t, r, "GET")
		fmt.Fprint(w, `{"number":1, "labels": [{"url": "u", "name": "n", "color": "c"}]}`)
	})

	issue, _, err := client.Issues.Get("o", "r", 1)
	if err != nil {
		t.Errorf("Issues.Get returned error: %v", err)
	}

	want := &Issue{
		Number: Int(1),
		Labels: []Label{{
			URL:   String("u"),
			Name:  String("n"),
			Color: String("c"),
		}},
	}
	if !reflect.DeepEqual(issue, want) {
		t.Errorf("Issues.Get returned %+v, want %+v", issue, want)
	}
}

func TestIssuesService_Get_invalidOwner(t *testing.T) {
	_, _, err := client.Issues.Get("%", "r", 1)
	testURLParseError(t, err)
}

func TestIssuesService_Create(t *testing.T) {
	setup()
	defer teardown()

	input := &IssueRequest{
		Title:    String("t"),
		Body:     String("b"),
		Assignee: String("a"),
		Labels:   &[]string{"l1", "l2"},
	}

	mux.HandleFunc("/repos/o/r/issues", func(w http.ResponseWriter, r *http.Request) {
		v := new(IssueRequest)
		json.NewDecoder(r.Body).Decode(v)

		testMethod(t, r, "POST")
		if !reflect.DeepEqual(v, input) {
			t.Errorf("Request body = %+v, want %+v", v, input)
		}

		fmt.Fprint(w, `{"number":1}`)
	})

	issue, _, err := client.Issues.Create("o", "r", input)
	if err != nil {
		t.Errorf("Issues.Create returned error: %v", err)
	}

	want := &Issue{Number: Int(1)}
	if !reflect.DeepEqual(issue, want) {
		t.Errorf("Issues.Create returned %+v, want %+v", issue, want)
	}
}

func TestIssuesService_Create_invalidOwner(t *testing.T) {
	_, _, err := client.Issues.Create("%", "r", nil)
	testURLParseError(t, err)
}

func TestIssuesService_Edit(t *testing.T) {
	setup()
	defer teardown()

	input := &IssueRequest{Title: String("t")}

	mux.HandleFunc("/repos/o/r/issues/1", func(w http.ResponseWriter, r *http.Request) {
		v := new(IssueRequest)
		json.NewDecoder(r.Body).Decode(v)

		testMethod(t, r, "PATCH")
		if !reflect.DeepEqual(v, input) {
			t.Errorf("Request body = %+v, want %+v", v, input)
		}

		fmt.Fprint(w, `{"number":1}`)
	})

	issue, _, err := client.Issues.Edit("o", "r", 1, input)
	if err != nil {
		t.Errorf("Issues.Edit returned error: %v", err)
	}

	want := &Issue{Number: Int(1)}
	if !reflect.DeepEqual(issue, want) {
		t.Errorf("Issues.Edit returned %+v, want %+v", issue, want)
	}
}

func TestIssuesService_Edit_invalidOwner(t *testing.T) {
	_, _, err := client.Issues.Edit("%", "r", 1, nil)
	testURLParseError(t, err)
}
<|endoftext|>


<commit_before>package moka_test

import (
	"github.com/gcapizzi/moka"
	. "github.com/gcapizzi/moka/syntax"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Moka", func() {
	var collaborator CollaboratorDouble
	var subject Subject

	BeforeEach(func() {
		collaborator = NewCollaboratorDouble()
		subject = NewSubject(collaborator)
	})

	It("supports allowing a method call on a double", func() {
		AllowDouble(collaborator).To(ReceiveCallTo("Query").With("arg").AndReturn("result"))
		Expect(subject.DelegateQuery("arg")).To(Equal("result"))
	})

	It("supports expecting a method call on a double", func() {
		ExpectDouble(collaborator).To(ReceiveCallTo("Command").With("arg").AndReturn("result"))

		result := subject.DelegateCommand("arg")

		Expect(result).To(Equal("result"))
		VerifyCalls(collaborator)
	})
})

type Collaborator interface {
	Query(string) string
	Command(string) string
}

type CollaboratorDouble struct {
	moka.Double
}

func NewCollaboratorDouble() CollaboratorDouble {
	return CollaboratorDouble{Double: moka.NewStrictDouble()}
}

func (d CollaboratorDouble) Query(arg string) string {
	return d.Call("Query", arg)[0].(string)
}

func (d CollaboratorDouble) Command(arg string) string {
	return d.Call("Command", arg)[0].(string)
}

type Subject struct {
	collaborator Collaborator
}

func NewSubject(collaborator Collaborator) Subject {
	return Subject{collaborator: collaborator}
}

func (s Subject) DelegateQuery(arg string) string {
	return s.collaborator.Query(arg)
}

func (s Subject) DelegateCommand(arg string) string {
	return s.collaborator.Command(arg)
}
<commit_msg>Separate the 'exercise' phase from the 'verify' phase in the AllowDouble() test<commit_after>package moka_test

import (
	"github.com/gcapizzi/moka"
	. "github.com/gcapizzi/moka/syntax"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Moka", func() {
	var collaborator CollaboratorDouble
	var subject Subject

	BeforeEach(func() {
		collaborator = NewCollaboratorDouble()
		subject = NewSubject(collaborator)
	})

	It("supports allowing a method call on a double", func() {
		AllowDouble(collaborator).To(ReceiveCallTo("Query").With("arg").AndReturn("result"))

		result := subject.DelegateQuery("arg")

		Expect(result).To(Equal("result"))
	})

	It("supports expecting a method call on a double", func() {
		ExpectDouble(collaborator).To(ReceiveCallTo("Command").With("arg").AndReturn("result"))

		result := subject.DelegateCommand("arg")

		Expect(result).To(Equal("result"))
		VerifyCalls(collaborator)
	})
})

type Collaborator interface {
	Query(string) string
	Command(string) string
}

type CollaboratorDouble struct {
	moka.Double
}

func NewCollaboratorDouble() CollaboratorDouble {
	return CollaboratorDouble{Double: moka.NewStrictDouble()}
}

func (d CollaboratorDouble) Query(arg string) string {
	return d.Call("Query", arg)[0].(string)
}

func (d CollaboratorDouble) Command(arg string) string {
	return d.Call("Command", arg)[0].(string)
}

type Subject struct {
	collaborator Collaborator
}

func NewSubject(collaborator Collaborator) Subject {
	return Subject{collaborator: collaborator}
}

func (s Subject) DelegateQuery(arg string) string {
	return s.collaborator.Query(arg)
}

func (s Subject) DelegateCommand(arg string) string {
	return s.collaborator.Command(arg)
}
<|endoftext|>


<commit_before>package tee

import (
	"log"
	"os"
)

//-----------------------------------------------------------------------------
// Tee: Copy written data to output file and Stdout / Stderr
//-----------------------------------------------------------------------------

type Tee struct {
	outFile string
	useStdErr bool
	out *os.File
}

// Close Tee
func (t *Tee) Close() error {
	if t.out != nil {
		return t.out.Close()
	}

	return nil
}

// Initialize a Tee
func NewTee(outFile string, useStdErr bool) *Tee {

	// t := &Tee{outFile: outFile, useStdErr: useStdErr}
	t := &Tee{outFile, useStdErr, nil}

	// Copy to STDOUT to file (or to stdout)
	if (outFile == "") || (outFile == "-") {
		t.outFile = ""
	} else {
		out, err := os.Create(outFile)
		if err != nil {
			log.Fatal(err)
		}
		t.out = out
	}

	return t
}

// Write to Tee
func (t *Tee) Write(buf []byte) (n int, err error) {

	if t.out != nil {
		n, err = t.out.Write(buf)
	} else {
		n = len(buf)
		err = nil
	}

	// Also write to stdout / stderr
	if t.useStdErr {
		os.Stderr.Write(buf)
	} else {
		os.Stdout.Write(buf)
	}

	return n, err
}
<commit_msg>Project updated<commit_after>package tee

import (
	"log"
	"os"
)

//-----------------------------------------------------------------------------
// Tee: Copy written data to output file and Stdout / Stderr
//-----------------------------------------------------------------------------

type Tee struct {
	outFile string
	useStdErr bool
	out *os.File
}

// Close Tee
func (t *Tee) Close() error {
	if t.out != nil {
		return t.out.Close()
	}

	return nil
}

// Initialize a Tee
func NewTee(outFile string, useStdErr bool) *Tee {
	t := &Tee{outFile, useStdErr, nil}

	// Copy to STDOUT to file (or to stdout)
	if (outFile == "") || (outFile == "-") {
		t.outFile = ""
	} else {
		out, err := os.Create(outFile)
		if err != nil {
			log.Fatal(err)
		}
		t.out = out
	}

	return t
}

// Write to Tee
func (t *Tee) Write(buf []byte) (n int, err error) {

	if t.out != nil {
		n, err = t.out.Write(buf)
	} else {
		n = len(buf)
		err = nil
	}

	// Also write to stdout / stderr
	if t.useStdErr {
		os.Stderr.Write(buf)
	} else {
		os.Stdout.Write(buf)
	}

	return n, err
}
<|endoftext|>


<commit_before>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"sqlflow.org/sqlflow/go/codegen/optimize"
	"sqlflow.org/sqlflow/go/codegen/pai"
	"sqlflow.org/sqlflow/go/codegen/tensorflow"
	"sqlflow.org/sqlflow/go/codegen/xgboost"
	"sqlflow.org/sqlflow/go/database"
	"sqlflow.org/sqlflow/go/executor"
	"sqlflow.org/sqlflow/go/ir"
	"sqlflow.org/sqlflow/go/log"
	"sqlflow.org/sqlflow/go/parser"
	"sqlflow.org/sqlflow/go/pipe"
	pb "sqlflow.org/sqlflow/go/proto"
)

// EndOfExecution will push to the pipe when one SQL statement execution is finished.
type EndOfExecution struct {
	StartTime int64
	EndTime   int64
	Statement string
}

// RunSQLProgram run a SQL program.
//
// TODO(wangkuiyi): Make RunSQLProgram return an error in addition to
// *pipe.Reader, and remove the calls to log.Printf.
func RunSQLProgram(sqlProgram string, modelDir string, session *pb.Session) *pipe.Reader {
	rd, wr := pipe.Pipe()
	go func() {
		var db *database.DB
		var err error
		defer wr.Close()
		if db, err = database.OpenAndConnectDB(session.DbConnStr); err != nil {
			wr.Write(fmt.Errorf("create DB failed: %v", err))
			return
		}
		defer db.Close()
		err = runSQLProgram(wr, sqlProgram, db, modelDir, session)
		if err != nil {
			if e := wr.Write(fmt.Errorf("runSQLProgram error: %v", err)); e != nil {
				log.GetDefaultLogger().Errorf("runSQLProgram error(piping): %v", e)
			}
		}
	}()
	return rd
}

func initializeAndCheckAttributes(stmt ir.SQLFlowStmt) error {
	switch s := stmt.(type) {
	case *ir.TrainStmt:
		if s.GetModelKind() == ir.XGBoost {
			return xgboost.InitializeAttributes(s)
		} else if s.GetModelKind() == ir.KMeans {
			return pai.InitializeKMeansAttributes(s)
		}
		return tensorflow.InitializeAttributes(s)
	case *ir.OptimizeStmt:
		return optimize.InitializeAttributes(s)
	}
	return nil
}

// ResolveSQLProgram accepts parse result from parser and returns a list of SQLFlowStmt
func ResolveSQLProgram(sqlStmts []*parser.SQLFlowStmt, logger *log.Logger) ([]ir.SQLFlowStmt, error) {
	spIRs := []ir.SQLFlowStmt{}
	var err error
	for _, sql := range sqlStmts {
		var r ir.SQLFlowStmt
		if sql.IsExtendedSyntax() {
			if sql.Train {
				logger.Info("resolveSQL:train")
				r, err = ir.GenerateTrainStmt(sql.SQLFlowSelectStmt)
			} else if sql.ShowTrain {
				logger.Info("resolveSQL:showTrain")
				r, err = ir.GenerateShowTrainStmt(sql.SQLFlowSelectStmt)
			} else if sql.Explain {
				logger.Info("resolveSQL:explain")
				// since getTrainStmtFromModel is false, use empty cwd is fine.
				r, err = ir.GenerateExplainStmt(sql.SQLFlowSelectStmt, "", "", "", false)
			} else if sql.Predict {
				logger.Info("resolveSQL:predict")
				r, err = ir.GeneratePredictStmt(sql.SQLFlowSelectStmt, "", "", "", false)
			} else if sql.Evaluate {
				logger.Info("resolveSQL:evaluate")
				r, err = ir.GenerateEvaluateStmt(sql.SQLFlowSelectStmt, "", "", "", false)
			} else if sql.Optimize {
				logger.Info("resolveSQL:optimize")
				r, err = ir.GenerateOptimizeStmt(sql.SQLFlowSelectStmt)
			} else if sql.Run {
				logger.Info("resolveSQL:run")
				r, err = ir.GenerateRunStmt(sql.SQLFlowSelectStmt)
			} else {
				return nil, fmt.Errorf("unknown extended SQL statement type")
			}
		} else {
			logger.Info("resolveSQL:standard")
			standardSQL := ir.NormalStmt(sql.Original)
			r = &standardSQL
		}
		if err != nil {
			return nil, err
		}
		// TODO(yancey1989): enable the attribute checker when cover pai codegen.
		// if err = initializeAndCheckAttributes(r); err != nil {
		// 	return nil, err
		// }
		r.SetOriginalSQL(sql.Original)
		logger.Infof("Original SQL is:%s", r.GetOriginalSQL())
		spIRs = append(spIRs, r)
	}
	return spIRs, nil
}

func runSQLProgram(wr *pipe.Writer, sqlProgram string, db *database.DB, modelDir string, session *pb.Session) error {
	sqlProgram, err := parser.RemoveCommentInSQLStatement(sqlProgram)
	if err != nil {
		return err
	}

	stmts, err := parser.Parse(db.DriverName, sqlProgram)
	if err != nil {
		return err
	}
	// NOTE(tony): We generate IR and execute its translated program one-by-one since IR generation may depend on the execution
	// of the previous statement. For example, consider a SQL program
	//
	//   create table some_table as (select ...);
	//   select * from some_table to train ...
	//
	// The IR generation on the second statement would fail since it requires inspection the schema of some_table,
	// which depends on the execution of create table some_table as (select ...);.
	sqls := RewriteStatementsWithHints(stmts, db.DriverName)
	for _, sql := range sqls {
		if err := runSingleSQLFlowStatement(wr, sql, db, modelDir, session); err != nil {
			return err
		}
	}
	return nil
}

func runSingleSQLFlowStatement(wr *pipe.Writer, sql *parser.SQLFlowStmt, db *database.DB, modelDir string, session *pb.Session) (e error) {
	defer func(startTime int64) {
		// NOTE(tony): EndOfExecution indicates a successful run,
		// so we only writes it when e != nil
		if e != nil {
			wr.Write(EndOfExecution{
				StartTime: startTime,
				EndTime:   time.Now().UnixNano(),
				Statement: sql.Original,
			})
		}
	}(time.Now().UnixNano())

	// use system default tmp dir
	cwd, err := ioutil.TempDir("/tmp", "sqlflow_models")
	if err != nil {
		return err
	}
	defer func(cwd string) {
		if err := os.RemoveAll(cwd); err != nil {
			e = fmt.Errorf("encounter an error when removing temp files: %v", err)
		}
	}(cwd)
	var r ir.SQLFlowStmt

	generateTrainStmtFromModel := executor.New(session.Submitter).GetTrainStmtFromModel()

	if sql.IsExtendedSyntax() {
		if sql.Train {
			loadPreTrainModel := generateTrainStmtFromModel
			r, err = ir.GenerateTrainStmtWithInferredColumns(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, loadPreTrainModel, true)
		} else if sql.ShowTrain {
			r, err = ir.GenerateShowTrainStmt(sql.SQLFlowSelectStmt)
		} else if sql.Explain {
			r, err = ir.GenerateExplainStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, generateTrainStmtFromModel)
		} else if sql.Predict {
			r, err = ir.GeneratePredictStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, generateTrainStmtFromModel)
		} else if sql.Evaluate {
			r, err = ir.GenerateEvaluateStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, generateTrainStmtFromModel)
		} else if sql.Optimize {
			r, err = ir.GenerateOptimizeStmt(sql.SQLFlowSelectStmt)
		} else if sql.Run {
			r, err = ir.GenerateRunStmt(sql.SQLFlowSelectStmt)
		}

	} else {
		standardSQL := ir.NormalStmt(sql.Original)
		r = &standardSQL
	}
	if err != nil {
		return err
	}
	if err = initializeAndCheckAttributes(r); err != nil {
		return err
	}
	r.SetOriginalSQL(sql.Original)
	// TODO(typhoonzero): can run feature.LogDerivationResult(wr, trainStmt) here to send
	// feature derivation logs to client, yet we disable it for now so that it's less annoying.

	exec := executor.New(session.Submitter)
	exec.Setup(wr, db, modelDir, cwd, session)
	return executor.Run(exec, r)
}

// RewriteStatementsWithHints combines the hints into the standard SQL(s)
//
// FIXME(weiguoz): I'm not happy with such an implementation.
// I mean it is not clean that sqlflow handles such database relative details.
func RewriteStatementsWithHints(stmts []*parser.SQLFlowStmt, dialect string) []*parser.SQLFlowStmt {
	hints, sqls := splitHints(stmts, dialect)
	if len(hints) > 0 {
		for _, sql := range sqls {
			if !sql.IsExtendedSyntax() {
				sql.Original = hints + sql.Original
			}
		}
	}
	return sqls
}

func splitHints(stmts []*parser.SQLFlowStmt, dialect string) (string, []*parser.SQLFlowStmt) {
	hints, sqls := "", []*parser.SQLFlowStmt{}
	for _, stmt := range stmts {
		if isHint(stmt, dialect) {
			hints += stmt.Original + "\n" // alisa's requirements
		} else {
			sqls = append(sqls, stmt)
		}
	}
	return hints, sqls
}

func isHint(stmt *parser.SQLFlowStmt, dialect string) bool {
	if !stmt.IsExtendedSyntax() {
		if dialect == "alisa" {
			return isAlisaHint(stmt.Original)
		}
		// TODO(weiguoz) handle if submitter is "maxcompute" or "hive"
	}
	return false
}

func isAlisaHint(sql string) bool {
	for {
		sql = strings.TrimSpace(sql)
		// TODO(weiguoz): Let's remove the following code if we clean the comments before
		if strings.HasPrefix(sql, "--") {
			eol := strings.IndexAny(sql, "\n\r")
			if eol != -1 {
				sql = sql[eol+1:]
			} else {
				break
			}
		} else {
			break
		}
	}
	return strings.HasPrefix(strings.ToLower(sql), "set ")
}
<commit_msg>Polish code (#2950)<commit_after>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"sqlflow.org/sqlflow/go/codegen/optimize"
	"sqlflow.org/sqlflow/go/codegen/pai"
	"sqlflow.org/sqlflow/go/codegen/tensorflow"
	"sqlflow.org/sqlflow/go/codegen/xgboost"
	"sqlflow.org/sqlflow/go/database"
	"sqlflow.org/sqlflow/go/executor"
	"sqlflow.org/sqlflow/go/ir"
	"sqlflow.org/sqlflow/go/log"
	"sqlflow.org/sqlflow/go/parser"
	"sqlflow.org/sqlflow/go/pipe"
	pb "sqlflow.org/sqlflow/go/proto"
)

// EndOfExecution will push to the pipe when one SQL statement execution is finished.
type EndOfExecution struct {
	StartTime int64
	EndTime   int64
	Statement string
}

// RunSQLProgram run a SQL program.
//
// TODO(wangkuiyi): Make RunSQLProgram return an error in addition to
// *pipe.Reader, and remove the calls to log.Printf.
func RunSQLProgram(sqlProgram string, modelDir string, session *pb.Session) *pipe.Reader {
	rd, wr := pipe.Pipe()
	go func() {
		var db *database.DB
		var err error
		defer wr.Close()
		if db, err = database.OpenAndConnectDB(session.DbConnStr); err != nil {
			wr.Write(fmt.Errorf("create DB failed: %v", err))
			return
		}
		defer db.Close()
		err = runSQLProgram(wr, sqlProgram, db, modelDir, session)
		if err != nil {
			if e := wr.Write(fmt.Errorf("runSQLProgram error: %v", err)); e != nil {
				log.GetDefaultLogger().Errorf("runSQLProgram error(piping): %v", e)
			}
		}
	}()
	return rd
}

func initializeAndCheckAttributes(stmt ir.SQLFlowStmt) error {
	switch s := stmt.(type) {
	case *ir.TrainStmt:
		if s.GetModelKind() == ir.XGBoost {
			return xgboost.InitializeAttributes(s)
		} else if s.GetModelKind() == ir.KMeans {
			return pai.InitializeKMeansAttributes(s)
		}
		return tensorflow.InitializeAttributes(s)
	case *ir.OptimizeStmt:
		return optimize.InitializeAttributes(s)
	}
	return nil
}

// ResolveSQLProgram accepts parse result from parser and returns a list of SQLFlowStmt
func ResolveSQLProgram(sqlStmts []*parser.SQLFlowStmt, logger *log.Logger) ([]ir.SQLFlowStmt, error) {
	spIRs := []ir.SQLFlowStmt{}
	var err error
	for _, sql := range sqlStmts {
		var r ir.SQLFlowStmt
		if sql.IsExtendedSyntax() {
			if sql.Train {
				logger.Info("resolveSQL:train")
				r, err = ir.GenerateTrainStmt(sql.SQLFlowSelectStmt)
			} else if sql.ShowTrain {
				logger.Info("resolveSQL:showTrain")
				r, err = ir.GenerateShowTrainStmt(sql.SQLFlowSelectStmt)
			} else if sql.Explain {
				logger.Info("resolveSQL:explain")
				// since getTrainStmtFromModel is false, use empty cwd is fine.
				r, err = ir.GenerateExplainStmt(sql.SQLFlowSelectStmt, "", "", "", false)
			} else if sql.Predict {
				logger.Info("resolveSQL:predict")
				r, err = ir.GeneratePredictStmt(sql.SQLFlowSelectStmt, "", "", "", false)
			} else if sql.Evaluate {
				logger.Info("resolveSQL:evaluate")
				r, err = ir.GenerateEvaluateStmt(sql.SQLFlowSelectStmt, "", "", "", false)
			} else if sql.Optimize {
				logger.Info("resolveSQL:optimize")
				r, err = ir.GenerateOptimizeStmt(sql.SQLFlowSelectStmt)
			} else if sql.Run {
				logger.Info("resolveSQL:run")
				r, err = ir.GenerateRunStmt(sql.SQLFlowSelectStmt)
			} else {
				return nil, fmt.Errorf("unknown extended SQL statement type")
			}
		} else {
			logger.Info("resolveSQL:standard")
			standardSQL := ir.NormalStmt(sql.Original)
			r = &standardSQL
		}
		if err != nil {
			return nil, err
		}
		// TODO(yancey1989): enable the attribute checker when cover pai codegen.
		// if err = initializeAndCheckAttributes(r); err != nil {
		// 	return nil, err
		// }
		r.SetOriginalSQL(sql.Original)
		logger.Infof("Original SQL is:%s", r.GetOriginalSQL())
		spIRs = append(spIRs, r)
	}
	return spIRs, nil
}

func runSQLProgram(wr *pipe.Writer, sqlProgram string, db *database.DB, modelDir string, session *pb.Session) error {
	sqlProgram, err := parser.RemoveCommentInSQLStatement(sqlProgram)
	if err != nil {
		return err
	}

	stmts, err := parser.Parse(db.DriverName, sqlProgram)
	if err != nil {
		return err
	}
	// NOTE(tony): We generate IR and execute its translated program one-by-one since IR generation may depend on the execution
	// of the previous statement. For example, consider a SQL program
	//
	//   create table some_table as (select ...);
	//   select * from some_table to train ...
	//
	// The IR generation on the second statement would fail since it requires inspection the schema of some_table,
	// which depends on the execution of create table some_table as (select ...);.
	sqls := RewriteStatementsWithHints(stmts, db.DriverName)
	for _, sql := range sqls {
		if err := runSingleSQLFlowStatement(wr, sql, db, modelDir, session); err != nil {
			return err
		}
	}
	return nil
}

func runSingleSQLFlowStatement(wr *pipe.Writer, sql *parser.SQLFlowStmt, db *database.DB, modelDir string, session *pb.Session) (e error) {
	defer func(startTime int64) {
		// NOTE(tony): EndOfExecution indicates a successful run,
		// so we only writes it when e != nil
		if e != nil {
			wr.Write(EndOfExecution{
				StartTime: startTime,
				EndTime:   time.Now().UnixNano(),
				Statement: sql.Original,
			})
		}
	}(time.Now().UnixNano())

	// use system default tmp dir
	cwd, err := ioutil.TempDir("/tmp", "sqlflow_models")
	if err != nil {
		return err
	}
	defer func(cwd string) {
		if err := os.RemoveAll(cwd); err != nil {
			e = fmt.Errorf("encounter an error when removing temp files: %v", err)
		}
	}(cwd)

	var r ir.SQLFlowStmt
	if sql.IsExtendedSyntax() {
		generateTrainStmtFromModel := executor.New(session.Submitter).GetTrainStmtFromModel()
		if sql.Train {
			// generateTrainStmtFromModel refers to if a pre-trained model
			r, err = ir.GenerateTrainStmtWithInferredColumns(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, generateTrainStmtFromModel, true)
		} else if sql.ShowTrain {
			r, err = ir.GenerateShowTrainStmt(sql.SQLFlowSelectStmt)
		} else if sql.Explain {
			r, err = ir.GenerateExplainStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, generateTrainStmtFromModel)
		} else if sql.Predict {
			r, err = ir.GeneratePredictStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, generateTrainStmtFromModel)
		} else if sql.Evaluate {
			r, err = ir.GenerateEvaluateStmt(sql.SQLFlowSelectStmt, session.DbConnStr, modelDir, cwd, generateTrainStmtFromModel)
		} else if sql.Optimize {
			r, err = ir.GenerateOptimizeStmt(sql.SQLFlowSelectStmt)
		} else if sql.Run {
			r, err = ir.GenerateRunStmt(sql.SQLFlowSelectStmt)
		}
	} else {
		standardSQL := ir.NormalStmt(sql.Original)
		r = &standardSQL
	}
	if err != nil {
		return err
	}
	if err = initializeAndCheckAttributes(r); err != nil {
		return err
	}
	r.SetOriginalSQL(sql.Original)
	// TODO(typhoonzero): can run feature.LogDerivationResult(wr, trainStmt) here to send
	// feature derivation logs to client, yet we disable it for now so that it's less annoying.

	exec := executor.New(session.Submitter)
	exec.Setup(wr, db, modelDir, cwd, session)
	return executor.Run(exec, r)
}

// RewriteStatementsWithHints combines the hints into the standard SQL(s)
//
// FIXME(weiguoz): I'm not happy with such an implementation.
// I mean it is not clean that sqlflow handles such database relative details.
func RewriteStatementsWithHints(stmts []*parser.SQLFlowStmt, dialect string) []*parser.SQLFlowStmt {
	hints, sqls := splitHints(stmts, dialect)
	if len(hints) > 0 {
		for _, sql := range sqls {
			if !sql.IsExtendedSyntax() {
				sql.Original = hints + sql.Original
			}
		}
	}
	return sqls
}

func splitHints(stmts []*parser.SQLFlowStmt, dialect string) (string, []*parser.SQLFlowStmt) {
	hints, sqls := "", []*parser.SQLFlowStmt{}
	for _, stmt := range stmts {
		if isHint(stmt, dialect) {
			hints += stmt.Original + "\n" // alisa's requirements
		} else {
			sqls = append(sqls, stmt)
		}
	}
	return hints, sqls
}

func isHint(stmt *parser.SQLFlowStmt, dialect string) bool {
	if !stmt.IsExtendedSyntax() {
		if dialect == "alisa" {
			return isAlisaHint(stmt.Original)
		}
		// TODO(weiguoz) handle if submitter is "maxcompute" or "hive"
	}
	return false
}

func isAlisaHint(sql string) bool {
	for {
		sql = strings.TrimSpace(sql)
		// TODO(weiguoz): Let's remove the following code if we clean the comments before
		if strings.HasPrefix(sql, "--") {
			eol := strings.IndexAny(sql, "\n\r")
			if eol != -1 {
				sql = sql[eol+1:]
			} else {
				break
			}
		} else {
			break
		}
	}
	return strings.HasPrefix(strings.ToLower(sql), "set ")
}
<|endoftext|>


<commit_before>/*
Copyright (c) 2017 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rdm

import (
	"context"
	"flag"
	"fmt"
	"strings"

	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/types"
)

type attach struct {
	*flags.VirtualMachineFlag

	device string
}

func init() {
	cli.Register("vm.rdm.attach", &attach{})
}

func (cmd *attach) Register(ctx context.Context, f *flag.FlagSet) {

	cmd.VirtualMachineFlag, ctx = flags.NewVirtualMachineFlag(ctx)
	cmd.VirtualMachineFlag.Register(ctx, f)

	f.StringVar(&cmd.device, "device", "", "Device Name")
}

func (cmd *attach) Description() string {
	return `Attach DEVICE to VM with RDM.

Examples:
  govc vm.rdm.attach -vm VM -device /vmfs/devices/disks/naa.000000000000000000000000000000000`
}

func (cmd *attach) Process(ctx context.Context) error {
	if err := cmd.VirtualMachineFlag.Process(ctx); err != nil {
		return err
	}
	return nil
}

//This piece of code was developped mainly thanks to the project govmax on github.com
//This file in particular https://github.com/codedellemc/govmax/blob/master/api/v1/vmomi.go
func (cmd *attach) Run(ctx context.Context, f *flag.FlagSet) error {
	vm, err := cmd.VirtualMachine()
	if err != nil {
		return err
	}

	if vm == nil {
		return flag.ErrHelp
	}

	devices, err := vm.Device(ctx)
	if err != nil {
		return err
	}

	controller, err := devices.FindSCSIController("")
	if err != nil {
		return err
	}

	vmConfigOptions, err := vm.QueryConfigTarget(ctx)
	if err != nil {
		return err
	}

	for _, scsiDisk := range vmConfigOptions.ScsiDisk {
		if !strings.Contains(scsiDisk.Disk.CanonicalName, cmd.device) {
			continue
		}
		var backing types.VirtualDiskRawDiskMappingVer1BackingInfo
		backing.CompatibilityMode = string(types.VirtualDiskCompatibilityModePhysicalMode)
		backing.DeviceName = scsiDisk.Disk.DeviceName
		for _, descriptor := range scsiDisk.Disk.Descriptor {
			if strings.HasPrefix(descriptor.Id, "vml.") {
				backing.LunUuid = descriptor.Id
				break
			}
		}
		var device types.VirtualDisk
		device.Backing = &backing
		device.ControllerKey = controller.VirtualController.Key

		var unitNumber *int32
		scsiCtrlUnitNumber := controller.VirtualController.UnitNumber
		var u int32
		for u = 0; u < 16; u++ {
			free := true
			for _, d := range devices {
				if d.GetVirtualDevice().ControllerKey == device.GetVirtualDevice().ControllerKey {
					if u == *(d.GetVirtualDevice().UnitNumber) || u == *scsiCtrlUnitNumber {
						free = false
					}
				}
			}
			if free {
				unitNumber = &u
				break
			}
		}
		device.UnitNumber = unitNumber

		spec := types.VirtualMachineConfigSpec{}

		config := &types.VirtualDeviceConfigSpec{
			Device:    &device,
			Operation: types.VirtualDeviceConfigSpecOperationAdd,
		}

		config.FileOperation = types.VirtualDeviceConfigSpecFileOperationCreate

		spec.DeviceChange = append(spec.DeviceChange, config)

		task, err := vm.Reconfigure(ctx, spec)
		if err != nil {
			return err
		}

		err = task.Wait(ctx)
		if err != nil {
			return fmt.Errorf("Error adding device %+v \n with backing %+v \nLogged Item:  %s", device, backing, err)
		}
		return nil

	}
	return fmt.Errorf("Error: No LUN with device name containing %s found", cmd.device)
}
<commit_msg>Fix typo<commit_after>/*
Copyright (c) 2017 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rdm

import (
	"context"
	"flag"
	"fmt"
	"strings"

	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/govc/flags"
	"github.com/vmware/govmomi/vim25/types"
)

type attach struct {
	*flags.VirtualMachineFlag

	device string
}

func init() {
	cli.Register("vm.rdm.attach", &attach{})
}

func (cmd *attach) Register(ctx context.Context, f *flag.FlagSet) {

	cmd.VirtualMachineFlag, ctx = flags.NewVirtualMachineFlag(ctx)
	cmd.VirtualMachineFlag.Register(ctx, f)

	f.StringVar(&cmd.device, "device", "", "Device Name")
}

func (cmd *attach) Description() string {
	return `Attach DEVICE to VM with RDM.

Examples:
  govc vm.rdm.attach -vm VM -device /vmfs/devices/disks/naa.000000000000000000000000000000000`
}

func (cmd *attach) Process(ctx context.Context) error {
	if err := cmd.VirtualMachineFlag.Process(ctx); err != nil {
		return err
	}
	return nil
}

//This piece of code was developed mainly thanks to the project govmax on github.com
//This file in particular https://github.com/codedellemc/govmax/blob/master/api/v1/vmomi.go
func (cmd *attach) Run(ctx context.Context, f *flag.FlagSet) error {
	vm, err := cmd.VirtualMachine()
	if err != nil {
		return err
	}

	if vm == nil {
		return flag.ErrHelp
	}

	devices, err := vm.Device(ctx)
	if err != nil {
		return err
	}

	controller, err := devices.FindSCSIController("")
	if err != nil {
		return err
	}

	vmConfigOptions, err := vm.QueryConfigTarget(ctx)
	if err != nil {
		return err
	}

	for _, scsiDisk := range vmConfigOptions.ScsiDisk {
		if !strings.Contains(scsiDisk.Disk.CanonicalName, cmd.device) {
			continue
		}
		var backing types.VirtualDiskRawDiskMappingVer1BackingInfo
		backing.CompatibilityMode = string(types.VirtualDiskCompatibilityModePhysicalMode)
		backing.DeviceName = scsiDisk.Disk.DeviceName
		for _, descriptor := range scsiDisk.Disk.Descriptor {
			if strings.HasPrefix(descriptor.Id, "vml.") {
				backing.LunUuid = descriptor.Id
				break
			}
		}
		var device types.VirtualDisk
		device.Backing = &backing
		device.ControllerKey = controller.VirtualController.Key

		var unitNumber *int32
		scsiCtrlUnitNumber := controller.VirtualController.UnitNumber
		var u int32
		for u = 0; u < 16; u++ {
			free := true
			for _, d := range devices {
				if d.GetVirtualDevice().ControllerKey == device.GetVirtualDevice().ControllerKey {
					if u == *(d.GetVirtualDevice().UnitNumber) || u == *scsiCtrlUnitNumber {
						free = false
					}
				}
			}
			if free {
				unitNumber = &u
				break
			}
		}
		device.UnitNumber = unitNumber

		spec := types.VirtualMachineConfigSpec{}

		config := &types.VirtualDeviceConfigSpec{
			Device:    &device,
			Operation: types.VirtualDeviceConfigSpecOperationAdd,
		}

		config.FileOperation = types.VirtualDeviceConfigSpecFileOperationCreate

		spec.DeviceChange = append(spec.DeviceChange, config)

		task, err := vm.Reconfigure(ctx, spec)
		if err != nil {
			return err
		}

		err = task.Wait(ctx)
		if err != nil {
			return fmt.Errorf("Error adding device %+v \n with backing %+v \nLogged Item:  %s", device, backing, err)
		}
		return nil

	}
	return fmt.Errorf("Error: No LUN with device name containing %s found", cmd.device)
}
<|endoftext|>