<commit_before>package chanrpc

import (
	"errors"
	"fmt"
	"github.com/name5566/leaf/conf"
	"github.com/name5566/leaf/log"
	"runtime"
)

// one server per goroutine (goroutine not safe)
// one client per goroutine (goroutine not safe)
type Server struct {
	// id -> function
	//
	// function:
	// func(args []interface{})
	// func(args []interface{}) interface{}
	// func(args []interface{}) []interface{}
	functions map[interface{}]interface{}
	ChanCall  chan *CallInfo
}

type CallInfo struct {
	f       interface{}
	args    []interface{}
	chanRet chan *RetInfo
	cb      interface{}
}

type RetInfo struct {
	// nil
	// interface{}
	// []interface{}
	ret interface{}
	err error
	// callback:
	// func(err error)
	// func(ret interface{}, err error)
	// func(ret []interface{}, err error)
	cb interface{}
}

type Client struct {
	s               *Server
	chanSyncRet     chan *RetInfo
	ChanAsynRet     chan *RetInfo
	pendingAsynCall int
}

func NewServer(l int) *Server {
	s := new(Server)
	s.functions = make(map[interface{}]interface{})
	s.ChanCall = make(chan *CallInfo, l)
	return s
}

func assert(i interface{}) []interface{} {
	if i == nil {
		return nil
	} else {
		return i.([]interface{})
	}
}

// you must call the function before calling Open and Go
func (s *Server) Register(id interface{}, f interface{}) {
	switch f.(type) {
	case func([]interface{}):
	case func([]interface{}) interface{}:
	case func([]interface{}) []interface{}:
	default:
		panic(fmt.Sprintf("function id %v: definition of function is invalid", id))
	}

	if _, ok := s.functions[id]; ok {
		panic(fmt.Sprintf("function id %v: already registered", id))
	}

	s.functions[id] = f
}

func (s *Server) ret(ci *CallInfo, ri *RetInfo) (err error) {
	if ci.chanRet == nil {
		return
	}

	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	ri.cb = ci.cb
	ci.chanRet <- ri
	return
}

func (s *Server) exec(ci *CallInfo) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if conf.LenStackBuf > 0 {
				buf := make([]byte, conf.LenStackBuf)
				l := runtime.Stack(buf, false)
				err = fmt.Errorf("%v: %s", r, buf[:l])
			} else {
				err = fmt.Errorf("%v", r)
			}

			s.ret(ci, &RetInfo{err: fmt.Errorf("%v", r)})
		}
	}()

	// execute
	switch ci.f.(type) {
	case func([]interface{}):
		ci.f.(func([]interface{}))(ci.args)
		return s.ret(ci, &RetInfo{})
	case func([]interface{}) interface{}:
		ret := ci.f.(func([]interface{}) interface{})(ci.args)
		return s.ret(ci, &RetInfo{ret: ret})
	case func([]interface{}) []interface{}:
		ret := ci.f.(func([]interface{}) []interface{})(ci.args)
		return s.ret(ci, &RetInfo{ret: ret})
	}

	panic("bug")
}

func (s *Server) Exec(ci *CallInfo) {
	err := s.exec(ci)
	if err != nil {
		log.Error("%v", err)
	}
}

// goroutine safe
func (s *Server) Go(id interface{}, args ...interface{}) {
	f := s.functions[id]
	if f == nil {
		return
	}

	defer recover()

	s.ChanCall <- &CallInfo{
		f:    f,
		args: args,
	}
}

// goroutine safe
func (s *Server) Call0(id interface{}, args ...interface{}) error {
	return s.Open(0).Call0(id, args...)
}

// goroutine safe
func (s *Server) Call1(id interface{}, args ...interface{}) (interface{}, error) {
	return s.Open(0).Call1(id, args...)
}

// goroutine safe
func (s *Server) CallN(id interface{}, args ...interface{}) ([]interface{}, error) {
	return s.Open(0).CallN(id, args...)
}

func (s *Server) Close() {
	close(s.ChanCall)

	for ci := range s.ChanCall {
		s.ret(ci, &RetInfo{
			err: errors.New("chanrpc server closed"),
		})
	}
}

// goroutine safe
func (s *Server) Open(l int) *Client {
	c := NewClient(l)
	c.Attach(s)
	return c
}

func NewClient(l int) *Client {
	c := new(Client)
	c.chanSyncRet = make(chan *RetInfo, 1)
	c.ChanAsynRet = make(chan *RetInfo, l)
	return c
}

func (c *Client) Attach(s *Server) {
	c.s = s
}

func (c *Client) call(ci *CallInfo, block bool) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	if block {
		c.s.ChanCall <- ci
	} else {
		select {
		case c.s.ChanCall <- ci:
		default:
			err = errors.New("chanrpc channel full")
		}
	}
	return
}

func (c *Client) f(id interface{}, n int) (f interface{}, err error) {
	if c.s == nil {
		err = errors.New("server not attached")
		return
	}

	f = c.s.functions[id]
	if f == nil {
		err = fmt.Errorf("function id %v: function not registered", id)
		return
	}

	var ok bool
	switch n {
	case 0:
		_, ok = f.(func([]interface{}))
	case 1:
		_, ok = f.(func([]interface{}) interface{})
	case 2:
		_, ok = f.(func([]interface{}) []interface{})
	default:
		panic("bug")
	}

	if !ok {
		err = fmt.Errorf("function id %v: return type mismatch", id)
	}
	return
}

func (c *Client) Call0(id interface{}, args ...interface{}) error {
	f, err := c.f(id, 0)
	if err != nil {
		return err
	}

	err = c.call(&CallInfo{
		f:       f,
		args:    args,
		chanRet: c.chanSyncRet,
	}, true)
	if err != nil {
		return err
	}

	ri := <-c.chanSyncRet
	return ri.err
}

func (c *Client) Call1(id interface{}, args ...interface{}) (interface{}, error) {
	f, err := c.f(id, 1)
	if err != nil {
		return nil, err
	}

	err = c.call(&CallInfo{
		f:       f,
		args:    args,
		chanRet: c.chanSyncRet,
	}, true)
	if err != nil {
		return nil, err
	}

	ri := <-c.chanSyncRet
	return ri.ret, ri.err
}

func (c *Client) CallN(id interface{}, args ...interface{}) ([]interface{}, error) {
	f, err := c.f(id, 2)
	if err != nil {
		return nil, err
	}

	err = c.call(&CallInfo{
		f:       f,
		args:    args,
		chanRet: c.chanSyncRet,
	}, true)
	if err != nil {
		return nil, err
	}

	ri := <-c.chanSyncRet
	return assert(ri.ret), ri.err
}

func (c *Client) asynCall(id interface{}, args []interface{}, cb interface{}, n int) {
	f, err := c.f(id, n)
	if err != nil {
		c.ChanAsynRet <- &RetInfo{err: err, cb: cb}
		return
	}

	err = c.call(&CallInfo{
		f:       f,
		args:    args,
		chanRet: c.ChanAsynRet,
		cb:      cb,
	}, false)
	if err != nil {
		c.ChanAsynRet <- &RetInfo{err: err, cb: cb}
		return
	}
}

func (c *Client) AsynCall(id interface{}, _args ...interface{}) {
	if len(_args) < 1 {
		panic("callback function not found")
	}

	args := _args[:len(_args)-1]
	cb := _args[len(_args)-1]

	var n int
	switch cb.(type) {
	case func(error):
		n = 0
	case func(interface{}, error):
		n = 1
	case func([]interface{}, error):
		n = 2
	default:
		panic("definition of callback function is invalid")
	}

	// too many calls
	if c.pendingAsynCall >= cap(c.ChanAsynRet) {
		execCb(&RetInfo{err: errors.New("too many calls"), cb: cb})
		return
	}

	c.asynCall(id, args, cb, n)
	c.pendingAsynCall++
}

func execCb(ri *RetInfo) {
	defer func() {
		if r := recover(); r != nil {
			if conf.LenStackBuf > 0 {
				buf := make([]byte, conf.LenStackBuf)
				l := runtime.Stack(buf, false)
				log.Error("%v: %s", r, buf[:l])
			} else {
				log.Error("%v", r)
			}
		}
	}()

	// execute
	switch ri.cb.(type) {
	case func(error):
		ri.cb.(func(error))(ri.err)
	case func(interface{}, error):
		ri.cb.(func(interface{}, error))(ri.ret, ri.err)
	case func([]interface{}, error):
		ri.cb.(func([]interface{}, error))(assert(ri.ret), ri.err)
	default:
		panic("bug")
	}
	return
}

func (c *Client) Cb(ri *RetInfo) {
	c.pendingAsynCall--
	execCb(ri)
}

func (c *Client) Close() {
	for c.pendingAsynCall > 0 {
		c.Cb(<-c.ChanAsynRet)
	}
}

func (c *Client) Idle() bool {
	return c.pendingAsynCall == 0
}
<commit_msg>fix recover bugs.<commit_after>package chanrpc

import (
	"errors"
	"fmt"
	"github.com/name5566/leaf/conf"
	"github.com/name5566/leaf/log"
	"runtime"
)

// one server per goroutine (goroutine not safe)
// one client per goroutine (goroutine not safe)
type Server struct {
	// id -> function
	//
	// function:
	// func(args []interface{})
	// func(args []interface{}) interface{}
	// func(args []interface{}) []interface{}
	functions map[interface{}]interface{}
	ChanCall  chan *CallInfo
}

type CallInfo struct {
	f       interface{}
	args    []interface{}
	chanRet chan *RetInfo
	cb      interface{}
}

type RetInfo struct {
	// nil
	// interface{}
	// []interface{}
	ret interface{}
	err error
	// callback:
	// func(err error)
	// func(ret interface{}, err error)
	// func(ret []interface{}, err error)
	cb interface{}
}

type Client struct {
	s               *Server
	chanSyncRet     chan *RetInfo
	ChanAsynRet     chan *RetInfo
	pendingAsynCall int
}

func NewServer(l int) *Server {
	s := new(Server)
	s.functions = make(map[interface{}]interface{})
	s.ChanCall = make(chan *CallInfo, l)
	return s
}

func assert(i interface{}) []interface{} {
	if i == nil {
		return nil
	} else {
		return i.([]interface{})
	}
}

// you must call the function before calling Open and Go
func (s *Server) Register(id interface{}, f interface{}) {
	switch f.(type) {
	case func([]interface{}):
	case func([]interface{}) interface{}:
	case func([]interface{}) []interface{}:
	default:
		panic(fmt.Sprintf("function id %v: definition of function is invalid", id))
	}

	if _, ok := s.functions[id]; ok {
		panic(fmt.Sprintf("function id %v: already registered", id))
	}

	s.functions[id] = f
}

func (s *Server) ret(ci *CallInfo, ri *RetInfo) (err error) {
	if ci.chanRet == nil {
		return
	}

	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	ri.cb = ci.cb
	ci.chanRet <- ri
	return
}

func (s *Server) exec(ci *CallInfo) (err error) {
	defer func() {
		if r := recover(); r != nil {
			if conf.LenStackBuf > 0 {
				buf := make([]byte, conf.LenStackBuf)
				l := runtime.Stack(buf, false)
				err = fmt.Errorf("%v: %s", r, buf[:l])
			} else {
				err = fmt.Errorf("%v", r)
			}

			s.ret(ci, &RetInfo{err: fmt.Errorf("%v", r)})
		}
	}()

	// execute
	switch ci.f.(type) {
	case func([]interface{}):
		ci.f.(func([]interface{}))(ci.args)
		return s.ret(ci, &RetInfo{})
	case func([]interface{}) interface{}:
		ret := ci.f.(func([]interface{}) interface{})(ci.args)
		return s.ret(ci, &RetInfo{ret: ret})
	case func([]interface{}) []interface{}:
		ret := ci.f.(func([]interface{}) []interface{})(ci.args)
		return s.ret(ci, &RetInfo{ret: ret})
	}

	panic("bug")
}

func (s *Server) Exec(ci *CallInfo) {
	err := s.exec(ci)
	if err != nil {
		log.Error("%v", err)
	}
}

// goroutine safe
func (s *Server) Go(id interface{}, args ...interface{}) {
	f := s.functions[id]
	if f == nil {
		return
	}

	defer func() {
		recover()
	}()

	s.ChanCall <- &CallInfo{
		f:    f,
		args: args,
	}
}

// goroutine safe
func (s *Server) Call0(id interface{}, args ...interface{}) error {
	return s.Open(0).Call0(id, args...)
}

// goroutine safe
func (s *Server) Call1(id interface{}, args ...interface{}) (interface{}, error) {
	return s.Open(0).Call1(id, args...)
}

// goroutine safe
func (s *Server) CallN(id interface{}, args ...interface{}) ([]interface{}, error) {
	return s.Open(0).CallN(id, args...)
}

func (s *Server) Close() {
	close(s.ChanCall)

	for ci := range s.ChanCall {
		s.ret(ci, &RetInfo{
			err: errors.New("chanrpc server closed"),
		})
	}
}

// goroutine safe
func (s *Server) Open(l int) *Client {
	c := NewClient(l)
	c.Attach(s)
	return c
}

func NewClient(l int) *Client {
	c := new(Client)
	c.chanSyncRet = make(chan *RetInfo, 1)
	c.ChanAsynRet = make(chan *RetInfo, l)
	return c
}

func (c *Client) Attach(s *Server) {
	c.s = s
}

func (c *Client) call(ci *CallInfo, block bool) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = r.(error)
		}
	}()

	if block {
		c.s.ChanCall <- ci
	} else {
		select {
		case c.s.ChanCall <- ci:
		default:
			err = errors.New("chanrpc channel full")
		}
	}
	return
}

func (c *Client) f(id interface{}, n int) (f interface{}, err error) {
	if c.s == nil {
		err = errors.New("server not attached")
		return
	}

	f = c.s.functions[id]
	if f == nil {
		err = fmt.Errorf("function id %v: function not registered", id)
		return
	}

	var ok bool
	switch n {
	case 0:
		_, ok = f.(func([]interface{}))
	case 1:
		_, ok = f.(func([]interface{}) interface{})
	case 2:
		_, ok = f.(func([]interface{}) []interface{})
	default:
		panic("bug")
	}

	if !ok {
		err = fmt.Errorf("function id %v: return type mismatch", id)
	}
	return
}

func (c *Client) Call0(id interface{}, args ...interface{}) error {
	f, err := c.f(id, 0)
	if err != nil {
		return err
	}

	err = c.call(&CallInfo{
		f:       f,
		args:    args,
		chanRet: c.chanSyncRet,
	}, true)
	if err != nil {
		return err
	}

	ri := <-c.chanSyncRet
	return ri.err
}

func (c *Client) Call1(id interface{}, args ...interface{}) (interface{}, error) {
	f, err := c.f(id, 1)
	if err != nil {
		return nil, err
	}

	err = c.call(&CallInfo{
		f:       f,
		args:    args,
		chanRet: c.chanSyncRet,
	}, true)
	if err != nil {
		return nil, err
	}

	ri := <-c.chanSyncRet
	return ri.ret, ri.err
}

func (c *Client) CallN(id interface{}, args ...interface{}) ([]interface{}, error) {
	f, err := c.f(id, 2)
	if err != nil {
		return nil, err
	}

	err = c.call(&CallInfo{
		f:       f,
		args:    args,
		chanRet: c.chanSyncRet,
	}, true)
	if err != nil {
		return nil, err
	}

	ri := <-c.chanSyncRet
	return assert(ri.ret), ri.err
}

func (c *Client) asynCall(id interface{}, args []interface{}, cb interface{}, n int) {
	f, err := c.f(id, n)
	if err != nil {
		c.ChanAsynRet <- &RetInfo{err: err, cb: cb}
		return
	}

	err = c.call(&CallInfo{
		f:       f,
		args:    args,
		chanRet: c.ChanAsynRet,
		cb:      cb,
	}, false)
	if err != nil {
		c.ChanAsynRet <- &RetInfo{err: err, cb: cb}
		return
	}
}

func (c *Client) AsynCall(id interface{}, _args ...interface{}) {
	if len(_args) < 1 {
		panic("callback function not found")
	}

	args := _args[:len(_args)-1]
	cb := _args[len(_args)-1]

	var n int
	switch cb.(type) {
	case func(error):
		n = 0
	case func(interface{}, error):
		n = 1
	case func([]interface{}, error):
		n = 2
	default:
		panic("definition of callback function is invalid")
	}

	// too many calls
	if c.pendingAsynCall >= cap(c.ChanAsynRet) {
		execCb(&RetInfo{err: errors.New("too many calls"), cb: cb})
		return
	}

	c.asynCall(id, args, cb, n)
	c.pendingAsynCall++
}

func execCb(ri *RetInfo) {
	defer func() {
		if r := recover(); r != nil {
			if conf.LenStackBuf > 0 {
				buf := make([]byte, conf.LenStackBuf)
				l := runtime.Stack(buf, false)
				log.Error("%v: %s", r, buf[:l])
			} else {
				log.Error("%v", r)
			}
		}
	}()

	// execute
	switch ri.cb.(type) {
	case func(error):
		ri.cb.(func(error))(ri.err)
	case func(interface{}, error):
		ri.cb.(func(interface{}, error))(ri.ret, ri.err)
	case func([]interface{}, error):
		ri.cb.(func([]interface{}, error))(assert(ri.ret), ri.err)
	default:
		panic("bug")
	}
	return
}

func (c *Client) Cb(ri *RetInfo) {
	c.pendingAsynCall--
	execCb(ri)
}

func (c *Client) Close() {
	for c.pendingAsynCall > 0 {
		c.Cb(<-c.ChanAsynRet)
	}
}

func (c *Client) Idle() bool {
	return c.pendingAsynCall == 0
}
<|endoftext|>


<commit_before>package main_test

import (
	"fmt"
	"net/url"
	"os/exec"
	"strings"

	"github.com/cloudfoundry-incubator/auctioneer"
	"github.com/cloudfoundry-incubator/bbs"
	bbstestrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	legacybbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"testing"
	"time"
)

var auctioneerProcess ifrit.Process

var auctioneerPath string

var dotNetStack = "dot-net"
var dotNetRootFSURL = models.PreloadedRootFS(dotNetStack)
var linuxStack = "linux"
var linuxRootFSURL = models.PreloadedRootFS(linuxStack)
var dotNetCell, linuxCell *FakeCell

var auctioneerServerPort int
var auctioneerAddress string
var runner *ginkgomon.Runner

var etcdPort int
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var etcdClient storeadapter.StoreAdapter

var consulRunner *consulrunner.ClusterRunner
var consulSession *consuladapter.Session

var auctioneerClient auctioneer.Client

var bbsArgs bbstestrunner.Args
var bbsBinPath string
var bbsURL *url.URL
var bbsRunner *ginkgomon.Runner
var bbsProcess ifrit.Process
var bbsClient bbs.Client

var legacyBBS *legacybbs.BBS
var logger lager.Logger

func TestAuctioneer(t *testing.T) {
	// these integration tests can take a bit, especially under load;
	// 1 second is too harsh
	SetDefaultEventuallyTimeout(10 * time.Second)

	RegisterFailHandler(Fail)
	RunSpecs(t, "Auctioneer Cmd Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	bbsConfig, err := gexec.Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	compiledAuctioneerPath, err := gexec.Build("github.com/cloudfoundry-incubator/auctioneer/cmd/auctioneer", "-race")
	Expect(err).NotTo(HaveOccurred())
	return []byte(strings.Join([]string{compiledAuctioneerPath, bbsConfig}, ","))
}, func(pathsByte []byte) {
	path := string(pathsByte)
	compiledAuctioneerPath := strings.Split(path, ",")[0]
	bbsBinPath = strings.Split(path, ",")[1]

	bbsBinPath = strings.Split(path, ",")[1]
	auctioneerPath = string(compiledAuctioneerPath)

	auctioneerServerPort = 1800 + GinkgoParallelNode()
	auctioneerAddress = fmt.Sprintf("http://127.0.0.1:%d", auctioneerServerPort)

	etcdPort = 5001 + GinkgoParallelNode()
	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)
	etcdClient = etcdRunner.Adapter(nil)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	auctioneerClient = auctioneer.NewClient(auctioneerAddress)

	logger = lagertest.NewTestLogger("test")

	consulRunner.Start()
	consulRunner.WaitUntilReady()

	bbsAddress := fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode())

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsClient = bbs.NewClient(bbsURL.String())

	etcdUrl := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
	bbsArgs = bbstestrunner.Args{
		Address:           bbsAddress,
		AuctioneerAddress: auctioneerAddress,
		EtcdCluster:       etcdUrl,
	}
	bbsRunner = bbstestrunner.New(bbsBinPath, bbsArgs)
	bbsProcess = ginkgomon.Invoke(bbsRunner)
})

var _ = BeforeEach(func() {
	etcdRunner.Start()

	consulRunner.Reset()
	consulSession = consulRunner.NewSession("a-session")

	legacyBBS = legacybbs.NewBBS(etcdClient, consulSession, "http://receptor.bogus.com", clock.NewClock(), logger)

	runner = ginkgomon.New(ginkgomon.Config{
		Name: "auctioneer",
		Command: exec.Command(
			auctioneerPath,
			"-bbsAddress", bbsURL.String(),
			"-etcdCluster", fmt.Sprintf("http://127.0.0.1:%d", etcdPort),
			"-listenAddr", fmt.Sprintf("0.0.0.0:%d", auctioneerServerPort),
			"-lockRetryInterval", "1s",
			"-consulCluster", consulRunner.ConsulCluster(),
		),
		StartCheck: "auctioneer.started",
	})

	dotNetCell = SpinUpFakeCell(legacyBBS, "dot-net-cell", dotNetStack)
	linuxCell = SpinUpFakeCell(legacyBBS, "linux-cell", linuxStack)
})

var _ = AfterEach(func() {
	ginkgomon.Kill(auctioneerProcess)
	ginkgomon.Kill(bbsProcess)

	etcdRunner.Stop()

	dotNetCell.Stop()
	linuxCell.Stop()
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
	if consulRunner != nil {
		consulRunner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<commit_msg>Start bbs process in a BeforeEach for integration test<commit_after>package main_test

import (
	"fmt"
	"net/url"
	"os/exec"
	"strings"

	"github.com/cloudfoundry-incubator/auctioneer"
	"github.com/cloudfoundry-incubator/bbs"
	bbstestrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	legacybbs "github.com/cloudfoundry-incubator/runtime-schema/bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"testing"
	"time"
)

var auctioneerProcess ifrit.Process

var auctioneerPath string

var dotNetStack = "dot-net"
var dotNetRootFSURL = models.PreloadedRootFS(dotNetStack)
var linuxStack = "linux"
var linuxRootFSURL = models.PreloadedRootFS(linuxStack)
var dotNetCell, linuxCell *FakeCell

var auctioneerServerPort int
var auctioneerAddress string
var runner *ginkgomon.Runner

var etcdPort int
var etcdRunner *etcdstorerunner.ETCDClusterRunner
var etcdClient storeadapter.StoreAdapter

var consulRunner *consulrunner.ClusterRunner
var consulSession *consuladapter.Session

var auctioneerClient auctioneer.Client

var bbsArgs bbstestrunner.Args
var bbsBinPath string
var bbsURL *url.URL
var bbsRunner *ginkgomon.Runner
var bbsProcess ifrit.Process
var bbsClient bbs.Client

var legacyBBS *legacybbs.BBS
var logger lager.Logger

func TestAuctioneer(t *testing.T) {
	// these integration tests can take a bit, especially under load;
	// 1 second is too harsh
	SetDefaultEventuallyTimeout(10 * time.Second)

	RegisterFailHandler(Fail)
	RunSpecs(t, "Auctioneer Cmd Suite")
}

var _ = SynchronizedBeforeSuite(func() []byte {
	bbsConfig, err := gexec.Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
	Expect(err).NotTo(HaveOccurred())

	compiledAuctioneerPath, err := gexec.Build("github.com/cloudfoundry-incubator/auctioneer/cmd/auctioneer", "-race")
	Expect(err).NotTo(HaveOccurred())
	return []byte(strings.Join([]string{compiledAuctioneerPath, bbsConfig}, ","))
}, func(pathsByte []byte) {
	path := string(pathsByte)
	compiledAuctioneerPath := strings.Split(path, ",")[0]
	bbsBinPath = strings.Split(path, ",")[1]

	bbsBinPath = strings.Split(path, ",")[1]
	auctioneerPath = string(compiledAuctioneerPath)

	auctioneerServerPort = 1800 + GinkgoParallelNode()
	auctioneerAddress = fmt.Sprintf("http://127.0.0.1:%d", auctioneerServerPort)

	etcdPort = 5001 + GinkgoParallelNode()
	etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)
	etcdClient = etcdRunner.Adapter(nil)

	consulRunner = consulrunner.NewClusterRunner(
		9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
		1,
		"http",
	)

	auctioneerClient = auctioneer.NewClient(auctioneerAddress)

	logger = lagertest.NewTestLogger("test")

	consulRunner.Start()
	consulRunner.WaitUntilReady()

	bbsAddress := fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode())

	bbsURL = &url.URL{
		Scheme: "http",
		Host:   bbsAddress,
	}

	bbsClient = bbs.NewClient(bbsURL.String())

	etcdUrl := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
	bbsArgs = bbstestrunner.Args{
		Address:           bbsAddress,
		AuctioneerAddress: auctioneerAddress,
		EtcdCluster:       etcdUrl,
	}
})

var _ = BeforeEach(func() {
	etcdRunner.Start()
	bbsRunner = bbstestrunner.New(bbsBinPath, bbsArgs)
	bbsProcess = ginkgomon.Invoke(bbsRunner)

	consulRunner.Reset()
	consulSession = consulRunner.NewSession("a-session")

	legacyBBS = legacybbs.NewBBS(etcdClient, consulSession, "http://receptor.bogus.com", clock.NewClock(), logger)

	runner = ginkgomon.New(ginkgomon.Config{
		Name: "auctioneer",
		Command: exec.Command(
			auctioneerPath,
			"-bbsAddress", bbsURL.String(),
			"-etcdCluster", fmt.Sprintf("http://127.0.0.1:%d", etcdPort),
			"-listenAddr", fmt.Sprintf("0.0.0.0:%d", auctioneerServerPort),
			"-lockRetryInterval", "1s",
			"-consulCluster", consulRunner.ConsulCluster(),
		),
		StartCheck: "auctioneer.started",
	})

	dotNetCell = SpinUpFakeCell(legacyBBS, "dot-net-cell", dotNetStack)
	linuxCell = SpinUpFakeCell(legacyBBS, "linux-cell", linuxStack)
})

var _ = AfterEach(func() {
	ginkgomon.Kill(auctioneerProcess)
	etcdRunner.Stop()
	ginkgomon.Kill(bbsProcess)
	dotNetCell.Stop()
	linuxCell.Stop()
})

var _ = SynchronizedAfterSuite(func() {
	if etcdRunner != nil {
		etcdRunner.Stop()
	}
	if consulRunner != nil {
		consulRunner.Stop()
	}
}, func() {
	gexec.CleanupBuildArtifacts()
})
<|endoftext|>


<commit_before>package deployment

import (
	"bytes"
	"fmt"
	"testing"

	"github.com/liimaorg/liimactl/client"
	"github.com/spf13/pflag"
)

//Tests the command "deployment get"
func TestNewDeploymentGetCmd(t *testing.T) {

	//Tests
	tests := []struct {
		name string   //Name of the test
		args []string //Arguments
		want string   //Wanted testresult
	}{
		{"Test1", []string{"get", "--appServer=testApp"}, "------\ntestApp   1970-01-01T01:00 \n"},
		{"Test2", []string{"get", "--appServer=testApp2", "--environment=T"}, "------\ntestApp2 T  1970-01-01T01:00 \n"},
	}

	//Init config
	var flags *pflag.FlagSet
	liimacli := &client.Cli{}
	config, err := initConfig(flags)
	if err != nil {
		fmt.Println(err)
	}

	//Create mock client
	liimacli.Client, err = client.NewMockClient(config)

	//Run tests
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {

			//Create command
			cmd := NewDeploymentCmd(liimacli)

			//Set commands output to buffer
			buf := new(bytes.Buffer)
			cmd.SetOutput(buf)

			//Set test arguments
			cmd.SetArgs(tt.args)

			//Execute command
			err = cmd.Execute()
			if err != nil {
				t.Errorf("Execute() failed with %v", err)
			}
			//Check result
			if got := buf.String(); got != tt.want {
				t.Errorf("Commands-Output = %v, want %v", got, tt.want)
			}
		})
	}

}

//Tests the command "deployment create"
func TestNewDeploymentCreateCmd(t *testing.T) {

	//Tests
	tests := []struct {
		name string   //Name of the test
		args []string //Arguments
		want string   //Wanted testresult
	}{
		{"Test1", []string{"create", "--appServer=testApp", "--environment=T", "--appName=test1", "--version=1.1.1"}, "SUCCESS\n"},
	}

	//Init config
	var flags *pflag.FlagSet
	liimacli := &client.Cli{}
	config, err := initConfig(flags)
	if err != nil {
		fmt.Println(err)
	}

	//Create mock client
	liimacli.Client, err = client.NewMockClient(config)

	//Run tests
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {

			//Create command
			cmd := NewDeploymentCmd(liimacli)

			//Set commands output to buffer
			buf := new(bytes.Buffer)
			cmd.SetOutput(buf)

			//Set test arguments
			cmd.SetArgs(tt.args)

			//Execute command
			err = cmd.Execute()
			if err != nil {
				t.Errorf("Execute() failed with %v", err)
			}

			//Check result
			if got := buf.String(); got != tt.want {
				t.Errorf("got: %v, want: %v", got, tt.want)
			}
		})
	}

}

// initConfig reads in config file and ENV variables if set.
func initConfig(flags *pflag.FlagSet) (*client.Config, error) {

	var config client.Config
	return &config, nil
}
<commit_msg>fix deployment_test<commit_after>package deployment

import (
	"bytes"
	"fmt"
	"strings"
	"testing"

	"github.com/liimaorg/liimactl/client"
	"github.com/spf13/pflag"
)

//Tests the command "deployment get"
func TestNewDeploymentGetCmd(t *testing.T) {

	//Tests
	tests := []struct {
		name string   //Name of the test
		args []string //Arguments
		want string   //Wanted testresult
	}{
		{"Test1", []string{"get", "--appServer=testApp"}, "------\ntestApp"},
		{"Test2", []string{"get", "--appServer=testApp2", "--environment=T"}, "------\ntestApp2 T"},
	}

	//Init config
	var flags *pflag.FlagSet
	liimacli := &client.Cli{}
	config, err := initConfig(flags)
	if err != nil {
		fmt.Println(err)
	}

	//Create mock client
	liimacli.Client, err = client.NewMockClient(config)

	//Run tests
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {

			//Create command
			cmd := NewDeploymentCmd(liimacli)

			//Set commands output to buffer
			buf := new(bytes.Buffer)
			cmd.SetOutput(buf)

			//Set test arguments
			cmd.SetArgs(tt.args)

			//Execute command
			err = cmd.Execute()
			if err != nil {
				t.Errorf("Execute() failed with %v", err)
			}
			//Check result
			if got := buf.String(); !strings.HasPrefix(got, tt.want) {
				t.Errorf("Commands-Output = %v, want %v", got, tt.want)
			}
		})
	}

}

//Tests the command "deployment create"
func TestNewDeploymentCreateCmd(t *testing.T) {

	//Tests
	tests := []struct {
		name string   //Name of the test
		args []string //Arguments
		want string   //Wanted testresult
	}{
		{"Test1", []string{"create", "--appServer=testApp", "--environment=T", "--appName=test1", "--version=1.1.1"}, "SUCCESS\n"},
	}

	//Init config
	var flags *pflag.FlagSet
	liimacli := &client.Cli{}
	config, err := initConfig(flags)
	if err != nil {
		fmt.Println(err)
	}

	//Create mock client
	liimacli.Client, err = client.NewMockClient(config)

	//Run tests
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {

			//Create command
			cmd := NewDeploymentCmd(liimacli)

			//Set commands output to buffer
			buf := new(bytes.Buffer)
			cmd.SetOutput(buf)

			//Set test arguments
			cmd.SetArgs(tt.args)

			//Execute command
			err = cmd.Execute()
			if err != nil {
				t.Errorf("Execute() failed with %v", err)
			}

			//Check result
			if got := buf.String(); got != tt.want {
				t.Errorf("got: %v, want: %v", got, tt.want)
			}
		})
	}

}

// initConfig reads in config file and ENV variables if set.
func initConfig(flags *pflag.FlagSet) (*client.Config, error) {

	var config client.Config
	return &config, nil
}
<|endoftext|>


<commit_before>package agollo

import (
	"testing"
	"os"
	"strconv"
	"time"
	"github.com/zouyx/agollo/test"
)

func TestInit(t *testing.T) {
	config:=GetAppConfig()

	test.NotNil(t,config)
	test.Equal(t,"test",config.AppId)
	test.Equal(t,"dev",config.Cluster)
	test.Equal(t,"application",config.NamespaceName)
	test.Equal(t,"localhost:8888",config.Ip)

	apolloConfig:=GetCurrentApolloConfig()
	test.Equal(t,"test",apolloConfig.AppId)
	test.Equal(t,"dev",apolloConfig.Cluster)
	test.Equal(t,"application",apolloConfig.NamespaceName)

}

func TestInitRefreshInterval_1(t *testing.T) {
	os.Setenv(refresh_interval_key,"joe")

	err:=initRefreshInterval()
	test.NotNil(t,err)

	interval:="3"
	os.Setenv(refresh_interval_key,interval)
	err=initRefreshInterval()
	test.Nil(t,err)
	i,_:=strconv.Atoi(interval)
	test.Equal(t,time.Duration(i),refresh_interval)

}

func TestGetConfigUrl(t *testing.T) {
	appConfig:=getTestAppConfig()
	url:=getConfigUrl(appConfig)
	test.StartWith(t,"http://localhost:8888/configs/test/dev/application?releaseKey=&ip=",url)
}

func TestGetNotifyUrl(t *testing.T) {
	appConfig:=getTestAppConfig()
	url:=getNotifyUrl("notifys",appConfig)
	test.Equal(t,"http://localhost:8888/notifications/v2?appId=test&cluster=dev&notifications=notifys",url)
}

func TestGetServicesConfigUrl(t *testing.T) {
	appConfig:=getTestAppConfig()
	url:=getServicesConfigUrl(appConfig)
	ip:=getInternal()
	test.Equal(t,"http://localhost:8888/services/config?appId=test&ip="+ip,url)
}

func getTestAppConfig() *AppConfig {
	jsonStr:=`{
    "appId": "test",
    "cluster": "dev",
    "namespaceName": "application",
    "ip": "localhost:8888",
    "releaseKey": "1"
	}`
	config,_:=createAppConfigWithJson(jsonStr)

	return config
}

func TestSyncServerIpList(t *testing.T) {
	go runMockServicesConfigServer(normalServicesConfigResponse)
	defer closeMockServicesConfigServer()

	time.Sleep(1*time.Second)

	err:=syncServerIpList()

	test.Nil(t,err)

	test.Equal(t,10,len(servers))

}<commit_msg>refactor app config<commit_after>package agollo

import (
	"testing"
	"os"
	"strconv"
	"time"
	"github.com/zouyx/agollo/test"
)

func TestInit(t *testing.T) {
	config:=GetAppConfig()

	test.NotNil(t,config)
	test.Equal(t,"test",config.AppId)
	test.Equal(t,"dev",config.Cluster)
	test.Equal(t,"application",config.NamespaceName)
	test.Equal(t,"localhost:8888",config.Ip)

	apolloConfig:=GetCurrentApolloConfig()
	test.Equal(t,"test",apolloConfig.AppId)
	test.Equal(t,"dev",apolloConfig.Cluster)
	test.Equal(t,"application",apolloConfig.NamespaceName)

}

func TestInitRefreshInterval_1(t *testing.T) {
	os.Setenv(refresh_interval_key,"joe")

	err:=initRefreshInterval()
	test.NotNil(t,err)

	interval:="3"
	os.Setenv(refresh_interval_key,interval)
	err=initRefreshInterval()
	test.Nil(t,err)
	i,_:=strconv.Atoi(interval)
	test.Equal(t,time.Duration(i),refresh_interval)

}

func TestGetConfigUrl(t *testing.T) {
	appConfig:=getTestAppConfig()
	url:=getConfigUrl(appConfig)
	test.StartWith(t,"http://localhost:8888/configs/test/dev/application?releaseKey=&ip=",url)
}

func TestGetConfigUrlByHost(t *testing.T) {
	appConfig:=getTestAppConfig()
	url:=getConfigUrlByHost(appConfig,"http://baidu.com/")
	test.StartWith(t,"http://baidu.com/configs/test/dev/application?releaseKey=&ip=",url)
}

func TestGetNotifyUrl(t *testing.T) {
	appConfig:=getTestAppConfig()
	url:=getNotifyUrl("notifys",appConfig)
	test.Equal(t,"http://localhost:8888/notifications/v2?appId=test&cluster=dev&notifications=notifys",url)
}

func TestGetNotifyUrlByHost(t *testing.T) {
	appConfig:=getTestAppConfig()
	url:=getNotifyUrlByHost("notifys",appConfig,"http://baidu.com/")
	test.Equal(t,"http://baidu.com/notifications/v2?appId=test&cluster=dev&notifications=notifys",url)
}

func TestGetServicesConfigUrl(t *testing.T) {
	appConfig:=getTestAppConfig()
	url:=getServicesConfigUrl(appConfig)
	ip:=getInternal()
	test.Equal(t,"http://localhost:8888/services/config?appId=test&ip="+ip,url)
}

func getTestAppConfig() *AppConfig {
	jsonStr:=`{
    "appId": "test",
    "cluster": "dev",
    "namespaceName": "application",
    "ip": "localhost:8888",
    "releaseKey": "1"
	}`
	config,_:=createAppConfigWithJson(jsonStr)

	return config
}

func TestSyncServerIpList(t *testing.T) {
	go runMockServicesConfigServer(normalServicesConfigResponse)
	defer closeMockServicesConfigServer()

	time.Sleep(1*time.Second)

	err:=syncServerIpList()

	test.Nil(t,err)

	test.Equal(t,10,len(servers))

}<|endoftext|>


<commit_before>package dmplugin

import (
	"errors"
	"fmt"
	"io"
	"sync"
	"syscall"

	"github.intel.com/hpdd/logging/alert"
	"github.intel.com/hpdd/logging/debug"
	pb "github.intel.com/hpdd/policy/pdm/pdm"
	"golang.org/x/net/context"
)

type (
	// DataMoverClient is the data mover client to the HSM agent
	DataMoverClient struct {
		rpcClient pb.DataMoverClient
		stop      chan struct{}
		status    chan *pb.ActionStatus
		mover     Mover
	}

	// Action is a data movement action
	Action struct {
		status       chan *pb.ActionStatus
		item         *pb.ActionItem
		fileID       []byte
		actualLength *uint64
	}

	// Mover defines an interface for data mover implementations
	Mover interface {
		FsName() string
		ArchiveID() uint32
	}

	// Archiver defines an interface for data movers capable of
	// fulfilling Archive requests
	Archiver interface {
		Archive(*Action) error
	}

	// Restorer defines an interface for data movers capable of
	// fulfilling Restore requests
	Restorer interface {
		Restore(*Action) error
	}

	// Remover defines an interface for data movers capable of
	// fulfilling Remove requests
	Remover interface {
		Remove(*Action) error
	}
)

type key int

var handleKey key

func withHandle(ctx context.Context, handle *pb.Handle) context.Context {
	return context.WithValue(ctx, handleKey, handle)
}

func getHandle(ctx context.Context) (*pb.Handle, bool) {
	handle, ok := ctx.Value(handleKey).(*pb.Handle)
	return handle, ok
}

// Update sends an action status update
func (a *Action) Update(offset, length, max int64) error {
	a.status <- &pb.ActionStatus{
		Id:     a.item.Id,
		Offset: uint64(offset),
		Length: uint64(length),
	}
	return nil
}

// Complete signals that the action has completed
func (a *Action) Complete() error {
	status := &pb.ActionStatus{
		Id:        a.item.Id,
		Completed: true,
		Offset:    a.item.Offset,
		Length:    a.item.Length,
		FileId:    a.fileID,
	}
	if a.actualLength != nil {
		status.Length = *a.actualLength
	}
	a.status <- status
	return nil
}

func getErrno(err error) int32 {
	if errno, ok := err.(syscall.Errno); ok {
		return int32(errno)
	}
	return -1
}

// Fail signals that the action has failed
func (a *Action) Fail(err error) error {
	alert.Warnf("fail: id:%d %v", a.item.Id, err)
	a.status <- &pb.ActionStatus{
		Id:        a.item.Id,
		Completed: true,

		Error: getErrno(err),
	}
	return nil
}

// ID returns the action item's ID
func (a *Action) ID() uint64 {
	return a.item.Id
}

// Offset returns the current offset of the action item
func (a *Action) Offset() int64 {
	return int64(a.item.Offset)
}

// Length returns the expected length of the action item's file
func (a *Action) Length() int64 {
	return int64(a.item.Length)
}

// Data returns a byte slice of the action item's data
func (a *Action) Data() []byte {
	return a.item.Data
}

// PrimaryPath returns the action item's primary file path
func (a *Action) PrimaryPath() string {
	return a.item.PrimaryPath
}

// WritePath returns the action item's write path (e.g. for restores)
func (a *Action) WritePath() string {
	return a.item.WritePath
}

// FileID returns the action item's file id
func (a *Action) FileID() string {
	return string(a.item.FileId)
}

// SetFileID sets the action's file id
func (a *Action) SetFileID(id []byte) {
	a.fileID = id
}

// SetActualLength sets the action's actual file length
func (a *Action) SetActualLength(length uint64) {
	a.actualLength = &length
}

// NewMover returns a new *DataMoverClient
func NewMover(cli pb.DataMoverClient, mover Mover) *DataMoverClient {
	return &DataMoverClient{
		rpcClient: cli,
		mover:     mover,
		stop:      make(chan struct{}),
		status:    make(chan *pb.ActionStatus, 2),
	}
}

// Run begins listening for and processing incoming action items
func (dm *DataMoverClient) Run() {
	var wg sync.WaitGroup
	ctx, cancel := context.WithCancel(context.Background())

	handle, err := dm.registerEndpoint(ctx)
	if err != nil {
		alert.Fatal(err)
	}
	ctx = withHandle(ctx, handle)
	actions := dm.processActions(ctx)
	dm.processStatus(ctx)

	for i := 0; i < 2; i++ {
		wg.Add(1)
		go func(i int) {
			dm.handler(fmt.Sprintf("handler-%d", i), actions)
			wg.Done()
		}(i)
		dm.processStatus(ctx)
	}

	<-dm.stop
	debug.Printf("Shutting down Data Mover")
	cancel()
	wg.Wait()
	close(dm.status)
}

// Stop signals to the client that it should stop processing and shut down
func (dm *DataMoverClient) Stop() {
	close(dm.stop)
}

func (dm *DataMoverClient) registerEndpoint(ctx context.Context) (*pb.Handle, error) {

	handle, err := dm.rpcClient.Register(ctx, &pb.Endpoint{
		FsUrl:   dm.mover.FsName(),
		Archive: dm.mover.ArchiveID(),
	})
	if err != nil {
		return nil, err
	}
	debug.Printf("Registered archive %d,  cookie %x", dm.mover.ArchiveID(), handle.Id)
	return handle, nil
}

func (dm *DataMoverClient) processActions(ctx context.Context) chan *pb.ActionItem {
	actions := make(chan *pb.ActionItem)

	go func() {
		handle, ok := getHandle(ctx)
		if !ok {
			alert.Fatal("No context")
		}
		stream, err := dm.rpcClient.GetActions(ctx, handle)
		if err != nil {
			alert.Fatalf("GetActions() failed: %v", err)
		}
		for {
			action, err := stream.Recv()
			if err == io.EOF {
				return
			}
			if err != nil {
				close(actions)
				alert.Fatalf("Failed to receive a message: %v", err)
			}
			// debug.Printf("Got message id:%d op: %v %v", action.Id, action.Op, action.PrimaryPath)

			actions <- action
		}
	}()

	return actions

}

func (dm *DataMoverClient) processStatus(ctx context.Context) {
	go func() {
		handle, ok := getHandle(ctx)
		if !ok {
			alert.Fatal("No context")
		}

		acks, err := dm.rpcClient.StatusStream(ctx)
		if err != nil {
			alert.Fatalf("StatusStream() failed: %v", err)
		}
		for reply := range dm.status {
			reply.Handle = handle
			// debug.Printf("Sent reply  %x error: %#v", reply.Id, reply.Error)
			err := acks.Send(reply)
			if err != nil {
				alert.Fatalf("Failed to ack message %x: %v", reply.Id, err)
			}
		}
	}()
	return
}

func (dm *DataMoverClient) handler(name string, actions chan *pb.ActionItem) {
	for item := range actions {
		var ret error
		action := &Action{
			status: dm.status,
			item:   item,
		}

		ret = errors.New("Command not supported")

		switch item.Op {
		case pb.Command_ARCHIVE:
			if archiver, ok := dm.mover.(Archiver); ok {
				ret = archiver.Archive(action)
			}
		case pb.Command_RESTORE:
			if restorer, ok := dm.mover.(Restorer); ok {
				ret = restorer.Restore(action)
			}
		case pb.Command_REMOVE:
			if remover, ok := dm.mover.(Remover); ok {
				ret = remover.Remove(action)
			}
		case pb.Command_CANCEL:
			// TODO: Cancel in-progress action using a context
		default:
			ret = errors.New("Unknown cmmand")
		}

		//		rate.Mark(1)
		// debug.Printf("completed (action: %v) %v ", action, ret)
		if ret != nil {
			action.Fail(ret)
		} else {
			action.Complete()
		}
	}
	debug.Printf("%s: stopping", name)
}
<commit_msg>increase plugin threads to 4<commit_after>package dmplugin

import (
	"errors"
	"fmt"
	"io"
	"sync"
	"syscall"

	"github.intel.com/hpdd/logging/alert"
	"github.intel.com/hpdd/logging/debug"
	pb "github.intel.com/hpdd/policy/pdm/pdm"
	"golang.org/x/net/context"
)

type (
	// DataMoverClient is the data mover client to the HSM agent
	DataMoverClient struct {
		rpcClient pb.DataMoverClient
		stop      chan struct{}
		status    chan *pb.ActionStatus
		mover     Mover
	}

	// Action is a data movement action
	Action struct {
		status       chan *pb.ActionStatus
		item         *pb.ActionItem
		fileID       []byte
		actualLength *uint64
	}

	// Mover defines an interface for data mover implementations
	Mover interface {
		FsName() string
		ArchiveID() uint32
	}

	// Archiver defines an interface for data movers capable of
	// fulfilling Archive requests
	Archiver interface {
		Archive(*Action) error
	}

	// Restorer defines an interface for data movers capable of
	// fulfilling Restore requests
	Restorer interface {
		Restore(*Action) error
	}

	// Remover defines an interface for data movers capable of
	// fulfilling Remove requests
	Remover interface {
		Remove(*Action) error
	}
)

type key int

var handleKey key

func withHandle(ctx context.Context, handle *pb.Handle) context.Context {
	return context.WithValue(ctx, handleKey, handle)
}

func getHandle(ctx context.Context) (*pb.Handle, bool) {
	handle, ok := ctx.Value(handleKey).(*pb.Handle)
	return handle, ok
}

// Update sends an action status update
func (a *Action) Update(offset, length, max int64) error {
	a.status <- &pb.ActionStatus{
		Id:     a.item.Id,
		Offset: uint64(offset),
		Length: uint64(length),
	}
	return nil
}

// Complete signals that the action has completed
func (a *Action) Complete() error {
	status := &pb.ActionStatus{
		Id:        a.item.Id,
		Completed: true,
		Offset:    a.item.Offset,
		Length:    a.item.Length,
		FileId:    a.fileID,
	}
	if a.actualLength != nil {
		status.Length = *a.actualLength
	}
	a.status <- status
	return nil
}

func getErrno(err error) int32 {
	if errno, ok := err.(syscall.Errno); ok {
		return int32(errno)
	}
	return -1
}

// Fail signals that the action has failed
func (a *Action) Fail(err error) error {
	alert.Warnf("fail: id:%d %v", a.item.Id, err)
	a.status <- &pb.ActionStatus{
		Id:        a.item.Id,
		Completed: true,

		Error: getErrno(err),
	}
	return nil
}

// ID returns the action item's ID
func (a *Action) ID() uint64 {
	return a.item.Id
}

// Offset returns the current offset of the action item
func (a *Action) Offset() int64 {
	return int64(a.item.Offset)
}

// Length returns the expected length of the action item's file
func (a *Action) Length() int64 {
	return int64(a.item.Length)
}

// Data returns a byte slice of the action item's data
func (a *Action) Data() []byte {
	return a.item.Data
}

// PrimaryPath returns the action item's primary file path
func (a *Action) PrimaryPath() string {
	return a.item.PrimaryPath
}

// WritePath returns the action item's write path (e.g. for restores)
func (a *Action) WritePath() string {
	return a.item.WritePath
}

// FileID returns the action item's file id
func (a *Action) FileID() string {
	return string(a.item.FileId)
}

// SetFileID sets the action's file id
func (a *Action) SetFileID(id []byte) {
	a.fileID = id
}

// SetActualLength sets the action's actual file length
func (a *Action) SetActualLength(length uint64) {
	a.actualLength = &length
}

// NewMover returns a new *DataMoverClient
func NewMover(cli pb.DataMoverClient, mover Mover) *DataMoverClient {
	return &DataMoverClient{
		rpcClient: cli,
		mover:     mover,
		stop:      make(chan struct{}),
		status:    make(chan *pb.ActionStatus, 2),
	}
}

// Run begins listening for and processing incoming action items
func (dm *DataMoverClient) Run() {
	var wg sync.WaitGroup
	ctx, cancel := context.WithCancel(context.Background())

	handle, err := dm.registerEndpoint(ctx)
	if err != nil {
		alert.Fatal(err)
	}
	ctx = withHandle(ctx, handle)
	actions := dm.processActions(ctx)
	dm.processStatus(ctx)

	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func(i int) {
			dm.handler(fmt.Sprintf("handler-%d", i), actions)
			wg.Done()
		}(i)
		dm.processStatus(ctx)
	}

	<-dm.stop
	debug.Printf("Shutting down Data Mover")
	cancel()
	wg.Wait()
	close(dm.status)
}

// Stop signals to the client that it should stop processing and shut down
func (dm *DataMoverClient) Stop() {
	close(dm.stop)
}

func (dm *DataMoverClient) registerEndpoint(ctx context.Context) (*pb.Handle, error) {

	handle, err := dm.rpcClient.Register(ctx, &pb.Endpoint{
		FsUrl:   dm.mover.FsName(),
		Archive: dm.mover.ArchiveID(),
	})
	if err != nil {
		return nil, err
	}
	debug.Printf("Registered archive %d,  cookie %x", dm.mover.ArchiveID(), handle.Id)
	return handle, nil
}

func (dm *DataMoverClient) processActions(ctx context.Context) chan *pb.ActionItem {
	actions := make(chan *pb.ActionItem)

	go func() {
		handle, ok := getHandle(ctx)
		if !ok {
			alert.Fatal("No context")
		}
		stream, err := dm.rpcClient.GetActions(ctx, handle)
		if err != nil {
			alert.Fatalf("GetActions() failed: %v", err)
		}
		for {
			action, err := stream.Recv()
			if err == io.EOF {
				return
			}
			if err != nil {
				close(actions)
				alert.Fatalf("Failed to receive a message: %v", err)
			}
			// debug.Printf("Got message id:%d op: %v %v", action.Id, action.Op, action.PrimaryPath)

			actions <- action
		}
	}()

	return actions

}

func (dm *DataMoverClient) processStatus(ctx context.Context) {
	go func() {
		handle, ok := getHandle(ctx)
		if !ok {
			alert.Fatal("No context")
		}

		acks, err := dm.rpcClient.StatusStream(ctx)
		if err != nil {
			alert.Fatalf("StatusStream() failed: %v", err)
		}
		for reply := range dm.status {
			reply.Handle = handle
			// debug.Printf("Sent reply  %x error: %#v", reply.Id, reply.Error)
			err := acks.Send(reply)
			if err != nil {
				alert.Fatalf("Failed to ack message %x: %v", reply.Id, err)
			}
		}
	}()
	return
}

func (dm *DataMoverClient) handler(name string, actions chan *pb.ActionItem) {
	for item := range actions {
		var ret error
		action := &Action{
			status: dm.status,
			item:   item,
		}

		ret = errors.New("Command not supported")

		switch item.Op {
		case pb.Command_ARCHIVE:
			if archiver, ok := dm.mover.(Archiver); ok {
				ret = archiver.Archive(action)
			}
		case pb.Command_RESTORE:
			if restorer, ok := dm.mover.(Restorer); ok {
				ret = restorer.Restore(action)
			}
		case pb.Command_REMOVE:
			if remover, ok := dm.mover.(Remover); ok {
				ret = remover.Remove(action)
			}
		case pb.Command_CANCEL:
			// TODO: Cancel in-progress action using a context
		default:
			ret = errors.New("Unknown cmmand")
		}

		//		rate.Mark(1)
		// debug.Printf("completed (action: %v) %v ", action, ret)
		if ret != nil {
			action.Fail(ret)
		} else {
			action.Complete()
		}
	}
	debug.Printf("%s: stopping", name)
}
<|endoftext|>


<commit_before>package ast

import (
	"fmt"
	"github.com/gtfierro/hod/lang/token"
	"github.com/gtfierro/hod/turtle"
	"github.com/kr/pretty"
	"strings"
)

type QueryType uint

const (
	SELECT_QUERY QueryType = 1 << iota
	INSERT_QUERY
	DELETE_QUERY
)

var debug = false

func SetDebug() {
	debug = true
}
func ClearDebug() {
	debug = false
}

type Query struct {
	Select    SelectClause
	From      FromClause
	Count     bool
	Insert    InsertClause
	Where     WhereClause
	Variables []string
	Type      QueryType
}

func (q Query) IsInsert() bool {
	return (q.Type & INSERT_QUERY) == INSERT_QUERY
}

func (q Query) IsSelect() bool {
	return (q.Type & SELECT_QUERY) == SELECT_QUERY
}

func (q Query) CopyWithNewTerms(terms []Triple) Query {
	newq := Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Type:      q.Type,
	}
	newq.Where.Terms = make([]Triple, len(terms))
	copy(newq.Where.Terms, terms)

	newq.Insert.Terms = make([]Triple, len(terms))
	copy(newq.Insert.Terms, terms)
	return newq
}

func (q Query) Copy() *Query {
	return &Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Where:     q.Where,
		Insert:    q.Insert,
		Count:     q.Count,
		Type:      q.Type,
	}
}

func NewQuery(selectclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: selectclause.(SelectClause),
		Count:  count,
		Type:   SELECT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewInsertQuery(insertclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
		fmt.Printf("%# v", pretty.Formatter(insertclause.(InsertClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: SelectClause{AllVars: true},
		Insert: insertclause.(InsertClause),
		Count:  count,
		Type:   INSERT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewQueryMulti(selectclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		From:   fromclause.(FromClause),
		Select: selectclause.(SelectClause),
		Count:  count,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewInsertQueryMulti(insertclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
		fmt.Printf("%# v", pretty.Formatter(insertclause.(InsertClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: SelectClause{AllVars: true},
		From:   fromclause.(FromClause),
		Insert: insertclause.(InsertClause),
		Count:  count,
		Type:   INSERT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func (q *Query) PopulateVars() {
	vars := make(map[string]int)
	// get all variables
	for _, triple := range q.Where.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	for _, triple := range q.Insert.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	if q.Where.GraphGroup != nil {
		VarsFromGroup(*q.Where.GraphGroup, vars)
	}
	q.Variables = []string{} // clear
	for varname := range vars {
		q.Variables = append(q.Variables, varname)
	}
}

func (q Query) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range q.Where.Terms {
		q.Where.Terms[idx] = f(triple)
	}
	for idx, triple := range q.Insert.Terms {
		q.Insert.Terms[idx] = f(triple)
	}
	if q.Where.GraphGroup != nil {
		q.Where.GraphGroup.IterTriples(f)
	}
}

func AddIfVar(uri turtle.URI, m map[string]int) {
	if uri.IsVariable() {
		m[uri.String()] = 1
	}
}

func VarsFromGroup(group GraphGroup, m map[string]int) {
	for _, triple := range group.Terms {
		AddIfVar(triple.Subject, m)
		AddIfVar(triple.Object, m)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, m)
		}
	}
	for _, union := range group.Unions {
		VarsFromGroup(union, m)
	}
}

func (grp GraphGroup) Expand() [][]Triple {
	var terms = make([]Triple, len(grp.Terms))
	copy(terms, grp.Terms)
	var groups [][]Triple

	if len(grp.Unions) > 0 {
		for _, union := range grp.Unions {
			for _, subgroup := range union.Expand() {
				groups = append(groups, append(terms, subgroup...))
			}
		}
	} else {
		groups = append(groups, terms)
	}
	return groups
}

func (grp GraphGroup) Iter(f func(t turtle.URI)) {
	for _, triple := range grp.Terms {
		f(triple.Subject)
		f(triple.Object)
		for _, path := range triple.Predicates {
			f(path.Predicate)
		}
	}
	for _, union := range grp.Unions {
		union.Iter(f)
	}
}

func (grp *GraphGroup) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range grp.Terms {
		grp.Terms[idx] = f(triple)
	}
	for _, union := range grp.Unions {
		union.IterTriples(f)
	}
}

type SelectClause struct {
	Vars    []string
	AllVars bool
}

func NewAllSelectClause() (SelectClause, error) {
	return SelectClause{AllVars: true}, nil
}

func NewSelectClause(varlist interface{}) (SelectClause, error) {
	return SelectClause{Vars: varlist.([]string)}, nil
}

type InsertClause struct {
	Terms []Triple
}

func NewInsertClause(triples interface{}) (InsertClause, error) {
	return InsertClause{
		Terms: triples.([]Triple),
	}, nil
}

type FromClause struct {
	Databases []string
	AllDBs    bool
}

func (f FromClause) String() string {
	if f.AllDBs {
		return "*"
	}
	return strings.Join(f.Databases, " ")
}

func NewAllFromClause() (FromClause, error) {
	return FromClause{AllDBs: true}, nil
}

func NewFromClause(dblist interface{}) (FromClause, error) {
	return FromClause{Databases: dblist.([]string)}, nil
}

func (from FromClause) Empty() bool {
	return len(from.Databases) == 0 && !from.AllDBs
}

type WhereClause struct {
	Terms      []Triple
	GraphGroup *GraphGroup
}

func NewWhereClause(triples interface{}) (WhereClause, error) {
	return WhereClause{
		Terms: triples.([]Triple),
	}, nil
}

func NewWhereClauseWithGraphGroup(triples, group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		Terms:      triples.([]Triple),
		GraphGroup: &g,
	}, nil
}

func NewWhereClauseGraphGroup(group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		GraphGroup: &g,
	}, nil
}

type GraphGroup struct {
	Terms  []Triple
	Unions []GraphGroup
}

func GraphGroupFromTriples(triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms: triples.([]Triple),
	}, nil
}

func GraphGroupUnion(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Unions: []GraphGroup{left.(GraphGroup), right.(GraphGroup)},
	}, nil
}

func AddTriplesToGraphGroup(left, triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, triples.([]Triple)...),
		Unions: left.(GraphGroup).Unions,
	}, nil
}

func MergeGraphGroups(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, right.(GraphGroup).Terms...),
		Unions: append(left.(GraphGroup).Unions, right.(GraphGroup).Unions...),
	}, nil
}

type Triple struct {
	Subject    turtle.URI
	Predicates []PathPattern
	Object     turtle.URI
}

func (t Triple) String() string {
	s := "<" + t.Subject.String() + "|"
	for _, pp := range t.Predicates {
		s += " " + pp.String()
	}
	return s + " | " + t.Object.String() + ">"
}

func (t Triple) Copy() Triple {
	var p = make([]PathPattern, len(t.Predicates))
	copy(p, t.Predicates)
	return Triple{
		Subject:    t.Subject,
		Object:     t.Object,
		Predicates: p,
	}
}

func NewTriple(subject, predicates, object interface{}) (Triple, error) {
	return Triple{
		Subject:    subject.(turtle.URI),
		Predicates: predicates.([]PathPattern),
		Object:     object.(turtle.URI),
	}, nil
}

func NewTripleBlock(triple interface{}) ([]Triple, error) {
	return []Triple{triple.(Triple)}, nil
}

func AppendTripleBlock(block, triple interface{}) ([]Triple, error) {
	return append(block.([]Triple), triple.(Triple)), nil
}

func NewURI(value interface{}) (turtle.URI, error) {
	return turtle.ParseURI(value.(string)), nil
}

func NewVarList(_var interface{}) ([]string, error) {
	return []string{_var.(string)}, nil
}

func AppendVar(varlist, _var interface{}) ([]string, error) {
	return append(varlist.([]string), _var.(string)), nil
}

func NewStringList(_str interface{}) ([]string, error) {
	return []string{_str.(string)}, nil
}

func AppendString(strlist, _str interface{}) ([]string, error) {
	return append(strlist.([]string), _str.(string)), nil
}

func ParseString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func ParseQuotedString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func NewPathSequence(_pred interface{}) ([]PathPattern, error) {
	return []PathPattern{_pred.(PathPattern)}, nil
}

func AppendPathSequence(_seq, _pred interface{}) ([]PathPattern, error) {
	return append(_seq.([]PathPattern), _pred.(PathPattern)), nil
}

func NewPathPattern(_pred interface{}) (PathPattern, error) {
	pred, _ := ParseString(_pred)
	if pred == "a" {
		pred = "rdf:type"
	}
	return PathPattern{
		Predicate: turtle.ParseURI(pred),
		Pattern:   PATTERN_SINGLE,
	}, nil
}

func AddPathMod(_pred, _mod interface{}) (PathPattern, error) {
	return PathPattern{
		Predicate: _pred.(PathPattern).Predicate,
		Pattern:   _mod.(Pattern),
	}, nil
}

type PathPattern struct {
	Predicate turtle.URI
	Pattern   Pattern
}

func PathFromVar(_var interface{}) ([]PathPattern, error) {
	return []PathPattern{
		PathPattern{
			Predicate: turtle.ParseURI(_var.(string)),
			Pattern:   PATTERN_SINGLE,
		},
	}, nil
}

func (pp PathPattern) String() string {
	return pp.Predicate.String() + pp.Pattern.String()
}

type Pattern uint

const (
	PATTERN_SINGLE = iota + 1
	PATTERN_ZERO_ONE
	PATTERN_ONE_PLUS
	PATTERN_ZERO_PLUS
)

func (p Pattern) String() string {
	switch p {
	case PATTERN_SINGLE:
		return ""
	case PATTERN_ZERO_ONE:
		return "?"
	case PATTERN_ONE_PLUS:
		return "+"
	case PATTERN_ZERO_PLUS:
		return "*"
	}
	return "unknown"
}
<commit_msg>add Dump() method to ast.Query<commit_after>package ast

import (
	"fmt"
	"github.com/gtfierro/hod/lang/token"
	"github.com/gtfierro/hod/turtle"
	"github.com/kr/pretty"
	"strings"
)

type QueryType uint

const (
	SELECT_QUERY QueryType = 1 << iota
	INSERT_QUERY
	DELETE_QUERY
)

var debug = false

func SetDebug() {
	debug = true
}
func ClearDebug() {
	debug = false
}

type Query struct {
	Select    SelectClause
	From      FromClause
	Count     bool
	Insert    InsertClause
	Where     WhereClause
	Variables []string
	Type      QueryType
}

func (q Query) Dump() {
	for _, triple := range q.Where.Terms {
		fmt.Println(triple.String())
	}
	fmt.Println("----")
}

func (q Query) IsInsert() bool {
	return (q.Type & INSERT_QUERY) == INSERT_QUERY
}

func (q Query) IsSelect() bool {
	return (q.Type & SELECT_QUERY) == SELECT_QUERY
}

func (q Query) CopyWithNewTerms(terms []Triple) Query {
	newq := Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Type:      q.Type,
	}
	newq.Where.Terms = make([]Triple, len(terms))
	copy(newq.Where.Terms, terms)

	newq.Insert.Terms = make([]Triple, len(terms))
	copy(newq.Insert.Terms, terms)
	return newq
}

func (q Query) Copy() *Query {
	return &Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Where:     q.Where,
		Insert:    q.Insert,
		Count:     q.Count,
		Type:      q.Type,
	}
}

func NewQuery(selectclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: selectclause.(SelectClause),
		Count:  count,
		Type:   SELECT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewInsertQuery(insertclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
		fmt.Printf("%# v", pretty.Formatter(insertclause.(InsertClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: SelectClause{AllVars: true},
		Insert: insertclause.(InsertClause),
		Count:  count,
		Type:   INSERT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewQueryMulti(selectclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		From:   fromclause.(FromClause),
		Select: selectclause.(SelectClause),
		Count:  count,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func NewInsertQueryMulti(insertclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
		fmt.Printf("%# v", pretty.Formatter(insertclause.(InsertClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: SelectClause{AllVars: true},
		From:   fromclause.(FromClause),
		Insert: insertclause.(InsertClause),
		Count:  count,
		Type:   INSERT_QUERY,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	q.PopulateVars()
	if q.Select.AllVars {
		q.Select.Vars = q.Variables
	}
	return q, nil
}

func (q *Query) PopulateVars() {
	vars := make(map[string]int)
	// get all variables
	for _, triple := range q.Where.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	for _, triple := range q.Insert.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	if q.Where.GraphGroup != nil {
		VarsFromGroup(*q.Where.GraphGroup, vars)
	}
	q.Variables = []string{} // clear
	for varname := range vars {
		q.Variables = append(q.Variables, varname)
	}
}

func (q Query) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range q.Where.Terms {
		q.Where.Terms[idx] = f(triple)
	}
	for idx, triple := range q.Insert.Terms {
		q.Insert.Terms[idx] = f(triple)
	}
	if q.Where.GraphGroup != nil {
		q.Where.GraphGroup.IterTriples(f)
	}
}

func AddIfVar(uri turtle.URI, m map[string]int) {
	if uri.IsVariable() {
		m[uri.String()] = 1
	}
}

func VarsFromGroup(group GraphGroup, m map[string]int) {
	for _, triple := range group.Terms {
		AddIfVar(triple.Subject, m)
		AddIfVar(triple.Object, m)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, m)
		}
	}
	for _, union := range group.Unions {
		VarsFromGroup(union, m)
	}
}

func (grp GraphGroup) Expand() [][]Triple {
	var terms = make([]Triple, len(grp.Terms))
	copy(terms, grp.Terms)
	var groups [][]Triple

	if len(grp.Unions) > 0 {
		for _, union := range grp.Unions {
			for _, subgroup := range union.Expand() {
				groups = append(groups, append(terms, subgroup...))
			}
		}
	} else {
		groups = append(groups, terms)
	}
	return groups
}

func (grp GraphGroup) Iter(f func(t turtle.URI)) {
	for _, triple := range grp.Terms {
		f(triple.Subject)
		f(triple.Object)
		for _, path := range triple.Predicates {
			f(path.Predicate)
		}
	}
	for _, union := range grp.Unions {
		union.Iter(f)
	}
}

func (grp *GraphGroup) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range grp.Terms {
		grp.Terms[idx] = f(triple)
	}
	for _, union := range grp.Unions {
		union.IterTriples(f)
	}
}

type SelectClause struct {
	Vars    []string
	AllVars bool
}

func NewAllSelectClause() (SelectClause, error) {
	return SelectClause{AllVars: true}, nil
}

func NewSelectClause(varlist interface{}) (SelectClause, error) {
	return SelectClause{Vars: varlist.([]string)}, nil
}

type InsertClause struct {
	Terms []Triple
}

func NewInsertClause(triples interface{}) (InsertClause, error) {
	return InsertClause{
		Terms: triples.([]Triple),
	}, nil
}

type FromClause struct {
	Databases []string
	AllDBs    bool
}

func (f FromClause) String() string {
	if f.AllDBs {
		return "*"
	}
	return strings.Join(f.Databases, " ")
}

func NewAllFromClause() (FromClause, error) {
	return FromClause{AllDBs: true}, nil
}

func NewFromClause(dblist interface{}) (FromClause, error) {
	return FromClause{Databases: dblist.([]string)}, nil
}

func (from FromClause) Empty() bool {
	return len(from.Databases) == 0 && !from.AllDBs
}

type WhereClause struct {
	Terms      []Triple
	GraphGroup *GraphGroup
}

func NewWhereClause(triples interface{}) (WhereClause, error) {
	return WhereClause{
		Terms: triples.([]Triple),
	}, nil
}

func NewWhereClauseWithGraphGroup(triples, group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		Terms:      triples.([]Triple),
		GraphGroup: &g,
	}, nil
}

func NewWhereClauseGraphGroup(group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		GraphGroup: &g,
	}, nil
}

type GraphGroup struct {
	Terms  []Triple
	Unions []GraphGroup
}

func GraphGroupFromTriples(triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms: triples.([]Triple),
	}, nil
}

func GraphGroupUnion(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Unions: []GraphGroup{left.(GraphGroup), right.(GraphGroup)},
	}, nil
}

func AddTriplesToGraphGroup(left, triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, triples.([]Triple)...),
		Unions: left.(GraphGroup).Unions,
	}, nil
}

func MergeGraphGroups(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, right.(GraphGroup).Terms...),
		Unions: append(left.(GraphGroup).Unions, right.(GraphGroup).Unions...),
	}, nil
}

type Triple struct {
	Subject    turtle.URI
	Predicates []PathPattern
	Object     turtle.URI
}

func (t Triple) String() string {
	s := "<" + t.Subject.String() + "|"
	for _, pp := range t.Predicates {
		s += " " + pp.String()
	}
	return s + " | " + t.Object.String() + ">"
}

func (t Triple) Copy() Triple {
	var p = make([]PathPattern, len(t.Predicates))
	copy(p, t.Predicates)
	return Triple{
		Subject:    t.Subject,
		Object:     t.Object,
		Predicates: p,
	}
}

func NewTriple(subject, predicates, object interface{}) (Triple, error) {
	return Triple{
		Subject:    subject.(turtle.URI),
		Predicates: predicates.([]PathPattern),
		Object:     object.(turtle.URI),
	}, nil
}

func NewTripleBlock(triple interface{}) ([]Triple, error) {
	return []Triple{triple.(Triple)}, nil
}

func AppendTripleBlock(block, triple interface{}) ([]Triple, error) {
	return append(block.([]Triple), triple.(Triple)), nil
}

func NewURI(value interface{}) (turtle.URI, error) {
	return turtle.ParseURI(value.(string)), nil
}

func NewVarList(_var interface{}) ([]string, error) {
	return []string{_var.(string)}, nil
}

func AppendVar(varlist, _var interface{}) ([]string, error) {
	return append(varlist.([]string), _var.(string)), nil
}

func NewStringList(_str interface{}) ([]string, error) {
	return []string{_str.(string)}, nil
}

func AppendString(strlist, _str interface{}) ([]string, error) {
	return append(strlist.([]string), _str.(string)), nil
}

func ParseString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func ParseQuotedString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func NewPathSequence(_pred interface{}) ([]PathPattern, error) {
	return []PathPattern{_pred.(PathPattern)}, nil
}

func AppendPathSequence(_seq, _pred interface{}) ([]PathPattern, error) {
	return append(_seq.([]PathPattern), _pred.(PathPattern)), nil
}

func NewPathPattern(_pred interface{}) (PathPattern, error) {
	pred, _ := ParseString(_pred)
	if pred == "a" {
		pred = "rdf:type"
	}
	return PathPattern{
		Predicate: turtle.ParseURI(pred),
		Pattern:   PATTERN_SINGLE,
	}, nil
}

func AddPathMod(_pred, _mod interface{}) (PathPattern, error) {
	return PathPattern{
		Predicate: _pred.(PathPattern).Predicate,
		Pattern:   _mod.(Pattern),
	}, nil
}

type PathPattern struct {
	Predicate turtle.URI
	Pattern   Pattern
}

func PathFromVar(_var interface{}) ([]PathPattern, error) {
	return []PathPattern{
		PathPattern{
			Predicate: turtle.ParseURI(_var.(string)),
			Pattern:   PATTERN_SINGLE,
		},
	}, nil
}

func (pp PathPattern) String() string {
	return pp.Predicate.String() + pp.Pattern.String()
}

type Pattern uint

const (
	PATTERN_SINGLE = iota + 1
	PATTERN_ZERO_ONE
	PATTERN_ONE_PLUS
	PATTERN_ZERO_PLUS
)

func (p Pattern) String() string {
	switch p {
	case PATTERN_SINGLE:
		return ""
	case PATTERN_ZERO_ONE:
		return "?"
	case PATTERN_ONE_PLUS:
		return "+"
	case PATTERN_ZERO_PLUS:
		return "*"
	}
	return "unknown"
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package reformat provides a module that "wraps" an existing module and transforms it's output.
This can be useful for adding extra formatting simple bar modules.

For example, a time module might use strftime-style format strings,
which don't allow for colours or borders. You can add those using reformat:

 t := localtime.New(...)
 r := reformat.New(t).Format(func(o bar.Output) bar.Output {
   return o.Background("red").Padding(20)
 })
*/
package reformat

import (
	"sync"

	"github.com/soumya92/barista/bar"
	l "github.com/soumya92/barista/logging"
	"github.com/soumya92/barista/outputs"
)

// FormatFunc takes the module's output and returns a modified version.
type FormatFunc func(bar.Output) bar.Output

// Original returns the original output unchanged.
func Original(in bar.Output) bar.Output {
	return in
}

// Hide replaces all outputs with nil, hiding them from the bar.
func Hide(in bar.Output) bar.Output {
	return nil
}

// SegmentFunc is a reformatting function at the segment level.
type SegmentFunc func(*bar.Segment) *bar.Segment

// EachSegment transforms each segment individually.
func EachSegment(f SegmentFunc) FormatFunc {
	return func(in bar.Output) bar.Output {
		if in == nil {
			return in
		}
		out := outputs.Group()
		for _, s := range in.Segments() {
			out.Append(f(s.Clone()))
		}
		return out
	}
}

// SkipErrors wraps a segment transformation function so that
// error segments pass through unchanged.
func SkipErrors(f SegmentFunc) SegmentFunc {
	return func(in *bar.Segment) *bar.Segment {
		if in.GetError() != nil {
			return in
		}
		return f(in)
	}
}

// Module stores the original module, the re-formatting function, and
// helpers required to allow dynamic re-formatting.
type Module struct {
	module        bar.Module
	sink          bar.Sink
	mu            sync.Mutex
	formatter     FormatFunc
	restarted     chan bool
	lastOutput    bar.Output
	lastFormatted bar.Output
	started       bool
	finished      bool
}

// New wraps an existing bar.Module, allowing the format to be changed
// before being sent to the bar.
func New(original bar.Module) *Module {
	m := &Module{
		module:    original,
		restarted: make(chan bool),
		formatter: Original,
	}
	l.Label(m, l.ID(original))
	return m
}

// Format sets the reformat function.
func (m *Module) Format(f FormatFunc) *Module {
	l.Fine("%s.Format(%s)", l.ID(m), l.ID(f))
	m.mu.Lock()
	defer m.mu.Unlock()
	if f == nil {
		f = Original
	}
	m.formatter = f
	if m.started {
		go m.sink.Output(m.lastOutput)
	}
	return m
}

// Stream sets up the output pipeline to filter outputs when hidden.
func (m *Module) Stream(s bar.Sink) {
	wSink := wrappedSink(m, s)
	m.mu.Lock()
	m.sink = wSink
	m.mu.Unlock()
	for {
		m.module.Stream(wSink)
		m.mu.Lock()
		m.finished = true
		m.started = false
		m.mu.Unlock()
		<-m.restarted
		m.mu.Lock()
		nonErrorOutput := outputs.Group()
		formatted := m.formatter(m.lastOutput)
		if formatted != nil {
			for _, o := range formatted.Segments() {
				if o.GetError() == nil {
					nonErrorOutput.Append(o)
				}
			}
		}
		s.Output(nonErrorOutput)
		m.mu.Unlock()
	}
}

// Click passes through the click event if supported by the wrapped module.
func (m *Module) Click(e bar.Event) {
	m.mu.Lock()
	if m.finished && isRestartableClick(e) {
		l.Log("%s restarted", l.ID(m))
		m.finished = false
		m.restarted <- true
		m.mu.Unlock()
		return
	}
	m.mu.Unlock()
	if clickable, ok := m.module.(bar.Clickable); ok {
		clickable.Click(e)
	}
}

// isRestartableClick mimics the function in barista main.
// Modules that have finished can still be reformatted,
// so the wrapping module needs to keep running.
func isRestartableClick(e bar.Event) bool {
	return e.Button == bar.ButtonLeft ||
		e.Button == bar.ButtonRight ||
		e.Button == bar.ButtonMiddle
}

func wrappedSink(m *Module, s bar.Sink) bar.Sink {
	return func(o bar.Output) {
		m.mu.Lock()
		m.lastOutput = o
		m.lastFormatted = m.formatter(o)
		out := m.lastFormatted
		m.started = true
		m.mu.Unlock()
		s.Output(out)
	}
}
<commit_msg>reformat: only cache original output, not formatted output<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

/*
Package reformat provides a module that "wraps" an existing module and transforms it's output.
This can be useful for adding extra formatting simple bar modules.

For example, a time module might use strftime-style format strings,
which don't allow for colours or borders. You can add those using reformat:

 t := localtime.New(...)
 r := reformat.New(t).Format(func(o bar.Output) bar.Output {
   return o.Background("red").Padding(20)
 })
*/
package reformat

import (
	"sync"

	"github.com/soumya92/barista/bar"
	l "github.com/soumya92/barista/logging"
	"github.com/soumya92/barista/outputs"
)

// FormatFunc takes the module's output and returns a modified version.
type FormatFunc func(bar.Output) bar.Output

// Original returns the original output unchanged.
func Original(in bar.Output) bar.Output {
	return in
}

// Hide replaces all outputs with nil, hiding them from the bar.
func Hide(in bar.Output) bar.Output {
	return nil
}

// SegmentFunc is a reformatting function at the segment level.
type SegmentFunc func(*bar.Segment) *bar.Segment

// EachSegment transforms each segment individually.
func EachSegment(f SegmentFunc) FormatFunc {
	return func(in bar.Output) bar.Output {
		if in == nil {
			return in
		}
		out := outputs.Group()
		for _, s := range in.Segments() {
			out.Append(f(s.Clone()))
		}
		return out
	}
}

// SkipErrors wraps a segment transformation function so that
// error segments pass through unchanged.
func SkipErrors(f SegmentFunc) SegmentFunc {
	return func(in *bar.Segment) *bar.Segment {
		if in.GetError() != nil {
			return in
		}
		return f(in)
	}
}

// Module stores the original module, the re-formatting function, and
// helpers required to allow dynamic re-formatting.
type Module struct {
	module     bar.Module
	sink       bar.Sink
	mu         sync.Mutex
	formatter  FormatFunc
	restarted  chan bool
	lastOutput bar.Output
	started    bool
	finished   bool
}

// New wraps an existing bar.Module, allowing the format to be changed
// before being sent to the bar.
func New(original bar.Module) *Module {
	m := &Module{
		module:    original,
		restarted: make(chan bool),
		formatter: Original,
	}
	l.Label(m, l.ID(original))
	return m
}

// Format sets the reformat function.
func (m *Module) Format(f FormatFunc) *Module {
	l.Fine("%s.Format(%s)", l.ID(m), l.ID(f))
	m.mu.Lock()
	defer m.mu.Unlock()
	if f == nil {
		f = Original
	}
	m.formatter = f
	if m.started {
		go m.sink.Output(m.lastOutput)
	}
	return m
}

// Stream sets up the output pipeline to filter outputs when hidden.
func (m *Module) Stream(s bar.Sink) {
	wSink := wrappedSink(m, s)
	m.mu.Lock()
	m.sink = wSink
	m.mu.Unlock()
	for {
		m.module.Stream(wSink)
		m.mu.Lock()
		m.finished = true
		m.started = false
		m.mu.Unlock()
		<-m.restarted
		m.mu.Lock()
		nonErrorOutput := outputs.Group()
		formatted := m.formatter(m.lastOutput)
		if formatted != nil {
			for _, o := range formatted.Segments() {
				if o.GetError() == nil {
					nonErrorOutput.Append(o)
				}
			}
		}
		s.Output(nonErrorOutput)
		m.mu.Unlock()
	}
}

// Click passes through the click event if supported by the wrapped module.
func (m *Module) Click(e bar.Event) {
	m.mu.Lock()
	if m.finished && isRestartableClick(e) {
		l.Log("%s restarted", l.ID(m))
		m.finished = false
		m.restarted <- true
		m.mu.Unlock()
		return
	}
	m.mu.Unlock()
	if clickable, ok := m.module.(bar.Clickable); ok {
		clickable.Click(e)
	}
}

// isRestartableClick mimics the function in barista main.
// Modules that have finished can still be reformatted,
// so the wrapping module needs to keep running.
func isRestartableClick(e bar.Event) bool {
	return e.Button == bar.ButtonLeft ||
		e.Button == bar.ButtonRight ||
		e.Button == bar.ButtonMiddle
}

func wrappedSink(m *Module, s bar.Sink) bar.Sink {
	return func(o bar.Output) {
		m.mu.Lock()
		m.lastOutput = o
		f := m.formatter
		m.started = true
		m.mu.Unlock()
		s.Output(f(o))
	}
}
<|endoftext|>


<commit_before>package triggers

type Donators struct{}

func (d *Donators) Triggers() []string {
    return []string{
        "donators",
        "donations",
        "donate",
        "supporters",
        "support",
        "patreon",
        "patreons",
        "credits",
    }
}

func (d *Donators) Response(trigger string, content string) string {
    return "<:robyulblush:327206930437373952> **These awesome people support me:**\nKakkela 💕\nSunny 💓\nsomicidal minaiac 💞\nOokami 💖\nKeldra 💗\nThank you so much!\n_You want to be in this list? <https://www.patreon.com/sekl>!_"
}
<commit_msg>[donators] adds TN!<commit_after>package triggers

type Donators struct{}

func (d *Donators) Triggers() []string {
    return []string{
        "donators",
        "donations",
        "donate",
        "supporters",
        "support",
        "patreon",
        "patreons",
        "credits",
    }
}

func (d *Donators) Response(trigger string, content string) string {
    return "<:robyulblush:327206930437373952> **These awesome people support me:**\nKakkela 💕\nSunny 💓\nsomicidal minaiac 💞\nOokami 💖\nKeldra 💗\nTN 💝\nThank you so much!\n_You want to be in this list? <https://www.patreon.com/sekl>!_"
}
<|endoftext|>


<commit_before>package albumId

type album struct {
	id        string
	albumID   string
	albumName string
}

type albums []album

var albumList = albums{
	{"1", "18918", "jay"},
	{"2", "18915", "范特西"},
	{"3", "18907", "八度空间"},
	{"4", "18905", "叶惠美"},
	{"5", "18918", "七里香"},
	{"6", "18896", "十一月的萧邦"},
	{"7", "18893", "依然范特西"},
	{"8", "18886", "我很忙"},
	{"9", "18877", "魔杰座"},
	{"10", "18875", "跨时代"},
	{"11", "18869", "惊叹号"},
	{"12", "2263029", "十二新作"},
	{"13", "3084335", "哎呦，不错哦"},
	{"14", "34720827", "周杰伦的床边故事"},
}

func GetAlbumID(idx string) interface{} {
	for _, album := range albumList {
		if idx == album.albumID {
			mAlbum := map[string]string{"albumID": album.albumID, "albumName": album.albumName}
			return mAlbum
		}
	}
	return "Choose a correct album."
}
<commit_msg>Update export field<commit_after>package albumId

// Album export album struct
type Album struct {
	ID        string
	AlbumID   string
	AlbumName string
}

// AlbumList export album list
var AlbumList = []Album{
	{"1", "18918", "jay"},
	{"2", "18915", "范特西"},
	{"3", "18907", "八度空间"},
	{"4", "18905", "叶惠美"},
	{"5", "18918", "七里香"},
	{"6", "18896", "十一月的萧邦"},
	{"7", "18893", "依然范特西"},
	{"8", "18886", "我很忙"},
	{"9", "18877", "魔杰座"},
	{"10", "18875", "跨时代"},
	{"11", "18869", "惊叹号"},
	{"12", "2263029", "十二新作"},
	{"13", "3084335", "哎呦，不错哦"},
	{"14", "34720827", "周杰伦的床边故事"},
}

// GetAlbumID for get album's id from user's input
func GetAlbumID(idx string) (aAlbum Album) {
	for _, album := range AlbumList {
		if idx == album.ID {
			aAlbum.AlbumID = album.AlbumID
			aAlbum.AlbumName = album.AlbumName
		}
	}
	return
}
<|endoftext|>


<commit_before>package alg

// TriangulateConvex takes a face, in the form of a slice
// of indices, and outputs those indicies split into triangles
// based on the assumption that the face is convex. This involves
// forming pairs of indices and drawing an edge back to the first
// index repeatedly.
//
// If given less than 3 indices, returns an empty slice.
//
// Example input: [0,1,2,3,4]
// Example output: [[0,1,2][0,2,3][0,3,4]]
//
// Visual Example:
//      ____0____
//     4         1
//      \       /
//       3-----2
//        - - -
//      ____0____
//     4   / \   1
//      \ /   \ /
//       3-----2
func TriangulateConvex(face []int) [][3]int {
	if len(face) < 3 {
		return [][3]int{}
	}
	tris := make([][3]int, len(face)-2)
	for i := 0; i < len(tris); i++ {
		tris[i][0] = face[0]
		tris[i][1] = face[i+1]
		tris[i][2] = face[i+2]
	}
	return tris
}
<commit_msg>Additional notes on TriangulateConvex<commit_after>package alg

// TriangulateConvex takes a face, in the form of a slice
// of indices, and outputs those indicies split into triangles
// based on the assumption that the face is convex. This involves
// forming pairs of indices and drawing an edge back to the first
// index repeatedly.
//
// If given less than 3 indices, returns an empty slice.
//
// Example input: [0,1,2,3,4]
// Example output: [[0,1,2][0,2,3][0,3,4]]
//
// Visual Example:
//      ____0____
//     4         1
//      \       /
//       3-----2
//        - - -
//      ____0____
//     4   / \   1
//      \ /   \ /
//       3-----2
//
// This makes additional assumptions that the points represented
// by the indices are coplanar, and that there are no holes present
// in the face.
func TriangulateConvex(face []int) [][3]int {
	if len(face) < 3 {
		return [][3]int{}
	}
	tris := make([][3]int, len(face)-2)
	for i := 0; i < len(tris); i++ {
		tris[i][0] = face[0]
		tris[i][1] = face[i+1]
		tris[i][2] = face[i+2]
	}
	return tris
}
<|endoftext|>