<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package pcap2log

import (
	"fmt"
	"github.com/kr/pretty"
	"io"
	"log"
)

var _ = pretty.Print

type Order struct {
	id       uint
	optionId OptionId
	price    uint
	size     uint
	side     MarketSide
}

type simulator struct {
	w                io.Writer
	ops              []OrderOperation
	orders           map[uint]Order
	assumeSubscribed bool
}

func NewSimulator(w io.Writer) simulator {
	return simulator{
		w:                w,
		orders:           make(map[uint]Order),
		assumeSubscribed: true,
	}
}

func (s *simulator) addMessage(qom *QOMessage, typeChar byte) {
	if qom.typ == MessageTypeUnknown {
		return
	}
	//log.Printf("addMessage(%#v, %c)\n", qom, typeChar)
	s.outMessageNorm(qom, typeChar)
	s.addMessageOperations(qom)
	s.processOperations()
}

type OrderOp byte

const (
	OrderOpUnknown OrderOp = iota
	OrderOpAdd
	OrderOpRemove
	OrderOpUpdate
)

type OrderOperation struct {
	op       OrderOp
	orderId  uint
	optionId OptionId
	size     uint
	price    uint
	side     MarketSide
}

func (s *simulator) addMessageOperations(m *QOMessage) {
	switch m.typ {
	case MessageTypeQuoteAdd:
		s.addOp(OrderOpAdd, m.side1, m.optionId)
		s.addOp(OrderOpAdd, m.side2, m.optionId)
	case MessageTypeQuoteReplace:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
		s.addOp(OrderOpAdd, m.side1, OptionIdUnknown)
		s.addOp(OrderOpRemove, m.side2, OptionIdUnknown)
		s.addOp(OrderOpAdd, m.side2, OptionIdUnknown)
	case MessageTypeQuoteDelete:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
		s.addOp(OrderOpRemove, m.side2, OptionIdUnknown)
	case MessageTypeOrderAdd:
		s.addOp(OrderOpAdd, m.side1, m.optionId)
	case MessageTypeOrderExecute, MessageTypeOrderExecuteWPrice, MessageTypeOrderCancel:
		s.addOp(OrderOpUpdate, m.side1, OptionIdUnknown)
	case MessageTypeOrderUpdate:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
		order := m.side1
		order.refNumDelta = order.origRefNumDelta
		s.addOp(OrderOpAdd, order, OptionIdUnknown)
	case MessageTypeOrderReplace:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
		s.addOp(OrderOpAdd, m.side1, OptionIdUnknown)
	case MessageTypeOrderDelete:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
	case MessageTypeBlockOrderDelete:
		for _, r := range m.bssdRefs {
			s.addOp(OrderOpRemove, OrderSide{origRefNumDelta: r}, OptionIdUnknown)
		}
	default:
		log.Fatalf("Unexpected message type %d in %+v\n", m.typ, m)
	}
}

func (s *simulator) addOp(op OrderOp, order OrderSide, optionId OptionId) {
	var o OrderOperation
	switch op {
	case OrderOpAdd:
		o = OrderOperation{
			op:       op,
			optionId: optionId,
			orderId:  order.refNumDelta,
			size:     order.size,
			price:    order.price,
			side:     order.side,
		}
	case OrderOpRemove:
		o = OrderOperation{
			op:      op,
			orderId: order.origRefNumDelta,
		}
	case OrderOpUpdate:
		o = OrderOperation{
			op:      op,
			orderId: order.origRefNumDelta,
			size:    order.size,
		}
	default:
		log.Fatal("Unexpected order operation")
	}
	s.ops = append(s.ops, o)
}

func (s *simulator) processOperations() {
	var prev struct {
		valid     bool
		nextValid bool
		optionId  OptionId
		side      MarketSide
	}
	for i := range s.ops {
		prev.valid, prev.nextValid = prev.nextValid, false
		op := &s.ops[i]
		order, orderFound := s.orders[op.orderId]
		s.outOrderLookup(op, order, orderFound)
		if op.op == OrderOpAdd {
			if orderFound {
				log.Fatalf("Order already exist when adding op=%#v order=%#v", op, order)
			}
			if op.optionId == OptionIdUnknown {
				if s.ops[i-1].op != OrderOpRemove {
					log.Fatal("Unexpected add operation", op)
				}
				if !prev.valid || i == 0 {
					continue
				}
				op.optionId = prev.optionId
				if op.side == MarketSideUnknown {
					op.side = prev.side
				}
			}
			if s.assumeSubscribed {
				s.subscibe(op.optionId)
			} else if !s.subscribed(op.optionId) {
				continue
			}
		} else {
			if !orderFound {
				continue
			}
			if op.op == OrderOpRemove {
				prev.nextValid = true
				prev.optionId = order.optionId
				prev.side = order.side
			}
		}
		s.updateOrders(op, order)
	}
	s.ops = s.ops[:0]
}

func (s *simulator) updateOrders(op *OrderOperation, order Order) {
	switch op.op {
	case OrderOpAdd:
		order = Order{
			id:       op.orderId,
			optionId: op.optionId,
			price:    op.price,
			size:     op.size,
			side:     op.side,
		}
		s.orders[op.orderId] = order
	case OrderOpUpdate:
		order.size -= op.size
		s.orders[op.orderId] = order
	case OrderOpRemove:
		order = Order{
			id: op.orderId,
		}
		delete(s.orders, op.orderId)
	default:
		log.Fatal("Unexpected order operation", op)
	}
	s.outOrderUpdate(op, order)
}

func (s *simulator) subscibe(optionId OptionId) {
}

func (s *simulator) subscribed(optionId OptionId) bool {
	return true
}

// output functions

func (s *simulator) Printf(format string, vs ...interface{}) {
	if _, err := fmt.Fprintf(s.w, format, vs...); err != nil {
		log.Fatal("output error", err)
	}
}

func (s *simulator) Printfln(format string, vs ...interface{}) {
	s.Printf(format, vs...)
	if _, err := fmt.Fprintln(s.w); err != nil {
		log.Fatal("output error", err)
	}
}

func (s *simulator) outMessageNorm(m *QOMessage, typeChar byte) {
	out := func(name, f string, vs ...interface{}) {
		s.Printf("NORM %s %c ", name, typeChar)
		s.Printfln(f, vs...)
	}
	ord, bid, ask := &m.side1, &m.side1, &m.side2
	if bid.side == MarketSideSell {
		bid, ask = ask, bid
	}
	switch m.typ {
	case MessageTypeQuoteAdd:
		out("QBID", "%08x %08x %08x %08x", m.optionId, bid.refNumDelta, bid.size, bid.price)
		out("QASK", "%08x %08x %08x %08x", m.optionId, ask.refNumDelta, ask.size, ask.price)
	case MessageTypeQuoteReplace:
		out("QBID", "%08x %08x %08x %08x", bid.refNumDelta, bid.origRefNumDelta, bid.size, bid.price)
		out("QASK", "%08x %08x %08x %08x", ask.refNumDelta, ask.origRefNumDelta, ask.size, ask.price)
	case MessageTypeQuoteDelete:
		out("QBID", "%08x", bid.origRefNumDelta)
		out("QASK", "%08x", ask.origRefNumDelta)
	case MessageTypeOrderAdd:
		out("ORDER", "%c %08x %08x %08x %08x", ord.side, m.optionId, ord.refNumDelta, ord.size, ord.price)
	case MessageTypeOrderExecute, MessageTypeOrderExecuteWPrice, MessageTypeOrderCancel:
		out("ORDER", "%08x %08x", ord.origRefNumDelta, ord.size)
	case MessageTypeOrderUpdate:
		out("ORDER", "%08x %08x %08x", ord.origRefNumDelta, ord.size, ord.price)
	case MessageTypeOrderReplace:
		out("ORDER", "%08x %08x %08x %08x", ord.refNumDelta, ord.origRefNumDelta, ord.size, ord.price)
	case MessageTypeOrderDelete:
		out("ORDER", "%08x", ord.origRefNumDelta)
	case MessageTypeBlockOrderDelete:
		for _, r := range m.bssdRefs {
			out("ORDER", "%08x", r)
		}
	default:
		log.Fatalf("Unexpected message type %d in %+v\n", m.typ, m)
	}
}

func bool2int(b bool) int {
	if b {
		return 1
	} else {
		return 0
	}
}

func marketSide2int(ms MarketSide) int {
	if ms == MarketSideSell {
		return 1
	} else {
		return 0
	}
}

func (s *simulator) outOrderLookup(op *OrderOperation, order Order, isFound bool) {
	if op.op == OrderOpAdd {
		s.Printfln("ORDL 1 %08x %08x", op.orderId, op.optionId)
		order.optionId = op.optionId
	} else {
		s.Printfln("ORDL 0 %08x", op.orderId)
	}
	s.Printfln("ORDRESP %d %d %d %08x %08x %08x",
		bool2int(!isFound && op.op != OrderOpAdd),
		bool2int(op.op == OrderOpAdd),
		marketSide2int(order.side),
		order.size,
		order.price,
		order.optionId,
	)

}

func (s *simulator) outOrderUpdate(op *OrderOperation, order Order) {
	s.Printfln("ORDU %08x %08x %d %08x %08x",
		order.id,
		order.optionId,
		marketSide2int(order.side),
		order.price,
		order.size,
	)
}
<commit_msg>pcap2log: add per-option state<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package pcap2log

import (
	"fmt"
	"github.com/kr/pretty"
	"io"
	"log"
)

var _ = pretty.Print

type Order struct {
	id       uint
	optionId OptionId
	price    uint
	size     uint
	side     MarketSide
}

type simulator struct {
	w                io.Writer
	ops              []OrderOperation
	orders           map[uint]Order
	options          map[OptionId]*OptionState
	assumeSubscribed bool
}

func NewSimulator(w io.Writer) simulator {
	return simulator{
		w:                w,
		orders:           make(map[uint]Order),
		options:          make(map[OptionId]*OptionState),
		assumeSubscribed: true,
	}
}

func (s *simulator) addMessage(qom *QOMessage, typeChar byte) {
	if qom.typ == MessageTypeUnknown {
		return
	}
	//log.Printf("addMessage(%#v, %c)\n", qom, typeChar)
	s.outMessageNorm(qom, typeChar)
	s.addMessageOperations(qom)
	s.processOperations()
}

type OrderOp byte

const (
	OrderOpUnknown OrderOp = iota
	OrderOpAdd
	OrderOpRemove
	OrderOpUpdate
)

type OrderOperation struct {
	op       OrderOp
	orderId  uint
	optionId OptionId
	size     uint
	price    uint
	side     MarketSide
}

func (s *simulator) addMessageOperations(m *QOMessage) {
	switch m.typ {
	case MessageTypeQuoteAdd:
		s.addOp(OrderOpAdd, m.side1, m.optionId)
		s.addOp(OrderOpAdd, m.side2, m.optionId)
	case MessageTypeQuoteReplace:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
		s.addOp(OrderOpAdd, m.side1, OptionIdUnknown)
		s.addOp(OrderOpRemove, m.side2, OptionIdUnknown)
		s.addOp(OrderOpAdd, m.side2, OptionIdUnknown)
	case MessageTypeQuoteDelete:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
		s.addOp(OrderOpRemove, m.side2, OptionIdUnknown)
	case MessageTypeOrderAdd:
		s.addOp(OrderOpAdd, m.side1, m.optionId)
	case MessageTypeOrderExecute, MessageTypeOrderExecuteWPrice, MessageTypeOrderCancel:
		s.addOp(OrderOpUpdate, m.side1, OptionIdUnknown)
	case MessageTypeOrderUpdate:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
		order := m.side1
		order.refNumDelta = order.origRefNumDelta
		s.addOp(OrderOpAdd, order, OptionIdUnknown)
	case MessageTypeOrderReplace:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
		s.addOp(OrderOpAdd, m.side1, OptionIdUnknown)
	case MessageTypeOrderDelete:
		s.addOp(OrderOpRemove, m.side1, OptionIdUnknown)
	case MessageTypeBlockOrderDelete:
		for _, r := range m.bssdRefs {
			s.addOp(OrderOpRemove, OrderSide{origRefNumDelta: r}, OptionIdUnknown)
		}
	default:
		log.Fatalf("Unexpected message type %d in %+v\n", m.typ, m)
	}
}

func (s *simulator) addOp(op OrderOp, order OrderSide, optionId OptionId) {
	var o OrderOperation
	switch op {
	case OrderOpAdd:
		o = OrderOperation{
			op:       op,
			optionId: optionId,
			orderId:  order.refNumDelta,
			size:     order.size,
			price:    order.price,
			side:     order.side,
		}
	case OrderOpRemove:
		o = OrderOperation{
			op:      op,
			orderId: order.origRefNumDelta,
		}
	case OrderOpUpdate:
		o = OrderOperation{
			op:      op,
			orderId: order.origRefNumDelta,
			size:    order.size,
		}
	default:
		log.Fatal("Unexpected order operation")
	}
	s.ops = append(s.ops, o)
}

func (s *simulator) processOperations() {
	var prev struct {
		valid     bool
		nextValid bool
		optionId  OptionId
		side      MarketSide
	}
	for i := range s.ops {
		prev.valid, prev.nextValid = prev.nextValid, false
		op := &s.ops[i]
		order, orderFound := s.orders[op.orderId]
		s.outOrderLookup(op, order, orderFound)
		if op.op == OrderOpAdd {
			if orderFound {
				log.Fatalf("Order already exist when adding op=%#v order=%#v", op, order)
			}
			if op.optionId == OptionIdUnknown {
				if s.ops[i-1].op != OrderOpRemove {
					log.Fatal("Unexpected add operation", op)
				}
				if !prev.valid || i == 0 {
					continue
				}
				op.optionId = prev.optionId
				if op.side == MarketSideUnknown {
					op.side = prev.side
				}
			}
			if s.assumeSubscribed {
				s.subscibe(op.optionId)
			} else if !s.subscribed(op.optionId) {
				continue
			}
		} else {
			if !orderFound {
				continue
			}
			if op.optionId != OptionIdUnknown || op.side != MarketSideUnknown || op.price != 0 {
				log.Fatalf("unexpected operation parameters %#v\n", op)
			}
			op.optionId = order.optionId
			op.side = order.side
			op.price = order.price
			if op.op == OrderOpRemove {
				prev.nextValid = true
				prev.optionId = order.optionId
				prev.side = order.side
			}
		}
		s.updateOrders(op, order)
		s.updateOptionState(*op)
	}
	s.ops = s.ops[:0]
}

func (s *simulator) updateOrders(op *OrderOperation, order Order) {
	switch op.op {
	case OrderOpAdd:
		order = Order{
			id:       op.orderId,
			optionId: op.optionId,
			price:    op.price,
			size:     op.size,
			side:     op.side,
		}
		s.orders[op.orderId] = order
	case OrderOpUpdate:
		order.size -= op.size
		s.orders[op.orderId] = order
	case OrderOpRemove:
		order = Order{
			id: op.orderId,
		}
		delete(s.orders, op.orderId)
	default:
		log.Fatal("Unexpected order operation", op)
	}
	s.outOrderUpdate(op, order)
}

func (s *simulator) updateOptionState(op OrderOperation) {
	if op.optionId == OptionIdUnknown || op.side == MarketSideUnknown {
		log.Fatalf("unexpected operation parameters op=%#v order=%#v newOrder=%#v\n", op)
	}
	optionState := s.options[op.optionId]
	if optionState == nil {
		log.Fatal("unexpectedly not subscribed to", op.optionId)
	}
	var delta int
	if op.op == OrderOpAdd {
		delta = int(op.size)
	} else {
		delta = int(-op.size)
	}

	side := optionState.side(op.side)
	side.updateLevel(op.price, delta)
}

func (s *simulator) subscibe(optionId OptionId) {
	if !s.subscribed(optionId) {
		s.options[optionId] = NewOptionState(optionId)
	}
}

func (s *simulator) subscribed(optionId OptionId) bool {
	_, ok := s.options[optionId]
	return ok
}

type OptionState struct {
	bid *OptionSideState
	ask *OptionSideState
}

func NewOptionState(id OptionId) *OptionState {
	return &OptionState{
		bid: NewOptionSideState(MarketSideBuy),
		ask: NewOptionSideState(MarketSideSell),
	}
}

func (s *OptionState) side(ms MarketSide) (res *OptionSideState) {
	switch ms {
	case MarketSideBuy:
		res = s.bid
	case MarketSideSell:
		res = s.ask
	default:
		log.Fatal("Unexpected market side", ms, "for option", s)
	}
	return
}

type OptionSideState struct {
	levels map[uint]uint
}

func NewOptionSideState(side MarketSide) *OptionSideState {
	s := OptionSideState{
		levels: make(map[uint]uint),
	}
	return &s
}

func (s *OptionSideState) updateLevel(price uint, delta int) {
	ssize := int(s.levels[price]) + delta
	if ssize < 0 {
		log.Fatal("size become negative")
	}
	if ssize == 0 {
		delete(s.levels, price)
	} else {
		s.levels[price] = uint(ssize)
	}
}

// output functions

func (s *simulator) Printf(format string, vs ...interface{}) {
	if _, err := fmt.Fprintf(s.w, format, vs...); err != nil {
		log.Fatal("output error", err)
	}
}

func (s *simulator) Printfln(format string, vs ...interface{}) {
	s.Printf(format, vs...)
	if _, err := fmt.Fprintln(s.w); err != nil {
		log.Fatal("output error", err)
	}
}

func (s *simulator) outMessageNorm(m *QOMessage, typeChar byte) {
	out := func(name, f string, vs ...interface{}) {
		s.Printf("NORM %s %c ", name, typeChar)
		s.Printfln(f, vs...)
	}
	ord, bid, ask := &m.side1, &m.side1, &m.side2
	if bid.side == MarketSideSell {
		bid, ask = ask, bid
	}
	switch m.typ {
	case MessageTypeQuoteAdd:
		out("QBID", "%08x %08x %08x %08x", m.optionId, bid.refNumDelta, bid.size, bid.price)
		out("QASK", "%08x %08x %08x %08x", m.optionId, ask.refNumDelta, ask.size, ask.price)
	case MessageTypeQuoteReplace:
		out("QBID", "%08x %08x %08x %08x", bid.refNumDelta, bid.origRefNumDelta, bid.size, bid.price)
		out("QASK", "%08x %08x %08x %08x", ask.refNumDelta, ask.origRefNumDelta, ask.size, ask.price)
	case MessageTypeQuoteDelete:
		out("QBID", "%08x", bid.origRefNumDelta)
		out("QASK", "%08x", ask.origRefNumDelta)
	case MessageTypeOrderAdd:
		out("ORDER", "%c %08x %08x %08x %08x", ord.side, m.optionId, ord.refNumDelta, ord.size, ord.price)
	case MessageTypeOrderExecute, MessageTypeOrderExecuteWPrice, MessageTypeOrderCancel:
		out("ORDER", "%08x %08x", ord.origRefNumDelta, ord.size)
	case MessageTypeOrderUpdate:
		out("ORDER", "%08x %08x %08x", ord.origRefNumDelta, ord.size, ord.price)
	case MessageTypeOrderReplace:
		out("ORDER", "%08x %08x %08x %08x", ord.refNumDelta, ord.origRefNumDelta, ord.size, ord.price)
	case MessageTypeOrderDelete:
		out("ORDER", "%08x", ord.origRefNumDelta)
	case MessageTypeBlockOrderDelete:
		for _, r := range m.bssdRefs {
			out("ORDER", "%08x", r)
		}
	default:
		log.Fatalf("Unexpected message type %d in %+v\n", m.typ, m)
	}
}

func bool2int(b bool) int {
	if b {
		return 1
	} else {
		return 0
	}
}

func marketSide2int(ms MarketSide) int {
	if ms == MarketSideSell {
		return 1
	} else {
		return 0
	}
}

func (s *simulator) outOrderLookup(op *OrderOperation, order Order, isFound bool) {
	if op.op == OrderOpAdd {
		s.Printfln("ORDL 1 %08x %08x", op.orderId, op.optionId)
		order.optionId = op.optionId
	} else {
		s.Printfln("ORDL 0 %08x", op.orderId)
	}
	s.Printfln("ORDRESP %d %d %d %08x %08x %08x",
		bool2int(!isFound && op.op != OrderOpAdd),
		bool2int(op.op == OrderOpAdd),
		marketSide2int(order.side),
		order.size,
		order.price,
		order.optionId,
	)

}

func (s *simulator) outOrderUpdate(op *OrderOperation, order Order) {
	s.Printfln("ORDU %08x %08x %d %08x %08x",
		order.id,
		order.optionId,
		marketSide2int(order.side),
		order.price,
		order.size,
	)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The OpenEBS Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package negative

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1"
	"github.com/openebs/maya/tests/cstor"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = Describe("[cstor] [-ve] TEST PROVISION WITHOUT DISK", func() {
	var (
		err error
	)

	AfterEach(func() {

		By("deleting storagepoolclaim")
		_, err = ops.SPCClient.Delete(spcObj.Name, &metav1.DeleteOptions{})
		Expect(err).To(BeNil(), "while deleting the storagepoolclaim {%s}", spcObj.Name)

		By("fetching ndm pods")
		podList, err := ops.PodClient.WithNamespace(openebsNamespace).
			List(metav1.ListOptions{
				LabelSelector: "name=openebs-ndm",
			})
		Expect(err).ShouldNot(HaveOccurred(), "while fetching ndm pods")

		By("deleting ndm pods to bring back bds")
		for _, pod := range podList.Items {
			err = ops.PodClient.WithNamespace(openebsNamespace).
				Delete(pod.Name, &metav1.DeleteOptions{})
			Expect(err).ShouldNot(HaveOccurred(), "while deleting ndm pods")
		}
	})

	When("creating storagepoolclaim without disk", func() {
		It("should not create any cstorpool", func() {

			By("fetching bds")
			bdList, err := ops.BDClient.List(metav1.ListOptions{})
			Expect(err).ShouldNot(HaveOccurred(), "while fetching bds")

			By("deleting bd to remove disk")
			for _, bd := range bdList.Items {
				err = ops.BDClient.WithNamespace(openebsNamespace).
					Delete(bd.Name, &metav1.DeleteOptions{})
				Expect(err).ShouldNot(HaveOccurred(), "while deleting bd")
			}

			By("building storagepoolclaim with invalid disk type")
			spcObj = spc.NewBuilder().
				WithGenerateName(spcName).
				WithDiskType(string(apis.TypeSparseCPV)).
				WithMaxPool(cstor.PoolCount).
				WithOverProvisioning(false).
				WithPoolType(string(apis.PoolTypeStripedCPV)).
				Build().Object

			By("creating above storagepoolclaim")
			spcObj, err = ops.SPCClient.Create(spcObj)
			Expect(err).To(BeNil(), "while creating storagepoolclaim {%s}", spcObj.Name)

			By("verifying cstorpool count as 0")
			cspCount := ops.GetHealthyCSPCountEventually(spcObj.Name, cstor.PoolCount)
			Expect(cspCount).To(Equal(0), "while checking cstorpool count")

		})
	})
})
<commit_msg>fix(BDD, cstor): fix negative cstor pool provisioning (#1421)<commit_after>/*
Copyright 2019 The OpenEBS Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package negative

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	apis "github.com/openebs/maya/pkg/apis/openebs.io/v1alpha1"
	spc "github.com/openebs/maya/pkg/storagepoolclaim/v1alpha1"
	"github.com/openebs/maya/tests/cstor"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var _ = Describe("[cstor] [-ve] TEST PROVISION WITHOUT DISK", func() {
	var (
		err error
	)

	AfterEach(func() {

		By("deleting storagepoolclaim")
		_, err = ops.SPCClient.Delete(spcObj.Name, &metav1.DeleteOptions{})
		Expect(err).To(BeNil(), "while deleting the storagepoolclaim {%s}", spcObj.Name)

		By("fetching ndm pods")
		podList, err := ops.PodClient.WithNamespace(openebsNamespace).
			List(metav1.ListOptions{
				LabelSelector: "name=openebs-ndm",
			})
		Expect(err).ShouldNot(HaveOccurred(), "while fetching ndm pods")

		By("deleting ndm pods to bring back bds")
		for _, pod := range podList.Items {
			err = ops.PodClient.WithNamespace(openebsNamespace).
				Delete(pod.Name, &metav1.DeleteOptions{})
			Expect(err).ShouldNot(HaveOccurred(), "while deleting ndm pods")
		}
	})

	When("creating storagepoolclaim without disk", func() {
		It("should not create any cstorpool", func() {

			By("fetching bds")
			bdList, err := ops.BDClient.List(metav1.ListOptions{})
			Expect(err).ShouldNot(HaveOccurred(), "while fetching bds")

			By("deleting bd to remove disk")
			for _, bd := range bdList.Items {
				err = ops.BDClient.WithNamespace(openebsNamespace).
					Delete(bd.Name, &metav1.DeleteOptions{})
				Expect(err).ShouldNot(HaveOccurred(), "while deleting bd")
			}

			By("building storagepoolclaim with invalid disk type")
			spcObj = spc.NewBuilder().
				WithGenerateName(spcName).
				WithDiskType(string(apis.TypeSparseCPV)).
				WithMaxPool(cstor.PoolCount).
				WithOverProvisioning(false).
				WithPoolType(string(apis.PoolTypeStripedCPV)).
				Build().Object

			By("creating above storagepoolclaim")
			spcObj, err = ops.SPCClient.Create(spcObj)
			Expect(err).To(BeNil(), "while creating storagepoolclaim {%s}", spcObj.Name)

			By("verifying cstorpool count as 0")
			cspCount := ops.GetHealthyCSPCount(spcObj.Name, cstor.PoolCount)
			Expect(cspCount).To(Equal(0), "while checking cstorpool count")

		})
	})
})
<|endoftext|>


<commit_before>package main

import "fmt"

func passByView(arr []int)  {
	arr[0] = 0xff
}

func main() {
	// test failure
	//autoArray := [1024]int { iota }
	arr1 := [...]int { 0, 1, 2, 3, 4, 5, 6, 7 }
	fmt.Println(arr1[2:4]) // [2, 4)
	fmt.Println(arr1[:4]) // [2, 4)
	fmt.Println(arr1[2:]) // [2, 4)
	fmt.Println(arr1[:]) // [2, 4)

	s1 := arr1[:]
	passByView(s1)
	fmt.Println(s1)

	s2 := arr1[2:4]
	passByView(s2)
	fmt.Println(s2)
	fmt.Println("arr1 =", arr1)

	// re-slice
	fmt.Println(s2[1:])

	// out of range
	//fmt.Println(arr1[4:3]) // error

	fmt.Println("arr1 =", arr1)
	slice1 := arr1[2:4]
	fmt.Println("slice1 =", slice1) // { arr1[2], arr1[3], [arr1[4]], [...] }
	func() {
		defer func() {
			if r := recover(); r != nil {
				fmt.Println(">>> Error:", r)
			}
		}()

		for i := 0; i < 10; i++ {
			fmt.Println(slice1[:i])
		}
	}()

	fmt.Println("arr1 =", arr1)
	fmt.Printf("slice1 = %v, len(slice1) = %d, cap(slice1) = %d\n", slice1, len(slice1), cap(slice1))

	// append slice
	slice2 := arr1[3:7]
	fmt.Println("arr1 =", arr1)
	fmt.Printf("slice2 = %v, len(slice2) = %d, cap(slice2) = %d\n", slice2, len(slice2), cap(slice2))
	append1 := append(slice2, -1)
	append2 := append(append1, -2)
	append3 := append(append2, -3)
	fmt.Printf("append1 = %v, len(append1) = %d, cap(append1) = %d\n", append1, len(append1), cap(append1))
	fmt.Printf("append2 = %v, len(append2) = %d, cap(append2) = %d\n", append2, len(append2), cap(append2))
	fmt.Printf("append3 = %v, len(append3) = %d, cap(append3) = %d\n", append3, len(append3), cap(append3))
	fmt.Printf("arr1 = %v, len(arr1) = %d, cap(arr1) = %d\n", arr1, len(arr1), cap(arr1))

	// directly definition slice
	var slice3 []int
	fmt.Println(slice3, slice3 == nil)

	// append to slice
	for i:= 0; i < 100; i++ {
		slice3 = append(slice3, i)
		fmt.Printf("len(slice3) = %d, cap(slice3) = %d\n", len(slice3), cap(slice3))
	}

	// make slice
	slice4 := make([]int, 16)
	slice5 := make([]int, 16, 64)
	fmt.Printf("slice4 = %v, len(slice4) = %d, cap(slice4) = %d\n", slice4, len(slice4), cap(slice4))
	fmt.Printf("slice5 = %v, len(slice5) = %d, cap(slice5) = %d\n", slice5, len(slice5), cap(slice5))

	// copy
	fmt.Println(copy(slice4, slice2))
	fmt.Printf("slice4 = %v, len(slice4) = %d, cap(slice4) = %d\n", slice4, len(slice4), cap(slice4))

	// delete
	slice4 = append(slice4[:3], slice4[4:]...)
	fmt.Printf("slice4 = %v, len(slice4) = %d, cap(slice4) = %d\n", slice4, len(slice4), cap(slice4))
}
<commit_msg>shift and pop<commit_after>package main

import "fmt"

func passByView(arr []int)  {
	arr[0] = 0xff
}

func main() {
	// test failure
	//autoArray := [1024]int { iota }
	arr1 := [...]int { 0, 1, 2, 3, 4, 5, 6, 7 }
	fmt.Println(arr1[2:4]) // [2, 4)
	fmt.Println(arr1[:4]) // [2, 4)
	fmt.Println(arr1[2:]) // [2, 4)
	fmt.Println(arr1[:]) // [2, 4)

	s1 := arr1[:]
	passByView(s1)
	fmt.Println(s1)

	s2 := arr1[2:4]
	passByView(s2)
	fmt.Println(s2)
	fmt.Println("arr1 =", arr1)

	// re-slice
	fmt.Println(s2[1:])

	// out of range
	//fmt.Println(arr1[4:3]) // error

	fmt.Println("arr1 =", arr1)
	slice1 := arr1[2:4]
	fmt.Println("slice1 =", slice1) // { arr1[2], arr1[3], [arr1[4]], [...] }
	func() {
		defer func() {
			if r := recover(); r != nil {
				fmt.Println(">>> Error:", r)
			}
		}()

		for i := 0; i < 10; i++ {
			fmt.Println(slice1[:i])
		}
	}()

	fmt.Println("arr1 =", arr1)
	fmt.Printf("slice1 = %v, len(slice1) = %d, cap(slice1) = %d\n", slice1, len(slice1), cap(slice1))

	// append slice
	slice2 := arr1[3:7]
	fmt.Println("arr1 =", arr1)
	fmt.Printf("slice2 = %v, len(slice2) = %d, cap(slice2) = %d\n", slice2, len(slice2), cap(slice2))
	append1 := append(slice2, -1)
	append2 := append(append1, -2)
	append3 := append(append2, -3)
	fmt.Printf("append1 = %v, len(append1) = %d, cap(append1) = %d\n", append1, len(append1), cap(append1))
	fmt.Printf("append2 = %v, len(append2) = %d, cap(append2) = %d\n", append2, len(append2), cap(append2))
	fmt.Printf("append3 = %v, len(append3) = %d, cap(append3) = %d\n", append3, len(append3), cap(append3))
	fmt.Printf("arr1 = %v, len(arr1) = %d, cap(arr1) = %d\n", arr1, len(arr1), cap(arr1))

	// directly definition slice
	var slice3 []int
	fmt.Println(slice3, slice3 == nil)

	// append to slice
	for i:= 0; i < 100; i++ {
		slice3 = append(slice3, i)
		fmt.Printf("len(slice3) = %d, cap(slice3) = %d\n", len(slice3), cap(slice3))
	}

	sliceInit := []int {2, 4, 6, 8}
	fmt.Printf("sliceInit = %v, len(sliceInit) = %d, cap(sliceInit) = %d\n", sliceInit, len(sliceInit), cap(sliceInit))

	// make slice
	slice4 := make([]int, 16)
	slice5 := make([]int, 16, 64)
	fmt.Printf("slice4 = %v, len(slice4) = %d, cap(slice4) = %d\n", slice4, len(slice4), cap(slice4))
	fmt.Printf("slice5 = %v, len(slice5) = %d, cap(slice5) = %d\n", slice5, len(slice5), cap(slice5))

	// copy
	fmt.Println(copy(slice4, slice2))
	fmt.Printf("slice4 = %v, len(slice4) = %d, cap(slice4) = %d\n", slice4, len(slice4), cap(slice4))

	// delete
	slice4 = append(slice4[:3], slice4[4:]...)
	fmt.Printf("slice4 = %v, len(slice4) = %d, cap(slice4) = %d\n", slice4, len(slice4), cap(slice4))

	// shift
	front := sliceInit[0]
	sliceInit = sliceInit[1:]
	fmt.Printf("front = %d, sliceInit = %d\n", front, sliceInit)

	// pop
	back := sliceInit[len(sliceInit) - 1]
	sliceInit = sliceInit[:len(sliceInit) - 1]
	fmt.Printf("back = %d, sliceInit = %d\n", back, sliceInit)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"

	"code.google.com/p/go.crypto/ssh"
)

var (
	config string
	wt     sync.WaitGroup
)

type Jobs struct {
	Servers  []string `json:"servers"`
	Pem_file string   `json:"pem"`
	Tasks    []string `json:"tasks"`
	User     string   `json:"user"`
}

// ssh key support
type keychain struct {
	keys []ssh.Signer
}

func (k *keychain) Key(i int) (ssh.PublicKey, error) {
	if i < 0 || i >= len(k.keys) {
		return nil, nil
	}
	return k.keys[i].PublicKey(), nil
}

func (k *keychain) Sign(i int, rand io.Reader, data []byte) (sig []byte, err error) {
	return k.keys[i].Sign(rand, data)
}

func (k *keychain) add(key ssh.Signer) {
	k.keys = append(k.keys, key)
}

func (k *keychain) loadPEM(file string) error {
	buf, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	key, err := ssh.ParsePrivateKey(buf)
	if err != nil {
		return err
	}
	k.add(key)
	return nil
}

// end of ssh key support

func (j *Jobs) runTasks() {

	// load ssh key
	k := new(keychain)
	err := k.loadPEM(j.Pem_file)

	if err != nil {
		panic("Cannot load key [" + j.Pem_file + "]: " + err.Error())
	}

	// not ready
	ts := time.Now().Format(time.RFC3339)
	for i := 0; i < len(j.Servers); i++ {
		wt.Add(1)
		// create log file
		log_file := "logs/provi_log__" + j.Servers[i] + "--" + ts + ".log"

		outfile, err := os.Create(log_file)
		if err != nil {
			log.Fatal(err)
		}
		defer outfile.Close()

		// create ssh session
		config := &ssh.ClientConfig{
			User: j.User,
			Auth: []ssh.ClientAuth{
				ssh.ClientAuthKeyring(k),
			},
		}

		client, err := ssh.Dial("tcp", j.Servers[i]+":22", config)
		if err != nil {
			log.Fatal("Failed to connect to [" + j.Servers[i] + "] with error: " + err.Error())
		}

		go func(x int, s *ssh.ClientConn, o *os.File) {
			defer wt.Done()
			j.runTask(x, s, o)
		}(i, client, outfile)
	}
	wt.Wait()
}

func (p *Jobs) runTask(i int, s *ssh.ClientConn, outfile *os.File) {
	for j := 0; j < len(p.Tasks); j++ {
		p.runCmd(i, p.Tasks[j], s, outfile)
	}
}

func (p *Jobs) runCmd(i int, cmd string, client *ssh.ClientConn, outfile *os.File) {
	session, err := client.NewSession()
	if err != nil {
		log.Fatal("Failed to create SSH session to server [" + p.Servers[i] + "] with error: " + err.Error())
	}
	defer session.Close()

	session.Stdout = outfile
	session.Stderr = outfile

	// Set up terminal modes
	modes := ssh.TerminalModes{
		ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}
	// Request pseudo terminal
	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		log.Fatalf("request for pseudo terminal failed: %s", err)
	}

	err = session.Start(cmd)
	if err != nil {
		log.Fatal("run command ["+cmd+"] on server ("+p.Servers[i]+") failed with -> ", err)
	}
	session.Wait()

	log.Println(cmd, " done for server ", p.Servers[i])
}

func init() {
	flag.StringVar(&config, "config", "", "Config JSON for the run")
}

func run(config string) error {

	var t Jobs

	// read json file
	jsonBlob, e := ioutil.ReadFile(config)
	if e != nil {
		fmt.Printf("File error: %v\n", e)
		os.Exit(1)
	}

	err := json.Unmarshal(jsonBlob, &t)
	if err != nil {
		fmt.Println("error:", err)
		return err
	}

	t.runTasks()

	return nil
}

func main() {
	flag.Parse()

	if config == "" {
		config = "./config.json"
	}

	run(config)
}
<commit_msg>update to latest ssh package<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"

	"code.google.com/p/go.crypto/ssh"
)

var (
	config string
	wt     sync.WaitGroup
)

type Jobs struct {
	Servers  []string `json:"servers"`
	Pem_file string   `json:"pem"`
	Tasks    []string `json:"tasks"`
	User     string   `json:"user"`
}

// ssh key support
type keychain struct {
	keys []ssh.Signer
}

func (k *keychain) Key(i int) (ssh.PublicKey, error) {
	if i < 0 || i >= len(k.keys) {
		return nil, nil
	}
	return k.keys[i].PublicKey(), nil
}

/*
func (k *keychain) Sign(i int, rand io.Reader, data []byte) (sig []byte, err error) {
	return k.keys[i].Sign(rand, data)
}*/

func (k *keychain) add(key ssh.Signer) {
	k.keys = append(k.keys, key)
}

func (k *keychain) loadPEM(file string) (ssh.Signer, error) {
	buf, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	key, err := ssh.ParsePrivateKey(buf)
	if err != nil {
		return nil, err
	}
	k.add(key)
	return key, nil
}

// end of ssh key support

func (j *Jobs) runTasks() {

	// load ssh key
	k := new(keychain)
	key, err := k.loadPEM(j.Pem_file)

	if err != nil {
		panic("Cannot load key [" + j.Pem_file + "]: " + err.Error())
	}

	// not ready
	ts := time.Now().Format(time.RFC3339)
	for i := 0; i < len(j.Servers); i++ {
		wt.Add(1)
		// create log file
		log_file := "logs/provi_log__" + j.Servers[i] + "--" + ts + ".log"

		outfile, err := os.Create(log_file)
		if err != nil {
			log.Fatal(err)
		}
		defer outfile.Close()

		// create ssh session
		config := &ssh.ClientConfig{
			User: j.User,
			Auth: []ssh.AuthMethod{
				ssh.PublicKeys(key),
			},
		}

		client, err := ssh.Dial("tcp", j.Servers[i]+":22", config)
		if err != nil {
			log.Fatal("Failed to connect to [" + j.Servers[i] + "] with error: " + err.Error())
		}

		go func(x int, s *ssh.Client, o *os.File) {
			defer wt.Done()
			j.runTask(x, s, o)
		}(i, client, outfile)
	}
	wt.Wait()
}

func (p *Jobs) runTask(i int, s *ssh.Client, outfile *os.File) {
	for j := 0; j < len(p.Tasks); j++ {
		p.runCmd(i, p.Tasks[j], s, outfile)
	}
}

func (p *Jobs) runCmd(i int, cmd string, client *ssh.Client, outfile *os.File) {
	session, err := client.NewSession()
	if err != nil {
		log.Fatal("Failed to create SSH session to server [" + p.Servers[i] + "] with error: " + err.Error())
	}
	defer session.Close()

	session.Stdout = outfile
	session.Stderr = outfile

	// Set up terminal modes
	modes := ssh.TerminalModes{
		ssh.ECHO:          0,     // disable echoing
		ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
		ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	}
	// Request pseudo terminal
	if err := session.RequestPty("xterm", 80, 40, modes); err != nil {
		log.Fatalf("request for pseudo terminal failed: %s", err)
	}

	err = session.Start(cmd)
	if err != nil {
		log.Fatal("run command ["+cmd+"] on server ("+p.Servers[i]+") failed with -> ", err)
	}
	session.Wait()

	log.Println(cmd, " done for server ", p.Servers[i])
}

func init() {
	flag.StringVar(&config, "config", "", "Config JSON for the run")
}

func run(config string) error {

	var t Jobs

	// read json file
	jsonBlob, e := ioutil.ReadFile(config)
	if e != nil {
		fmt.Printf("File error: %v\n", e)
		os.Exit(1)
	}

	err := json.Unmarshal(jsonBlob, &t)
	if err != nil {
		fmt.Println("error:", err)
		return err
	}

	t.runTasks()

	return nil
}

func main() {
	flag.Parse()

	if config == "" {
		config = "./config.json"
	}

	run(config)
}
<|endoftext|>


<commit_before>package gocbcore

import (
	"fmt"
)

type generalError struct {
	message string
}

func (e generalError) Error() string {
	return e.message
}

type timeoutError struct {
}

func (e timeoutError) Error() string {
	return "The operation has timed out."
}
func (e timeoutError) Timeout() bool {
	return true
}

type networkError struct {
}

func (e networkError) Error() string {
	return "Network error."
}

func (e networkError) NetworkError() bool {
	return true
}

type overloadError struct {
}

func (e overloadError) Error() string {
	return "Queue overflow."
}
func (e overloadError) Overload() bool {
	return true
}

type memdError struct {
	code StatusCode
}

func (e memdError) Error() string {
	switch e.code {
	case StatusSuccess:
		return "Success."
	case StatusKeyNotFound:
		return "Key not found."
	case StatusKeyExists:
		return "Key already exists."
	case StatusTooBig:
		return "Document value was too large."
	case StatusNotStored:
		return "The document could not be stored."
	case StatusBadDelta:
		return "An invalid delta was passed."
	case StatusNotMyVBucket:
		return "Operation sent to incorrect server."
	case StatusNoBucket:
		return "Not connected to a bucket."
	case StatusAuthStale:
		return "The authenication context is stale. Try re-authenticating."
	case StatusAuthError:
		return "Authentication Error."
	case StatusAuthContinue:
		return "Auth Continue."
	case StatusRangeError:
		return "Requested value is outside range."
	case StatusAccessError:
		return "No access."
	case StatusNotInitialized:
		return "The cluster is being initialized. Requests are blocked."
	case StatusRollback:
		return "A rollback is required."
	case StatusUnknownCommand:
		return "An unknown command was received."
	case StatusOutOfMemory:
		return "The server is out of memory."
	case StatusNotSupported:
		return "The server does not support this command."
	case StatusInternalError:
		return "Internal server error."
	case StatusBusy:
		return "The server is busy. Try again later."
	case StatusTmpFail:
		return "A temporary failure occurred.  Try again later."
	default:
		return fmt.Sprintf("An unknown error occurred (%d).", e.code)
	}
}
func (e memdError) KeyNotFound() bool {
	return e.code == StatusKeyNotFound
}
func (e memdError) KeyExists() bool {
	return e.code == StatusKeyExists
}
func (e memdError) Temporary() bool {
	return e.code == StatusOutOfMemory || e.code == StatusTmpFail
}
func (e memdError) AuthError() bool {
	return e.code == StatusAuthError
}
func (e memdError) ValueTooBig() bool {
	return e.code == StatusTooBig
}
func (e memdError) NotStored() bool {
	return e.code == StatusNotStored
}
func (e memdError) BadDelta() bool {
	return e.code == StatusBadDelta
}

type agentError struct {
	message string
}

func (e agentError) Error() string {
	return e.message
}

type streamEndError struct {
	code StreamEndStatus
}

func (e streamEndError) Error() string {
	switch e.code {
	case StreamEndOK:
		return "Success."
	case StreamEndClosed:
		return "Stream closed."
	case StreamEndStateChanged:
		return "State changed."
	case StreamEndDisconnected:
		return "Disconnected."
	case StreamEndTooSlow:
		return "Too slow."
	default:
		return fmt.Sprintf("Stream closed for unknown reason: (%d).", e.code)
	}
}

func (e streamEndError) Closed() bool {
	return e.code == StreamEndClosed
}
func (e streamEndError) StateChanged() bool {
	return e.code == StreamEndStateChanged
}
func (e streamEndError) Disconnected() bool {
	return e.code == StreamEndDisconnected
}
func (e streamEndError) TooSlow() bool {
	return e.code == StreamEndTooSlow
}<commit_msg>Add a function to the StreamEndError to check for stream end ok<commit_after>package gocbcore

import (
	"fmt"
)

type generalError struct {
	message string
}

func (e generalError) Error() string {
	return e.message
}

type timeoutError struct {
}

func (e timeoutError) Error() string {
	return "The operation has timed out."
}
func (e timeoutError) Timeout() bool {
	return true
}

type networkError struct {
}

func (e networkError) Error() string {
	return "Network error."
}

func (e networkError) NetworkError() bool {
	return true
}

type overloadError struct {
}

func (e overloadError) Error() string {
	return "Queue overflow."
}
func (e overloadError) Overload() bool {
	return true
}

type memdError struct {
	code StatusCode
}

func (e memdError) Error() string {
	switch e.code {
	case StatusSuccess:
		return "Success."
	case StatusKeyNotFound:
		return "Key not found."
	case StatusKeyExists:
		return "Key already exists."
	case StatusTooBig:
		return "Document value was too large."
	case StatusNotStored:
		return "The document could not be stored."
	case StatusBadDelta:
		return "An invalid delta was passed."
	case StatusNotMyVBucket:
		return "Operation sent to incorrect server."
	case StatusNoBucket:
		return "Not connected to a bucket."
	case StatusAuthStale:
		return "The authenication context is stale. Try re-authenticating."
	case StatusAuthError:
		return "Authentication Error."
	case StatusAuthContinue:
		return "Auth Continue."
	case StatusRangeError:
		return "Requested value is outside range."
	case StatusAccessError:
		return "No access."
	case StatusNotInitialized:
		return "The cluster is being initialized. Requests are blocked."
	case StatusRollback:
		return "A rollback is required."
	case StatusUnknownCommand:
		return "An unknown command was received."
	case StatusOutOfMemory:
		return "The server is out of memory."
	case StatusNotSupported:
		return "The server does not support this command."
	case StatusInternalError:
		return "Internal server error."
	case StatusBusy:
		return "The server is busy. Try again later."
	case StatusTmpFail:
		return "A temporary failure occurred.  Try again later."
	default:
		return fmt.Sprintf("An unknown error occurred (%d).", e.code)
	}
}
func (e memdError) KeyNotFound() bool {
	return e.code == StatusKeyNotFound
}
func (e memdError) KeyExists() bool {
	return e.code == StatusKeyExists
}
func (e memdError) Temporary() bool {
	return e.code == StatusOutOfMemory || e.code == StatusTmpFail
}
func (e memdError) AuthError() bool {
	return e.code == StatusAuthError
}
func (e memdError) ValueTooBig() bool {
	return e.code == StatusTooBig
}
func (e memdError) NotStored() bool {
	return e.code == StatusNotStored
}
func (e memdError) BadDelta() bool {
	return e.code == StatusBadDelta
}

type agentError struct {
	message string
}

func (e agentError) Error() string {
	return e.message
}

type streamEndError struct {
	code StreamEndStatus
}

func (e streamEndError) Error() string {
	switch e.code {
	case StreamEndOK:
		return "Success."
	case StreamEndClosed:
		return "Stream closed."
	case StreamEndStateChanged:
		return "State changed."
	case StreamEndDisconnected:
		return "Disconnected."
	case StreamEndTooSlow:
		return "Too slow."
	default:
		return fmt.Sprintf("Stream closed for unknown reason: (%d).", e.code)
	}
}

func (e streamEndError) Success() bool {
	return e.code == StreamEndOK
}
func (e streamEndError) Closed() bool {
	return e.code == StreamEndClosed
}
func (e streamEndError) StateChanged() bool {
	return e.code == StreamEndStateChanged
}
func (e streamEndError) Disconnected() bool {
	return e.code == StreamEndDisconnected
}
func (e streamEndError) TooSlow() bool {
	return e.code == StreamEndTooSlow
}<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tests

// This file contains Test functions which initiate the tests. The tests can be either
// coded in this file or use scenarios from perf_scenarios.go.

import (
	"testing"

	"go.opentelemetry.io/collector/testbed/testbed"
)

func TestMetricNoBackend10kDPSOpenCensus(t *testing.T) {
	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)
	tc := testbed.NewTestCase(
		t,
		dataProvider,
		testbed.NewOCMetricDataSender(testbed.DefaultHost, 55678),
		testbed.NewOCDataReceiver(testbed.DefaultOCPort),
		&testbed.ChildProcess{},
		&testbed.PerfTestValidator{},
		performanceResultsSummary,
	)
	defer tc.Stop()

	tc.SetResourceLimits(testbed.ResourceSpec{ExpectedMaxCPU: 200, ExpectedMaxRAM: 200})
	tc.StartAgent()

	tc.StartLoad(testbed.LoadOptions{DataItemsPerSecond: 10_000})

	tc.Sleep(tc.Duration)
}

func TestMetric10kDPS(t *testing.T) {
	tests := []struct {
		name         string
		sender       testbed.DataSender
		receiver     testbed.DataReceiver
		resourceSpec testbed.ResourceSpec
	}{
		{
			"OpenCensus",
			testbed.NewOCMetricDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t)),
			testbed.NewOCDataReceiver(testbed.GetAvailablePort(t)),
			testbed.ResourceSpec{
				ExpectedMaxCPU: 85,
				ExpectedMaxRAM: 75,
			},
		},
		{
			"OTLP",
			testbed.NewOTLPMetricDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t)),
			testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t)),
			testbed.ResourceSpec{
				ExpectedMaxCPU: 50,
				ExpectedMaxRAM: 65,
			},
		},
		{
			"OTLP-HTTP",
			testbed.NewOTLPHTTPMetricDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t)),
			testbed.NewOTLPHTTPDataReceiver(testbed.GetAvailablePort(t)),
			testbed.ResourceSpec{
				ExpectedMaxCPU: 50,
				ExpectedMaxRAM: 65,
			},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			Scenario10kItemsPerSecond(
				t,
				test.sender,
				test.receiver,
				test.resourceSpec,
				performanceResultsSummary,
				nil,
				nil,
			)
		})
	}

}
<commit_msg>Increase RaM limits for TestMetric10kDPS (#3124)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tests

// This file contains Test functions which initiate the tests. The tests can be either
// coded in this file or use scenarios from perf_scenarios.go.

import (
	"testing"

	"go.opentelemetry.io/collector/testbed/testbed"
)

func TestMetricNoBackend10kDPSOpenCensus(t *testing.T) {
	options := testbed.LoadOptions{DataItemsPerSecond: 10_000, ItemsPerBatch: 10}
	dataProvider := testbed.NewPerfTestDataProvider(options)
	tc := testbed.NewTestCase(
		t,
		dataProvider,
		testbed.NewOCMetricDataSender(testbed.DefaultHost, 55678),
		testbed.NewOCDataReceiver(testbed.DefaultOCPort),
		&testbed.ChildProcess{},
		&testbed.PerfTestValidator{},
		performanceResultsSummary,
	)
	defer tc.Stop()

	tc.SetResourceLimits(testbed.ResourceSpec{ExpectedMaxCPU: 200, ExpectedMaxRAM: 200})
	tc.StartAgent()

	tc.StartLoad(testbed.LoadOptions{DataItemsPerSecond: 10_000})

	tc.Sleep(tc.Duration)
}

func TestMetric10kDPS(t *testing.T) {
	tests := []struct {
		name         string
		sender       testbed.DataSender
		receiver     testbed.DataReceiver
		resourceSpec testbed.ResourceSpec
	}{
		{
			"OpenCensus",
			testbed.NewOCMetricDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t)),
			testbed.NewOCDataReceiver(testbed.GetAvailablePort(t)),
			testbed.ResourceSpec{
				ExpectedMaxCPU: 85,
				ExpectedMaxRAM: 85,
			},
		},
		{
			"OTLP",
			testbed.NewOTLPMetricDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t)),
			testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t)),
			testbed.ResourceSpec{
				ExpectedMaxCPU: 50,
				ExpectedMaxRAM: 85,
			},
		},
		{
			"OTLP-HTTP",
			testbed.NewOTLPHTTPMetricDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t)),
			testbed.NewOTLPHTTPDataReceiver(testbed.GetAvailablePort(t)),
			testbed.ResourceSpec{
				ExpectedMaxCPU: 50,
				ExpectedMaxRAM: 85,
			},
		},
	}

	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			Scenario10kItemsPerSecond(
				t,
				test.sender,
				test.receiver,
				test.resourceSpec,
				performanceResultsSummary,
				nil,
				nil,
			)
		})
	}

}
<|endoftext|>


<commit_before>// Copyright 2016 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.

package grpc_testing

import (
	"net"
	"time"

	pb_testproto "github.com/mwitkow/go-grpc-middleware/testing/testproto"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

// InterceptorTestSuite is a testify/Suite that starts a gRPC PingService server and a client.
type InterceptorTestSuite struct {
	suite.Suite

	ServerOpts []grpc.ServerOption
	ClientOpts []grpc.DialOption

	ServerListener net.Listener
	Server         *grpc.Server
	clientConn     *grpc.ClientConn
	Client         pb_testproto.TestServiceClient
	ctx            context.Context
}

func (s *InterceptorTestSuite) SetupSuite() {
	var err error
	s.ServerListener, err = net.Listen("tcp", "127.0.0.1:0")
	require.NoError(s.T(), err, "must be able to allocate a port for serverListener")

	// This is the point where we hook up the interceptor
	s.Server = grpc.NewServer(s.ServerOpts...)
	pb_testproto.RegisterTestServiceServer(s.Server, &TestPingService{T: s.T()})

	go func() {
		s.Server.Serve(s.ServerListener)
	}()
	clientOpts := append(s.ClientOpts, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithTimeout(2*time.Second))
	s.clientConn, err = grpc.Dial(s.ServerAddr(), clientOpts...)
	require.NoError(s.T(), err, "must not error on client Dial")
	s.Client = pb_testproto.NewTestServiceClient(s.clientConn)

}

func (s *InterceptorTestSuite) ServerAddr() string {
	return s.ServerListener.Addr().String()
}

func (s *InterceptorTestSuite) SimpleCtx() context.Context {
	ctx, _ := context.WithTimeout(context.TODO(), 2 * time.Second)
	return ctx
}

func (s *InterceptorTestSuite) TearDownSuite() {
	if s.ServerListener != nil {
		s.Server.Stop()
		s.T().Logf("stopped grpc.Server at: %v", s.ServerAddr())
		s.ServerListener.Close()

	}
	if s.clientConn != nil {
		s.clientConn.Close()
	}
}
<commit_msg>extend interceptor suite to allow for different clients and shadowed service impls.<commit_after>// Copyright 2016 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.

package grpc_testing

import (
	"net"
	"time"

	pb_testproto "github.com/mwitkow/go-grpc-middleware/testing/testproto"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
)

// InterceptorTestSuite is a testify/Suite that starts a gRPC PingService server and a client.
type InterceptorTestSuite struct {
	suite.Suite

	TestService pb_testproto.TestServiceServer
	ServerOpts []grpc.ServerOption
	ClientOpts []grpc.DialOption

	ServerListener net.Listener
	Server         *grpc.Server
	clientConn     *grpc.ClientConn
	Client         pb_testproto.TestServiceClient
}

func (s *InterceptorTestSuite) SetupSuite() {
	var err error
	s.ServerListener, err = net.Listen("tcp", "127.0.0.1:0")
	require.NoError(s.T(), err, "must be able to allocate a port for serverListener")

	// This is the point where we hook up the interceptor
	s.Server = grpc.NewServer(s.ServerOpts...)
	// Crete a service of the instantiator hasn't provided one.
	if s.TestService == nil {
		s.TestService = &TestPingService{T: s.T()}
	}
	pb_testproto.RegisterTestServiceServer(s.Server, s.TestService)

	go func() {
		s.Server.Serve(s.ServerListener)
	}()
	s.Client = s.NewClient(s.ClientOpts...)
}

func (s *InterceptorTestSuite) NewClient(dialOpts ...grpc.DialOption) pb_testproto.TestServiceClient {
	newDialOpts := append(dialOpts, grpc.WithInsecure(), grpc.WithBlock(), grpc.WithTimeout(2*time.Second))
	clientConn, err := grpc.Dial(s.ServerAddr(), newDialOpts...)
	require.NoError(s.T(), err, "must not error on client Dial")
	return pb_testproto.NewTestServiceClient(clientConn)
}

func (s *InterceptorTestSuite) ServerAddr() string {
	return s.ServerListener.Addr().String()
}

func (s *InterceptorTestSuite) SimpleCtx() context.Context {
	ctx, _ := context.WithTimeout(context.TODO(), 2 * time.Second)
	return ctx
}

func (s *InterceptorTestSuite) TearDownSuite() {
	if s.ServerListener != nil {
		s.Server.Stop()
		s.T().Logf("stopped grpc.Server at: %v", s.ServerAddr())
		s.ServerListener.Close()

	}
	if s.clientConn != nil {
		s.clientConn.Close()
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Davis Webb
// Copyright 2015 Luke Shumaker
// Copyright 2015 Guntas Grewal

package store

import (
	"fmt"
	"github.com/jinzhu/gorm"
	he "httpentity"
	"httpentity/util"
	"io"
	"strings"
)

var _ he.Entity = &Group{}
var _ he.NetEntity = &Group{}
var dirGroups he.Entity = newDirGroups()

// Model /////////////////////////////////////////////////////////////

type Group struct {
	Id        string
	Addresses []GroupAddress
}

func (o Group) schema(db *gorm.DB) {
	db.CreateTable(&o)
}

type GroupAddress struct {
	Id      int64
	GroupId string
	Medium  string
	Address string
}

func (o GroupAddress) schema(db *gorm.DB) {
	table := db.CreateTable(&o)
	table.AddForeignKey("group_id", "groups(id)", "RESTRICT", "RESTRICT")
	table.AddForeignKey("medium", "media(id)", "RESTRICT", "RESTRICT")
	table.AddUniqueIndex("uniqueness_idx", "medium", "address")
}

func GetGroupById(db *gorm.DB, id string) *Group {
	var o Group
	if result := db.First(&o, "id = ?", id); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return &o
}

func GetGroupByAddress(db *gorm.DB, address string) *Group {
	var o Group
	if result := db.Joins("inner join groups on group_addresses.group_id = groups.id").Where("group_addresses.address = ?", address).Find(&o); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return &o
}

func getGroupAddressesByMediumAndGroupId(db *gorm.DB, medium string, groupId string) *GroupAddress {
	var o GroupAddress
	if result := db.Where("medium =? and group_id =?", medium, groupId).Find(&o); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return &o
}

func getGroupAddressesByMedium(db *gorm.DB, medium string) *GroupAddress {
	var o GroupAddress
	if result := db.Where("medium =?", medium).Find(&o); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return &o
}

func NewGroup(db *gorm.DB, name string) *Group {
	o := Group{Id: name}
	if err := db.Create(&o).Error; err != nil {
		panic(err)
	}
	return &o
}

func (o *Group) Subentity(name string, req he.Request) he.Entity {
	panic("TODO: API: (*Group).Subentity()")
}

func (o *Group) Methods() map[string]func(he.Request) he.Response {
	return map[string]func(he.Request) he.Response{
		"GET": func(req he.Request) he.Response {
			// TODO: permission check
			return he.StatusOK(o)
		},
		"PUT": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"PUT\"]")
		},
		"PATCH": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"PATCH\"]")
		},
		"DELETE": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"DELETE\"]")
		},
	}
}

// View //////////////////////////////////////////////////////////////

func (o *Group) Encoders() map[string]func(io.Writer) error {
	return defaultEncoders(o)
}

// Directory ("Controller") //////////////////////////////////////////

type t_dirGroups struct {
	methods map[string]func(he.Request) he.Response
}

func newDirGroups() t_dirGroups {
	r := t_dirGroups{}
	r.methods = map[string]func(he.Request) he.Response{
		"POST": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			badbody := he.StatusUnsupportedMediaType(heutil.NetString(fmt.Sprintf("submitted body not what expected")))
			hash, ok := req.Entity.(map[string]interface{}); if !ok { return badbody }
			groupname, ok := hash["groupname"].(string)    ; if !ok { return badbody }

			groupname = strings.ToLower(groupname)

			group := NewGroup(db, groupname)
			if group == nil {
				return he.StatusConflict(heutil.NetString("a group with that name already exists"))
			} else {
				return he.StatusCreated(r, groupname, req)
			}
		},
	}
	return r
}

func (d t_dirGroups) Methods() map[string]func(he.Request) he.Response {
	return d.methods
}

func (d t_dirGroups) Subentity(name string, req he.Request) he.Entity {
	db := req.Things["db"].(*gorm.DB)
	return GetGroupById(db, name)
}
<commit_msg>added group get and put functionality<commit_after>// Copyright 2015 Davis Webb
// Copyright 2015 Luke Shumaker
// Copyright 2015 Guntas Grewal

package store

import (
	"fmt"
	"github.com/jinzhu/gorm"
	he "httpentity"
	"httpentity/util"
	"io"
	"strings"
)

var _ he.Entity = &Group{}
var _ he.NetEntity = &Group{}
var dirGroups he.Entity = newDirGroups()

// Model /////////////////////////////////////////////////////////////

type Group struct {
	Id        string
	Addresses []GroupAddress
}

func (o Group) schema(db *gorm.DB) {
	db.CreateTable(&o)
}

type GroupAddress struct {
	Id      int64
	GroupId string
	Medium  string
	Address string
}

func (o GroupAddress) schema(db *gorm.DB) {
	table := db.CreateTable(&o)
	table.AddForeignKey("group_id", "groups(id)", "RESTRICT", "RESTRICT")
	table.AddForeignKey("medium", "media(id)", "RESTRICT", "RESTRICT")
	table.AddUniqueIndex("uniqueness_idx", "medium", "address")
}

func GetGroupById(db *gorm.DB, id string) *Group {
	var o Group
	if result := db.First(&o, "id = ?", id); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return &o
}

func GetUsersInGroup(db *gorm.DB, groupId string) *[]User {
	var users []User
	err := db.Joins("inner join user_addresses on user_addresses.user_id = users.id").Joins(
	"inner join subscriptions on subscriptions.address_id = user_addresses.id").Where(
	"subscriptions.group_id = ?", groupId).Find(&users)
	if err != nil {
		panic("could not get users in group")
	}
	return &users
}

func GetGroupByAddress(db *gorm.DB, address string) *Group {
	var o Group
	if result := db.Joins("inner join groups on group_addresses.group_id = groups.id").Where("group_addresses.address = ?", address).Find(&o); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return &o
}

func getGroupAddressesByMediumAndGroupId(db *gorm.DB, medium string, groupId string) *GroupAddress {
	var o GroupAddress
	if result := db.Where("medium =? and group_id =?", medium, groupId).Find(&o); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return &o
}

func getGroupAddressesByMedium(db *gorm.DB, medium string) *GroupAddress {
	var o GroupAddress
	if result := db.Where("medium =?", medium).Find(&o); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return &o
}

func NewGroup(db *gorm.DB, name string) *Group {
	o := Group{Id: name}
	if err := db.Create(&o).Error; err != nil {
		panic(err)
	}
	return &o
}

func (o *Group) Subentity(name string, req he.Request) he.Entity {
	panic("TODO: API: (*Group).Subentity()")
}

func (o *Group) Methods() map[string]func(he.Request) he.Response {
	return map[string]func(he.Request) he.Response{
		"GET": func(req he.Request) he.Response {
			return he.StatusOK(o)
	},
		"PUT": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			sess := req.Things["session"].(*Session)
			users = GetUsersInGroup(db,  o.Id)
			flag := false
			for i := o; i < range *users; i++ {
				if sess.UserId == users[i].Id {
					flag = true
					break
				}
			}
			if !flag {
                                return he.StatusForbidden(heutil.NetString("Unauthorized user"))
                        }

			var new_group Group
			err := safeDecodeJSON(req.Entity, &new_group)
			if err != nil {
				return err.Response()
			}
			if o.Id != new_group.Id {
				return he.StatusConflict(heutil.NetString("Cannot change group id"))
			}
			*o = new_group
			o.Save(db)
			return he.StatusOK(o)
		},
		"PATCH": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"PATCH\"]")
		},
		"DELETE": func(req he.Request) he.Response {
			panic("TODO: API: (*Group).Methods()[\"DELETE\"]")
		},
	}
}

// View //////////////////////////////////////////////////////////////

func (o *Group) Encoders() map[string]func(io.Writer) error {
	return defaultEncoders(o)
}

// Directory ("Controller") //////////////////////////////////////////

type t_dirGroups struct {
	methods map[string]func(he.Request) he.Response
}

func newDirGroups() t_dirGroups {
	r := t_dirGroups{}
	r.methods = map[string]func(he.Request) he.Response{
		"POST": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			badbody := he.StatusUnsupportedMediaType(heutil.NetString(fmt.Sprintf("submitted body not what expected")))
			hash, ok := req.Entity.(map[string]interface{}); if !ok { return badbody }
			groupname, ok := hash["groupname"].(string)    ; if !ok { return badbody }

			groupname = strings.ToLower(groupname)

			group := NewGroup(db, groupname)
			if group == nil {
				return he.StatusConflict(heutil.NetString("a group with that name already exists"))
			} else {
				return he.StatusCreated(r, groupname, req)
			}
		},
	}
	return r
}

func (d t_dirGroups) Methods() map[string]func(he.Request) he.Response {
	return d.methods
}

func (d t_dirGroups) Subentity(name string, req he.Request) he.Entity {
	db := req.Things["db"].(*gorm.DB)
	return GetGroupById(db, name)
}
<|endoftext|>


<commit_before>// Copyright 2015 Davis Webb
// Copyright 2015 Luke Shumaker
// Copyright 2015 Guntas Grewal

package store

import (
	"github.com/jinzhu/gorm"
	he "httpentity"
	"httpentity/util" // heutil
	"io"
	"jsonpatch"
	"strings"
)

var _ he.Entity = &Group{}
var _ he.NetEntity = &Group{}
var dirGroups he.Entity = newDirGroups()

// Model /////////////////////////////////////////////////////////////

type Group struct {
	Id            string         `json:"group_id"`
	Existence     int            `json:"existence"` // 1 -> public, 2 -> confirmed, 3 -> member
	Read          int            `json:"read"`      // 1 -> public, 2 -> confirmed, 3 -> member
	Post          int            `json:"post"`      // 1 -> public, 2 -> confirmed, 3 -> moderator
	Join          int            `json:"join"`      // 1 -> auto join, 2 -> confirm to join
	Subscriptions []Subscription `json:"subscriptions"`
}

func (o Group) dbSchema(db *gorm.DB) error {
	return db.CreateTable(&o).Error
}

func (o Group) dbSeed(db *gorm.DB) error {
	return db.Create(&Group{
		Id:            "test",
		Existence:     1,
		Read:          1,
		Post:          1,
		Join:          1,
		Subscriptions: []Subscription{},
	}).Error
}

func GetGroupById(db *gorm.DB, id string) *Group {
	var o Group
	if result := db.First(&o, "id = ?", id); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	db.Model(&o).Related(&o.Subscriptions)
	return &o
}

func GetGroupsByMember(db *gorm.DB, user User) []Group {
	// turn the User's list of addresses into a list of address IDs
	user_address_ids := make([]int64, len(user.Addresses))
	for i, user_address := range user.Addresses {
		user_address_ids[i] = user_address.Id
	}
	// use the list of address IDs to get a list of subscriptions
	var subscriptions []Subscription
	if len(user_address_ids) > 0 {
		if result := db.Where("address_id IN (?)", user_address_ids).Find(&subscriptions); result.Error != nil {
			if result.RecordNotFound() {
				return nil
			}
			panic(result.Error)
		}
	} else {
		subscriptions = make([]Subscription, 0)
	}
	// turn the list of subscriptions into a list of group IDs
	group_ids := make([]string, len(subscriptions))
	for i, subscription := range subscriptions {
		group_ids[i] = subscription.GroupId
	}
	// use the list of group IDs to get the groups
	var groups []Group
	if result := db.Where(group_ids).Find(&groups); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	// return them
	return groups
}

func GetPublicAndSubscribedGroups(db *gorm.DB, user User) []Group {
	groups := GetGroupsByMember(db, user)
	// also get public groups
	var publicgroups []Group
	if result := db.Where(&Group{Existence: 1}).Find(&publicgroups); result.Error != nil {
		if !result.RecordNotFound() {
			panic(result.Error)
		}
	}
	// merge public groups and subscribed groups
	for _, publicgroup := range publicgroups {
		for _, group := range groups {
			if group.Id == publicgroup.Id {
				break
			}
		}
		groups = append(groups, publicgroup)
	}

	// return them
	return groups
}

func GetAllGroups(db *gorm.DB) []Group {
	var o []Group
	if result := db.Find(&o); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return o
}

func NewGroup(db *gorm.DB, name string, existence int, read int, post int, join int) *Group {
	if name == "" {
		panic("name can't be empty")
	}
	subscriptions := make([]Subscription, 0)
	o := Group{
		Id:            name,
		Existence:     CheckInput(existence, 1, 3, 1),
		Read:          CheckInput(read, 1, 3, 1),
		Post:          CheckInput(post, 1, 3, 1),
		Join:          CheckInput(existence, 1, 2, 1),
		Subscriptions: subscriptions,
	}
	if err := db.Create(&o).Error; err != nil {
		panic(err)
	}
	return &o
}

func CheckInput(input int, min int, max int, defaultt int) int {
	if input < min || input > max {
		return defaultt
	}
	return input
}

func (o *Group) Save(db *gorm.DB) {
	if err := db.Save(o).Error; err != nil {
		panic(err)
	}
}

func (o *Group) Subentity(name string, req he.Request) he.Entity {
	panic("TODO: API: (*Group).Subentity()")
}

func (o *Group) Methods() map[string]func(he.Request) he.Response {
	return map[string]func(he.Request) he.Response{
		"GET": func(req he.Request) he.Response {
			return he.StatusOK(o)
		},
		"PUT": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)

			var new_group Group
			httperr := safeDecodeJSON(req.Entity, &new_group)
			if httperr != nil {
				return *httperr
			}
			if o.Id != new_group.Id {
				return he.StatusConflict(heutil.NetString("Cannot change group id"))
			}
			*o = new_group
			o.Save(db)
			return he.StatusOK(o)
		},
		"PATCH": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)

			patch, ok := req.Entity.(jsonpatch.Patch)
			if !ok {
				return he.StatusUnsupportedMediaType(heutil.NetString("PATCH request must have a patch media type"))
			}
			var new_group Group
			err := patch.Apply(o, &new_group)
			if err != nil {
				return he.StatusConflict(heutil.NetPrintf("%v", err))
			}
			if o.Id != new_group.Id {
				return he.StatusConflict(heutil.NetString("Cannot change user id"))
			}
			*o = new_group
			o.Save(db)
			return he.StatusOK(o)
		},
		"DELETE": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			db.Delete(o)
			return he.StatusNoContent()
		},
	}
}

// View //////////////////////////////////////////////////////////////

func (o *Group) Encoders() map[string]func(io.Writer) error {
	return defaultEncoders(o)
}

// Directory ("Controller") //////////////////////////////////////////

type t_dirGroups struct {
	methods map[string]func(he.Request) he.Response
}

func newDirGroups() t_dirGroups {
	r := t_dirGroups{}
	r.methods = map[string]func(he.Request) he.Response{
		"GET": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			sess := req.Things["session"].(*Session)
			var groups []Group
			if sess == nil {
				groups = []Group{}
			} else {
				//groups = GetAllGroups(db)
				groups = GetPublicAndSubscribedGroups(db, *GetUserById(db, sess.UserId))
			}
			generic := make([]interface{}, len(groups))
			type EnumerateGroup struct {
				Id            string         `json:"id"`
				Existence     string         `json:"existence"`
				Read          string         `json:"read"`
				Post          string         `json:"post"`
				Join          string         `json:"join"`
				Subscriptions []Subscription `json:"subscriptions"`
			}

			for i, group := range groups {
				var enum EnumerateGroup
				enum.Id = group.Id
				enum.Existence = Existence(group.Existence).String()
				enum.Read = Read(group.Read).String()
				enum.Post = Post(group.Post).String()
				enum.Join = Join(group.Join).String()
				enum.Subscriptions = group.Subscriptions
				generic[i] = enum
			}
			return he.StatusOK(heutil.NetList(generic))
		},
		"POST": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			type postfmt struct {
				Groupname string    `json:"groupname"`
				Existence Existence `json:"existence"`
				Read      Read      `json:"read"`
				Post      Post      `json:"post"`
				Join      Join      `json:"join"`
			}
			var entity postfmt
			httperr := safeDecodeJSON(req.Entity, &entity)
			if httperr != nil {
				return *httperr
			}

			if entity.Groupname == "" {
				return he.StatusUnsupportedMediaType(heutil.NetString("groupname can't be emtpy"))
			}

			entity.Groupname = strings.ToLower(entity.Groupname)
			group := NewGroup(
				db,
				entity.Groupname,
				int(entity.Existence),
				int(entity.Read),
				int(entity.Post),
				int(entity.Join),
			)
			sess := req.Things["session"].(*Session)
			address := GetAddressByIdAndMedium(db, sess.UserId, "noop")
			if address != nil {
				subscription := Subscription{
					AddressId: address.Id,
					GroupId:   group.Id,
					Confirmed: true,
				}
				db.Create(&subscription)
			}
			if group == nil {
				return he.StatusConflict(heutil.NetString("a group with that name already exists"))
			} else {
				return he.StatusCreated(r, group.Id, req)
			}
		},
	}
	return r
}

func (d t_dirGroups) Methods() map[string]func(he.Request) he.Response {
	return d.methods
}

func (d t_dirGroups) Subentity(name string, req he.Request) he.Entity {
	name = strings.ToLower(name)
	db := req.Things["db"].(*gorm.DB)
	// TODO: permissions check
	//sess := req.Things["session"].(*Session)
	return GetGroupById(db, name)
}
<commit_msg>added permissions check for getting a specific group<commit_after>// Copyright 2015 Davis Webb
// Copyright 2015 Luke Shumaker
// Copyright 2015 Guntas Grewal

package store

import (
	"github.com/jinzhu/gorm"
	he "httpentity"
	"httpentity/util" // heutil
	"io"
	"jsonpatch"
	"strings"
)

var _ he.Entity = &Group{}
var _ he.NetEntity = &Group{}
var dirGroups he.Entity = newDirGroups()

// Model /////////////////////////////////////////////////////////////

type Group struct {
	Id            string         `json:"group_id"`
	Existence     int            `json:"existence"` // 1 -> public, 2 -> confirmed, 3 -> member
	Read          int            `json:"read"`      // 1 -> public, 2 -> confirmed, 3 -> member
	Post          int            `json:"post"`      // 1 -> public, 2 -> confirmed, 3 -> moderator
	Join          int            `json:"join"`      // 1 -> auto join, 2 -> confirm to join
	Subscriptions []Subscription `json:"subscriptions"`
}

func (o Group) dbSchema(db *gorm.DB) error {
	return db.CreateTable(&o).Error
}

func (o Group) dbSeed(db *gorm.DB) error {
	return db.Create(&Group{
		Id:            "test",
		Existence:     1,
		Read:          1,
		Post:          1,
		Join:          1,
		Subscriptions: []Subscription{},
	}).Error
}

func GetGroupById(db *gorm.DB, id string) *Group {
	var o Group
	if result := db.First(&o, "id = ?", id); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	db.Model(&o).Related(&o.Subscriptions)
	return &o
}

func GetGroupsByMember(db *gorm.DB, user User) []Group {
	// turn the User's list of addresses into a list of address IDs
	user_address_ids := make([]int64, len(user.Addresses))
	for i, user_address := range user.Addresses {
		user_address_ids[i] = user_address.Id
	}
	// use the list of address IDs to get a list of subscriptions
	var subscriptions []Subscription
	if len(user_address_ids) > 0 {
		if result := db.Where("address_id IN (?)", user_address_ids).Find(&subscriptions); result.Error != nil {
			if result.RecordNotFound() {
				return nil
			}
			panic(result.Error)
		}
	} else {
		subscriptions = make([]Subscription, 0)
	}
	// turn the list of subscriptions into a list of group IDs
	group_ids := make([]string, len(subscriptions))
	for i, subscription := range subscriptions {
		group_ids[i] = subscription.GroupId
	}
	// use the list of group IDs to get the groups
	var groups []Group
	if result := db.Where(group_ids).Find(&groups); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	// return them
	return groups
}

func GetPublicAndSubscribedGroups(db *gorm.DB, user User) []Group {
	groups := GetGroupsByMember(db, user)
	// also get public groups
	var publicgroups []Group
	if result := db.Where(&Group{Existence: 1}).Find(&publicgroups); result.Error != nil {
		if !result.RecordNotFound() {
			panic(result.Error)
		}
	}
	// merge public groups and subscribed groups
	for _, publicgroup := range publicgroups {
		for _, group := range groups {
			if group.Id == publicgroup.Id {
				break
			}
		}
		groups = append(groups, publicgroup)
	}

	// return them
	return groups
}

func GetAllGroups(db *gorm.DB) []Group {
	var o []Group
	if result := db.Find(&o); result.Error != nil {
		if result.RecordNotFound() {
			return nil
		}
		panic(result.Error)
	}
	return o
}

func NewGroup(db *gorm.DB, name string, existence int, read int, post int, join int) *Group {
	if name == "" {
		panic("name can't be empty")
	}
	subscriptions := make([]Subscription, 0)
	o := Group{
		Id:            name,
		Existence:     CheckInput(existence, 1, 3, 1),
		Read:          CheckInput(read, 1, 3, 1),
		Post:          CheckInput(post, 1, 3, 1),
		Join:          CheckInput(existence, 1, 2, 1),
		Subscriptions: subscriptions,
	}
	if err := db.Create(&o).Error; err != nil {
		panic(err)
	}
	return &o
}

func CheckInput(input int, min int, max int, defaultt int) int {
	if input < min || input > max {
		return defaultt
	}
	return input
}

func (o *Group) Save(db *gorm.DB) {
	if err := db.Save(o).Error; err != nil {
		panic(err)
	}
}

func IsSubscribed(db *gorm.DB, userid string, group Group) bool {
	subscriptions := GetSubscriptionsGroupById(db, group.Id)
	address_ids := make([]int64, len(subscriptions))
	for i, subscription := range subscriptions {
		address_ids[i] = subscription.AddressId
	}
	var addresses []UserAddress
	if len(address_ids) > 0 {
		if result := db.Where("id IN (?)", address_ids).Find(&addresses); result.Error != nil {
			if !result.RecordNotFound() {
				panic("cant find any subscriptions corresponding user address")
			}
		}
	} else {
		// no subscriptions so user cannot possibly be subscribed
		return false
	}
	for _, address := range addresses {
		if address.UserId == userid {
			return true
		}
	}
	// could not find user in subscribed user addresses, therefore, he/she isn't subscribed
	return false
}

func (o *Group) Subentity(name string, req he.Request) he.Entity {
	panic("TODO: API: (*Group).Subentity()")
}

func (o *Group) Methods() map[string]func(he.Request) he.Response {
	return map[string]func(he.Request) he.Response{
		"GET": func(req he.Request) he.Response {
			// check permissions
			db := req.Things["db"].(*gorm.DB)
			sess := req.Things["session"].(*Session)
			if o.Read != 1 || !IsSubscribed(db, sess.UserId, *o) {
				return he.StatusForbidden(heutil.NetString("Unauthorized user"))
			}
			o.Subscriptions = GetSubscriptionsGroupById(db, o.Id)
			return he.StatusOK(o)
		},
		"PUT": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)

			var new_group Group
			httperr := safeDecodeJSON(req.Entity, &new_group)
			if httperr != nil {
				return *httperr
			}
			if o.Id != new_group.Id {
				return he.StatusConflict(heutil.NetString("Cannot change group id"))
			}
			*o = new_group
			o.Save(db)
			return he.StatusOK(o)
		},
		"PATCH": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)

			patch, ok := req.Entity.(jsonpatch.Patch)
			if !ok {
				return he.StatusUnsupportedMediaType(heutil.NetString("PATCH request must have a patch media type"))
			}
			var new_group Group
			err := patch.Apply(o, &new_group)
			if err != nil {
				return he.StatusConflict(heutil.NetPrintf("%v", err))
			}
			if o.Id != new_group.Id {
				return he.StatusConflict(heutil.NetString("Cannot change user id"))
			}
			*o = new_group
			o.Save(db)
			return he.StatusOK(o)
		},
		"DELETE": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			db.Delete(o)
			return he.StatusNoContent()
		},
	}
}

// View //////////////////////////////////////////////////////////////

func (o *Group) Encoders() map[string]func(io.Writer) error {
	return defaultEncoders(o)
}

// Directory ("Controller") //////////////////////////////////////////

type t_dirGroups struct {
	methods map[string]func(he.Request) he.Response
}

func newDirGroups() t_dirGroups {
	r := t_dirGroups{}
	r.methods = map[string]func(he.Request) he.Response{
		"GET": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			sess := req.Things["session"].(*Session)
			var groups []Group
			if sess == nil {
				groups = []Group{}
			} else {
				//groups = GetAllGroups(db)
				groups = GetPublicAndSubscribedGroups(db, *GetUserById(db, sess.UserId))
			}
			generic := make([]interface{}, len(groups))
			type EnumerateGroup struct {
				Id            string         `json:"id"`
				Existence     string         `json:"existence"`
				Read          string         `json:"read"`
				Post          string         `json:"post"`
				Join          string         `json:"join"`
				Subscriptions []Subscription `json:"subscriptions"`
			}

			for i, group := range groups {
				var enum EnumerateGroup
				enum.Id = group.Id
				enum.Existence = Existence(group.Existence).String()
				enum.Read = Read(group.Read).String()
				enum.Post = Post(group.Post).String()
				enum.Join = Join(group.Join).String()
				enum.Subscriptions = group.Subscriptions
				generic[i] = enum
			}
			return he.StatusOK(heutil.NetList(generic))
		},
		"POST": func(req he.Request) he.Response {
			db := req.Things["db"].(*gorm.DB)
			type postfmt struct {
				Groupname string    `json:"groupname"`
				Existence Existence `json:"existence"`
				Read      Read      `json:"read"`
				Post      Post      `json:"post"`
				Join      Join      `json:"join"`
			}
			var entity postfmt
			httperr := safeDecodeJSON(req.Entity, &entity)
			if httperr != nil {
				return *httperr
			}

			if entity.Groupname == "" {
				return he.StatusUnsupportedMediaType(heutil.NetString("groupname can't be emtpy"))
			}

			entity.Groupname = strings.ToLower(entity.Groupname)
			group := NewGroup(
				db,
				entity.Groupname,
				int(entity.Existence),
				int(entity.Read),
				int(entity.Post),
				int(entity.Join),
			)
			sess := req.Things["session"].(*Session)
			address := GetAddressByIdAndMedium(db, sess.UserId, "noop")
			if address != nil {
				subscription := Subscription{
					AddressId: address.Id,
					GroupId:   group.Id,
					Confirmed: true,
				}
				db.Create(&subscription)
			}
			if group == nil {
				return he.StatusConflict(heutil.NetString("a group with that name already exists"))
			} else {
				return he.StatusCreated(r, group.Id, req)
			}
		},
	}
	return r
}

func (d t_dirGroups) Methods() map[string]func(he.Request) he.Response {
	return d.methods
}

func (d t_dirGroups) Subentity(name string, req he.Request) he.Entity {
	name = strings.ToLower(name)
	db := req.Things["db"].(*gorm.DB)
	// TODO: permissions check
	//sess := req.Things["session"].(*Session)
	return GetGroupById(db, name)
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"testing"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/common"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/discovery"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/fuse"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pfs/route"
	"github.com/pachyderm/pachyderm/src/pkg/btrfs"
	"github.com/pachyderm/pachyderm/src/pkg/executil"
	"github.com/pachyderm/pachyderm/src/pkg/grpctest"
	"github.com/stretchr/testify/require"
)

const (
	// TODO(pedge): large numbers of shards takes forever because
	// we are doing tons of btrfs operations on init, is there anything
	// we can do about that?
	testShardsPerServer = 8
	testNumServers      = 8
	testSize            = 1000
)

var (
	counter int32
)

func init() {
	// TODO(pedge): needed in tests? will not be needed for golang 1.5 for sure
	runtime.GOMAXPROCS(runtime.NumCPU())
	executil.SetDebug(true)
	common.ForceLogColors()
}

func TestBtrfsFFI(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewFFIAPI())
	runTest(t, driver, testSimple)
}

func TestBtrfsExec(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewExecAPI())
	runTest(t, driver, testSimple)
}

func TestFuseMount(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewExecAPI())
	runTest(t, driver, testMount)
}

func getBtrfsRootDir(t *testing.T) string {
	// TODO(pedge)
	rootDir := os.Getenv("PFS_BTRFS_ROOT")
	if rootDir == "" {
		t.Fatal("PFS_BTRFS_ROOT not set")
	}
	return rootDir
}

func testGetVersion(t *testing.T, apiClient pfs.ApiClient) {
	getVersionResponse, err := pfsutil.GetVersion(apiClient)
	require.NoError(t, err)
	require.Equal(t, common.VersionString(), pfs.VersionString(getVersionResponse.Version))
}

func testSimple(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	getCommitInfoResponse, err := pfsutil.GetCommitInfo(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Nil(t, getCommitInfoResponse.CommitInfo.ParentCommit)

	branchResponse, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	var wg sync.WaitGroup
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, iErr := pfsutil.PutFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(t, iErr)
			_, iErr = pfsutil.PutFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(t, iErr)
		}()
	}
	wg.Wait()

	err = pfsutil.Commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	wg = sync.WaitGroup{}
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			iErr := pfsutil.GetFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), buffer)
			require.NoError(t, iErr)
			require.Equal(t, fmt.Sprintf("hello%d", i), buffer.String())
			buffer = bytes.NewBuffer(nil)
			iErr = pfsutil.GetFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), buffer)
			require.NoError(t, iErr)
			require.Equal(t, fmt.Sprintf("hello%d", i), buffer.String())
		}()
	}
	wg.Wait()

	listFilesResponse, err := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))
	listFilesResponse, err = pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))

	var fileInfos [7][]*pfs.FileInfo
	wg = sync.WaitGroup{}
	for i := 0; i < 7; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			listFilesResponse, iErr := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
			require.NoError(t, iErr)
			fileInfos[i] = listFilesResponse.FileInfo
		}()
	}
	wg.Wait()
	count := 0
	for i := 0; i < 7; i++ {
		count += len(fileInfos[i])
	}
	require.Equal(t, testSize, count)
}

func testMount(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	//directory, err := ioutil.TempDir("", "testMount")
	//require.NoError(t, err)
	directory := "/compile/testMount"
	mounter := fuse.NewMounter()
	go func() {
		err = mounter.Mount(apiClient, repositoryName, directory, 0, 1)
		require.NoError(t, err)
	}()
	mounter.Ready()

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	branchResponse, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	_, err = os.Stat(filepath.Join(directory, newCommitID))
	require.NoError(t, err)

	err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
	require.NoError(t, err)

	_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", strings.NewReader("bar"))
	require.NoError(t, err)

	err = pfsutil.Commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
	require.NoError(t, err)
	require.Equal(t, "foo", string(data))

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
	require.NoError(t, err)
	require.Equal(t, "bar", string(data))
}

func testRepositoryName() string {
	// TODO could be nice to add callee to this string to make it easy to
	// recover results for debugging
	return fmt.Sprintf("test-%d", atomic.AddInt32(&counter, 1))
}

func runTest(
	t *testing.T,
	driver drive.Driver,
	f func(t *testing.T, apiClient pfs.ApiClient),
) {
	grpctest.Run(
		t,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			discoveryClient := discovery.NewMockClient()
			i := 0
			addresses := make([]string, testNumServers)
			for address := range servers {
				shards := make([]string, testShardsPerServer)
				for j := 0; j < testShardsPerServer; j++ {
					shards[j] = fmt.Sprintf("%d", (i*testShardsPerServer)+j)
				}
				_ = discoveryClient.Set(address+"-master", strings.Join(shards, ","), 0)
				addresses[i] = address
				i++
			}
			_ = discoveryClient.Set("all-addresses", strings.Join(addresses, ","), 0)
			for address, server := range servers {
				combinedAPIServer := NewCombinedAPIServer(
					route.NewSharder(
						testShardsPerServer*testNumServers,
					),
					route.NewRouter(
						route.NewDiscoveryAddresser(
							discoveryClient,
						),
						route.NewDialer(),
						address,
					),
					driver,
				)
				pfs.RegisterApiServer(server, combinedAPIServer)
				pfs.RegisterInternalApiServer(server, combinedAPIServer)
			}
		},
		func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			for _, c := range clientConns {
				if c != clientConn {
					_ = c.Close()
				}
			}
			f(
				t,
				pfs.NewApiClient(
					clientConn,
				),
			)
		},
	)
}
<commit_msg>skip test for now<commit_after>package server

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"testing"

	"google.golang.org/grpc"

	"github.com/pachyderm/pachyderm/src/common"
	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/discovery"
	"github.com/pachyderm/pachyderm/src/pfs/drive"
	"github.com/pachyderm/pachyderm/src/pfs/fuse"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pfs/route"
	"github.com/pachyderm/pachyderm/src/pkg/btrfs"
	"github.com/pachyderm/pachyderm/src/pkg/executil"
	"github.com/pachyderm/pachyderm/src/pkg/grpctest"
	"github.com/stretchr/testify/require"
)

const (
	// TODO(pedge): large numbers of shards takes forever because
	// we are doing tons of btrfs operations on init, is there anything
	// we can do about that?
	testShardsPerServer = 8
	testNumServers      = 8
	testSize            = 1000
)

var (
	counter int32
)

func init() {
	// TODO(pedge): needed in tests? will not be needed for golang 1.5 for sure
	runtime.GOMAXPROCS(runtime.NumCPU())
	executil.SetDebug(true)
	common.ForceLogColors()
}

func TestBtrfsFFI(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewFFIAPI())
	runTest(t, driver, testSimple)
}

func TestBtrfsExec(t *testing.T) {
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewExecAPI())
	runTest(t, driver, testSimple)
}

func TestFuseMount(t *testing.T) {
	t.Skip()
	t.Parallel()
	driver := drive.NewBtrfsDriver(getBtrfsRootDir(t), btrfs.NewExecAPI())
	runTest(t, driver, testMount)
}

func getBtrfsRootDir(t *testing.T) string {
	// TODO(pedge)
	rootDir := os.Getenv("PFS_BTRFS_ROOT")
	if rootDir == "" {
		t.Fatal("PFS_BTRFS_ROOT not set")
	}
	return rootDir
}

func testGetVersion(t *testing.T, apiClient pfs.ApiClient) {
	getVersionResponse, err := pfsutil.GetVersion(apiClient)
	require.NoError(t, err)
	require.Equal(t, common.VersionString(), pfs.VersionString(getVersionResponse.Version))
}

func testSimple(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	getCommitInfoResponse, err := pfsutil.GetCommitInfo(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Nil(t, getCommitInfoResponse.CommitInfo.ParentCommit)

	branchResponse, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_WRITE, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/b")
	require.NoError(t, err)
	err = pfsutil.MakeDirectory(apiClient, repositoryName, newCommitID, "a/c")
	require.NoError(t, err)

	var wg sync.WaitGroup
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			_, iErr := pfsutil.PutFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(t, iErr)
			_, iErr = pfsutil.PutFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), strings.NewReader(fmt.Sprintf("hello%d", i)))
			require.NoError(t, iErr)
		}()
	}
	wg.Wait()

	err = pfsutil.Commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	getCommitInfoResponse, err = pfsutil.GetCommitInfo(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)
	require.NotNil(t, getCommitInfoResponse)
	require.Equal(t, newCommitID, getCommitInfoResponse.CommitInfo.Commit.Id)
	require.Equal(t, pfs.CommitType_COMMIT_TYPE_READ, getCommitInfoResponse.CommitInfo.CommitType)
	require.Equal(t, "scratch", getCommitInfoResponse.CommitInfo.ParentCommit.Id)

	wg = sync.WaitGroup{}
	for i := 0; i < testSize; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			buffer := bytes.NewBuffer(nil)
			iErr := pfsutil.GetFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/b/file%d", i), buffer)
			require.NoError(t, iErr)
			require.Equal(t, fmt.Sprintf("hello%d", i), buffer.String())
			buffer = bytes.NewBuffer(nil)
			iErr = pfsutil.GetFile(apiClient, repositoryName, newCommitID, fmt.Sprintf("a/c/file%d", i), buffer)
			require.NoError(t, iErr)
			require.Equal(t, fmt.Sprintf("hello%d", i), buffer.String())
		}()
	}
	wg.Wait()

	listFilesResponse, err := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))
	listFilesResponse, err = pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/c", 0, 1)
	require.NoError(t, err)
	require.Equal(t, testSize, len(listFilesResponse.FileInfo))

	var fileInfos [7][]*pfs.FileInfo
	wg = sync.WaitGroup{}
	for i := 0; i < 7; i++ {
		i := i
		wg.Add(1)
		go func() {
			defer wg.Done()
			listFilesResponse, iErr := pfsutil.ListFiles(apiClient, repositoryName, newCommitID, "a/b", uint64(i), 7)
			require.NoError(t, iErr)
			fileInfos[i] = listFilesResponse.FileInfo
		}()
	}
	wg.Wait()
	count := 0
	for i := 0; i < 7; i++ {
		count += len(fileInfos[i])
	}
	require.Equal(t, testSize, count)
}

func testMount(t *testing.T, apiClient pfs.ApiClient) {
	repositoryName := testRepositoryName()

	err := pfsutil.InitRepository(apiClient, repositoryName)
	require.NoError(t, err)

	//directory, err := ioutil.TempDir("", "testMount")
	//require.NoError(t, err)
	directory := "/compile/testMount"
	mounter := fuse.NewMounter()
	go func() {
		err = mounter.Mount(apiClient, repositoryName, directory, 0, 1)
		require.NoError(t, err)
	}()
	mounter.Ready()

	_, err = os.Stat(filepath.Join(directory, "scratch"))
	require.NoError(t, err)

	branchResponse, err := pfsutil.Branch(apiClient, repositoryName, "scratch")
	require.NoError(t, err)
	require.NotNil(t, branchResponse)
	newCommitID := branchResponse.Commit.Id

	_, err = os.Stat(filepath.Join(directory, newCommitID))
	require.NoError(t, err)

	err = ioutil.WriteFile(filepath.Join(directory, newCommitID, "foo"), []byte("foo"), 0666)
	require.NoError(t, err)

	_, err = pfsutil.PutFile(apiClient, repositoryName, newCommitID, "bar", strings.NewReader("bar"))
	require.NoError(t, err)

	err = pfsutil.Commit(apiClient, repositoryName, newCommitID)
	require.NoError(t, err)

	data, err := ioutil.ReadFile(filepath.Join(directory, newCommitID, "foo"))
	require.NoError(t, err)
	require.Equal(t, "foo", string(data))

	data, err = ioutil.ReadFile(filepath.Join(directory, newCommitID, "bar"))
	require.NoError(t, err)
	require.Equal(t, "bar", string(data))
}

func testRepositoryName() string {
	// TODO could be nice to add callee to this string to make it easy to
	// recover results for debugging
	return fmt.Sprintf("test-%d", atomic.AddInt32(&counter, 1))
}

func runTest(
	t *testing.T,
	driver drive.Driver,
	f func(t *testing.T, apiClient pfs.ApiClient),
) {
	grpctest.Run(
		t,
		testNumServers,
		func(servers map[string]*grpc.Server) {
			discoveryClient := discovery.NewMockClient()
			i := 0
			addresses := make([]string, testNumServers)
			for address := range servers {
				shards := make([]string, testShardsPerServer)
				for j := 0; j < testShardsPerServer; j++ {
					shards[j] = fmt.Sprintf("%d", (i*testShardsPerServer)+j)
				}
				_ = discoveryClient.Set(address+"-master", strings.Join(shards, ","), 0)
				addresses[i] = address
				i++
			}
			_ = discoveryClient.Set("all-addresses", strings.Join(addresses, ","), 0)
			for address, server := range servers {
				combinedAPIServer := NewCombinedAPIServer(
					route.NewSharder(
						testShardsPerServer*testNumServers,
					),
					route.NewRouter(
						route.NewDiscoveryAddresser(
							discoveryClient,
						),
						route.NewDialer(),
						address,
					),
					driver,
				)
				pfs.RegisterApiServer(server, combinedAPIServer)
				pfs.RegisterInternalApiServer(server, combinedAPIServer)
			}
		},
		func(t *testing.T, clientConns map[string]*grpc.ClientConn) {
			var clientConn *grpc.ClientConn
			for _, c := range clientConns {
				clientConn = c
				break
			}
			for _, c := range clientConns {
				if c != clientConn {
					_ = c.Close()
				}
			}
			f(
				t,
				pfs.NewApiClient(
					clientConn,
				),
			)
		},
	)
}
<|endoftext|>