<commit_before>package gocbcore

import (
	"strings"
	"sync"
	"time"
)

type memdInitFunc func(*memdPipeline, time.Time) error

type CloseHandler func(*memdPipeline)
type BadRouteHandler func(*memdPipeline, *memdQRequest, *memdResponse)

type Callback func(*memdResponse, error)

type memdPipeline struct {
	lock sync.RWMutex

	queue *memdQueue

	address  string
	conn     memdReadWriteCloser
	isClosed bool
	ioDoneCh chan bool

	opList  memdOpMap

	handleBadRoute BadRouteHandler
	handleDeath    CloseHandler
}

func CreateMemdPipeline(address string) *memdPipeline {
	return &memdPipeline{
		address:  address,
		queue:    createMemdQueue(),
		ioDoneCh: make(chan bool, 1),
	}
}

func (s *memdPipeline) Address() string {
	return s.address
}

func (s *memdPipeline) Hostname() string {
	return strings.Split(s.address, ":")[0]
}

func (s *memdPipeline) IsClosed() bool {
	return s.isClosed
}

func (s *memdPipeline) SetHandlers(badRouteFn BadRouteHandler, deathFn CloseHandler) {
	s.lock.Lock()

	if s.isClosed {
		// We died between authentication and here, immediately notify the deathFn
		s.lock.Unlock()
		deathFn(s)
		return
	}

	s.handleBadRoute = badRouteFn
	s.handleDeath = deathFn
	s.lock.Unlock()
}

func (pipeline *memdPipeline) ExecuteRequest(req *memdQRequest, deadline time.Time) (respOut *memdResponse, errOut error) {
	if req.Callback != nil {
		panic("Tried to synchronously dispatch an operation with an async handler.")
	}

	signal := make(chan bool)

	req.Callback = func(resp *memdResponse, err error) {
		respOut = resp
		errOut = err
		signal <- true
	}

	if !pipeline.queue.QueueRequest(req) {
		return nil, &generalError{"Failed to dispatch operation."}
	}

	timeoutTmr := AcquireTimer(deadline.Sub(time.Now()))
	select {
	case <-signal:
		ReleaseTimer(timeoutTmr, false)
		return
	case <-timeoutTmr.C:
		ReleaseTimer(timeoutTmr, true)
		req.Cancel()
		return nil, &timeoutError{}
	}
}

func (pipeline *memdPipeline) dispatchRequest(req *memdQRequest) error {
	// We do a cursory check of the server to avoid dispatching operations on the network
	//   that have already knowingly been cancelled.  This doesn't guarentee a cancelled
	//   operation from being sent, but it does reduce network IO when possible.
	if req.QueueOwner() != pipeline.queue {
		// Even though we failed to dispatch, this is not actually an error,
		//   we just consume the operation since its already been handled elsewhere
		return nil
	}

	pipeline.opList.Add(req)

	err := pipeline.conn.WritePacket(&req.memdRequest)
	if err != nil {
		logDebugf("Got write error")
		pipeline.opList.Remove(req)
		return err
	}

	return nil
}

func (s *memdPipeline) resolveRequest(resp *memdResponse) {
	opIndex := resp.Opaque

	// Find the request that goes with this response
	req := s.opList.FindAndMaybeRemove(opIndex)

	if req == nil {
		// There is no known request that goes with this response.  Ignore it.
		logDebugf("Received response with no corresponding request.")
		return
	}

	if !req.Persistent || (resp.Magic == ResMagic && resp.Status != StatusSuccess) {
		if !s.queue.UnqueueRequest(req) {
			// While we found a valid request, the request does not appear to be queued
			//   with this server anymore, this probably means that it has been cancelled.
			logDebugf("Received response for cancelled request.")
			return
		}
	}

	if resp.Magic == ResMagic && resp.Status == StatusNotMyVBucket {
		// If possible, lets backchannel our NMV back to the Agent of this memdQueueConn
		//   instance.  This is primarily meant to enhance performance, and allow the
		//   agent to be instantly notified upon a new configuration arriving.  If the
		//   backchannel isn't available, we just Callback with the NMV error.
		logDebugf("Received NMV response.")
		s.lock.RLock()
		badRouteFn := s.handleBadRoute
		s.lock.RUnlock()
		if badRouteFn != nil {
			badRouteFn(s, req, resp)
			return
		}
	}

	// Call the requests callback handler...  Ignore Status field for incoming requests.
	logDebugf("Dispatching response callback.")
	if resp.Magic == ReqMagic || resp.Status == StatusSuccess {
		req.Callback(resp, nil)
	} else {
		req.Callback(nil, &memdError{resp.Status})
	}
}

func (pipeline *memdPipeline) ioLoop() {
	killSig := make(chan bool)

	// Reading
	go func() {
		logDebugf("Reader loop starting...")
		for {
			resp := &memdResponse{}
			err := pipeline.conn.ReadPacket(resp)
			if err != nil {
				logDebugf("Server read error: %v", err)
				killSig <- true
				break
			}

			logDebugf("Got response to resolve.")
			pipeline.resolveRequest(resp)
		}
	}()

	// Writing
	logDebugf("Writer loop starting...")
	for {
		select {
		case req := <-pipeline.queue.reqsCh:
			logDebugf("Got a request to dispatch.")
			err := pipeline.dispatchRequest(req)
			if err != nil {
				// We can assume that the server is not fully drained yet, as the drainer blocks
				//   waiting for the IO goroutines to finish first.
				pipeline.queue.reqsCh <- req

				// We must wait for the receive goroutine to die as well before we can continue.
				<-killSig

				return
			}
		case <-killSig:
			return
		}
	}
}

func (pipeline *memdPipeline) Run() {
	logDebugf("Beginning pipeline runner")

	// Run the IO loop.  This will block until the connection has been closed.
	pipeline.ioLoop()

	// Now we must signal drainers that we are done!
	pipeline.ioDoneCh <- true

	// Signal the creator that we died :(
	pipeline.lock.Lock()
	pipeline.isClosed = true
	deathFn := pipeline.handleDeath
	pipeline.lock.Unlock()
	if deathFn != nil {
		deathFn(pipeline)
	} else {
		pipeline.Drain(nil)
	}
}

func (pipeline *memdPipeline) Close() {
	pipeline.conn.Close()
}

func (pipeline *memdPipeline) Drain(reqCb drainedReqCallback) {
	// If the user does no pass a drain callback, we handle the requests
	//   by immediately failing them with a network error.
	if reqCb == nil {
		reqCb = func(req *memdQRequest) {
			req.Callback(nil, networkError{})
		}
	}

	// Make sure the connection is closed, which will signal the ioLoop
	//   to stop running and signal on ioDoneCh
	pipeline.conn.Close()

	// Drain the request queue, this will block until the io thread signals
	//   on ioDoneCh, and the queues have been completely emptied
	pipeline.queue.Drain(reqCb, pipeline.ioDoneCh)

	// As a last step, immediately notify all the requests that were
	//   on-the-wire that a network error has occurred.
	pipeline.opList.Drain(func(r *memdQRequest) {
		if pipeline.queue.UnqueueRequest(r) {
			r.Callback(nil, networkError{})
		}
	})
}
<commit_msg>Closing a connection should drain any pending requests.<commit_after>package gocbcore

import (
	"strings"
	"sync"
	"time"
)

type memdInitFunc func(*memdPipeline, time.Time) error

type CloseHandler func(*memdPipeline)
type BadRouteHandler func(*memdPipeline, *memdQRequest, *memdResponse)

type Callback func(*memdResponse, error)

type memdPipeline struct {
	lock sync.RWMutex

	queue *memdQueue

	address  string
	conn     memdReadWriteCloser
	isClosed bool
	ioDoneCh chan bool

	opList  memdOpMap

	handleBadRoute BadRouteHandler
	handleDeath    CloseHandler
}

func CreateMemdPipeline(address string) *memdPipeline {
	return &memdPipeline{
		address:  address,
		queue:    createMemdQueue(),
		ioDoneCh: make(chan bool, 1),
	}
}

func (s *memdPipeline) Address() string {
	return s.address
}

func (s *memdPipeline) Hostname() string {
	return strings.Split(s.address, ":")[0]
}

func (s *memdPipeline) IsClosed() bool {
	return s.isClosed
}

func (s *memdPipeline) SetHandlers(badRouteFn BadRouteHandler, deathFn CloseHandler) {
	s.lock.Lock()

	if s.isClosed {
		// We died between authentication and here, immediately notify the deathFn
		s.lock.Unlock()
		deathFn(s)
		return
	}

	s.handleBadRoute = badRouteFn
	s.handleDeath = deathFn
	s.lock.Unlock()
}

func (pipeline *memdPipeline) ExecuteRequest(req *memdQRequest, deadline time.Time) (respOut *memdResponse, errOut error) {
	if req.Callback != nil {
		panic("Tried to synchronously dispatch an operation with an async handler.")
	}

	signal := make(chan bool)

	req.Callback = func(resp *memdResponse, err error) {
		respOut = resp
		errOut = err
		signal <- true
	}

	if !pipeline.queue.QueueRequest(req) {
		return nil, &generalError{"Failed to dispatch operation."}
	}

	timeoutTmr := AcquireTimer(deadline.Sub(time.Now()))
	select {
	case <-signal:
		ReleaseTimer(timeoutTmr, false)
		return
	case <-timeoutTmr.C:
		ReleaseTimer(timeoutTmr, true)
		req.Cancel()
		return nil, &timeoutError{}
	}
}

func (pipeline *memdPipeline) dispatchRequest(req *memdQRequest) error {
	// We do a cursory check of the server to avoid dispatching operations on the network
	//   that have already knowingly been cancelled.  This doesn't guarentee a cancelled
	//   operation from being sent, but it does reduce network IO when possible.
	if req.QueueOwner() != pipeline.queue {
		// Even though we failed to dispatch, this is not actually an error,
		//   we just consume the operation since its already been handled elsewhere
		return nil
	}

	pipeline.opList.Add(req)

	err := pipeline.conn.WritePacket(&req.memdRequest)
	if err != nil {
		logDebugf("Got write error")
		pipeline.opList.Remove(req)
		return err
	}

	return nil
}

func (s *memdPipeline) resolveRequest(resp *memdResponse) {
	opIndex := resp.Opaque

	// Find the request that goes with this response
	req := s.opList.FindAndMaybeRemove(opIndex)

	if req == nil {
		// There is no known request that goes with this response.  Ignore it.
		logDebugf("Received response with no corresponding request.")
		return
	}

	if !req.Persistent || (resp.Magic == ResMagic && resp.Status != StatusSuccess) {
		if !s.queue.UnqueueRequest(req) {
			// While we found a valid request, the request does not appear to be queued
			//   with this server anymore, this probably means that it has been cancelled.
			logDebugf("Received response for cancelled request.")
			return
		}
	}

	if resp.Magic == ResMagic && resp.Status == StatusNotMyVBucket {
		// If possible, lets backchannel our NMV back to the Agent of this memdQueueConn
		//   instance.  This is primarily meant to enhance performance, and allow the
		//   agent to be instantly notified upon a new configuration arriving.  If the
		//   backchannel isn't available, we just Callback with the NMV error.
		logDebugf("Received NMV response.")
		s.lock.RLock()
		badRouteFn := s.handleBadRoute
		s.lock.RUnlock()
		if badRouteFn != nil {
			badRouteFn(s, req, resp)
			return
		}
	}

	// Call the requests callback handler...  Ignore Status field for incoming requests.
	logDebugf("Dispatching response callback.")
	if resp.Magic == ReqMagic || resp.Status == StatusSuccess {
		req.Callback(resp, nil)
	} else {
		req.Callback(nil, &memdError{resp.Status})
	}
}

func (pipeline *memdPipeline) ioLoop() {
	killSig := make(chan bool)

	// Reading
	go func() {
		logDebugf("Reader loop starting...")
		for {
			resp := &memdResponse{}
			err := pipeline.conn.ReadPacket(resp)
			if err != nil {
				logDebugf("Server read error: %v", err)
				killSig <- true
				break
			}

			logDebugf("Got response to resolve.")
			pipeline.resolveRequest(resp)
		}
	}()

	// Writing
	logDebugf("Writer loop starting...")
	for {
		select {
		case req := <-pipeline.queue.reqsCh:
			logDebugf("Got a request to dispatch.")
			err := pipeline.dispatchRequest(req)
			if err != nil {
				// We can assume that the server is not fully drained yet, as the drainer blocks
				//   waiting for the IO goroutines to finish first.
				pipeline.queue.reqsCh <- req

				// We must wait for the receive goroutine to die as well before we can continue.
				<-killSig

				return
			}
		case <-killSig:
			return
		}
	}
}

func (pipeline *memdPipeline) Run() {
	logDebugf("Beginning pipeline runner")

	// Run the IO loop.  This will block until the connection has been closed.
	pipeline.ioLoop()

	// Now we must signal drainers that we are done!
	pipeline.ioDoneCh <- true

	// Signal the creator that we died :(
	pipeline.lock.Lock()
	pipeline.isClosed = true
	deathFn := pipeline.handleDeath
	pipeline.lock.Unlock()
	if deathFn != nil {
		deathFn(pipeline)
	} else {
		pipeline.Drain(nil)
	}
}

func (pipeline *memdPipeline) Close() {
	pipeline.Drain(nil)
}

func (pipeline *memdPipeline) Drain(reqCb drainedReqCallback) {
	// If the user does no pass a drain callback, we handle the requests
	//   by immediately failing them with a network error.
	if reqCb == nil {
		reqCb = func(req *memdQRequest) {
			req.Callback(nil, networkError{})
		}
	}

	// Make sure the connection is closed, which will signal the ioLoop
	//   to stop running and signal on ioDoneCh
	pipeline.conn.Close()

	// Drain the request queue, this will block until the io thread signals
	//   on ioDoneCh, and the queues have been completely emptied
	pipeline.queue.Drain(reqCb, pipeline.ioDoneCh)

	// As a last step, immediately notify all the requests that were
	//   on-the-wire that a network error has occurred.
	pipeline.opList.Drain(func(r *memdQRequest) {
		if pipeline.queue.UnqueueRequest(r) {
			r.Callback(nil, networkError{})
		}
	})
}
<|endoftext|>


<commit_before>// Diffie-Hellman-Merkle key exchange
//
// Step 1:   PrivateKey(p *big.Int) *big.Int
// Step 2:   PublicKey(private, p, *big.Int, g int64) *big.Int
// Step 2.1: NewPair(p, *big.Int, g int64) (private, public *big.Int)
// Step 3:   SecretKey(private1, public2, p *big.Int) *big.Int
//
// Private keys should be generated randomly.

package diffiehellman

import (
	"math/big"
	"testing"
)

type testCase struct {
	g    int64    // prime, generator
	p    *big.Int // prime, modulus
	a, b *big.Int // private keys
	A, B *big.Int // public keys
	s    *big.Int // secret key
}

// WP example
var smallTest = testCase{
	5,
	big.NewInt(23),
	big.NewInt(6), big.NewInt(15),
	big.NewInt(8), big.NewInt(19),
	big.NewInt(2),
}

// 1024 bit example modulus from cryptopp.com wiki, random private keys
var biggerTest = testCase{
	2,
	mph("ab359aa76a6773ed7a93b214db0c25d0160817b8a893c001c761e198a3694509" +
		"ebe87a5313e0349d95083e5412c9fc815bfd61f95ddece43376550fdc624e92f" +
		"f38a415783b97261204e05d65731bba1ccff0e84c8cd2097b75feca1029261ae" +
		"19a389a2e15d2939314b184aef707b82eb94412065181d23e04bf065f4ac413f"),

	mph("2f6afe91cb53ecfa463d45cd800c948f7cb83bb9ddc62a07b5b3fd302d0cdf52" +
		"18ae53ad015a632d137001a3b34239d54715606a292b6cf895b09d7dcf1bdf7a"),

	mph("3651007bfa8a8b1cbaed2ae9326327599249c3bb6e9d8744b7407f3d4732cb8a" +
		"0708d95c382747bad640d444f135e7e599618d11b15b9ef32e7ac7194e547f4b"),

	mph("57d5489e3858cbd8fae75120907d1521f8e935cce2206d285b11762847e2a4c4" +
		"a341a4eea18bdd8b40036c8d0004ffc323d5ae19da55176b08ff6f2d0ac97c65" +
		"357c1f16756a6901ff926c8544c8af0a90ed2705966851f79a115cb77aac66be" +
		"ceff569aadd7f02859539c28d55c08c62a03e45613bc52d205ace0704ea7c610"),

	mph("6b189a36db5ca3ff83b66fb2c226078294c323f4c7cef35c506c237b0db7906d" +
		"64cceb05af15a3603a30fd49834d3a6971d917f520d9a577c159d3b7d2bd8813" +
		"5d19db47a9618340e4a51ec8845dbf5d50a4c6f14d6161def1eeaacecee8018f" +
		"8816113a294959399989b759f4618e35dbffc570ab2a5a74ac59fccef35f684c"),

	mph("64f74babc466f8e56d9b77ce2cc65d65fe1603b931c018b98a2a615d66172590" +
		"803a94ac230db02de4b8ae567497c1844a6f7bd5bed69b95d3137acc1a6462d5" +
		"aeba5b2b83a0e6b8ed4c072e5135a57c87b654ebe04cf128bbff49ee06df33b7" +
		"8615e0067fdc9df22f7673b1e0501fb57598c7bff9ff173ddff47270fbd6f98f"),
}

// must parse hex, short name contrived just to make test data line up with
// tab width 4.
func mph(h string) *big.Int {
	p, ok := new(big.Int).SetString(h, 16)
	if !ok {
		panic("invalid hex: " + h)
	}
	return p
}

var tests = []testCase{
	smallTest,
	biggerTest,
}

var _one = big.NewInt(1)

// test that PrivateKey returns numbers in range, returns different numbers.
func TestPrivateKey(t *testing.T) {
	priv := func(p *big.Int) *big.Int {
		a := PrivateKey(p)
		if a.Cmp(_one) <= 0 || a.Cmp(p) >= 0 {
			t.Fatalf("PrivateKey(%d) = %d, out of range (1, %d)", p, a, p)
		}
		return a
	}
	ms := map[string]bool{}
	mb := map[string]bool{}
	for i := 0; i < 100; i++ {
		ms[priv(smallTest.p).String()] = true
		mb[priv(biggerTest.p).String()] = true
	}
	if len(ms) == 1 {
		t.Fatalf("For prime %d same key generated every time.  "+
			"Want random keys.", smallTest.p)
	}
	if len(mb) < 100 {
		t.Fatal("For prime %d duplicate keys generated.  "+
			"Want unique keys.", biggerTest.p)
	}
}

// test that PublicKey returns known results.
func TestPublicKey(t *testing.T) {
	tp := func(a, A, p *big.Int, g int64) {
		if k := PublicKey(a, p, g); k.Cmp(A) != 0 {
			t.Fatalf("PublicKey(%x,\n%x,\n%d)\n= %x,\nwant %x.",
				a, p, g, k, A)
		}
	}
	for _, test := range tests {
		tp(test.a, test.A, test.p, test.g)
		tp(test.b, test.B, test.p, test.g)
	}
}

// test that SecretKey returns known results.
func TestSecretKeys(t *testing.T) {
	tp := func(a, B, p, s *big.Int) {
		if k := SecretKey(a, B, p); k.Cmp(s) != 0 {
			t.Fatalf("SecretKey(%x,\n%x,\n%x)\n= %x,\nwant %x.",
				a, B, p, k, s)
		}
	}
	for _, test := range tests {
		tp(test.a, test.B, test.p, test.s)
		tp(test.b, test.A, test.p, test.s)
	}
}

// test that NewPair produces working keys
func TestNewPair(t *testing.T) {
	p, g := biggerTest.p, biggerTest.g
	test := func(a, A *big.Int) {
		if a.Cmp(_one) <= 0 || a.Cmp(p) >= 0 {
			t.Fatalf("NewPair(%d, %d) private key = %d, out of range (1, %d)",
				p, g, a, p)
		}
		if A.Cmp(_one) <= 0 || A.Cmp(p) >= 0 {
			t.Fatalf("NewPair(%d, %d) public key = %d, out of range (1, %d)",
				p, g, A, p)
		}
	}
	a, A := NewPair(p, g)
	test(a, A)
	for i := 0; i < 20; i++ {
		b, B := NewPair(p, g)
		test(b, B)
		sa := SecretKey(a, B, p)
		sb := SecretKey(b, A, p)
		if sa.Cmp(sb) != 0 {
			t.Fatalf("NewPair() produced non-working keys.")
		}
		a, A = b, B
	}
}

func BenchmarkPrivateKey(b *testing.B) {
	for i := 0; i < b.N; i++ {
		PrivateKey(biggerTest.p)
	}
}

func BenchmarkPublicKey(b *testing.B) {
	for i := 0; i < b.N; i++ {
		PublicKey(biggerTest.a, biggerTest.p, biggerTest.g)
	}
}

func BenchmarkNewPair(b *testing.B) {
	for i := 0; i < b.N; i++ {
		NewPair(biggerTest.p, biggerTest.g)
	}
}

func BenchmarkSecretKey(b *testing.B) {
	for i := 0; i < b.N; i++ {
		SecretKey(biggerTest.a, biggerTest.B, biggerTest.p)
	}
}
<commit_msg>diffie-hellman: remove extra commas in expected signatures<commit_after>// Diffie-Hellman-Merkle key exchange
//
// Step 1:   PrivateKey(p *big.Int) *big.Int
// Step 2:   PublicKey(private, p *big.Int, g int64) *big.Int
// Step 2.1: NewPair(p *big.Int, g int64) (private, public *big.Int)
// Step 3:   SecretKey(private1, public2, p *big.Int) *big.Int
//
// Private keys should be generated randomly.

package diffiehellman

import (
	"math/big"
	"testing"
)

type testCase struct {
	g    int64    // prime, generator
	p    *big.Int // prime, modulus
	a, b *big.Int // private keys
	A, B *big.Int // public keys
	s    *big.Int // secret key
}

// WP example
var smallTest = testCase{
	5,
	big.NewInt(23),
	big.NewInt(6), big.NewInt(15),
	big.NewInt(8), big.NewInt(19),
	big.NewInt(2),
}

// 1024 bit example modulus from cryptopp.com wiki, random private keys
var biggerTest = testCase{
	2,
	mph("ab359aa76a6773ed7a93b214db0c25d0160817b8a893c001c761e198a3694509" +
		"ebe87a5313e0349d95083e5412c9fc815bfd61f95ddece43376550fdc624e92f" +
		"f38a415783b97261204e05d65731bba1ccff0e84c8cd2097b75feca1029261ae" +
		"19a389a2e15d2939314b184aef707b82eb94412065181d23e04bf065f4ac413f"),

	mph("2f6afe91cb53ecfa463d45cd800c948f7cb83bb9ddc62a07b5b3fd302d0cdf52" +
		"18ae53ad015a632d137001a3b34239d54715606a292b6cf895b09d7dcf1bdf7a"),

	mph("3651007bfa8a8b1cbaed2ae9326327599249c3bb6e9d8744b7407f3d4732cb8a" +
		"0708d95c382747bad640d444f135e7e599618d11b15b9ef32e7ac7194e547f4b"),

	mph("57d5489e3858cbd8fae75120907d1521f8e935cce2206d285b11762847e2a4c4" +
		"a341a4eea18bdd8b40036c8d0004ffc323d5ae19da55176b08ff6f2d0ac97c65" +
		"357c1f16756a6901ff926c8544c8af0a90ed2705966851f79a115cb77aac66be" +
		"ceff569aadd7f02859539c28d55c08c62a03e45613bc52d205ace0704ea7c610"),

	mph("6b189a36db5ca3ff83b66fb2c226078294c323f4c7cef35c506c237b0db7906d" +
		"64cceb05af15a3603a30fd49834d3a6971d917f520d9a577c159d3b7d2bd8813" +
		"5d19db47a9618340e4a51ec8845dbf5d50a4c6f14d6161def1eeaacecee8018f" +
		"8816113a294959399989b759f4618e35dbffc570ab2a5a74ac59fccef35f684c"),

	mph("64f74babc466f8e56d9b77ce2cc65d65fe1603b931c018b98a2a615d66172590" +
		"803a94ac230db02de4b8ae567497c1844a6f7bd5bed69b95d3137acc1a6462d5" +
		"aeba5b2b83a0e6b8ed4c072e5135a57c87b654ebe04cf128bbff49ee06df33b7" +
		"8615e0067fdc9df22f7673b1e0501fb57598c7bff9ff173ddff47270fbd6f98f"),
}

// must parse hex, short name contrived just to make test data line up with
// tab width 4.
func mph(h string) *big.Int {
	p, ok := new(big.Int).SetString(h, 16)
	if !ok {
		panic("invalid hex: " + h)
	}
	return p
}

var tests = []testCase{
	smallTest,
	biggerTest,
}

var _one = big.NewInt(1)

// test that PrivateKey returns numbers in range, returns different numbers.
func TestPrivateKey(t *testing.T) {
	priv := func(p *big.Int) *big.Int {
		a := PrivateKey(p)
		if a.Cmp(_one) <= 0 || a.Cmp(p) >= 0 {
			t.Fatalf("PrivateKey(%d) = %d, out of range (1, %d)", p, a, p)
		}
		return a
	}
	ms := map[string]bool{}
	mb := map[string]bool{}
	for i := 0; i < 100; i++ {
		ms[priv(smallTest.p).String()] = true
		mb[priv(biggerTest.p).String()] = true
	}
	if len(ms) == 1 {
		t.Fatalf("For prime %d same key generated every time.  "+
			"Want random keys.", smallTest.p)
	}
	if len(mb) < 100 {
		t.Fatal("For prime %d duplicate keys generated.  "+
			"Want unique keys.", biggerTest.p)
	}
}

// test that PublicKey returns known results.
func TestPublicKey(t *testing.T) {
	tp := func(a, A, p *big.Int, g int64) {
		if k := PublicKey(a, p, g); k.Cmp(A) != 0 {
			t.Fatalf("PublicKey(%x,\n%x,\n%d)\n= %x,\nwant %x.",
				a, p, g, k, A)
		}
	}
	for _, test := range tests {
		tp(test.a, test.A, test.p, test.g)
		tp(test.b, test.B, test.p, test.g)
	}
}

// test that SecretKey returns known results.
func TestSecretKeys(t *testing.T) {
	tp := func(a, B, p, s *big.Int) {
		if k := SecretKey(a, B, p); k.Cmp(s) != 0 {
			t.Fatalf("SecretKey(%x,\n%x,\n%x)\n= %x,\nwant %x.",
				a, B, p, k, s)
		}
	}
	for _, test := range tests {
		tp(test.a, test.B, test.p, test.s)
		tp(test.b, test.A, test.p, test.s)
	}
}

// test that NewPair produces working keys
func TestNewPair(t *testing.T) {
	p, g := biggerTest.p, biggerTest.g
	test := func(a, A *big.Int) {
		if a.Cmp(_one) <= 0 || a.Cmp(p) >= 0 {
			t.Fatalf("NewPair(%d, %d) private key = %d, out of range (1, %d)",
				p, g, a, p)
		}
		if A.Cmp(_one) <= 0 || A.Cmp(p) >= 0 {
			t.Fatalf("NewPair(%d, %d) public key = %d, out of range (1, %d)",
				p, g, A, p)
		}
	}
	a, A := NewPair(p, g)
	test(a, A)
	for i := 0; i < 20; i++ {
		b, B := NewPair(p, g)
		test(b, B)
		sa := SecretKey(a, B, p)
		sb := SecretKey(b, A, p)
		if sa.Cmp(sb) != 0 {
			t.Fatalf("NewPair() produced non-working keys.")
		}
		a, A = b, B
	}
}

func BenchmarkPrivateKey(b *testing.B) {
	for i := 0; i < b.N; i++ {
		PrivateKey(biggerTest.p)
	}
}

func BenchmarkPublicKey(b *testing.B) {
	for i := 0; i < b.N; i++ {
		PublicKey(biggerTest.a, biggerTest.p, biggerTest.g)
	}
}

func BenchmarkNewPair(b *testing.B) {
	for i := 0; i < b.N; i++ {
		NewPair(biggerTest.p, biggerTest.g)
	}
}

func BenchmarkSecretKey(b *testing.B) {
	for i := 0; i < b.N; i++ {
		SecretKey(biggerTest.a, biggerTest.B, biggerTest.p)
	}
}
<|endoftext|>


<commit_before>package gfile

import(
	"io/ioutil"
	"testing"
	"github.com/gogf/gf/g/test/gtest"

)

func TestGetContents(t *testing.T) {
	gtest.Case(t,func(){
		var(
			filepaths string= "./testfile/havefile1/GetContents.txt"
		)

		gtest.Assert(GetContents(filepaths),"abcdefghijkmln")

	})
}

func TestGetBinContents(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths1 string="./testfile/havefile1/GetContents.txt" //存在文件
			filepaths2 string="./testfile/havefile1/GetContents_no.txt"  //不存大文件
			readcontent []byte
		)
		readcontent=GetBinContents(filepaths1)
		gtest.Assert(readcontent,[]byte("abcdefghijkmln"))

		readcontent=GetBinContents(filepaths2)
		//文件不存在时@todo:等断言功能优化后，再来修改这里
		//if readcontent!=nil{
		//	t.Error("文件应不存在")
		//}
		gtest.Assert(GetBinContents(filepaths2),nil)

	})
}

//暂时不知道用途 @todo:继续添加
func TestTruncate(t *testing.T) {
	gtest.Case(t , func() {

	})
}

func TestPutContents(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths string="./testfile/havefile1/PutContents.txt"
			err error
			readcontent []byte
		)

		err=PutContents(filepaths,"test!")
		gtest.Assert(err,nil)

		//==================判断是否真正写入
		readcontent, err=ioutil.ReadFile(filepaths)
		gtest.Assert(err,nil)
		gtest.Assert(string(readcontent),"test!")
	})
}

func TestPutContentsAppend(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths string="./testfile/havefile1/PutContents.txt"
			err error
			readcontent []byte
		)

		err=PutContentsAppend(filepaths,"hello")
		gtest.Assert(err,nil)

		//==================判断是否真正写入
		readcontent, err=ioutil.ReadFile(filepaths)
		gtest.Assert(err,nil)
		gtest.Assert(string(readcontent),"test!hello")
	})

}

func TestPutBinContents(t *testing.T){
	gtest.Case(t , func() {
		var(
			filepaths string="./testfile/havefile1/PutContents.txt"
			err error
			readcontent []byte
		)

		err=PutBinContents(filepaths,[]byte("test!!"))
		gtest.Assert(err,nil)

		//==================判断是否真正写入
		readcontent, err=ioutil.ReadFile(filepaths)
		gtest.Assert(err,nil)
		gtest.Assert(string(readcontent),"test!!")
	})
}

func TestPutBinContentsAppend(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths string="./testfile/havefile1/PutContents.txt"  //原文件内容: yy
			err error
			readcontent []byte
		)

		err=PutBinContentsAppend(filepaths,[]byte("word"))
		gtest.Assert(err,nil)

		//==================判断是否真正写入
		readcontent, err=ioutil.ReadFile(filepaths)
		gtest.Assert(err,nil)
		gtest.Assert(string(readcontent),"test!!word")
	})
}

func TestGetBinContentsByTwoOffsetsByPath(t *testing.T) {
	gtest.Case(t, func() {
		var (
			filepaths   string = "./testfile/havefile1/GetContents.txt" //原文件内容: abcdefghijk
			readcontent []byte
		)

		readcontent = GetBinContentsByTwoOffsetsByPath(filepaths, 2, 5)

		gtest.Assert(string(readcontent), "cde")
	})

}

<commit_msg>添加内容测试<commit_after>package gfile

import(
	"io/ioutil"
	"testing"
	"github.com/gogf/gf/g/test/gtest"

)

func TestGetContents(t *testing.T) {
	gtest.Case(t,func(){
		var(
			filepaths string= "./testfile/havefile1/GetContents.txt"
		)

		gtest.Assert(GetContents(filepaths),"abcdefghijkmln")

	})
}

func TestGetBinContents(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths1 string="./testfile/havefile1/GetContents.txt" //存在文件
			filepaths2 string="./testfile/havefile1/GetContents_no.txt"  //不存大文件
			readcontent []byte
		)
		readcontent=GetBinContents(filepaths1)
		gtest.Assert(readcontent,[]byte("abcdefghijkmln"))

		readcontent=GetBinContents(filepaths2)
		//文件不存在时@todo:等断言功能优化后，再来修改这里
		//if readcontent!=nil{
		//	t.Error("文件应不存在")
		//}
		gtest.Assert(GetBinContents(filepaths2),nil)

	})
}

//截断文件为指定的大小
func TestTruncate(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths1 string="./testfile/havefile1/GetContents.txt" //存在文件
			err error
		)
		err=Truncate(filepaths1,200)
		gtest.Assert(err,nil)

	})
}

func TestPutContents(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths string="./testfile/havefile1/PutContents.txt"
			err error
			readcontent []byte
		)

		err=PutContents(filepaths,"test!")
		gtest.Assert(err,nil)

		//==================判断是否真正写入
		readcontent, err=ioutil.ReadFile(filepaths)
		gtest.Assert(err,nil)
		gtest.Assert(string(readcontent),"test!")
	})
}

func TestPutContentsAppend(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths string="./testfile/havefile1/PutContents.txt"
			err error
			readcontent []byte
		)

		err=PutContentsAppend(filepaths,"hello")
		gtest.Assert(err,nil)

		//==================判断是否真正写入
		readcontent, err=ioutil.ReadFile(filepaths)
		gtest.Assert(err,nil)
		gtest.Assert(string(readcontent),"test!hello")
	})

}

func TestPutBinContents(t *testing.T){
	gtest.Case(t , func() {
		var(
			filepaths string="./testfile/havefile1/PutContents.txt"
			err error
			readcontent []byte
		)

		err=PutBinContents(filepaths,[]byte("test!!"))
		gtest.Assert(err,nil)

		//==================判断是否真正写入
		readcontent, err=ioutil.ReadFile(filepaths)
		gtest.Assert(err,nil)
		gtest.Assert(string(readcontent),"test!!")
	})
}

func TestPutBinContentsAppend(t *testing.T) {
	gtest.Case(t , func() {
		var(
			filepaths string="./testfile/havefile1/PutContents.txt"  //原文件内容: yy
			err error
			readcontent []byte
		)

		err=PutBinContentsAppend(filepaths,[]byte("word"))
		gtest.Assert(err,nil)

		//==================判断是否真正写入
		readcontent, err=ioutil.ReadFile(filepaths)
		gtest.Assert(err,nil)
		gtest.Assert(string(readcontent),"test!!word")
	})
}

func TestGetBinContentsByTwoOffsetsByPath(t *testing.T) {
	gtest.Case(t, func() {
		var (
			filepaths   string = "./testfile/havefile1/GetContents.txt" //原文件内容: abcdefghijk
			readcontent []byte
		)

		readcontent = GetBinContentsByTwoOffsetsByPath(filepaths, 2, 5)

		gtest.Assert(string(readcontent), "cde")
	})

}

func TestGetNextCharOffsetByPath(t *testing.T) {
	gtest.Case(t, func() {
		var (
			filepaths   string = "./testfile/havefile1/GetContents.txt" //原文件内容: abcdefghijk
			localindex int64

		)

		localindex = GetNextCharOffsetByPath(filepaths,'d', 1)
		gtest.Assert(localindex, 3)
	})
}

<|endoftext|>


<commit_before>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// deploy command tests
// Author: bratseth

package cmd

import (
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/vespa-engine/vespa/client/go/mock"
	"github.com/vespa-engine/vespa/client/go/vespa"
)

func TestPrepareZip(t *testing.T) {
	assertPrepare("testdata/applications/withTarget/target/application.zip",
		[]string{"prepare", "testdata/applications/withTarget/target/application.zip"}, t)
}

func TestActivateZip(t *testing.T) {
	assertActivate("testdata/applications/withTarget/target/application.zip",
		[]string{"activate", "testdata/applications/withTarget/target/application.zip"}, t)
}

func TestDeployZip(t *testing.T) {
	assertDeploy("testdata/applications/withTarget/target/application.zip",
		[]string{"deploy", "testdata/applications/withTarget/target/application.zip"}, t)
}

func TestDeployZipWithURLTargetArgument(t *testing.T) {
	applicationPackage := "testdata/applications/withTarget/target/application.zip"
	arguments := []string{"deploy", "testdata/applications/withTarget/target/application.zip", "-t", "http://target:19071"}

	client := &mock.HTTPClient{}
	cli, stdout, _ := newTestCLI(t)
	cli.httpClient = client
	assert.Nil(t, cli.Run(arguments...))
	assert.Equal(t,
		"\nSuccess: Deployed "+applicationPackage+"\n",
		stdout.String())
	assertDeployRequestMade("http://target:19071", client, t)
}

func TestDeployZipWitLocalTargetArgument(t *testing.T) {
	assertDeploy("testdata/applications/withTarget/target/application.zip",
		[]string{"deploy", "testdata/applications/withTarget/target/application.zip", "-t", "local"}, t)
}

func TestDeploySourceDirectory(t *testing.T) {
	assertDeploy("testdata/applications/withSource/src/main/application",
		[]string{"deploy", "testdata/applications/withSource/src/main/application"}, t)
}

func TestDeployApplicationDirectoryWithSource(t *testing.T) {
	assertDeploy("testdata/applications/withSource/src/main/application",
		[]string{"deploy", "testdata/applications/withSource"}, t)
}

func TestDeployApplicationDirectoryWithPomAndTarget(t *testing.T) {
	assertDeploy("testdata/applications/withTarget/target/application.zip",
		[]string{"deploy", "testdata/applications/withTarget"}, t)
}

func TestDeployApplicationDirectoryWithPomAndEmptyTarget(t *testing.T) {
	cli, _, stderr := newTestCLI(t)
	assert.NotNil(t, cli.Run("deploy", "testdata/applications/withEmptyTarget"))
	assert.Equal(t,
		"Error: found pom.xml, but target/application.zip does not exist: run 'mvn package' first\n",
		stderr.String())
}

func TestDeployApplicationPackageErrorWithUnexpectedNonJson(t *testing.T) {
	assertApplicationPackageError(t, "deploy", 401,
		"Raw text error",
		"Raw text error")
}

func TestDeployApplicationPackageErrorWithUnexpectedJson(t *testing.T) {
	assertApplicationPackageError(t, "deploy", 401,
		`{
    "some-unexpected-json": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"
}`,
		`{"some-unexpected-json": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"}`)
}

func TestDeployApplicationPackageErrorWithExpectedFormat(t *testing.T) {
	assertApplicationPackageError(t, "deploy", 400,
		"Invalid XML, error in services.xml:\nelement \"nosuch\" not allowed here",
		`{
         "error-code": "INVALID_APPLICATION_PACKAGE",
         "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"
     }`)
}

func TestPrepareApplicationPackageErrorWithExpectedFormat(t *testing.T) {
	assertApplicationPackageError(t, "prepare", 400,
		"Invalid XML, error in services.xml:\nelement \"nosuch\" not allowed here",
		`{
         "error-code": "INVALID_APPLICATION_PACKAGE",
         "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"
     }`)
}

func TestDeployError(t *testing.T) {
	assertDeployServerError(t, 501, "Deploy service error")
}

func assertDeploy(applicationPackage string, arguments []string, t *testing.T) {
	t.Helper()
	cli, stdout, _ := newTestCLI(t)
	client := &mock.HTTPClient{}
	cli.httpClient = client
	assert.Nil(t, cli.Run(arguments...))
	assert.Equal(t,
		"\nSuccess: Deployed "+applicationPackage+"\n",
		stdout.String())
	assertDeployRequestMade("http://127.0.0.1:19071", client, t)
}

func assertPrepare(applicationPackage string, arguments []string, t *testing.T) {
	t.Helper()
	client := &mock.HTTPClient{}
	client.NextResponseString(200, `{"session-id":"42","message":"Session 42 for tenant 'default' prepared.","log":[{"level":"WARNING","message":"Warning message 1","time": 1430134091319}]}`)
	cli, stdout, _ := newTestCLI(t)
	cli.httpClient = client
	assert.Nil(t, cli.Run(arguments...))
	assert.Equal(t,
		"Success: Prepared "+applicationPackage+" with session 42\n",
		stdout.String())

	assertPackageUpload(0, "http://127.0.0.1:19071/application/v2/tenant/default/session", client, t)
	sessionURL := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/prepared"
	assert.Equal(t, sessionURL, client.Requests[1].URL.String())
	assert.Equal(t, "PUT", client.Requests[1].Method)
}

func assertActivate(applicationPackage string, arguments []string, t *testing.T) {
	t.Helper()
	client := &mock.HTTPClient{}
	cli, stdout, _ := newTestCLI(t)
	cli.httpClient = client
	if err := cli.config.writeSessionID(vespa.DefaultApplication, 42); err != nil {
		t.Fatal(err)
	}
	assert.Nil(t, cli.Run(arguments...))
	assert.Equal(t,
		"Success: Activated "+applicationPackage+" with session 42\n",
		stdout.String())
	url := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/active"
	assert.Equal(t, url, client.LastRequest.URL.String())
	assert.Equal(t, "PUT", client.LastRequest.Method)
}

func assertPackageUpload(requestNumber int, url string, client *mock.HTTPClient, t *testing.T) {
	t.Helper()
	req := client.LastRequest
	if requestNumber >= 0 {
		req = client.Requests[requestNumber]
	}
	assert.Equal(t, url, req.URL.String())
	assert.Equal(t, "application/zip", req.Header.Get("Content-Type"))
	assert.Equal(t, "POST", req.Method)
	var body = req.Body
	assert.NotNil(t, body)
	buf := make([]byte, 7) // Just check the first few bytes
	body.Read(buf)
	assert.Equal(t, "PK\x03\x04\x14\x00\b", string(buf))
}

func assertDeployRequestMade(target string, client *mock.HTTPClient, t *testing.T) {
	t.Helper()
	assertPackageUpload(-1, target+"/application/v2/tenant/default/prepareandactivate", client, t)
}

func assertApplicationPackageError(t *testing.T, cmd string, status int, expectedMessage string, returnBody string) {
	t.Helper()
	client := &mock.HTTPClient{}
	client.NextResponseString(status, returnBody)
	cli, _, stderr := newTestCLI(t)
	cli.httpClient = client
	assert.NotNil(t, cli.Run(cmd, "testdata/applications/withTarget/target/application.zip"))
	assert.Equal(t,
		"Error: invalid application package (Status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n",
		stderr.String())
}

func assertDeployServerError(t *testing.T, status int, errorMessage string) {
	t.Helper()
	client := &mock.HTTPClient{}
	client.NextResponseString(status, errorMessage)
	cli, _, stderr := newTestCLI(t)
	cli.httpClient = client
	assert.NotNil(t, cli.Run("deploy", "testdata/applications/withTarget/target/application.zip"))
	assert.Equal(t,
		"Error: error from deploy api at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n",
		stderr.String())
}
<commit_msg>Only return session-id from upload, then full prepared output after prepare<commit_after>// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
// deploy command tests
// Author: bratseth

package cmd

import (
	"strconv"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/vespa-engine/vespa/client/go/mock"
	"github.com/vespa-engine/vespa/client/go/vespa"
)

func TestPrepareZip(t *testing.T) {
	assertPrepare("testdata/applications/withTarget/target/application.zip",
		[]string{"prepare", "testdata/applications/withTarget/target/application.zip"}, t)
}

func TestActivateZip(t *testing.T) {
	assertActivate("testdata/applications/withTarget/target/application.zip",
		[]string{"activate", "testdata/applications/withTarget/target/application.zip"}, t)
}

func TestDeployZip(t *testing.T) {
	assertDeploy("testdata/applications/withTarget/target/application.zip",
		[]string{"deploy", "testdata/applications/withTarget/target/application.zip"}, t)
}

func TestDeployZipWithURLTargetArgument(t *testing.T) {
	applicationPackage := "testdata/applications/withTarget/target/application.zip"
	arguments := []string{"deploy", "testdata/applications/withTarget/target/application.zip", "-t", "http://target:19071"}

	client := &mock.HTTPClient{}
	cli, stdout, _ := newTestCLI(t)
	cli.httpClient = client
	assert.Nil(t, cli.Run(arguments...))
	assert.Equal(t,
		"\nSuccess: Deployed "+applicationPackage+"\n",
		stdout.String())
	assertDeployRequestMade("http://target:19071", client, t)
}

func TestDeployZipWitLocalTargetArgument(t *testing.T) {
	assertDeploy("testdata/applications/withTarget/target/application.zip",
		[]string{"deploy", "testdata/applications/withTarget/target/application.zip", "-t", "local"}, t)
}

func TestDeploySourceDirectory(t *testing.T) {
	assertDeploy("testdata/applications/withSource/src/main/application",
		[]string{"deploy", "testdata/applications/withSource/src/main/application"}, t)
}

func TestDeployApplicationDirectoryWithSource(t *testing.T) {
	assertDeploy("testdata/applications/withSource/src/main/application",
		[]string{"deploy", "testdata/applications/withSource"}, t)
}

func TestDeployApplicationDirectoryWithPomAndTarget(t *testing.T) {
	assertDeploy("testdata/applications/withTarget/target/application.zip",
		[]string{"deploy", "testdata/applications/withTarget"}, t)
}

func TestDeployApplicationDirectoryWithPomAndEmptyTarget(t *testing.T) {
	cli, _, stderr := newTestCLI(t)
	assert.NotNil(t, cli.Run("deploy", "testdata/applications/withEmptyTarget"))
	assert.Equal(t,
		"Error: found pom.xml, but target/application.zip does not exist: run 'mvn package' first\n",
		stderr.String())
}

func TestDeployApplicationPackageErrorWithUnexpectedNonJson(t *testing.T) {
	assertApplicationPackageError(t, "deploy", 401,
		"Raw text error",
		"Raw text error")
}

func TestDeployApplicationPackageErrorWithUnexpectedJson(t *testing.T) {
	assertApplicationPackageError(t, "deploy", 401,
		`{
    "some-unexpected-json": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"
}`,
		`{"some-unexpected-json": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"}`)
}

func TestDeployApplicationPackageErrorWithExpectedFormat(t *testing.T) {
	assertApplicationPackageError(t, "deploy", 400,
		"Invalid XML, error in services.xml:\nelement \"nosuch\" not allowed here",
		`{
         "error-code": "INVALID_APPLICATION_PACKAGE",
         "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"
     }`)
}

func TestPrepareApplicationPackageErrorWithExpectedFormat(t *testing.T) {
	assertApplicationPackageError(t, "prepare", 400,
		"Invalid XML, error in services.xml:\nelement \"nosuch\" not allowed here",
		`{
         "error-code": "INVALID_APPLICATION_PACKAGE",
         "message": "Invalid XML, error in services.xml: element \"nosuch\" not allowed here"
     }`)
}

func TestDeployError(t *testing.T) {
	assertDeployServerError(t, 501, "Deploy service error")
}

func assertDeploy(applicationPackage string, arguments []string, t *testing.T) {
	t.Helper()
	cli, stdout, _ := newTestCLI(t)
	client := &mock.HTTPClient{}
	cli.httpClient = client
	assert.Nil(t, cli.Run(arguments...))
	assert.Equal(t,
		"\nSuccess: Deployed "+applicationPackage+"\n",
		stdout.String())
	assertDeployRequestMade("http://127.0.0.1:19071", client, t)
}

func assertPrepare(applicationPackage string, arguments []string, t *testing.T) {
	t.Helper()
	client := &mock.HTTPClient{}
	client.NextResponseString(200, `{"session-id":"42"}`)
	client.NextResponseString(200, `{"session-id":"42","message":"Session 42 for tenant 'default' prepared.","log":[{"level":"WARNING","message":"Warning message 1","time": 1430134091319}]}`)
	cli, stdout, _ := newTestCLI(t)
	cli.httpClient = client
	assert.Nil(t, cli.Run(arguments...))
	assert.Equal(t,
		"Success: Prepared "+applicationPackage+" with session 42\n",
		stdout.String())

	assertPackageUpload(0, "http://127.0.0.1:19071/application/v2/tenant/default/session", client, t)
	sessionURL := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/prepared"
	assert.Equal(t, sessionURL, client.Requests[1].URL.String())
	assert.Equal(t, "PUT", client.Requests[1].Method)
}

func assertActivate(applicationPackage string, arguments []string, t *testing.T) {
	t.Helper()
	client := &mock.HTTPClient{}
	cli, stdout, _ := newTestCLI(t)
	cli.httpClient = client
	if err := cli.config.writeSessionID(vespa.DefaultApplication, 42); err != nil {
		t.Fatal(err)
	}
	assert.Nil(t, cli.Run(arguments...))
	assert.Equal(t,
		"Success: Activated "+applicationPackage+" with session 42\n",
		stdout.String())
	url := "http://127.0.0.1:19071/application/v2/tenant/default/session/42/active"
	assert.Equal(t, url, client.LastRequest.URL.String())
	assert.Equal(t, "PUT", client.LastRequest.Method)
}

func assertPackageUpload(requestNumber int, url string, client *mock.HTTPClient, t *testing.T) {
	t.Helper()
	req := client.LastRequest
	if requestNumber >= 0 {
		req = client.Requests[requestNumber]
	}
	assert.Equal(t, url, req.URL.String())
	assert.Equal(t, "application/zip", req.Header.Get("Content-Type"))
	assert.Equal(t, "POST", req.Method)
	var body = req.Body
	assert.NotNil(t, body)
	buf := make([]byte, 7) // Just check the first few bytes
	body.Read(buf)
	assert.Equal(t, "PK\x03\x04\x14\x00\b", string(buf))
}

func assertDeployRequestMade(target string, client *mock.HTTPClient, t *testing.T) {
	t.Helper()
	assertPackageUpload(-1, target+"/application/v2/tenant/default/prepareandactivate", client, t)
}

func assertApplicationPackageError(t *testing.T, cmd string, status int, expectedMessage string, returnBody string) {
	t.Helper()
	client := &mock.HTTPClient{}
	client.NextResponseString(status, returnBody)
	cli, _, stderr := newTestCLI(t)
	cli.httpClient = client
	assert.NotNil(t, cli.Run(cmd, "testdata/applications/withTarget/target/application.zip"))
	assert.Equal(t,
		"Error: invalid application package (Status "+strconv.Itoa(status)+")\n"+expectedMessage+"\n",
		stderr.String())
}

func assertDeployServerError(t *testing.T, status int, errorMessage string) {
	t.Helper()
	client := &mock.HTTPClient{}
	client.NextResponseString(status, errorMessage)
	cli, _, stderr := newTestCLI(t)
	cli.httpClient = client
	assert.NotNil(t, cli.Run("deploy", "testdata/applications/withTarget/target/application.zip"))
	assert.Equal(t,
		"Error: error from deploy api at 127.0.0.1:19071 (Status "+strconv.Itoa(status)+"):\n"+errorMessage+"\n",
		stderr.String())
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package install installs the experimental API group, making it available as
// an option to all of the API encoding/decoding machinery.
package install

import (
	"k8s.io/apimachinery/pkg/apimachinery/announced"
	"k8s.io/apimachinery/pkg/apimachinery/registered"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/authentication"
	"k8s.io/kubernetes/pkg/apis/authentication/v1beta1"
)

func init() {
	Install(api.GroupFactoryRegistry, api.Registry, api.Scheme)
}

// Install registers the API group and adds types to a scheme
func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
	if err := announced.NewGroupMetaFactory(
		&announced.GroupMetaFactoryArgs{
			GroupName:                  authentication.GroupName,
			VersionPreferenceOrder:     []string{v1beta1.SchemeGroupVersion.Version},
			ImportPrefix:               "k8s.io/kubernetes/pkg/apis/authentication",
			RootScopedKinds:            sets.NewString("TokenReview"),
			AddInternalObjectsToScheme: authentication.AddToScheme,
		},
		announced.VersionToSchemeFunc{
			v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme,
		},
	).Announce(groupFactoryRegistry).RegisterAndEnable(registry, scheme); err != nil {
		panic(err)
	}
}
<commit_msg>install authentication.k8s.io/v1, add tests<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package install installs the experimental API group, making it available as
// an option to all of the API encoding/decoding machinery.
package install

import (
	"k8s.io/apimachinery/pkg/apimachinery/announced"
	"k8s.io/apimachinery/pkg/apimachinery/registered"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/authentication"
	"k8s.io/kubernetes/pkg/apis/authentication/v1"
	"k8s.io/kubernetes/pkg/apis/authentication/v1beta1"
)

func init() {
	Install(api.GroupFactoryRegistry, api.Registry, api.Scheme)
}

// Install registers the API group and adds types to a scheme
func Install(groupFactoryRegistry announced.APIGroupFactoryRegistry, registry *registered.APIRegistrationManager, scheme *runtime.Scheme) {
	if err := announced.NewGroupMetaFactory(
		&announced.GroupMetaFactoryArgs{
			GroupName:                  authentication.GroupName,
			VersionPreferenceOrder:     []string{v1.SchemeGroupVersion.Version, v1beta1.SchemeGroupVersion.Version},
			ImportPrefix:               "k8s.io/kubernetes/pkg/apis/authentication",
			RootScopedKinds:            sets.NewString("TokenReview"),
			AddInternalObjectsToScheme: authentication.AddToScheme,
		},
		announced.VersionToSchemeFunc{
			v1beta1.SchemeGroupVersion.Version: v1beta1.AddToScheme,
			v1.SchemeGroupVersion.Version:      v1.AddToScheme,
		},
	).Announce(groupFactoryRegistry).RegisterAndEnable(registry, scheme); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

const (
	// Healthy means that the device is healthy
	Healthy = "Healthy"
	// UnHealthy means that the device is unhealthy
	Unhealthy = "Unhealthy"

	// Current version of the API supported by kubelet
	Version = "v1beta1"
	// DevicePluginPath is the folder the Device Plugin is expecting sockets to be on
	// Only privileged pods have access to this path
	// Note: Placeholder until we find a "standard path"
	DevicePluginPath = "/var/lib/kubelet/device-plugins/"
	// KubeletSocket is the path of the Kubelet registry socket
	KubeletSocket = DevicePluginPath + "kubelet.sock"
	// Timeout duration in secs for PreStartContainer RPC
	KubeletPreStartContainerRPCTimeoutInSecs = 30
)

var SupportedVersions = [...]string{"v1beta1"}
<commit_msg>fix golint failures<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

const (
	// Healthy means that the device is healthy
	Healthy = "Healthy"
	// Unhealthy means that the device is unhealthy
	Unhealthy = "Unhealthy"

	// Version means current version of the API supported by kubelet
	Version = "v1beta1"
	// DevicePluginPath is the folder the Device Plugin is expecting sockets to be on
	// Only privileged pods have access to this path
	// Note: Placeholder until we find a "standard path"
	DevicePluginPath = "/var/lib/kubelet/device-plugins/"
	// KubeletSocket is the path of the Kubelet registry socket
	KubeletSocket = DevicePluginPath + "kubelet.sock"
	// KubeletPreStartContainerRPCTimeoutInSecs is the timeout duration in secs for PreStartContainer RPC
	KubeletPreStartContainerRPCTimeoutInSecs = 30
)

// SupportedVersions provides a list of supported version
var SupportedVersions = [...]string{"v1beta1"}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package middleware defines shared middleware for handlers.
package middleware

import (
	"context"
	"net/http"
	"time"

	"github.com/google/exposure-notifications-verification-server/pkg/controller"
	"github.com/google/exposure-notifications-verification-server/pkg/database"
	"github.com/google/exposure-notifications-verification-server/pkg/render"

	"github.com/google/exposure-notifications-server/pkg/logging"

	"firebase.google.com/go/auth"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
)

// RequireVerified requires a user to have verified their login email.
// MUST first run RequireAuth to populate user.
func RequireVerified(ctx context.Context, client *auth.Client, db *database.Database, h *render.Renderer, ttl time.Duration) mux.MiddlewareFunc {
	logger := logging.FromContext(ctx).Named("middleware.RequireVerified")

	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			ctx := r.Context()

			session := controller.SessionFromContext(ctx)
			if session == nil {
				logger.Errorw("session does not exist")
				controller.MissingSession(w, r, h)
				return
			}

			flash := controller.Flash(session)

			user := controller.UserFromContext(ctx)
			if user == nil {
				logger.Debugw("no user found when checking email verification")
				flash.Error("Log in first to verify email.")
				controller.ClearSessionFirebaseCookie(session)
				controller.Unauthorized(w, r, h)
				return
			}

			m := controller.TemplateMapFromContext(ctx)

			fbUser, err := client.GetUserByEmail(ctx, user.Email)
			if err != nil {
				delete(m, "currentUser") // Remove user from the template map.
				logger.Debugw("firebase user does not exist")
				flash.Error("That user does not exist.")
				controller.ClearSessionFirebaseCookie(session)
				controller.Unauthorized(w, r, h)
				return
			}

			realm := controller.RealmFromContext(ctx)
			if NeedsEmailVerification(session, realm, fbUser) {
				logger.Debugw("user email not verified")
				http.Redirect(w, r, "/login/verify-email", http.StatusSeeOther)
				return
			}

			next.ServeHTTP(w, r)
		})
	}
}

func NeedsEmailVerification(session *sessions.Session, realm *database.Realm, fbUser *auth.UserRecord) bool {
	if (realm == nil || realm.EmailVerifiedMode == database.MFARequired) && !fbUser.EmailVerified {
		return true
	}

	if realm.EmailVerifiedMode == database.MFAOptionalPrompt &&
		!controller.EmailVerificationPromptedFromSession(session) &&
		!fbUser.EmailVerified {
		return true
	}

	return false
}
<commit_msg>Handle nil realm in logic (#591)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package middleware defines shared middleware for handlers.
package middleware

import (
	"context"
	"net/http"
	"time"

	"github.com/google/exposure-notifications-verification-server/pkg/controller"
	"github.com/google/exposure-notifications-verification-server/pkg/database"
	"github.com/google/exposure-notifications-verification-server/pkg/render"

	"github.com/google/exposure-notifications-server/pkg/logging"

	"firebase.google.com/go/auth"
	"github.com/gorilla/mux"
	"github.com/gorilla/sessions"
)

// RequireVerified requires a user to have verified their login email.
// MUST first run RequireAuth to populate user.
func RequireVerified(ctx context.Context, client *auth.Client, db *database.Database, h *render.Renderer, ttl time.Duration) mux.MiddlewareFunc {
	logger := logging.FromContext(ctx).Named("middleware.RequireVerified")

	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			ctx := r.Context()

			session := controller.SessionFromContext(ctx)
			if session == nil {
				logger.Errorw("session does not exist")
				controller.MissingSession(w, r, h)
				return
			}

			flash := controller.Flash(session)

			user := controller.UserFromContext(ctx)
			if user == nil {
				logger.Debugw("no user found when checking email verification")
				flash.Error("Log in first to verify email.")
				controller.ClearSessionFirebaseCookie(session)
				controller.Unauthorized(w, r, h)
				return
			}

			m := controller.TemplateMapFromContext(ctx)

			fbUser, err := client.GetUserByEmail(ctx, user.Email)
			if err != nil {
				delete(m, "currentUser") // Remove user from the template map.
				logger.Debugw("firebase user does not exist")
				flash.Error("That user does not exist.")
				controller.ClearSessionFirebaseCookie(session)
				controller.Unauthorized(w, r, h)
				return
			}

			realm := controller.RealmFromContext(ctx)
			if NeedsEmailVerification(session, realm, fbUser) {
				logger.Debugw("user email not verified")
				http.Redirect(w, r, "/login/verify-email", http.StatusSeeOther)
				return
			}

			next.ServeHTTP(w, r)
		})
	}
}

func NeedsEmailVerification(session *sessions.Session, realm *database.Realm, fbUser *auth.UserRecord) bool {
	if realm == nil || realm.EmailVerifiedMode == database.MFARequired {
		return !fbUser.EmailVerified
	}

	if realm.EmailVerifiedMode == database.MFAOptionalPrompt &&
		!controller.EmailVerificationPromptedFromSession(session) &&
		!fbUser.EmailVerified {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Nuclio Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rpc

import (
	"io"
	"net"
	"os"
	"os/exec"
	"testing"

	"github.com/nuclio/nuclio/pkg/errors"
	"github.com/nuclio/nuclio/pkg/functionconfig"
	"github.com/nuclio/nuclio/pkg/platformconfig"
	"github.com/nuclio/nuclio/pkg/processor"
	"github.com/nuclio/nuclio/pkg/processor/runtime"

	"github.com/nuclio/logger"
	"github.com/nuclio/zap"
	"github.com/stretchr/testify/suite"
)

type testRuntime struct {
	*AbstractRuntime
	wrapperProcess *os.Process
	wrapperConn    net.Conn
}

// NewRuntime returns a new Python runtime
func newTestRuntime(parentLogger logger.Logger, configuration *runtime.Configuration) (*testRuntime, error) {
	var err error

	newTestRuntime := &testRuntime{}

	newTestRuntime.AbstractRuntime, err = NewAbstractRuntime(parentLogger.GetChild("logger"),
		configuration,
		newTestRuntime)

	if err != nil {
		return nil, errors.Wrap(err, "Failed to create runtime")
	}

	return newTestRuntime, nil
}

func (r *testRuntime) RunWrapper(socketPath string) (*os.Process, error) {
	var err error
	cmd := exec.Command("sleep", "999999")
	if err = cmd.Start(); err != nil {
		return nil, err
	}
	r.wrapperProcess = cmd.Process

	// Connect to runtime
	r.wrapperConn, err = net.Dial("unix", socketPath)
	if err != nil {
		return nil, err
	}

	return cmd.Process, nil
}

func (r *testRuntime) GetEventEncoder(writer io.Writer) EventEncoder {
	return NewEventJSONEncoder(r.Logger, writer)
}

type RuntimeSuite struct {
	suite.Suite
	testRuntimeInstance *testRuntime
}

func (suite *RuntimeSuite) TestRestart() {
	var err error

	loggerInstance := suite.createLogger()
	configInstance := suite.createConfig(loggerInstance)

	suite.testRuntimeInstance, err = newTestRuntime(loggerInstance, configInstance)
	suite.Require().NoError(err, "Can't create runtime")

	err = suite.testRuntimeInstance.Start()
	suite.Require().NoError(err, "Can't start runtime")

	oldPid := suite.testRuntimeInstance.wrapperProcess.Pid
	err = suite.testRuntimeInstance.Restart()
	suite.Require().NoError(err, "Can't restart runtime")
	suite.Require().NotEqual(oldPid, suite.testRuntimeInstance.wrapperProcess.Pid, "Wrapper process didn't change")
}

func (suite *RuntimeSuite) TearDownTest() {
	if suite.testRuntimeInstance != nil && suite.testRuntimeInstance.wrapperProcess != nil {
		suite.testRuntimeInstance.wrapperProcess.Kill() // nolint: errcheck
	}
}

func (suite *RuntimeSuite) createLogger() logger.Logger {
	loggerInstance, err := nucliozap.NewNuclioZapTest("rpc-runtime-test")
	suite.Require().NoError(err, "Can't create logger")

	return loggerInstance
}

func (suite *RuntimeSuite) createConfig(loggerInstance logger.Logger) *runtime.Configuration {
	return &runtime.Configuration{
		FunctionLogger: loggerInstance,
		Configuration: &processor.Configuration{
			Config: functionconfig.Config{
				Meta: functionconfig.Meta{
					Namespace: "test",
				},
				Spec: functionconfig.Spec{},
			},
			PlatformConfig: &platformconfig.Config{
				Kind: "docker",
			},
		},
	}
}

func TestRuntime(t *testing.T) {
	suite.Run(t, new(RuntimeSuite))
}
<commit_msg>CI runtime abstract fix (#1481)<commit_after>/*
Copyright 2017 The Nuclio Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rpc

import (
	"io"
	"net"
	"os"
	"os/exec"
	"testing"

	"github.com/nuclio/nuclio/pkg/errors"
	"github.com/nuclio/nuclio/pkg/functionconfig"
	"github.com/nuclio/nuclio/pkg/platformconfig"
	"github.com/nuclio/nuclio/pkg/processor"
	"github.com/nuclio/nuclio/pkg/processor/runtime"

	"github.com/nuclio/logger"
	"github.com/nuclio/zap"
	"github.com/stretchr/testify/suite"
)

type testRuntime struct {
	*AbstractRuntime
	wrapperProcess *os.Process
	wrapperConn    net.Conn
}

// NewRuntime returns a new Python runtime
func newTestRuntime(parentLogger logger.Logger, configuration *runtime.Configuration) (*testRuntime, error) {
	var err error

	newTestRuntime := &testRuntime{}

	newTestRuntime.AbstractRuntime, err = NewAbstractRuntime(parentLogger.GetChild("logger"),
		configuration,
		newTestRuntime)

	if err != nil {
		return nil, errors.Wrap(err, "Failed to create runtime")
	}

	return newTestRuntime, nil
}

func (r *testRuntime) RunWrapper(socketPath string) (*os.Process, error) {
	var err error
	cmd := exec.Command("sleep", "999999")
	if err = cmd.Start(); err != nil {
		return nil, err
	}
	r.wrapperProcess = cmd.Process

	// Connect to runtime
	r.wrapperConn, err = net.Dial("unix", socketPath)
	if err != nil {
		return nil, err
	}

	return cmd.Process, nil
}

func (r *testRuntime) GetEventEncoder(writer io.Writer) EventEncoder {
	return NewEventJSONEncoder(r.Logger, writer)
}

type RuntimeSuite struct {
	suite.Suite
	testRuntimeInstance *testRuntime
}

func (suite *RuntimeSuite) TestRestart() {
	var err error

	loggerInstance := suite.createLogger()
	configInstance := suite.createConfig(loggerInstance)

	suite.testRuntimeInstance, err = newTestRuntime(loggerInstance, configInstance)
	suite.Require().NoError(err, "Can't create runtime")

	err = suite.testRuntimeInstance.Start()
	suite.Require().NoError(err, "Can't start runtime")

	oldPid := suite.testRuntimeInstance.wrapperProcess.Pid
	err = suite.testRuntimeInstance.Restart()
	suite.Require().NoError(err, "Can't restart runtime")
	suite.Require().NotEqual(oldPid, suite.testRuntimeInstance.wrapperProcess.Pid, "Wrapper process didn't change")
}

func (suite *RuntimeSuite) TearDownTest() {
	if suite.testRuntimeInstance != nil && suite.testRuntimeInstance.wrapperProcess != nil {
		suite.testRuntimeInstance.Stop() // nolint: errcheck
	}
}

func (suite *RuntimeSuite) createLogger() logger.Logger {
	loggerInstance, err := nucliozap.NewNuclioZapTest("rpc-runtime-test")
	suite.Require().NoError(err, "Can't create logger")

	return loggerInstance
}

func (suite *RuntimeSuite) createConfig(loggerInstance logger.Logger) *runtime.Configuration {
	return &runtime.Configuration{
		FunctionLogger: loggerInstance,
		Configuration: &processor.Configuration{
			Config: functionconfig.Config{
				Meta: functionconfig.Meta{
					Namespace: "test",
				},
				Spec: functionconfig.Spec{},
			},
			PlatformConfig: &platformconfig.Config{
				Kind: "docker",
			},
		},
	}
}

func TestRuntime(t *testing.T) {
	suite.Run(t, new(RuntimeSuite))
}
<|endoftext|>


<commit_before>// Author: Liam Stanley <me@liamstanley.io>
// Docs: https://marill.liam.sh/
// Repo: https://github.com/lrstanley/marill

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"text/template"
	"time"

	"github.com/lrstanley/marill/utils"
	"github.com/tdewolff/minify"
	"github.com/tdewolff/minify/css"
	"github.com/tdewolff/minify/html"
	"github.com/tdewolff/minify/js"
)

// JSONOutput is the generated json that will be embedded in Angular.
type JSONOutput struct {
	Version     string
	VersionFull string
	GitRevision string
	MinScore    float64
	Out         []*JSONTestResult
	Successful  int
	Failed      int
	Success     bool
	HostFile    string
	TimeScanned string
}

// Bytes returns a bytes array representation of JSONOutput.
func (j *JSONOutput) Bytes() []byte {
	jsonBytes, err := json.Marshal(j)
	if err != nil {
		panic(err)
	}

	return jsonBytes
}

// String returns a string representation of JSONOutput.
func (j *JSONOutput) String() string {
	jsonBytes, err := json.Marshal(j)
	if err != nil {
		panic(err)
	}

	return fmt.Sprintf("%s", jsonBytes)
}

// StringPretty returns a prettified/indented representation of JSONOutput.
func (j *JSONOutput) StringPretty() string {
	jsonBytes, err := json.MarshalIndent(j, "", "    ")
	if err != nil {
		panic(err)
	}

	return fmt.Sprintf("%s", jsonBytes)
}

// JSONTestResult is a wrapper around the test results, providing string
// representations of some errors and other items that during JSON conversion
// get converted to structs.
type JSONTestResult struct {
	*TestResult
	Assets      []*JSONTestResource
	ErrorString string // string representation of any errors
	URLString   string // string representation of the resulting URL.
}

type JSONTestResource struct {
	URL           string
	Code          int
	ContentLength int64
	Error         string
	Time          *utils.TimerResult
}

func genJSONOutput(scan *Scan) (*JSONOutput, error) {
	htmlConvertedResults := make([]*JSONTestResult, len(scan.results))
	var hosts string

	for i := 0; i < len(scan.results); i++ {
		htmlConvertedResults[i] = &JSONTestResult{TestResult: scan.results[i]}

		// Make the footprint of assets much smaller.
		if htmlConvertedResults[i].Result.Assets != nil && len(htmlConvertedResults[i].Result.Assets) > 0 {
			for j := 0; j < len(htmlConvertedResults[i].Result.Assets); j++ {
				var errString string
				err := htmlConvertedResults[i].Result.Assets[j].Error
				if err != nil {
					errString = htmlConvertedResults[i].Result.Assets[j].Error.Error()
				}

				htmlConvertedResults[i].Assets = append(htmlConvertedResults[i].Assets, &JSONTestResource{
					URL:           htmlConvertedResults[i].Result.Assets[j].URL,
					Code:          htmlConvertedResults[i].Result.Assets[j].Response.Code,
					ContentLength: htmlConvertedResults[i].Result.Assets[j].Response.ContentLength,
					Error:         errString,
					Time:          htmlConvertedResults[i].Result.Assets[j].Time,
				})
			}
		}

		if htmlConvertedResults[i].Result.Error != nil {
			htmlConvertedResults[i].ErrorString = htmlConvertedResults[i].Result.Error.Error()
			// make it so errors are still true, but it doesn't bloat the json
			htmlConvertedResults[i].Result.Error = errors.New(htmlConvertedResults[i].ErrorString)
		}

		if htmlConvertedResults[i].Result.Response.URL != nil {
			htmlConvertedResults[i].URLString = htmlConvertedResults[i].Result.Response.URL.String()
		}

		// trim out some of the bulk here
		if len(htmlConvertedResults[i].Result.Response.Body) > 200 {
			htmlConvertedResults[i].Result.Response.Body = htmlConvertedResults[i].Result.Response.Body[0:200] + " [...]"
		}

		if htmlConvertedResults[i].Result.Request.IP != "" {
			hosts += fmt.Sprintf("%s %s\n", htmlConvertedResults[i].Result.Request.IP, htmlConvertedResults[i].Result.Request.URL.Host)
		}
	}

	jsonOut := &JSONOutput{
		VersionFull: getVersion(),
		Version:     version,
		GitRevision: commithash,
		MinScore:    8.0,
		Out:         htmlConvertedResults,
		Successful:  scan.successful,
		Failed:      scan.failed,
		HostFile:    strings.TrimRight(hosts, "\n"),
		Success:     true,
		TimeScanned: time.Now().Format(time.RFC3339),
	}

	return jsonOut, nil
}

func genHTMLOutput(input *JSONOutput) ([]byte, error) {
	m := minify.New()
	m.AddFunc("text/css", css.Minify)
	m.AddFunc("text/html", html.Minify)
	m.AddFunc("text/javascript", js.Minify)

	// above the necessary static files
	htmlRawTmpl, err := Asset("data/html/index.html")
	if err != nil {
		return nil, err
	}
	jsRawTmpl, err := Asset("data/html/main.js")
	if err != nil {
		return nil, err
	}
	cssRawTmpl, err := Asset("data/html/main.css")
	if err != nil {
		return nil, err
	}

	// minify js and css
	jsTmpl, err := m.String("text/javascript", string(jsRawTmpl))
	if err != nil {
		return nil, err
	}

	cssTmpl, err := m.String("text/css", string(cssRawTmpl))
	if err != nil {
		return nil, err
	}

	jsonStr := fmt.Sprintf("%s", input.String())
	tmpl := template.New("html")
	tmpl.Delims("{[", "]}")
	tmpl = template.Must(tmpl.Parse(string(htmlRawTmpl)))

	var buf bytes.Buffer
	tmpl.Execute(&buf, struct {
		JSON string
		JS   string
		CSS  string
	}{
		JSON: jsonStr,
		JS:   jsTmpl,
		CSS:  cssTmpl,
	})

	htmlBytes, err := m.Bytes("text/html", buf.Bytes())
	if err != nil {
		return nil, err
	}

	return htmlBytes, nil
}
<commit_msg>add ContentType header as key for assets<commit_after>// Author: Liam Stanley <me@liamstanley.io>
// Docs: https://marill.liam.sh/
// Repo: https://github.com/lrstanley/marill

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"text/template"
	"time"

	"github.com/lrstanley/marill/utils"
	"github.com/tdewolff/minify"
	"github.com/tdewolff/minify/css"
	"github.com/tdewolff/minify/html"
	"github.com/tdewolff/minify/js"
)

// JSONOutput is the generated json that will be embedded in Angular.
type JSONOutput struct {
	Version     string
	VersionFull string
	GitRevision string
	MinScore    float64
	Out         []*JSONTestResult
	Successful  int
	Failed      int
	Success     bool
	HostFile    string
	TimeScanned string
}

// Bytes returns a bytes array representation of JSONOutput.
func (j *JSONOutput) Bytes() []byte {
	jsonBytes, err := json.Marshal(j)
	if err != nil {
		panic(err)
	}

	return jsonBytes
}

// String returns a string representation of JSONOutput.
func (j *JSONOutput) String() string {
	jsonBytes, err := json.Marshal(j)
	if err != nil {
		panic(err)
	}

	return fmt.Sprintf("%s", jsonBytes)
}

// StringPretty returns a prettified/indented representation of JSONOutput.
func (j *JSONOutput) StringPretty() string {
	jsonBytes, err := json.MarshalIndent(j, "", "    ")
	if err != nil {
		panic(err)
	}

	return fmt.Sprintf("%s", jsonBytes)
}

// JSONTestResult is a wrapper around the test results, providing string
// representations of some errors and other items that during JSON conversion
// get converted to structs.
type JSONTestResult struct {
	*TestResult
	Assets      []*JSONTestResource
	ErrorString string // string representation of any errors
	URLString   string // string representation of the resulting URL.
}

type JSONTestResource struct {
	URL           string
	Code          int
	ContentLength int64
	Error         string
	Time          *utils.TimerResult
	ContentType   string
}

func genJSONOutput(scan *Scan) (*JSONOutput, error) {
	htmlConvertedResults := make([]*JSONTestResult, len(scan.results))
	var hosts string

	for i := 0; i < len(scan.results); i++ {
		htmlConvertedResults[i] = &JSONTestResult{TestResult: scan.results[i]}

		// Make the footprint of assets much smaller.
		if htmlConvertedResults[i].Result.Assets != nil && len(htmlConvertedResults[i].Result.Assets) > 0 {
			for j := 0; j < len(htmlConvertedResults[i].Result.Assets); j++ {
				var errString string
				err := htmlConvertedResults[i].Result.Assets[j].Error
				if err != nil {
					errString = htmlConvertedResults[i].Result.Assets[j].Error.Error()
				}

				htmlConvertedResults[i].Assets = append(htmlConvertedResults[i].Assets, &JSONTestResource{
					URL:           htmlConvertedResults[i].Result.Assets[j].URL,
					Code:          htmlConvertedResults[i].Result.Assets[j].Response.Code,
					ContentLength: htmlConvertedResults[i].Result.Assets[j].Response.ContentLength,
					Error:         errString,
					Time:          htmlConvertedResults[i].Result.Assets[j].Time,
					ContentType:   htmlConvertedResults[i].Result.Assets[j].Response.Headers.Get("Content-Type"),
				})
			}
		}

		if htmlConvertedResults[i].Result.Error != nil {
			htmlConvertedResults[i].ErrorString = htmlConvertedResults[i].Result.Error.Error()
			// make it so errors are still true, but it doesn't bloat the json
			htmlConvertedResults[i].Result.Error = errors.New(htmlConvertedResults[i].ErrorString)
		}

		if htmlConvertedResults[i].Result.Response.URL != nil {
			htmlConvertedResults[i].URLString = htmlConvertedResults[i].Result.Response.URL.String()
		}

		// trim out some of the bulk here
		if len(htmlConvertedResults[i].Result.Response.Body) > 200 {
			htmlConvertedResults[i].Result.Response.Body = htmlConvertedResults[i].Result.Response.Body[0:200] + " [...]"
		}

		if htmlConvertedResults[i].Result.Request.IP != "" {
			hosts += fmt.Sprintf("%s %s\n", htmlConvertedResults[i].Result.Request.IP, htmlConvertedResults[i].Result.Request.URL.Host)
		}
	}

	jsonOut := &JSONOutput{
		VersionFull: getVersion(),
		Version:     version,
		GitRevision: commithash,
		MinScore:    8.0,
		Out:         htmlConvertedResults,
		Successful:  scan.successful,
		Failed:      scan.failed,
		HostFile:    strings.TrimRight(hosts, "\n"),
		Success:     true,
		TimeScanned: time.Now().Format(time.RFC3339),
	}

	return jsonOut, nil
}

func genHTMLOutput(input *JSONOutput) ([]byte, error) {
	m := minify.New()
	m.AddFunc("text/css", css.Minify)
	m.AddFunc("text/html", html.Minify)
	m.AddFunc("text/javascript", js.Minify)

	// above the necessary static files
	htmlRawTmpl, err := Asset("data/html/index.html")
	if err != nil {
		return nil, err
	}
	jsRawTmpl, err := Asset("data/html/main.js")
	if err != nil {
		return nil, err
	}
	cssRawTmpl, err := Asset("data/html/main.css")
	if err != nil {
		return nil, err
	}

	// minify js and css
	jsTmpl, err := m.String("text/javascript", string(jsRawTmpl))
	if err != nil {
		return nil, err
	}

	cssTmpl, err := m.String("text/css", string(cssRawTmpl))
	if err != nil {
		return nil, err
	}

	jsonStr := fmt.Sprintf("%s", input.String())
	tmpl := template.New("html")
	tmpl.Delims("{[", "]}")
	tmpl = template.Must(tmpl.Parse(string(htmlRawTmpl)))

	var buf bytes.Buffer
	tmpl.Execute(&buf, struct {
		JSON string
		JS   string
		CSS  string
	}{
		JSON: jsonStr,
		JS:   jsTmpl,
		CSS:  cssTmpl,
	})

	htmlBytes, err := m.Bytes("text/html", buf.Bytes())
	if err != nil {
		return nil, err
	}

	return htmlBytes, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pipelinerun

import (
	"testing"
	"time"

	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	alpha1 "github.com/tektoncd/pipeline/pkg/client/informers/externalversions/pipeline/v1alpha1"
	fakepipelineruninformer "github.com/tektoncd/pipeline/pkg/client/injection/informers/pipeline/v1alpha1/pipelinerun/fake"
	tb "github.com/tektoncd/pipeline/test/builder"
	corev1 "k8s.io/api/core/v1"
	"knative.dev/pkg/apis"
	"knative.dev/pkg/metrics/metricstest"

	// Required to setup metrics env for testing
	_ "knative.dev/pkg/metrics/testing"
	rtesting "knative.dev/pkg/reconciler/testing"
)

func TestUninitializedMetrics(t *testing.T) {
	metrics := Recorder{}

	durationCountError := metrics.DurationAndCount(&v1alpha1.PipelineRun{})
	prCountError := metrics.RunningPipelineRuns(nil)

	assertErrNotNil(durationCountError, "DurationAndCount recording expected to return error but got nil", t)
	assertErrNotNil(prCountError, "Current PR count recording expected to return error but got nil", t)
}

func TestRecordPipelineRunDurationCount(t *testing.T) {
	startTime := time.Now()

	testData := []struct {
		name              string
		taskRun           *v1alpha1.PipelineRun
		expectedTags      map[string]string
		expectedCountTags map[string]string
		expectedDuration  float64
		expectedCount     int64
	}{{
		name: "for_succeeded_pipeline",
		taskRun: tb.PipelineRun("pipelinerun-1", tb.PipelineRunNamespace("ns"),
			tb.PipelineRunSpec("pipeline-1"),
			tb.PipelineRunStatus(
				tb.PipelineRunStartTime(startTime),
				tb.PipelineRunCompletionTime(startTime.Add(1*time.Minute)),
				tb.PipelineRunStatusCondition(apis.Condition{
					Type:   apis.ConditionSucceeded,
					Status: corev1.ConditionTrue,
				}),
			)),
		expectedTags: map[string]string{
			"pipeline":    "pipeline-1",
			"pipelinerun": "pipelinerun-1",
			"namespace":   "ns",
			"status":      "success",
		},
		expectedCountTags: map[string]string{
			"status": "success",
		},
		expectedDuration: 60,
		expectedCount:    1,
	}, {
		name: "for_failed_pipeline",
		taskRun: tb.PipelineRun("pipelinerun-1", tb.PipelineRunNamespace("ns"),
			tb.PipelineRunSpec("pipeline-1"),
			tb.PipelineRunStatus(
				tb.PipelineRunStartTime(startTime),
				tb.PipelineRunCompletionTime(startTime.Add(1*time.Minute)),
				tb.PipelineRunStatusCondition(apis.Condition{
					Type:   apis.ConditionSucceeded,
					Status: corev1.ConditionFalse,
				}),
			)),
		expectedTags: map[string]string{
			"pipeline":    "pipeline-1",
			"pipelinerun": "pipelinerun-1",
			"namespace":   "ns",
			"status":      "failed",
		},
		expectedCountTags: map[string]string{
			"status": "failed",
		},
		expectedDuration: 60,
		expectedCount:    1,
	}}

	for _, test := range testData {
		t.Run(test.name, func(t *testing.T) {
			unregisterMetrics()

			metrics, err := NewRecorder()
			assertErrIsNil(err, "Recorder initialization failed", t)

			err = metrics.DurationAndCount(test.taskRun)
			assertErrIsNil(err, "DurationAndCount recording recording got an error", t)
			metricstest.CheckDistributionData(t, "pipelinerun_duration_seconds", test.expectedTags, 1, test.expectedDuration, test.expectedDuration)
			metricstest.CheckCountData(t, "pipelinerun_count", test.expectedCountTags, test.expectedCount)

		})
	}
}

func TestRecordRunningPipelineRunsCount(t *testing.T) {
	unregisterMetrics()

	ctx, _ := rtesting.SetupFakeContext(t)
	informer := fakepipelineruninformer.Get(ctx)
	addPipelineRun(informer, "pipelinerun-1", "pipeline-1", "ns", corev1.ConditionTrue, t)
	addPipelineRun(informer, "pipelinerun-2", "pipeline-2", "ns", corev1.ConditionFalse, t)
	addPipelineRun(informer, "pipelinerun-3", "pipeline-3", "ns", corev1.ConditionUnknown, t)

	metrics, err := NewRecorder()
	assertErrIsNil(err, "Recorder initialization failed", t)

	err = metrics.RunningPipelineRuns(informer.Lister())
	assertErrIsNil(err, "RunningPrsCount recording expected to return nil but got error", t)
	metricstest.CheckLastValueData(t, "running_pipelineruns_count", map[string]string{}, 1)

}

func addPipelineRun(informer alpha1.PipelineRunInformer, run, pipeline, ns string, status corev1.ConditionStatus, t *testing.T) {
	t.Helper()

	err := informer.Informer().GetIndexer().Add(tb.PipelineRun(run, tb.PipelineRunNamespace(ns),
		tb.PipelineRunSpec(pipeline),
		tb.PipelineRunStatus(
			tb.PipelineRunStatusCondition(apis.Condition{
				Type:   apis.ConditionSucceeded,
				Status: status,
			}),
		)))

	if err != nil {
		t.Errorf("Failed to add the pipelinerun")
	}
}

func assertErrNotNil(err error, message string, t *testing.T) {
	t.Helper()
	if err == nil {
		t.Errorf(message)
	}
}

func assertErrIsNil(err error, message string, t *testing.T) {
	t.Helper()
	if err != nil {
		t.Errorf(message)
	}
}

func unregisterMetrics() {
	metricstest.Unregister("pipelinerun_duration_seconds", "pipelinerun_count", "running_pipelineruns_count")
}
<commit_msg>Use pipelineRun instead of taskRun in metrics_test…<commit_after>/*
Copyright 2019 The Tekton Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package pipelinerun

import (
	"testing"
	"time"

	"github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1"
	alpha1 "github.com/tektoncd/pipeline/pkg/client/informers/externalversions/pipeline/v1alpha1"
	fakepipelineruninformer "github.com/tektoncd/pipeline/pkg/client/injection/informers/pipeline/v1alpha1/pipelinerun/fake"
	tb "github.com/tektoncd/pipeline/test/builder"
	corev1 "k8s.io/api/core/v1"
	"knative.dev/pkg/apis"
	"knative.dev/pkg/metrics/metricstest"

	// Required to setup metrics env for testing
	_ "knative.dev/pkg/metrics/testing"
	rtesting "knative.dev/pkg/reconciler/testing"
)

func TestUninitializedMetrics(t *testing.T) {
	metrics := Recorder{}

	durationCountError := metrics.DurationAndCount(&v1alpha1.PipelineRun{})
	prCountError := metrics.RunningPipelineRuns(nil)

	assertErrNotNil(durationCountError, "DurationAndCount recording expected to return error but got nil", t)
	assertErrNotNil(prCountError, "Current PR count recording expected to return error but got nil", t)
}

func TestRecordPipelineRunDurationCount(t *testing.T) {
	startTime := time.Now()

	testData := []struct {
		name              string
		pipelineRun       *v1alpha1.PipelineRun
		expectedTags      map[string]string
		expectedCountTags map[string]string
		expectedDuration  float64
		expectedCount     int64
	}{{
		name: "for_succeeded_pipeline",
		pipelineRun: tb.PipelineRun("pipelinerun-1", tb.PipelineRunNamespace("ns"),
			tb.PipelineRunSpec("pipeline-1"),
			tb.PipelineRunStatus(
				tb.PipelineRunStartTime(startTime),
				tb.PipelineRunCompletionTime(startTime.Add(1*time.Minute)),
				tb.PipelineRunStatusCondition(apis.Condition{
					Type:   apis.ConditionSucceeded,
					Status: corev1.ConditionTrue,
				}),
			)),
		expectedTags: map[string]string{
			"pipeline":    "pipeline-1",
			"pipelinerun": "pipelinerun-1",
			"namespace":   "ns",
			"status":      "success",
		},
		expectedCountTags: map[string]string{
			"status": "success",
		},
		expectedDuration: 60,
		expectedCount:    1,
	}, {
		name: "for_failed_pipeline",
		pipelineRun: tb.PipelineRun("pipelinerun-1", tb.PipelineRunNamespace("ns"),
			tb.PipelineRunSpec("pipeline-1"),
			tb.PipelineRunStatus(
				tb.PipelineRunStartTime(startTime),
				tb.PipelineRunCompletionTime(startTime.Add(1*time.Minute)),
				tb.PipelineRunStatusCondition(apis.Condition{
					Type:   apis.ConditionSucceeded,
					Status: corev1.ConditionFalse,
				}),
			)),
		expectedTags: map[string]string{
			"pipeline":    "pipeline-1",
			"pipelinerun": "pipelinerun-1",
			"namespace":   "ns",
			"status":      "failed",
		},
		expectedCountTags: map[string]string{
			"status": "failed",
		},
		expectedDuration: 60,
		expectedCount:    1,
	}}

	for _, test := range testData {
		t.Run(test.name, func(t *testing.T) {
			unregisterMetrics()

			metrics, err := NewRecorder()
			assertErrIsNil(err, "Recorder initialization failed", t)

			err = metrics.DurationAndCount(test.pipelineRun)
			assertErrIsNil(err, "DurationAndCount recording recording got an error", t)
			metricstest.CheckDistributionData(t, "pipelinerun_duration_seconds", test.expectedTags, 1, test.expectedDuration, test.expectedDuration)
			metricstest.CheckCountData(t, "pipelinerun_count", test.expectedCountTags, test.expectedCount)

		})
	}
}

func TestRecordRunningPipelineRunsCount(t *testing.T) {
	unregisterMetrics()

	ctx, _ := rtesting.SetupFakeContext(t)
	informer := fakepipelineruninformer.Get(ctx)
	addPipelineRun(informer, "pipelinerun-1", "pipeline-1", "ns", corev1.ConditionTrue, t)
	addPipelineRun(informer, "pipelinerun-2", "pipeline-2", "ns", corev1.ConditionFalse, t)
	addPipelineRun(informer, "pipelinerun-3", "pipeline-3", "ns", corev1.ConditionUnknown, t)

	metrics, err := NewRecorder()
	assertErrIsNil(err, "Recorder initialization failed", t)

	err = metrics.RunningPipelineRuns(informer.Lister())
	assertErrIsNil(err, "RunningPrsCount recording expected to return nil but got error", t)
	metricstest.CheckLastValueData(t, "running_pipelineruns_count", map[string]string{}, 1)

}

func addPipelineRun(informer alpha1.PipelineRunInformer, run, pipeline, ns string, status corev1.ConditionStatus, t *testing.T) {
	t.Helper()

	err := informer.Informer().GetIndexer().Add(tb.PipelineRun(run, tb.PipelineRunNamespace(ns),
		tb.PipelineRunSpec(pipeline),
		tb.PipelineRunStatus(
			tb.PipelineRunStatusCondition(apis.Condition{
				Type:   apis.ConditionSucceeded,
				Status: status,
			}),
		)))

	if err != nil {
		t.Errorf("Failed to add the pipelinerun")
	}
}

func assertErrNotNil(err error, message string, t *testing.T) {
	t.Helper()
	if err == nil {
		t.Errorf(message)
	}
}

func assertErrIsNil(err error, message string, t *testing.T) {
	t.Helper()
	if err != nil {
		t.Errorf(message)
	}
}

func unregisterMetrics() {
	metricstest.Unregister("pipelinerun_duration_seconds", "pipelinerun_count", "running_pipelineruns_count")
}
<|endoftext|>


<commit_before>package main

import (
	"bosun.org/graphite"
	"encoding/json"
	"fmt"
	"github.com/Dieterbe/go-metrics"
	"github.com/raintank/raintank-metric/schema"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
	//"strings"
)

type Hit struct {
	Index  string                  `json:"_index"`
	Type   string                  `json:"_type"`
	Id     string                  `json:"_id"`
	Score  float64                 `json:"_score"`
	Source schema.MetricDefinition `json:"_source"`
}

type EsResult struct {
	Took     int
	TimedOut bool
	_shards  struct {
		total      int
		successful int
		failed     int
	}
	Hits struct {
		Total    int
		MaxScore int
		Hits     []Hit
	}
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type stat struct {
	def       schema.MetricDefinition
	firstSeen int64
}

var targets map[string]stat
var targetKeys []string
var targetsLock sync.Mutex

func init() {
	targets = make(map[string]stat)
	targetKeys = make([]string, 0)
}

func main() {
	if len(os.Args) != 4 {
		log.Fatal("usage: graphite-watcher <elasticsearch-addr> <metrics-addr> <graphite-addr>")
	}
	addr, _ := net.ResolveTCPAddr("tcp", os.Args[2])
	go metrics.Graphite(metrics.DefaultRegistry, 10e9, "graphite-watcher.", addr)
	lag := metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015))
	metrics.Register("lag", lag)
	numMetrics := metrics.NewGauge()
	metrics.Register("num_metrics", numMetrics)

	// for a metric to exist in ES at t=Y, there must at least have been 1 point for that metric
	// at a time X where X < Y.  Hence, we confidently say that if we see a metric at Y, we can
	// demand data to show up for that metric at >=Y
	// for our data check to be useful we need metrics to show up in ES soon after being in the pipeline,
	// which seems to be true (see nsqadmin)
	updateTargets := func() {
		getEsTick := time.NewTicker(time.Second * time.Duration(10))
		for ts := range getEsTick.C {
			tsUnix := ts.Unix()
			res, err := http.Get("http://" + os.Args[1] + "/metric/_search?q=*:*&size=10000000")
			perror(err)
			defer res.Body.Close()
			body, err := ioutil.ReadAll(res.Body)
			perror(err)
			var data EsResult
			err = json.Unmarshal(body, &data)
			perror(err)
			amount := len(data.Hits.Hits)
			numMetrics.Update(int64(amount))
			for _, h := range data.Hits.Hits {
				//fmt.Println(h.Source.Name)
				targetsLock.Lock()
				if _, ok := targets[h.Source.Name]; !ok {
					targets[h.Source.Name] = stat{h.Source, tsUnix}
					targetKeys = append(targetKeys, h.Source.Name)
				}
				targetsLock.Unlock()
			}
		}
	}

	test := func(wg *sync.WaitGroup, curTs int64, met stat) {
		g := graphite.HostHeader{Host: "http://" + os.Args[3] + "/render", Header: http.Header{}}
		g.Header.Add("X-Org-Id", strconv.FormatInt(int64(met.def.OrgId), 10))
		g.Header.Set("User-Agent", "graphite-watcher")
		q := graphite.Request{Targets: []string{met.def.Name}}
		series, err := g.Query(&q)
		perror(err)
		for _, serie := range series {
			if met.def.Name != serie.Target {
				fmt.Println("ERROR: name != target name:", met.def.Name, serie.Target)
			}

			lastTs := int64(0)
			oldestNull := int64(math.MaxInt64)
			if len(serie.Datapoints) == 0 {
				fmt.Println("ERROR: series for", met.def.Name, "contains no points!")
			}
			for _, p := range serie.Datapoints {
				ts, err := p[1].Int64()
				if err != nil {
					fmt.Println("ERROR: could not parse timestamp", p)
				}
				if ts <= lastTs {
					fmt.Println("ERROR: timestamp must be bigger than last", lastTs, ts)
				}
				if lastTs == 0 && (ts < curTs-24*3600-60 || ts > curTs-24*3600+60) {
					fmt.Println("ERROR: first point", p, "should have been about 24h ago")
				}
				if lastTs != 0 && ts != lastTs+int64(met.def.Interval) {
					fmt.Println("ERROR: point", p, " is not interval ", met.def.Interval, "apart from previous point")
				}
				_, err = p[0].Float64()
				if err != nil && ts > met.firstSeen {
					if ts < oldestNull {
						oldestNull = ts
					}
					if ts < curTs-30 {
						// will show up in lag metric too
						fmt.Println("ERROR: point has a recent null value", p)
					}
				}
				lastTs = ts
			}
			if lastTs < curTs-int64(met.def.Interval) || lastTs > curTs+int64(met.def.Interval) {
				fmt.Println("ERROR: last point at ", lastTs, "is out of range")
			}
			// if there was no null, we treat the point after the last one we had as null
			if oldestNull == math.MaxInt64 {
				oldestNull = lastTs + int64(met.def.Interval)
			}
			// lag cannot be < 0
			if oldestNull > curTs {
				oldestNull = curTs
			}
			// lag is from first null, even if there were non-nulls after it
			//fmt.Printf("%60s - lag %d\n", name, curTs-oldestNull)
			lag.Update(curTs - oldestNull)
		}
		wg.Done()
	}

	go updateTargets()

	tick := time.NewTicker(time.Millisecond * time.Duration(100))
	wg := &sync.WaitGroup{}
	for ts := range tick.C {
		var t stat
		targetsLock.Lock()
		if len(targetKeys) > 0 {
			key := targetKeys[rand.Intn(len(targets))]
			t = targets[key]
		}
		targetsLock.Unlock()
		if len(targetKeys) > 0 {
			wg.Add(1)
			go test(wg, ts.Unix(), t)
		}
	}
	wg.Wait()
}
<commit_msg>better err<commit_after>package main

import (
	"bosun.org/graphite"
	"encoding/json"
	"fmt"
	"github.com/Dieterbe/go-metrics"
	"github.com/raintank/raintank-metric/schema"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
	//"strings"
)

type Hit struct {
	Index  string                  `json:"_index"`
	Type   string                  `json:"_type"`
	Id     string                  `json:"_id"`
	Score  float64                 `json:"_score"`
	Source schema.MetricDefinition `json:"_source"`
}

type EsResult struct {
	Took     int
	TimedOut bool
	_shards  struct {
		total      int
		successful int
		failed     int
	}
	Hits struct {
		Total    int
		MaxScore int
		Hits     []Hit
	}
}

func perror(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

type stat struct {
	def       schema.MetricDefinition
	firstSeen int64
}

var targets map[string]stat
var targetKeys []string
var targetsLock sync.Mutex

func init() {
	targets = make(map[string]stat)
	targetKeys = make([]string, 0)
}

func main() {
	if len(os.Args) != 4 {
		log.Fatal("usage: graphite-watcher <elasticsearch-addr> <metrics-addr> <graphite-addr>")
	}
	addr, _ := net.ResolveTCPAddr("tcp", os.Args[2])
	go metrics.Graphite(metrics.DefaultRegistry, 10e9, "graphite-watcher.", addr)
	lag := metrics.NewHistogram(metrics.NewExpDecaySample(1028, 0.015))
	metrics.Register("lag", lag)
	numMetrics := metrics.NewGauge()
	metrics.Register("num_metrics", numMetrics)

	// for a metric to exist in ES at t=Y, there must at least have been 1 point for that metric
	// at a time X where X < Y.  Hence, we confidently say that if we see a metric at Y, we can
	// demand data to show up for that metric at >=Y
	// for our data check to be useful we need metrics to show up in ES soon after being in the pipeline,
	// which seems to be true (see nsqadmin)
	updateTargets := func() {
		getEsTick := time.NewTicker(time.Second * time.Duration(10))
		for ts := range getEsTick.C {
			tsUnix := ts.Unix()
			res, err := http.Get("http://" + os.Args[1] + "/metric/_search?q=*:*&size=10000000")
			perror(err)
			defer res.Body.Close()
			body, err := ioutil.ReadAll(res.Body)
			perror(err)
			var data EsResult
			err = json.Unmarshal(body, &data)
			perror(err)
			amount := len(data.Hits.Hits)
			numMetrics.Update(int64(amount))
			for _, h := range data.Hits.Hits {
				//fmt.Println(h.Source.Name)
				targetsLock.Lock()
				if _, ok := targets[h.Source.Name]; !ok {
					targets[h.Source.Name] = stat{h.Source, tsUnix}
					targetKeys = append(targetKeys, h.Source.Name)
				}
				targetsLock.Unlock()
			}
		}
	}

	test := func(wg *sync.WaitGroup, curTs int64, met stat) {
		g := graphite.HostHeader{Host: "http://" + os.Args[3] + "/render", Header: http.Header{}}
		g.Header.Add("X-Org-Id", strconv.FormatInt(int64(met.def.OrgId), 10))
		g.Header.Set("User-Agent", "graphite-watcher")
		q := graphite.Request{Targets: []string{met.def.Name}}
		series, err := g.Query(&q)
		perror(err)
		for _, serie := range series {
			if met.def.Name != serie.Target {
				fmt.Println("ERROR: name != target name:", met.def.Name, serie.Target)
			}

			lastTs := int64(0)
			oldestNull := int64(math.MaxInt64)
			if len(serie.Datapoints) == 0 {
				fmt.Println("ERROR: series for", met.def.Name, "contains no points!")
			}
			for _, p := range serie.Datapoints {
				ts, err := p[1].Int64()
				if err != nil {
					fmt.Println("ERROR: could not parse timestamp", p)
				}
				if ts <= lastTs {
					fmt.Println("ERROR: timestamp must be bigger than last", lastTs, ts)
				}
				if lastTs == 0 && (ts < curTs-24*3600-60 || ts > curTs-24*3600+60) {
					fmt.Println("ERROR: first point", p, "should have been about 24h ago")
				}
				if lastTs != 0 && ts != lastTs+int64(met.def.Interval) {
					fmt.Println("ERROR: point", p, " is not interval ", met.def.Interval, "apart from previous point")
				}
				_, err = p[0].Float64()
				if err != nil && ts > met.firstSeen {
					if ts < oldestNull {
						oldestNull = ts
					}
					if ts < curTs-30 {
						// will show up in lag metric too
						fmt.Println("ERROR: ", met.def.Name, " at", curTs, "seeing a null for ts", p[1])
					}
				}
				lastTs = ts
			}
			if lastTs < curTs-int64(met.def.Interval) || lastTs > curTs+int64(met.def.Interval) {
				fmt.Println("ERROR: last point at ", lastTs, "is out of range")
			}
			// if there was no null, we treat the point after the last one we had as null
			if oldestNull == math.MaxInt64 {
				oldestNull = lastTs + int64(met.def.Interval)
			}
			// lag cannot be < 0
			if oldestNull > curTs {
				oldestNull = curTs
			}
			// lag is from first null, even if there were non-nulls after it
			//fmt.Printf("%60s - lag %d\n", name, curTs-oldestNull)
			lag.Update(curTs - oldestNull)
		}
		wg.Done()
	}

	go updateTargets()

	tick := time.NewTicker(time.Millisecond * time.Duration(100))
	wg := &sync.WaitGroup{}
	for ts := range tick.C {
		var t stat
		targetsLock.Lock()
		if len(targetKeys) > 0 {
			key := targetKeys[rand.Intn(len(targets))]
			t = targets[key]
		}
		targetsLock.Unlock()
		if len(targetKeys) > 0 {
			wg.Add(1)
			go test(wg, ts.Unix(), t)
		}
	}
	wg.Wait()
}
<|endoftext|>