<commit_before>package formula

import (
	"bytes"
	"fmt"
	"strconv"
	"text/scanner"
)

// This VM is quite simple, having only a general purpose
// register (R) and a boolean status register (S).
// Some instructions might contain an integer value (V):
//
//  N - set R = n
//  ADD - set R = R + V
//  SUB - set R = R - V
//  MULT - set R = R * V
//  DIV - set R = R / V
//  MOD - set R = R % V
//  JMPT - jump by V if S is true
//  JMPF - jump by V if S is false
//  EQ - set S = (R == V)
//  NEQ - set S = (R != V)
//  LT - set S = (R < V)
//  LTE - set S = (R <= V)
//  GT - set S = (R > V)
//  GTE - set S = (R >= V)
//  RET - end execution and return V
//
// If the end of the program is reached without finding
// a ret instruction, the last value of S is returned.
// as an integer.

type opCode uint8

const (
	// Instructions altering R
	opN opCode = iota + 1
	opADD
	opSUB
	opMULT
	opDIV
	opMOD
	// Special instructions
	opRET
	// Jump instructions
	opJMPT
	opJMPF
	// Comparison instructions
	opEQ
	opNEQ
	opLT
	opLTE
	opGT
	opGTE
)

func (o opCode) String() string {
	names := []string{"N", "ADD", "SUB", "MULT", "DIV", "MOD", "RET", "JMPT", "JMPF", "EQ", "NEQ", "LT", "LTE", "GT", "GTE"}
	return names[int(o)-1]
}

func (o opCode) Alters() bool {
	return o <= opMOD
}

func (o opCode) IsSpecial() bool {
	return o == opRET
}

func (o opCode) IsJump() bool {
	return o == opJMPT || o == opJMPF
}

func (o opCode) Compares() bool {
	return o >= opEQ
}

type instruction struct {
	opCode opCode
	value  int
}

type program []*instruction

func invalid(s *scanner.Scanner, what, val string) (program, error) {
	return nil, fmt.Errorf("invalid %s in formula at %s: %q", what, s.Pos(), val)
}

func jumpTarget(s *scanner.Scanner, code []byte, chr byte) int {
	// look for matching chr
	offset := s.Pos().Offset
	paren := 0
	target := -1
	for ii, v := range code[offset:] {
		if v == '(' {
			paren++
		} else if v == ')' {
			paren--
			if paren < 0 {
				target = offset + ii
				break
			}
		} else if v == chr && paren == 0 {
			target = offset + ii
			break
		}
	}
	return target
}

func makeJump(s *scanner.Scanner, code []byte, p *program, op opCode, jumps map[int][]int, chr byte) {
	// end of conditional, put the placeholder for a jump
	// and complete it once we reach the matching chr. Store the
	// current position of the jump in its value, so
	// calculating the relative offset is quicker.
	pos := len(*p)
	inst := &instruction{opCode: op, value: pos}
	*p = append(*p, inst)
	target := jumpTarget(s, code, chr)
	jumps[target] = append(jumps[target], pos)
}

func resolveJumps(s *scanner.Scanner, p program, jumps map[int][]int) {
	// check for incomplete jumps to this location.
	// the pc should point at the next instruction
	// to be added and the jump is relative.
	pc := len(p)
	offset := s.Pos().Offset - 1
	for _, v := range jumps[offset] {
		inst := p[v]
		inst.value = pc - inst.value - 1
	}
	delete(jumps, offset)
}

func compileVmFormulaString(code string) (Formula, error) {
	return compileVmFormula([]byte(code))
}

func compileVmFormula(code []byte) (Formula, error) {
	p, err := vmCompile(code)
	if err != nil {
		return nil, err
	}
	p = vmOptimize(p)
	return makeVmFunc(p), nil
}

func vmCompile(code []byte) (program, error) {
	var s scanner.Scanner
	var err error
	s.Init(bytes.NewReader(code))
	s.Error = func(s *scanner.Scanner, msg string) {
		err = fmt.Errorf("error parsing plural formula %s: %s", s.Pos(), msg)
	}
	s.Mode = scanner.ScanIdents | scanner.ScanInts
	tok := s.Scan()
	var p program
	var op bytes.Buffer
	var logic bytes.Buffer
	jumps := make(map[int][]int)
	for tok != scanner.EOF && err == nil {
		switch tok {
		case scanner.Ident:
			if n := s.TokenText(); n != "n" {
				return invalid(&s, "ident", n)
			}
			if op.Len() > 0 {
				return invalid(&s, "ident", "RHS variables are not supported")
			}
			p = append(p, &instruction{opCode: opN})
		case scanner.Int:
			val, _ := strconv.Atoi(s.TokenText())
			if op.Len() == 0 {
				// return statement
				p = append(p, &instruction{opCode: opRET, value: val})
			} else {
				var opc opCode
				switch op.String() {
				case "+":
					opc = opADD
				case "-":
					opc = opSUB
				case "*":
					opc = opMULT
				case "/":
					opc = opDIV
				case "%":
					opc = opMOD
				case "==":
					opc = opEQ
				case "!=":
					opc = opNEQ
				case "<":
					opc = opLT
				case "<=":
					opc = opLTE
				case ">":
					opc = opGT
				case ">=":
					opc = opGTE
				default:
					return invalid(&s, "op", op.String())
				}
				p = append(p, &instruction{opCode: opc, value: val})
				op.Reset()
			}
		case '?':
			resolveJumps(&s, p, jumps)
			makeJump(&s, code, &p, opJMPF, jumps, ':')
		case ':':
			resolveJumps(&s, p, jumps)
		case '!', '=', '<', '>', '%':
			op.WriteRune(tok)
		case '&', '|':
			// logic operations
			if logic.Len() == 0 {
				logic.WriteRune(tok)
			} else if logic.Len() == 1 {
				b := logic.Bytes()[0]
				if b != byte(tok) {
					return invalid(&s, "token", string(tok))
				}
				if b == '&' {
					makeJump(&s, code, &p, opJMPF, jumps, ':')
				} else {
					makeJump(&s, code, &p, opJMPT, jumps, '?')
				}
				logic.Reset()
			} else {
				return invalid(&s, "token", string(tok))
			}
		case '(':
		case ')':
			resolveJumps(&s, p, jumps)
		default:
			return invalid(&s, "token", string(tok))
		}
		tok = s.Scan()
	}
	return p, nil
}

func removeInstructions(p program, start int, count int) program {
	p = append(p[:start], p[start+count:]...)
	// Check for jumps that might be affected by the removal
	for kk := start; kk >= 0; kk-- {
		if in := p[kk]; in.opCode.IsJump() && kk+in.value > start {
			in.value -= count
		}
	}
	return p
}

func vmOptimize(p program) program {
	// The optimizer is quite simple. Each pass is documented
	// at its beginning.

	//A first pass looks
	// for multiple comparison instructions that are preceeded
	// by exactly the same instructions and it removes the second
	// group of instructions.
	cmp := -1
	count := len(p)
	ii := 0
	for ; ii < count; ii++ {
		v := p[ii]
		if v.opCode.Compares() {
			if cmp >= 0 {
				delta := ii - cmp
				jj := cmp - 1
				for ; jj >= 0; jj-- {
					i1 := p[jj]
					if !i1.opCode.Alters() {
						break
					}
					i2 := p[jj+delta]
					if i1.opCode != i2.opCode || i1.value != i2.value {
						break
					}
				}
				equal := (cmp - 1) - jj
				if equal > 0 {
					ii -= equal
					count -= equal
					p = removeInstructions(p, ii, equal)
					continue
				}
			}
			cmp = ii
		}
	}
	// A second pass then looks for
	// instructions that set R = N when R is already
	// equal to N and it removes the second instruction.
	n := -1
	for ii = 0; ii < count; ii++ {
		v := p[ii]
		if v.opCode == opN {
			if n >= 0 {
				p = removeInstructions(p, ii, 1)
				ii--
				count--
				continue
			}
			n = ii
		} else if v.opCode.Alters() {
			n = -1
		}
	}
	// Third pass looks for jumps which end up in a jump of the same type,
	// add adjusts the value to make just one jump.
	for ii := 0; ii < count; ii++ {
		v := insts[ii]
		if v.opCode.IsJump() {
			for true {
				t := ii + v.value + 1
				if t >= count {
					break
				}
				nv := insts[t]
				if nv.opCode != v.opCode {
					break
				}
				v.value += nv.value
			}
		}
	}
	return insts
}

func makeVmFunc(p program) Formula {
	count := len(p)
	return func(n int) int {
		return vmExec(p, count, n)
	}
}

func vmExec(p program, count int, n int) int {
	var R int
	var S bool
	for ii := 0; ii < count; ii++ {
		i := p[ii]
		switch i.opCode {
		case opN:
			R = n
		case opADD:
			R += i.value
		case opSUB:
			R -= i.value
		case opMULT:
			R *= i.value
		case opDIV:
			R /= i.value
		case opMOD:
			R %= i.value
		case opRET:
			return i.value
		case opJMPT:
			if S {
				ii += i.value
			}
		case opJMPF:
			if !S {
				ii += i.value
			}
		case opEQ:
			S = R == i.value
		case opNEQ:
			S = R != i.value
		case opLT:
			S = R < i.value
		case opLTE:
			S = R <= i.value
		case opGT:
			S = R > i.value
		case opGTE:
			S = R >= i.value
		}
	}
	return bint(S)
}
<commit_msg>Perform one more optimization in the third pass<commit_after>package formula

import (
	"bytes"
	"fmt"
	"strconv"
	"text/scanner"
)

// This VM is quite simple, having only a general purpose
// register (R) and a boolean status register (S).
// Some instructions might contain an integer value (V):
//
//  N - set R = n
//  ADD - set R = R + V
//  SUB - set R = R - V
//  MULT - set R = R * V
//  DIV - set R = R / V
//  MOD - set R = R % V
//  JMPT - jump by V if S is true
//  JMPF - jump by V if S is false
//  EQ - set S = (R == V)
//  NEQ - set S = (R != V)
//  LT - set S = (R < V)
//  LTE - set S = (R <= V)
//  GT - set S = (R > V)
//  GTE - set S = (R >= V)
//  RET - end execution and return V
//
// If the end of the program is reached without finding
// a ret instruction, the last value of S is returned.
// as an integer.

type opCode uint8

const (
	// Instructions altering R
	opN opCode = iota + 1
	opADD
	opSUB
	opMULT
	opDIV
	opMOD
	// Special instructions
	opRET
	// Jump instructions
	opJMPT
	opJMPF
	// Comparison instructions
	opEQ
	opNEQ
	opLT
	opLTE
	opGT
	opGTE
)

func (o opCode) String() string {
	names := []string{"N", "ADD", "SUB", "MULT", "DIV", "MOD", "RET", "JMPT", "JMPF", "EQ", "NEQ", "LT", "LTE", "GT", "GTE"}
	return names[int(o)-1]
}

func (o opCode) Alters() bool {
	return o <= opMOD
}

func (o opCode) IsSpecial() bool {
	return o == opRET
}

func (o opCode) IsJump() bool {
	return o == opJMPT || o == opJMPF
}

func (o opCode) Compares() bool {
	return o >= opEQ
}

type instruction struct {
	opCode opCode
	value  int
}

type program []*instruction

func invalid(s *scanner.Scanner, what, val string) (program, error) {
	return nil, fmt.Errorf("invalid %s in formula at %s: %q", what, s.Pos(), val)
}

func jumpTarget(s *scanner.Scanner, code []byte, chr byte) int {
	// look for matching chr
	offset := s.Pos().Offset
	paren := 0
	target := -1
	for ii, v := range code[offset:] {
		if v == '(' {
			paren++
		} else if v == ')' {
			paren--
			if paren < 0 {
				target = offset + ii
				break
			}
		} else if v == chr && paren == 0 {
			target = offset + ii
			break
		}
	}
	return target
}

func makeJump(s *scanner.Scanner, code []byte, p *program, op opCode, jumps map[int][]int, chr byte) {
	// end of conditional, put the placeholder for a jump
	// and complete it once we reach the matching chr. Store the
	// current position of the jump in its value, so
	// calculating the relative offset is quicker.
	pos := len(*p)
	inst := &instruction{opCode: op, value: pos}
	*p = append(*p, inst)
	target := jumpTarget(s, code, chr)
	jumps[target] = append(jumps[target], pos)
}

func resolveJumps(s *scanner.Scanner, p program, jumps map[int][]int) {
	// check for incomplete jumps to this location.
	// the pc should point at the next instruction
	// to be added and the jump is relative.
	pc := len(p)
	offset := s.Pos().Offset - 1
	for _, v := range jumps[offset] {
		inst := p[v]
		inst.value = pc - inst.value - 1
	}
	delete(jumps, offset)
}

func compileVmFormulaString(code string) (Formula, error) {
	return compileVmFormula([]byte(code))
}

func compileVmFormula(code []byte) (Formula, error) {
	p, err := vmCompile(code)
	if err != nil {
		return nil, err
	}
	p = vmOptimize(p)
	return makeVmFunc(p), nil
}

func vmCompile(code []byte) (program, error) {
	var s scanner.Scanner
	var err error
	s.Init(bytes.NewReader(code))
	s.Error = func(s *scanner.Scanner, msg string) {
		err = fmt.Errorf("error parsing plural formula %s: %s", s.Pos(), msg)
	}
	s.Mode = scanner.ScanIdents | scanner.ScanInts
	tok := s.Scan()
	var p program
	var op bytes.Buffer
	var logic bytes.Buffer
	jumps := make(map[int][]int)
	for tok != scanner.EOF && err == nil {
		switch tok {
		case scanner.Ident:
			if n := s.TokenText(); n != "n" {
				return invalid(&s, "ident", n)
			}
			if op.Len() > 0 {
				return invalid(&s, "ident", "RHS variables are not supported")
			}
			p = append(p, &instruction{opCode: opN})
		case scanner.Int:
			val, _ := strconv.Atoi(s.TokenText())
			if op.Len() == 0 {
				// return statement
				p = append(p, &instruction{opCode: opRET, value: val})
			} else {
				var opc opCode
				switch op.String() {
				case "+":
					opc = opADD
				case "-":
					opc = opSUB
				case "*":
					opc = opMULT
				case "/":
					opc = opDIV
				case "%":
					opc = opMOD
				case "==":
					opc = opEQ
				case "!=":
					opc = opNEQ
				case "<":
					opc = opLT
				case "<=":
					opc = opLTE
				case ">":
					opc = opGT
				case ">=":
					opc = opGTE
				default:
					return invalid(&s, "op", op.String())
				}
				p = append(p, &instruction{opCode: opc, value: val})
				op.Reset()
			}
		case '?':
			resolveJumps(&s, p, jumps)
			makeJump(&s, code, &p, opJMPF, jumps, ':')
		case ':':
			resolveJumps(&s, p, jumps)
		case '!', '=', '<', '>', '%':
			op.WriteRune(tok)
		case '&', '|':
			// logic operations
			if logic.Len() == 0 {
				logic.WriteRune(tok)
			} else if logic.Len() == 1 {
				b := logic.Bytes()[0]
				if b != byte(tok) {
					return invalid(&s, "token", string(tok))
				}
				if b == '&' {
					makeJump(&s, code, &p, opJMPF, jumps, ':')
				} else {
					makeJump(&s, code, &p, opJMPT, jumps, '?')
				}
				logic.Reset()
			} else {
				return invalid(&s, "token", string(tok))
			}
		case '(':
		case ')':
			resolveJumps(&s, p, jumps)
		default:
			return invalid(&s, "token", string(tok))
		}
		tok = s.Scan()
	}
	return p, nil
}

func removeInstructions(p program, start int, count int) program {
	p = append(p[:start], p[start+count:]...)
	// Check for jumps that might be affected by the removal
	for kk := start; kk >= 0; kk-- {
		if in := p[kk]; in.opCode.IsJump() && kk+in.value > start {
			in.value -= count
		}
	}
	return p
}

func vmOptimize(p program) program {
	// The optimizer is quite simple. Each pass is documented
	// at its beginning.

	//A first pass looks
	// for multiple comparison instructions that are preceeded
	// by exactly the same instructions and it removes the second
	// group of instructions.
	cmp := -1
	count := len(p)
	ii := 0
	for ; ii < count; ii++ {
		v := p[ii]
		if v.opCode.Compares() {
			if cmp >= 0 {
				delta := ii - cmp
				jj := cmp - 1
				for ; jj >= 0; jj-- {
					i1 := p[jj]
					if !i1.opCode.Alters() {
						break
					}
					i2 := p[jj+delta]
					if i1.opCode != i2.opCode || i1.value != i2.value {
						break
					}
				}
				equal := (cmp - 1) - jj
				if equal > 0 {
					ii -= equal
					count -= equal
					p = removeInstructions(p, ii, equal)
					continue
				}
			}
			cmp = ii
		}
	}
	// A second pass then looks for
	// instructions that set R = N when R is already
	// equal to N and it removes the second instruction.
	n := -1
	for ii = 0; ii < count; ii++ {
		v := p[ii]
		if v.opCode == opN {
			if n >= 0 {
				p = removeInstructions(p, ii, 1)
				ii--
				count--
				continue
			}
			n = ii
		} else if v.opCode.Alters() {
			n = -1
		}
	}
	// Third pass does two jump related optimizations. It looks for jumps
	// which end up in a jump of the same type, and adjusts the value to
	// make just one jump. It also checks jumps which end up in N instruction
	// since the R register might already be set with the required value,
	// meaning the N can be jumped over.
	for ii := 0; ii < count; ii++ {
		v := p[ii]
		if v.opCode.IsJump() {
			for true {
				t := ii + v.value + 1
				if t >= count {
					break
				}
				nv := p[t]
				if nv.opCode != v.opCode {
					break
				}
				v.value += nv.value
			}
			t := ii + v.value + 1
			if p[t].opCode == opN {
				// find opN before ii
				ni := -1
				for jj := ii - 1; jj >= 0; jj-- {
					if p[jj].opCode == opN {
						ni = jj
						break
					}
				}
				if ni >= 0 {
					end := ii - ni
					equal := 1
					for jj := 1; jj < end; jj++ {
						i1, i2 := p[ni+jj], p[t+jj]
						if i1.opCode.Alters() && i1.opCode == i2.opCode && i1.value == i2.value {
							equal++
						} else {
							if !i1.opCode.Compares() || !i2.opCode.Compares() {
								equal = 0
							}
							break
						}
					}
					v.value += equal
				}
			}
		}
	}
	return p
}

func makeVmFunc(p program) Formula {
	count := len(p)
	return func(n int) int {
		return vmExec(p, count, n)
	}
}

func vmExec(p program, count int, n int) int {
	var R int
	var S bool
	for ii := 0; ii < count; ii++ {
		i := p[ii]
		switch i.opCode {
		case opN:
			R = n
		case opADD:
			R += i.value
		case opSUB:
			R -= i.value
		case opMULT:
			R *= i.value
		case opDIV:
			R /= i.value
		case opMOD:
			R %= i.value
		case opRET:
			return i.value
		case opJMPT:
			if S {
				ii += i.value
			}
		case opJMPF:
			if !S {
				ii += i.value
			}
		case opEQ:
			S = R == i.value
		case opNEQ:
			S = R != i.value
		case opLT:
			S = R < i.value
		case opLTE:
			S = R <= i.value
		case opGT:
			S = R > i.value
		case opGTE:
			S = R >= i.value
		}
	}
	return bint(S)
}
<|endoftext|>


<commit_before>package revocations
import (
	"testing"
	"github.com/leanovate/microzon-auth-go/config"
	"github.com/leanovate/microzon-auth-go/logging"
	"github.com/leanovate/microzon-auth-go/store/memory_backend"
	"time"
	"github.com/leanovate/microzon-auth-go/common"
)

func BenchmarkRevocationsManagerFill(b *testing.B) {
	storeConfig := config.NewStoreConfig(logging.NewSimpleLoggerNull())
	store, _ := memory_backend.NewMemoryStore(storeConfig, logging.NewSimpleLoggerNull())

	revocations, _ := NewRevocationsManager(store, logging.NewSimpleLoggerNull())

	now := time.Now()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		revocations.AddRevocation(common.RawSha256FromData("data"), now)
	}
}<commit_msg>Made bench comparable<commit_after>package revocations
import (
	"testing"
	"github.com/leanovate/microzon-auth-go/config"
	"github.com/leanovate/microzon-auth-go/logging"
	"github.com/leanovate/microzon-auth-go/store/memory_backend"
	"time"
	"github.com/leanovate/microzon-auth-go/common"
	"fmt"
)

func BenchmarkRevocationsManagerFill(b *testing.B) {
	storeConfig := config.NewStoreConfig(logging.NewSimpleLoggerNull())
	store, _ := memory_backend.NewMemoryStore(storeConfig, logging.NewSimpleLoggerNull())

	revocations, _ := NewRevocationsManager(store, logging.NewSimpleLoggerNull())

	now := time.Now()
	hashes := make([]common.RawSha256, b.N)
	for i := 0; i < b.N; i++ {
		hashes[i] = common.RawSha256FromData(fmt.Sprintf("data%d", i))
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		revocations.AddRevocation(hashes[i], now)
		if !revocations.IsRevoked(hashes[i]) {
			b.Fail()
		}
	}
}<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

const (
	// TODO(crassirostris): Once test is stable, decrease allowed loses
	loadTestMaxAllowedLostFraction    = 0.05
	loadTestMaxAllowedFluentdRestarts = 1
)

// TODO(crassirostris): Remove Flaky once test is stable
var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:StackdriverLogging]", func() {
	f := framework.NewDefaultFramework("gcl-logging-load")

	It("should create a constant load with long-living pods and ensure logs delivery", func() {
		gclLogsProvider, err := newGclLogsProvider(f)
		framework.ExpectNoError(err, "Failed to create GCL logs provider")

		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
		nodeCount := len(nodes)
		podCount := 30 * nodeCount
		loggingDuration := 10 * time.Minute
		linesPerSecond := 1000 * nodeCount
		linesPerPod := linesPerSecond * int(loggingDuration.Seconds()) / podCount
		ingestionTimeout := 30 * time.Minute

		By("Running logs generator pods")
		pods := []*loggingPod{}
		for podIdx := 0; podIdx < podCount; podIdx++ {
			node := nodes[podIdx%len(nodes)]
			podName := fmt.Sprintf("logs-generator-%d-%d", linesPerPod, podIdx)
			pods = append(pods, createLoggingPod(f, podName, node.Name, linesPerPod, loggingDuration))

			defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
		}

		By("Waiting for pods to succeed")
		time.Sleep(loggingDuration)

		By("Waiting for all log lines to be ingested")
		config := &loggingTestConfig{
			LogsProvider:              gclLogsProvider,
			Pods:                      pods,
			IngestionTimeout:          ingestionTimeout,
			MaxAllowedLostFraction:    loadTestMaxAllowedLostFraction,
			MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
		}
		err = waitForFullLogsIngestion(f, config)
		if err != nil {
			framework.Failf("Failed to ingest logs: %v", err)
		} else {
			framework.Logf("Successfully ingested all logs")
		}
	})

	It("should create a constant load with short-living pods and ensure logs delivery", func() {
		gclLogsProvider, err := newGclLogsProvider(f)
		framework.ExpectNoError(err, "Failed to create GCL logs provider")

		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
		maxPodCount := 10
		jobDuration := 1 * time.Minute
		linesPerPodPerSecond := 100
		testDuration := 10 * time.Minute
		ingestionTimeout := 30 * time.Minute

		podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount))
		podRunCount := int(testDuration.Seconds())/int(podRunDelay.Seconds()) - 1
		linesPerPod := linesPerPodPerSecond * int(jobDuration.Seconds())

		By("Running short-living pods")
		pods := []*loggingPod{}
		for runIdx := 0; runIdx < podRunCount; runIdx++ {
			for nodeIdx, node := range nodes {
				podName := fmt.Sprintf("job-logs-generator-%d-%d-%d-%d", maxPodCount, linesPerPod, runIdx, nodeIdx)
				pods = append(pods, createLoggingPod(f, podName, node.Name, linesPerPod, jobDuration))

				defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
			}
			time.Sleep(podRunDelay)
		}

		By("Waiting for the last pods to finish")
		time.Sleep(jobDuration)

		By("Waiting for all log lines to be ingested")
		config := &loggingTestConfig{
			LogsProvider:              gclLogsProvider,
			Pods:                      pods,
			IngestionTimeout:          ingestionTimeout,
			MaxAllowedLostFraction:    loadTestMaxAllowedLostFraction,
			MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
		}
		err = waitForFullLogsIngestion(f, config)
		if err != nil {
			framework.Failf("Failed to ingest logs: %v", err)
		} else {
			framework.Logf("Successfully ingested all logs")
		}
	})
})
<commit_msg>Lower allowed loss limit for Stackdriver Logging load tests<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"time"

	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
)

const (
	loadTestMaxAllowedLostFraction    = 0.01
	loadTestMaxAllowedFluentdRestarts = 1
)

var _ = framework.KubeDescribe("Cluster level logging using GCL [Feature:StackdriverLogging]", func() {
	f := framework.NewDefaultFramework("gcl-logging-load")

	It("should create a constant load with long-living pods and ensure logs delivery", func() {
		gclLogsProvider, err := newGclLogsProvider(f)
		framework.ExpectNoError(err, "Failed to create GCL logs provider")

		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
		nodeCount := len(nodes)
		podCount := 30 * nodeCount
		loggingDuration := 10 * time.Minute
		linesPerSecond := 1000 * nodeCount
		linesPerPod := linesPerSecond * int(loggingDuration.Seconds()) / podCount
		ingestionTimeout := 30 * time.Minute

		By("Running logs generator pods")
		pods := []*loggingPod{}
		for podIdx := 0; podIdx < podCount; podIdx++ {
			node := nodes[podIdx%len(nodes)]
			podName := fmt.Sprintf("logs-generator-%d-%d", linesPerPod, podIdx)
			pods = append(pods, createLoggingPod(f, podName, node.Name, linesPerPod, loggingDuration))

			defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
		}

		By("Waiting for pods to succeed")
		time.Sleep(loggingDuration)

		By("Waiting for all log lines to be ingested")
		config := &loggingTestConfig{
			LogsProvider:              gclLogsProvider,
			Pods:                      pods,
			IngestionTimeout:          ingestionTimeout,
			MaxAllowedLostFraction:    loadTestMaxAllowedLostFraction,
			MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
		}
		err = waitForFullLogsIngestion(f, config)
		if err != nil {
			framework.Failf("Failed to ingest logs: %v", err)
		} else {
			framework.Logf("Successfully ingested all logs")
		}
	})

	It("should create a constant load with short-living pods and ensure logs delivery", func() {
		gclLogsProvider, err := newGclLogsProvider(f)
		framework.ExpectNoError(err, "Failed to create GCL logs provider")

		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet).Items
		maxPodCount := 10
		jobDuration := 1 * time.Minute
		linesPerPodPerSecond := 100
		testDuration := 10 * time.Minute
		ingestionTimeout := 30 * time.Minute

		podRunDelay := time.Duration(int64(jobDuration) / int64(maxPodCount))
		podRunCount := int(testDuration.Seconds())/int(podRunDelay.Seconds()) - 1
		linesPerPod := linesPerPodPerSecond * int(jobDuration.Seconds())

		By("Running short-living pods")
		pods := []*loggingPod{}
		for runIdx := 0; runIdx < podRunCount; runIdx++ {
			for nodeIdx, node := range nodes {
				podName := fmt.Sprintf("job-logs-generator-%d-%d-%d-%d", maxPodCount, linesPerPod, runIdx, nodeIdx)
				pods = append(pods, createLoggingPod(f, podName, node.Name, linesPerPod, jobDuration))

				defer f.PodClient().Delete(podName, &meta_v1.DeleteOptions{})
			}
			time.Sleep(podRunDelay)
		}

		By("Waiting for the last pods to finish")
		time.Sleep(jobDuration)

		By("Waiting for all log lines to be ingested")
		config := &loggingTestConfig{
			LogsProvider:              gclLogsProvider,
			Pods:                      pods,
			IngestionTimeout:          ingestionTimeout,
			MaxAllowedLostFraction:    loadTestMaxAllowedLostFraction,
			MaxAllowedFluentdRestarts: loadTestMaxAllowedFluentdRestarts,
		}
		err = waitForFullLogsIngestion(f, config)
		if err != nil {
			framework.Failf("Failed to ingest logs: %v", err)
		} else {
			framework.Logf("Successfully ingested all logs")
		}
	})
})
<|endoftext|>


<commit_before>package main

import (
	"cli"
	"log"
	"net"
	//"time"

	"telnet"
)

const (
	cmdSE   = 240
	cmdSB   = 250
	cmdWill = 251
	cmdWont = 252
	cmdDo   = 253
	cmdDont = 254
	cmdIAC  = 255
)

const (
	optEcho           = 1
	optSupressGoAhead = 3
	optNaws           = 31 // rfc1073
	optLinemode       = 34
)

func listenTelnet(addr string, cliServer *cli.Server) {

	handler := func(conn net.Conn) {
		handleTelnet(conn, cliServer)
	}

	telnetServer := telnet.Server{Addr: addr, Handler: handler}

	log.Printf("serving telnet on TCP %s", addr)

	if err := telnetServer.ListenAndServe(); err != nil {
		log.Fatalf("telnet server on address %s: error: %s", addr, err)
	}
}

func handleTelnet(conn net.Conn, cliServer *cli.Server) {
	defer conn.Close()

	log.Printf("handleTelnet: new telnet connection from: %s", conn.RemoteAddr())

	charMode(conn)

	cliClient := cli.NewClient(conn)

	go cli.InputLoop(cliServer, cliClient)

	cli.OutputLoop(cliClient)

	log.Printf("handleTelnet: terminating connection: remote=%s", conn.RemoteAddr())
}

func charMode(conn net.Conn) {
	log.Printf("charMode: entering telnet character mode")
	cmd := []byte{cmdIAC, cmdWill, optEcho, cmdIAC, cmdWill, optSupressGoAhead, cmdIAC, cmdDont, optLinemode}
	if wr, err := conn.Write(cmd); err != nil {
		log.Printf("charMode: len=%d err=%v", wr, err)
	}
}
<commit_msg>Telnet nop.<commit_after>package main

import (
	"cli"
	"log"
	"net"
	//"time"

	"telnet"
)

// https://tools.ietf.org/html/rfc854 TELNET PROTOCOL SPECIFICATION
const (
	cmdSE   = 240
	cmdNOP  = 241
	cmdSB   = 250
	cmdWill = 251
	cmdWont = 252
	cmdDo   = 253
	cmdDont = 254
	cmdIAC  = 255
)

const (
	optEcho           = 1
	optSupressGoAhead = 3
	optNaws           = 31 // rfc1073
	optLinemode       = 34
)

func listenTelnet(addr string, cliServer *cli.Server) {

	handler := func(conn net.Conn) {
		handleTelnet(conn, cliServer)
	}

	telnetServer := telnet.Server{Addr: addr, Handler: handler}

	log.Printf("serving telnet on TCP %s", addr)

	if err := telnetServer.ListenAndServe(); err != nil {
		log.Fatalf("telnet server on address %s: error: %s", addr, err)
	}
}

func handleTelnet(conn net.Conn, cliServer *cli.Server) {
	defer conn.Close()

	log.Printf("handleTelnet: new telnet connection from: %s", conn.RemoteAddr())

	charMode(conn)

	cliClient := cli.NewClient(conn)

	go cli.InputLoop(cliServer, cliClient)

	cli.OutputLoop(cliClient)

	log.Printf("handleTelnet: terminating connection: remote=%s", conn.RemoteAddr())
}

func charMode(conn net.Conn) {
	log.Printf("charMode: entering telnet character mode")
	cmd := []byte{cmdIAC, cmdWill, optEcho, cmdIAC, cmdWill, optSupressGoAhead, cmdIAC, cmdDont, optLinemode}
	if wr, err := conn.Write(cmd); err != nil {
		log.Printf("charMode: len=%d err=%v", wr, err)
	}
}
<|endoftext|>


<commit_before>package renter

import (
	"bytes"
	"crypto/rand"
	"strconv"
	"testing"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/hostdb"
)

// TestRepair tests that the repair method can repeatedly improve the
// redundancy of an unavailable file until it becomes available.
func TestRepair(t *testing.T) {
	// generate data
	const dataSize = 777
	data := make([]byte, dataSize)
	rand.Read(data)

	// create Reed-Solomon encoder
	rsc, err := NewRSCode(8, 2)
	if err != nil {
		t.Fatal(err)
	}

	// create hosts
	const pieceSize = 10
	hosts := make([]hostdb.Uploader, rsc.NumPieces())
	for i := range hosts {
		hosts[i] = &testHost{
			ip:       modules.NetAddress(strconv.Itoa(i)),
			failRate: 2, // 50% failure rate
		}
	}
	// make one host always fail
	hosts[0].(*testHost).failRate = 1

	// upload data to hosts
	f := newFile("foo", rsc, pieceSize, dataSize)
	r := bytes.NewReader(data)
	for chunk, pieces := range f.incompleteChunks() {
		err = f.repair(chunk, pieces, r, hosts)
		if err != nil {
			t.Fatal(err)
		}
	}

	// file should not be available after first pass
	if f.available() {
		t.Fatalf("file should not be available: %v%%", f.uploadProgress())
	}

	// repair until file becomes available
	const maxAttempts = 20
	for i := 0; i < maxAttempts; i++ {
		for chunk, pieces := range f.incompleteChunks() {
			err = f.repair(chunk, pieces, r, hosts)
			if err != nil {
				t.Fatal(err)
			}
		}
		if f.available() {
			break
		}
	}
	if !f.available() {
		t.Fatalf("file not repaired to availability after %v attempts: %v", maxAttempts, err)
	}
}
<commit_msg>add TestOfflineChunks<commit_after>package renter

import (
	"bytes"
	"crypto/rand"
	"reflect"
	"strconv"
	"testing"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/renter/hostdb"
	"github.com/NebulousLabs/Sia/types"
)

// TestRepair tests that the repair method can repeatedly improve the
// redundancy of an unavailable file until it becomes available.
func TestRepair(t *testing.T) {
	// generate data
	const dataSize = 777
	data := make([]byte, dataSize)
	rand.Read(data)

	// create Reed-Solomon encoder
	rsc, err := NewRSCode(8, 2)
	if err != nil {
		t.Fatal(err)
	}

	// create hosts
	const pieceSize = 10
	hosts := make([]hostdb.Uploader, rsc.NumPieces())
	for i := range hosts {
		hosts[i] = &testHost{
			ip:       modules.NetAddress(strconv.Itoa(i)),
			failRate: 2, // 50% failure rate
		}
	}
	// make one host always fail
	hosts[0].(*testHost).failRate = 1

	// upload data to hosts
	f := newFile("foo", rsc, pieceSize, dataSize)
	r := bytes.NewReader(data)
	for chunk, pieces := range f.incompleteChunks() {
		err = f.repair(chunk, pieces, r, hosts)
		if err != nil {
			t.Fatal(err)
		}
	}

	// file should not be available after first pass
	if f.available() {
		t.Fatalf("file should not be available: %v%%", f.uploadProgress())
	}

	// repair until file becomes available
	const maxAttempts = 20
	for i := 0; i < maxAttempts; i++ {
		for chunk, pieces := range f.incompleteChunks() {
			err = f.repair(chunk, pieces, r, hosts)
			if err != nil {
				t.Fatal(err)
			}
		}
		if f.available() {
			break
		}
	}
	if !f.available() {
		t.Fatalf("file not repaired to availability after %v attempts: %v", maxAttempts, err)
	}
}

// offlineHostDB is a mocked hostDB, used for testing the offlineChunks method
// of the file type. It is implemented as a map from NetAddresses to booleans,
// where the bool indicates whether the host is active.
type offlineHostDB map[modules.NetAddress]bool

func (hdb offlineHostDB) ActiveHosts() (hosts []modules.HostSettings) {
	for addr, active := range hdb {
		if active {
			hosts = append(hosts, modules.HostSettings{IPAddress: addr})
		}
	}
	return
}

func (hdb offlineHostDB) AllHosts() (hosts []modules.HostSettings) {
	for addr := range hdb {
		hosts = append(hosts, modules.HostSettings{IPAddress: addr})
	}
	return
}

// stub implementations of hostDB methods
func (hdb offlineHostDB) AveragePrice() types.Currency {
	return types.NewCurrency64(0)
}
func (hdb offlineHostDB) NewPool(uint64, types.BlockHeight) (hostdb.HostPool, error) {
	return nil, nil
}
func (hdb offlineHostDB) Renew(types.FileContractID, types.BlockHeight) (types.FileContractID, error) {
	return types.FileContractID{}, nil
}

// TestOfflineChunks tests the offlineChunks method of the file type.
func TestOfflineChunks(t *testing.T) {
	// Create a mock hostdb.
	hdb := &offlineHostDB{
		"foo": false,
		"bar": false,
		"baz": true,
	}
	rsc, _ := NewRSCode(1, 1)
	f := &file{
		erasureCode: rsc,
		contracts: map[types.FileContractID]fileContract{
			{0}: {IP: "foo", Pieces: []pieceData{{0, 0, 0}, {1, 0, 0}}},
			{1}: {IP: "bar", Pieces: []pieceData{{0, 1, 0}}},
			{2}: {IP: "baz", Pieces: []pieceData{{1, 1, 0}}},
		},
	}

	// pieces 0.0, 0.1, and 1.0 are offline. Since redundancy is 1,
	// offlineChunks should report only chunk 0 as needing repair.
	expChunks := map[uint64][]uint64{
		0: {0, 1},
	}
	chunks := f.offlineChunks(hdb)
	if !reflect.DeepEqual(chunks, expChunks) {
		t.Fatalf("offlineChunks did not return correct chunks: expected %v, got %v", expChunks, chunks)
	}
}
<|endoftext|>


<commit_before>package tools

import (
	"bytes"
	"encoding/csv"
	"github.com/scascketta/capmetricsd/Godeps/_workspace/src/github.com/boltdb/bolt"
	"github.com/scascketta/capmetricsd/Godeps/_workspace/src/github.com/golang/protobuf/proto"
	"github.com/scascketta/capmetricsd/daemon/agency"
	"github.com/scascketta/capmetricsd/daemon/agency/capmetro"
	"log"
	"os"
	"strconv"
	"time"
)

func readBoltData(db *bolt.DB, min, max string) (*[]agency.VehicleLocation, error) {
	locations := []agency.VehicleLocation{}

	err := db.View(func(tx *bolt.Tx) error {
		topBucket := tx.Bucket([]byte(capmetro.BucketName))

		err := topBucket.ForEach(func(tripID, _ []byte) error {
			tripBucket := topBucket.Bucket(tripID)
			c := tripBucket.Cursor()

			for k, v := c.Seek([]byte(min)); k != nil && bytes.Compare(k, []byte(max)) <= 0; k, v = c.Next() {
				var loc agency.VehicleLocation
				if err := proto.Unmarshal(v, &loc); err != nil {
					return err
				}

				locations = append(locations, loc)
			}

			return nil
		})

		return err
	})

	return &locations, err
}

func writeData(dest string, locations *[]agency.VehicleLocation) error {
	log.Printf("Writing %d vehicle locations to %s.\n", len(*locations), dest)

	headers := []string{"vehicle_id", "timestamp", "speed", "route_id", "trip_id", "latitude", "longitude"}

	f, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer f.Close()

	w := csv.NewWriter(f)
	if err = w.Write(headers); err != nil {
		log.Println("Error writing CSV header record")
		return err
	}

	for _, loc := range *locations {
		t := time.Unix(loc.GetTimestamp(), 0).UTC()
		record := []string{
			loc.GetVehicleId(),
			t.Local().Format(Iso8601Format),
			strconv.FormatFloat(float64(loc.GetSpeed()), 'f', -1, 32),
			loc.GetRouteId(),
			loc.GetTripId(),
			strconv.FormatFloat(float64(loc.GetLatitude()), 'f', -1, 32),
			strconv.FormatFloat(float64(loc.GetLongitude()), 'f', -1, 32),
		}
		if err = w.Write(record); err != nil {
			log.Println("Error writing CSV records")
			return err
		}
	}

	return nil
}

func GetData(dbPath, dest string, min string, max string) error {
	log.Printf("Get data between %s and %s\n", min, max)

	log.Println("dbPath: ", dbPath)
	db, err := bolt.Open(dbPath, 0600, nil)
	if err != nil {
		return err
	}

	locations, err := readBoltData(db, min, max)
	if err != nil {
		return err
	}

	err = writeData(dest, locations)
	return err
}
<commit_msg>Flush CSV writer after writing last record then check for err<commit_after>package tools

import (
	"bytes"
	"encoding/csv"
	"github.com/scascketta/capmetricsd/Godeps/_workspace/src/github.com/boltdb/bolt"
	"github.com/scascketta/capmetricsd/Godeps/_workspace/src/github.com/golang/protobuf/proto"
	"github.com/scascketta/capmetricsd/daemon/agency"
	"github.com/scascketta/capmetricsd/daemon/agency/capmetro"
	"log"
	"os"
	"strconv"
	"time"
)

func readBoltData(db *bolt.DB, min, max string) (*[]agency.VehicleLocation, error) {
	locations := []agency.VehicleLocation{}

	err := db.View(func(tx *bolt.Tx) error {
		topBucket := tx.Bucket([]byte(capmetro.BucketName))

		err := topBucket.ForEach(func(tripID, _ []byte) error {
			tripBucket := topBucket.Bucket(tripID)
			c := tripBucket.Cursor()

			for k, v := c.Seek([]byte(min)); k != nil && bytes.Compare(k, []byte(max)) <= 0; k, v = c.Next() {
				var loc agency.VehicleLocation
				if err := proto.Unmarshal(v, &loc); err != nil {
					return err
				}

				locations = append(locations, loc)
			}

			return nil
		})

		return err
	})

	return &locations, err
}

func writeData(dest string, locations *[]agency.VehicleLocation) error {
	log.Printf("Writing %d vehicle locations to %s.\n", len(*locations), dest)

	headers := []string{"vehicle_id", "timestamp", "speed", "route_id", "trip_id", "latitude", "longitude"}

	f, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer f.Close()

	w := csv.NewWriter(f)
	if err = w.Write(headers); err != nil {
		log.Println("Error writing CSV header record")
		return err
	}

	for _, loc := range *locations {
		t := time.Unix(loc.GetTimestamp(), 0).UTC()
		record := []string{
			loc.GetVehicleId(),
			t.Local().Format(Iso8601Format),
			strconv.FormatFloat(float64(loc.GetSpeed()), 'f', -1, 32),
			loc.GetRouteId(),
			loc.GetTripId(),
			strconv.FormatFloat(float64(loc.GetLatitude()), 'f', -1, 32),
			strconv.FormatFloat(float64(loc.GetLongitude()), 'f', -1, 32),
		}
		if err = w.Write(record); err != nil {
			log.Println("Error writing CSV records")
			return err
		}
	}

	w.Flush()
	if err := w.Error(); err != nil {
		return err
	}

	return nil
}

func GetData(dbPath, dest string, min string, max string) error {
	log.Printf("Get data between %s and %s\n", min, max)

	log.Println("dbPath: ", dbPath)
	db, err := bolt.Open(dbPath, 0600, nil)
	if err != nil {
		return err
	}

	locations, err := readBoltData(db, min, max)
	if err != nil {
		return err
	}

	err = writeData(dest, locations)
	return err
}
<|endoftext|>


<commit_before>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"path/filepath"
	"testing"

	"github.com/docker/machine/libmachine/state"
	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/minikube/pkg/kapi"
	"k8s.io/minikube/test/integration/util"
)

func TestContainerd(t *testing.T) {
	if isTestNoneDriver(t) {
		t.Skip("Can't run containerd backend with none driver")
	}
	if shouldRunInParallel(t) {
		t.Parallel()
	}

	t.Run("GvisorUntrustedWorkload", testGvisorUntrustedWorkload)
	t.Run("GvisorRuntimeClass", testGvisorRuntimeClass)
	t.Run("GvisorRestart", testGvisorRestart)
}

func testGvisorUntrustedWorkload(t *testing.T) {
	p := profileName(t)
	if shouldRunInParallel(t) {
		t.Parallel()
	}
	mk := NewMinikubeRunner(t, p, "--wait=false")
	defer mk.TearDown(t)

	mk.MustRun("addons enable gvisor", true)

	t.Log("waiting for gvisor controller to come up")
	if err := waitForGvisorControllerRunning(p); err != nil {
		t.Fatalf("waiting for gvisor controller to be up: %v", err)
	}

	createUntrustedWorkload(t, p)
	t.Log("making sure untrusted workload is Running")
	if err := waitForUntrustedNginxRunning(p); err != nil {
		t.Fatalf("waiting for nginx to be up: %v", err)
	}
	deleteUntrustedWorkload(t, p)
}

func testGvisorRuntimeClass(t *testing.T) {
	p := profileName(t)
	if shouldRunInParallel(t) {
		t.Parallel()
	}
	mk := NewMinikubeRunner(t, p, "--wait=false")
	defer mk.TearDown(t)

	mk.MustRun("addons enable gvisor", true)

	t.Log("waiting for gvisor controller to come up")
	if err := waitForGvisorControllerRunning(p); err != nil {
		t.Fatalf("waiting for gvisor controller to be up: %v", err)
	}

	createGvisorWorkload(t, p)
	t.Log("making sure gvisor workload is Running")
	if err := waitForGvisorNginxRunning(p); err != nil {
		t.Fatalf("waiting for nginx to be up: %v", err)
	}
	deleteGvisorWorkload(t, p)
}

func testGvisorRestart(t *testing.T) {
	p := profileName(t)
	if shouldRunInParallel(t) {
		t.Parallel()
	}
	mk := NewMinikubeRunner(t, p, "--wait=false")
	defer mk.TearDown(t)

	mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
	mk.MustRun("cache add gcr.io/k8s-minikube/gvisor-addon:latest")
	mk.MustRun("addons enable gvisor")

	t.Log("waiting for gvisor controller to come up")
	if err := waitForGvisorControllerRunning(p); err != nil {
		t.Errorf("waiting for gvisor controller to be up: %v", err)
	}

	createUntrustedWorkload(t, p)
	t.Log("making sure untrusted workload is Running")
	if err := waitForUntrustedNginxRunning(p); err != nil {
		t.Errorf("waiting for nginx to be up: %v", err)
	}
	deleteUntrustedWorkload(t, p)

	mk.MustRun("delete")
	mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
	mk.CheckStatus(state.Running.String())

	t.Log("waiting for gvisor controller to come up")
	if err := waitForGvisorControllerRunning(p); err != nil {
		t.Errorf("waiting for gvisor controller to be up: %v", err)
	}

	createUntrustedWorkload(t, p)
	t.Log("making sure untrusted workload is Running")
	if err := waitForUntrustedNginxRunning(p); err != nil {
		t.Errorf("waiting for nginx to be up: %v", err)
	}
	deleteUntrustedWorkload(t, p)
}

func createUntrustedWorkload(t *testing.T, profile string) {
	kr := util.NewKubectlRunner(t, profile)
	untrustedPath := filepath.Join(*testdataDir, "nginx-untrusted.yaml")
	t.Log("creating pod with untrusted workload annotation")
	if _, err := kr.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil {
		t.Fatalf("creating untrusted nginx resource: %v", err)
	}
}

func deleteUntrustedWorkload(t *testing.T, profile string) {
	kr := util.NewKubectlRunner(t, profile)
	untrustedPath := filepath.Join(*testdataDir, "nginx-untrusted.yaml")
	if _, err := kr.RunCommand([]string{"delete", "-f", untrustedPath}); err != nil {
		t.Logf("error deleting untrusted nginx resource: %v", err)
	}
}

func createGvisorWorkload(t *testing.T, profile string) {
	kr := util.NewKubectlRunner(t, profile)
	gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
	t.Log("creating pod with gvisor workload annotation")
	if _, err := kr.RunCommand([]string{"replace", "-f", gvisorPath, "--force"}); err != nil {
		t.Fatalf("creating gvisor nginx resource: %v", err)
	}
}

func deleteGvisorWorkload(t *testing.T, profile string) {
	kr := util.NewKubectlRunner(t, profile)
	gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
	if _, err := kr.RunCommand([]string{"delete", "-f", gvisorPath}); err != nil {
		t.Logf("error deleting gvisor nginx resource: %v", err)
	}
}

// waitForGvisorControllerRunning waits for the gvisor controller pod to be running.
func waitForGvisorControllerRunning(p string) error {
	client, err := kapi.Client(p)
	if err != nil {
		return errors.Wrap(err, "getting kubernetes client")
	}

	selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"}))
	if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
		return errors.Wrap(err, "waiting for gvisor controller pod to stabilize")
	}
	return nil
}

// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start
// running.
func waitForUntrustedNginxRunning(miniProfile string) error {
	client, err := kapi.Client(miniProfile)
	if err != nil {
		return errors.Wrap(err, "getting kubernetes client")
	}

	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "untrusted": "true"}))
	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
		return errors.Wrap(err, "waiting for nginx pods")
	}
	return nil
}

// waitForGvisorNginxRunning waits for the nginx pod with gvisor runtime class
// to start running.
func waitForGvisorNginxRunning(miniProfile string) error {
	client, err := kapi.Client(miniProfile)
	if err != nil {
		return errors.Wrap(err, "getting kubernetes client")
	}

	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "runtime": "gvisor"}))
	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
		return errors.Wrap(err, "waiting for nginx pods")
	}
	return nil
}
<commit_msg>Update gVisor tests to actually start a cluster.<commit_after>// +build integration

/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"path/filepath"
	"testing"

	"github.com/docker/machine/libmachine/state"
	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/minikube/pkg/kapi"
	"k8s.io/minikube/test/integration/util"
)

// TestGvisorWorkload tests the gVisor addon for pods with the untrusted
// workload annotation and gvisor runtime class.
func TestGvisorWorkload(t *testing.T) {
	if isTestNoneDriver(t) {
		t.Skip("Can't run containerd backend with none driver")
	}

	p := profileName(t)
	mk := NewMinikubeRunner(t, p, "--wait=false")
	defer mk.TearDown(t)

	mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
	mk.MustRun("cache add gcr.io/k8s-minikube/gvisor-addon:latest")
	// NOTE: addons are enabled globally.
	mk.MustRun("addons enable gvisor")

	t.Log("waiting for gvisor controller to come up")
	if err := waitForGvisorControllerRunning(p); err != nil {
		t.Fatalf("waiting for gvisor controller to be up: %v", err)
	}

	// Test a pod with the untrusted workload annotation.
	createUntrustedWorkload(t, p)
	t.Log("making sure untrusted workload is Running")
	if err := waitForUntrustedNginxRunning(p); err != nil {
		t.Fatalf("waiting for untrusted-workload nginx to be up: %v", err)
	}
	deleteUntrustedWorkload(t, p)

	// Test a pod with the gvisor runtime class.
	createGvisorWorkload(t, p)
	t.Log("making sure gvisor workload is Running")
	if err := waitForGvisorNginxRunning(p); err != nil {
		t.Fatalf("waiting for gvisor nginx to be up: %v", err)
	}
	deleteGvisorWorkload(t, p)
}

// TestGvisorRestart tests the gVisor addon is functional after a VM restart.
func TestGvisorRestart(t *testing.T) {
	if isTestNoneDriver(t) {
		t.Skip("Can't run containerd backend with none driver")
	}

	p := profileName(t)
	mk := NewMinikubeRunner(t, p, "--wait=false")
	defer mk.TearDown(t)

	mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
	mk.MustRun("cache add gcr.io/k8s-minikube/gvisor-addon:latest")
	// NOTE: addons are enabled globally.
	mk.MustRun("addons enable gvisor")

	t.Log("waiting for gvisor controller to come up")
	if err := waitForGvisorControllerRunning(p); err != nil {
		t.Errorf("waiting for gvisor controller to be up: %v", err)
	}

	createUntrustedWorkload(t, p)
	t.Log("making sure untrusted workload is Running")
	if err := waitForUntrustedNginxRunning(p); err != nil {
		t.Errorf("waiting for nginx to be up: %v", err)
	}
	deleteUntrustedWorkload(t, p)

	mk.MustRun("delete")
	mk.MustStart("--container-runtime=containerd", "--docker-opt containerd=/var/run/containerd/containerd.sock")
	mk.CheckStatus(state.Running.String())

	t.Log("waiting for gvisor controller to come up")
	if err := waitForGvisorControllerRunning(p); err != nil {
		t.Errorf("waiting for gvisor controller to be up: %v", err)
	}

	createUntrustedWorkload(t, p)
	t.Log("making sure untrusted workload is Running")
	if err := waitForUntrustedNginxRunning(p); err != nil {
		t.Errorf("waiting for nginx to be up: %v", err)
	}
	deleteUntrustedWorkload(t, p)
}

func createUntrustedWorkload(t *testing.T, profile string) {
	kr := util.NewKubectlRunner(t, profile)
	untrustedPath := filepath.Join(*testdataDir, "nginx-untrusted.yaml")
	t.Log("creating pod with untrusted workload annotation")
	if _, err := kr.RunCommand([]string{"replace", "-f", untrustedPath, "--force"}); err != nil {
		t.Fatalf("creating untrusted nginx resource: %v", err)
	}
}

func deleteUntrustedWorkload(t *testing.T, profile string) {
	kr := util.NewKubectlRunner(t, profile)
	untrustedPath := filepath.Join(*testdataDir, "nginx-untrusted.yaml")
	if _, err := kr.RunCommand([]string{"delete", "-f", untrustedPath}); err != nil {
		t.Logf("error deleting untrusted nginx resource: %v", err)
	}
}

func createGvisorWorkload(t *testing.T, profile string) {
	kr := util.NewKubectlRunner(t, profile)
	gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
	t.Log("creating pod with gvisor workload annotation")
	if _, err := kr.RunCommand([]string{"replace", "-f", gvisorPath, "--force"}); err != nil {
		t.Fatalf("creating gvisor nginx resource: %v", err)
	}
}

func deleteGvisorWorkload(t *testing.T, profile string) {
	kr := util.NewKubectlRunner(t, profile)
	gvisorPath := filepath.Join(*testdataDir, "nginx-gvisor.yaml")
	if _, err := kr.RunCommand([]string{"delete", "-f", gvisorPath}); err != nil {
		t.Logf("error deleting gvisor nginx resource: %v", err)
	}
}

// waitForGvisorControllerRunning waits for the gvisor controller pod to be running.
func waitForGvisorControllerRunning(p string) error {
	client, err := kapi.Client(p)
	if err != nil {
		return errors.Wrap(err, "getting kubernetes client")
	}

	selector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "gvisor"}))
	if err := kapi.WaitForPodsWithLabelRunning(client, "kube-system", selector); err != nil {
		return errors.Wrap(err, "waiting for gvisor controller pod to stabilize")
	}
	return nil
}

// waitForUntrustedNginxRunning waits for the untrusted nginx pod to start
// running.
func waitForUntrustedNginxRunning(miniProfile string) error {
	client, err := kapi.Client(miniProfile)
	if err != nil {
		return errors.Wrap(err, "getting kubernetes client")
	}

	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "untrusted": "true"}))
	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
		return errors.Wrap(err, "waiting for nginx pods")
	}
	return nil
}

// waitForGvisorNginxRunning waits for the nginx pod with gvisor runtime class
// to start running.
func waitForGvisorNginxRunning(miniProfile string) error {
	client, err := kapi.Client(miniProfile)
	if err != nil {
		return errors.Wrap(err, "getting kubernetes client")
	}

	selector := labels.SelectorFromSet(labels.Set(map[string]string{"run": "nginx", "runtime": "gvisor"}))
	if err := kapi.WaitForPodsWithLabelRunning(client, "default", selector); err != nil {
		return errors.Wrap(err, "waiting for nginx pods")
	}
	return nil
}
<|endoftext|>


<commit_before>package serverlogger

import (
	"errors"
	"fmt"
	"log"
	"os"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"

	liblog "github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/logbuf"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/lib/srpc/serverutil"
	proto "github.com/Symantec/Dominator/proto/logger"
)

type loggerMapT struct {
	*serverutil.PerUserMethodLimiter
	sync.Mutex
	loggerMap map[string]*Logger
}

type grabWriter struct {
	data []byte
}

var loggerMap *loggerMapT = &loggerMapT{
	loggerMap: make(map[string]*Logger),
	PerUserMethodLimiter: serverutil.NewPerUserMethodLimiter(
		map[string]uint{
			"Debug":         1,
			"Print":         1,
			"SetDebugLevel": 1,
			"Watch":         1,
		}),
}

func init() {
	srpc.RegisterName("Logger", loggerMap)
}

func getCallerName(depth int) string {
	if pc, _, _, ok := runtime.Caller(depth); !ok {
		return "UNKNOWN"
	} else if fi := runtime.FuncForPC(pc); fi == nil {
		return "UNKNOWN"
	} else if splitName := strings.Split(fi.Name(), "."); len(splitName) < 1 {
		return "UNKNOWN"
	} else {
		return splitName[len(splitName)-1]
	}
}

func (w *grabWriter) Write(p []byte) (int, error) {
	w.data = p
	return len(p), nil
}

func newLogger(name string, options logbuf.Options, flags int) *Logger {
	loggerMap.Lock()
	defer loggerMap.Unlock()
	if _, ok := loggerMap.loggerMap[name]; ok {
		panic("logger already exists: " + name)
	}
	circularBuffer := logbuf.NewWithOptions(options)
	logger := &Logger{
		circularBuffer: circularBuffer,
		flags:          flags,
		level:          int16(*initialLogDebugLevel),
		streamers:      make(map[*streamerType]struct{}),
	}
	if logger.level < -1 {
		logger.level = -1
	}
	logger.maxLevel = logger.level
	// Ensure this satisfies the published interface.
	var debugLogger liblog.FullDebugLogger
	debugLogger = logger
	_ = debugLogger
	loggerMap.loggerMap[name] = logger
	return logger
}

func (l *Logger) checkAuth(authInfo *srpc.AuthInformation) error {
	if authInfo.HaveMethodAccess {
		return nil
	}
	if accessChecker := l.accessChecker; accessChecker == nil {
		return errors.New("no access to resource")
	} else if accessChecker(getCallerName(3), authInfo) {
		return nil
	} else {
		return errors.New("no access to resource")
	}
}

func (l *Logger) debug(level int16, v ...interface{}) {
	if l.maxLevel >= level {
		l.log(level, fmt.Sprint(v...), false)
	}
}

func (l *Logger) debugf(level int16, format string, v ...interface{}) {
	if l.maxLevel >= level {
		l.log(level, fmt.Sprintf(format, v...), false)
	}
}

func (l *Logger) debugln(level int16, v ...interface{}) {
	if l.maxLevel >= level {
		l.log(level, fmt.Sprintln(v...), false)
	}
}

func (l *Logger) fatals(msg string) {
	l.log(-1, msg, true)
	os.Exit(1)
}

func (l *Logger) log(level int16, msg string, dying bool) {
	buffer := &grabWriter{}
	rawLogger := log.New(buffer, "", l.flags)
	rawLogger.Output(4, msg)
	if l.level >= level {
		l.circularBuffer.Write(buffer.data)
	}
	recalculateLevels := false
	l.mutex.Lock()
	defer l.mutex.Unlock()
	for streamer := range l.streamers {
		if streamer.debugLevel >= level &&
			(streamer.includeRegex == nil ||
				streamer.includeRegex.Match(buffer.data)) &&
			(streamer.excludeRegex == nil ||
				!streamer.excludeRegex.Match(buffer.data)) {
			select {
			case streamer.output <- buffer.data:
			default:
				delete(l.streamers, streamer)
				close(streamer.output)
				recalculateLevels = true
			}
		}
	}
	if dying {
		for streamer := range l.streamers {
			delete(l.streamers, streamer)
			close(streamer.output)
		}
		l.circularBuffer.Flush()
		time.Sleep(time.Millisecond * 10)
	} else if recalculateLevels {
		l.updateMaxLevel()
	}
}

func (l *Logger) makeStreamer(request proto.WatchRequest) (
	*streamerType, error) {
	if request.DebugLevel < -1 {
		request.DebugLevel = -1
	}
	streamer := &streamerType{debugLevel: request.DebugLevel}
	if request.ExcludeRegex != "" {
		var err error
		streamer.excludeRegex, err = regexp.Compile(request.ExcludeRegex)
		if err != nil {
			return nil, err
		}
	}
	if request.IncludeRegex != "" {
		var err error
		streamer.includeRegex, err = regexp.Compile(request.IncludeRegex)
		if err != nil {
			return nil, err
		}
	}
	return streamer, nil
}

func (l *Logger) panics(msg string) {
	l.log(-1, msg, true)
	panic(msg)
}

func (l *Logger) prints(msg string) {
	l.log(-1, msg, false)
}

func (l *Logger) setLevel(maxLevel int16) {
	if maxLevel < -1 {
		maxLevel = -1
	}
	l.level = maxLevel
	l.mutex.Lock()
	l.updateMaxLevel()
	l.mutex.Unlock()
}

func (l *Logger) updateMaxLevel() {
	maxLevel := l.level
	for streamer := range l.streamers {
		if streamer.debugLevel > maxLevel {
			maxLevel = streamer.debugLevel
		}
	}
	l.maxLevel = maxLevel
}

func (l *Logger) watch(conn *srpc.Conn, streamer *streamerType) {
	channel := make(chan []byte, 256)
	streamer.output = channel
	l.mutex.Lock()
	l.streamers[streamer] = struct{}{}
	l.updateMaxLevel()
	l.mutex.Unlock()
	timer := time.NewTimer(time.Millisecond * 100)
	flushPending := false
	closeNotifier := conn.GetCloseNotifier()
	for keepGoing := true; keepGoing; {
		select {
		case <-closeNotifier:
			keepGoing = false
			break
		case data, ok := <-channel:
			if !ok {
				keepGoing = false
				break
			}
			if _, err := conn.Write(data); err != nil {
				keepGoing = false
				break
			}
			if !flushPending {
				timer.Reset(time.Millisecond * 100)
				flushPending = true
			}
		case <-timer.C:
			if conn.Flush() != nil {
				keepGoing = false
				break
			}
			flushPending = false
		}
	}
	if flushPending {
		conn.Flush()
	}
	l.mutex.Lock()
	delete(l.streamers, streamer)
	l.updateMaxLevel()
	l.mutex.Unlock()
	// Drain the channel.
	for {
		select {
		case <-channel:
		default:
			return
		}
	}
}

func (t *loggerMapT) getLogger(name string,
	authInfo *srpc.AuthInformation) (*Logger, error) {
	loggerMap.Lock()
	defer loggerMap.Unlock()
	if logger, ok := loggerMap.loggerMap[name]; !ok {
		return nil, errors.New("unknown logger: " + name)
	} else if err := logger.checkAuth(authInfo); err != nil {
		return nil, err
	} else {
		return logger, nil
	}
}

func (t *loggerMapT) Debug(conn *srpc.Conn,
	request proto.DebugRequest, reply *proto.DebugResponse) error {
	authInfo := conn.GetAuthInformation()
	if logger, err := t.getLogger(request.Name, authInfo); err != nil {
		return err
	} else {
		logger.Debugf(request.Level, "Logger.Debug(%d): %s\n",
			request.Level, strings.Join(request.Args, " "))
		return nil
	}
}

func (t *loggerMapT) Print(conn *srpc.Conn,
	request proto.PrintRequest,
	reply *proto.PrintResponse) error {
	authInfo := conn.GetAuthInformation()
	if logger, err := t.getLogger(request.Name, authInfo); err != nil {
		return err
	} else {
		logger.Println("Logger.Print():", strings.Join(request.Args, " "))
		return nil
	}
}

func (t *loggerMapT) SetDebugLevel(conn *srpc.Conn,
	request proto.SetDebugLevelRequest,
	reply *proto.SetDebugLevelResponse) error {
	authInfo := conn.GetAuthInformation()
	if logger, err := t.getLogger(request.Name, authInfo); err != nil {
		return err
	} else {
		logger.Printf("Logger.SetDebugLevel(%d)\n", request.Level)
		logger.SetLevel(request.Level)
		return nil
	}
}

func (t *loggerMapT) Watch(conn *srpc.Conn, decoder srpc.Decoder,
	encoder srpc.Encoder) error {
	var request proto.WatchRequest
	if err := decoder.Decode(&request); err != nil {
		return err
	}
	authInfo := conn.GetAuthInformation()
	if logger, err := t.getLogger(request.Name, authInfo); err != nil {
		return encoder.Encode(proto.WatchResponse{Error: err.Error()})
	} else if streamer, err := logger.makeStreamer(request); err != nil {
		return encoder.Encode(proto.WatchResponse{Error: err.Error()})
	} else {
		if err := encoder.Encode(proto.WatchResponse{}); err != nil {
			return err
		}
		if err := conn.Flush(); err != nil {
			return err
		}
		logger.watch(conn, streamer)
		return srpc.ErrorCloseClient
	}
}
<commit_msg>Switch lib/log/serverlogger to use connection Decode and Encode methods.<commit_after>package serverlogger

import (
	"errors"
	"fmt"
	"log"
	"os"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"time"

	liblog "github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/logbuf"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/lib/srpc/serverutil"
	proto "github.com/Symantec/Dominator/proto/logger"
)

type loggerMapT struct {
	*serverutil.PerUserMethodLimiter
	sync.Mutex
	loggerMap map[string]*Logger
}

type grabWriter struct {
	data []byte
}

var loggerMap *loggerMapT = &loggerMapT{
	loggerMap: make(map[string]*Logger),
	PerUserMethodLimiter: serverutil.NewPerUserMethodLimiter(
		map[string]uint{
			"Debug":         1,
			"Print":         1,
			"SetDebugLevel": 1,
			"Watch":         1,
		}),
}

func init() {
	srpc.RegisterName("Logger", loggerMap)
}

func getCallerName(depth int) string {
	if pc, _, _, ok := runtime.Caller(depth); !ok {
		return "UNKNOWN"
	} else if fi := runtime.FuncForPC(pc); fi == nil {
		return "UNKNOWN"
	} else if splitName := strings.Split(fi.Name(), "."); len(splitName) < 1 {
		return "UNKNOWN"
	} else {
		return splitName[len(splitName)-1]
	}
}

func (w *grabWriter) Write(p []byte) (int, error) {
	w.data = p
	return len(p), nil
}

func newLogger(name string, options logbuf.Options, flags int) *Logger {
	loggerMap.Lock()
	defer loggerMap.Unlock()
	if _, ok := loggerMap.loggerMap[name]; ok {
		panic("logger already exists: " + name)
	}
	circularBuffer := logbuf.NewWithOptions(options)
	logger := &Logger{
		circularBuffer: circularBuffer,
		flags:          flags,
		level:          int16(*initialLogDebugLevel),
		streamers:      make(map[*streamerType]struct{}),
	}
	if logger.level < -1 {
		logger.level = -1
	}
	logger.maxLevel = logger.level
	// Ensure this satisfies the published interface.
	var debugLogger liblog.FullDebugLogger
	debugLogger = logger
	_ = debugLogger
	loggerMap.loggerMap[name] = logger
	return logger
}

func (l *Logger) checkAuth(authInfo *srpc.AuthInformation) error {
	if authInfo.HaveMethodAccess {
		return nil
	}
	if accessChecker := l.accessChecker; accessChecker == nil {
		return errors.New("no access to resource")
	} else if accessChecker(getCallerName(3), authInfo) {
		return nil
	} else {
		return errors.New("no access to resource")
	}
}

func (l *Logger) debug(level int16, v ...interface{}) {
	if l.maxLevel >= level {
		l.log(level, fmt.Sprint(v...), false)
	}
}

func (l *Logger) debugf(level int16, format string, v ...interface{}) {
	if l.maxLevel >= level {
		l.log(level, fmt.Sprintf(format, v...), false)
	}
}

func (l *Logger) debugln(level int16, v ...interface{}) {
	if l.maxLevel >= level {
		l.log(level, fmt.Sprintln(v...), false)
	}
}

func (l *Logger) fatals(msg string) {
	l.log(-1, msg, true)
	os.Exit(1)
}

func (l *Logger) log(level int16, msg string, dying bool) {
	buffer := &grabWriter{}
	rawLogger := log.New(buffer, "", l.flags)
	rawLogger.Output(4, msg)
	if l.level >= level {
		l.circularBuffer.Write(buffer.data)
	}
	recalculateLevels := false
	l.mutex.Lock()
	defer l.mutex.Unlock()
	for streamer := range l.streamers {
		if streamer.debugLevel >= level &&
			(streamer.includeRegex == nil ||
				streamer.includeRegex.Match(buffer.data)) &&
			(streamer.excludeRegex == nil ||
				!streamer.excludeRegex.Match(buffer.data)) {
			select {
			case streamer.output <- buffer.data:
			default:
				delete(l.streamers, streamer)
				close(streamer.output)
				recalculateLevels = true
			}
		}
	}
	if dying {
		for streamer := range l.streamers {
			delete(l.streamers, streamer)
			close(streamer.output)
		}
		l.circularBuffer.Flush()
		time.Sleep(time.Millisecond * 10)
	} else if recalculateLevels {
		l.updateMaxLevel()
	}
}

func (l *Logger) makeStreamer(request proto.WatchRequest) (
	*streamerType, error) {
	if request.DebugLevel < -1 {
		request.DebugLevel = -1
	}
	streamer := &streamerType{debugLevel: request.DebugLevel}
	if request.ExcludeRegex != "" {
		var err error
		streamer.excludeRegex, err = regexp.Compile(request.ExcludeRegex)
		if err != nil {
			return nil, err
		}
	}
	if request.IncludeRegex != "" {
		var err error
		streamer.includeRegex, err = regexp.Compile(request.IncludeRegex)
		if err != nil {
			return nil, err
		}
	}
	return streamer, nil
}

func (l *Logger) panics(msg string) {
	l.log(-1, msg, true)
	panic(msg)
}

func (l *Logger) prints(msg string) {
	l.log(-1, msg, false)
}

func (l *Logger) setLevel(maxLevel int16) {
	if maxLevel < -1 {
		maxLevel = -1
	}
	l.level = maxLevel
	l.mutex.Lock()
	l.updateMaxLevel()
	l.mutex.Unlock()
}

func (l *Logger) updateMaxLevel() {
	maxLevel := l.level
	for streamer := range l.streamers {
		if streamer.debugLevel > maxLevel {
			maxLevel = streamer.debugLevel
		}
	}
	l.maxLevel = maxLevel
}

func (l *Logger) watch(conn *srpc.Conn, streamer *streamerType) {
	channel := make(chan []byte, 256)
	streamer.output = channel
	l.mutex.Lock()
	l.streamers[streamer] = struct{}{}
	l.updateMaxLevel()
	l.mutex.Unlock()
	timer := time.NewTimer(time.Millisecond * 100)
	flushPending := false
	closeNotifier := conn.GetCloseNotifier()
	for keepGoing := true; keepGoing; {
		select {
		case <-closeNotifier:
			keepGoing = false
			break
		case data, ok := <-channel:
			if !ok {
				keepGoing = false
				break
			}
			if _, err := conn.Write(data); err != nil {
				keepGoing = false
				break
			}
			if !flushPending {
				timer.Reset(time.Millisecond * 100)
				flushPending = true
			}
		case <-timer.C:
			if conn.Flush() != nil {
				keepGoing = false
				break
			}
			flushPending = false
		}
	}
	if flushPending {
		conn.Flush()
	}
	l.mutex.Lock()
	delete(l.streamers, streamer)
	l.updateMaxLevel()
	l.mutex.Unlock()
	// Drain the channel.
	for {
		select {
		case <-channel:
		default:
			return
		}
	}
}

func (t *loggerMapT) getLogger(name string,
	authInfo *srpc.AuthInformation) (*Logger, error) {
	loggerMap.Lock()
	defer loggerMap.Unlock()
	if logger, ok := loggerMap.loggerMap[name]; !ok {
		return nil, errors.New("unknown logger: " + name)
	} else if err := logger.checkAuth(authInfo); err != nil {
		return nil, err
	} else {
		return logger, nil
	}
}

func (t *loggerMapT) Debug(conn *srpc.Conn,
	request proto.DebugRequest, reply *proto.DebugResponse) error {
	authInfo := conn.GetAuthInformation()
	if logger, err := t.getLogger(request.Name, authInfo); err != nil {
		return err
	} else {
		logger.Debugf(request.Level, "Logger.Debug(%d): %s\n",
			request.Level, strings.Join(request.Args, " "))
		return nil
	}
}

func (t *loggerMapT) Print(conn *srpc.Conn,
	request proto.PrintRequest,
	reply *proto.PrintResponse) error {
	authInfo := conn.GetAuthInformation()
	if logger, err := t.getLogger(request.Name, authInfo); err != nil {
		return err
	} else {
		logger.Println("Logger.Print():", strings.Join(request.Args, " "))
		return nil
	}
}

func (t *loggerMapT) SetDebugLevel(conn *srpc.Conn,
	request proto.SetDebugLevelRequest,
	reply *proto.SetDebugLevelResponse) error {
	authInfo := conn.GetAuthInformation()
	if logger, err := t.getLogger(request.Name, authInfo); err != nil {
		return err
	} else {
		logger.Printf("Logger.SetDebugLevel(%d)\n", request.Level)
		logger.SetLevel(request.Level)
		return nil
	}
}

func (t *loggerMapT) Watch(conn *srpc.Conn) error {
	var request proto.WatchRequest
	if err := conn.Decode(&request); err != nil {
		return err
	}
	authInfo := conn.GetAuthInformation()
	if logger, err := t.getLogger(request.Name, authInfo); err != nil {
		return conn.Encode(proto.WatchResponse{Error: err.Error()})
	} else if streamer, err := logger.makeStreamer(request); err != nil {
		return conn.Encode(proto.WatchResponse{Error: err.Error()})
	} else {
		if err := conn.Encode(proto.WatchResponse{}); err != nil {
			return err
		}
		if err := conn.Flush(); err != nil {
			return err
		}
		logger.watch(conn, streamer)
		return srpc.ErrorCloseClient
	}
}
<|endoftext|>


<commit_before>package runtime

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry/cf-smoke-tests/smoke"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Runtime:", func() {
	var testConfig = smoke.GetConfig()
	var appName string
	var appUrl string
	var expectedNullResponse string

	BeforeEach(func() {
		appName = testConfig.RuntimeApp
		if appName == "" {
			appName = generator.PrefixedRandomName("SMOKES", "APP")
		}

		appUrl = "https://" + appName + "." + testConfig.AppsDomain

		Eventually(func() error {
			var err error
			expectedNullResponse, err = getBodySkipSSL(testConfig.SkipSSLValidation, appUrl)
			return err
		}, CF_TIMEOUT_IN_SECONDS).Should(BeNil())
	})

	AfterEach(func() {
		smoke.AppReport(appName, CF_TIMEOUT_IN_SECONDS)
		if testConfig.Cleanup {
			Expect(cf.Cf("delete", appName, "-f", "-r").Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))
		}
	})

	Context("linux apps", func() {
		It("can be pushed, scaled and deleted", func() {
			Expect(cf.Cf("push", appName, "-p", SIMPLE_RUBY_APP_BITS_PATH, "-d", testConfig.AppsDomain, "--no-start").Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
			smoke.SetBackend(appName)
			Expect(cf.Cf("start", appName).Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))

			runPushTests(appName, appUrl, expectedNullResponse, testConfig)
		})
	})

	Context("windows apps", func() {
		It("can be pushed, scaled and deleted", func() {
			smoke.SkipIfWindows(testConfig)

			Expect(cf.Cf("push", appName, "-p", SIMPLE_DOTNET_APP_BITS_PATH, "-d", testConfig.AppsDomain, "-s", "windows2012R2", "-b", "hwc_buildpack", "--no-start").Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
			smoke.EnableDiego(appName)
			Expect(cf.Cf("start", appName).Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))

			runPushTests(appName, appUrl, expectedNullResponse, testConfig)
		})
	})
})

func runPushTests(appName, appUrl, expectedNullResponse string, testConfig *smoke.Config) {
	Eventually(func() (string, error) {
		return getBodySkipSSL(testConfig.SkipSSLValidation, appUrl)
	}, CF_TIMEOUT_IN_SECONDS).Should(ContainSubstring("It just needed to be restarted!"))

	instances := 2
	maxAttempts := 30

	ExpectAppToScale(appName, instances)

	ExpectAllAppInstancesToStart(appName, instances, maxAttempts)

	ExpectAllAppInstancesToBeReachable(appUrl, instances, maxAttempts)

	if testConfig.Cleanup {
		Expect(cf.Cf("delete", appName, "-f", "-r").Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))

		Eventually(func() *Session {
			appStatusSession := cf.Cf("app", appName)
			Expect(appStatusSession.Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(1))
			return appStatusSession
		}, 5).Should(Say("not found"))

		Eventually(func() (string, error) {
			return getBodySkipSSL(testConfig.SkipSSLValidation, appUrl)
		}, CF_TIMEOUT_IN_SECONDS).Should(ContainSubstring(string(expectedNullResponse)))
	}
}

func ExpectAppToScale(appName string, instances int) {
	Expect(cf.Cf("scale", appName, "-i", strconv.Itoa(instances)).Wait(CF_SCALE_TIMEOUT_IN_SECONDS)).To(Exit(0))
}

// Gets app status (up to maxAttempts) until all instances are up
func ExpectAllAppInstancesToStart(appName string, instances int, maxAttempts int) {
	var found bool
	expectedOutput := fmt.Sprintf("instances: %d/%d", instances, instances)

	outputMatchers := make([]*regexp.Regexp, instances)
	for i := 0; i < instances; i++ {
		outputMatchers[i] = regexp.MustCompile(fmt.Sprintf(`#%d\s+running`, i))
	}

	for i := 0; i < maxAttempts; i++ {
		session := cf.Cf("app", appName)
		Expect(session.Wait(CF_APP_STATUS_TIMEOUT_IN_SECONDS)).To(Exit(0))

		output := string(session.Out.Contents())
		found = strings.Contains(output, expectedOutput)

		if found {
			for _, matcher := range outputMatchers {
				matches := matcher.FindStringSubmatch(output)
				if matches == nil {
					found = false
					break
				}
			}
		}

		if found {
			break
		}
		time.Sleep(500 * time.Millisecond)
	}

	Expect(found).To(BeTrue(), fmt.Sprintf("Wanted to see '%s' (all instances running) in %d attempts, but didn't", expectedOutput, maxAttempts))
}

// Curls the appUrl (up to maxAttempts) until all instances have been seen
func ExpectAllAppInstancesToBeReachable(appUrl string, instances int, maxAttempts int) {
	matcher := regexp.MustCompile(`instance[ _]index["]{0,1}:[ ]{0,1}(\d+)`)

	branchesSeen := make([]bool, instances)
	var sawAll bool
	var testConfig = smoke.GetConfig()
	for i := 0; i < maxAttempts; i++ {
		var output string
		Eventually(func() error {
			var err error
			output, err = getBodySkipSSL(testConfig.SkipSSLValidation, appUrl)
			return err
		}, CF_TIMEOUT_IN_SECONDS).Should(BeNil())

		matches := matcher.FindStringSubmatch(output)
		if matches == nil {
			Fail("Expected app curl output to include an instance_index; got " + output)
		}
		indexString := matches[1]
		index, err := strconv.Atoi(indexString)
		if err != nil {
			Fail("Failed to parse instance index value " + indexString)
		}
		branchesSeen[index] = true

		if allTrue(branchesSeen) {
			sawAll = true
			break
		}
	}

	Expect(sawAll).To(BeTrue(), fmt.Sprintf("Expected to hit all %d app instances in %d attempts, but didn't", instances, maxAttempts))
}

func allTrue(bools []bool) bool {
	for _, curr := range bools {
		if !curr {
			return false
		}
	}
	return true
}

func getBodySkipSSL(skip bool, url string) (string, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: skip},
	}
	client := &http.Client{Transport: transport}
	resp, err := client.Get(url)

	if err != nil {
		return "", err
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}
<commit_msg>Allow running instances label to be separated from value with more than one space<commit_after>package runtime

import (
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry/cf-smoke-tests/smoke"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
)

var _ = Describe("Runtime:", func() {
	var testConfig = smoke.GetConfig()
	var appName string
	var appUrl string
	var expectedNullResponse string

	BeforeEach(func() {
		appName = testConfig.RuntimeApp
		if appName == "" {
			appName = generator.PrefixedRandomName("SMOKES", "APP")
		}

		appUrl = "https://" + appName + "." + testConfig.AppsDomain

		Eventually(func() error {
			var err error
			expectedNullResponse, err = getBodySkipSSL(testConfig.SkipSSLValidation, appUrl)
			return err
		}, CF_TIMEOUT_IN_SECONDS).Should(BeNil())
	})

	AfterEach(func() {
		smoke.AppReport(appName, CF_TIMEOUT_IN_SECONDS)
		if testConfig.Cleanup {
			Expect(cf.Cf("delete", appName, "-f", "-r").Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))
		}
	})

	Context("linux apps", func() {
		It("can be pushed, scaled and deleted", func() {
			Expect(cf.Cf("push", appName, "-p", SIMPLE_RUBY_APP_BITS_PATH, "-d", testConfig.AppsDomain, "--no-start").Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
			smoke.SetBackend(appName)
			Expect(cf.Cf("start", appName).Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))

			runPushTests(appName, appUrl, expectedNullResponse, testConfig)
		})
	})

	Context("windows apps", func() {
		It("can be pushed, scaled and deleted", func() {
			smoke.SkipIfWindows(testConfig)

			Expect(cf.Cf("push", appName, "-p", SIMPLE_DOTNET_APP_BITS_PATH, "-d", testConfig.AppsDomain, "-s", "windows2012R2", "-b", "hwc_buildpack", "--no-start").Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))
			smoke.EnableDiego(appName)
			Expect(cf.Cf("start", appName).Wait(CF_PUSH_TIMEOUT_IN_SECONDS)).To(Exit(0))

			runPushTests(appName, appUrl, expectedNullResponse, testConfig)
		})
	})
})

func runPushTests(appName, appUrl, expectedNullResponse string, testConfig *smoke.Config) {
	Eventually(func() (string, error) {
		return getBodySkipSSL(testConfig.SkipSSLValidation, appUrl)
	}, CF_TIMEOUT_IN_SECONDS).Should(ContainSubstring("It just needed to be restarted!"))

	instances := 2
	maxAttempts := 30

	ExpectAppToScale(appName, instances)

	ExpectAllAppInstancesToStart(appName, instances, maxAttempts)

	ExpectAllAppInstancesToBeReachable(appUrl, instances, maxAttempts)

	if testConfig.Cleanup {
		Expect(cf.Cf("delete", appName, "-f", "-r").Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(0))

		Eventually(func() *Session {
			appStatusSession := cf.Cf("app", appName)
			Expect(appStatusSession.Wait(CF_TIMEOUT_IN_SECONDS)).To(Exit(1))
			return appStatusSession
		}, 5).Should(Say("not found"))

		Eventually(func() (string, error) {
			return getBodySkipSSL(testConfig.SkipSSLValidation, appUrl)
		}, CF_TIMEOUT_IN_SECONDS).Should(ContainSubstring(string(expectedNullResponse)))
	}
}

func ExpectAppToScale(appName string, instances int) {
	Expect(cf.Cf("scale", appName, "-i", strconv.Itoa(instances)).Wait(CF_SCALE_TIMEOUT_IN_SECONDS)).To(Exit(0))
}

// Gets app status (up to maxAttempts) until all instances are up
func ExpectAllAppInstancesToStart(appName string, instances int, maxAttempts int) {
	var found bool
	expectedOutput := regexp.MustCompile(fmt.Sprintf(`instances:\s+%d/%d`, instances, instances))

	outputMatchers := make([]*regexp.Regexp, instances)
	for i := 0; i < instances; i++ {
		outputMatchers[i] = regexp.MustCompile(fmt.Sprintf(`#%d\s+running`, i))
	}

	for i := 0; i < maxAttempts; i++ {
		session := cf.Cf("app", appName)
		Expect(session.Wait(CF_APP_STATUS_TIMEOUT_IN_SECONDS)).To(Exit(0))

		output := string(session.Out.Contents())
		found = expectedOutput.MatchString(output)

		if found {
			for _, matcher := range outputMatchers {
				matches := matcher.FindStringSubmatch(output)
				if matches == nil {
					found = false
					break
				}
			}
		}

		if found {
			break
		}
		time.Sleep(500 * time.Millisecond)
	}

	Expect(found).To(BeTrue(), fmt.Sprintf("Wanted to see '%s' (all instances running) in %d attempts, but didn't", expectedOutput, maxAttempts))
}

// Curls the appUrl (up to maxAttempts) until all instances have been seen
func ExpectAllAppInstancesToBeReachable(appUrl string, instances int, maxAttempts int) {
	matcher := regexp.MustCompile(`instance[ _]index["]{0,1}:[ ]{0,1}(\d+)`)

	branchesSeen := make([]bool, instances)
	var sawAll bool
	var testConfig = smoke.GetConfig()
	for i := 0; i < maxAttempts; i++ {
		var output string
		Eventually(func() error {
			var err error
			output, err = getBodySkipSSL(testConfig.SkipSSLValidation, appUrl)
			return err
		}, CF_TIMEOUT_IN_SECONDS).Should(BeNil())

		matches := matcher.FindStringSubmatch(output)
		if matches == nil {
			Fail("Expected app curl output to include an instance_index; got " + output)
		}
		indexString := matches[1]
		index, err := strconv.Atoi(indexString)
		if err != nil {
			Fail("Failed to parse instance index value " + indexString)
		}
		branchesSeen[index] = true

		if allTrue(branchesSeen) {
			sawAll = true
			break
		}
	}

	Expect(sawAll).To(BeTrue(), fmt.Sprintf("Expected to hit all %d app instances in %d attempts, but didn't", instances, maxAttempts))
}

func allTrue(bools []bool) bool {
	for _, curr := range bools {
		if !curr {
			return false
		}
	}
	return true
}

func getBodySkipSSL(skip bool, url string) (string, error) {
	transport := &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: skip},
	}
	client := &http.Client{Transport: transport}
	resp, err := client.Get(url)

	if err != nil {
		return "", err
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return "", err
	}
	return string(body), nil
}
<|endoftext|>