<commit_before>package main

import (
	"os"
	"reflect"
	"testing"
)

func TestIsDir(t *testing.T) {
	// setUp
	os.Mkdir("tmp", 0777)

	// test
	if !IsDir("tmp") {
		t.Error("IsDir() returned false for directory tmp")
	}

	// cleanUp
	err := os.Remove("tmp")
	if err != nil {
		t.Error("An error occured trying to remove tmp/,", err)
	}
}

func TestIsGoFile(t *testing.T) {
	filename := "test.go"
	if !IsGoFile(filename) {
		t.Error("IsGoFile() returned false for filename test.go")
	}
}

func TestParseFilenames(t *testing.T) {
	// setUp
	os.Mkdir("tmp", 0777)

	os.Create("tmp/test.txt")
	os.Create("tmp/test2.md")

	test_data := []string{"test.txt", "test2.md"}

	currentDirectory, err := os.Getwd()
	if err != nil {
		t.Error("An error occured trying to get current working directory,", err)
	}

	// test
	filenames := ParseFilenames(currentDirectory + "/tmp")
	if !reflect.DeepEqual(filenames, test_data) {
		t.Error("ParseFilenames() did not return expected value.")
	}

	// cleanUp
	err = os.Remove("tmp/test.txt")
	if err != nil {
		t.Error("An error occured trying to remove tmp/test.txt,", err)
	}

	err = os.Remove("tmp/test2.md")
	if err != nil {
		t.Error("An error occured trying to remove tmp/test2.md,", err)
	}

	err = os.Remove("tmp")
	if err != nil {
		t.Error("An error occured trying to remove tmp/,", err)
	}
}

func TestCheckDirForGo(t *testing.T) {
	// setUp
	os.Mkdir("tmp", 0777)

	os.Create("tmp/test.go")

	currentDirectory, err := os.Getwd()
	if err != nil {
		t.Error("An error occured trying to get current working directory,", err)
	}

	// test
	if !CheckDirForGo(currentDirectory + "/tmp") {
		t.Error("CheckDirForGo() did not properly identify a directory with a .go file in it.")
	}

	// cleanUp
	err = os.Remove("tmp/test.go")
	if err != nil {
		t.Error("An error occured trying to remove tmp/test.go,", err)
	}

	err = os.Remove("tmp")
	if err != nil {
		t.Error("An error occured trying to remove tmp/,", err)
	}
}

// Wercker seems to remove version control when it gets golang packages in its default build step
// This means wercker errors on go get -u with version control not found.
// The git clone goupdatebuild step in wercker.yml combined with the ssh-key for git access
// solve this problem and are required for TestUpdatePackage() to function properly on wercker
func TestUpdatePackage(t *testing.T) {
	GOPATH := os.Getenv("GOPATH")
	packPath := GOPATH + "/src/github.com/ehabit/goupdate"
	updated := UpdatePackage(packPath)
	if !updated {
		t.Error("An error occured trying to update this Go package,", packPath)
	}
}
<commit_msg>Simplified TestUpdatePackage, reduced by 1 assignment<commit_after>package main

import (
	"os"
	"reflect"
	"testing"
)

func TestIsDir(t *testing.T) {
	// setUp
	os.Mkdir("tmp", 0777)

	// test
	if !IsDir("tmp") {
		t.Error("IsDir() returned false for directory tmp")
	}

	// cleanUp
	err := os.Remove("tmp")
	if err != nil {
		t.Error("An error occured trying to remove tmp/,", err)
	}
}

func TestIsGoFile(t *testing.T) {
	filename := "test.go"
	if !IsGoFile(filename) {
		t.Error("IsGoFile() returned false for filename test.go")
	}
}

func TestParseFilenames(t *testing.T) {
	// setUp
	os.Mkdir("tmp", 0777)

	os.Create("tmp/test.txt")
	os.Create("tmp/test2.md")

	test_data := []string{"test.txt", "test2.md"}

	currentDirectory, err := os.Getwd()
	if err != nil {
		t.Error("An error occured trying to get current working directory,", err)
	}

	// test
	filenames := ParseFilenames(currentDirectory + "/tmp")
	if !reflect.DeepEqual(filenames, test_data) {
		t.Error("ParseFilenames() did not return expected value.")
	}

	// cleanUp
	err = os.Remove("tmp/test.txt")
	if err != nil {
		t.Error("An error occured trying to remove tmp/test.txt,", err)
	}

	err = os.Remove("tmp/test2.md")
	if err != nil {
		t.Error("An error occured trying to remove tmp/test2.md,", err)
	}

	err = os.Remove("tmp")
	if err != nil {
		t.Error("An error occured trying to remove tmp/,", err)
	}
}

func TestCheckDirForGo(t *testing.T) {
	// setUp
	os.Mkdir("tmp", 0777)

	os.Create("tmp/test.go")

	currentDirectory, err := os.Getwd()
	if err != nil {
		t.Error("An error occured trying to get current working directory,", err)
	}

	// test
	if !CheckDirForGo(currentDirectory + "/tmp") {
		t.Error("CheckDirForGo() did not properly identify a directory with a .go file in it.")
	}

	// cleanUp
	err = os.Remove("tmp/test.go")
	if err != nil {
		t.Error("An error occured trying to remove tmp/test.go,", err)
	}

	err = os.Remove("tmp")
	if err != nil {
		t.Error("An error occured trying to remove tmp/,", err)
	}
}

// Wercker seems to remove version control when it gets golang packages in its default build step
// This means wercker errors on go get -u with version control not found.
// The git clone goupdatebuild step in wercker.yml combined with the ssh-key for git access
// solve this problem and are required for TestUpdatePackage() to function properly on wercker
func TestUpdatePackage(t *testing.T) {
	packPath := os.Getenv("GOPATH") + "/src/github.com/ehabit/goupdate"
	updated := UpdatePackage(packPath)
	if !updated {
		t.Error("An error occured trying to update this Go package,", packPath)
	}
}
<|endoftext|>


<commit_before>package tdb

import (
	"time"
)

// see https://en.wikipedia.org/wiki/Radix_tree
type tree struct {
	root        *node
	_bytes      int
	_length     int
	ctxRemovals map[int64]int
}

type node struct {
	key        []byte
	edges      edges
	data       []sequence
	removedFor []int64
}

type edge struct {
	label  []byte
	target *node
}

func newByteTree() *tree {
	return &tree{&node{}, 0, 0, make(map[int64]int)}
}

func (bt *tree) bytes() int {
	return bt._bytes
}

func (bt *tree) length(ctx int64) int {
	return bt._length - bt.ctxRemovals[ctx]
}

func (bt *tree) walk(ctx int64, fn func(key []byte, data []sequence) bool) {
	nodes := make([]*node, 0, bt._length)
	nodes = append(nodes, bt.root)
	for {
		if len(nodes) == 0 {
			break
		}
		n := nodes[0]
		nodes = nodes[1:]
		if n.data != nil {
			alreadyRemoved := n.wasRemovedFor(ctx)
			if !alreadyRemoved {
				keep := fn(n.key, n.data)
				if !keep {
					if n.doRemoveFor(ctx) {
						bt.ctxRemovals[ctx]++
					}
				}
			}
		}
		for _, e := range n.edges {
			nodes = append(nodes, e.target)
		}
	}
}

func (bt *tree) remove(ctx int64, fullKey []byte) []sequence {
	// TODO: basic shape of this is very similar to update, dry violation
	n := bt.root
	key := fullKey
	// Try to update on existing edge
nodeLoop:
	for {
		for _, edge := range n.edges {
			labelLength := len(edge.label)
			keyLength := len(key)
			i := 0
			for ; i < keyLength && i < labelLength; i++ {
				if edge.label[i] != key[i] {
					break
				}
			}
			if i == keyLength && keyLength == labelLength {
				// found it
				alreadyRemoved := edge.target.wasRemovedFor(ctx)
				if alreadyRemoved {
					return nil
				}
				if edge.target.doRemoveFor(ctx) {
					bt.ctxRemovals[ctx]++
				}
				return edge.target.data
			} else if i == labelLength && labelLength < keyLength {
				// descend
				n = edge.target
				key = key[labelLength:]
				continue nodeLoop
			}
		}

		// not found
		return nil
	}
}

func (bt *tree) update(t *table, truncateBefore time.Time, key []byte, vals tsparams) int {
	bytesAdded, newNode := bt.doUpdate(t, truncateBefore, key, vals)
	bt._bytes += bytesAdded
	if newNode {
		bt._length++
	}
	return bytesAdded
}

func (bt *tree) doUpdate(t *table, truncateBefore time.Time, fullKey []byte, vals tsparams) (int, bool) {
	n := bt.root
	key := fullKey
	// Try to update on existing edge
nodeLoop:
	for {
		for _, edge := range n.edges {
			labelLength := len(edge.label)
			keyLength := len(key)
			i := 0
			for ; i < keyLength && i < labelLength; i++ {
				if edge.label[i] != key[i] {
					break
				}
			}
			if i == keyLength && keyLength == labelLength {
				// update existing node
				return edge.target.doUpdate(t, truncateBefore, vals), false
			} else if i == labelLength && labelLength < keyLength {
				// descend
				n = edge.target
				key = key[labelLength:]
				continue nodeLoop
			} else if i > 0 {
				// common substring, split on that
				return edge.split(t, truncateBefore, i, fullKey, key, vals), true
			}
		}

		// Create new edge
		target := &node{key: fullKey}
		n.edges = append(n.edges, &edge{key, target})
		return target.doUpdate(t, truncateBefore, vals) + len(key), true
	}
}

func (n *node) doUpdate(t *table, truncateBefore time.Time, vals tsparams) int {
	bytesAdded := 0
	// Grow sequences to match number of fields in table
	for i := len(n.data); i < len(t.Fields); i++ {
		n.data = append(n.data, nil)
	}
	for i, field := range t.Fields {
		current := n.data[i]
		previousSize := len(current)
		updated := current.update(vals, field, t.Resolution, truncateBefore)
		n.data[i] = updated
		bytesAdded += len(updated) - previousSize
	}
	return bytesAdded
}

func (n *node) wasRemovedFor(ctx int64) bool {
	if ctx == 0 {
		return false
	}
	for _, _ctx := range n.removedFor {
		if _ctx == ctx {
			return true
		}
	}
	return false
}

func (n *node) doRemoveFor(ctx int64) bool {
	if ctx == 0 {
		return false
	}
	n.removedFor = append(n.removedFor, ctx)
	return true
}

func (e *edge) split(t *table, truncateBefore time.Time, splitOn int, fullKey []byte, key []byte, vals tsparams) int {
	newNode := &node{edges: edges{&edge{e.label[splitOn:], e.target}}}
	newLeaf := newNode
	if splitOn != len(key) {
		newLeaf = &node{key: fullKey}
		newNode.edges = append(newNode.edges, &edge{key[splitOn:], newLeaf})
	}
	e.label = e.label[:splitOn]
	e.target = newNode
	return len(key) - splitOn + newLeaf.doUpdate(t, truncateBefore, vals)
}

type edges []*edge
<commit_msg>Protected removal tracking with mutex<commit_after>package tdb

import (
	"sync"
	"time"
)

// see https://en.wikipedia.org/wiki/Radix_tree
type tree struct {
	root        *node
	_bytes      int
	_length     int
	ctxRemovals map[int64]int
	removalMx   sync.RWMutex
}

type node struct {
	key        []byte
	edges      edges
	data       []sequence
	removedFor []int64
}

type edge struct {
	label  []byte
	target *node
}

func newByteTree() *tree {
	return &tree{root: &node{}, ctxRemovals: make(map[int64]int)}
}

func (bt *tree) bytes() int {
	return bt._bytes
}

func (bt *tree) length(ctx int64) int {
	return bt._length - bt.ctxRemovals[ctx]
}

func (bt *tree) walk(ctx int64, fn func(key []byte, data []sequence) bool) {
	nodes := make([]*node, 0, bt._length)
	nodes = append(nodes, bt.root)
	for {
		if len(nodes) == 0 {
			break
		}
		n := nodes[0]
		nodes = nodes[1:]
		if n.data != nil {
			alreadyRemoved := n.wasRemovedFor(bt, ctx)
			if !alreadyRemoved {
				keep := fn(n.key, n.data)
				if !keep {
					n.doRemoveFor(bt, ctx)
				}
			}
		}
		for _, e := range n.edges {
			nodes = append(nodes, e.target)
		}
	}
}

func (bt *tree) remove(ctx int64, fullKey []byte) []sequence {
	// TODO: basic shape of this is very similar to update, dry violation
	n := bt.root
	key := fullKey
	// Try to update on existing edge
nodeLoop:
	for {
		for _, edge := range n.edges {
			labelLength := len(edge.label)
			keyLength := len(key)
			i := 0
			for ; i < keyLength && i < labelLength; i++ {
				if edge.label[i] != key[i] {
					break
				}
			}
			if i == keyLength && keyLength == labelLength {
				// found it
				alreadyRemoved := edge.target.wasRemovedFor(bt, ctx)
				if alreadyRemoved {
					return nil
				}
				edge.target.doRemoveFor(bt, ctx)
				return edge.target.data
			} else if i == labelLength && labelLength < keyLength {
				// descend
				n = edge.target
				key = key[labelLength:]
				continue nodeLoop
			}
		}

		// not found
		return nil
	}
}

func (bt *tree) update(t *table, truncateBefore time.Time, key []byte, vals tsparams) int {
	bytesAdded, newNode := bt.doUpdate(t, truncateBefore, key, vals)
	bt._bytes += bytesAdded
	if newNode {
		bt._length++
	}
	return bytesAdded
}

func (bt *tree) doUpdate(t *table, truncateBefore time.Time, fullKey []byte, vals tsparams) (int, bool) {
	n := bt.root
	key := fullKey
	// Try to update on existing edge
nodeLoop:
	for {
		for _, edge := range n.edges {
			labelLength := len(edge.label)
			keyLength := len(key)
			i := 0
			for ; i < keyLength && i < labelLength; i++ {
				if edge.label[i] != key[i] {
					break
				}
			}
			if i == keyLength && keyLength == labelLength {
				// update existing node
				return edge.target.doUpdate(t, truncateBefore, vals), false
			} else if i == labelLength && labelLength < keyLength {
				// descend
				n = edge.target
				key = key[labelLength:]
				continue nodeLoop
			} else if i > 0 {
				// common substring, split on that
				return edge.split(t, truncateBefore, i, fullKey, key, vals), true
			}
		}

		// Create new edge
		target := &node{key: fullKey}
		n.edges = append(n.edges, &edge{key, target})
		return target.doUpdate(t, truncateBefore, vals) + len(key), true
	}
}

func (n *node) doUpdate(t *table, truncateBefore time.Time, vals tsparams) int {
	bytesAdded := 0
	// Grow sequences to match number of fields in table
	for i := len(n.data); i < len(t.Fields); i++ {
		n.data = append(n.data, nil)
	}
	for i, field := range t.Fields {
		current := n.data[i]
		previousSize := len(current)
		updated := current.update(vals, field, t.Resolution, truncateBefore)
		n.data[i] = updated
		bytesAdded += len(updated) - previousSize
	}
	return bytesAdded
}

func (n *node) wasRemovedFor(bt *tree, ctx int64) bool {
	if ctx == 0 {
		return false
	}
	bt.removalMx.RLock()
	for _, _ctx := range n.removedFor {
		if _ctx == ctx {
			bt.removalMx.RUnlock()
			return true
		}
	}
	bt.removalMx.RUnlock()
	return false
}

func (n *node) doRemoveFor(bt *tree, ctx int64) {
	if ctx == 0 {
		return
	}
	bt.removalMx.Lock()
	n.removedFor = append(n.removedFor, ctx)
	bt.ctxRemovals[ctx]++
	bt.removalMx.Unlock()
}

func (e *edge) split(t *table, truncateBefore time.Time, splitOn int, fullKey []byte, key []byte, vals tsparams) int {
	newNode := &node{edges: edges{&edge{e.label[splitOn:], e.target}}}
	newLeaf := newNode
	if splitOn != len(key) {
		newLeaf = &node{key: fullKey}
		newNode.edges = append(newNode.edges, &edge{key[splitOn:], newLeaf})
	}
	e.label = e.label[:splitOn]
	e.target = newNode
	return len(key) - splitOn + newLeaf.doUpdate(t, truncateBefore, vals)
}

type edges []*edge
<|endoftext|>


<commit_before>package watcher

import (
	"github.com/smartystreets/goconvey/web/server/contract"
	"os"
)

type Scanner struct {
	fs      contract.FileSystem
	watcher contract.Watcher
	current int64
	working int64
}

// TODO: we must find out if a new directory has been added or if one has been removed!

func (self *Scanner) Scan(root string) (changed bool) {
	self.calculateWorkingChecksum(root)
	return self.determineIfChanged()
}

func (self *Scanner) determineIfChanged() bool {
	defer func() {
		self.current = self.working
		self.working = 0
	}()

	return self.working != self.current
}

func (self *Scanner) calculateWorkingChecksum(root string) {
	self.fs.Walk(root, func(path string, info os.FileInfo, err error) error {
		step := newWalkStep(root, path, info)

		if step.isIgnored(self.watcher) {
			return nil
		}

		if step.isNewWatchedFolder(self.watcher) {
			self.working += step.sum()
		} else if step.isWatchedFile(self.watcher) {
			self.working += step.sum()
		}

		return nil
	})
}

func NewScanner(fs contract.FileSystem, watcher contract.Watcher) *Scanner {
	self := &Scanner{}
	self.fs = fs
	self.watcher = watcher
	return self
}
<commit_msg>Moved some code around.<commit_after>package watcher

import (
	"github.com/smartystreets/goconvey/web/server/contract"
	"os"
)

type Scanner struct {
	fs      contract.FileSystem
	watcher contract.Watcher
	current int64
	working int64
}

// TODO: we must find out if a new directory has been added or if one has been removed!

func (self *Scanner) Scan(root string) (changed bool) {
	self.calculateWorkingChecksum(root)
	return self.determineIfChanged()
}

func (self *Scanner) calculateWorkingChecksum(root string) {
	self.fs.Walk(root, func(path string, info os.FileInfo, err error) error {
		step := newWalkStep(root, path, info)

		if step.isIgnored(self.watcher) {
			return nil
		}

		if step.isNewWatchedFolder(self.watcher) {
			self.working += step.sum()
		} else if step.isWatchedFile(self.watcher) {
			self.working += step.sum()
		}

		return nil
	})
}

func (self *Scanner) determineIfChanged() bool {
	defer func() {
		self.current = self.working
		self.working = 0
	}()

	return self.working != self.current
}

func NewScanner(fs contract.FileSystem, watcher contract.Watcher) *Scanner {
	self := &Scanner{}
	self.fs = fs
	self.watcher = watcher
	return self
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"github.com/google/cayley/graph"
	"github.com/google/cayley/graph/iterator"
	"github.com/google/cayley/quad"
)

type Morphism struct {
	Name     string
	Reversal func() Morphism
	Apply    graph.MorphismFunc
}

type iteratorMorphism interface {
	Apply(graph.Iterator) graph.Iterator
}

type Path struct {
	stack []Morphism
	it    graph.Iterator
	qs    graph.QuadStore
}

func V(qs graph.QuadStore, nodes ...string) *Path {
	fixed := qs.FixedIterator()
	for _, n := range nodes {
		fixed.Add(qs.ValueOf(n))
	}
	return &Path{
		it: fixed,
		qs: qs,
	}
}

func PathFromIterator(qs graph.QuadStore, it graph.Iterator) *Path {
	return &Path{
		it: it,
		qs: qs,
	}
}

func M(qs graph.QuadStore) *Path {
	return &Path{
		it: nil,
		qs: qs,
	}
}

func (p *Path) Reverse() *Path {
	newPath := M(p.qs)
	for i := len(p.stack) - 1; i >= 0; i-- {
		newPath.stack = append(newPath.stack, p.stack[i].Reversal())
	}
	return newPath
}

func (p *Path) IsConcrete() bool { return p.it != nil }

func (p *Path) Tag(tags ...string) *Path {
	p.stack = append(p.stack, TagMorphism(tags...))
	return p
}

func (p *Path) Out(via ...interface{}) *Path {
	p.stack = append(p.stack, OutMorphism(p.qs, via...))
	return p
}
func (p *Path) In(via ...interface{}) *Path {
	p.stack = append(p.stack, InMorphism(p.qs, via...))
	return p
}

func (p *Path) BuildIterator() graph.Iterator {
	f := p.MorphismFunc()
	return f(p.it)
}

func (p *Path) MorphismFunc() graph.MorphismFunc {
	return func(it graph.Iterator) graph.Iterator {
		i := it.Clone()
		for _, m := range p.stack {
			i = m.Apply(i)
		}
		return i
	}
}

func TagMorphism(tags ...string) Morphism {
	return Morphism{
		"tag",
		func() Morphism { return TagMorphism(tags...) },
		func(it graph.Iterator) graph.Iterator {
			for _, t := range tags {
				it.Tagger().Add(t)
			}
			return it
		}}
}

func OutMorphism(qs graph.QuadStore, via ...interface{}) Morphism {
	path := buildViaPath(qs, via...)
	return Morphism{
		"out",
		func() Morphism { return InMorphism(qs, via...) },
		inOutIterator(path, false),
	}
}

func InMorphism(qs graph.QuadStore, via ...interface{}) Morphism {
	path := buildViaPath(qs, via...)
	return Morphism{
		"in",
		func() Morphism { return OutMorphism(qs, via...) },
		inOutIterator(path, true),
	}
}

func inOutIterator(viaPath *Path, reverse bool) graph.MorphismFunc {
	return func(base graph.Iterator) graph.Iterator {
		in, out := quad.Subject, quad.Object
		if reverse {
			in, out = out, in
		}
		lto := iterator.NewLinksTo(viaPath.qs, base, in)
		and := iterator.NewAnd()
		and.AddSubIterator(iterator.NewLinksTo(viaPath.qs, viaPath.BuildIterator(), quad.Predicate))
		and.AddSubIterator(lto)
		return iterator.NewHasA(viaPath.qs, and, out)
	}
}

func buildViaPath(qs graph.QuadStore, via ...interface{}) *Path {
	if len(via) == 0 {
		return PathFromIterator(qs, qs.NodesAllIterator())
	} else if len(via) == 1 {
		v := via[0]
		if path, ok := v.(*Path); ok {
			return path
		} else if str, ok := v.(string); ok {
			return V(qs, str)
		} else {
			panic("Invalid type passed to buildViaPath.")
		}
	}
	var strings []string
	for _, s := range via {
		if str, ok := s.(string); ok {
			strings = append(strings, str)
		} else {
			panic("Non-string type passed to long Via path")
		}
	}
	return V(qs, strings...)
}
<commit_msg>unify morphisms and paths<commit_after>// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"github.com/google/cayley/graph"
	"github.com/google/cayley/graph/iterator"
	"github.com/google/cayley/quad"
)

type Morphism struct {
	Name     string
	Reversal func() Morphism
	Apply    graph.MorphismFunc
}

type iteratorMorphism interface {
	Apply(graph.Iterator) graph.Iterator
}

type Path struct {
	stack []Morphism
	qs    graph.QuadStore
}

func V(qs graph.QuadStore, nodes ...string) *Path {
	return &Path{
		stack: []Morphism{
			IsMorphism(qs, nodes...),
		},
		qs: qs,
	}
}

func PathFromIterator(qs graph.QuadStore, it graph.Iterator) *Path {
	return &Path{
		stack: []Morphism{
			intersectIteratorMorphism(it),
		},
		qs: qs,
	}
}

func M(qs graph.QuadStore) *Path {
	return &Path{
		qs: qs,
	}
}

func (p *Path) Reverse() *Path {
	newPath := M(p.qs)
	for i := len(p.stack) - 1; i >= 0; i-- {
		newPath.stack = append(newPath.stack, p.stack[i].Reversal())
	}
	return newPath
}

func (p *Path) Tag(tags ...string) *Path {
	p.stack = append(p.stack, TagMorphism(tags...))
	return p
}

func (p *Path) Out(via ...interface{}) *Path {
	p.stack = append(p.stack, OutMorphism(p.qs, via...))
	return p
}
func (p *Path) In(via ...interface{}) *Path {
	p.stack = append(p.stack, InMorphism(p.qs, via...))
	return p
}

func (p *Path) BuildIterator() graph.Iterator {
	f := p.MorphismFunc()
	return f(p.qs.NodesAllIterator())
}

func (p *Path) MorphismFunc() graph.MorphismFunc {
	return func(it graph.Iterator) graph.Iterator {
		i := it.Clone()
		for _, m := range p.stack {
			i = m.Apply(i)
		}
		return i
	}
}

func IsMorphism(qs graph.QuadStore, nodes ...string) Morphism {
	return Morphism{
		"is",
		func() Morphism { return IsMorphism(qs, nodes...) },
		func(it graph.Iterator) graph.Iterator {
			fixed := qs.FixedIterator()
			for _, n := range nodes {
				fixed.Add(qs.ValueOf(n))
			}
			and := iterator.NewAnd()
			and.AddSubIterator(fixed)
			and.AddSubIterator(it)
			return and
		},
	}
}

func TagMorphism(tags ...string) Morphism {
	return Morphism{
		"tag",
		func() Morphism { return TagMorphism(tags...) },
		func(it graph.Iterator) graph.Iterator {
			for _, t := range tags {
				it.Tagger().Add(t)
			}
			return it
		}}
}

func OutMorphism(qs graph.QuadStore, via ...interface{}) Morphism {
	path := buildViaPath(qs, via...)
	return Morphism{
		"out",
		func() Morphism { return InMorphism(qs, via...) },
		inOutIterator(path, false),
	}
}

func InMorphism(qs graph.QuadStore, via ...interface{}) Morphism {
	path := buildViaPath(qs, via...)
	return Morphism{
		"in",
		func() Morphism { return OutMorphism(qs, via...) },
		inOutIterator(path, true),
	}
}

func intersectIteratorMorphism(it graph.Iterator) Morphism {
	return Morphism{
		"iterator",
		func() Morphism { return intersectIteratorMorphism(it) },
		func(subIt graph.Iterator) graph.Iterator {
			and := iterator.NewAnd()
			and.AddSubIterator(it)
			and.AddSubIterator(subIt)
			return and
		},
	}
}

func AndMorphism(path *Path) Morphism {
	return Morphism{
		"and",
		func() Morphism { return AndMorphism(path) },
		func(it graph.Iterator) graph.Iterator {
			subIt := path.BuildIterator()
			and := iterator.NewAnd()
			and.AddSubIterator(it)
			and.AddSubIterator(subIt)
			return and
		},
	}
}

func inOutIterator(viaPath *Path, reverse bool) graph.MorphismFunc {
	return func(base graph.Iterator) graph.Iterator {
		in, out := quad.Subject, quad.Object
		if reverse {
			in, out = out, in
		}
		lto := iterator.NewLinksTo(viaPath.qs, base, in)
		and := iterator.NewAnd()
		and.AddSubIterator(iterator.NewLinksTo(viaPath.qs, viaPath.BuildIterator(), quad.Predicate))
		and.AddSubIterator(lto)
		return iterator.NewHasA(viaPath.qs, and, out)
	}
}

func buildViaPath(qs graph.QuadStore, via ...interface{}) *Path {
	if len(via) == 0 {
		return PathFromIterator(qs, qs.NodesAllIterator())
	} else if len(via) == 1 {
		v := via[0]
		if path, ok := v.(*Path); ok {
			return path
		} else if str, ok := v.(string); ok {
			return V(qs, str)
		} else {
			panic("Invalid type passed to buildViaPath.")
		}
	}
	var strings []string
	for _, s := range via {
		if str, ok := s.(string); ok {
			strings = append(strings, str)
		} else {
			panic("Non-string type passed to long Via path")
		}
	}
	return V(qs, strings...)
}
<|endoftext|>


<commit_before>package sharings

import (
	"bytes"
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/contacts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/permissions"
	"github.com/cozy/cozy-stack/tests/testutils"
	"github.com/labstack/echo"
	"github.com/stretchr/testify/assert"
)

const iocozytests = "io.cozy.tests"

var setup *testutils.TestSetup
var ts *httptest.Server
var aliceInstance *instance.Instance
var aliceAppToken string
var bobContact *contacts.Contact

func TestCreateSharingSuccess(t *testing.T) {
	assert.NotEmpty(t, aliceAppToken)
	assert.NotNil(t, bobContact)

	v := echo.Map{
		"data": echo.Map{
			"type": consts.Sharings,
			"attributes": echo.Map{
				"description": "this is a test",
				"rules": []interface{}{
					echo.Map{
						"title":   "test one",
						"doctype": iocozytests,
						"values":  []string{"foobar"},
					},
				},
			},
			"relationships": echo.Map{
				"recipients": echo.Map{
					"data": []interface{}{
						echo.Map{
							"id":      bobContact.ID(),
							"doctype": bobContact.DocType(),
						},
					},
				},
			},
		},
	}
	body, _ := json.Marshal(v)
	r := bytes.NewReader(body)

	req, err := http.NewRequest(http.MethodPost, ts.URL+"/sharings/", r)
	assert.NoError(t, err)
	req.Header.Add(echo.HeaderContentType, "application/vnd.api+json")
	req.Header.Add(echo.HeaderAuthorization, "Bearer "+aliceAppToken)
	res, err := http.DefaultClient.Do(req)
	assert.NoError(t, err)
	assert.Equal(t, http.StatusCreated, res.StatusCode)
	defer res.Body.Close()

	var result map[string]interface{}
	assert.NoError(t, json.NewDecoder(res.Body).Decode(&result))
	data := result["data"].(map[string]interface{})
	assert.Equal(t, data["type"], consts.Sharings)
	sid := data["id"].(string)
	assert.NotEmpty(t, sid)
	assert.NotEmpty(t, data["meta"].(map[string]interface{})["rev"])
	self := "/sharings/" + sid
	assert.Equal(t, data["links"].(map[string]interface{})["self"], self)

	attrs := data["attributes"].(map[string]interface{})
	assert.Equal(t, attrs["description"], "this is a test")
	assert.Equal(t, attrs["app_slug"], "testapp")
	assert.Equal(t, attrs["owner"], true)
	assert.NotEmpty(t, attrs["created_at"])
	assert.NotEmpty(t, attrs["updated_at"])
	assert.Nil(t, attrs["credentials"])

	members := attrs["members"].([]interface{})
	assert.Len(t, members, 2)
	owner := members[0].(map[string]interface{})
	assert.Equal(t, owner["status"], "owner")
	assert.Equal(t, owner["name"], "Alice")
	assert.Equal(t, owner["email"], "alice@example.net")
	assert.Equal(t, owner["instance"], aliceInstance.Domain)
	recipient := members[1].(map[string]interface{})
	assert.Equal(t, recipient["status"], "mail-not-sent")
	assert.Equal(t, recipient["name"], "Bob")
	assert.Equal(t, recipient["email"], "bob@example.net")

	rules := attrs["rules"].([]interface{})
	assert.Len(t, rules, 1)
	rule := rules[0].(map[string]interface{})
	assert.Equal(t, rule["title"], "test one")
	assert.Equal(t, rule["doctype"], iocozytests)
	assert.Equal(t, rule["values"], []interface{}{"foobar"})
}

func TestMain(m *testing.M) {
	config.UseTestFile()
	config.GetConfig().Assets = "../../assets"
	testutils.NeedCouchdb()

	// Prepare Alice's instance
	setup = testutils.NewSetup(m, "sharing_test_alice")
	var settings couchdb.JSONDoc
	settings.M = map[string]interface{}{
		"email":       "alice@example.net",
		"public_name": "Alice",
	}
	aliceInstance = setup.GetTestInstance(&instance.Options{
		Settings: settings,
	})
	aliceAppToken = generateAppToken(aliceInstance, "testapp")
	bobContact = createContact(aliceInstance, "Bob", "bob@example.net")

	// Routing
	routes := map[string]func(*echo.Group){
		"/sharings": Routes,
	}
	ts = setup.GetTestServerMultipleRoutes(routes)

	os.Exit(setup.Run())
}

func createContact(inst *instance.Instance, name, email string) *contacts.Contact {
	c := &contacts.Contact{
		FullName: name,
		Email: []contacts.Email{
			contacts.Email{Address: email},
		},
	}
	err := couchdb.CreateDoc(inst, c)
	if err != nil {
		return nil
	}
	return c
}

func generateAppToken(inst *instance.Instance, slug string) string {
	rules := permissions.Set{
		permissions.Rule{
			Type:  iocozytests,
			Verbs: permissions.ALL,
		},
	}
	permReq := permissions.Permission{
		Permissions: rules,
		Type:        permissions.TypeWebapp,
		SourceID:    consts.Apps + "/" + slug,
	}
	err := couchdb.CreateDoc(inst, &permReq)
	if err != nil {
		return ""
	}
	manifest := &apps.WebappManifest{
		DocSlug:        slug,
		DocPermissions: rules,
	}
	err = couchdb.CreateNamedDocWithDB(inst, manifest)
	if err != nil {
		return ""
	}
	return inst.BuildAppToken(manifest, "")
}
<commit_msg>Add a test for sending the invitation mail<commit_after>package sharings

import (
	"bytes"
	"encoding/json"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/contacts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/permissions"
	"github.com/cozy/cozy-stack/tests/testutils"
	"github.com/labstack/echo"
	"github.com/stretchr/testify/assert"
)

const iocozytests = "io.cozy.tests"

var setup *testutils.TestSetup
var ts *httptest.Server
var aliceInstance *instance.Instance
var aliceAppToken string
var bobContact *contacts.Contact
var sharingLink string

func assertSharingIsCorrectOnSharer(t *testing.T, body io.Reader) {
	var result map[string]interface{}
	assert.NoError(t, json.NewDecoder(body).Decode(&result))
	data := result["data"].(map[string]interface{})
	assert.Equal(t, data["type"], consts.Sharings)
	sid := data["id"].(string)
	assert.NotEmpty(t, sid)
	assert.NotEmpty(t, data["meta"].(map[string]interface{})["rev"])
	self := "/sharings/" + sid
	assert.Equal(t, data["links"].(map[string]interface{})["self"], self)

	attrs := data["attributes"].(map[string]interface{})
	assert.Equal(t, attrs["description"], "this is a test")
	assert.Equal(t, attrs["app_slug"], "testapp")
	assert.Equal(t, attrs["owner"], true)
	assert.NotEmpty(t, attrs["created_at"])
	assert.NotEmpty(t, attrs["updated_at"])
	assert.Nil(t, attrs["credentials"])

	members := attrs["members"].([]interface{})
	assert.Len(t, members, 2)
	owner := members[0].(map[string]interface{})
	assert.Equal(t, owner["status"], "owner")
	assert.Equal(t, owner["name"], "Alice")
	assert.Equal(t, owner["email"], "alice@example.net")
	assert.Equal(t, owner["instance"], aliceInstance.Domain)
	recipient := members[1].(map[string]interface{})
	assert.Equal(t, recipient["status"], "mail-not-sent")
	assert.Equal(t, recipient["name"], "Bob")
	assert.Equal(t, recipient["email"], "bob@example.net")

	rules := attrs["rules"].([]interface{})
	assert.Len(t, rules, 1)
	rule := rules[0].(map[string]interface{})
	assert.Equal(t, rule["title"], "test one")
	assert.Equal(t, rule["doctype"], iocozytests)
	assert.Equal(t, rule["values"], []interface{}{"foobar"})
}

func assertInvitationMailWasSent(t *testing.T) {
	var jobs []jobs.Job
	couchReq := &couchdb.FindRequest{
		UseIndex: "by-worker-and-state",
		Selector: mango.And(
			mango.Equal("worker", "sendmail"),
			mango.Exists("state"),
		),
		Limit: 1,
	}
	err := couchdb.FindDocs(aliceInstance, consts.Jobs, couchReq, &jobs)
	assert.NoError(t, err)
	assert.Len(t, jobs, 1)
	var msg map[string]interface{}
	err = json.Unmarshal(jobs[0].Message, &msg)
	assert.NoError(t, err)
	assert.Equal(t, msg["mode"], "from")
	assert.Equal(t, msg["template_name"], "sharing_request")
	values := msg["template_values"].(map[string]interface{})
	assert.Equal(t, values["RecipientName"], "Bob")
	assert.Equal(t, values["SharerPublicName"], "Alice")
	assert.Equal(t, values["Description"], "this is a test")
	sharingLink = values["SharingLink"].(string)
	assert.Contains(t, sharingLink, "/discovery?state=")
}

func TestCreateSharingSuccess(t *testing.T) {
	assert.NotEmpty(t, aliceAppToken)
	assert.NotNil(t, bobContact)

	v := echo.Map{
		"data": echo.Map{
			"type": consts.Sharings,
			"attributes": echo.Map{
				"description": "this is a test",
				"rules": []interface{}{
					echo.Map{
						"title":   "test one",
						"doctype": iocozytests,
						"values":  []string{"foobar"},
					},
				},
			},
			"relationships": echo.Map{
				"recipients": echo.Map{
					"data": []interface{}{
						echo.Map{
							"id":      bobContact.ID(),
							"doctype": bobContact.DocType(),
						},
					},
				},
			},
		},
	}
	body, _ := json.Marshal(v)
	r := bytes.NewReader(body)

	req, err := http.NewRequest(http.MethodPost, ts.URL+"/sharings/", r)
	assert.NoError(t, err)
	req.Header.Add(echo.HeaderContentType, "application/vnd.api+json")
	req.Header.Add(echo.HeaderAuthorization, "Bearer "+aliceAppToken)
	res, err := http.DefaultClient.Do(req)
	assert.NoError(t, err)
	assert.Equal(t, http.StatusCreated, res.StatusCode)
	defer res.Body.Close()

	assertSharingIsCorrectOnSharer(t, res.Body)
	assertInvitationMailWasSent(t)
}

func TestMain(m *testing.M) {
	config.UseTestFile()
	config.GetConfig().Assets = "../../assets"
	testutils.NeedCouchdb()

	// Prepare Alice's instance
	setup = testutils.NewSetup(m, "sharing_test_alice")
	var settings couchdb.JSONDoc
	settings.M = map[string]interface{}{
		"email":       "alice@example.net",
		"public_name": "Alice",
	}
	aliceInstance = setup.GetTestInstance(&instance.Options{
		Settings: settings,
	})
	aliceAppToken = generateAppToken(aliceInstance, "testapp")
	bobContact = createContact(aliceInstance, "Bob", "bob@example.net")

	// Routing
	routes := map[string]func(*echo.Group){
		"/sharings": Routes,
	}
	ts = setup.GetTestServerMultipleRoutes(routes)

	os.Exit(setup.Run())
}

func createContact(inst *instance.Instance, name, email string) *contacts.Contact {
	c := &contacts.Contact{
		FullName: name,
		Email: []contacts.Email{
			contacts.Email{Address: email},
		},
	}
	err := couchdb.CreateDoc(inst, c)
	if err != nil {
		return nil
	}
	return c
}

func generateAppToken(inst *instance.Instance, slug string) string {
	rules := permissions.Set{
		permissions.Rule{
			Type:  iocozytests,
			Verbs: permissions.ALL,
		},
	}
	permReq := permissions.Permission{
		Permissions: rules,
		Type:        permissions.TypeWebapp,
		SourceID:    consts.Apps + "/" + slug,
	}
	err := couchdb.CreateDoc(inst, &permReq)
	if err != nil {
		return ""
	}
	manifest := &apps.WebappManifest{
		DocSlug:        slug,
		DocPermissions: rules,
	}
	err = couchdb.CreateNamedDocWithDB(inst, manifest)
	if err != nil {
		return ""
	}
	return inst.BuildAppToken(manifest, "")
}
<|endoftext|>


<commit_before>package jwt

import (
    "github.com/kataras/iris"
    "github.com/kataras/iris/core/router"
)

type jwtRouter struct {
    standard Standard
}

func (r *jwtRouter) Routes(server *iris.Application) {

    jwtRoutes := server.Party("/jwt")
    {
        jwtRoutes.Post("/generate", r.generate)

        jwtRoutes.Post("/auth", r.auth)

        jwtRoutes.Put("/upgrade", r.upgrade)
    }
}

func NewRouter(standard Standard) *router.Router {
    return &jwtRouter{
        standard: standard,
    }
}<commit_msg>fix bug<commit_after>package jwt

import (
    "github.com/kataras/iris"
    "github.com/kataras/iris/core/router"
)

type jwtRouter struct {
    standard Standard
}

func (r *jwtRouter) Routes(server *iris.Application) {

    jwtRoutes := server.Party("/jwt")
    {
        jwtRoutes.Post("/generate", r.generate)

        jwtRoutes.Post("/auth", r.auth)

        jwtRoutes.Put("/upgrade", r.upgrade)
    }
}

func NewRouter(standard Standard) *jwtRouter {
    return &jwtRouter{
        standard: standard,
    }
}<|endoftext|>


<commit_before>// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Program wpr records and replays web traffic.
package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"webpagereplay"
)

const longUsage = `%s [record|replay] [options] archive_file

   To record web pages:
     1. Start this program in record mode.
        $ GOPATH=$PWD go run src/wpr.go record archive.json
     2. Load the web pages you want to record in a web browser. It is important to
        clear browser caches before this so that all subresources are requested
        from the network.
     3. Kill the process to stop recording.

   To replay web pages:
     1. Start this program in replay mode with a previously recorded archive.
        $ GOPATH=$PWD go run src/wpr.go replay archive.json
     2. Load recorded pages in a web browser. A 404 will be served for any pages or
        resources not in the recorded archive.`

type CommonConfig struct {
	// Info aboth this command.
	cmd cli.Command

	// Flags common to all commands.
	host                                     string
	httpPort, httpsPort, httpSecureProxyPort int
	certFile, keyFile                        string
	injectScripts                            string

	// Computed state.
	root_cert    tls.Certificate
	transformers []webpagereplay.ResponseTransformer
}

type RecordCommand struct {
	common CommonConfig
	cmd    cli.Command
}

type ReplayCommand struct {
	common CommonConfig
	cmd    cli.Command

	// Custom flags for replay.
	rulesFile string
}

func (common *CommonConfig) Flags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:        "host",
			Value:       "",
			Usage:       "IP address to bind all servers to. Defaults to 127.0.0.1 if not specified.",
			Destination: &common.host,
		},
		cli.IntFlag{
			Name:        "http_port",
			Value:       0,
			Usage:       "Port number to listen on for HTTP requests, or 0 to disable.",
			Destination: &common.httpPort,
		},
		cli.IntFlag{
			Name:        "https_port",
			Value:       0,
			Usage:       "Port number to listen on for HTTPS requests, or 0 to disable.",
			Destination: &common.httpsPort,
		},
		cli.IntFlag{
			Name:        "https_to_http_port",
			Value:       0,
			Usage:       "Port number to listen on for HTTP proxy requests over an HTTPS connection, or 0 to disable.",
			Destination: &common.httpSecureProxyPort,
		},
		cli.StringFlag{
			Name:        "https_cert_file",
			Value:       "wpr_cert.pem",
			Usage:       "File containing a PEM-encoded X509 certificate to use with SSL.",
			Destination: &common.certFile,
		},
		cli.StringFlag{
			Name:        "https_key_file",
			Value:       "wpr_key.pem",
			Usage:       "File containing a PEM-encoded private key to use with SSL.",
			Destination: &common.keyFile,
		},
		cli.StringFlag{
			Name:  "inject_scripts",
			Value: "deterministic.js",
			Usage: "A comma separated list of JavaScript sources to inject in all pages. " +
				"By default a script is injected that eliminates sources of entropy " +
				"such as Date() and Math.random() deterministic. " +
				"CAUTION: Without deterministic.js, many pages will not replay.",
			Destination: &common.injectScripts,
		},
	}
}

func (common *CommonConfig) CheckArgs(c *cli.Context) error {
	if len(c.Args()) > 1 {
		return errors.New("too many args")
	}
	if len(c.Args()) != 1 {
		return errors.New("must specify archive_file")
	}
	if common.httpPort == 0 && common.httpsPort == 0 && common.httpSecureProxyPort == 0 {
		return errors.New("must specify at least one port flag")
	}

	// Load common configs.
	log.Printf("Loading cert from %v\n", common.certFile)
	log.Printf("Loading key from %v\n", common.keyFile)
	var err error
	common.root_cert, err = tls.LoadX509KeyPair(common.certFile, common.keyFile)
	if err != nil {
		return fmt.Errorf("error opening cert or key files: %v", err)
	}
	err = webpagereplay.InstallRoot(common.root_cert.Certificate[0])
	if err != nil {
		return fmt.Errorf("Install root failed: %v", err)
	}
	for _, scriptFile := range strings.Split(common.injectScripts, ",") {
		log.Printf("Loading script from %v\n", scriptFile)
		// Replace {{WPR_TIME_SEED_TIMESTAMP}} with current timestamp.
		current_time_ms := time.Now().Unix() * 1000
		replacements := map[string]string{"{{WPR_TIME_SEED_TIMESTAMP}}": strconv.FormatInt(current_time_ms, 10)}
		si, err := webpagereplay.NewScriptInjectorFromFile(scriptFile, replacements)
		if err != nil {
			return fmt.Errorf("error opening script %s: %v", scriptFile, err)
		}
		common.transformers = append(common.transformers, si)
	}

	return nil
}

func (r *RecordCommand) Flags() []cli.Flag {
	return r.common.Flags()
}

func (r *ReplayCommand) Flags() []cli.Flag {
	return append(r.common.Flags(),
		cli.StringFlag{
			Name:        "rules_file",
			Value:       "",
			Usage:       "File containing rules to apply to responses during replay",
			Destination: &r.rulesFile,
		})
}

func startServers(tlsconfig *tls.Config, httpHandler, httpsHandler http.Handler, common *CommonConfig) {
	type Server struct {
		Scheme string
		*http.Server
	}

	servers := []*Server{}
	if common.httpPort > 0 {
		servers = append(servers, &Server{
			Scheme: "http",
			Server: &http.Server{
				Addr:    fmt.Sprintf("%v:%v", common.host, common.httpPort),
				Handler: httpHandler,
			},
		})
	}
	if common.httpsPort > 0 {
		servers = append(servers, &Server{
			Scheme: "https",
			Server: &http.Server{
				Addr:      fmt.Sprintf("%v:%v", common.host, common.httpsPort),
				Handler:   httpsHandler,
				TLSConfig: tlsconfig,
			},
		})
	}
	if common.httpSecureProxyPort > 0 {
		servers = append(servers, &Server{
			Scheme: "https",
			Server: &http.Server{
				Addr:      fmt.Sprintf("%v:%v", common.host, common.httpSecureProxyPort),
				Handler:   httpHandler, // this server proxies HTTP requests over an HTTPS connection
				TLSConfig: nil,         // use the default since this is as a proxy, not a MITM server
			},
		})
	}

	for _, s := range servers {
		log.Printf("Starting server on %s://%s", s.Scheme, s.Addr)
		s := s
		go func() {
			var err error
			switch s.Scheme {
			case "http":
				err = s.ListenAndServe()
			case "https":
				err = s.ListenAndServeTLS(common.certFile, common.keyFile)
			default:
				panic(fmt.Sprintf("unknown s.Scheme: %s", s.Scheme))
			}
			if err != nil {
				log.Printf("Failed to start server on %s://%s: %v", s.Scheme, s.Addr, err)
			}
		}()
	}

	log.Printf("Use Ctrl-C to exit.")
	select {}
}

func (r *RecordCommand) Run(c *cli.Context) {
	archiveFileName := c.Args().First()
	archive, err := webpagereplay.OpenWritableArchive(archiveFileName)
	if err != nil {
		cli.ShowSubcommandHelp(c)
		os.Exit(1)
	}
	defer archive.Close()
	log.Printf("Opened archive %s", archiveFileName)

	// Install a SIGINT handler to close the archive before shutting down.
	go func() {
		sigchan := make(chan os.Signal, 1)
		signal.Notify(sigchan, os.Interrupt)
		<-sigchan
		log.Printf("Shutting down")
		log.Printf("Writing archive file to %s", archiveFileName)
		if err := archive.Close(); err != nil {
			log.Printf("Error flushing archive: %v", err)
		}
		webpagereplay.RemoveRoot()
		os.Exit(0)
	}()

	httpHandler := webpagereplay.NewRecordingProxy(archive, "http", r.common.transformers)
	httpsHandler := webpagereplay.NewRecordingProxy(archive, "https", r.common.transformers)
	tlsconfig, err := webpagereplay.RecordTLSConfig(r.common.root_cert, archive)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating TLSConfig: %v", err)
		os.Exit(1)
	}
	startServers(tlsconfig, httpHandler, httpsHandler, &r.common)
}

func (r *ReplayCommand) Run(c *cli.Context) {
	archiveFileName := c.Args().First()
	archive, err := webpagereplay.OpenArchive(archiveFileName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error openining archive file: %v", err)
		os.Exit(1)
	}
	log.Printf("Opened archive %s", archiveFileName)

	if r.rulesFile != "" {
		t, err := webpagereplay.NewRuleBasedTransformer(r.rulesFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening rules file %s: %v\n", r.rulesFile, err)
			os.Exit(1)
		}
		r.common.transformers = append(r.common.transformers, t)
		log.Printf("Loaded replay rules from %s", r.rulesFile)
	}

	httpHandler := webpagereplay.NewReplayingProxy(archive, "http", r.common.transformers)
	httpsHandler := webpagereplay.NewReplayingProxy(archive, "https", r.common.transformers)
	tlsconfig, err := webpagereplay.ReplayTLSConfig(r.common.root_cert, archive)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating TLSConfig: %v", err)
		os.Exit(1)
	}
	startServers(tlsconfig, httpHandler, httpsHandler, &r.common)
}

func main() {
	progName := filepath.Base(os.Args[0])

	var record RecordCommand
	var replay ReplayCommand

	record.cmd = cli.Command{
		Name:   "record",
		Usage:  "Record web pages to an archive",
		Flags:  record.Flags(),
		Before: record.common.CheckArgs,
		Action: record.Run,
	}

	replay.cmd = cli.Command{
		Name:   "replay",
		Usage:  "Replay a previously-recorded web page archive",
		Flags:  replay.Flags(),
		Before: replay.common.CheckArgs,
		Action: replay.Run,
	}

	app := cli.NewApp()
	app.Commands = []cli.Command{record.cmd, replay.cmd}
	app.Usage = "Web Page Replay"
	app.UsageText = fmt.Sprintf(longUsage, progName)
	app.HideVersion = true
	app.Version = ""
	app.Writer = os.Stderr
	app.RunAndExitOnError()
}
<commit_msg>[wpr-go] Pick an availble port to start servers if no port is specified<commit_after>// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

// Program wpr records and replays web traffic.
package main

import (
	"crypto/tls"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/codegangsta/cli"
	"webpagereplay"
)

const longUsage = `%s [record|replay] [options] archive_file

   To record web pages:
     1. Start this program in record mode.
        $ GOPATH=$PWD go run src/wpr.go record archive.json
     2. Load the web pages you want to record in a web browser. It is important to
        clear browser caches before this so that all subresources are requested
        from the network.
     3. Kill the process to stop recording.

   To replay web pages:
     1. Start this program in replay mode with a previously recorded archive.
        $ GOPATH=$PWD go run src/wpr.go replay archive.json
     2. Load recorded pages in a web browser. A 404 will be served for any pages or
        resources not in the recorded archive.`

type CommonConfig struct {
	// Info aboth this command.
	cmd cli.Command

	// Flags common to all commands.
	host                                     string
	httpPort, httpsPort, httpSecureProxyPort int
	certFile, keyFile                        string
	injectScripts                            string

	// Computed state.
	root_cert    tls.Certificate
	transformers []webpagereplay.ResponseTransformer
}

type RecordCommand struct {
	common CommonConfig
	cmd    cli.Command
}

type ReplayCommand struct {
	common CommonConfig
	cmd    cli.Command

	// Custom flags for replay.
	rulesFile string
}

func (common *CommonConfig) Flags() []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:        "host",
			Value:       "",
			Usage:       "IP address to bind all servers to. Defaults to 127.0.0.1 if not specified.",
			Destination: &common.host,
		},
		cli.IntFlag{
			Name:        "http_port",
			Value:       -1,
			Usage:       "Port number to listen on for HTTP requests, 0 to use any port, or -1 to disable.",
			Destination: &common.httpPort,
		},
		cli.IntFlag{
			Name:        "https_port",
			Value:       -1,
			Usage:       "Port number to listen on for HTTPS requests, 0 to use any port, or -1 to disable.",
			Destination: &common.httpsPort,
		},
		cli.IntFlag{
			Name:        "https_to_http_port",
			Value:       -1,
			Usage:       "Port number to listen on for HTTP proxy requests over an HTTPS connection, 0 to use any port, or -1 to disable.",
			Destination: &common.httpSecureProxyPort,
		},
		cli.StringFlag{
			Name:        "https_cert_file",
			Value:       "wpr_cert.pem",
			Usage:       "File containing a PEM-encoded X509 certificate to use with SSL.",
			Destination: &common.certFile,
		},
		cli.StringFlag{
			Name:        "https_key_file",
			Value:       "wpr_key.pem",
			Usage:       "File containing a PEM-encoded private key to use with SSL.",
			Destination: &common.keyFile,
		},
		cli.StringFlag{
			Name:  "inject_scripts",
			Value: "deterministic.js",
			Usage: "A comma separated list of JavaScript sources to inject in all pages. " +
				"By default a script is injected that eliminates sources of entropy " +
				"such as Date() and Math.random() deterministic. " +
				"CAUTION: Without deterministic.js, many pages will not replay.",
			Destination: &common.injectScripts,
		},
	}
}

func (common *CommonConfig) CheckArgs(c *cli.Context) error {
	if len(c.Args()) > 1 {
		return errors.New("too many args")
	}
	if len(c.Args()) != 1 {
		return errors.New("must specify archive_file")
	}
	if common.httpPort == -1 && common.httpsPort == -1 && common.httpSecureProxyPort == -1 {
		return errors.New("must specify at least one port flag")
	}

	// Load common configs.
	log.Printf("Loading cert from %v\n", common.certFile)
	log.Printf("Loading key from %v\n", common.keyFile)
	var err error
	common.root_cert, err = tls.LoadX509KeyPair(common.certFile, common.keyFile)
	if err != nil {
		return fmt.Errorf("error opening cert or key files: %v", err)
	}
	err = webpagereplay.InstallRoot(common.root_cert.Certificate[0])
	if err != nil {
		return fmt.Errorf("Install root failed: %v", err)
	}
	for _, scriptFile := range strings.Split(common.injectScripts, ",") {
		log.Printf("Loading script from %v\n", scriptFile)
		// Replace {{WPR_TIME_SEED_TIMESTAMP}} with current timestamp.
		current_time_ms := time.Now().Unix() * 1000
		replacements := map[string]string{"{{WPR_TIME_SEED_TIMESTAMP}}": strconv.FormatInt(current_time_ms, 10)}
		si, err := webpagereplay.NewScriptInjectorFromFile(scriptFile, replacements)
		if err != nil {
			return fmt.Errorf("error opening script %s: %v", scriptFile, err)
		}
		common.transformers = append(common.transformers, si)
	}

	return nil
}

func (r *RecordCommand) Flags() []cli.Flag {
	return r.common.Flags()
}

func (r *ReplayCommand) Flags() []cli.Flag {
	return append(r.common.Flags(),
		cli.StringFlag{
			Name:        "rules_file",
			Value:       "",
			Usage:       "File containing rules to apply to responses during replay",
			Destination: &r.rulesFile,
		})
}

func getAvailablePort() int {
	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	if err != nil {
		panic(err)
	}
	listener, err := net.ListenTCP("tcp", addr)
	if err != nil {
		panic(err)
	}
	defer listener.Close()
	return listener.Addr().(*net.TCPAddr).Port
}

func startServers(tlsconfig *tls.Config, httpHandler, httpsHandler http.Handler, common *CommonConfig) {
	type Server struct {
		Scheme string
		*http.Server
	}

	servers := []*Server{}

	if common.httpPort > -1 {
		if common.httpPort == 0 {
			common.httpPort = getAvailablePort()
		}
		servers = append(servers, &Server{
			Scheme: "http",
			Server: &http.Server{
				Addr:    fmt.Sprintf("%v:%v", common.host, common.httpPort),
				Handler: httpHandler,
			},
		})
	}
	if common.httpsPort > -1 {
		if common.httpsPort == 0 {
			common.httpsPort = getAvailablePort()
		}
		servers = append(servers, &Server{
			Scheme: "https",
			Server: &http.Server{
				Addr:      fmt.Sprintf("%v:%v", common.host, common.httpsPort),
				Handler:   httpsHandler,
				TLSConfig: tlsconfig,
			},
		})
	}
	if common.httpSecureProxyPort > -1 {
		if common.httpSecureProxyPort == 0 {
			common.httpSecureProxyPort = getAvailablePort()
		}
		servers = append(servers, &Server{
			Scheme: "https",
			Server: &http.Server{
				Addr:      fmt.Sprintf("%v:%v", common.host, common.httpSecureProxyPort),
				Handler:   httpHandler, // this server proxies HTTP requests over an HTTPS connection
				TLSConfig: nil,         // use the default since this is as a proxy, not a MITM server
			},
		})
	}

	for _, s := range servers {
		log.Printf("Starting server on %s://%s", s.Scheme, s.Addr)
		s := s
		go func() {
			var err error
			switch s.Scheme {
			case "http":
				err = s.ListenAndServe()
			case "https":
				err = s.ListenAndServeTLS(common.certFile, common.keyFile)
			default:
				panic(fmt.Sprintf("unknown s.Scheme: %s", s.Scheme))
			}
			if err != nil {
				log.Printf("Failed to start server on %s://%s: %v", s.Scheme, s.Addr, err)
			}
		}()
	}

	log.Printf("Use Ctrl-C to exit.")
	select {}
}

func (r *RecordCommand) Run(c *cli.Context) {
	archiveFileName := c.Args().First()
	archive, err := webpagereplay.OpenWritableArchive(archiveFileName)
	if err != nil {
		cli.ShowSubcommandHelp(c)
		os.Exit(1)
	}
	defer archive.Close()
	log.Printf("Opened archive %s", archiveFileName)

	// Install a SIGINT handler to close the archive before shutting down.
	go func() {
		sigchan := make(chan os.Signal, 1)
		signal.Notify(sigchan, os.Interrupt)
		<-sigchan
		log.Printf("Shutting down")
		log.Printf("Writing archive file to %s", archiveFileName)
		if err := archive.Close(); err != nil {
			log.Printf("Error flushing archive: %v", err)
		}
		webpagereplay.RemoveRoot()
		os.Exit(0)
	}()

	httpHandler := webpagereplay.NewRecordingProxy(archive, "http", r.common.transformers)
	httpsHandler := webpagereplay.NewRecordingProxy(archive, "https", r.common.transformers)
	tlsconfig, err := webpagereplay.RecordTLSConfig(r.common.root_cert, archive)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating TLSConfig: %v", err)
		os.Exit(1)
	}
	startServers(tlsconfig, httpHandler, httpsHandler, &r.common)
}

func (r *ReplayCommand) Run(c *cli.Context) {
	archiveFileName := c.Args().First()
	archive, err := webpagereplay.OpenArchive(archiveFileName)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error openining archive file: %v", err)
		os.Exit(1)
	}
	log.Printf("Opened archive %s", archiveFileName)

	if r.rulesFile != "" {
		t, err := webpagereplay.NewRuleBasedTransformer(r.rulesFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error opening rules file %s: %v\n", r.rulesFile, err)
			os.Exit(1)
		}
		r.common.transformers = append(r.common.transformers, t)
		log.Printf("Loaded replay rules from %s", r.rulesFile)
	}

	httpHandler := webpagereplay.NewReplayingProxy(archive, "http", r.common.transformers)
	httpsHandler := webpagereplay.NewReplayingProxy(archive, "https", r.common.transformers)
	tlsconfig, err := webpagereplay.ReplayTLSConfig(r.common.root_cert, archive)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error creating TLSConfig: %v", err)
		os.Exit(1)
	}
	startServers(tlsconfig, httpHandler, httpsHandler, &r.common)
}

func main() {
	progName := filepath.Base(os.Args[0])

	var record RecordCommand
	var replay ReplayCommand

	record.cmd = cli.Command{
		Name:   "record",
		Usage:  "Record web pages to an archive",
		Flags:  record.Flags(),
		Before: record.common.CheckArgs,
		Action: record.Run,
	}

	replay.cmd = cli.Command{
		Name:   "replay",
		Usage:  "Replay a previously-recorded web page archive",
		Flags:  replay.Flags(),
		Before: replay.common.CheckArgs,
		Action: replay.Run,
	}

	app := cli.NewApp()
	app.Commands = []cli.Command{record.cmd, replay.cmd}
	app.Usage = "Web Page Replay"
	app.UsageText = fmt.Sprintf(longUsage, progName)
	app.HideVersion = true
	app.Version = ""
	app.Writer = os.Stderr
	app.RunAndExitOnError()
}
<|endoftext|>


<commit_before>package timestamp

// Helper for using timestamp in the log output

/* Copyright (c) 2016-2017, Alexander I.Grafov
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of kvlog nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ॐ तारे तुत्तारे तुरे स्व */

import (
	"time"

	"github.com/grafov/kiwi"
)

var TimestampKey = "timestamp"

// Set adds "timestamp" field to the context.
func Set(format string) *kiwi.Pair {
	return &kiwi.Pair{
		Key:  TimestampKey,
		Val:  "",
		Eval: func() string { return time.Now().Format(format) },
		Type: kiwi.TimeVal,
	}
}
<commit_msg>Set "at" as default key for the timestamp<commit_after>package timestamp

// Helper for using timestamp in the log output

/* Copyright (c) 2016-2018, Alexander I.Grafov <grafov@gmail.com>
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
  list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
  this list of conditions and the following disclaimer in the documentation
  and/or other materials provided with the distribution.

* Neither the name of kvlog nor the names of its
  contributors may be used to endorse or promote products derived from
  this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

ॐ तारे तुत्तारे तुरे स्व */

import (
	"time"

	"github.com/grafov/kiwi"
)

var TimestampKey = "at"

// Set adds "timestamp" field to the context.
func Set(format string) *kiwi.Pair {
	return &kiwi.Pair{
		Key:  TimestampKey,
		Val:  "",
		Eval: func() string { return time.Now().Format(format) },
		Type: kiwi.TimeVal,
	}
}
<|endoftext|>


<commit_before>package models

import (
	"log"
	"testing"
	"time"
)

func TestGetCharacterKnownAssociates(t *testing.T) {
	_, err := database.Exec(`
			INSERT IGNORE INTO evedata.characterAssociations VALUES
			 (1001, 1002, 3, 1, UTC_TIMESTAMP);
		`)
	if err != nil {
		t.Error(err)
		return
	}

	_, err = GetCharacterKnownAssociates(1001)
	if err != nil {
		t.Error(err)
		return
	}
}

func TestGetCorporationKnownAssociates(t *testing.T) {
	_, err := GetCorporationKnownAssociates(1001)
	if err != nil {
		t.Error(err)
		return
	}
}

func TestGetAllianceKnownAssociates(t *testing.T) {
	_, err := GetAllianceKnownAssociates(1001)
	if err != nil {
		t.Error(err)
		return
	}
}

func TestBuildRelationships(t *testing.T) {
	err := UpdateCorporationHistory(1001, 147035273, 100000222, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}
	err = UpdateCorporationHistory(1002, 147035273, 100000222, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}
	err = UpdateCorporationHistory(1003, 147035273, 100000222, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}

	err = BuildRelationships()
	if err != nil {
		t.Error(err)
	}
}
<commit_msg>Update association test<commit_after>package models

import (
	"log"
	"testing"
	"time"
)

func TestGetCharacterKnownAssociates(t *testing.T) {
	_, err := database.Exec(`
			INSERT IGNORE INTO evedata.characterAssociations VALUES
			 (1001, 1002, 3, 1, UTC_TIMESTAMP);
		`)
	if err != nil {
		t.Error(err)
		return
	}

	_, err = GetCharacterKnownAssociates(1001)
	if err != nil {
		t.Error(err)
		return
	}
}

func TestGetCorporationKnownAssociates(t *testing.T) {
	_, err := GetCorporationKnownAssociates(1001)
	if err != nil {
		t.Error(err)
		return
	}
}

func TestGetAllianceKnownAssociates(t *testing.T) {
	_, err := GetAllianceKnownAssociates(1001)
	if err != nil {
		t.Error(err)
		return
	}
}

func TestBuildRelationships(t *testing.T) {

	err := UpdateCharacter(1001, "dude", 1, 1, 147035273, 0, 1, "male", -10, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}

	err = UpdateCharacter(1002, "dude 2", 1, 1, 147035273, 0, 2, "Female", -10, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}

	err = UpdateCharacter(1003, "dude 3", 1, 1, 147035273, 0, 2, "Female", -10, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}

	err = UpdateCorporationHistory(1001, 147035273, 100000222, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}
	err = UpdateCorporationHistory(1002, 147035273, 100000223, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}
	err = UpdateCorporationHistory(1003, 147035273, 100000224, time.Now())
	if err != nil {
		log.Fatal(err)
		return
	}

	err = BuildRelationships()
	if err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file was auto-generated via go generate.
// DO NOT UPDATE MANUALLY
package main_test

import "testing"
import "os"

import "v.io/core/veyron/lib/testutil"
import "v.io/core/veyron/lib/testutil/v23tests"

func TestMain(m *testing.M) {
	testutil.Init()
	cleanup := v23tests.UseSharedBinDir()
	r := m.Run()
	cleanup()
	os.Exit(r)
}

func TestV23Tunneld(t *testing.T) {
	v23tests.RunTest(t, V23TestTunneld)
}
<commit_msg>apps: cleanup of GO.PACKAGE indentation and v23 tests header<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This file was auto-generated via go generate.
// DO NOT UPDATE MANUALLY
package main_test

import "testing"
import "os"

import "v.io/core/veyron/lib/testutil"
import "v.io/core/veyron/lib/testutil/v23tests"

func TestMain(m *testing.M) {
	testutil.Init()
	cleanup := v23tests.UseSharedBinDir()
	r := m.Run()
	cleanup()
	os.Exit(r)
}

func TestV23Tunneld(t *testing.T) {
	v23tests.RunTest(t, V23TestTunneld)
}
<|endoftext|>


<commit_before>// Copyright © 2017 Makoto Ito
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package word2vec

import (
	"github.com/ynqa/word-embedding/models"
	"github.com/ynqa/word-embedding/utils/fileio"
	"gopkg.in/cheggaaa/pb.v1"
)

// Word2Vec stores the model, and optimizer.
type Word2Vec struct {
	models.Common
	// Skip-Gram, or CBOW.
	Model Model
	// Hierarchical Softmax, or Negative Sampling.
	Opt   Optimizer
}

// PreTrain prepares word statistical info for training.
func (w Word2Vec) PreTrain() error {
	return w.Opt.PreTrain()
}

// Run executes training words' vector.
func (w Word2Vec) Run() error {
	progressor := pb.StartNew(GetWords())
	if err := fileio.Load(w.Common.InputFile, w.iterator(progressor)); err != nil {
		return err
	}
	progressor.Finish()
	return nil
}

func (w Word2Vec) iterator(progressor *pb.ProgressBar) func(words []string) {
	return func(words []string) {
		for index := range words {
			w.Model.Train(words, index, w.Opt.Update)
			progressor.Increment()
		}
	}
}

// Save calls word map itself.
func (w Word2Vec) Save() error {
	return GlobalWordMap.Save(w.Common.OutputFile)
}
<commit_msg>Adapt go fmt<commit_after>// Copyright © 2017 Makoto Ito
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package word2vec

import (
	"github.com/ynqa/word-embedding/models"
	"github.com/ynqa/word-embedding/utils/fileio"
	"gopkg.in/cheggaaa/pb.v1"
)

// Word2Vec stores the model, and optimizer.
type Word2Vec struct {
	models.Common
	// Skip-Gram, or CBOW.
	Model Model
	// Hierarchical Softmax, or Negative Sampling.
	Opt Optimizer
}

// PreTrain prepares word statistical info for training.
func (w Word2Vec) PreTrain() error {
	return w.Opt.PreTrain()
}

// Run executes training words' vector.
func (w Word2Vec) Run() error {
	progressor := pb.StartNew(GetWords())
	if err := fileio.Load(w.Common.InputFile, w.iterator(progressor)); err != nil {
		return err
	}
	progressor.Finish()
	return nil
}

func (w Word2Vec) iterator(progressor *pb.ProgressBar) func(words []string) {
	return func(words []string) {
		for index := range words {
			w.Model.Train(words, index, w.Opt.Update)
			progressor.Increment()
		}
	}
}

// Save calls word map itself.
func (w Word2Vec) Save() error {
	return GlobalWordMap.Save(w.Common.OutputFile)
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venafi

import (
	"crypto"
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"testing"
	"time"

	"github.com/Venafi/vcert/pkg/certificate"
	"github.com/Venafi/vcert/pkg/endpoint"
	"github.com/Venafi/vcert/pkg/venafi/fake"

	internalvanafiapi "github.com/jetstack/cert-manager/pkg/internal/venafi/api"
	internalfake "github.com/jetstack/cert-manager/pkg/internal/venafi/fake"
	"github.com/jetstack/cert-manager/pkg/util"
	"github.com/jetstack/cert-manager/pkg/util/pki"
)

func checkCertificateIssued(t *testing.T, csrPEM []byte, resp []byte) {
	if len(resp) == 0 {
		t.Errorf("expected IssueResponse to be non-nil")
		t.FailNow()
		return
	}

	csr, err := pki.DecodeX509CertificateRequestBytes(csrPEM)
	if err != nil {
		t.Errorf("failed to decode CSR PEM: %s", err)
		return
	}

	crt, err := pki.DecodeX509CertificateBytes(resp)
	if err != nil {
		t.Errorf("unable to decode x509 certificate: %v", err)
		return
	}

	ok, err := pki.PublicKeyMatchesCSR(crt.PublicKey, csr)
	if err != nil {
		t.Errorf("error checking private key: %v", err)
		return
	}
	if !ok {
		t.Errorf("private key does not match certificate")
	}

	// validate the common name is correct
	expectedCN := csr.Subject.CommonName
	if expectedCN != crt.Subject.CommonName {
		t.Errorf("expected common name to be %q but it was %q", expectedCN, crt.Subject.CommonName)
	}

	// validate the dns names are correct
	expectedDNSNames := csr.DNSNames
	if !util.EqualUnsorted(crt.DNSNames, expectedDNSNames) {
		t.Errorf("expected dns names to be %q but it was %q", expectedDNSNames, crt.DNSNames)
	}
}

func checkNoCertificateIssued(t *testing.T, csrPEM []byte, resp []byte) {
	if len(resp) > 0 {
		t.Errorf("expected no response with error but got=%s", resp)
	}
}

func generateCSR(t *testing.T, sk crypto.Signer, commonName string, dnsNames []string) []byte {
	template := x509.CertificateRequest{
		Subject: pkix.Name{
			CommonName: commonName,
		},
		SignatureAlgorithm: x509.SHA256WithRSA,
		DNSNames:           dnsNames,
	}

	csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &template, sk)
	if err != nil {
		t.Error(err)
		t.FailNow()
	}

	csr := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes})
	return csr
}

func TestSign(t *testing.T) {
	sk, err := pki.GenerateRSAPrivateKey(2048)
	if err != nil {
		t.Error(err)
		t.FailNow()
	}

	csrPEM := generateCSR(t, sk, "common-name", []string{
		"foo.example.com", "bar.example.com"})

	csrNonePEM := generateCSR(t, sk, "", []string{})

	tests := map[string]testSignT{
		"if reading the zone configuration fails then error": {
			csrPEM:      csrPEM,
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
			client: internalfake.Connector{
				ReadZoneConfigurationFunc: func() (*endpoint.ZoneConfiguration, error) {
					return nil, errors.New("zone configuration error")
				},
			}.Default(),
		},
		"if validating the certificate fails then error": {
			csrPEM:      csrPEM,
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
			client: internalfake.Connector{
				ReadZoneConfigurationFunc: func() (*endpoint.ZoneConfiguration, error) {
					return &endpoint.ZoneConfiguration{
						Policy: endpoint.Policy{
							SubjectCNRegexes: []string{"foo"},
						},
					}, nil
				},
			}.Default(),
		},
		"a badly formed CSR should error": {
			csrPEM:      []byte("a badly formed CSR"),
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
		},
		"if requesting the certificate fails, sign should error": {
			csrPEM: csrPEM,
			client: internalfake.Connector{
				RequestCertificateFunc: func(*certificate.Request) (string, error) {
					return "", errors.New("request error")
				},
			}.Default(),
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
		},
		"if retrive certificate fails, sign should error": {
			csrPEM: csrPEM,
			client: internalfake.Connector{
				RetrieveCertificateFunc: func(*certificate.Request) (*certificate.PEMCollection, error) {
					return nil, errors.New("request error")
				},
			}.Default(),
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
		},
		"if no Common Name, DNS Name, or URI SANs in CSR then error": {
			csrPEM:      csrNonePEM,
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
		},
		"obtain a certificate with DNS names specified": {
			csrPEM:      csrPEM,
			checkFn:     checkCertificateIssued,
			expectedErr: false,
		},
		"obtain a certificate with custom fields specified": {
			csrPEM:       csrPEM,
			customFields: []internalvanafiapi.CustomField{{Name: "test", Value: "ok"}},
			client: internalfake.Connector{
				RetrieveCertificateFunc: func(r *certificate.Request) (*certificate.PEMCollection, error) {
					if len(r.CustomFields) == 0 {
						return nil, errors.New("custom fields not set")
					}
					if r.CustomFields[0].Name != "test" || r.CustomFields[0].Value != "ok" {
						return nil, errors.New("custom fields content not correct")
					}
					return internalfake.Connector{}.Default().RetrieveCertificate(r) // hack to return to normal
				},
			}.Default(),
			checkFn:     checkCertificateIssued,
			expectedErr: false,
		},
		"If invalid custom field type found the error": {
			csrPEM:       csrPEM,
			customFields: []internalvanafiapi.CustomField{{Name: "test", Value: "ok", Type: "Bool"}},
			checkFn:      checkNoCertificateIssued,
			expectedErr:  true,
		},
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			test.runTest(t)
		})
	}
}

type testSignT struct {
	csrPEM []byte
	client connector

	expectedErr bool

	customFields []internalvanafiapi.CustomField

	checkFn func(*testing.T, []byte, []byte)
}

func (s *testSignT) runTest(t *testing.T) {
	client := s.client
	if client == nil {
		client = fake.NewConnector(true, nil)
	}

	v := &Venafi{
		client: client,
	}

	resp, err := v.Sign(s.csrPEM, time.Minute, s.customFields)
	if err != nil && !s.expectedErr {
		t.Errorf("expected to not get an error, but got: %v", err)
	}
	if err == nil && s.expectedErr {
		t.Errorf("expected to get an error but did not get one")
	}

	if s.checkFn != nil {
		s.checkFn(t, s.csrPEM, resp)
	}
}
<commit_msg>spelling: retrieve<commit_after>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package venafi

import (
	"crypto"
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/pem"
	"errors"
	"testing"
	"time"

	"github.com/Venafi/vcert/pkg/certificate"
	"github.com/Venafi/vcert/pkg/endpoint"
	"github.com/Venafi/vcert/pkg/venafi/fake"

	internalvanafiapi "github.com/jetstack/cert-manager/pkg/internal/venafi/api"
	internalfake "github.com/jetstack/cert-manager/pkg/internal/venafi/fake"
	"github.com/jetstack/cert-manager/pkg/util"
	"github.com/jetstack/cert-manager/pkg/util/pki"
)

func checkCertificateIssued(t *testing.T, csrPEM []byte, resp []byte) {
	if len(resp) == 0 {
		t.Errorf("expected IssueResponse to be non-nil")
		t.FailNow()
		return
	}

	csr, err := pki.DecodeX509CertificateRequestBytes(csrPEM)
	if err != nil {
		t.Errorf("failed to decode CSR PEM: %s", err)
		return
	}

	crt, err := pki.DecodeX509CertificateBytes(resp)
	if err != nil {
		t.Errorf("unable to decode x509 certificate: %v", err)
		return
	}

	ok, err := pki.PublicKeyMatchesCSR(crt.PublicKey, csr)
	if err != nil {
		t.Errorf("error checking private key: %v", err)
		return
	}
	if !ok {
		t.Errorf("private key does not match certificate")
	}

	// validate the common name is correct
	expectedCN := csr.Subject.CommonName
	if expectedCN != crt.Subject.CommonName {
		t.Errorf("expected common name to be %q but it was %q", expectedCN, crt.Subject.CommonName)
	}

	// validate the dns names are correct
	expectedDNSNames := csr.DNSNames
	if !util.EqualUnsorted(crt.DNSNames, expectedDNSNames) {
		t.Errorf("expected dns names to be %q but it was %q", expectedDNSNames, crt.DNSNames)
	}
}

func checkNoCertificateIssued(t *testing.T, csrPEM []byte, resp []byte) {
	if len(resp) > 0 {
		t.Errorf("expected no response with error but got=%s", resp)
	}
}

func generateCSR(t *testing.T, sk crypto.Signer, commonName string, dnsNames []string) []byte {
	template := x509.CertificateRequest{
		Subject: pkix.Name{
			CommonName: commonName,
		},
		SignatureAlgorithm: x509.SHA256WithRSA,
		DNSNames:           dnsNames,
	}

	csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &template, sk)
	if err != nil {
		t.Error(err)
		t.FailNow()
	}

	csr := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrBytes})
	return csr
}

func TestSign(t *testing.T) {
	sk, err := pki.GenerateRSAPrivateKey(2048)
	if err != nil {
		t.Error(err)
		t.FailNow()
	}

	csrPEM := generateCSR(t, sk, "common-name", []string{
		"foo.example.com", "bar.example.com"})

	csrNonePEM := generateCSR(t, sk, "", []string{})

	tests := map[string]testSignT{
		"if reading the zone configuration fails then error": {
			csrPEM:      csrPEM,
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
			client: internalfake.Connector{
				ReadZoneConfigurationFunc: func() (*endpoint.ZoneConfiguration, error) {
					return nil, errors.New("zone configuration error")
				},
			}.Default(),
		},
		"if validating the certificate fails then error": {
			csrPEM:      csrPEM,
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
			client: internalfake.Connector{
				ReadZoneConfigurationFunc: func() (*endpoint.ZoneConfiguration, error) {
					return &endpoint.ZoneConfiguration{
						Policy: endpoint.Policy{
							SubjectCNRegexes: []string{"foo"},
						},
					}, nil
				},
			}.Default(),
		},
		"a badly formed CSR should error": {
			csrPEM:      []byte("a badly formed CSR"),
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
		},
		"if requesting the certificate fails, sign should error": {
			csrPEM: csrPEM,
			client: internalfake.Connector{
				RequestCertificateFunc: func(*certificate.Request) (string, error) {
					return "", errors.New("request error")
				},
			}.Default(),
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
		},
		"if retrieve certificate fails, sign should error": {
			csrPEM: csrPEM,
			client: internalfake.Connector{
				RetrieveCertificateFunc: func(*certificate.Request) (*certificate.PEMCollection, error) {
					return nil, errors.New("request error")
				},
			}.Default(),
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
		},
		"if no Common Name, DNS Name, or URI SANs in CSR then error": {
			csrPEM:      csrNonePEM,
			checkFn:     checkNoCertificateIssued,
			expectedErr: true,
		},
		"obtain a certificate with DNS names specified": {
			csrPEM:      csrPEM,
			checkFn:     checkCertificateIssued,
			expectedErr: false,
		},
		"obtain a certificate with custom fields specified": {
			csrPEM:       csrPEM,
			customFields: []internalvanafiapi.CustomField{{Name: "test", Value: "ok"}},
			client: internalfake.Connector{
				RetrieveCertificateFunc: func(r *certificate.Request) (*certificate.PEMCollection, error) {
					if len(r.CustomFields) == 0 {
						return nil, errors.New("custom fields not set")
					}
					if r.CustomFields[0].Name != "test" || r.CustomFields[0].Value != "ok" {
						return nil, errors.New("custom fields content not correct")
					}
					return internalfake.Connector{}.Default().RetrieveCertificate(r) // hack to return to normal
				},
			}.Default(),
			checkFn:     checkCertificateIssued,
			expectedErr: false,
		},
		"If invalid custom field type found the error": {
			csrPEM:       csrPEM,
			customFields: []internalvanafiapi.CustomField{{Name: "test", Value: "ok", Type: "Bool"}},
			checkFn:      checkNoCertificateIssued,
			expectedErr:  true,
		},
	}

	for name, test := range tests {
		t.Run(name, func(t *testing.T) {
			test.runTest(t)
		})
	}
}

type testSignT struct {
	csrPEM []byte
	client connector

	expectedErr bool

	customFields []internalvanafiapi.CustomField

	checkFn func(*testing.T, []byte, []byte)
}

func (s *testSignT) runTest(t *testing.T) {
	client := s.client
	if client == nil {
		client = fake.NewConnector(true, nil)
	}

	v := &Venafi{
		client: client,
	}

	resp, err := v.Sign(s.csrPEM, time.Minute, s.customFields)
	if err != nil && !s.expectedErr {
		t.Errorf("expected to not get an error, but got: %v", err)
	}
	if err == nil && s.expectedErr {
		t.Errorf("expected to get an error but did not get one")
	}

	if s.checkFn != nil {
		s.checkFn(t, s.csrPEM, resp)
	}
}
<|endoftext|>


<commit_before>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	"github.com/openshift/origin/pkg/test/ginkgo"
)

type testCategorizer struct {
	by        string
	substring string
}

func testPodSandboxCreation(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	const testName = "[sig-network] pods should successfully create sandboxes"
	// we can further refine this signal by subdividing different failure modes if it is pertinent.  Right now I'm seeing
	// 1. error reading container (probably exited) json message: EOF
	// 2. dial tcp 10.0.76.225:6443: i/o timeout
	// 3. error getting pod: pods "terminate-cmd-rpofb45fa14c-96bb-40f7-bd9e-346721740cac" not found
	// 4. write child: broken pipe
	bySubStrings := []testCategorizer{
		{by: " by reading container", substring: "error reading container (probably exited) json message: EOF"},
		{by: " by not timing out", substring: "i/o timeout"},
		{by: " by writing network status", substring: "error setting the networks status"},
		{by: " by getting pod", substring: " error getting pod: pods"},
		{by: " by writing child", substring: "write child: broken pipe"},
		{by: " by other", substring: " "}, // always matches
	}

	failures := []string{}
	flakes := []string{}
	eventsForPods := getEventsByPod(events)
	for _, event := range events {
		if !strings.Contains(event.Message, "reason/FailedCreatePodSandBox Failed to create pod sandbox") {
			continue
		}
		if strings.Contains(event.Message, "Multus") && strings.Contains(event.Message, "error getting pod") && (strings.Contains(event.Message, "connection refused") || strings.Contains(event.Message, "i/o timeout")) {
			flakes = append(flakes, fmt.Sprintf("%v - multus is unable to get pods due to LB disruption https://bugzilla.redhat.com/show_bug.cgi?id=1927264 - %v", event.Locator, event.Message))
			continue
		}
		deletionTime := getPodDeletionTime(eventsForPods[event.Locator], event.Locator)
		if deletionTime == nil {
			// this indicates a failure to create the sandbox that should not happen
			failures = append(failures, fmt.Sprintf("%v - never deleted - %v", event.Locator, event.Message))
		} else {
			timeBetweenDeleteAndFailure := event.From.Sub(*deletionTime)
			switch {
			case timeBetweenDeleteAndFailure < 1*time.Second:
				// nothing here, one second is close enough to be ok, the kubelet and CNI just didn't know
			case timeBetweenDeleteAndFailure < 5*time.Second:
				// withing five seconds, it ought to be long enough to know, but it's close enough to flake and not fail
				flakes = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			case deletionTime.Before(event.From):
				// something went wrong.  More than five seconds after the pod ws deleted, the CNI is trying to set up pod sandboxes and can't
				failures = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			default:
				// something went wrong.  deletion happend after we had a failure to create the pod sandbox
				failures = append(failures, fmt.Sprintf("%v - deletion came AFTER sandbox failure - %v", event.Locator, event.Message))
			}
		}
	}
	if len(failures) == 0 && len(flakes) == 0 {
		successes := []*ginkgo.JUnitTestCase{}
		for _, by := range bySubStrings {
			successes = append(successes, &ginkgo.JUnitTestCase{Name: testName + by.by})
		}
		return successes
	}

	ret := []*ginkgo.JUnitTestCase{}
	failuresBySubtest, flakesBySubtest := categorizeBySubset(bySubStrings, failures, flakes)

	// now iterate the individual failures to create failure entries
	for by, subFailures := range failuresBySubtest {
		failure := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFailures, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFailures), strings.Join(subFailures, "\n")),
			},
		}
		ret = append(ret, failure)
	}
	for by, subFlakes := range flakesBySubtest {
		flake := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFlakes, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFlakes), strings.Join(subFlakes, "\n")),
			},
		}
		ret = append(ret, flake)
		// write a passing test to trigger detection of this issue as a flake. Doing this first to try to see how frequent the issue actually is
		success := &ginkgo.JUnitTestCase{
			Name: testName + by,
		}
		ret = append(ret, success)
	}

	return append(ret)
}

// categorizeBySubset returns a map keyed by category for failures and flakes.  If a category is present in both failures and flakes, all are listed under failures.
func categorizeBySubset(categorizers []testCategorizer, failures, flakes []string) (map[string][]string, map[string][]string) {
	failuresBySubtest := map[string][]string{}
	flakesBySubtest := map[string][]string{}
	for _, failure := range failures {
		for _, by := range categorizers {
			if strings.Contains(failure, by.substring) {
				failuresBySubtest[by.by] = append(failuresBySubtest[by.by], failure)
				break // break after first match so we only add each failure one bucket
			}
		}
	}

	for _, flake := range flakes {
		for _, by := range categorizers {
			if strings.Contains(flake, by.substring) {
				if _, isFailure := failuresBySubtest[by.by]; isFailure {
					failuresBySubtest[by.by] = append(failuresBySubtest[by.by], flake)
				} else {
					flakesBySubtest[by.by] = append(flakesBySubtest[by.by], flake)
				}
				break // break after first match so we only add each failure one bucket
			}
		}
	}
	return failuresBySubtest, flakesBySubtest
}

// getEventsByPod returns map keyed by pod locator with all events associated with it.
func getEventsByPod(events monitorapi.Intervals) map[string]monitorapi.Intervals {
	eventsByPods := map[string]monitorapi.Intervals{}
	for _, event := range events {
		if !strings.Contains(event.Locator, "pod/") {
			continue
		}
		eventsByPods[event.Locator] = append(eventsByPods[event.Locator], event)
	}
	return eventsByPods
}

func getPodDeletionTime(events monitorapi.Intervals, podLocator string) *time.Time {
	for _, event := range events {
		if event.Locator == podLocator && event.Message == "reason/Deleted" {
			return &event.From
		}
	}
	return nil
}
<commit_msg>testPodSandboxCreation: skip sandbox errors for pods which were not deleted during network update<commit_after>package synthetictests

import (
	"fmt"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/intervalcreation"
	"github.com/openshift/origin/pkg/monitor/monitorapi"

	"github.com/openshift/origin/pkg/test/ginkgo"
)

type testCategorizer struct {
	by        string
	substring string
}

func testPodSandboxCreation(events monitorapi.Intervals) []*ginkgo.JUnitTestCase {
	const testName = "[sig-network] pods should successfully create sandboxes"
	// we can further refine this signal by subdividing different failure modes if it is pertinent.  Right now I'm seeing
	// 1. error reading container (probably exited) json message: EOF
	// 2. dial tcp 10.0.76.225:6443: i/o timeout
	// 3. error getting pod: pods "terminate-cmd-rpofb45fa14c-96bb-40f7-bd9e-346721740cac" not found
	// 4. write child: broken pipe
	bySubStrings := []testCategorizer{
		{by: " by reading container", substring: "error reading container (probably exited) json message: EOF"},
		{by: " by not timing out", substring: "i/o timeout"},
		{by: " by writing network status", substring: "error setting the networks status"},
		{by: " by getting pod", substring: " error getting pod: pods"},
		{by: " by writing child", substring: "write child: broken pipe"},
		{by: " by other", substring: " "}, // always matches
	}

	failures := []string{}
	flakes := []string{}
	operatorsProgressing := intervalcreation.IntervalsFromEvents_OperatorProgressing(events, time.Time{}, time.Time{})
	networkOperatorProgressing := operatorsProgressing.Filter(func(ev monitorapi.EventInterval) bool {
		return ev.Locator == "clusteroperator/network" || ev.Locator == "clusteroperator/machine-config"
	})
	eventsForPods := getEventsByPod(events)
	for _, event := range events {
		if !strings.Contains(event.Message, "reason/FailedCreatePodSandBox Failed to create pod sandbox") {
			continue
		}
		if strings.Contains(event.Message, "Multus") && strings.Contains(event.Message, "error getting pod") && (strings.Contains(event.Message, "connection refused") || strings.Contains(event.Message, "i/o timeout")) {
			flakes = append(flakes, fmt.Sprintf("%v - multus is unable to get pods due to LB disruption https://bugzilla.redhat.com/show_bug.cgi?id=1927264 - %v", event.Locator, event.Message))
			continue
		}
		deletionTime := getPodDeletionTime(eventsForPods[event.Locator], event.Locator)
		if deletionTime == nil {
			// mark sandboxes errors as flakes if networking is being updated
			match := -1
			for i := range networkOperatorProgressing {
				matchesFrom := event.From.After(networkOperatorProgressing[i].From)
				matchesTo := event.To.Before(networkOperatorProgressing[i].To)
				if matchesFrom && matchesTo {
					match = i
					break
				}
			}
			if match != -1 {
				flakes = append(flakes, fmt.Sprintf("%v - never deleted - network rollout - %v", event.Locator, event.Message))
			} else {
				failures = append(failures, fmt.Sprintf("%v - never deleted - %v", event.Locator, event.Message))
			}
		} else {
			timeBetweenDeleteAndFailure := event.From.Sub(*deletionTime)
			switch {
			case timeBetweenDeleteAndFailure < 1*time.Second:
				// nothing here, one second is close enough to be ok, the kubelet and CNI just didn't know
			case timeBetweenDeleteAndFailure < 5*time.Second:
				// withing five seconds, it ought to be long enough to know, but it's close enough to flake and not fail
				flakes = append(flakes, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			case deletionTime.Before(event.From):
				// something went wrong.  More than five seconds after the pod ws deleted, the CNI is trying to set up pod sandboxes and can't
				failures = append(failures, fmt.Sprintf("%v - %0.2f seconds after deletion - %v", event.Locator, timeBetweenDeleteAndFailure.Seconds(), event.Message))
			default:
				// something went wrong.  deletion happend after we had a failure to create the pod sandbox
				failures = append(failures, fmt.Sprintf("%v - deletion came AFTER sandbox failure - %v", event.Locator, event.Message))
			}
		}
	}
	if len(failures) == 0 && len(flakes) == 0 {
		successes := []*ginkgo.JUnitTestCase{}
		for _, by := range bySubStrings {
			successes = append(successes, &ginkgo.JUnitTestCase{Name: testName + by.by})
		}
		return successes
	}

	ret := []*ginkgo.JUnitTestCase{}
	failuresBySubtest, flakesBySubtest := categorizeBySubset(bySubStrings, failures, flakes)

	// now iterate the individual failures to create failure entries
	for by, subFailures := range failuresBySubtest {
		failure := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFailures, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFailures), strings.Join(subFailures, "\n")),
			},
		}
		ret = append(ret, failure)
	}
	for by, subFlakes := range flakesBySubtest {
		flake := &ginkgo.JUnitTestCase{
			Name:      testName + by,
			SystemOut: strings.Join(subFlakes, "\n"),
			FailureOutput: &ginkgo.FailureOutput{
				Output: fmt.Sprintf("%d failures to create the sandbox\n\n%v", len(subFlakes), strings.Join(subFlakes, "\n")),
			},
		}
		ret = append(ret, flake)
		// write a passing test to trigger detection of this issue as a flake. Doing this first to try to see how frequent the issue actually is
		success := &ginkgo.JUnitTestCase{
			Name: testName + by,
		}
		ret = append(ret, success)
	}

	return append(ret)
}

// categorizeBySubset returns a map keyed by category for failures and flakes.  If a category is present in both failures and flakes, all are listed under failures.
func categorizeBySubset(categorizers []testCategorizer, failures, flakes []string) (map[string][]string, map[string][]string) {
	failuresBySubtest := map[string][]string{}
	flakesBySubtest := map[string][]string{}
	for _, failure := range failures {
		for _, by := range categorizers {
			if strings.Contains(failure, by.substring) {
				failuresBySubtest[by.by] = append(failuresBySubtest[by.by], failure)
				break // break after first match so we only add each failure one bucket
			}
		}
	}

	for _, flake := range flakes {
		for _, by := range categorizers {
			if strings.Contains(flake, by.substring) {
				if _, isFailure := failuresBySubtest[by.by]; isFailure {
					failuresBySubtest[by.by] = append(failuresBySubtest[by.by], flake)
				} else {
					flakesBySubtest[by.by] = append(flakesBySubtest[by.by], flake)
				}
				break // break after first match so we only add each failure one bucket
			}
		}
	}
	return failuresBySubtest, flakesBySubtest
}

// getEventsByPod returns map keyed by pod locator with all events associated with it.
func getEventsByPod(events monitorapi.Intervals) map[string]monitorapi.Intervals {
	eventsByPods := map[string]monitorapi.Intervals{}
	for _, event := range events {
		if !strings.Contains(event.Locator, "pod/") {
			continue
		}
		eventsByPods[event.Locator] = append(eventsByPods[event.Locator], event)
	}
	return eventsByPods
}

func getPodDeletionTime(events monitorapi.Intervals, podLocator string) *time.Time {
	for _, event := range events {
		if event.Locator == podLocator && event.Message == "reason/Deleted" {
			return &event.From
		}
	}
	return nil
}
<|endoftext|>