<commit_before>package otp

import (
	"bytes"
	"crypto/rand"
	"testing"
)

func TestNewPad(t *testing.T) {
	m := make([]byte, 32)
	_, err := rand.Read(m)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Make sure we can properly create a new pad
	if _, err := NewPad(m, 8, 1); err != nil {
		t.Fatalf("bad: %s", err)
	}

	// Make sure an error is thrown if the pad size and the page size are not
	// cleanly divisible
	if _, err := NewPad(m, 7, 1); err == nil {
		t.Fatalf("Expected page size error")
	}

	// Make sure an error is thrown if the startPage is out of bounds
	if _, err := NewPad(m, 8, 5); err == nil {
		t.Fatalf("Expected out of bounds error")
	}
}

func TestTotalPages(t *testing.T) {
	m := make([]byte, 32)
	_, err := rand.Read(m)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	p, err := NewPad(m, 8, 1)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}

	totalPages := p.TotalPages()
	if totalPages != 4 {
		t.Fatalf("Expected 4 pages, got %d", totalPages)
	}
}

func TestPages(t *testing.T) {
	m := make([]byte, 32)
	_, err := rand.Read(m)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	p, err := NewPad(m, 8, 1)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}

	// Error is thrown if we attempt to look at pages < 1
	if _, err := p.PreviousPage(); err == nil {
		t.Fatalf("Expected out of bounds error")
	}

	// Current page is properly returned
	if !bytes.Equal(p.CurrentPage(), m[0:8]) {
		t.Fatalf("First page does not match expected bytes")
	}

	// Advancing the page works properly
	nextPage, err := p.NextPage()
	if err != nil {
		t.Fatalf("bad: %s", err)
	}

	if !bytes.Equal(nextPage, m[8:16]) {
		t.Fatalf("Page does not match expected bytes")
	}

	// Page pointer is updated after page advance
	if !bytes.Equal(p.CurrentPage(), m[8:16]) {
		t.Fatalf("Page pointer not properly advanced")
	}

	// Previous page is properly returned
	previousPage, err := p.PreviousPage()
	if err != nil {
		t.Fatalf("bad: %s", err)
	}

	if !bytes.Equal(previousPage, m[0:8]) {
		t.Fatalf("Previous page did not match expected bytes")
	}

	// By advancing the page twice more, we should be at the last page, where
	// requests for yet another page should fail.
	p.NextPage()
	p.NextPage()
	if _, err := p.NextPage(); err == nil {
		t.Fatalf("Expected out of bounds error")
	}
}
<commit_msg>Updated tests for new interfaces<commit_after>package otp

import (
	"crypto/rand"
	"encoding/base64"
	"testing"
)

func TestNewPad(t *testing.T) {
	m := make([]byte, 32)
	_, err := rand.Read(m)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Make sure we can properly create a new pad
	if _, err := NewPad(m, 8, 1); err != nil {
		t.Fatalf("bad: %s", err)
	}

	// Make sure an error is thrown if the pad size and the page size are not
	// cleanly divisible
	if _, err := NewPad(m, 7, 1); err == nil {
		t.Fatalf("Expected page size error")
	}

	// Make sure an error is thrown if the startPage is out of bounds
	if _, err := NewPad(m, 8, 5); err == nil {
		t.Fatalf("Expected out of bounds error")
	}
}

func TestPages(t *testing.T) {
	m := make([]byte, 32)
	_, err := rand.Read(m)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	p, err := NewPad(m, 8, 1)
	if err != nil {
		t.Fatalf("bad: %s", err)
	}

	// Make sure the total number of pages is correct
	totalPages := p.TotalPages()
	if totalPages != 4 {
		t.Fatalf("Expected 4 pages, got %d", totalPages)
	}

	// Current page is properly returned
	page := p.CurrentPage()
	if page != 1 {
		t.Fatalf("Expected page pointer to be 1, got %d", page)
	}

	// Advancing the page works properly
	if err := p.NextPage(); err != nil {
		t.Fatalf("bad: %s", err)
	}

	// Page pointer is updated after page advance
	page = p.CurrentPage()
	if page != 2 {
		t.Fatalf("Expected page pointer to be 2, got %d", page)
	}

	// Explicitly setting the page works properly
	if err := p.SetPage(4); err != nil {
		t.Fatalf("err: %s", err)
	}

	// Advancing the page past the end of the pad should throw an error
	if err := p.NextPage(); err == nil {
		t.Fatalf("Expected pad depleted error")
	}
}

func TestEncryption(t *testing.T) {
	m := []byte("123456789abcdefghijklmno")
	p, err := NewPad(m, 4, 1)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	encrypted1, err := p.Encrypt([]byte("test"))
	encoded1 := base64.StdEncoding.EncodeToString(encrypted1)
	if encoded1 != "pZemqA==" {
		t.Fatalf("bad: %s", encoded1)
	}

	if err := p.NextPage(); err != nil {
		t.Fatalf("err: %s", err)
	}

	encrypted2, err := p.Encrypt([]byte("test"))
	encoded2 := base64.StdEncoding.EncodeToString(encrypted2)
	if encoded2 != "qZuqrA==" {
		t.Fatalf("bad: %s", encoded2)
	}

	if err := p.SetPage(6); err != nil {
		t.Fatalf("err: %s", err)
	}

	encrypted3, err := p.Encrypt([]byte("test"))
	encoded3 := base64.StdEncoding.EncodeToString(encrypted3)
	if encoded3 != "4NLh4w==" {
		t.Fatalf("bad: %s", encoded3)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"github.com/yookoala/realpath"
)

const (
	helpString = `
usage: overalls -project=[path] -covermode[mode] OPTIONS

overalls recursively traverses your projects directory structure
running 'go test -covermode=count -coverprofile=profile.coverprofile'
in each directory with go test files, concatenates them into one
coverprofile in your root directory named 'overalls.coverprofile'

OPTIONS
  -project
	Your project path relative to the '$GOPATH/src' directory
	example: -project=github.com/go-playground/overalls

  -covermode
    Mode to run when testing files.
    default:count

OPTIONAL

  -go-binary
    An alternative 'go' binary to run the tests, for example to use 'richgo' for
		more human-friendly output.
    example: -go-binary=richgo
    default: 'go'

  -ignore
    A comma separated list of directory names to ignore, relative to project path.
    example: -ignore=[.git,.hiddentdir...]
    default: '.git,vendor'

  -debug
    A flag indicating whether to print debug messages.
    example: -debug
    default:false

  -concurrency
    Limit the number of packages being processed at one time.
    The minimum value must be 2 or more when set.
    example: -concurrency=5
    default: unlimited
`
)

const (
	defaultIgnores = ".git,vendor"
	outFilename    = "overalls.coverprofile"
	pkgFilename    = "profile.coverprofile"
)

var (
	modeRegex       = regexp.MustCompile("mode: [a-z]+\n")
	srcPath         string
	projectPath     string
	goBinary        string
	ignoreFlag      string
	projectFlag     string
	coverFlag       string
	helpFlag        bool
	debugFlag       bool
	concurrencyFlag int
	isLimited       bool
	emptyStruct     struct{}
	ignores         = map[string]struct{}{}
	flagArgs        []string
)

func help() {
	fmt.Printf(helpString)
}

func init() {
	flag.StringVar(&goBinary, "go-binary", "go", "Use an alternative test runner such as 'richgo'")
	flag.StringVar(&projectFlag, "project", "", "-project [path]: relative to the '$GOPATH/src' directory")
	flag.StringVar(&coverFlag, "covermode", "count", "Mode to run when testing files")
	flag.StringVar(&ignoreFlag, "ignore", defaultIgnores, "-ignore [dir1,dir2...]: comma separated list of directory names to ignore")
	flag.IntVar(&concurrencyFlag, "concurrency", -1, "-concurrency [int]: number of packages to process concurrently, The minimum value must be 2 or more when set.")
	flag.BoolVar(&debugFlag, "debug", false, "-debug [true|false]")
	flag.BoolVar(&helpFlag, "help", false, "-help")
}

func parseFlags(logger *log.Logger) {

	flag.Parse()

	if helpFlag {
		help()
		os.Exit(0)
	}

	if debugFlag {
		fmt.Println("GOPATH:", os.Getenv("GOPATH"))
	}

	fmt.Println("|", projectFlag)
	projectFlag = filepath.Clean(projectFlag)

	if debugFlag {
		fmt.Println("Project Path:", projectFlag)
	}

	if len(projectFlag) == 0 || projectFlag == "." {
		fmt.Printf("\n**invalid project path '%s'\n", projectFlag)
		help()
		os.Exit(1)
	}

	pkg, err := build.Default.Import(projectFlag, "", build.FindOnly)
	if err != nil {
		fmt.Printf("\n**could not find project path '%s' in GOPATH '%s'\n", projectFlag, os.Getenv("GOPATH"))
		os.Exit(1)
	}
	srcPath = pkg.SrcRoot

	flagArgs = flag.Args()

	switch coverFlag {
	case "set", "atomic":
	case "count":
		for _, flg := range flagArgs {
			if flg == "-race" {
				logger.Println("\n*****\n** WARNING: some common patterns in parallel code can trigger race conditions when using coverprofile=count and the -race flag; in which case coverprofile=atomic should be used.\n*****")
				break
			}
		}
	default:
		fmt.Printf("\n**invalid covermode '%s'\n", coverFlag)
		os.Exit(1)
	}

	arr := strings.Split(ignoreFlag, ",")
	for _, v := range arr {
		ignores[v] = emptyStruct
	}

	isLimited = concurrencyFlag != -1

	if isLimited && concurrencyFlag < 1 {
		fmt.Printf("\n**invalid concurrency value '%d', value must be at least 1\n", concurrencyFlag)
		os.Exit(1)
	}
}

func main() {
	logger := log.New(os.Stdout, "", log.LstdFlags)
	runMain(logger)
}

func runMain(logger *log.Logger) {
	parseFlags(logger)

	var err error
	var wd string

	projectPath = filepath.Join(srcPath, projectFlag)

	if err = os.Chdir(projectPath); err != nil {
		logger.Printf("\n**invalid project path '%s'\n%s\n", projectFlag, err)
		help()
		os.Exit(1)
	}

	if debugFlag {
		wd, err = os.Getwd()
		if err != nil {
			fmt.Println(err)
		}

		logger.Println("Working DIR:", wd)
	}

	testFiles(logger)
}

func scanOutput(r io.ReadCloser, fn func(...interface{})) {
	defer r.Close()
	bs := bufio.NewScanner(r)
	for bs.Scan() {
		fn(bs.Text())
	}
	if err := bs.Err(); err != nil {
		fn(fmt.Sprintf("Scan error: %v", err.Error()))
	}
}

func processDIR(logger *log.Logger, wg *sync.WaitGroup, fullPath, relPath string, out chan<- []byte, semaphore chan struct{}) {
	defer wg.Done()

	if isLimited {
		semaphore <- struct{}{}
	}

	// 1 for "test", 4 for covermode, coverprofile, outputdir, relpath
	args := make([]string, 1, 1+len(flagArgs)+4)
	args[0] = "test"
	args = append(args, "-covermode="+coverFlag, "-coverprofile="+pkgFilename, "-outputdir="+fullPath+"/", relPath)
	args = append(args, flagArgs...)
	fmt.Printf("Test args: %+v\n", args)

	cmd := exec.Command(goBinary, args...)
	if debugFlag {
		logger.Println("Processing:", strings.Join(cmd.Args, " "))
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		logger.Fatal("Unable to get process stdout")
	}

	go scanOutput(stdout, logger.Print)

	stderr, err := cmd.StderrPipe()
	if err != nil {
		logger.Fatal("Unable to get process stderr")
	}

	go scanOutput(stderr, logger.Print)

	if err := cmd.Run(); err != nil {
		logger.Fatal("ERROR:", err)
	}

	b, err := ioutil.ReadFile(relPath + "/profile.coverprofile")
	if err != nil {
		logger.Fatal("ERROR:", err)
	}

	out <- b

	if isLimited {
		<-semaphore
	}
}

// walk is like filepath.Walk, but it follows symlinks and only calls walkFunc on directories.
func walkDirectories(path string, walkFunc func(path string, info os.FileInfo) error) error {
	seen := make(map[string]bool)

	var walkHelper func(path string) error
	walkHelper = func(path string) error {
		qualifiedPath, err := realpath.Realpath(path)
		if err != nil {
			return err
		}

		// Prevent circular links.
		if seen[qualifiedPath] {
			return nil
		}
		seen[qualifiedPath] = true

		// Skip anything that isn't a directory.
		file, err := os.Stat(path)
		if err != nil {
			return err
		}
		if !file.IsDir() {
			return nil
		}

		err = walkFunc(path, file)
		if err != nil {
			if err == filepath.SkipDir {
				return nil
			}
			return err
		}

		files, err := ioutil.ReadDir(path)
		if err != nil {
			return err
		}

		for _, file := range files {
			err = walkHelper(filepath.Join(path, file.Name()))
			if err != nil {
				return err
			}
		}
		return nil
	}
	return walkHelper(path)
}

func testFiles(logger *log.Logger) {

	var semaphore chan struct{}

	if isLimited {
		semaphore = make(chan struct{}, concurrencyFlag)
	}

	out := make(chan []byte)
	wg := &sync.WaitGroup{}

	walker := func(path string, info os.FileInfo) error {
		rel, err := filepath.Rel(projectPath, path)
		if err != nil {
			logger.Fatalf("Could not make path '%s' relative to project path '%s'", path, projectPath)
		}

		if _, ignore := ignores[rel]; ignore {
			return filepath.SkipDir
		}

		rel = "./" + rel

		if files, err := filepath.Glob(rel + "/*_test.go"); len(files) == 0 || err != nil {

			if err != nil {
				logger.Fatal("Error checking for test files")
			}

			if debugFlag {
				logger.Println("No Go Test files in DIR:", rel, "skipping")
			}

			return nil
		}

		wg.Add(1)

		go processDIR(logger, wg, path, rel, out, semaphore)

		return nil
	}

	if err := walkDirectories(projectPath, walker); err != nil {
		logger.Fatalf("\n**could not walk project path '%s'\n%s\n", projectPath, err)
	}

	go func() {
		wg.Wait()
		close(out)

		if isLimited {
			close(semaphore)
		}
	}()

	buff := bytes.NewBufferString("")

	for cover := range out {
		buff.Write(cover)
	}

	final := buff.String()
	final = modeRegex.ReplaceAllString(final, "")
	final = "mode: " + coverFlag + "\n" + final

	if err := ioutil.WriteFile(outFilename, []byte(final), 0644); err != nil {
		logger.Fatal("ERROR Writing \""+outFilename+"\"", err)
	}
}
<commit_msg>Fix pull request comment : split '-- <go test arguments> -args <program arguments>'<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"sync"

	"github.com/yookoala/realpath"
)

const (
	helpString = `
usage: overalls -project=[path] -covermode[mode] OPTIONS

overalls recursively traverses your projects directory structure
running 'go test -covermode=count -coverprofile=profile.coverprofile'
in each directory with go test files, concatenates them into one
coverprofile in your root directory named 'overalls.coverprofile'

OPTIONS
  -project
	Your project path relative to the '$GOPATH/src' directory
	example: -project=github.com/go-playground/overalls

  -covermode
    Mode to run when testing files.
    default:count

OPTIONAL

  -go-binary
    An alternative 'go' binary to run the tests, for example to use 'richgo' for
		more human-friendly output.
    example: -go-binary=richgo
    default: 'go'

  -ignore
    A comma separated list of directory names to ignore, relative to project path.
    example: -ignore=[.git,.hiddentdir...]
    default: '.git,vendor'

  -debug
    A flag indicating whether to print debug messages.
    example: -debug
    default:false

  -concurrency
    Limit the number of packages being processed at one time.
    The minimum value must be 2 or more when set.
    example: -concurrency=5
    default: unlimited
`
)

const (
	defaultIgnores = ".git,vendor"
	outFilename    = "overalls.coverprofile"
	pkgFilename    = "profile.coverprofile"
)

var (
	modeRegex       = regexp.MustCompile("mode: [a-z]+\n")
	srcPath         string
	projectPath     string
	goBinary        string
	ignoreFlag      string
	projectFlag     string
	coverFlag       string
	helpFlag        bool
	debugFlag       bool
	concurrencyFlag int
	isLimited       bool
	emptyStruct     struct{}
	ignores         = map[string]struct{}{}
	flagArgs        []string
)

func help() {
	fmt.Printf(helpString)
}

func init() {
	flag.StringVar(&goBinary, "go-binary", "go", "Use an alternative test runner such as 'richgo'")
	flag.StringVar(&projectFlag, "project", "", "-project [path]: relative to the '$GOPATH/src' directory")
	flag.StringVar(&coverFlag, "covermode", "count", "Mode to run when testing files")
	flag.StringVar(&ignoreFlag, "ignore", defaultIgnores, "-ignore [dir1,dir2...]: comma separated list of directory names to ignore")
	flag.IntVar(&concurrencyFlag, "concurrency", -1, "-concurrency [int]: number of packages to process concurrently, The minimum value must be 2 or more when set.")
	flag.BoolVar(&debugFlag, "debug", false, "-debug [true|false]")
	flag.BoolVar(&helpFlag, "help", false, "-help")
}

func parseFlags(logger *log.Logger) {

	flag.Parse()

	if helpFlag {
		help()
		os.Exit(0)
	}

	if debugFlag {
		fmt.Println("GOPATH:", os.Getenv("GOPATH"))
	}

	fmt.Println("|", projectFlag)
	projectFlag = filepath.Clean(projectFlag)

	if debugFlag {
		fmt.Println("Project Path:", projectFlag)
	}

	if len(projectFlag) == 0 || projectFlag == "." {
		fmt.Printf("\n**invalid project path '%s'\n", projectFlag)
		help()
		os.Exit(1)
	}

	pkg, err := build.Default.Import(projectFlag, "", build.FindOnly)
	if err != nil {
		fmt.Printf("\n**could not find project path '%s' in GOPATH '%s'\n", projectFlag, os.Getenv("GOPATH"))
		os.Exit(1)
	}
	srcPath = pkg.SrcRoot

	flagArgs = flag.Args()

	switch coverFlag {
	case "set", "atomic":
	case "count":
		for _, flg := range flagArgs {
			if flg == "-race" {
				logger.Println("\n*****\n** WARNING: some common patterns in parallel code can trigger race conditions when using coverprofile=count and the -race flag; in which case coverprofile=atomic should be used.\n*****")
				break
			}
		}
	default:
		fmt.Printf("\n**invalid covermode '%s'\n", coverFlag)
		os.Exit(1)
	}

	arr := strings.Split(ignoreFlag, ",")
	for _, v := range arr {
		ignores[v] = emptyStruct
	}

	isLimited = concurrencyFlag != -1

	if isLimited && concurrencyFlag < 1 {
		fmt.Printf("\n**invalid concurrency value '%d', value must be at least 1\n", concurrencyFlag)
		os.Exit(1)
	}
}

func main() {
	logger := log.New(os.Stdout, "", log.LstdFlags)
	runMain(logger)
}

func runMain(logger *log.Logger) {
	parseFlags(logger)

	var err error
	var wd string

	projectPath = filepath.Join(srcPath, projectFlag)

	if err = os.Chdir(projectPath); err != nil {
		logger.Printf("\n**invalid project path '%s'\n%s\n", projectFlag, err)
		help()
		os.Exit(1)
	}

	if debugFlag {
		wd, err = os.Getwd()
		if err != nil {
			fmt.Println(err)
		}

		logger.Println("Working DIR:", wd)
	}

	testFiles(logger)
}

func scanOutput(r io.ReadCloser, fn func(...interface{})) {
	defer r.Close()
	bs := bufio.NewScanner(r)
	for bs.Scan() {
		fn(bs.Text())
	}
	if err := bs.Err(); err != nil {
		fn(fmt.Sprintf("Scan error: %v", err.Error()))
	}
}

func processDIR(logger *log.Logger, wg *sync.WaitGroup, fullPath, relPath string, out chan<- []byte, semaphore chan struct{}) {
	defer wg.Done()

	if isLimited {
		semaphore <- struct{}{}
	}

	// 1 for "test", 4 for covermode, coverprofile, outputdir, relpath
	args := make([]string, 1, 1+len(flagArgs)+4)
	args[0] = "test"
	// To split '-- <go test arguments> -args <program arguments>'
	for i, arg := range flagArgs {
		if arg == "-args" {
			args = append(args, flagArgs[:i]...)
			flagArgs = flagArgs[i:]
			break
		}
	}
	args = append(args, "-covermode="+coverFlag, "-coverprofile="+pkgFilename, "-outputdir="+fullPath+"/", relPath)
	args = append(args, flagArgs...)
	fmt.Printf("Test args: %+v\n", args)

	cmd := exec.Command(goBinary, args...)
	if debugFlag {
		logger.Println("Processing:", strings.Join(cmd.Args, " "))
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		logger.Fatal("Unable to get process stdout")
	}

	go scanOutput(stdout, logger.Print)

	stderr, err := cmd.StderrPipe()
	if err != nil {
		logger.Fatal("Unable to get process stderr")
	}

	go scanOutput(stderr, logger.Print)

	if err := cmd.Run(); err != nil {
		logger.Fatal("ERROR:", err)
	}

	b, err := ioutil.ReadFile(relPath + "/profile.coverprofile")
	if err != nil {
		logger.Fatal("ERROR:", err)
	}

	out <- b

	if isLimited {
		<-semaphore
	}
}

// walk is like filepath.Walk, but it follows symlinks and only calls walkFunc on directories.
func walkDirectories(path string, walkFunc func(path string, info os.FileInfo) error) error {
	seen := make(map[string]bool)

	var walkHelper func(path string) error
	walkHelper = func(path string) error {
		qualifiedPath, err := realpath.Realpath(path)
		if err != nil {
			return err
		}

		// Prevent circular links.
		if seen[qualifiedPath] {
			return nil
		}
		seen[qualifiedPath] = true

		// Skip anything that isn't a directory.
		file, err := os.Stat(path)
		if err != nil {
			return err
		}
		if !file.IsDir() {
			return nil
		}

		err = walkFunc(path, file)
		if err != nil {
			if err == filepath.SkipDir {
				return nil
			}
			return err
		}

		files, err := ioutil.ReadDir(path)
		if err != nil {
			return err
		}

		for _, file := range files {
			err = walkHelper(filepath.Join(path, file.Name()))
			if err != nil {
				return err
			}
		}
		return nil
	}
	return walkHelper(path)
}

func testFiles(logger *log.Logger) {

	var semaphore chan struct{}

	if isLimited {
		semaphore = make(chan struct{}, concurrencyFlag)
	}

	out := make(chan []byte)
	wg := &sync.WaitGroup{}

	walker := func(path string, info os.FileInfo) error {
		rel, err := filepath.Rel(projectPath, path)
		if err != nil {
			logger.Fatalf("Could not make path '%s' relative to project path '%s'", path, projectPath)
		}

		if _, ignore := ignores[rel]; ignore {
			return filepath.SkipDir
		}

		rel = "./" + rel

		if files, err := filepath.Glob(rel + "/*_test.go"); len(files) == 0 || err != nil {

			if err != nil {
				logger.Fatal("Error checking for test files")
			}

			if debugFlag {
				logger.Println("No Go Test files in DIR:", rel, "skipping")
			}

			return nil
		}

		wg.Add(1)

		go processDIR(logger, wg, path, rel, out, semaphore)

		return nil
	}

	if err := walkDirectories(projectPath, walker); err != nil {
		logger.Fatalf("\n**could not walk project path '%s'\n%s\n", projectPath, err)
	}

	go func() {
		wg.Wait()
		close(out)

		if isLimited {
			close(semaphore)
		}
	}()

	buff := bytes.NewBufferString("")

	for cover := range out {
		buff.Write(cover)
	}

	final := buff.String()
	final = modeRegex.ReplaceAllString(final, "")
	final = "mode: " + coverFlag + "\n" + final

	if err := ioutil.WriteFile(outFilename, []byte(final), 0644); err != nil {
		logger.Fatal("ERROR Writing \""+outFilename+"\"", err)
	}
}
<|endoftext|>


<commit_before>package p2p

import (
	"fmt"
	"net"
	"sync/atomic"
	"time"

	"github.com/tendermint/go-crypto"
	cmn "github.com/tendermint/tmlibs/common"
	"github.com/tendermint/tmlibs/log"

	tmconn "github.com/tendermint/tendermint/p2p/conn"
)

var testIPSuffix uint32 = 0

// Peer is an interface representing a peer connected on a reactor.
type Peer interface {
	cmn.Service

	ID() ID             // peer's cryptographic ID
	RemoteIP() net.IP   // remote IP of the connection
	IsOutbound() bool   // did we dial the peer
	IsPersistent() bool // do we redial this peer when we disconnect
	NodeInfo() NodeInfo // peer's info
	Status() tmconn.ConnectionStatus

	Send(byte, []byte) bool
	TrySend(byte, []byte) bool

	Set(string, interface{})
	Get(string) interface{}
}

//----------------------------------------------------------

// peerConn contains the raw connection and its config.
type peerConn struct {
	outbound   bool
	persistent bool
	config     *PeerConfig
	conn       net.Conn // source connection
	ips        []net.IP
}

// ID only exists for SecretConnection.
// NOTE: Will panic if conn is not *SecretConnection.
func (pc peerConn) ID() ID {
	return PubKeyToID(pc.conn.(*tmconn.SecretConnection).RemotePubKey())
}

// Return the IP from the connection RemoteAddr
func (pc peerConn) RemoteIP() net.IP {
	if len(pc.ips) > 0 {
		return pc.ips[0]
	}

	if pc.conn.RemoteAddr().String() == "pipe" {
		pc.ips = []net.IP{
			net.IP{172, 16, 0, byte(atomic.AddUint32(&testIPSuffix, 1))},
		}

		return pc.ips[0]
	}

	host, _, err := net.SplitHostPort(pc.conn.RemoteAddr().String())
	if err != nil {
		panic(err)
	}

	ips, err := net.LookupIP(host)
	if err != nil {
		panic(err)
	}

	pc.ips = ips

	return ips[0]
}

// peer implements Peer.
//
// Before using a peer, you will need to perform a handshake on connection.
type peer struct {
	cmn.BaseService

	// raw peerConn and the multiplex connection
	peerConn
	mconn *tmconn.MConnection

	// peer's node info and the channel it knows about
	// channels = nodeInfo.Channels
	// cached to avoid copying nodeInfo in hasChannel
	nodeInfo NodeInfo
	channels []byte

	// User data
	Data *cmn.CMap
}

func newPeer(pc peerConn, nodeInfo NodeInfo,
	reactorsByCh map[byte]Reactor, chDescs []*tmconn.ChannelDescriptor,
	onPeerError func(Peer, interface{})) *peer {

	p := &peer{
		peerConn: pc,
		nodeInfo: nodeInfo,
		channels: nodeInfo.Channels,
		Data:     cmn.NewCMap(),
	}
	p.mconn = createMConnection(pc.conn, p, reactorsByCh, chDescs, onPeerError, pc.config.MConfig)
	p.BaseService = *cmn.NewBaseService(nil, "Peer", p)
	return p
}

// PeerConfig is a Peer configuration.
type PeerConfig struct {
	AuthEnc bool `mapstructure:"auth_enc"` // authenticated encryption

	// times are in seconds
	HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"`
	DialTimeout      time.Duration `mapstructure:"dial_timeout"`

	MConfig *tmconn.MConnConfig `mapstructure:"connection"`

	DialFail   bool            `mapstructure:"dial_fail"` // for testing
	Fuzz       bool            `mapstructure:"fuzz"`      // fuzz connection (for testing)
	FuzzConfig *FuzzConnConfig `mapstructure:"fuzz_config"`
}

// DefaultPeerConfig returns the default config.
func DefaultPeerConfig() *PeerConfig {
	return &PeerConfig{
		AuthEnc:          true,
		HandshakeTimeout: 20, // * time.Second,
		DialTimeout:      3,  // * time.Second,
		MConfig:          tmconn.DefaultMConnConfig(),
		DialFail:         false,
		Fuzz:             false,
		FuzzConfig:       DefaultFuzzConnConfig(),
	}
}

func newOutboundPeerConn(addr *NetAddress, config *PeerConfig, persistent bool, ourNodePrivKey crypto.PrivKey) (peerConn, error) {
	var pc peerConn

	conn, err := dial(addr, config)
	if err != nil {
		return pc, cmn.ErrorWrap(err, "Error creating peer")
	}

	pc, err = newPeerConn(conn, config, true, persistent, ourNodePrivKey)
	if err != nil {
		if err2 := conn.Close(); err2 != nil {
			return pc, cmn.ErrorWrap(err, err2.Error())
		}
		return pc, err
	}

	// ensure dialed ID matches connection ID
	if config.AuthEnc && addr.ID != pc.ID() {
		if err2 := conn.Close(); err2 != nil {
			return pc, cmn.ErrorWrap(err, err2.Error())
		}
		return pc, ErrSwitchAuthenticationFailure{addr, pc.ID()}
	}
	return pc, nil
}

func newInboundPeerConn(conn net.Conn, config *PeerConfig, ourNodePrivKey crypto.PrivKey) (peerConn, error) {

	// TODO: issue PoW challenge

	return newPeerConn(conn, config, false, false, ourNodePrivKey)
}

func newPeerConn(rawConn net.Conn,
	config *PeerConfig, outbound, persistent bool,
	ourNodePrivKey crypto.PrivKey) (pc peerConn, err error) {

	conn := rawConn

	// Fuzz connection
	if config.Fuzz {
		// so we have time to do peer handshakes and get set up
		conn = FuzzConnAfterFromConfig(conn, 10*time.Second, config.FuzzConfig)
	}

	if config.AuthEnc {
		// Set deadline for secret handshake
		if err := conn.SetDeadline(time.Now().Add(config.HandshakeTimeout * time.Second)); err != nil {
			return pc, cmn.ErrorWrap(err, "Error setting deadline while encrypting connection")
		}

		// Encrypt connection
		conn, err = tmconn.MakeSecretConnection(conn, ourNodePrivKey)
		if err != nil {
			return pc, cmn.ErrorWrap(err, "Error creating peer")
		}
	}

	// Only the information we already have
	return peerConn{
		config:     config,
		outbound:   outbound,
		persistent: persistent,
		conn:       conn,
	}, nil
}

//---------------------------------------------------
// Implements cmn.Service

// SetLogger implements BaseService.
func (p *peer) SetLogger(l log.Logger) {
	p.Logger = l
	p.mconn.SetLogger(l)
}

// OnStart implements BaseService.
func (p *peer) OnStart() error {
	if err := p.BaseService.OnStart(); err != nil {
		return err
	}
	err := p.mconn.Start()
	return err
}

// OnStop implements BaseService.
func (p *peer) OnStop() {
	p.BaseService.OnStop()
	p.mconn.Stop() // stop everything and close the conn
}

//---------------------------------------------------
// Implements Peer

// ID returns the peer's ID - the hex encoded hash of its pubkey.
func (p *peer) ID() ID {
	return p.nodeInfo.ID
}

// IsOutbound returns true if the connection is outbound, false otherwise.
func (p *peer) IsOutbound() bool {
	return p.peerConn.outbound
}

// IsPersistent returns true if the peer is persitent, false otherwise.
func (p *peer) IsPersistent() bool {
	return p.peerConn.persistent
}

// NodeInfo returns a copy of the peer's NodeInfo.
func (p *peer) NodeInfo() NodeInfo {
	return p.nodeInfo
}

// Status returns the peer's ConnectionStatus.
func (p *peer) Status() tmconn.ConnectionStatus {
	return p.mconn.Status()
}

// Send msg bytes to the channel identified by chID byte. Returns false if the
// send queue is full after timeout, specified by MConnection.
func (p *peer) Send(chID byte, msgBytes []byte) bool {
	if !p.IsRunning() {
		// see Switch#Broadcast, where we fetch the list of peers and loop over
		// them - while we're looping, one peer may be removed and stopped.
		return false
	} else if !p.hasChannel(chID) {
		return false
	}
	return p.mconn.Send(chID, msgBytes)
}

// TrySend msg bytes to the channel identified by chID byte. Immediately returns
// false if the send queue is full.
func (p *peer) TrySend(chID byte, msgBytes []byte) bool {
	if !p.IsRunning() {
		return false
	} else if !p.hasChannel(chID) {
		return false
	}
	return p.mconn.TrySend(chID, msgBytes)
}

// Get the data for a given key.
func (p *peer) Get(key string) interface{} {
	return p.Data.Get(key)
}

// Set sets the data for the given key.
func (p *peer) Set(key string, data interface{}) {
	p.Data.Set(key, data)
}

// hasChannel returns true if the peer reported
// knowing about the given chID.
func (p *peer) hasChannel(chID byte) bool {
	for _, ch := range p.channels {
		if ch == chID {
			return true
		}
	}
	// NOTE: probably will want to remove this
	// but could be helpful while the feature is new
	p.Logger.Debug("Unknown channel for peer", "channel", chID, "channels", p.channels)
	return false
}

//---------------------------------------------------
// methods used by the Switch

// CloseConn should be called by the Switch if the peer was created but never started.
func (pc *peerConn) CloseConn() {
	pc.conn.Close() // nolint: errcheck
}

// HandshakeTimeout performs the Tendermint P2P handshake between a given node and the peer
// by exchanging their NodeInfo. It sets the received nodeInfo on the peer.
// NOTE: blocking
func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration) (peerNodeInfo NodeInfo, err error) {
	// Set deadline for handshake so we don't block forever on conn.ReadFull
	if err := pc.conn.SetDeadline(time.Now().Add(timeout)); err != nil {
		return peerNodeInfo, cmn.ErrorWrap(err, "Error setting deadline")
	}

	var trs, _ = cmn.Parallel(
		func(_ int) (val interface{}, err error, abort bool) {
			_, err = cdc.MarshalBinaryWriter(pc.conn, ourNodeInfo)
			return
		},
		func(_ int) (val interface{}, err error, abort bool) {
			_, err = cdc.UnmarshalBinaryReader(pc.conn, &peerNodeInfo, int64(MaxNodeInfoSize()))
			return
		},
	)
	if err := trs.FirstError(); err != nil {
		return peerNodeInfo, cmn.ErrorWrap(err, "Error during handshake")
	}

	// Remove deadline
	if err := pc.conn.SetDeadline(time.Time{}); err != nil {
		return peerNodeInfo, cmn.ErrorWrap(err, "Error removing deadline")
	}

	return peerNodeInfo, nil
}

// Addr returns peer's remote network address.
func (p *peer) Addr() net.Addr {
	return p.peerConn.conn.RemoteAddr()
}

// CanSend returns true if the send queue is not full, false otherwise.
func (p *peer) CanSend(chID byte) bool {
	if !p.IsRunning() {
		return false
	}
	return p.mconn.CanSend(chID)
}

// String representation.
func (p *peer) String() string {
	if p.outbound {
		return fmt.Sprintf("Peer{%v %v out}", p.mconn, p.ID())
	}

	return fmt.Sprintf("Peer{%v %v in}", p.mconn, p.ID())
}

//------------------------------------------------------------------
// helper funcs

func dial(addr *NetAddress, config *PeerConfig) (net.Conn, error) {
	if config.DialFail {
		return nil, fmt.Errorf("dial err (peerConfig.DialFail == true)")
	}

	conn, err := addr.DialTimeout(config.DialTimeout * time.Second)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func createMConnection(conn net.Conn, p *peer, reactorsByCh map[byte]Reactor, chDescs []*tmconn.ChannelDescriptor,
	onPeerError func(Peer, interface{}), config *tmconn.MConnConfig) *tmconn.MConnection {

	onReceive := func(chID byte, msgBytes []byte) {
		reactor := reactorsByCh[chID]
		if reactor == nil {
			// Note that its ok to panic here as it's caught in the conn._recover,
			// which does onPeerError.
			panic(cmn.Fmt("Unknown channel %X", chID))
		}
		reactor.Receive(chID, p, msgBytes)
	}

	onError := func(r interface{}) {
		onPeerError(p, r)
	}

	return tmconn.NewMConnectionWithConfig(conn, chDescs, onReceive, onError, config)
}
<commit_msg>Return fake IP even when there is no conn<commit_after>package p2p

import (
	"fmt"
	"net"
	"sync/atomic"
	"time"

	"github.com/tendermint/go-crypto"
	cmn "github.com/tendermint/tmlibs/common"
	"github.com/tendermint/tmlibs/log"

	tmconn "github.com/tendermint/tendermint/p2p/conn"
)

var testIPSuffix uint32 = 0

// Peer is an interface representing a peer connected on a reactor.
type Peer interface {
	cmn.Service

	ID() ID             // peer's cryptographic ID
	RemoteIP() net.IP   // remote IP of the connection
	IsOutbound() bool   // did we dial the peer
	IsPersistent() bool // do we redial this peer when we disconnect
	NodeInfo() NodeInfo // peer's info
	Status() tmconn.ConnectionStatus

	Send(byte, []byte) bool
	TrySend(byte, []byte) bool

	Set(string, interface{})
	Get(string) interface{}
}

//----------------------------------------------------------

// peerConn contains the raw connection and its config.
type peerConn struct {
	outbound   bool
	persistent bool
	config     *PeerConfig
	conn       net.Conn // source connection
	ips        []net.IP
}

// ID only exists for SecretConnection.
// NOTE: Will panic if conn is not *SecretConnection.
func (pc peerConn) ID() ID {
	return PubKeyToID(pc.conn.(*tmconn.SecretConnection).RemotePubKey())
}

// Return the IP from the connection RemoteAddr
func (pc peerConn) RemoteIP() net.IP {
	if len(pc.ips) > 0 {
		return pc.ips[0]
	}

	if pc.conn == nil || pc.conn.RemoteAddr().String() == "pipe" {
		pc.ips = []net.IP{
			net.IP{172, 16, 0, byte(atomic.AddUint32(&testIPSuffix, 1))},
		}

		return pc.ips[0]
	}

	host, _, err := net.SplitHostPort(pc.conn.RemoteAddr().String())
	if err != nil {
		panic(err)
	}

	ips, err := net.LookupIP(host)
	if err != nil {
		panic(err)
	}

	pc.ips = ips

	return ips[0]
}

// peer implements Peer.
//
// Before using a peer, you will need to perform a handshake on connection.
type peer struct {
	cmn.BaseService

	// raw peerConn and the multiplex connection
	peerConn
	mconn *tmconn.MConnection

	// peer's node info and the channel it knows about
	// channels = nodeInfo.Channels
	// cached to avoid copying nodeInfo in hasChannel
	nodeInfo NodeInfo
	channels []byte

	// User data
	Data *cmn.CMap
}

func newPeer(pc peerConn, nodeInfo NodeInfo,
	reactorsByCh map[byte]Reactor, chDescs []*tmconn.ChannelDescriptor,
	onPeerError func(Peer, interface{})) *peer {

	p := &peer{
		peerConn: pc,
		nodeInfo: nodeInfo,
		channels: nodeInfo.Channels,
		Data:     cmn.NewCMap(),
	}
	p.mconn = createMConnection(pc.conn, p, reactorsByCh, chDescs, onPeerError, pc.config.MConfig)
	p.BaseService = *cmn.NewBaseService(nil, "Peer", p)
	return p
}

// PeerConfig is a Peer configuration.
type PeerConfig struct {
	AuthEnc bool `mapstructure:"auth_enc"` // authenticated encryption

	// times are in seconds
	HandshakeTimeout time.Duration `mapstructure:"handshake_timeout"`
	DialTimeout      time.Duration `mapstructure:"dial_timeout"`

	MConfig *tmconn.MConnConfig `mapstructure:"connection"`

	DialFail   bool            `mapstructure:"dial_fail"` // for testing
	Fuzz       bool            `mapstructure:"fuzz"`      // fuzz connection (for testing)
	FuzzConfig *FuzzConnConfig `mapstructure:"fuzz_config"`
}

// DefaultPeerConfig returns the default config.
func DefaultPeerConfig() *PeerConfig {
	return &PeerConfig{
		AuthEnc:          true,
		HandshakeTimeout: 20, // * time.Second,
		DialTimeout:      3,  // * time.Second,
		MConfig:          tmconn.DefaultMConnConfig(),
		DialFail:         false,
		Fuzz:             false,
		FuzzConfig:       DefaultFuzzConnConfig(),
	}
}

func newOutboundPeerConn(addr *NetAddress, config *PeerConfig, persistent bool, ourNodePrivKey crypto.PrivKey) (peerConn, error) {
	var pc peerConn

	conn, err := dial(addr, config)
	if err != nil {
		return pc, cmn.ErrorWrap(err, "Error creating peer")
	}

	pc, err = newPeerConn(conn, config, true, persistent, ourNodePrivKey)
	if err != nil {
		if err2 := conn.Close(); err2 != nil {
			return pc, cmn.ErrorWrap(err, err2.Error())
		}
		return pc, err
	}

	// ensure dialed ID matches connection ID
	if config.AuthEnc && addr.ID != pc.ID() {
		if err2 := conn.Close(); err2 != nil {
			return pc, cmn.ErrorWrap(err, err2.Error())
		}
		return pc, ErrSwitchAuthenticationFailure{addr, pc.ID()}
	}
	return pc, nil
}

func newInboundPeerConn(conn net.Conn, config *PeerConfig, ourNodePrivKey crypto.PrivKey) (peerConn, error) {

	// TODO: issue PoW challenge

	return newPeerConn(conn, config, false, false, ourNodePrivKey)
}

func newPeerConn(rawConn net.Conn,
	config *PeerConfig, outbound, persistent bool,
	ourNodePrivKey crypto.PrivKey) (pc peerConn, err error) {

	conn := rawConn

	// Fuzz connection
	if config.Fuzz {
		// so we have time to do peer handshakes and get set up
		conn = FuzzConnAfterFromConfig(conn, 10*time.Second, config.FuzzConfig)
	}

	if config.AuthEnc {
		// Set deadline for secret handshake
		if err := conn.SetDeadline(time.Now().Add(config.HandshakeTimeout * time.Second)); err != nil {
			return pc, cmn.ErrorWrap(err, "Error setting deadline while encrypting connection")
		}

		// Encrypt connection
		conn, err = tmconn.MakeSecretConnection(conn, ourNodePrivKey)
		if err != nil {
			return pc, cmn.ErrorWrap(err, "Error creating peer")
		}
	}

	// Only the information we already have
	return peerConn{
		config:     config,
		outbound:   outbound,
		persistent: persistent,
		conn:       conn,
	}, nil
}

//---------------------------------------------------
// Implements cmn.Service

// SetLogger implements BaseService.
func (p *peer) SetLogger(l log.Logger) {
	p.Logger = l
	p.mconn.SetLogger(l)
}

// OnStart implements BaseService.
func (p *peer) OnStart() error {
	if err := p.BaseService.OnStart(); err != nil {
		return err
	}
	err := p.mconn.Start()
	return err
}

// OnStop implements BaseService.
func (p *peer) OnStop() {
	p.BaseService.OnStop()
	p.mconn.Stop() // stop everything and close the conn
}

//---------------------------------------------------
// Implements Peer

// ID returns the peer's ID - the hex encoded hash of its pubkey.
func (p *peer) ID() ID {
	return p.nodeInfo.ID
}

// IsOutbound returns true if the connection is outbound, false otherwise.
func (p *peer) IsOutbound() bool {
	return p.peerConn.outbound
}

// IsPersistent returns true if the peer is persitent, false otherwise.
func (p *peer) IsPersistent() bool {
	return p.peerConn.persistent
}

// NodeInfo returns a copy of the peer's NodeInfo.
func (p *peer) NodeInfo() NodeInfo {
	return p.nodeInfo
}

// Status returns the peer's ConnectionStatus.
func (p *peer) Status() tmconn.ConnectionStatus {
	return p.mconn.Status()
}

// Send msg bytes to the channel identified by chID byte. Returns false if the
// send queue is full after timeout, specified by MConnection.
func (p *peer) Send(chID byte, msgBytes []byte) bool {
	if !p.IsRunning() {
		// see Switch#Broadcast, where we fetch the list of peers and loop over
		// them - while we're looping, one peer may be removed and stopped.
		return false
	} else if !p.hasChannel(chID) {
		return false
	}
	return p.mconn.Send(chID, msgBytes)
}

// TrySend msg bytes to the channel identified by chID byte. Immediately returns
// false if the send queue is full.
func (p *peer) TrySend(chID byte, msgBytes []byte) bool {
	if !p.IsRunning() {
		return false
	} else if !p.hasChannel(chID) {
		return false
	}
	return p.mconn.TrySend(chID, msgBytes)
}

// Get the data for a given key.
func (p *peer) Get(key string) interface{} {
	return p.Data.Get(key)
}

// Set sets the data for the given key.
func (p *peer) Set(key string, data interface{}) {
	p.Data.Set(key, data)
}

// hasChannel returns true if the peer reported
// knowing about the given chID.
func (p *peer) hasChannel(chID byte) bool {
	for _, ch := range p.channels {
		if ch == chID {
			return true
		}
	}
	// NOTE: probably will want to remove this
	// but could be helpful while the feature is new
	p.Logger.Debug("Unknown channel for peer", "channel", chID, "channels", p.channels)
	return false
}

//---------------------------------------------------
// methods used by the Switch

// CloseConn should be called by the Switch if the peer was created but never started.
func (pc *peerConn) CloseConn() {
	pc.conn.Close() // nolint: errcheck
}

// HandshakeTimeout performs the Tendermint P2P handshake between a given node and the peer
// by exchanging their NodeInfo. It sets the received nodeInfo on the peer.
// NOTE: blocking
func (pc *peerConn) HandshakeTimeout(ourNodeInfo NodeInfo, timeout time.Duration) (peerNodeInfo NodeInfo, err error) {
	// Set deadline for handshake so we don't block forever on conn.ReadFull
	if err := pc.conn.SetDeadline(time.Now().Add(timeout)); err != nil {
		return peerNodeInfo, cmn.ErrorWrap(err, "Error setting deadline")
	}

	var trs, _ = cmn.Parallel(
		func(_ int) (val interface{}, err error, abort bool) {
			_, err = cdc.MarshalBinaryWriter(pc.conn, ourNodeInfo)
			return
		},
		func(_ int) (val interface{}, err error, abort bool) {
			_, err = cdc.UnmarshalBinaryReader(pc.conn, &peerNodeInfo, int64(MaxNodeInfoSize()))
			return
		},
	)
	if err := trs.FirstError(); err != nil {
		return peerNodeInfo, cmn.ErrorWrap(err, "Error during handshake")
	}

	// Remove deadline
	if err := pc.conn.SetDeadline(time.Time{}); err != nil {
		return peerNodeInfo, cmn.ErrorWrap(err, "Error removing deadline")
	}

	return peerNodeInfo, nil
}

// Addr returns peer's remote network address.
func (p *peer) Addr() net.Addr {
	return p.peerConn.conn.RemoteAddr()
}

// CanSend returns true if the send queue is not full, false otherwise.
func (p *peer) CanSend(chID byte) bool {
	if !p.IsRunning() {
		return false
	}
	return p.mconn.CanSend(chID)
}

// String representation.
func (p *peer) String() string {
	if p.outbound {
		return fmt.Sprintf("Peer{%v %v out}", p.mconn, p.ID())
	}

	return fmt.Sprintf("Peer{%v %v in}", p.mconn, p.ID())
}

//------------------------------------------------------------------
// helper funcs

func dial(addr *NetAddress, config *PeerConfig) (net.Conn, error) {
	if config.DialFail {
		return nil, fmt.Errorf("dial err (peerConfig.DialFail == true)")
	}

	conn, err := addr.DialTimeout(config.DialTimeout * time.Second)
	if err != nil {
		return nil, err
	}
	return conn, nil
}

func createMConnection(conn net.Conn, p *peer, reactorsByCh map[byte]Reactor, chDescs []*tmconn.ChannelDescriptor,
	onPeerError func(Peer, interface{}), config *tmconn.MConnConfig) *tmconn.MConnection {

	onReceive := func(chID byte, msgBytes []byte) {
		reactor := reactorsByCh[chID]
		if reactor == nil {
			// Note that its ok to panic here as it's caught in the conn._recover,
			// which does onPeerError.
			panic(cmn.Fmt("Unknown channel %X", chID))
		}
		reactor.Receive(chID, p, msgBytes)
	}

	onError := func(r interface{}) {
		onPeerError(p, r)
	}

	return tmconn.NewMConnectionWithConfig(conn, chDescs, onReceive, onError, config)
}
<|endoftext|>


<commit_before>package extension

import (
	"unicode"

	"github.com/yuin/goldmark"
	gast "github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

// TypographicPunctuation is a key of the punctuations that can be replaced with
// typographic entities.
type TypographicPunctuation int

const (
	// LeftSingleQuote is '
	LeftSingleQuote TypographicPunctuation = iota + 1
	// RightSingleQuote is '
	RightSingleQuote
	// LeftDoubleQuote is "
	LeftDoubleQuote
	// RightDoubleQuote is "
	RightDoubleQuote
	// EnDash is --
	EnDash
	// EmDash is ---
	EmDash
	// Ellipsis is ...
	Ellipsis
	// LeftAngleQuote is <<
	LeftAngleQuote
	// RightAngleQuote is >>
	RightAngleQuote
	// Apostrophe is '
	Apostrophe

	typographicPunctuationMax
)

// An TypographerConfig struct is a data structure that holds configuration of the
// Typographer extension.
type TypographerConfig struct {
	Substitutions [][]byte
}

func newDefaultSubstitutions() [][]byte {
	replacements := make([][]byte, typographicPunctuationMax)
	replacements[LeftSingleQuote] = []byte("‘")
	replacements[RightSingleQuote] = []byte("’")
	replacements[LeftDoubleQuote] = []byte("“")
	replacements[RightDoubleQuote] = []byte("”")
	replacements[EnDash] = []byte("–")
	replacements[EmDash] = []byte("—")
	replacements[Ellipsis] = []byte("…")
	replacements[LeftAngleQuote] = []byte("«")
	replacements[RightAngleQuote] = []byte("»")
	replacements[Apostrophe] = []byte("’")

	return replacements
}

// SetOption implements SetOptioner.
func (b *TypographerConfig) SetOption(name parser.OptionName, value interface{}) {
	switch name {
	case optTypographicSubstitutions:
		b.Substitutions = value.([][]byte)
	}
}

// A TypographerOption interface sets options for the TypographerParser.
type TypographerOption interface {
	parser.Option
	SetTypographerOption(*TypographerConfig)
}

const optTypographicSubstitutions parser.OptionName = "TypographicSubstitutions"

// TypographicSubstitutions is a list of the substitutions for the Typographer extension.
type TypographicSubstitutions map[TypographicPunctuation][]byte

type withTypographicSubstitutions struct {
	value [][]byte
}

func (o *withTypographicSubstitutions) SetParserOption(c *parser.Config) {
	c.Options[optTypographicSubstitutions] = o.value
}

func (o *withTypographicSubstitutions) SetTypographerOption(p *TypographerConfig) {
	p.Substitutions = o.value
}

// WithTypographicSubstitutions is a functional otpion that specify replacement text
// for punctuations.
func WithTypographicSubstitutions(values map[TypographicPunctuation][]byte) TypographerOption {
	replacements := newDefaultSubstitutions()
	for k, v := range values {
		replacements[k] = v
	}

	return &withTypographicSubstitutions{replacements}
}

type typographerDelimiterProcessor struct {
}

func (p *typographerDelimiterProcessor) IsDelimiter(b byte) bool {
	return b == '\'' || b == '"'
}

func (p *typographerDelimiterProcessor) CanOpenCloser(opener, closer *parser.Delimiter) bool {
	return opener.Char == closer.Char
}

func (p *typographerDelimiterProcessor) OnMatch(consumes int) gast.Node {
	return nil
}

var defaultTypographerDelimiterProcessor = &typographerDelimiterProcessor{}

type typographerParser struct {
	TypographerConfig
}

// NewTypographerParser return a new InlineParser that parses
// typographer expressions.
func NewTypographerParser(opts ...TypographerOption) parser.InlineParser {
	p := &typographerParser{
		TypographerConfig: TypographerConfig{
			Substitutions: newDefaultSubstitutions(),
		},
	}
	for _, o := range opts {
		o.SetTypographerOption(&p.TypographerConfig)
	}
	return p
}

func (s *typographerParser) Trigger() []byte {
	return []byte{'\'', '"', '-', '.', '<', '>'}
}

func (s *typographerParser) Parse(parent gast.Node, block text.Reader, pc parser.Context) gast.Node {
	before := block.PrecendingCharacter()
	line, _ := block.PeekLine()
	c := line[0]
	if len(line) > 2 {
		if c == '-' {
			if s.Substitutions[EmDash] != nil && line[1] == '-' && line[2] == '-' { // ---
				node := gast.NewString(s.Substitutions[EmDash])
				node.SetCode(true)
				block.Advance(3)
				return node
			}
		} else if c == '.' {
			if s.Substitutions[Ellipsis] != nil && line[1] == '.' && line[2] == '.' { // ...
				node := gast.NewString(s.Substitutions[Ellipsis])
				node.SetCode(true)
				block.Advance(3)
				return node
			}
			return nil
		}
	}
	if len(line) > 1 {
		if c == '<' {
			if s.Substitutions[LeftAngleQuote] != nil && line[1] == '<' { // <<
				node := gast.NewString(s.Substitutions[LeftAngleQuote])
				node.SetCode(true)
				block.Advance(2)
				return node
			}
			return nil
		} else if c == '>' {
			if s.Substitutions[RightAngleQuote] != nil && line[1] == '>' { // >>
				node := gast.NewString(s.Substitutions[RightAngleQuote])
				node.SetCode(true)
				block.Advance(2)
				return node
			}
			return nil
		} else if s.Substitutions[EnDash] != nil && c == '-' && line[1] == '-' { // --
			node := gast.NewString(s.Substitutions[EnDash])
			node.SetCode(true)
			block.Advance(2)
			return node
		}
	}
	if c == '\'' || c == '"' {
		d := parser.ScanDelimiter(line, before, 1, defaultTypographerDelimiterProcessor)
		if d == nil {
			return nil
		}
		if c == '\'' {
			if s.Substitutions[Apostrophe] != nil {
				// Handle decade abbrevations such as '90s
				if d.CanOpen && !d.CanClose && len(line) > 3 && util.IsNumeric(line[1]) && util.IsNumeric(line[2]) && line[3] == 's' {
					after := util.ToRune(line, 4)
					if len(line) == 3 || unicode.IsSpace(after) || unicode.IsPunct(after) {
						node := gast.NewString(s.Substitutions[Apostrophe])
						node.SetCode(true)
						block.Advance(1)
						return node
					}
				}
				// Convert normal apostrophes. This is probably more flexible than necessary but
				// converts any apostrophe in between two alphanumerics.
				if len(line) > 1 && (unicode.IsDigit(before) || unicode.IsLetter(before)) && (util.IsAlphaNumeric(line[1])) {
					node := gast.NewString(s.Substitutions[Apostrophe])
					node.SetCode(true)
					block.Advance(1)
					return node
				}
			}
			if s.Substitutions[LeftSingleQuote] != nil && d.CanOpen && !d.CanClose {
				node := gast.NewString(s.Substitutions[LeftSingleQuote])
				node.SetCode(true)
				block.Advance(1)
				return node
			}
			if s.Substitutions[RightSingleQuote] != nil && d.CanClose && !d.CanOpen {
				node := gast.NewString(s.Substitutions[RightSingleQuote])
				node.SetCode(true)
				block.Advance(1)
				return node
			}
		}
		if c == '"' {
			if s.Substitutions[LeftDoubleQuote] != nil && d.CanOpen && !d.CanClose {
				node := gast.NewString(s.Substitutions[LeftDoubleQuote])
				node.SetCode(true)
				block.Advance(1)
				return node
			}
			if s.Substitutions[RightDoubleQuote] != nil && d.CanClose && !d.CanOpen {
				node := gast.NewString(s.Substitutions[RightDoubleQuote])
				node.SetCode(true)
				block.Advance(1)
				return node
			}
		}
	}
	return nil
}

func (s *typographerParser) CloseBlock(parent gast.Node, pc parser.Context) {
	// nothing to do
}

type typographer struct {
	options []TypographerOption
}

// Typographer is an extension that replaces punctuations with typographic entities.
var Typographer = &typographer{}

// NewTypographer returns a new Extender that replaces punctuations with typographic entities.
func NewTypographer(opts ...TypographerOption) goldmark.Extender {
	return &typographer{
		options: opts,
	}
}

func (e *typographer) Extend(m goldmark.Markdown) {
	m.Parser().AddOptions(parser.WithInlineParsers(
		util.Prioritized(NewTypographerParser(e.options...), 9999),
	))
}
<commit_msg>Fixes #121<commit_after>package extension

import (
	"unicode"

	"github.com/yuin/goldmark"
	gast "github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/parser"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

// TypographicPunctuation is a key of the punctuations that can be replaced with
// typographic entities.
type TypographicPunctuation int

const (
	// LeftSingleQuote is '
	LeftSingleQuote TypographicPunctuation = iota + 1
	// RightSingleQuote is '
	RightSingleQuote
	// LeftDoubleQuote is "
	LeftDoubleQuote
	// RightDoubleQuote is "
	RightDoubleQuote
	// EnDash is --
	EnDash
	// EmDash is ---
	EmDash
	// Ellipsis is ...
	Ellipsis
	// LeftAngleQuote is <<
	LeftAngleQuote
	// RightAngleQuote is >>
	RightAngleQuote
	// Apostrophe is '
	Apostrophe

	typographicPunctuationMax
)

// An TypographerConfig struct is a data structure that holds configuration of the
// Typographer extension.
type TypographerConfig struct {
	Substitutions [][]byte
}

func newDefaultSubstitutions() [][]byte {
	replacements := make([][]byte, typographicPunctuationMax)
	replacements[LeftSingleQuote] = []byte("‘")
	replacements[RightSingleQuote] = []byte("’")
	replacements[LeftDoubleQuote] = []byte("“")
	replacements[RightDoubleQuote] = []byte("”")
	replacements[EnDash] = []byte("–")
	replacements[EmDash] = []byte("—")
	replacements[Ellipsis] = []byte("…")
	replacements[LeftAngleQuote] = []byte("«")
	replacements[RightAngleQuote] = []byte("»")
	replacements[Apostrophe] = []byte("’")

	return replacements
}

// SetOption implements SetOptioner.
func (b *TypographerConfig) SetOption(name parser.OptionName, value interface{}) {
	switch name {
	case optTypographicSubstitutions:
		b.Substitutions = value.([][]byte)
	}
}

// A TypographerOption interface sets options for the TypographerParser.
type TypographerOption interface {
	parser.Option
	SetTypographerOption(*TypographerConfig)
}

const optTypographicSubstitutions parser.OptionName = "TypographicSubstitutions"

// TypographicSubstitutions is a list of the substitutions for the Typographer extension.
type TypographicSubstitutions map[TypographicPunctuation][]byte

type withTypographicSubstitutions struct {
	value [][]byte
}

func (o *withTypographicSubstitutions) SetParserOption(c *parser.Config) {
	c.Options[optTypographicSubstitutions] = o.value
}

func (o *withTypographicSubstitutions) SetTypographerOption(p *TypographerConfig) {
	p.Substitutions = o.value
}

// WithTypographicSubstitutions is a functional otpion that specify replacement text
// for punctuations.
func WithTypographicSubstitutions(values map[TypographicPunctuation][]byte) TypographerOption {
	replacements := newDefaultSubstitutions()
	for k, v := range values {
		replacements[k] = v
	}

	return &withTypographicSubstitutions{replacements}
}

type typographerDelimiterProcessor struct {
}

func (p *typographerDelimiterProcessor) IsDelimiter(b byte) bool {
	return b == '\'' || b == '"'
}

func (p *typographerDelimiterProcessor) CanOpenCloser(opener, closer *parser.Delimiter) bool {
	return opener.Char == closer.Char
}

func (p *typographerDelimiterProcessor) OnMatch(consumes int) gast.Node {
	return nil
}

var defaultTypographerDelimiterProcessor = &typographerDelimiterProcessor{}

type typographerParser struct {
	TypographerConfig
}

// NewTypographerParser return a new InlineParser that parses
// typographer expressions.
func NewTypographerParser(opts ...TypographerOption) parser.InlineParser {
	p := &typographerParser{
		TypographerConfig: TypographerConfig{
			Substitutions: newDefaultSubstitutions(),
		},
	}
	for _, o := range opts {
		o.SetTypographerOption(&p.TypographerConfig)
	}
	return p
}

func (s *typographerParser) Trigger() []byte {
	return []byte{'\'', '"', '-', '.', '<', '>'}
}

func (s *typographerParser) Parse(parent gast.Node, block text.Reader, pc parser.Context) gast.Node {
	before := block.PrecendingCharacter()
	line, _ := block.PeekLine()
	c := line[0]
	if len(line) > 2 {
		if c == '-' {
			if s.Substitutions[EmDash] != nil && line[1] == '-' && line[2] == '-' { // ---
				node := gast.NewString(s.Substitutions[EmDash])
				node.SetCode(true)
				block.Advance(3)
				return node
			}
		} else if c == '.' {
			if s.Substitutions[Ellipsis] != nil && line[1] == '.' && line[2] == '.' { // ...
				node := gast.NewString(s.Substitutions[Ellipsis])
				node.SetCode(true)
				block.Advance(3)
				return node
			}
			return nil
		}
	}
	if len(line) > 1 {
		if c == '<' {
			if s.Substitutions[LeftAngleQuote] != nil && line[1] == '<' { // <<
				node := gast.NewString(s.Substitutions[LeftAngleQuote])
				node.SetCode(true)
				block.Advance(2)
				return node
			}
			return nil
		} else if c == '>' {
			if s.Substitutions[RightAngleQuote] != nil && line[1] == '>' { // >>
				node := gast.NewString(s.Substitutions[RightAngleQuote])
				node.SetCode(true)
				block.Advance(2)
				return node
			}
			return nil
		} else if s.Substitutions[EnDash] != nil && c == '-' && line[1] == '-' { // --
			node := gast.NewString(s.Substitutions[EnDash])
			node.SetCode(true)
			block.Advance(2)
			return node
		}
	}
	if c == '\'' || c == '"' {
		d := parser.ScanDelimiter(line, before, 1, defaultTypographerDelimiterProcessor)
		if d == nil {
			return nil
		}
		if c == '\'' {
			if s.Substitutions[Apostrophe] != nil {
				// Handle decade abbrevations such as '90s
				if d.CanOpen && !d.CanClose && len(line) > 3 && util.IsNumeric(line[1]) && util.IsNumeric(line[2]) && line[3] == 's' {
					after := util.ToRune(line, 4)
					if len(line) == 3 || unicode.IsSpace(after) || unicode.IsPunct(after) {
						node := gast.NewString(s.Substitutions[Apostrophe])
						node.SetCode(true)
						block.Advance(1)
						return node
					}
				}
				// Convert normal apostrophes. This is probably more flexible than necessary but
				// converts any apostrophe in between two alphanumerics.
				if len(line) > 1 && (unicode.IsDigit(before) || unicode.IsLetter(before)) && (unicode.IsLetter(util.ToRune(line, 1))) {
					node := gast.NewString(s.Substitutions[Apostrophe])
					node.SetCode(true)
					block.Advance(1)
					return node
				}
			}
			if s.Substitutions[LeftSingleQuote] != nil && d.CanOpen && !d.CanClose {
				node := gast.NewString(s.Substitutions[LeftSingleQuote])
				node.SetCode(true)
				block.Advance(1)
				return node
			}
			if s.Substitutions[RightSingleQuote] != nil && d.CanClose && !d.CanOpen {
				node := gast.NewString(s.Substitutions[RightSingleQuote])
				node.SetCode(true)
				block.Advance(1)
				return node
			}
		}
		if c == '"' {
			if s.Substitutions[LeftDoubleQuote] != nil && d.CanOpen && !d.CanClose {
				node := gast.NewString(s.Substitutions[LeftDoubleQuote])
				node.SetCode(true)
				block.Advance(1)
				return node
			}
			if s.Substitutions[RightDoubleQuote] != nil && d.CanClose && !d.CanOpen {
				node := gast.NewString(s.Substitutions[RightDoubleQuote])
				node.SetCode(true)
				block.Advance(1)
				return node
			}
		}
	}
	return nil
}

func (s *typographerParser) CloseBlock(parent gast.Node, pc parser.Context) {
	// nothing to do
}

type typographer struct {
	options []TypographerOption
}

// Typographer is an extension that replaces punctuations with typographic entities.
var Typographer = &typographer{}

// NewTypographer returns a new Extender that replaces punctuations with typographic entities.
func NewTypographer(opts ...TypographerOption) goldmark.Extender {
	return &typographer{
		options: opts,
	}
}

func (e *typographer) Extend(m goldmark.Markdown) {
	m.Parser().AddOptions(parser.WithInlineParsers(
		util.Prioritized(NewTypographerParser(e.options...), 9999),
	))
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"

	"golang.org/x/net/trace"
	"golang.org/x/oauth2"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/sniff"
)

type Opts struct {
	LogLevel                    string        `long:"log-level" description:"log level to use"`
	WorkDir                     string        `long:"work-dir" description:"directory to work in" value-name:"PATH" required:"true"`
	RepositoryDiscoveryInterval time.Duration `long:"repository-discovery-interval" description:"how frequently to ask GitHub for all repos to check which ones we need to clone and dirscan" required:"true" value-name:"SCAN_INTERVAL" default:"1h"`
	ChangeDiscoveryInterval     time.Duration `long:"change-discovery-interval" description:"how frequently to fetch changes for repositories on disk and scan the changes" required:"true" value-name:"SCAN_INTERVAL" default:"1h"`
	MinFetchInterval            time.Duration `long:"min-fetch-interval" description:"the minimum frequency tofetch changes for repositories on disk and scan the changes" value-name:"MIN_FETCH_INTERVAL" default:"6h"`
	MaxFetchInterval            time.Duration `long:"max-fetch-interval" description:"the maximum frequency tofetch changes for repositories on disk and scan the changes" value-name:"MAX_FETCH_INTERVAL" default:"168h"`

	Whitelist []string `short:"i" long:"ignore-pattern" description:"List of regex patterns to ignore." env:"IGNORED_PATTERNS" env-delim:"," value-name:"REGEX"`

	RPCBindIP   string `long:"rpc-server-bind-ip" default:"0.0.0.0" description:"IP address on which to listen for RPC traffic."`
	RPCBindPort uint16 `long:"rpc-server-bind-port" default:"50051" description:"Port on which to listen for RPC traffic."`

	GitHub struct {
		AccessToken    string `short:"a" long:"access-token" description:"github api access token" env:"GITHUB_ACCESS_TOKEN" value-name:"TOKEN" required:"true"`
		PrivateKeyPath string `long:"github-private-key-path" description:"private key to use for GitHub auth" required:"true" value-name:"SSH_KEY"`
		PublicKeyPath  string `long:"github-public-key-path" description:"public key to use for GitHub auth" required:"true" value-name:"SSH_KEY"`
	} `group:"GitHub Options"`

	PubSub struct {
		ProjectName string `long:"pubsub-project-name" description:"GCP Project Name" value-name:"NAME" required:"true"`

		FetchHint struct {
			Subscription string `long:"fetch-hint-pubsub-subscription" description:"PubSub Topic receive messages from" value-name:"NAME" required:"true"`
		} `group:"PubSub Fetch Hint Options"`
	} `group:"PubSub Options"`

	Metrics struct {
		SentryDSN     string `long:"sentry-dsn" description:"DSN to emit to Sentry with" env:"SENTRY_DSN" value-name:"DSN"`
		DatadogAPIKey string `long:"datadog-api-key" description:"key to emit to datadog" env:"DATADOG_API_KEY" value-name:"KEY"`
		Environment   string `long:"environment" description:"environment tag for metrics" env:"ENVIRONMENT" value-name:"NAME" default:"development"`
	} `group:"Metrics Options"`

	Slack struct {
		WebhookUrl string `long:"slack-webhook-url" description:"Slack webhook URL" env:"SLACK_WEBHOOK_URL" value-name:"WEBHOOK"`
	} `group:"Slack Options"`

	MySQL struct {
		Username string `long:"mysql-username" description:"MySQL username" value-name:"USERNAME" required:"true"`
		Password string `long:"mysql-password" description:"MySQL password" value-name:"PASSWORD"`
		Hostname string `long:"mysql-hostname" description:"MySQL hostname" value-name:"HOSTNAME" required:"true"`
		Port     uint16 `long:"mysql-port" description:"MySQL port" value-name:"PORT" required:"true"`
		DBName   string `long:"mysql-dbname" description:"MySQL database name" value-name:"DBNAME" required:"true"`
	}

	RPC struct {
		ClientCACertificate string `long:"rpc-server-client-ca" description:"Path to client CA certificate" required:"true"`
		Certificate         string `long:"rpc-server-cert" description:"Path to RPC server certificate" required:"true"`
		PrivateKey          string `long:"rpc-server-private-key" description:"Path to RPC server private key" required:"true"`
	}
}

func main() {
	var opts Opts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Debug("starting")

	_, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	if opts.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(opts.Metrics.SentryDSN, opts.Metrics.Environment))
	}

	workdir := opts.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	githubHTTPClient := &http.Client{
		Timeout: 30 * time.Second,
		Transport: &oauth2.Transport{
			Source: oauth2.StaticTokenSource(
				&oauth2.Token{AccessToken: opts.GitHub.AccessToken},
			),
			Base: &http.Transport{
				DisableKeepAlives: true,
			},
		},
	}

	dbURI := db.NewDSN(opts.MySQL.Username, opts.MySQL.Password, opts.MySQL.DBName, opts.MySQL.Hostname, int(opts.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clock := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)
	ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

	scanRepository := db.NewScanRepository(database, clock)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	fetchIntervalUpdater := revok.NewFetchIntervalUpdater(
		repositoryRepository,
		opts.MinFetchInterval,
		opts.MaxFetchInterval,
	)
	credentialRepository := db.NewCredentialRepository(database)
	emitter := metrics.BuildEmitter(opts.Metrics.DatadogAPIKey, opts.Metrics.Environment)
	gitClient := gitclient.New(opts.GitHub.PrivateKeyPath, opts.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(opts.Whitelist...)
	notifier := notifications.NewSlackNotifier(opts.Slack.WebhookUrl, clock, repoWhitelist)
	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
		notifier,
		emitter,
	)

	repoDiscoverer := revok.NewRepoDiscoverer(
		logger,
		workdir,
		cloneMsgCh,
		ghClient,
		clock,
		opts.RepositoryDiscoveryInterval,
		repositoryRepository,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
	)

	changeDiscoverer := revok.NewChangeDiscoverer(
		logger,
		gitClient,
		clock,
		opts.ChangeDiscoveryInterval,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		fetchIntervalUpdater,
		emitter,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		ancestryScanner,
		notifier,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clock,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	pushEventProcessor := revok.NewPushEventProcessor(
		changeDiscoverer,
		repositoryRepository,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		repositoryRepository,
		clock,
		24*time.Hour,
		gitClient,
		sniffer,
	)

	certificate, err := tls.LoadX509KeyPair(
		opts.RPC.Certificate,
		opts.RPC.PrivateKey,
	)

	clientCertPool := x509.NewCertPool()
	bs, err := ioutil.ReadFile(opts.RPC.ClientCACertificate)
	if err != nil {
		log.Fatalf("failed to read client ca certificate: %s", err.Error())
	}

	ok := clientCertPool.AppendCertsFromPEM(bs)
	if !ok {
		log.Fatalf("failed to append client certs from pem: %s", err.Error())
	}

	grpcServer := revok.NewGRPCServer(
		logger,
		fmt.Sprintf("%s:%d", opts.RPCBindIP, opts.RPCBindPort),
		revok.NewRevokServer(logger, repositoryRepository),
		&tls.Config{
			ClientAuth:   tls.RequireAndVerifyClientCert,
			Certificates: []tls.Certificate{certificate},
			ClientCAs:    clientCertPool,
		},
	)

	pubSubClient, err := pubsub.NewClient(context.Background(), opts.PubSub.ProjectName)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}
	hintSubscription := pubSubClient.Subscription(opts.PubSub.FetchHint.Subscription)

	failedMessageRepo := db.NewFailedMessageRepository(database)
	ack := queue.NewAcker()

	retryHandler := queue.NewRetryHandler(failedMessageRepo, pushEventProcessor, ack)

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, []grouper.Member{
		{"repo-discoverer", repoDiscoverer},
		{"cloner", cloner},
		{"change-discoverer", changeDiscoverer},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"github-hint-handler", queue.NewPubSubSubscriber(logger, hintSubscription, retryHandler)},
		{"head-credential-counter", headCredentialCounter},
		{"grpc-server", grpcServer},
		{"debug", http_server.New("127.0.0.1:6060", debugHandler())},
	}))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<commit_msg>Fix typo in revok worker options<commit_after>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"

	"golang.org/x/net/trace"
	"golang.org/x/oauth2"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/sniff"
)

type Opts struct {
	LogLevel                    string        `long:"log-level" description:"log level to use"`
	WorkDir                     string        `long:"work-dir" description:"directory to work in" value-name:"PATH" required:"true"`
	RepositoryDiscoveryInterval time.Duration `long:"repository-discovery-interval" description:"how frequently to ask GitHub for all repos to check which ones we need to clone and dirscan" required:"true" value-name:"SCAN_INTERVAL" default:"1h"`
	ChangeDiscoveryInterval     time.Duration `long:"change-discovery-interval" description:"how frequently to fetch changes for repositories on disk and scan the changes" required:"true" value-name:"SCAN_INTERVAL" default:"1h"`
	MinFetchInterval            time.Duration `long:"min-fetch-interval" description:"the minimum frequency to fetch changes for repositories on disk and scan the changes" value-name:"MIN_FETCH_INTERVAL" default:"6h"`
	MaxFetchInterval            time.Duration `long:"max-fetch-interval" description:"the maximum frequency to fetch changes for repositories on disk and scan the changes" value-name:"MAX_FETCH_INTERVAL" default:"168h"`

	Whitelist []string `short:"i" long:"ignore-pattern" description:"List of regex patterns to ignore." env:"IGNORED_PATTERNS" env-delim:"," value-name:"REGEX"`

	RPCBindIP   string `long:"rpc-server-bind-ip" default:"0.0.0.0" description:"IP address on which to listen for RPC traffic."`
	RPCBindPort uint16 `long:"rpc-server-bind-port" default:"50051" description:"Port on which to listen for RPC traffic."`

	GitHub struct {
		AccessToken    string `short:"a" long:"access-token" description:"github api access token" env:"GITHUB_ACCESS_TOKEN" value-name:"TOKEN" required:"true"`
		PrivateKeyPath string `long:"github-private-key-path" description:"private key to use for GitHub auth" required:"true" value-name:"SSH_KEY"`
		PublicKeyPath  string `long:"github-public-key-path" description:"public key to use for GitHub auth" required:"true" value-name:"SSH_KEY"`
	} `group:"GitHub Options"`

	PubSub struct {
		ProjectName string `long:"pubsub-project-name" description:"GCP Project Name" value-name:"NAME" required:"true"`

		FetchHint struct {
			Subscription string `long:"fetch-hint-pubsub-subscription" description:"PubSub Topic receive messages from" value-name:"NAME" required:"true"`
		} `group:"PubSub Fetch Hint Options"`
	} `group:"PubSub Options"`

	Metrics struct {
		SentryDSN     string `long:"sentry-dsn" description:"DSN to emit to Sentry with" env:"SENTRY_DSN" value-name:"DSN"`
		DatadogAPIKey string `long:"datadog-api-key" description:"key to emit to datadog" env:"DATADOG_API_KEY" value-name:"KEY"`
		Environment   string `long:"environment" description:"environment tag for metrics" env:"ENVIRONMENT" value-name:"NAME" default:"development"`
	} `group:"Metrics Options"`

	Slack struct {
		WebhookUrl string `long:"slack-webhook-url" description:"Slack webhook URL" env:"SLACK_WEBHOOK_URL" value-name:"WEBHOOK"`
	} `group:"Slack Options"`

	MySQL struct {
		Username string `long:"mysql-username" description:"MySQL username" value-name:"USERNAME" required:"true"`
		Password string `long:"mysql-password" description:"MySQL password" value-name:"PASSWORD"`
		Hostname string `long:"mysql-hostname" description:"MySQL hostname" value-name:"HOSTNAME" required:"true"`
		Port     uint16 `long:"mysql-port" description:"MySQL port" value-name:"PORT" required:"true"`
		DBName   string `long:"mysql-dbname" description:"MySQL database name" value-name:"DBNAME" required:"true"`
	}

	RPC struct {
		ClientCACertificate string `long:"rpc-server-client-ca" description:"Path to client CA certificate" required:"true"`
		Certificate         string `long:"rpc-server-cert" description:"Path to RPC server certificate" required:"true"`
		PrivateKey          string `long:"rpc-server-private-key" description:"Path to RPC server private key" required:"true"`
	}
}

func main() {
	var opts Opts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Debug("starting")

	_, err := flags.Parse(&opts)
	if err != nil {
		os.Exit(1)
	}

	if opts.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(opts.Metrics.SentryDSN, opts.Metrics.Environment))
	}

	workdir := opts.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	githubHTTPClient := &http.Client{
		Timeout: 30 * time.Second,
		Transport: &oauth2.Transport{
			Source: oauth2.StaticTokenSource(
				&oauth2.Token{AccessToken: opts.GitHub.AccessToken},
			),
			Base: &http.Transport{
				DisableKeepAlives: true,
			},
		},
	}

	dbURI := db.NewDSN(opts.MySQL.Username, opts.MySQL.Password, opts.MySQL.DBName, opts.MySQL.Hostname, int(opts.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clock := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)
	ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

	scanRepository := db.NewScanRepository(database, clock)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	fetchIntervalUpdater := revok.NewFetchIntervalUpdater(
		repositoryRepository,
		opts.MinFetchInterval,
		opts.MaxFetchInterval,
	)
	credentialRepository := db.NewCredentialRepository(database)
	emitter := metrics.BuildEmitter(opts.Metrics.DatadogAPIKey, opts.Metrics.Environment)
	gitClient := gitclient.New(opts.GitHub.PrivateKeyPath, opts.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(opts.Whitelist...)
	notifier := notifications.NewSlackNotifier(opts.Slack.WebhookUrl, clock, repoWhitelist)
	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
		notifier,
		emitter,
	)

	repoDiscoverer := revok.NewRepoDiscoverer(
		logger,
		workdir,
		cloneMsgCh,
		ghClient,
		clock,
		opts.RepositoryDiscoveryInterval,
		repositoryRepository,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
	)

	changeDiscoverer := revok.NewChangeDiscoverer(
		logger,
		gitClient,
		clock,
		opts.ChangeDiscoveryInterval,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		fetchIntervalUpdater,
		emitter,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		ancestryScanner,
		notifier,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clock,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	pushEventProcessor := revok.NewPushEventProcessor(
		changeDiscoverer,
		repositoryRepository,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		repositoryRepository,
		clock,
		24*time.Hour,
		gitClient,
		sniffer,
	)

	certificate, err := tls.LoadX509KeyPair(
		opts.RPC.Certificate,
		opts.RPC.PrivateKey,
	)

	clientCertPool := x509.NewCertPool()
	bs, err := ioutil.ReadFile(opts.RPC.ClientCACertificate)
	if err != nil {
		log.Fatalf("failed to read client ca certificate: %s", err.Error())
	}

	ok := clientCertPool.AppendCertsFromPEM(bs)
	if !ok {
		log.Fatalf("failed to append client certs from pem: %s", err.Error())
	}

	grpcServer := revok.NewGRPCServer(
		logger,
		fmt.Sprintf("%s:%d", opts.RPCBindIP, opts.RPCBindPort),
		revok.NewRevokServer(logger, repositoryRepository),
		&tls.Config{
			ClientAuth:   tls.RequireAndVerifyClientCert,
			Certificates: []tls.Certificate{certificate},
			ClientCAs:    clientCertPool,
		},
	)

	pubSubClient, err := pubsub.NewClient(context.Background(), opts.PubSub.ProjectName)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}
	hintSubscription := pubSubClient.Subscription(opts.PubSub.FetchHint.Subscription)

	failedMessageRepo := db.NewFailedMessageRepository(database)
	ack := queue.NewAcker()

	retryHandler := queue.NewRetryHandler(failedMessageRepo, pushEventProcessor, ack)

	runner := sigmon.New(grouper.NewParallel(os.Interrupt, []grouper.Member{
		{"repo-discoverer", repoDiscoverer},
		{"cloner", cloner},
		{"change-discoverer", changeDiscoverer},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"github-hint-handler", queue.NewPubSubSubscriber(logger, hintSubscription, retryHandler)},
		{"head-credential-counter", headCredentialCounter},
		{"grpc-server", grpcServer},
		{"debug", http_server.New("127.0.0.1:6060", debugHandler())},
	}))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/net/trace"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/revokpb"
	"cred-alert/search"
	"cred-alert/sniff"
	"red/redrunner"
	"rolodex/rolodexpb"
)

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Info("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
	if err != nil {
		logger.Error("failed-opening-config-file", err)
		os.Exit(1)
	}

	cfg, err = config.LoadWorkerConfig(bs)

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbURI := db.NewDSN(cfg.MySQL.Username, cfg.MySQL.Password, cfg.MySQL.DBName, cfg.MySQL.Hostname, int(cfg.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clk := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clk)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(cfg.GitHub.PrivateKeyPath, cfg.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)
	formatter := notifications.NewSlackNotificationFormatter()
	notifier := notifications.NewSlackNotifier(clk, formatter)

	certificate, err := config.LoadCertificate(
		cfg.Identity.CertificatePath,
		cfg.Identity.PrivateKeyPath,
		cfg.Identity.PrivateKeyPassphrase,
	)
	if err != nil {
		log.Fatalln(err)
	}

	caCertPool, err := config.LoadCertificatePool(cfg.Identity.CACertificatePath)
	if err != nil {
		log.Fatalln(err)
	}

	rolodexServerAddr := fmt.Sprintf("%s:%d", cfg.Rolodex.ServerAddress, cfg.Rolodex.ServerPort)

	transportCreds := credentials.NewTLS(&tls.Config{
		Certificates: []tls.Certificate{certificate},
		RootCAs:      caCertPool,
	})

	dialOption := grpc.WithTransportCredentials(transportCreds)
	conn, err := grpc.Dial(rolodexServerAddr, dialOption)

	rolodexClient := rolodexpb.NewRolodexClient(conn)

	teamURLs := notifications.NewTeamURLs(
		cfg.Slack.DefaultURL,
		cfg.Slack.DefaultChannel,
		cfg.Slack.TeamURLs,
	)

	addressBook := notifications.NewRolodex(
		rolodexClient,
		teamURLs,
	)

	router := notifications.NewRouter(
		notifier,
		addressBook,
		repoWhitelist,
	)

	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
		router,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		ancestryScanner,
		router,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clk,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		repositoryRepository,
		clk,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", http_server.New("127.0.0.1:6060", debugHandler())},
	}

	looper := gitclient.NewLooper()
	searcher := search.NewSearcher(repositoryRepository, looper)

	grpcServer := redrunner.NewGRPCServer(
		logger,
		fmt.Sprintf("%s:%d", cfg.API.BindIP, cfg.API.BindPort),
		&tls.Config{
			ClientAuth:   tls.RequireAndVerifyClientCert,
			Certificates: []tls.Certificate{certificate},
			ClientCAs:    caCertPool,
		},
		func(server *grpc.Server) {
			revokpb.RegisterRevokServer(server, revok.NewServer(logger, repositoryRepository, searcher))
		},
	)

	members = append(members, grouper.Member{
		Name:   "grpc-server",
		Runner: grpcServer,
	})

	pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

	publicKey, err := crypto.ReadRSAPublicKey(cfg.PubSub.PublicKeyPath)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}
	pushEventProcessor := queue.NewPushEventProcessor(
		changeFetcher,
		crypto.NewRSAVerifier(publicKey),
		emitter,
	)

	members = append(members, grouper.Member{
		Name:   "github-hint-handler",
		Runner: queue.NewPubSubAcceptor(logger, subscription, pushEventProcessor, emitter),
	})

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clk,
			cfg.RepositoryDiscoveryInterval,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<commit_msg>[revok] use keepalive for rolodex connection<commit_after>package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/net/trace"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/revokpb"
	"cred-alert/search"
	"cred-alert/sniff"
	"red/redrunner"
	"rolodex/rolodexpb"
)

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Info("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
	if err != nil {
		logger.Error("failed-opening-config-file", err)
		os.Exit(1)
	}

	cfg, err = config.LoadWorkerConfig(bs)

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbURI := db.NewDSN(cfg.MySQL.Username, cfg.MySQL.Password, cfg.MySQL.DBName, cfg.MySQL.Hostname, int(cfg.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clk := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clk)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(cfg.GitHub.PrivateKeyPath, cfg.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)
	formatter := notifications.NewSlackNotificationFormatter()
	notifier := notifications.NewSlackNotifier(clk, formatter)

	certificate, err := config.LoadCertificate(
		cfg.Identity.CertificatePath,
		cfg.Identity.PrivateKeyPath,
		cfg.Identity.PrivateKeyPassphrase,
	)
	if err != nil {
		log.Fatalln(err)
	}

	caCertPool, err := config.LoadCertificatePool(cfg.Identity.CACertificatePath)
	if err != nil {
		log.Fatalln(err)
	}

	rolodexServerAddr := fmt.Sprintf("%s:%d", cfg.Rolodex.ServerAddress, cfg.Rolodex.ServerPort)

	transportCreds := credentials.NewTLS(&tls.Config{
		Certificates: []tls.Certificate{certificate},
		RootCAs:      caCertPool,
	})

	conn, err := grpc.Dial(
		rolodexServerAddr,
		grpc.WithDialer(keepAliveDial),
		grpc.WithTransportCredentials(transportCreds),
	)

	rolodexClient := rolodexpb.NewRolodexClient(conn)

	teamURLs := notifications.NewTeamURLs(
		cfg.Slack.DefaultURL,
		cfg.Slack.DefaultChannel,
		cfg.Slack.TeamURLs,
	)

	addressBook := notifications.NewRolodex(
		rolodexClient,
		teamURLs,
	)

	router := notifications.NewRouter(
		notifier,
		addressBook,
		repoWhitelist,
	)

	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
		router,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		ancestryScanner,
		router,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clk,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		repositoryRepository,
		clk,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", http_server.New("127.0.0.1:6060", debugHandler())},
	}

	looper := gitclient.NewLooper()
	searcher := search.NewSearcher(repositoryRepository, looper)

	grpcServer := redrunner.NewGRPCServer(
		logger,
		fmt.Sprintf("%s:%d", cfg.API.BindIP, cfg.API.BindPort),
		&tls.Config{
			ClientAuth:   tls.RequireAndVerifyClientCert,
			Certificates: []tls.Certificate{certificate},
			ClientCAs:    caCertPool,
		},
		func(server *grpc.Server) {
			revokpb.RegisterRevokServer(server, revok.NewServer(logger, repositoryRepository, searcher))
		},
	)

	members = append(members, grouper.Member{
		Name:   "grpc-server",
		Runner: grpcServer,
	})

	pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

	publicKey, err := crypto.ReadRSAPublicKey(cfg.PubSub.PublicKeyPath)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}
	pushEventProcessor := queue.NewPushEventProcessor(
		changeFetcher,
		crypto.NewRSAVerifier(publicKey),
		emitter,
	)

	members = append(members, grouper.Member{
		Name:   "github-hint-handler",
		Runner: queue.NewPubSubAcceptor(logger, subscription, pushEventProcessor, emitter),
	})

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clk,
			cfg.RepositoryDiscoveryInterval,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func keepAliveDial(addr string, timeout time.Duration) (net.Conn, error) {
	d := net.Dialer{
		Timeout:   timeout,
		KeepAlive: 60 * time.Second,
	}
	return d.Dial("tcp", addr)
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<|endoftext|>