<commit_before>// Copyright 2011 Julian Phillips.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package py

// #include "utils.h"
// static size_t __PyObject_VAR_SIZE(PyObject *obj, Py_ssize_t n) {
//     return _PyObject_VAR_SIZE((PyTypeObject *)obj, n);
// }
// static void _PyObject_INIT(PyObject *obj, PyObject *typ) {
//     PyObject_INIT(obj, (PyTypeObject *)typ);
// }
// static void _PyObject_INIT_VAR(PyObject *obj, PyObject *typ, Py_ssize_t n) {
//     PyObject_INIT_VAR(obj, (PyTypeObject *)typ, n);
// }
// static void __PyObject_GC_TRACK(PyObject *obj) {
//     _PyObject_GC_TRACK(obj);
// }
// static void setGcRefs(PyGC_Head *g, Py_ssize_t refs) {
//     g->gc.gc_refs = refs;
// }
import "C"

// This file is about as unsafe as you can get ... we are playing tricks on the
// Go runtime in here. :(
import "unsafe"

import "sync"

// Some sizes that we need for various calculations
const (
	upSize   = unsafe.Sizeof(unsafe.Pointer(nil))
	headSize = unsafe.Sizeof(C.PyGC_Head{})
	baseSize = unsafe.Sizeof(C.PyObject{})
)

// We need to keep track of things that we have allocated, and the proxies that
// we have created, memLock must be locked whilst using these variables.
var (
	memLock sync.Mutex
	allocated = make(map[uintptr][]unsafe.Pointer)
	gcProxies = make(map[uintptr]*C.PyObject)
)

func fromGc(g *C.PyGC_Head) *C.PyObject {
	p := unsafe.Pointer(g)
	o := unsafe.Pointer(uintptr(p) + headSize)
	return (*C.PyObject)(o)
}

func goGcMalloc(size uintptr) *C.PyObject {
	// first, lock memLock, and arrange for it to be unlocked on return
	memLock.Lock()
	defer memLock.Unlock()

	g := (*C.PyGC_Head)(unsafe.Pointer(_goMalloc(size + headSize)))
	C.setGcRefs(g, C._PyGC_REFS_UNTRACKED)
	p := fromGc(g)

	// We need to move the original tracked entry to be indexed by the offset
	// address from fromGc
	px := uintptr(unsafe.Pointer(p))
	gx := uintptr(unsafe.Pointer(g))
	allocated[px] = allocated[gx]
	delete(allocated, gx)

	// We can't access the internals of the GC Module to manipulate the
	// generation counts, so we have to use a proxy object instead.  We just
	// create a bare minimum object, initialise it, and then store it away to be
	// cleaned up later.
	proxy := C._PyObject_GC_Malloc(C.size_t(baseSize))
	if proxy == nil {
		delete(allocated, px)
		return nil
	}
	C._PyObject_INIT(proxy, c(BaseType))
	gcProxies[px] = proxy

	return p
}

// _goMalloc must be called with memLock already locked.
func _goMalloc(size uintptr) *C.PyObject {
	// We have to use []unsafe.Pointer instead of []byte, otherwise the Go
	// runtime will mark the memory as not containing pointers, and won't use it
	// to pin other Go allocations in memory - in which case we might as well
	// just stick with the Python allocator.
	n := (size + upSize - 1) / upSize
	s := make([]unsafe.Pointer, n)
	p := unsafe.Pointer(&s[0])

	// We need to keep a reference to the allocation, so that the Go runtime
	// doesn't free the memory before we are finished with it.
	allocated[uintptr(p)] = s

	return (*C.PyObject)(p)
}

func goMalloc(size uintptr) *C.PyObject {
	// first, lock memLock, and arrange for it to be unlocked on return
	memLock.Lock()
	defer memLock.Unlock()

	// now call _goMalloc to do the actual work
	return _goMalloc(size)
}

//export goGenericAlloc
func goGenericAlloc(t unsafe.Pointer, n C.Py_ssize_t) unsafe.Pointer {
	var obj *C.PyObject

	typ := newType((*C.PyObject)(t))
	size := uintptr(C.__PyObject_VAR_SIZE(c(typ), n))

	if typ.IsGc() {
		obj = goGcMalloc(size)
	} else {
		obj = goMalloc(size)
	}

	if obj == nil {
		raise(MemoryError.ErrV(None))
		return nil
	}

	if typ.o.tp_flags & C.Py_TPFLAGS_HEAPTYPE != 0 {
		typ.Incref()
	}

	if typ.o.tp_itemsize == 0 {
		C._PyObject_INIT(obj, c(typ))
	} else {
		C._PyObject_INIT_VAR(obj, c(typ), n)
	}

	if typ.IsGc() {
		C.__PyObject_GC_TRACK(obj)
	}

	return unsafe.Pointer(obj)
}

//export goGenericFree
func goGenericFree(o unsafe.Pointer) {
	// first, lock memLock, and arrange for it to be unlocked on return
	memLock.Lock()
	defer memLock.Unlock()

	// Remove the entry from allocated, to let the Go runtime reclaim the memory
	// in the next GC run.
	delete(allocated, uintptr(o))

	// We need to also delete the proxy, so that the Python GC counts get
	// updated as appropriate.
	proxy, ok := gcProxies[uintptr(o)]
	if ok {
		C.PyObject_GC_Del(unsafe.Pointer(proxy))
		delete(gcProxies, uintptr(o))
	}
}
<commit_msg>Fix for "function is not needed and will not be emitted" errors<commit_after>// Copyright 2011 Julian Phillips.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package py

// #include "utils.h"
// __attribute__ ((used)) static size_t __PyObject_VAR_SIZE(PyObject *obj, Py_ssize_t n){
//     return _PyObject_VAR_SIZE((PyTypeObject *)obj, n);
// }
// __attribute__ ((used)) static void _PyObject_INIT(PyObject *obj, PyObject *typ){
//     PyObject_INIT(obj, (PyTypeObject *)typ);
// }
// __attribute__ ((used)) static void _PyObject_INIT_VAR(PyObject *obj, PyObject *typ, Py_ssize_t n){
//     PyObject_INIT_VAR(obj, (PyTypeObject *)typ, n);
// }
// __attribute__ ((used)) static void __PyObject_GC_TRACK(PyObject *obj){
//     _PyObject_GC_TRACK(obj);
// }
// __attribute__ ((used)) static void setGcRefs(PyGC_Head *g, Py_ssize_t refs){
//     g->gc.gc_refs = refs;
// }
import "C"

// This file is about as unsafe as you can get ... we are playing tricks on the
// Go runtime in here. :(
import "unsafe"

import "sync"

// Some sizes that we need for various calculations
const (
	upSize   = unsafe.Sizeof(unsafe.Pointer(nil))
	headSize = unsafe.Sizeof(C.PyGC_Head{})
	baseSize = unsafe.Sizeof(C.PyObject{})
)

// We need to keep track of things that we have allocated, and the proxies that
// we have created, memLock must be locked whilst using these variables.
var (
	memLock sync.Mutex
	allocated = make(map[uintptr][]unsafe.Pointer)
	gcProxies = make(map[uintptr]*C.PyObject)
)

func fromGc(g *C.PyGC_Head) *C.PyObject {
	p := unsafe.Pointer(g)
	o := unsafe.Pointer(uintptr(p) + headSize)
	return (*C.PyObject)(o)
}

func goGcMalloc(size uintptr) *C.PyObject {
	// first, lock memLock, and arrange for it to be unlocked on return
	memLock.Lock()
	defer memLock.Unlock()

	g := (*C.PyGC_Head)(unsafe.Pointer(_goMalloc(size + headSize)))
	C.setGcRefs(g, C._PyGC_REFS_UNTRACKED)
	p := fromGc(g)

	// We need to move the original tracked entry to be indexed by the offset
	// address from fromGc
	px := uintptr(unsafe.Pointer(p))
	gx := uintptr(unsafe.Pointer(g))
	allocated[px] = allocated[gx]
	delete(allocated, gx)

	// We can't access the internals of the GC Module to manipulate the
	// generation counts, so we have to use a proxy object instead.  We just
	// create a bare minimum object, initialise it, and then store it away to be
	// cleaned up later.
	proxy := C._PyObject_GC_Malloc(C.size_t(baseSize))
	if proxy == nil {
		delete(allocated, px)
		return nil
	}
	C._PyObject_INIT(proxy, c(BaseType))
	gcProxies[px] = proxy

	return p
}

// _goMalloc must be called with memLock already locked.
func _goMalloc(size uintptr) *C.PyObject {
	// We have to use []unsafe.Pointer instead of []byte, otherwise the Go
	// runtime will mark the memory as not containing pointers, and won't use it
	// to pin other Go allocations in memory - in which case we might as well
	// just stick with the Python allocator.
	n := (size + upSize - 1) / upSize
	s := make([]unsafe.Pointer, n)
	p := unsafe.Pointer(&s[0])

	// We need to keep a reference to the allocation, so that the Go runtime
	// doesn't free the memory before we are finished with it.
	allocated[uintptr(p)] = s

	return (*C.PyObject)(p)
}

func goMalloc(size uintptr) *C.PyObject {
	// first, lock memLock, and arrange for it to be unlocked on return
	memLock.Lock()
	defer memLock.Unlock()

	// now call _goMalloc to do the actual work
	return _goMalloc(size)
}

//export goGenericAlloc
func goGenericAlloc(t unsafe.Pointer, n C.Py_ssize_t) unsafe.Pointer {
	var obj *C.PyObject

	typ := newType((*C.PyObject)(t))
	size := uintptr(C.__PyObject_VAR_SIZE(c(typ), n))

	if typ.IsGc() {
		obj = goGcMalloc(size)
	} else {
		obj = goMalloc(size)
	}

	if obj == nil {
		raise(MemoryError.ErrV(None))
		return nil
	}

	if typ.o.tp_flags & C.Py_TPFLAGS_HEAPTYPE != 0 {
		typ.Incref()
	}

	if typ.o.tp_itemsize == 0 {
		C._PyObject_INIT(obj, c(typ))
	} else {
		C._PyObject_INIT_VAR(obj, c(typ), n)
	}

	if typ.IsGc() {
		C.__PyObject_GC_TRACK(obj)
	}

	return unsafe.Pointer(obj)
}

//export goGenericFree
func goGenericFree(o unsafe.Pointer) {
	// first, lock memLock, and arrange for it to be unlocked on return
	memLock.Lock()
	defer memLock.Unlock()

	// Remove the entry from allocated, to let the Go runtime reclaim the memory
	// in the next GC run.
	delete(allocated, uintptr(o))

	// We need to also delete the proxy, so that the Python GC counts get
	// updated as appropriate.
	proxy, ok := gcProxies[uintptr(o)]
	if ok {
		C.PyObject_GC_Del(unsafe.Pointer(proxy))
		delete(gcProxies, uintptr(o))
	}
}
<|endoftext|>


<commit_before>package scoredb

import (
	"fmt"
	"sync"

	"github.com/asciimoo/privacyscore/penalty"
)

type ScoreCount struct {
	Label     string
	Count     uint
	BaseScore penalty.Score
}

const (
	step     = 10
	minScore = 0
	maxScore = 100
)

var scores = struct {
	sync.RWMutex
	DB         []*ScoreCount
	EntryCount uint
}{DB: make([]*ScoreCount, 0, maxScore-minScore), EntryCount: 0}

func init() {
	i := minScore
	scores.DB = append(scores.DB, &ScoreCount{"< 0", 0, 0})
	for i <= maxScore {
		var label string
		switch i {
		case maxScore:
			label = fmt.Sprintf("%v", i)
		default:
			label = fmt.Sprintf("%v-%v", i, i+step-1)
		}
		scores.DB = append(scores.DB, &ScoreCount{label, 0, penalty.Score(i)})
		i += step
	}
}

func Add(s penalty.Score) {
	s = s - (s % step)
	var idx int
	if s < minScore {
		idx = 0
	} else {
		idx = int((int(s)-minScore)/step) + 1
	}
	if idx >= len(scores.DB) {
		idx = len(scores.DB) - 1
	}
	scores.Lock()
	scores.EntryCount += 1
	scores.DB[idx].Count += 1
	scores.Unlock()
}

func GetAll() []*ScoreCount {
	return scores.DB
}

func GetTopEntryCount() uint {
	var c uint = 0
	for _, e := range scores.DB {
		if e.Count > c {
			c = e.Count
		}
	}
	return c
}
<commit_msg>[fix] scores below 0<commit_after>package scoredb

import (
	"fmt"
	"sync"

	"github.com/asciimoo/privacyscore/penalty"
)

type ScoreCount struct {
	Label     string
	Count     uint
	BaseScore penalty.Score
}

const (
	step     = 10
	minScore = 0
	maxScore = 100
)

var scores = struct {
	sync.RWMutex
	DB         []*ScoreCount
	EntryCount uint
}{DB: make([]*ScoreCount, 0, maxScore-minScore), EntryCount: 0}

func init() {
	i := minScore
	scores.DB = append(scores.DB, &ScoreCount{"< 0", 0, 0})
	for i <= maxScore {
		var label string
		switch i {
		case maxScore:
			label = fmt.Sprintf("%v", i)
		default:
			label = fmt.Sprintf("%v-%v", i, i+step-1)
		}
		scores.DB = append(scores.DB, &ScoreCount{label, 0, penalty.Score(i)})
		i += step
	}
}

func Add(s penalty.Score) {
	var idx int
	if s < minScore {
		idx = 0
	} else {
		s = s - (s % step)
		idx = int((int(s)-minScore)/step) + 1
	}
	if idx >= len(scores.DB) {
		idx = len(scores.DB) - 1
	}
	scores.Lock()
	scores.EntryCount += 1
	scores.DB[idx].Count += 1
	scores.Unlock()
}

func GetAll() []*ScoreCount {
	return scores.DB
}

func GetTopEntryCount() uint {
	var c uint = 0
	for _, e := range scores.DB {
		if e.Count > c {
			c = e.Count
		}
	}
	return c
}
<|endoftext|>


<commit_before>package sieve

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"testing"
)

var config struct {
	Events map[string]map[string]EventParserSpec `json:"events"`
}

func TestSampleLogs(t *testing.T) {
	parser := NewStackatoParser(config.Events)
	for process_pat, event_types := range parser.tree {
		for event_type, event_parser := range event_types {
			process := process_pat
			if process_pat == "*" {
				process = "foo"
			}
			event, err := parser.Parse(process, event_parser.Sample)
			if err != nil {
				t.Fatalf("parse error (%s) for %s", err, event_parser.Sample)
			}
			if event == nil {
				t.Fatalf("no event detected for: %s", event_parser.Sample)
			}
			if event.Process != process {
				t.Fatalf("misdetection process %s != %s -- for: %s", event.Process, process, event_parser.Sample)
			}
			if event.Type != event_type {
				t.Fatalf("misdetection type %s != %s -- for: %s", event.Type, event_type, event_parser.Sample)
			}
			// TODO: we should test the Desc field as well. meanwhile,
			// displaying the Desc field to the user.
			fmt.Printf("<< %19s | %16s >> -- [%8s] %s\n", event.Type, event.Process, event.Severity, event.Desc)
		}
	}
}

func init() {
	// XXX: I'm uncertain as to the simplest way to load
	// cloud_events.yml as JSON and json.Unmarshal into config.Events
	// .. all in Go. So, for now, I'll use ruby to do the JSON
	// converstion.
	fmt.Println("Loading etc/cloud_events.yml into test config struct")
	if output, err := exec.Command(
		"/usr/bin/ruby", "-ryaml", "-rjson", "-e",
		"puts YAML.load_file('../../etc/cloud_events.yml').to_json",
	).CombinedOutput(); err != nil {
		panic(fmt.Sprintf("Failed to run ruby: %v (output: %s)", err, output))
	} else {
		if err := json.Unmarshal(output, &config); err != nil {
			panic(err)
		}
	}

}
<commit_msg>fix reference to yml in test<commit_after>package sieve

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"testing"
)

var config struct {
	Events map[string]map[string]EventParserSpec `json:"events"`
}

func TestSampleLogs(t *testing.T) {
	parser := NewStackatoParser(config.Events)
	for process_pat, event_types := range parser.tree {
		for event_type, event_parser := range event_types {
			process := process_pat
			if process_pat == "*" {
				process = "foo"
			}
			event, err := parser.Parse(process, event_parser.Sample)
			if err != nil {
				t.Fatalf("parse error (%s) for %s", err, event_parser.Sample)
			}
			if event == nil {
				t.Fatalf("no event detected for: %s", event_parser.Sample)
			}
			if event.Process != process {
				t.Fatalf("misdetection process %s != %s -- for: %s", event.Process, process, event_parser.Sample)
			}
			if event.Type != event_type {
				t.Fatalf("misdetection type %s != %s -- for: %s", event.Type, event_type, event_parser.Sample)
			}
			// TODO: we should test the Desc field as well. meanwhile,
			// displaying the Desc field to the user.
			fmt.Printf("<< %19s | %16s >> -- [%8s] %s\n", event.Type, event.Process, event.Severity, event.Desc)
		}
	}
}

func init() {
	// XXX: I'm uncertain as to the simplest way to load
	// sieve.yml as JSON and json.Unmarshal into config.Events
	// .. all in Go. So, for now, I'll use ruby to do the JSON
	// converstion.
	fmt.Println("Loading etc/sieve.yml into test config struct")
	if output, err := exec.Command(
		"/usr/bin/ruby", "-ryaml", "-rjson", "-e",
		"puts YAML.load_file('../../etc/sieve.yml').to_json",
	).CombinedOutput(); err != nil {
		panic(fmt.Sprintf("Failed to run ruby: %v (output: %s)", err, output))
	} else {
		if err := json.Unmarshal(output, &config); err != nil {
			panic(err)
		}
	}

}
<|endoftext|>


<commit_before>package gorelic

import (
	"fmt"
	"github.com/yvasiyarov/newrelic_platform_go"
	"log"
)

const (
	// Send data to newrelic every 60 seconds
	NEWRELIC_POLL_INTERVAL = 60
	// Get garbage collector run statistic every 10 seconds
	// During GC stat pooling - mheap will be locked, so be carefull changing this value
	GC_POLL_INTERVAL_IN_SECONDS = 10
	// Get memory allocator statistic every 60 seconds
	// During this process stoptheword() is called, so be carefull changing this value
	MEMORY_ALLOCATOR_POLL_INTERVAL_IN_SECONDS = 60

	AGENT_GUID    = "com.github.yvasiyarov.GoRelic"
	AGENT_VERSION = "0.0.2"
	AGENT_NAME    = "Go daemon"
)

type Agent struct {
	NewrelicName                string
	NewrelicLicense             string
	NewrelicPollInterval        int
	Verbose                     bool
	CollectGcStat               bool
	CollectMemoryStat           bool
	GCPollInterval              int
	MemoryAllocatorPollInterval int
	AgentGUID                   string
	AgentVersion                string
	plugin                      *newrelic_platform_go.NewrelicPlugin
}

func NewAgent() *Agent {
	agent := &Agent{
		NewrelicName:                AGENT_NAME,
		NewrelicPollInterval:        NEWRELIC_POLL_INTERVAL,
		Verbose:                     false,
		CollectGcStat:               true,
		CollectMemoryStat:           true,
		GCPollInterval:              GC_POLL_INTERVAL_IN_SECONDS,
		MemoryAllocatorPollInterval: MEMORY_ALLOCATOR_POLL_INTERVAL_IN_SECONDS,
		AgentGUID:                   AGENT_GUID,
		AgentVersion:                AGENT_VERSION,
	}
	return agent
}

func (agent *Agent) Run() {
	if agent.NewrelicLicense == "" {
		log.Print("Please, pass a valid newrelic license key.")
		return
	}

	agent.plugin = newrelic_platform_go.NewNewrelicPlugin(agent.AgentVersion, agent.NewrelicLicense, agent.NewrelicPollInterval)
	component := newrelic_platform_go.NewPluginComponent(agent.NewrelicName, agent.AgentGUID)
	agent.plugin.AddComponent(component)

	addRuntimeMericsToComponent(component)

	if agent.CollectGcStat {
		addGCMericsToComponent(component, agent.GCPollInterval)
		agent.Debug(fmt.Sprintf("Init GC metrics collection. Poll interval %d seconds.", agent.GCPollInterval))
	}
	if agent.CollectMemoryStat {
		addMemoryMericsToComponent(component, agent.MemoryAllocatorPollInterval)
		agent.Debug(fmt.Sprintf("Init memory allocator metrics collection. Poll interval %d seconds.", agent.MemoryAllocatorPollInterval))
	}

	agent.plugin.Verbose = agent.Verbose
	go agent.plugin.Run()
}

func (agent *Agent) Debug(msg string) {
	if agent.Verbose {
		log.Println(msg)
	}
}
<commit_msg>return error from (*Agent).Run()<commit_after>package gorelic

import (
	"fmt"
	"github.com/yvasiyarov/newrelic_platform_go"
	"log"
	"errors"
)

const (
	// Send data to newrelic every 60 seconds
	NEWRELIC_POLL_INTERVAL = 60
	// Get garbage collector run statistic every 10 seconds
	// During GC stat pooling - mheap will be locked, so be carefull changing this value
	GC_POLL_INTERVAL_IN_SECONDS = 10
	// Get memory allocator statistic every 60 seconds
	// During this process stoptheword() is called, so be carefull changing this value
	MEMORY_ALLOCATOR_POLL_INTERVAL_IN_SECONDS = 60

	AGENT_GUID    = "com.github.yvasiyarov.GoRelic"
	AGENT_VERSION = "0.0.2"
	AGENT_NAME    = "Go daemon"
)

type Agent struct {
	NewrelicName                string
	NewrelicLicense             string
	NewrelicPollInterval        int
	Verbose                     bool
	CollectGcStat               bool
	CollectMemoryStat           bool
	GCPollInterval              int
	MemoryAllocatorPollInterval int
	AgentGUID                   string
	AgentVersion                string
	plugin                      *newrelic_platform_go.NewrelicPlugin
}

func NewAgent() *Agent {
	agent := &Agent{
		NewrelicName:                AGENT_NAME,
		NewrelicPollInterval:        NEWRELIC_POLL_INTERVAL,
		Verbose:                     false,
		CollectGcStat:               true,
		CollectMemoryStat:           true,
		GCPollInterval:              GC_POLL_INTERVAL_IN_SECONDS,
		MemoryAllocatorPollInterval: MEMORY_ALLOCATOR_POLL_INTERVAL_IN_SECONDS,
		AgentGUID:                   AGENT_GUID,
		AgentVersion:                AGENT_VERSION,
	}
	return agent
}

func (agent *Agent) Run() error {
	if agent.NewrelicLicense == "" {
		return errors.New("Please, pass a valid newrelic license key.")
	}

	agent.plugin = newrelic_platform_go.NewNewrelicPlugin(agent.AgentVersion, agent.NewrelicLicense, agent.NewrelicPollInterval)
	component := newrelic_platform_go.NewPluginComponent(agent.NewrelicName, agent.AgentGUID)
	agent.plugin.AddComponent(component)

	addRuntimeMericsToComponent(component)

	if agent.CollectGcStat {
		addGCMericsToComponent(component, agent.GCPollInterval)
		agent.Debug(fmt.Sprintf("Init GC metrics collection. Poll interval %d seconds.", agent.GCPollInterval))
	}
	if agent.CollectMemoryStat {
		addMemoryMericsToComponent(component, agent.MemoryAllocatorPollInterval)
		agent.Debug(fmt.Sprintf("Init memory allocator metrics collection. Poll interval %d seconds.", agent.MemoryAllocatorPollInterval))
	}

	agent.plugin.Verbose = agent.Verbose
	go agent.plugin.Run()
	return nil
}

func (agent *Agent) Debug(msg string) {
	if agent.Verbose {
		log.Println(msg)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/scanner"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"
)

var (
	// main operation modes
	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')")
	simplifyAST = flag.Bool("s", false, "simplify code")
	doDiff      = flag.Bool("d", false, "display diffs instead of rewriting files")

	// layout control
	comments  = flag.Bool("comments", true, "print comments")
	tabWidth  = flag.Int("tabwidth", 8, "tab width")
	tabIndent = flag.Bool("tabindent", true, "indent with tabs independent of -spaces")
	useSpaces = flag.Bool("spaces", true, "align with spaces instead of tabs")

	// debugging
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")
)

var (
	fset        = token.NewFileSet()
	exitCode    = 0
	rewrite     func(*ast.File) *ast.File
	parserMode  uint
	printerMode uint
)

func report(err os.Error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func initParserMode() {
	parserMode = uint(0)
	if *comments {
		parserMode |= parser.ParseComments
	}
}

func initPrinterMode() {
	printerMode = uint(0)
	if *tabIndent {
		printerMode |= printer.TabIndent
	}
	if *useSpaces {
		printerMode |= printer.UseSpaces
	}
}

func isGoFile(f *os.FileInfo) bool {
	// ignore non-Go files
	return f.IsRegular() && !strings.HasPrefix(f.Name, ".") && strings.HasSuffix(f.Name, ".go")
}

// If in == nil, the source is the contents of the file with the given filename.
func processFile(filename string, in io.Reader, out io.Writer) os.Error {
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	file, err := parser.ParseFile(fset, filename, src, parserMode)
	if err != nil {
		return err
	}

	if rewrite != nil {
		file = rewrite(file)
	}

	if *simplifyAST {
		simplify(file)
	}

	var buf bytes.Buffer
	_, err = (&printer.Config{printerMode, *tabWidth}).Fprint(&buf, fset, file)
	if err != nil {
		return err
	}
	res := buf.Bytes()

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			err = ioutil.WriteFile(filename, res, 0)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff %s gofmt/%s\n", filename, filename)
			out.Write(data)
		}
	}

	if !*list && !*write && !*doDiff {
		_, err = out.Write(res)
	}

	return err
}

type fileVisitor chan os.Error

func (v fileVisitor) VisitDir(path string, f *os.FileInfo) bool {
	return true
}

func (v fileVisitor) VisitFile(path string, f *os.FileInfo) {
	if isGoFile(f) {
		v <- nil // synchronize error handler
		if err := processFile(path, nil, os.Stdout); err != nil {
			v <- err
		}
	}
}

func walkDir(path string) {
	v := make(fileVisitor)
	go func() {
		filepath.Walk(path, v, v)
		close(v)
	}()
	for err := range v {
		if err != nil {
			report(err)
		}
	}
}

func main() {
	// call gofmtMain in a separate function
	// so that it can use defer and have them
	// run before the exit.
	gofmtMain()
	os.Exit(exitCode)
}

func gofmtMain() {
	flag.Usage = usage
	flag.Parse()
	if *tabWidth < 0 {
		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", *tabWidth)
		exitCode = 2
		return
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "creating cpu profile: %s\n", err)
			exitCode = 2
			return
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	initParserMode()
	initPrinterMode()
	initRewrite()

	if flag.NArg() == 0 {
		if err := processFile("<standard input>", os.Stdin, os.Stdout); err != nil {
			report(err)
		}
		return
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsRegular():
			if err := processFile(path, nil, os.Stdout); err != nil {
				report(err)
			}
		case dir.IsDirectory():
			walkDir(path)
		}
	}
}

func diff(b1, b2 []byte) (data []byte, err os.Error) {
	f1, err := ioutil.TempFile("", "gofmt")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "gofmt")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	return exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
}
<commit_msg>gofmt: fix -d regression from exec change<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"exec"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/scanner"
	"go/token"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"
)

var (
	// main operation modes
	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')")
	simplifyAST = flag.Bool("s", false, "simplify code")
	doDiff      = flag.Bool("d", false, "display diffs instead of rewriting files")

	// layout control
	comments  = flag.Bool("comments", true, "print comments")
	tabWidth  = flag.Int("tabwidth", 8, "tab width")
	tabIndent = flag.Bool("tabindent", true, "indent with tabs independent of -spaces")
	useSpaces = flag.Bool("spaces", true, "align with spaces instead of tabs")

	// debugging
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to this file")
)

var (
	fset        = token.NewFileSet()
	exitCode    = 0
	rewrite     func(*ast.File) *ast.File
	parserMode  uint
	printerMode uint
)

func report(err os.Error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func initParserMode() {
	parserMode = uint(0)
	if *comments {
		parserMode |= parser.ParseComments
	}
}

func initPrinterMode() {
	printerMode = uint(0)
	if *tabIndent {
		printerMode |= printer.TabIndent
	}
	if *useSpaces {
		printerMode |= printer.UseSpaces
	}
}

func isGoFile(f *os.FileInfo) bool {
	// ignore non-Go files
	return f.IsRegular() && !strings.HasPrefix(f.Name, ".") && strings.HasSuffix(f.Name, ".go")
}

// If in == nil, the source is the contents of the file with the given filename.
func processFile(filename string, in io.Reader, out io.Writer) os.Error {
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		in = f
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	file, err := parser.ParseFile(fset, filename, src, parserMode)
	if err != nil {
		return err
	}

	if rewrite != nil {
		file = rewrite(file)
	}

	if *simplifyAST {
		simplify(file)
	}

	var buf bytes.Buffer
	_, err = (&printer.Config{printerMode, *tabWidth}).Fprint(&buf, fset, file)
	if err != nil {
		return err
	}
	res := buf.Bytes()

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(out, filename)
		}
		if *write {
			err = ioutil.WriteFile(filename, res, 0)
			if err != nil {
				return err
			}
		}
		if *doDiff {
			data, err := diff(src, res)
			if err != nil {
				return fmt.Errorf("computing diff: %s", err)
			}
			fmt.Printf("diff %s gofmt/%s\n", filename, filename)
			out.Write(data)
		}
	}

	if !*list && !*write && !*doDiff {
		_, err = out.Write(res)
	}

	return err
}

type fileVisitor chan os.Error

func (v fileVisitor) VisitDir(path string, f *os.FileInfo) bool {
	return true
}

func (v fileVisitor) VisitFile(path string, f *os.FileInfo) {
	if isGoFile(f) {
		v <- nil // synchronize error handler
		if err := processFile(path, nil, os.Stdout); err != nil {
			v <- err
		}
	}
}

func walkDir(path string) {
	v := make(fileVisitor)
	go func() {
		filepath.Walk(path, v, v)
		close(v)
	}()
	for err := range v {
		if err != nil {
			report(err)
		}
	}
}

func main() {
	// call gofmtMain in a separate function
	// so that it can use defer and have them
	// run before the exit.
	gofmtMain()
	os.Exit(exitCode)
}

func gofmtMain() {
	flag.Usage = usage
	flag.Parse()
	if *tabWidth < 0 {
		fmt.Fprintf(os.Stderr, "negative tabwidth %d\n", *tabWidth)
		exitCode = 2
		return
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			fmt.Fprintf(os.Stderr, "creating cpu profile: %s\n", err)
			exitCode = 2
			return
		}
		defer f.Close()
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	initParserMode()
	initPrinterMode()
	initRewrite()

	if flag.NArg() == 0 {
		if err := processFile("<standard input>", os.Stdin, os.Stdout); err != nil {
			report(err)
		}
		return
	}

	for i := 0; i < flag.NArg(); i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsRegular():
			if err := processFile(path, nil, os.Stdout); err != nil {
				report(err)
			}
		case dir.IsDirectory():
			walkDir(path)
		}
	}
}

func diff(b1, b2 []byte) (data []byte, err os.Error) {
	f1, err := ioutil.TempFile("", "gofmt")
	if err != nil {
		return
	}
	defer os.Remove(f1.Name())
	defer f1.Close()

	f2, err := ioutil.TempFile("", "gofmt")
	if err != nil {
		return
	}
	defer os.Remove(f2.Name())
	defer f2.Close()

	f1.Write(b1)
	f2.Write(b2)

	data, err = exec.Command("diff", "-u", f1.Name(), f2.Name()).CombinedOutput()
	if len(data) > 0 {
		// diff exits with a non-zero status when the files don't match.
		// Ignore that failure as long as we get output.
		err = nil
	}
	return

}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package driverhub provides a handler for proxying connections to a Selenium server.
package driverhub

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/bazelbuild/rules_webtesting/go/httphelper"
	"github.com/bazelbuild/rules_webtesting/go/launcher/diagnostics"
	"github.com/bazelbuild/rules_webtesting/go/launcher/environment"
	"github.com/bazelbuild/rules_webtesting/go/launcher/errors"
	"github.com/bazelbuild/rules_webtesting/go/launcher/healthreporter"
	"github.com/bazelbuild/rules_webtesting/go/launcher/proxy"
	"github.com/bazelbuild/rules_webtesting/go/launcher/proxy/driverhub/debugger"
	"github.com/bazelbuild/rules_webtesting/go/launcher/webdriver"
	"github.com/bazelbuild/rules_webtesting/go/metadata"
	"github.com/bazelbuild/rules_webtesting/go/metadata/capabilities"
	"github.com/gorilla/mux"
)

const envTimeout = 5 * time.Minute // some environments such as Android take a long time to start up.

// WebDriverHub routes message to the various WebDriver sessions.
type WebDriverHub struct {
	*mux.Router
	environment.Env
	*metadata.Metadata
	*http.Client
	diagnostics.Diagnostics
	Proxy    *proxy.Proxy
	debugger *debugger.Debugger

	healthyOnce sync.Once

	mu               sync.RWMutex
	sessions         map[string]*WebDriverSession
	reusableSessions []*WebDriverSession
	nextID           int
}

// NewHandler creates a handler for /wd/hub paths that delegates to a WebDriver server instance provided by env.
func HTTPHandlerProvider(p *proxy.Proxy) (proxy.HTTPHandler, error) {
	var d *debugger.Debugger
	if p.Metadata.DebuggerPort != 0 {
		d = debugger.New(p.Metadata.DebuggerPort)
	}
	h := &WebDriverHub{
		Router:      mux.NewRouter(),
		Env:         p.Env,
		sessions:    map[string]*WebDriverSession{},
		Client:      &http.Client{},
		Diagnostics: p.Diagnostics,
		Metadata:    p.Metadata,
		Proxy:       p,
		debugger:    d,
	}

	h.Path("/wd/hub/session").Methods("POST").HandlerFunc(h.createSession)
	h.Path("/wd/hub/session").HandlerFunc(unknownMethod)
	h.PathPrefix("/wd/hub/session/{sessionID}").HandlerFunc(h.routeToSession)
	h.PathPrefix("/wd/hub/{command}").HandlerFunc(h.defaultForward)
	h.PathPrefix("/").HandlerFunc(unknownCommand)

	return h, nil
}

func (h *WebDriverHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h.debugger != nil {
		// allow debugger to pause for breakpoint, log to debugger front-end.
		h.debugger.Request(r)
	}
	// TODO(DrMarcII) add support for breakpointing on responses.
	h.Router.ServeHTTP(w, r)
}

// Name is the name of the component used in error messages.
func (h *WebDriverHub) Name() string {
	return "WebDriver Hub"
}

// Healthy returns nil if the WebDriverHub is ready for use, and an error otherwise.
func (h *WebDriverHub) Healthy(ctx context.Context) error {
	if h.debugger != nil {
		return h.debugger.Healthy(ctx)
	}
	return nil
}

// AddSession adds a session to WebDriverHub.
func (h *WebDriverHub) AddSession(id string, session *WebDriverSession) {
	h.mu.Lock()
	defer h.mu.Unlock()
	if h.sessions == nil {
		h.sessions = map[string]*WebDriverSession{}
	}
	h.sessions[id] = session
}

// RemoveSession removes a session from WebDriverHub.
func (h *WebDriverHub) RemoveSession(id string) {
	h.mu.Lock()
	defer h.mu.Unlock()
	if h.sessions == nil {
		return
	}
	delete(h.sessions, id)
}

// GetSession gets the session for a given WebDriver session id..
func (h *WebDriverHub) GetSession(id string) *WebDriverSession {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.sessions[id]
}

// NextID gets the next available internal id for a session.
func (h *WebDriverHub) NextID() int {
	h.mu.Lock()
	defer h.mu.Unlock()
	id := h.nextID
	h.nextID++
	return id
}

// GetActiveSessions returns the ids for all currently active sessions.
func (h *WebDriverHub) GetActiveSessions() []string {
	result := []string{}
	h.mu.RLock()
	defer h.mu.RUnlock()
	for id := range h.sessions {
		result = append(result, id)
	}
	return result
}

// Shutdown  shuts down any running sessions.
func (h *WebDriverHub) Shutdown(ctx context.Context) error {
	for _, id := range h.GetActiveSessions() {
		session := h.GetSession(id)
		session.quit(ctx, false)
	}
	for _, session := range h.reusableSessions {
		session.quit(ctx, false)
	}
	return nil
}

// GetReusableSession grabs a reusable session if one is available that matches caps.
func (h *WebDriverHub) GetReusableSession(ctx context.Context, caps capabilities.Spec) (*WebDriverSession, bool) {
	if !capabilities.CanReuseSession(caps) {
		return nil, false
	}

	h.mu.Lock()
	defer h.mu.Unlock()
	for i, session := range h.reusableSessions {
		if capabilities.SpecEquals(caps, session.RequestedCaps) {
			h.reusableSessions = append(h.reusableSessions[:i], h.reusableSessions[i+1:]...)
			if err := session.WebDriver.Healthy(ctx); err == nil {
				return session, true
			}
			return session, true
		}
	}
	return nil, false
}

// AddReusableSession adds a session that can be reused.
func (h *WebDriverHub) AddReusableSession(session *WebDriverSession) error {
	if !capabilities.CanReuseSession(session.RequestedCaps) {
		return errors.New(h.Name(), "session is not reusable.")
	}
	h.reusableSessions = append(h.reusableSessions, session)
	return nil
}

func (h *WebDriverHub) routeToSession(w http.ResponseWriter, r *http.Request) {
	sid := mux.Vars(r)["sessionID"]
	session := h.GetSession(sid)

	if session == nil {
		invalidSessionID(w, sid)
		return
	}
	session.ServeHTTP(w, r)
}

func (h *WebDriverHub) createSession(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	log.Print("Creating session\n\n")

	if err := h.waitForHealthyEnv(ctx); err != nil {
		sessionNotCreated(w, err)
		return
	}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		sessionNotCreated(w, err)
		return
	}

	j := struct {
		// OSS capabilities
		Desired map[string]interface{} `json:"desiredCapabilities"`
		// W3C capabilities
		Capabilities struct {
			Always map[string]interface{}   `json:"alwaysMatch"`
			First  []map[string]interface{} `json:"firstMatch"`
		} `json:"capabilities"`
	}{}

	if err := json.Unmarshal(body, &j); err != nil {
		sessionNotCreated(w, err)
		return
	}

	if j.Desired == nil && j.Capabilities.Always == nil {
		sessionNotCreated(w, errors.New(h.Name(), "new session request body missing capabilities"))
		return
	}

	requestedCaps := capabilities.Spec{
		OSSCaps: j.Desired,
		Always:  j.Capabilities.Always,
		First:   j.Capabilities.First,
	}

	id := h.NextID()

	caps, err := h.Env.StartSession(ctx, id, requestedCaps)
	if err != nil {
		sessionNotCreated(w, err)
		return
	}

	log.Printf("Caps: %+v", caps)

	var session *WebDriverSession

	if reusable, ok := h.GetReusableSession(ctx, caps); ok {
		reusable.Unpause(id)
		session = reusable
	} else {
		// TODO(DrMarcII) parameterize attempts based on browser metadata
		driver, err := webdriver.CreateSession(ctx, h.Env.WDAddress(ctx), 3, caps)
		if err != nil {
			if err2 := h.Env.StopSession(ctx, id); err2 != nil {
				log.Printf("error stopping session after failing to launch webdriver: %v", err2)
			}
			sessionNotCreated(w, err)
			return
		}

		s, err := CreateSession(id, h, driver, caps)
		if err != nil {
			sessionNotCreated(w, err)
			return
		}
		session = s
	}

	h.AddSession(session.WebDriver.SessionID(), session)

	var respJSON map[string]interface{}

	if session.WebDriver.W3C() {
		respJSON = map[string]interface{}{
			"value": map[string]interface{}{
				"capabilities": session.WebDriver.Capabilities(),
				"sessionId":    session.WebDriver.SessionID(),
			},
		}
	} else {
		respJSON = map[string]interface{}{
			"value":     session.WebDriver.Capabilities(),
			"sessionId": session.WebDriver.SessionID(),
			"status":    0,
		}
	}

	bytes, err := json.Marshal(respJSON)
	if err != nil {
		unknownError(w, err)
		return
	}

	w.Header().Set("Content-Type", contentType)
	w.Write(bytes)
}

func (h *WebDriverHub) defaultForward(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	if err := h.waitForHealthyEnv(ctx); err != nil {
		unknownError(w, err)
		return
	}

	if err := httphelper.Forward(r.Context(), h.Env.WDAddress(ctx), "/wd/hub/", w, r); err != nil {
		unknownError(w, err)
	}
}

func (h *WebDriverHub) waitForHealthyEnv(ctx context.Context) error {
	h.healthyOnce.Do(func() {
		healthyCtx, cancel := context.WithTimeout(ctx, envTimeout)
		defer cancel()
		// ignore error here as we will call and return Healthy below.
		healthreporter.WaitForHealthy(healthyCtx, h.Env)
	})
	return h.Env.Healthy(ctx)
}
<commit_msg>Change debugger field to Debugger. (#163)<commit_after>// Copyright 2016 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package driverhub provides a handler for proxying connections to a Selenium server.
package driverhub

import (
	"context"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/bazelbuild/rules_webtesting/go/httphelper"
	"github.com/bazelbuild/rules_webtesting/go/launcher/diagnostics"
	"github.com/bazelbuild/rules_webtesting/go/launcher/environment"
	"github.com/bazelbuild/rules_webtesting/go/launcher/errors"
	"github.com/bazelbuild/rules_webtesting/go/launcher/healthreporter"
	"github.com/bazelbuild/rules_webtesting/go/launcher/proxy"
	"github.com/bazelbuild/rules_webtesting/go/launcher/proxy/driverhub/debugger"
	"github.com/bazelbuild/rules_webtesting/go/launcher/webdriver"
	"github.com/bazelbuild/rules_webtesting/go/metadata"
	"github.com/bazelbuild/rules_webtesting/go/metadata/capabilities"
	"github.com/gorilla/mux"
)

const envTimeout = 5 * time.Minute // some environments such as Android take a long time to start up.

// WebDriverHub routes message to the various WebDriver sessions.
type WebDriverHub struct {
	*mux.Router
	environment.Env
	*metadata.Metadata
	*http.Client
	diagnostics.Diagnostics
	Proxy    *proxy.Proxy
	Debugger *debugger.Debugger

	healthyOnce sync.Once

	mu               sync.RWMutex
	sessions         map[string]*WebDriverSession
	reusableSessions []*WebDriverSession
	nextID           int
}

// NewHandler creates a handler for /wd/hub paths that delegates to a WebDriver server instance provided by env.
func HTTPHandlerProvider(p *proxy.Proxy) (proxy.HTTPHandler, error) {
	var d *debugger.Debugger
	if p.Metadata.DebuggerPort != 0 {
		d = debugger.New(p.Metadata.DebuggerPort)
	}
	h := &WebDriverHub{
		Router:      mux.NewRouter(),
		Env:         p.Env,
		sessions:    map[string]*WebDriverSession{},
		Client:      &http.Client{},
		Diagnostics: p.Diagnostics,
		Metadata:    p.Metadata,
		Proxy:       p,
		Debugger:    d,
	}

	h.Path("/wd/hub/session").Methods("POST").HandlerFunc(h.createSession)
	h.Path("/wd/hub/session").HandlerFunc(unknownMethod)
	h.PathPrefix("/wd/hub/session/{sessionID}").HandlerFunc(h.routeToSession)
	h.PathPrefix("/wd/hub/{command}").HandlerFunc(h.defaultForward)
	h.PathPrefix("/").HandlerFunc(unknownCommand)

	return h, nil
}

func (h *WebDriverHub) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h.Debugger != nil {
		// allow debugger to pause for breakpoint, log to debugger front-end.
		h.Debugger.Request(r)
	}
	// TODO(DrMarcII) add support for breakpointing on responses.
	h.Router.ServeHTTP(w, r)
}

// Name is the name of the component used in error messages.
func (h *WebDriverHub) Name() string {
	return "WebDriver Hub"
}

// Healthy returns nil if the WebDriverHub is ready for use, and an error otherwise.
func (h *WebDriverHub) Healthy(ctx context.Context) error {
	if h.Debugger != nil {
		return h.Debugger.Healthy(ctx)
	}
	return nil
}

// AddSession adds a session to WebDriverHub.
func (h *WebDriverHub) AddSession(id string, session *WebDriverSession) {
	h.mu.Lock()
	defer h.mu.Unlock()
	if h.sessions == nil {
		h.sessions = map[string]*WebDriverSession{}
	}
	h.sessions[id] = session
}

// RemoveSession removes a session from WebDriverHub.
func (h *WebDriverHub) RemoveSession(id string) {
	h.mu.Lock()
	defer h.mu.Unlock()
	if h.sessions == nil {
		return
	}
	delete(h.sessions, id)
}

// GetSession gets the session for a given WebDriver session id..
func (h *WebDriverHub) GetSession(id string) *WebDriverSession {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return h.sessions[id]
}

// NextID gets the next available internal id for a session.
func (h *WebDriverHub) NextID() int {
	h.mu.Lock()
	defer h.mu.Unlock()
	id := h.nextID
	h.nextID++
	return id
}

// GetActiveSessions returns the ids for all currently active sessions.
func (h *WebDriverHub) GetActiveSessions() []string {
	result := []string{}
	h.mu.RLock()
	defer h.mu.RUnlock()
	for id := range h.sessions {
		result = append(result, id)
	}
	return result
}

// Shutdown  shuts down any running sessions.
func (h *WebDriverHub) Shutdown(ctx context.Context) error {
	for _, id := range h.GetActiveSessions() {
		session := h.GetSession(id)
		session.quit(ctx, false)
	}
	for _, session := range h.reusableSessions {
		session.quit(ctx, false)
	}
	return nil
}

// GetReusableSession grabs a reusable session if one is available that matches caps.
func (h *WebDriverHub) GetReusableSession(ctx context.Context, caps capabilities.Spec) (*WebDriverSession, bool) {
	if !capabilities.CanReuseSession(caps) {
		return nil, false
	}

	h.mu.Lock()
	defer h.mu.Unlock()
	for i, session := range h.reusableSessions {
		if capabilities.SpecEquals(caps, session.RequestedCaps) {
			h.reusableSessions = append(h.reusableSessions[:i], h.reusableSessions[i+1:]...)
			if err := session.WebDriver.Healthy(ctx); err == nil {
				return session, true
			}
			return session, true
		}
	}
	return nil, false
}

// AddReusableSession adds a session that can be reused.
func (h *WebDriverHub) AddReusableSession(session *WebDriverSession) error {
	if !capabilities.CanReuseSession(session.RequestedCaps) {
		return errors.New(h.Name(), "session is not reusable.")
	}
	h.reusableSessions = append(h.reusableSessions, session)
	return nil
}

func (h *WebDriverHub) routeToSession(w http.ResponseWriter, r *http.Request) {
	sid := mux.Vars(r)["sessionID"]
	session := h.GetSession(sid)

	if session == nil {
		invalidSessionID(w, sid)
		return
	}
	session.ServeHTTP(w, r)
}

func (h *WebDriverHub) createSession(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	log.Print("Creating session\n\n")

	if err := h.waitForHealthyEnv(ctx); err != nil {
		sessionNotCreated(w, err)
		return
	}

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		sessionNotCreated(w, err)
		return
	}

	j := struct {
		// OSS capabilities
		Desired map[string]interface{} `json:"desiredCapabilities"`
		// W3C capabilities
		Capabilities struct {
			Always map[string]interface{}   `json:"alwaysMatch"`
			First  []map[string]interface{} `json:"firstMatch"`
		} `json:"capabilities"`
	}{}

	if err := json.Unmarshal(body, &j); err != nil {
		sessionNotCreated(w, err)
		return
	}

	if j.Desired == nil && j.Capabilities.Always == nil {
		sessionNotCreated(w, errors.New(h.Name(), "new session request body missing capabilities"))
		return
	}

	requestedCaps := capabilities.Spec{
		OSSCaps: j.Desired,
		Always:  j.Capabilities.Always,
		First:   j.Capabilities.First,
	}

	id := h.NextID()

	caps, err := h.Env.StartSession(ctx, id, requestedCaps)
	if err != nil {
		sessionNotCreated(w, err)
		return
	}

	log.Printf("Caps: %+v", caps)

	var session *WebDriverSession

	if reusable, ok := h.GetReusableSession(ctx, caps); ok {
		reusable.Unpause(id)
		session = reusable
	} else {
		// TODO(DrMarcII) parameterize attempts based on browser metadata
		driver, err := webdriver.CreateSession(ctx, h.Env.WDAddress(ctx), 3, caps)
		if err != nil {
			if err2 := h.Env.StopSession(ctx, id); err2 != nil {
				log.Printf("error stopping session after failing to launch webdriver: %v", err2)
			}
			sessionNotCreated(w, err)
			return
		}

		s, err := CreateSession(id, h, driver, caps)
		if err != nil {
			sessionNotCreated(w, err)
			return
		}
		session = s
	}

	h.AddSession(session.WebDriver.SessionID(), session)

	var respJSON map[string]interface{}

	if session.WebDriver.W3C() {
		respJSON = map[string]interface{}{
			"value": map[string]interface{}{
				"capabilities": session.WebDriver.Capabilities(),
				"sessionId":    session.WebDriver.SessionID(),
			},
		}
	} else {
		respJSON = map[string]interface{}{
			"value":     session.WebDriver.Capabilities(),
			"sessionId": session.WebDriver.SessionID(),
			"status":    0,
		}
	}

	bytes, err := json.Marshal(respJSON)
	if err != nil {
		unknownError(w, err)
		return
	}

	w.Header().Set("Content-Type", contentType)
	w.Write(bytes)
}

func (h *WebDriverHub) defaultForward(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	if err := h.waitForHealthyEnv(ctx); err != nil {
		unknownError(w, err)
		return
	}

	if err := httphelper.Forward(r.Context(), h.Env.WDAddress(ctx), "/wd/hub/", w, r); err != nil {
		unknownError(w, err)
	}
}

func (h *WebDriverHub) waitForHealthyEnv(ctx context.Context) error {
	h.healthyOnce.Do(func() {
		healthyCtx, cancel := context.WithTimeout(ctx, envTimeout)
		defer cancel()
		// ignore error here as we will call and return Healthy below.
		healthreporter.WaitForHealthy(healthyCtx, h.Env)
	})
	return h.Env.Healthy(ctx)
}
<|endoftext|>


<commit_before>package worker

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/koding/logging"
	"github.com/koding/rabbitmq"
	"log"
	"log/syslog"
	"os"

	"github.com/jinzhu/gorm"
	"github.com/rcrowley/go-metrics"
	"github.com/streadway/amqp"
	"labix.org/v2/mgo"
)

type Listener struct {
	Consumer             *rabbitmq.Consumer
	MaintenancePublisher *rabbitmq.Producer
	SourceExchangeName   string
	WorkerName           string
	Log                  logging.Logger
	Metrics              *Metrics
	Debug                bool
}

type Metrics struct {
	Registry metrics.Registry
	*MetricsCounter
}

type MetricsCounter struct {
	Messages, Success, Handler404, Mgo404, Gorm404, OtherError metrics.Counter
}

func NewListener(workerName string, sourceExchangeName string, log logging.Logger, debug bool) *Listener {
	return &Listener{
		WorkerName:         workerName,
		SourceExchangeName: sourceExchangeName,
		Log:                log,
		Metrics:            initializeMetrics(workerName),
		Debug:              debug,
	}
}

func initializeMetrics(name string) *Metrics {
	r := metrics.NewRegistry()

	messages := metrics.NewCounter()
	r.Register(name+"messages", messages)

	success := metrics.NewCounter()
	r.Register(name+"success", success)

	handler404 := metrics.NewCounter()
	r.Register(name+"handler404", success)

	mgo404 := metrics.NewCounter()
	r.Register(name+"mgo404", mgo404)

	gorm404 := metrics.NewCounter()
	r.Register(name+"gorm404", gorm404)

	otherError := metrics.NewCounter()
	r.Register(name+"otherError", otherError)

	return &Metrics{
		r,
		&MetricsCounter{
			Messages:   messages,
			Success:    success,
			Handler404: handler404,
			Mgo404:     mgo404,
			Gorm404:    gorm404,
			OtherError: otherError,
		},
	}
}

func (l *Listener) createConsumer(rmq *rabbitmq.RabbitMQ) *rabbitmq.Consumer {
	exchange := rabbitmq.Exchange{
		Name:    l.SourceExchangeName,
		Type:    "fanout",
		Durable: true,
	}

	queue := rabbitmq.Queue{
		Name:    fmt.Sprintf("%s:WorkerQueue", l.WorkerName),
		Durable: true,
	}

	binding := rabbitmq.BindingOptions{
		RoutingKey: "",
	}

	consumerOptions := rabbitmq.ConsumerOptions{
		Tag: fmt.Sprintf("%sWorkerConsumer", l.WorkerName),
	}

	Consumer, err := rmq.NewConsumer(exchange, queue, binding, consumerOptions)
	if err != nil {
		panic(err)
	}

	return Consumer
}

func (l *Listener) createMaintenancePublisher(rmq *rabbitmq.RabbitMQ) *rabbitmq.Producer {
	exchange := rabbitmq.Exchange{
		Name: "",
	}

	publishingOptions := rabbitmq.PublishingOptions{
		Tag:       fmt.Sprintf("%sWorkerConsumer", l.WorkerName),
		Immediate: false,
		// RoutingKey: "ApiMaintenanceQueue", // publish to ApiMaintenanceQueue
	}

	producer, err := rmq.NewProducer(
		exchange,
		rabbitmq.Queue{
			Name:    "ApiMaintenanceQueue",
			Durable: true,
		},
		publishingOptions,
	)
	if err != nil {
		panic(err)
	}

	return producer
}

func (l *Listener) Listen(rmq *rabbitmq.RabbitMQ, handler Handler) {
	// set consumer
	l.Consumer = l.createConsumer(rmq)
	l.MaintenancePublisher = l.createMaintenancePublisher(rmq)

	if err := l.Consumer.QOS(10); err != nil {
		panic(err)
	}

	l.Consumer.Consume(l.Start(handler))
}

func (l *Listener) Close() {
	l.Consumer.Shutdown()
	l.MaintenancePublisher.Shutdown()
}

var HandlerNotFoundErr = errors.New("Handler Not Found")

type Handler interface {
	HandleEvent(string, []byte) error
	ErrHandler
}

type ErrHandler interface {
	// bool is whether publishing the message to maintenance qeueue or not
	DefaultErrHandler(amqp.Delivery, error) bool
}

var timers = map[string]metrics.Timer{}

func (l *Listener) withMetrics(fn func(string, []byte) error, delivery amqp.Delivery) error {
	var timer metrics.Timer
	var err error
	var exists bool

	l.Metrics.Messages.Inc(1)

	timerName := l.WorkerName + delivery.Type
	timer, exists = timers[timerName]
	if !exists {
		timer = metrics.NewTimer()
		l.Metrics.Registry.Register(timerName, timer)
	}

	timer.Time(func() {
		err = fn(delivery.Type, delivery.Body)
	})

	return err
}

func (l *Listener) Start(handler Handler) func(delivery amqp.Delivery) {
	l.Log.Info("Worker Started to Consume")

	if l.Debug {
		go metrics.Log(l.Metrics.Registry, 1e10, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}

	w, _ := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics")
	go metrics.Syslog(metrics.DefaultRegistry, 1e10, w)

	return func(delivery amqp.Delivery) {
		err := l.withMetrics(handler.HandleEvent, delivery)

		switch err {
		case nil:
			l.Metrics.Success.Inc(1)
			delivery.Ack(false)
		case HandlerNotFoundErr:
			l.Metrics.Handler404.Inc(1)
			l.Log.Debug("unknown event type (%s) recieved, deleting message from RMQ", delivery.Type)

			delivery.Ack(false)
		case gorm.RecordNotFound:
			l.Metrics.Gorm404.Inc(1)
			l.Log.Warning("Record not found in our db (%s) recieved, deleting message from RMQ", string(delivery.Body))

			delivery.Ack(false)
		case mgo.ErrNotFound:
			l.Metrics.Mgo404.Inc(1)
			l.Log.Warning("Record not found in our mongo db (%s) recieved, deleting message from RMQ", string(delivery.Body))

			delivery.Ack(false)
		default:
			l.Metrics.OtherError.Inc(1)

			if handler.DefaultErrHandler(delivery, err) {
				data, err := json.Marshal(delivery)
				if err == nil {
					msg := amqp.Publishing{
						Body:  []byte(data),
						AppId: l.WorkerName,
					}
					l.MaintenancePublisher.Publish(msg)
				}
			}
		}
	}
}
<commit_msg>Social: remove timers global var<commit_after>package worker

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/koding/logging"
	"github.com/koding/rabbitmq"
	"log"
	"log/syslog"
	"os"

	"github.com/jinzhu/gorm"
	"github.com/rcrowley/go-metrics"
	"github.com/streadway/amqp"
	"labix.org/v2/mgo"
)

type Listener struct {
	Consumer             *rabbitmq.Consumer
	MaintenancePublisher *rabbitmq.Producer
	SourceExchangeName   string
	WorkerName           string
	Log                  logging.Logger
	Metrics              *Metrics
	Debug                bool
}

type Metrics struct {
	Registry metrics.Registry
	Timers   TimersArray
	*MetricsCounter
}

type TimersArray map[string]metrics.Timer

type MetricsCounter struct {
	Messages, Success, Handler404, Mgo404, Gorm404, OtherError metrics.Counter
}

func NewListener(workerName string, sourceExchangeName string, log logging.Logger, debug bool) *Listener {
	return &Listener{
		WorkerName:         workerName,
		SourceExchangeName: sourceExchangeName,
		Log:                log,
		Metrics:            initializeMetrics(workerName),
		Debug:              debug,
	}
}

func initializeMetrics(name string) *Metrics {
	r := metrics.NewRegistry()

	messages := metrics.NewCounter()
	r.Register(name+"messages", messages)

	success := metrics.NewCounter()
	r.Register(name+"success", success)

	handler404 := metrics.NewCounter()
	r.Register(name+"handler404", success)

	mgo404 := metrics.NewCounter()
	r.Register(name+"mgo404", mgo404)

	gorm404 := metrics.NewCounter()
	r.Register(name+"gorm404", gorm404)

	otherError := metrics.NewCounter()
	r.Register(name+"otherError", otherError)

	return &Metrics{
		r,
		TimersArray{},
		&MetricsCounter{
			Messages:   messages,
			Success:    success,
			Handler404: handler404,
			Mgo404:     mgo404,
			Gorm404:    gorm404,
			OtherError: otherError,
		},
	}
}

func (l *Listener) createConsumer(rmq *rabbitmq.RabbitMQ) *rabbitmq.Consumer {
	exchange := rabbitmq.Exchange{
		Name:    l.SourceExchangeName,
		Type:    "fanout",
		Durable: true,
	}

	queue := rabbitmq.Queue{
		Name:    fmt.Sprintf("%s:WorkerQueue", l.WorkerName),
		Durable: true,
	}

	binding := rabbitmq.BindingOptions{
		RoutingKey: "",
	}

	consumerOptions := rabbitmq.ConsumerOptions{
		Tag: fmt.Sprintf("%sWorkerConsumer", l.WorkerName),
	}

	Consumer, err := rmq.NewConsumer(exchange, queue, binding, consumerOptions)
	if err != nil {
		panic(err)
	}

	return Consumer
}

func (l *Listener) createMaintenancePublisher(rmq *rabbitmq.RabbitMQ) *rabbitmq.Producer {
	exchange := rabbitmq.Exchange{
		Name: "",
	}

	publishingOptions := rabbitmq.PublishingOptions{
		Tag:       fmt.Sprintf("%sWorkerConsumer", l.WorkerName),
		Immediate: false,
		// RoutingKey: "ApiMaintenanceQueue", // publish to ApiMaintenanceQueue
	}

	producer, err := rmq.NewProducer(
		exchange,
		rabbitmq.Queue{
			Name:    "ApiMaintenanceQueue",
			Durable: true,
		},
		publishingOptions,
	)
	if err != nil {
		panic(err)
	}

	return producer
}

func (l *Listener) Listen(rmq *rabbitmq.RabbitMQ, handler Handler) {
	// set consumer
	l.Consumer = l.createConsumer(rmq)
	l.MaintenancePublisher = l.createMaintenancePublisher(rmq)

	if err := l.Consumer.QOS(10); err != nil {
		panic(err)
	}

	l.Consumer.Consume(l.Start(handler))
}

func (l *Listener) Close() {
	l.Consumer.Shutdown()
	l.MaintenancePublisher.Shutdown()
}

var HandlerNotFoundErr = errors.New("Handler Not Found")

type Handler interface {
	HandleEvent(string, []byte) error
	ErrHandler
}

type ErrHandler interface {
	// bool is whether publishing the message to maintenance qeueue or not
	DefaultErrHandler(amqp.Delivery, error) bool
}

func (l *Listener) withMetrics(fn func(string, []byte) error, delivery amqp.Delivery) error {
	var timer metrics.Timer
	var err error
	var exists bool

	l.Metrics.Messages.Inc(1)

	timerName := l.WorkerName + delivery.Type
	timer, exists = l.Metrics.Timers[timerName]
	if !exists {
		timer = metrics.NewTimer()
		l.Metrics.Registry.Register(timerName, timer)

		l.Metrics.Timers[timerName] = timer
	}

	timer.Time(func() {
		err = fn(delivery.Type, delivery.Body)
	})

	return err
}

func (l *Listener) Start(handler Handler) func(delivery amqp.Delivery) {
	l.Log.Info("Worker Started to Consume")

	if l.Debug {
		go metrics.Log(l.Metrics.Registry, 1e10, log.New(os.Stderr, "metrics: ", log.Lmicroseconds))
	}

	w, _ := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics")
	go metrics.Syslog(metrics.DefaultRegistry, 1e10, w)

	return func(delivery amqp.Delivery) {
		err := l.withMetrics(handler.HandleEvent, delivery)

		switch err {
		case nil:
			l.Metrics.Success.Inc(1)
			delivery.Ack(false)
		case HandlerNotFoundErr:
			l.Metrics.Handler404.Inc(1)
			l.Log.Debug("unknown event type (%s) recieved, deleting message from RMQ", delivery.Type)

			delivery.Ack(false)
		case gorm.RecordNotFound:
			l.Metrics.Gorm404.Inc(1)
			l.Log.Warning("Record not found in our db (%s) recieved, deleting message from RMQ", string(delivery.Body))

			delivery.Ack(false)
		case mgo.ErrNotFound:
			l.Metrics.Mgo404.Inc(1)
			l.Log.Warning("Record not found in our mongo db (%s) recieved, deleting message from RMQ", string(delivery.Body))

			delivery.Ack(false)
		default:
			l.Metrics.OtherError.Inc(1)

			if handler.DefaultErrHandler(delivery, err) {
				data, err := json.Marshal(delivery)
				if err == nil {
					msg := amqp.Publishing{
						Body:  []byte(data),
						AppId: l.WorkerName,
					}
					l.MaintenancePublisher.Publish(msg)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package moh

import (
	"code.google.com/p/go.net/websocket"
	"log"
	"time"
)

type Subscriber struct {
	ws *websocket.Conn
	h  MessageHandler
}

// NewSubscriber opens a websocket connection to a Publisher and
// returns a pointer to newly created Subscriber.
// After creating a Subscriber you should subscribe to messages with Subscribe function.
func NewSubscriber(addr string, h MessageHandler) (*Subscriber, error) {
	url := "ws://" + addr + "/"
	origin := "http://localhost/" // dont know if this is required
	ws, err := websocket.Dial(url, "", origin)
	if err != nil {
		return nil, err
	}

	sub := &Subscriber{
		ws: ws,
		h:  h,
	}
	go sub.consumer()
	return sub, err
}

// Subscribe registers the Subscriber to receive messages matching with the key.
func (s *Subscriber) Subscribe(key string) error {
	return websocket.Message.Send(s.ws, key)
}

func (s *Subscriber) consumer() {
	for {
		var message []byte
		log.Println("Reading from websocket")
		err := websocket.Message.Receive(s.ws, &message)
		if err != nil {
			log.Println("Cannot read message from websocket")
			time.Sleep(100 * time.Millisecond)
			continue
		}
		log.Println("Received data:", message)
		s.h(message)
	}
}
<commit_msg>use more descriptive name for field: handler<commit_after>package moh

import (
	"code.google.com/p/go.net/websocket"
	"log"
	"time"
)

type Subscriber struct {
	ws      *websocket.Conn
	handler MessageHandler
}

// NewSubscriber opens a websocket connection to a Publisher and
// returns a pointer to newly created Subscriber.
// After creating a Subscriber you should subscribe to messages with Subscribe function.
func NewSubscriber(addr string, handler MessageHandler) (*Subscriber, error) {
	url := "ws://" + addr + "/"
	origin := "http://localhost/" // dont know if this is required
	ws, err := websocket.Dial(url, "", origin)
	if err != nil {
		return nil, err
	}

	sub := &Subscriber{
		ws:      ws,
		handler: handler,
	}
	go sub.consumer()
	return sub, err
}

// Subscribe registers the Subscriber to receive messages matching with the key.
func (s *Subscriber) Subscribe(key string) error {
	return websocket.Message.Send(s.ws, key)
}

func (s *Subscriber) consumer() {
	for {
		var message []byte
		log.Println("Reading from websocket")
		err := websocket.Message.Receive(s.ws, &message)
		if err != nil {
			log.Println("Cannot read message from websocket")
			time.Sleep(100 * time.Millisecond)
			continue
		}
		log.Println("Received data:", message)
		s.handler(message)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"labix.org/v2/mgo/bson"
)

var VMDeletionWarning1 = &Warning{
	ID: "vmDeletionWarning-1",

	Description: "Find users inactive for > 20 days, send email",

	Select: []bson.M{
		bson.M{"lastLoginDate": moreThanDaysQuery(20)},
		bson.M{"inactive.warning": bson.M{"$exists": false}},
	},

	ExemptCheckers: []*ExemptChecker{
		IsUserPaid, IsUserNotConfirmed, IsUserVMsEmpty, IsUserKodingEmployee,
	},

	Action: SendEmail,
}

var VMDeletionWarning2 = &Warning{
	ID: "vmDeletionWarning-2",

	Description: "Find users inactive for > 24 days, send email",

	PreviousWarning: VMDeletionWarning1,

	IntervalSinceLastWarning: time.Hour * 24 * 4, // 4 days since last warning

	Select: []bson.M{
		bson.M{"lastLoginDate": moreThanDaysQuery(24)},
		bson.M{"inactive.warning": VMDeletionWarning1.ID},
	},

	ExemptCheckers: []*ExemptChecker{
		IsTooSoon, IsUserPaid, IsUserNotConfirmed, IsUserVMsEmpty, IsUserKodingEmployee,
	},

	Action: SendEmail,
}

var DeleteInactiveUserVM = &Warning{
	ID: "deleteInactiveUserVm",

	Description: "Find users inactive for > 29 days, deleted ALL their vms",

	PreviousWarning: VMDeletionWarning2,

	IntervalSinceLastWarning: time.Hour * 24 * 4, // 4 days since last warning

	Select: []bson.M{
		bson.M{"lastLoginDate": moreThanDaysQuery(29)},
		bson.M{"inactive.warning": VMDeletionWarning2.ID},
	},

	ExemptCheckers: []*ExemptChecker{
		IsTooSoon, IsUserPaid, IsUserVMsEmpty, IsUserKodingEmployee,
	},

	Action: DeleteVMs,
}

var DeleteBlockedUserVM = &Warning{
	ID: "deleteBlockedUserVm",

	Description: "Find blocked users inactive > 14 days, delete ALL their vms",

	Select: []bson.M{
		bson.M{"lastLoginDate": moreThanDaysQuery(14)},
		bson.M{"inactive.warning": bson.M{"$exists": false}},
		bson.M{"status": "blocked"},
	},

	ExemptCheckers: []*ExemptChecker{IsUserVMsEmpty},

	Action: DeleteVMs,
}
<commit_msg>janitor: remove `IsUserNotConfirmed` check to catch users before we  added check to let unconfirmed users turn on VMs<commit_after>package main

import (
	"time"

	"labix.org/v2/mgo/bson"
)

var VMDeletionWarning1 = &Warning{
	ID: "vmDeletionWarning-1",

	Description: "Find users inactive for > 20 days, send email",

	Select: []bson.M{
		bson.M{"lastLoginDate": moreThanDaysQuery(20)},
		bson.M{"inactive.warning": bson.M{"$exists": false}},
	},

	ExemptCheckers: []*ExemptChecker{
		IsUserPaid, IsUserVMsEmpty, IsUserKodingEmployee,
	},

	Action: SendEmail,
}

var VMDeletionWarning2 = &Warning{
	ID: "vmDeletionWarning-2",

	Description: "Find users inactive for > 24 days, send email",

	PreviousWarning: VMDeletionWarning1,

	IntervalSinceLastWarning: time.Hour * 24 * 4, // 4 days since last warning

	Select: []bson.M{
		bson.M{"lastLoginDate": moreThanDaysQuery(24)},
		bson.M{"inactive.warning": VMDeletionWarning1.ID},
	},

	ExemptCheckers: []*ExemptChecker{
		IsTooSoon, IsUserPaid, IsUserVMsEmpty, IsUserKodingEmployee,
	},

	Action: SendEmail,
}

var DeleteInactiveUserVM = &Warning{
	ID: "deleteInactiveUserVm",

	Description: "Find users inactive for > 29 days, deleted ALL their vms",

	PreviousWarning: VMDeletionWarning2,

	IntervalSinceLastWarning: time.Hour * 24 * 4, // 4 days since last warning

	Select: []bson.M{
		bson.M{"lastLoginDate": moreThanDaysQuery(29)},
		bson.M{"inactive.warning": VMDeletionWarning2.ID},
	},

	ExemptCheckers: []*ExemptChecker{
		IsTooSoon, IsUserPaid, IsUserVMsEmpty, IsUserKodingEmployee,
	},

	Action: DeleteVMs,
}

var DeleteBlockedUserVM = &Warning{
	ID: "deleteBlockedUserVm",

	Description: "Find blocked users inactive > 14 days, delete ALL their vms",

	Select: []bson.M{
		bson.M{"lastLoginDate": moreThanDaysQuery(14)},
		bson.M{"inactive.warning": bson.M{"$exists": false}},
		bson.M{"status": "blocked"},
	},

	ExemptCheckers: []*ExemptChecker{IsUserVMsEmpty},

	Action: DeleteVMs,
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package onlineddl

import (
	"context"
	"fmt"
	"testing"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"

	"vitess.io/vitess/go/test/endtoend/cluster"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// VtgateExecQuery runs a query on VTGate using given query params
func VtgateExecQuery(t *testing.T, vtParams *mysql.ConnParams, query string, expectError string) *sqltypes.Result {
	t.Helper()

	ctx := context.Background()
	conn, err := mysql.Connect(ctx, vtParams)
	require.Nil(t, err)
	defer conn.Close()

	qr, err := conn.ExecuteFetch(query, 1000, true)
	if expectError == "" {
		require.NoError(t, err)
	} else {
		require.Error(t, err, "error should not be nil")
		assert.Contains(t, err.Error(), expectError, "Unexpected error")
	}
	return qr
}

// CheckRetryMigration attempts to retry a migration, and expects success/failure by counting affected rows
func CheckRetryMigration(t *testing.T, vtParams *mysql.ConnParams, shards []cluster.Shard, uuid string, expectRetryPossible bool) {
	retryQuery := fmt.Sprintf("alter vitess_migration '%s' retry", uuid)
	r := VtgateExecQuery(t, vtParams, retryQuery, "")

	if expectRetryPossible {
		assert.Equal(t, len(shards), int(r.RowsAffected))
	} else {
		assert.Equal(t, int(0), int(r.RowsAffected))
	}
}

// CheckCancelMigration attempts to cancel a migration, and expects success/failure by counting affected rows
func CheckCancelMigration(t *testing.T, vtParams *mysql.ConnParams, shards []cluster.Shard, uuid string, expectCancelPossible bool) {
	cancelQuery := fmt.Sprintf("alter vitess_migration '%s' cancel", uuid)
	r := VtgateExecQuery(t, vtParams, cancelQuery, "")

	if expectCancelPossible {
		assert.Equal(t, len(shards), int(r.RowsAffected))
	} else {
		assert.Equal(t, int(0), int(r.RowsAffected))
	}
}

// CheckCancelAllMigrations cancels all pending migrations and expect number of affected rows
func CheckCancelAllMigrations(t *testing.T, vtParams *mysql.ConnParams, expectCount int) {
	cancelQuery := fmt.Sprintf("alter vitess_migration cancel all")
	r := VtgateExecQuery(t, vtParams, cancelQuery, "")

	assert.Equal(t, expectCount, int(r.RowsAffected))
}
<commit_msg>using ALTER VITESS_MIGRATION ... CANCEL ALL in endtoend/onlineddl/<commit_after>/*
Copyright 2021 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package onlineddl

import (
	"context"
	"fmt"
	"testing"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"

	"vitess.io/vitess/go/test/endtoend/cluster"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

// VtgateExecQuery runs a query on VTGate using given query params
func VtgateExecQuery(t *testing.T, vtParams *mysql.ConnParams, query string, expectError string) *sqltypes.Result {
	t.Helper()

	ctx := context.Background()
	conn, err := mysql.Connect(ctx, vtParams)
	require.Nil(t, err)
	defer conn.Close()

	qr, err := conn.ExecuteFetch(query, 1000, true)
	if expectError == "" {
		require.NoError(t, err)
	} else {
		require.Error(t, err, "error should not be nil")
		assert.Contains(t, err.Error(), expectError, "Unexpected error")
	}
	return qr
}

// CheckRetryMigration attempts to retry a migration, and expects success/failure by counting affected rows
func CheckRetryMigration(t *testing.T, vtParams *mysql.ConnParams, shards []cluster.Shard, uuid string, expectRetryPossible bool) {
	retryQuery := fmt.Sprintf("alter vitess_migration '%s' retry", uuid)
	r := VtgateExecQuery(t, vtParams, retryQuery, "")

	if expectRetryPossible {
		assert.Equal(t, len(shards), int(r.RowsAffected))
	} else {
		assert.Equal(t, int(0), int(r.RowsAffected))
	}
}

// CheckCancelMigration attempts to cancel a migration, and expects success/failure by counting affected rows
func CheckCancelMigration(t *testing.T, vtParams *mysql.ConnParams, shards []cluster.Shard, uuid string, expectCancelPossible bool) {
	cancelQuery := fmt.Sprintf("alter vitess_migration '%s' cancel", uuid)
	r := VtgateExecQuery(t, vtParams, cancelQuery, "")

	if expectCancelPossible {
		assert.Equal(t, len(shards), int(r.RowsAffected))
	} else {
		assert.Equal(t, int(0), int(r.RowsAffected))
	}
}

// CheckCancelAllMigrations cancels all pending migrations and expect number of affected rows
func CheckCancelAllMigrations(t *testing.T, vtParams *mysql.ConnParams, expectCount int) {
	cancelQuery := "alter vitess_migration cancel all"
	r := VtgateExecQuery(t, vtParams, cancelQuery, "")

	assert.Equal(t, expectCount, int(r.RowsAffected))
}
<|endoftext|>


<commit_before>package fsm

import (
	"fmt"
	"log"

	"time"

	"io"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/swf"
	"github.com/juju/errors"
	. "github.com/sclasen/swfsm/sugar"
)

type FSMClient interface {
	WalkOpenWorkflowInfos(template *swf.ListOpenWorkflowExecutionsInput, workflowInfosFunc WorkflowInfosFunc) error
	GetState(id string) (string, interface{}, error)
	GetSnapshots(id string) ([]FSMSnapshot, error)
	Signal(id string, signal string, input interface{}) error
	Start(startTemplate swf.StartWorkflowExecutionInput, id string, input interface{}) (*swf.StartWorkflowExecutionOutput, error)
	RequestCancel(id string) error
}

type ClientSWFOps interface {
	ListOpenWorkflowExecutions(req *swf.ListOpenWorkflowExecutionsInput) (resp *swf.WorkflowExecutionInfos, err error)
	ListClosedWorkflowExecutions(req *swf.ListClosedWorkflowExecutionsInput) (resp *swf.WorkflowExecutionInfos, err error)
	GetWorkflowExecutionHistory(req *swf.GetWorkflowExecutionHistoryInput) (resp *swf.GetWorkflowExecutionHistoryOutput, err error)
	SignalWorkflowExecution(req *swf.SignalWorkflowExecutionInput) (resp *swf.SignalWorkflowExecutionOutput, err error)
	StartWorkflowExecution(req *swf.StartWorkflowExecutionInput) (resp *swf.StartWorkflowExecutionOutput, err error)
	TerminateWorkflowExecution(req *swf.TerminateWorkflowExecutionInput) (resp *swf.TerminateWorkflowExecutionOutput, err error)
	RequestCancelWorkflowExecution(req *swf.RequestCancelWorkflowExecutionInput) (resp *swf.RequestCancelWorkflowExecutionOutput, err error)
}

func NewFSMClient(f *FSM, c ClientSWFOps) FSMClient {
	return &client{
		f: f,
		c: c,
	}
}

type client struct {
	f *FSM
	c ClientSWFOps
}

type WorkflowInfosFunc func(infos *swf.WorkflowExecutionInfos) error

type stopWalkingError error

func StopWalking() stopWalkingError {
	return stopWalkingError(fmt.Errorf(""))
}

func IsStopWalking(err error) bool {
	_, ok := err.(stopWalkingError)
	return ok
}

func (c *client) WalkOpenWorkflowInfos(template *swf.ListOpenWorkflowExecutionsInput, workflowInfosFunc WorkflowInfosFunc) error {
	template.Domain = S(c.f.Domain)

	if template.StartTimeFilter == nil {
		template.StartTimeFilter = &swf.ExecutionTimeFilter{OldestDate: aws.Time(time.Unix(0, 0))}
	}

	if template.TypeFilter == nil && template.ExecutionFilter == nil && template.TagFilter == nil {
		template.TypeFilter = &swf.WorkflowTypeFilter{Name: S(c.f.Name)}
	}

	infos, err := c.c.ListOpenWorkflowExecutions(template)

	for {
		if err != nil {
			return err
		}

		err := workflowInfosFunc(infos)

		if err != nil {
			if IsStopWalking(err) {
				return nil
			}
			return err
		}

		if infos.NextPageToken == nil {
			break
		}

		template.NextPageToken = infos.NextPageToken
		infos, err = c.c.ListOpenWorkflowExecutions(template)
	}

	return nil
}

// TODO: make usable again
func (c *client) listIds(executionInfosFunc func() (*swf.WorkflowExecutionInfos, error)) ([]string, string, error) {
	executionInfos, err := executionInfosFunc()

	if err != nil {
		if ae, ok := err.(awserr.Error); ok {
			log.Printf("component=client fn=listIds at=list-infos-func error-type=%s message=%s", ae.Code(), ae.Message())
		} else {
			log.Printf("component=client fn=listIds at=list-infos-func error=%s", err)
		}
		return []string{}, "", err
	}

	ids := []string{}
	for _, info := range executionInfos.ExecutionInfos {
		ids = append(ids, *info.Execution.WorkflowID)
	}

	nextPageToken := ""
	if executionInfos.NextPageToken != nil {
		nextPageToken = *executionInfos.NextPageToken
	}

	return ids, nextPageToken, nil
}

func (c *client) findExecution(id string) (*swf.WorkflowExecution, error) {
	open, err := c.c.ListOpenWorkflowExecutions(&swf.ListOpenWorkflowExecutionsInput{
		Domain:          S(c.f.Domain),
		MaximumPageSize: aws.Int64(1),
		StartTimeFilter: &swf.ExecutionTimeFilter{OldestDate: aws.Time(time.Unix(0, 0))},
		ExecutionFilter: &swf.WorkflowExecutionFilter{
			WorkflowID: S(id),
		},
	})

	if err != nil {
		if ae, ok := err.(awserr.Error); ok {
			log.Printf("component=client fn=findExecution at=list-open error-type=%s message=%s", ae.Code(), ae.Message())
		} else {
			log.Printf("component=client fn=findExecution at=list-open error=%s", err)
		}
		return nil, err
	}

	if len(open.ExecutionInfos) == 1 {
		return open.ExecutionInfos[0].Execution, nil
	} else {
		closed, err := c.c.ListClosedWorkflowExecutions(&swf.ListClosedWorkflowExecutionsInput{
			Domain:          S(c.f.Domain),
			MaximumPageSize: aws.Int64(1),
			StartTimeFilter: &swf.ExecutionTimeFilter{OldestDate: aws.Time(time.Unix(0, 0))},
			ExecutionFilter: &swf.WorkflowExecutionFilter{
				WorkflowID: S(id),
			},
		})

		if err != nil {
			if ae, ok := err.(awserr.Error); ok {
				log.Printf("component=client fn=findExecution at=list-closed error-type=%s message=%s", ae.Code(), ae.Message())
			} else {
				log.Printf("component=client fn=findExecution at=list-closed error=%s", err)
			}
			return nil, err
		}

		if len(closed.ExecutionInfos) > 0 {
			return closed.ExecutionInfos[0].Execution, nil
		} else {
			return nil, errors.Trace(fmt.Errorf("workflow not found for id %s", id))
		}
	}
}

func (c *client) GetState(id string) (string, interface{}, error) {
	getState := func() (string, interface{}, error) {
		execution, err := c.findExecution(id)
		if err != nil {
			return "", nil, err
		}

		history, err := c.c.GetWorkflowExecutionHistory(&swf.GetWorkflowExecutionHistoryInput{
			Domain:       S(c.f.Domain),
			Execution:    execution,
			ReverseOrder: aws.Bool(true),
		})

		if err != nil {
			if ae, ok := err.(awserr.Error); ok {
				log.Printf("component=client fn=GetState at=get-history error-type=%s message=%s", ae.Code(), ae.Message())
			} else {
				log.Printf("component=client fn=GetState at=get-history error=%s", err)
			}
			return "", nil, err
		}

		serialized, err := c.f.findSerializedState(history.Events)

		if err != nil {
			log.Printf("component=client fn=GetState at=find-serialized-state error=%s", err)
			return "", nil, err
		}

		data := c.f.zeroStateData()
		err = c.f.Serializer.Deserialize(serialized.StateData, data)
		if err != nil {
			log.Printf("component=client fn=GetState at=deserialize-serialized-state error=%s", err)
			return "", nil, err
		}

		return serialized.StateName, data, nil
	}

	var err error
	for i := 0; i < 5; i++ {
		state, data, err := getState()
		if err != nil && strings.HasSuffix(err.Error(), io.EOF.Error()) {
			continue
		} else {
			return state, data, err
		}
	}

	return "", nil, err

}

func (c *client) Signal(id string, signal string, input interface{}) error {
	var serializedInput *string
	if input != nil {
		switch it := input.(type) {
		case string:
			serializedInput = S(it)
		default:
			ser, err := c.f.Serializer.Serialize(input)
			if err != nil {
				return errors.Trace(err)
			}
			serializedInput = S(ser)
		}
	}
	_, err := c.c.SignalWorkflowExecution(&swf.SignalWorkflowExecutionInput{
		Domain:     S(c.f.Domain),
		SignalName: S(signal),
		Input:      serializedInput,
		WorkflowID: S(id),
	})
	return err
}

func (c *client) Start(startTemplate swf.StartWorkflowExecutionInput, id string, input interface{}) (*swf.StartWorkflowExecutionOutput, error) {
	var serializedInput *string
	if input != nil {
		ser, err := c.f.Serializer.Serialize(input)
		if err != nil {
			return nil, errors.Trace(err)
		}
		serializedInput = S(ser)
	}
	startTemplate.Domain = S(c.f.Domain)
	startTemplate.WorkflowID = S(id)
	startTemplate.Input = serializedInput
	if len(startTemplate.TagList) == 0 {
		startTemplate.TagList = GetTagsIfTaggable(input)
	}
	return c.c.StartWorkflowExecution(&startTemplate)
}

func (c *client) RequestCancel(id string) error {
	_, err := c.c.RequestCancelWorkflowExecution(&swf.RequestCancelWorkflowExecutionInput{
		Domain:     S(c.f.Domain),
		WorkflowID: S(id),
	})
	return err
}

func (c *client) GetSnapshots(id string) ([]FSMSnapshot, error) {
	snapshots := []FSMSnapshot{}

	execution, err := c.findExecution(id)
	if err != nil {
		return snapshots, err
	}

	// TODO: extract out history walker and/or nextpagetoken support
	history, err := c.c.GetWorkflowExecutionHistory(&swf.GetWorkflowExecutionHistoryInput{
		Domain:       S(c.f.Domain),
		Execution:    execution,
		ReverseOrder: aws.Bool(true),
	})

	if err != nil {
		return snapshots, err
	}

	snapshot := FSMSnapshot{}
	for _, historyEvent := range history.Events {
		// TODO: how to deal with failure events?
		switch EventType := *historyEvent.EventType; EventType {
		case swf.EventTypeWorkflowExecutionStarted:
			snapshot.Event = &FSMSnapshotEvent{
				Type:  EventType,
				Name:  "start",
				Input: c.tryDeserialize(historyEvent.WorkflowExecutionStartedEventAttributes.Input),
			}
		case swf.EventTypeWorkflowExecutionSignaled:
			snapshot.Event = &FSMSnapshotEvent{
				Type:  EventType,
				Name:  *historyEvent.WorkflowExecutionSignaledEventAttributes.SignalName,
				Input: c.tryDeserialize(historyEvent.WorkflowExecutionSignaledEventAttributes.Input),
			}
		case swf.EventTypeActivityTaskScheduled:
			if snapshot.Event != nil && snapshot.Event.Input == c.pointerScheduledEventID(historyEvent.EventID) {
				snapshot.Event.Name = *historyEvent.ActivityTaskScheduledEventAttributes.ActivityType.Name
				snapshot.Event.Version = *historyEvent.ActivityTaskScheduledEventAttributes.ActivityType.Version
				snapshot.Event.Input = c.tryDeserialize(historyEvent.ActivityTaskScheduledEventAttributes.Input)
			}
		case swf.EventTypeActivityTaskCompleted:
			snapshot.Event = &FSMSnapshotEvent{
				Type:   EventType,
				Input:  c.pointerScheduledEventID(historyEvent.ActivityTaskCompletedEventAttributes.ScheduledEventID),
				Output: c.tryDeserialize(historyEvent.ActivityTaskCompletedEventAttributes.Result),
			}
		case swf.EventTypeStartChildWorkflowExecutionInitiated:
			snapshot.Event = &FSMSnapshotEvent{
				Type:   EventType,
				Name:   *historyEvent.StartChildWorkflowExecutionInitiatedEventAttributes.WorkflowType.Name,
				Input:  c.tryDeserialize(historyEvent.StartChildWorkflowExecutionInitiatedEventAttributes.Input),
				Target: *historyEvent.StartChildWorkflowExecutionInitiatedEventAttributes.WorkflowID,
			}
		case swf.EventTypeSignalExternalWorkflowExecutionInitiated:
			snapshot.Event = &FSMSnapshotEvent{
				Type:   EventType,
				Name:   *historyEvent.SignalExternalWorkflowExecutionInitiatedEventAttributes.SignalName,
				Input:  c.tryDeserialize(historyEvent.SignalExternalWorkflowExecutionInitiatedEventAttributes.Input),
				Target: *historyEvent.SignalExternalWorkflowExecutionInitiatedEventAttributes.WorkflowID,
			}
		case swf.EventTypeRequestCancelExternalWorkflowExecutionInitiated:
			snapshot.Event = &FSMSnapshotEvent{
				Type:   EventType,
				Target: *historyEvent.RequestCancelExternalWorkflowExecutionInitiatedEventAttributes.WorkflowID,
			}
		case swf.EventTypeTimerStarted:
			if snapshot.Event != nil && snapshot.Event.Input == c.pointerStartedEventID(historyEvent.EventID) {
				snapshot.Event.Input = *historyEvent.TimerStartedEventAttributes.StartToFireTimeout
			}
		case swf.EventTypeTimerFired:
			snapshot.Event = &FSMSnapshotEvent{
				Type:  EventType,
				Name:  *historyEvent.TimerFiredEventAttributes.TimerID,
				Input: c.pointerStartedEventID(historyEvent.TimerFiredEventAttributes.StartedEventID),
			}
		case swf.EventTypeWorkflowExecutionCancelRequested:
			snapshot.Event = &FSMSnapshotEvent{
				Type: EventType,
			}
		}

		state, err := c.f.statefulHistoryEventToSerializedState(historyEvent)
		if err != nil {
			break
		}

		if state != nil {
			snapshot.State = &FSMSnapshotState{
				ID:        *historyEvent.EventID,
				Timestamp: *historyEvent.EventTimestamp,
				Version:   state.StateVersion,
				Name:      state.StateName,
				Data:      c.f.zeroStateData(),
			}
			err = c.f.Serializer.Deserialize(state.StateData, snapshot.State.Data)
			if err != nil {
				return snapshots, err
			}

			snapshots = append(snapshots, snapshot)
			snapshot = FSMSnapshot{}
		}
	}

	return snapshots, err
}

func (c *client) tryDeserialize(serialized *string) interface{} {
	if serialized == nil || *serialized == "" {
		return ""
	}

	tryMap := make(map[string]interface{})
	err := c.f.systemSerializer.Deserialize(*serialized, &tryMap)
	if err == nil {
		return tryMap
	}
	log.Printf("component=client fn=trySystemDeserialize at=deserialize-map error=%s", err)

	return *serialized
}

func (c *client) pointerStartedEventID(id *int64) string {
	return fmt.Sprintf("StartedEventID->%d", *id)
}

func (c *client) pointerScheduledEventID(id *int64) string {
	return fmt.Sprintf("ScheduledEventID->%d", *id)
}
<commit_msg>add GetStateForRun that takes a runID too<commit_after>package fsm

import (
	"fmt"
	"log"

	"time"

	"io"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/swf"
	"github.com/juju/errors"
	. "github.com/sclasen/swfsm/sugar"
)

type FSMClient interface {
	WalkOpenWorkflowInfos(template *swf.ListOpenWorkflowExecutionsInput, workflowInfosFunc WorkflowInfosFunc) error
	GetState(id string) (string, interface{}, error)
	GetSnapshots(id string) ([]FSMSnapshot, error)
	Signal(id string, signal string, input interface{}) error
	Start(startTemplate swf.StartWorkflowExecutionInput, id string, input interface{}) (*swf.StartWorkflowExecutionOutput, error)
	RequestCancel(id string) error
}

type ClientSWFOps interface {
	ListOpenWorkflowExecutions(req *swf.ListOpenWorkflowExecutionsInput) (resp *swf.WorkflowExecutionInfos, err error)
	ListClosedWorkflowExecutions(req *swf.ListClosedWorkflowExecutionsInput) (resp *swf.WorkflowExecutionInfos, err error)
	GetWorkflowExecutionHistory(req *swf.GetWorkflowExecutionHistoryInput) (resp *swf.GetWorkflowExecutionHistoryOutput, err error)
	SignalWorkflowExecution(req *swf.SignalWorkflowExecutionInput) (resp *swf.SignalWorkflowExecutionOutput, err error)
	StartWorkflowExecution(req *swf.StartWorkflowExecutionInput) (resp *swf.StartWorkflowExecutionOutput, err error)
	TerminateWorkflowExecution(req *swf.TerminateWorkflowExecutionInput) (resp *swf.TerminateWorkflowExecutionOutput, err error)
	RequestCancelWorkflowExecution(req *swf.RequestCancelWorkflowExecutionInput) (resp *swf.RequestCancelWorkflowExecutionOutput, err error)
}

func NewFSMClient(f *FSM, c ClientSWFOps) FSMClient {
	return &client{
		f: f,
		c: c,
	}
}

type client struct {
	f *FSM
	c ClientSWFOps
}

type WorkflowInfosFunc func(infos *swf.WorkflowExecutionInfos) error

type stopWalkingError error

func StopWalking() stopWalkingError {
	return stopWalkingError(fmt.Errorf(""))
}

func IsStopWalking(err error) bool {
	_, ok := err.(stopWalkingError)
	return ok
}

func (c *client) WalkOpenWorkflowInfos(template *swf.ListOpenWorkflowExecutionsInput, workflowInfosFunc WorkflowInfosFunc) error {
	template.Domain = S(c.f.Domain)

	if template.StartTimeFilter == nil {
		template.StartTimeFilter = &swf.ExecutionTimeFilter{OldestDate: aws.Time(time.Unix(0, 0))}
	}

	if template.TypeFilter == nil && template.ExecutionFilter == nil && template.TagFilter == nil {
		template.TypeFilter = &swf.WorkflowTypeFilter{Name: S(c.f.Name)}
	}

	infos, err := c.c.ListOpenWorkflowExecutions(template)

	for {
		if err != nil {
			return err
		}

		err := workflowInfosFunc(infos)

		if err != nil {
			if IsStopWalking(err) {
				return nil
			}
			return err
		}

		if infos.NextPageToken == nil {
			break
		}

		template.NextPageToken = infos.NextPageToken
		infos, err = c.c.ListOpenWorkflowExecutions(template)
	}

	return nil
}

// TODO: make usable again
func (c *client) listIds(executionInfosFunc func() (*swf.WorkflowExecutionInfos, error)) ([]string, string, error) {
	executionInfos, err := executionInfosFunc()

	if err != nil {
		if ae, ok := err.(awserr.Error); ok {
			log.Printf("component=client fn=listIds at=list-infos-func error-type=%s message=%s", ae.Code(), ae.Message())
		} else {
			log.Printf("component=client fn=listIds at=list-infos-func error=%s", err)
		}
		return []string{}, "", err
	}

	ids := []string{}
	for _, info := range executionInfos.ExecutionInfos {
		ids = append(ids, *info.Execution.WorkflowID)
	}

	nextPageToken := ""
	if executionInfos.NextPageToken != nil {
		nextPageToken = *executionInfos.NextPageToken
	}

	return ids, nextPageToken, nil
}

func (c *client) findExecution(id string) (*swf.WorkflowExecution, error) {
	open, err := c.c.ListOpenWorkflowExecutions(&swf.ListOpenWorkflowExecutionsInput{
		Domain:          S(c.f.Domain),
		MaximumPageSize: aws.Int64(1),
		StartTimeFilter: &swf.ExecutionTimeFilter{OldestDate: aws.Time(time.Unix(0, 0))},
		ExecutionFilter: &swf.WorkflowExecutionFilter{
			WorkflowID: S(id),
		},
	})

	if err != nil {
		if ae, ok := err.(awserr.Error); ok {
			log.Printf("component=client fn=findExecution at=list-open error-type=%s message=%s", ae.Code(), ae.Message())
		} else {
			log.Printf("component=client fn=findExecution at=list-open error=%s", err)
		}
		return nil, err
	}

	if len(open.ExecutionInfos) == 1 {
		return open.ExecutionInfos[0].Execution, nil
	} else {
		closed, err := c.c.ListClosedWorkflowExecutions(&swf.ListClosedWorkflowExecutionsInput{
			Domain:          S(c.f.Domain),
			MaximumPageSize: aws.Int64(1),
			StartTimeFilter: &swf.ExecutionTimeFilter{OldestDate: aws.Time(time.Unix(0, 0))},
			ExecutionFilter: &swf.WorkflowExecutionFilter{
				WorkflowID: S(id),
			},
		})

		if err != nil {
			if ae, ok := err.(awserr.Error); ok {
				log.Printf("component=client fn=findExecution at=list-closed error-type=%s message=%s", ae.Code(), ae.Message())
			} else {
				log.Printf("component=client fn=findExecution at=list-closed error=%s", err)
			}
			return nil, err
		}

		if len(closed.ExecutionInfos) > 0 {
			return closed.ExecutionInfos[0].Execution, nil
		} else {
			return nil, errors.Trace(fmt.Errorf("workflow not found for id %s", id))
		}
	}
}
func (c *client) GetStateForRun(id, run string) (string, interface{}, error) {
	getState := func() (string, interface{}, error) {

		history, err := c.c.GetWorkflowExecutionHistory(&swf.GetWorkflowExecutionHistoryInput{
			Domain: S(c.f.Domain),
			Execution: &swf.WorkflowExecution{
				WorkflowID: S(id),
				RunID:      S(run),
			},
			ReverseOrder: aws.Bool(true),
		})

		if err != nil {
			if ae, ok := err.(awserr.Error); ok {
				log.Printf("component=client fn=GetState at=get-history error-type=%s message=%s", ae.Code(), ae.Message())
			} else {
				log.Printf("component=client fn=GetState at=get-history error=%s", err)
			}
			return "", nil, err
		}

		serialized, err := c.f.findSerializedState(history.Events)

		if err != nil {
			log.Printf("component=client fn=GetState at=find-serialized-state error=%s", err)
			return "", nil, err
		}

		data := c.f.zeroStateData()
		err = c.f.Serializer.Deserialize(serialized.StateData, data)
		if err != nil {
			log.Printf("component=client fn=GetState at=deserialize-serialized-state error=%s", err)
			return "", nil, err
		}

		return serialized.StateName, data, nil
	}

	var err error
	for i := 0; i < 5; i++ {
		state, data, err := getState()
		if err != nil && strings.HasSuffix(err.Error(), io.EOF.Error()) {
			continue
		} else {
			return state, data, err
		}
	}

	return "", nil, err
}

func (c *client) GetState(id string) (string, interface{}, error) {
	execution, err := c.findExecution(id)
	if err != nil {
		return "", nil, err
	}
	return c.GetStateForRun(id, *execution.RunID)
}

func (c *client) Signal(id string, signal string, input interface{}) error {
	var serializedInput *string
	if input != nil {
		switch it := input.(type) {
		case string:
			serializedInput = S(it)
		default:
			ser, err := c.f.Serializer.Serialize(input)
			if err != nil {
				return errors.Trace(err)
			}
			serializedInput = S(ser)
		}
	}
	_, err := c.c.SignalWorkflowExecution(&swf.SignalWorkflowExecutionInput{
		Domain:     S(c.f.Domain),
		SignalName: S(signal),
		Input:      serializedInput,
		WorkflowID: S(id),
	})
	return err
}

func (c *client) Start(startTemplate swf.StartWorkflowExecutionInput, id string, input interface{}) (*swf.StartWorkflowExecutionOutput, error) {
	var serializedInput *string
	if input != nil {
		ser, err := c.f.Serializer.Serialize(input)
		if err != nil {
			return nil, errors.Trace(err)
		}
		serializedInput = S(ser)
	}
	startTemplate.Domain = S(c.f.Domain)
	startTemplate.WorkflowID = S(id)
	startTemplate.Input = serializedInput
	if len(startTemplate.TagList) == 0 {
		startTemplate.TagList = GetTagsIfTaggable(input)
	}
	return c.c.StartWorkflowExecution(&startTemplate)
}

func (c *client) RequestCancel(id string) error {
	_, err := c.c.RequestCancelWorkflowExecution(&swf.RequestCancelWorkflowExecutionInput{
		Domain:     S(c.f.Domain),
		WorkflowID: S(id),
	})
	return err
}

func (c *client) GetSnapshots(id string) ([]FSMSnapshot, error) {
	snapshots := []FSMSnapshot{}

	execution, err := c.findExecution(id)
	if err != nil {
		return snapshots, err
	}

	// TODO: extract out history walker and/or nextpagetoken support
	history, err := c.c.GetWorkflowExecutionHistory(&swf.GetWorkflowExecutionHistoryInput{
		Domain:       S(c.f.Domain),
		Execution:    execution,
		ReverseOrder: aws.Bool(true),
	})

	if err != nil {
		return snapshots, err
	}

	snapshot := FSMSnapshot{}
	for _, historyEvent := range history.Events {
		// TODO: how to deal with failure events?
		switch EventType := *historyEvent.EventType; EventType {
		case swf.EventTypeWorkflowExecutionStarted:
			snapshot.Event = &FSMSnapshotEvent{
				Type:  EventType,
				Name:  "start",
				Input: c.tryDeserialize(historyEvent.WorkflowExecutionStartedEventAttributes.Input),
			}
		case swf.EventTypeWorkflowExecutionSignaled:
			snapshot.Event = &FSMSnapshotEvent{
				Type:  EventType,
				Name:  *historyEvent.WorkflowExecutionSignaledEventAttributes.SignalName,
				Input: c.tryDeserialize(historyEvent.WorkflowExecutionSignaledEventAttributes.Input),
			}
		case swf.EventTypeActivityTaskScheduled:
			if snapshot.Event != nil && snapshot.Event.Input == c.pointerScheduledEventID(historyEvent.EventID) {
				snapshot.Event.Name = *historyEvent.ActivityTaskScheduledEventAttributes.ActivityType.Name
				snapshot.Event.Version = *historyEvent.ActivityTaskScheduledEventAttributes.ActivityType.Version
				snapshot.Event.Input = c.tryDeserialize(historyEvent.ActivityTaskScheduledEventAttributes.Input)
			}
		case swf.EventTypeActivityTaskCompleted:
			snapshot.Event = &FSMSnapshotEvent{
				Type:   EventType,
				Input:  c.pointerScheduledEventID(historyEvent.ActivityTaskCompletedEventAttributes.ScheduledEventID),
				Output: c.tryDeserialize(historyEvent.ActivityTaskCompletedEventAttributes.Result),
			}
		case swf.EventTypeStartChildWorkflowExecutionInitiated:
			snapshot.Event = &FSMSnapshotEvent{
				Type:   EventType,
				Name:   *historyEvent.StartChildWorkflowExecutionInitiatedEventAttributes.WorkflowType.Name,
				Input:  c.tryDeserialize(historyEvent.StartChildWorkflowExecutionInitiatedEventAttributes.Input),
				Target: *historyEvent.StartChildWorkflowExecutionInitiatedEventAttributes.WorkflowID,
			}
		case swf.EventTypeSignalExternalWorkflowExecutionInitiated:
			snapshot.Event = &FSMSnapshotEvent{
				Type:   EventType,
				Name:   *historyEvent.SignalExternalWorkflowExecutionInitiatedEventAttributes.SignalName,
				Input:  c.tryDeserialize(historyEvent.SignalExternalWorkflowExecutionInitiatedEventAttributes.Input),
				Target: *historyEvent.SignalExternalWorkflowExecutionInitiatedEventAttributes.WorkflowID,
			}
		case swf.EventTypeRequestCancelExternalWorkflowExecutionInitiated:
			snapshot.Event = &FSMSnapshotEvent{
				Type:   EventType,
				Target: *historyEvent.RequestCancelExternalWorkflowExecutionInitiatedEventAttributes.WorkflowID,
			}
		case swf.EventTypeTimerStarted:
			if snapshot.Event != nil && snapshot.Event.Input == c.pointerStartedEventID(historyEvent.EventID) {
				snapshot.Event.Input = *historyEvent.TimerStartedEventAttributes.StartToFireTimeout
			}
		case swf.EventTypeTimerFired:
			snapshot.Event = &FSMSnapshotEvent{
				Type:  EventType,
				Name:  *historyEvent.TimerFiredEventAttributes.TimerID,
				Input: c.pointerStartedEventID(historyEvent.TimerFiredEventAttributes.StartedEventID),
			}
		case swf.EventTypeWorkflowExecutionCancelRequested:
			snapshot.Event = &FSMSnapshotEvent{
				Type: EventType,
			}
		}

		state, err := c.f.statefulHistoryEventToSerializedState(historyEvent)
		if err != nil {
			break
		}

		if state != nil {
			snapshot.State = &FSMSnapshotState{
				ID:        *historyEvent.EventID,
				Timestamp: *historyEvent.EventTimestamp,
				Version:   state.StateVersion,
				Name:      state.StateName,
				Data:      c.f.zeroStateData(),
			}
			err = c.f.Serializer.Deserialize(state.StateData, snapshot.State.Data)
			if err != nil {
				return snapshots, err
			}

			snapshots = append(snapshots, snapshot)
			snapshot = FSMSnapshot{}
		}
	}

	return snapshots, err
}

func (c *client) tryDeserialize(serialized *string) interface{} {
	if serialized == nil || *serialized == "" {
		return ""
	}

	tryMap := make(map[string]interface{})
	err := c.f.systemSerializer.Deserialize(*serialized, &tryMap)
	if err == nil {
		return tryMap
	}
	log.Printf("component=client fn=trySystemDeserialize at=deserialize-map error=%s", err)

	return *serialized
}

func (c *client) pointerStartedEventID(id *int64) string {
	return fmt.Sprintf("StartedEventID->%d", *id)
}

func (c *client) pointerScheduledEventID(id *int64) string {
	return fmt.Sprintf("ScheduledEventID->%d", *id)
}
<|endoftext|>


<commit_before>package google

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
)

func TestAccCloudRunService_cloudRunServiceUpdate(t *testing.T) {
	t.Parallel()

	project := getTestProjectFromEnv()
	name := "tftest-cloudrun-" + acctest.RandString(6)

	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccCloudRunService_cloudRunServiceUpdate(name, project, "10"),
			},
			{
				ResourceName:            "google_cloud_run_service.default",
				ImportStateId:           "us-central1/" + name,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"metadata.0.resource_version", "status.0.conditions"},
			},
			{
				Config: testAccCloudRunService_cloudRunServiceUpdate(name, project, "50"),
			},
			{
				ResourceName:            "google_cloud_run_service.default",
				ImportStateId:           "us-central1/" + name,
				ImportState:             true,
				ImportStateVerify:       true,
				ImportStateVerifyIgnore: []string{"metadata.0.resource_version", "status.0.conditions"},
			},
		},
	})
}

func testAccCloudRunService_cloudRunServiceUpdate(name, project, concurrency string) string {
	return fmt.Sprintf(`
resource "google_cloud_run_service" "default" {
  name          = "%s"
  location = "us-central1"

  metadata {
    namespace = "%s"
  }

  spec {
    containers {
	  image = "gcr.io/cloudrun/hello"
	  args = ["arrgs"]
	}
	container_concurrency = %s
  }
}
`, name, project, concurrency)
}
<commit_msg>comment out beta cloud run tests (#4120)<commit_after>package google

// Because Cloud Run is still in beta, we can't run any of the tests that call that
// resource without vendoring in the full beta provider.
<|endoftext|>