<commit_before>/*
Copyright 2017 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jsonnet

import (
	"errors"
	"fmt"
	"runtime/debug"

	"github.com/google/go-jsonnet/ast"
	"github.com/google/go-jsonnet/parser"
)

// Note: There are no garbage collection params because we're using the native
// Go garbage collector.

// VM is the core interpreter and is the touchpoint used to parse and execute
// Jsonnet.
type VM struct {
	MaxStack       int
	ext            vmExtMap
	tla            vmExtMap
	nativeFuncs    map[string]*NativeFunction
	importer       Importer
	ErrorFormatter ErrorFormatter
	StringOutput   bool
}

// External variable or top level argument provided before execution
type vmExt struct {
	// jsonnet code to evaluate or string to pass
	value string
	// isCode determines whether it should be evaluated as jsonnet code or
	// treated as string.
	isCode bool
}

type vmExtMap map[string]vmExt

// MakeVM creates a new VM with default parameters.
func MakeVM() *VM {
	return &VM{
		MaxStack:       500,
		ext:            make(vmExtMap),
		tla:            make(vmExtMap),
		nativeFuncs:    make(map[string]*NativeFunction),
		ErrorFormatter: &termErrorFormatter{pretty: false, maxStackTraceSize: 20},
		importer:       &FileImporter{},
	}
}

// ExtVar binds a Jsonnet external var to the given value.
func (vm *VM) ExtVar(key string, val string) {
	vm.ext[key] = vmExt{value: val, isCode: false}
}

// ExtCode binds a Jsonnet external code var to the given code.
func (vm *VM) ExtCode(key string, val string) {
	vm.ext[key] = vmExt{value: val, isCode: true}
}

// TLAVar binds a Jsonnet top level argument to the given value.
func (vm *VM) TLAVar(key string, val string) {
	vm.tla[key] = vmExt{value: val, isCode: false}
}

// TLACode binds a Jsonnet top level argument to the given code.
func (vm *VM) TLACode(key string, val string) {
	vm.tla[key] = vmExt{value: val, isCode: true}
}

// Importer sets Importer to use during evaluation (import callback).
func (vm *VM) Importer(i Importer) {
	vm.importer = i
}

type evalKind int

const (
	evalKindRegular evalKind = iota
	evalKindMulti            = iota
	evalKindStream           = iota
)

func (vm *VM) evaluateSnippet(filename string, snippet string, kind evalKind) (output interface{}, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("(CRASH) %v\n%s", r, debug.Stack())
		}
	}()
	node, err := snippetToAST(filename, snippet)
	if err != nil {
		return "", err
	}
	switch kind {
	case evalKindRegular:
		output, err = evaluate(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer, vm.StringOutput)
	case evalKindMulti:
		output, err = evaluateMulti(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer, vm.StringOutput)
	case evalKindStream:
		output, err = evaluateStream(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer)
	}
	if err != nil {
		return "", err
	}
	return output, nil
}

// NativeFunction registers a native function.
func (vm *VM) NativeFunction(f *NativeFunction) {
	vm.nativeFuncs[f.Name] = f
}

// EvaluateSnippet evaluates a string containing Jsonnet code, return a JSON
// string.
//
// The filename parameter is only used for error messages.
func (vm *VM) EvaluateSnippet(filename string, snippet string) (json string, formattedErr error) {
	output, err := vm.evaluateSnippet(filename, snippet, evalKindRegular)
	if err != nil {
		return "", errors.New(vm.ErrorFormatter.Format(err))
	}
	json = output.(string)
	return
}

// EvaluateSnippetStream evaluates a string containing Jsonnet code to an array.
// The array is returned as an array of JSON strings.
//
// The filename parameter is only used for error messages.
func (vm *VM) EvaluateSnippetStream(filename string, snippet string) (docs []string, formattedErr error) {
	output, err := vm.evaluateSnippet(filename, snippet, evalKindStream)
	if err != nil {
		return nil, errors.New(vm.ErrorFormatter.Format(err))
	}
	docs = output.([]string)
	return
}

// EvaluateSnippetMulti evaluates a string containing Jsonnet code to key-value
// pairs. The keys are field name strings and the values are JSON strings.
//
// The filename parameter is only used for error messages.
func (vm *VM) EvaluateSnippetMulti(filename string, snippet string) (files map[string]string, formattedErr error) {
	output, err := vm.evaluateSnippet(filename, snippet, evalKindMulti)
	if err != nil {
		return nil, errors.New(vm.ErrorFormatter.Format(err))
	}
	files = output.(map[string]string)
	return
}

func snippetToAST(filename string, snippet string) (ast.Node, error) {
	tokens, err := parser.Lex(filename, snippet)
	if err != nil {
		return nil, err
	}
	node, err := parser.Parse(tokens)
	if err != nil {
		return nil, err
	}
	err = desugarFile(&node)
	if err != nil {
		return nil, err
	}
	err = analyze(node)
	if err != nil {
		return nil, err
	}
	return node, nil
}

// SnippetToAST parses a snippet and returns the resulting AST.
func SnippetToAST(filename string, snippet string) (ast.Node, error) {
	return snippetToAST(filename, snippet)
}

// Version returns the Jsonnet version number.
func Version() string {
	return "v0.12.1"
}
<commit_msg>Expose Evaluate functions, to be able to reuse the AST from `SnippetToAST`.<commit_after>/*
Copyright 2017 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jsonnet

import (
	"errors"
	"fmt"
	"runtime/debug"

	"github.com/google/go-jsonnet/ast"
	"github.com/google/go-jsonnet/parser"
)

// Note: There are no garbage collection params because we're using the native
// Go garbage collector.

// VM is the core interpreter and is the touchpoint used to parse and execute
// Jsonnet.
type VM struct {
	MaxStack       int
	ext            vmExtMap
	tla            vmExtMap
	nativeFuncs    map[string]*NativeFunction
	importer       Importer
	ErrorFormatter ErrorFormatter
	StringOutput   bool
}

// External variable or top level argument provided before execution
type vmExt struct {
	// jsonnet code to evaluate or string to pass
	value string
	// isCode determines whether it should be evaluated as jsonnet code or
	// treated as string.
	isCode bool
}

type vmExtMap map[string]vmExt

// MakeVM creates a new VM with default parameters.
func MakeVM() *VM {
	return &VM{
		MaxStack:       500,
		ext:            make(vmExtMap),
		tla:            make(vmExtMap),
		nativeFuncs:    make(map[string]*NativeFunction),
		ErrorFormatter: &termErrorFormatter{pretty: false, maxStackTraceSize: 20},
		importer:       &FileImporter{},
	}
}

// ExtVar binds a Jsonnet external var to the given value.
func (vm *VM) ExtVar(key string, val string) {
	vm.ext[key] = vmExt{value: val, isCode: false}
}

// ExtCode binds a Jsonnet external code var to the given code.
func (vm *VM) ExtCode(key string, val string) {
	vm.ext[key] = vmExt{value: val, isCode: true}
}

// TLAVar binds a Jsonnet top level argument to the given value.
func (vm *VM) TLAVar(key string, val string) {
	vm.tla[key] = vmExt{value: val, isCode: false}
}

// TLACode binds a Jsonnet top level argument to the given code.
func (vm *VM) TLACode(key string, val string) {
	vm.tla[key] = vmExt{value: val, isCode: true}
}

// Importer sets Importer to use during evaluation (import callback).
func (vm *VM) Importer(i Importer) {
	vm.importer = i
}

type evalKind int

const (
	evalKindRegular evalKind = iota
	evalKindMulti            = iota
	evalKindStream           = iota
)

func (vm *VM) Evaluate(node ast.Node) (output interface{}, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("(CRASH) %v\n%s", r, debug.Stack())
		}
	}()
	return evaluate(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer, vm.StringOutput)
}

func (vm *VM) EvaluateStream(node ast.Node) (output interface{}, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("(CRASH) %v\n%s", r, debug.Stack())
		}
	}()
	return evaluateStream(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer)
}

func (vm *VM) EvaluateMulti(node ast.Node) (output interface{}, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("(CRASH) %v\n%s", r, debug.Stack())
		}
	}()
	return evaluateMulti(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer, vm.StringOutput)
}

func (vm *VM) evaluateSnippet(filename string, snippet string, kind evalKind) (output interface{}, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("(CRASH) %v\n%s", r, debug.Stack())
		}
	}()
	node, err := snippetToAST(filename, snippet)
	if err != nil {
		return "", err
	}
	switch kind {
	case evalKindRegular:
		output, err = evaluate(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer, vm.StringOutput)
	case evalKindMulti:
		output, err = evaluateMulti(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer, vm.StringOutput)
	case evalKindStream:
		output, err = evaluateStream(node, vm.ext, vm.tla, vm.nativeFuncs, vm.MaxStack, vm.importer)
	}
	if err != nil {
		return "", err
	}
	return output, nil
}

// NativeFunction registers a native function.
func (vm *VM) NativeFunction(f *NativeFunction) {
	vm.nativeFuncs[f.Name] = f
}

// EvaluateSnippet evaluates a string containing Jsonnet code, return a JSON
// string.
//
// The filename parameter is only used for error messages.
func (vm *VM) EvaluateSnippet(filename string, snippet string) (json string, formattedErr error) {
	output, err := vm.evaluateSnippet(filename, snippet, evalKindRegular)
	if err != nil {
		return "", errors.New(vm.ErrorFormatter.Format(err))
	}
	json = output.(string)
	return
}

// EvaluateSnippetStream evaluates a string containing Jsonnet code to an array.
// The array is returned as an array of JSON strings.
//
// The filename parameter is only used for error messages.
func (vm *VM) EvaluateSnippetStream(filename string, snippet string) (docs []string, formattedErr error) {
	output, err := vm.evaluateSnippet(filename, snippet, evalKindStream)
	if err != nil {
		return nil, errors.New(vm.ErrorFormatter.Format(err))
	}
	docs = output.([]string)
	return
}

// EvaluateSnippetMulti evaluates a string containing Jsonnet code to key-value
// pairs. The keys are field name strings and the values are JSON strings.
//
// The filename parameter is only used for error messages.
func (vm *VM) EvaluateSnippetMulti(filename string, snippet string) (files map[string]string, formattedErr error) {
	output, err := vm.evaluateSnippet(filename, snippet, evalKindMulti)
	if err != nil {
		return nil, errors.New(vm.ErrorFormatter.Format(err))
	}
	files = output.(map[string]string)
	return
}

func snippetToAST(filename string, snippet string) (ast.Node, error) {
	tokens, err := parser.Lex(filename, snippet)
	if err != nil {
		return nil, err
	}
	node, err := parser.Parse(tokens)
	if err != nil {
		return nil, err
	}
	err = desugarFile(&node)
	if err != nil {
		return nil, err
	}
	err = analyze(node)
	if err != nil {
		return nil, err
	}
	return node, nil
}

// SnippetToAST parses a snippet and returns the resulting AST.
func SnippetToAST(filename string, snippet string) (ast.Node, error) {
	return snippetToAST(filename, snippet)
}

// Version returns the Jsonnet version number.
func Version() string {
	return "v0.12.1"
}
<|endoftext|>


<commit_before>package structomancer

import "reflect"

type (
	structSpec struct {
		rType      reflect.Type
		rKind      reflect.Kind
		tagName    string
		fields     map[string]*FieldSpec
		fieldNames []string // cached
	}
)

func newStructSpec(t reflect.Type, tagName string) *structSpec {
	if !(IsStructType(t) || IsStructPtrType(t)) {
		panic("structomancer: unsupported type " + t.String())
	}

	var st reflect.Type
	if IsStructType(t) {
		st = t
	} else if IsStructPtrType(t) {
		st = t.Elem()
	} else {
		panic("structomancer: unsupported type " + t.String())
	}

	var fields []reflect.StructField
	for i := 0; i < st.NumField(); i++ {
		field := st.Field(i)

		// skip fields marked with "-", just like the json package
		if tag := field.Tag.Get(tagName); tag == "-" {
			continue
		}
		fields = append(fields, field)
	}

	fieldMap := make(map[string]*FieldSpec, len(fields))
	fieldNames := make([]string, len(fields))
	for i, field := range fields {
		fSpec := newFieldSpec(field, tagName)
		fieldMap[fSpec.Nickname()] = fSpec
		fieldNames[i] = fSpec.Nickname()
	}

	return &structSpec{
		rType:      t,
		rKind:      t.Kind(),
		tagName:    tagName,
		fields:     fieldMap,
		fieldNames: fieldNames,
	}
}

func (s *structSpec) Type() reflect.Type {
	return s.rType
}

func (s *structSpec) Kind() reflect.Kind {
	return s.rKind
}

func (s *structSpec) TagName() string {
	return s.tagName
}

func (s *structSpec) Fields() map[string]*FieldSpec {
	return s.fields
}

// Returns a *FieldSpec object representing the given field, or nil if one was not found.
func (s *structSpec) Field(sFieldName string) *FieldSpec {
	if f, exists := s.Fields()[sFieldName]; exists {
		return f
	} else {
		return nil
	}
}

// Returns the number of exported fields recognized by Structomancer in the struct type.
func (s *structSpec) NumFields() int {
	return len(s.Fields())
}

// Returns a slice of the exported field names recognized by Structomancer in the struct type.
func (s *structSpec) FieldNames() []string {
	return s.fieldNames
}
<commit_msg>More idiomatic handling of '-' struct tags<commit_after>package structomancer

import (
	"reflect"
	"strings"
)

type (
	structSpec struct {
		rType      reflect.Type
		rKind      reflect.Kind
		tagName    string
		fields     map[string]*FieldSpec
		fieldNames []string // cached
	}
)

func newStructSpec(t reflect.Type, tagName string) *structSpec {
	if !(IsStructType(t) || IsStructPtrType(t)) {
		panic("structomancer: unsupported type " + t.String())
	}

	var st reflect.Type
	if IsStructType(t) {
		st = t
	} else if IsStructPtrType(t) {
		st = t.Elem()
	} else {
		panic("structomancer: unsupported type " + t.String())
	}

	var fields []reflect.StructField
	for i := 0; i < st.NumField(); i++ {
		field := st.Field(i)

		// skip fields marked with "-", just like the json package
		if tag := field.Tag.Get(tagName); strings.HasPrefix(tag, "-") {
			continue
		}
		fields = append(fields, field)
	}

	fieldMap := make(map[string]*FieldSpec, len(fields))
	fieldNames := make([]string, len(fields))
	for i, field := range fields {
		fSpec := newFieldSpec(field, tagName)
		fieldMap[fSpec.Nickname()] = fSpec
		fieldNames[i] = fSpec.Nickname()
	}

	return &structSpec{
		rType:      t,
		rKind:      t.Kind(),
		tagName:    tagName,
		fields:     fieldMap,
		fieldNames: fieldNames,
	}
}

func (s *structSpec) Type() reflect.Type {
	return s.rType
}

func (s *structSpec) Kind() reflect.Kind {
	return s.rKind
}

func (s *structSpec) TagName() string {
	return s.tagName
}

func (s *structSpec) Fields() map[string]*FieldSpec {
	return s.fields
}

// Returns a *FieldSpec object representing the given field, or nil if one was not found.
func (s *structSpec) Field(sFieldName string) *FieldSpec {
	if f, exists := s.Fields()[sFieldName]; exists {
		return f
	} else {
		return nil
	}
}

// Returns the number of exported fields recognized by Structomancer in the struct type.
func (s *structSpec) NumFields() int {
	return len(s.Fields())
}

// Returns a slice of the exported field names recognized by Structomancer in the struct type.
func (s *structSpec) FieldNames() []string {
	return s.fieldNames
}
<|endoftext|>


<commit_before>package model

import "regexp"

type Route struct {
	Id       int64  `json:"-"`
	Name     string `json:"name"`
	Pattern  string `json:"pattern"`
	Broker   string `json:"broker"`
	From     string `json:"from" db:"fromName"`
	IsActive bool   `json:"is_active"`
	broker   Broker
	regex    *regexp.Regexp
}

func NewRoute(name, pattern string, broker Broker, isActive bool) *Route {
	return &Route{
		Name:     name,
		Pattern:  pattern,
		Broker:   broker.Name(),
		IsActive: isActive,
		broker:   broker,
		regex:    regexp.MustCompile(pattern),
	}
}

func (r *Route) SetBroker(broker Broker) *Route {
	r.broker = broker
	return r
}

func (r *Route) GetBroker() Broker {
	return r.broker
}

func (r *Route) SetFrom(from string) *Route {
	r.From = from
	return r
}

func (r *Route) Match(recipient string) bool {
	return r.IsActive && r.regex.MatchString(recipient)
}
<commit_msg>Fix the db tag of model.Route<commit_after>package model

import "regexp"

type Route struct {
	Id       int64  `json:"-"`
	Name     string `json:"name"`
	Pattern  string `json:"pattern"`
	Broker   string `json:"broker"`
	From     string `json:"from" db:"fromName"`
	IsActive bool   `json:"is_active" db:"isActive"`
	broker   Broker
	regex    *regexp.Regexp
}

func NewRoute(name, pattern string, broker Broker, isActive bool) *Route {
	return &Route{
		Name:     name,
		Pattern:  pattern,
		Broker:   broker.Name(),
		IsActive: isActive,
		broker:   broker,
		regex:    regexp.MustCompile(pattern),
	}
}

func (r *Route) SetBroker(broker Broker) *Route {
	r.broker = broker
	return r
}

func (r *Route) GetBroker() Broker {
	return r.broker
}

func (r *Route) SetFrom(from string) *Route {
	r.From = from
	return r
}

func (r *Route) Match(recipient string) bool {
	return r.IsActive && r.regex.MatchString(recipient)
}
<|endoftext|>


<commit_before>/*
 * The Clammit application intercepts HTTP POST requests with content-type
 * "multipart/form-data", forwards any "file" form-data elements to ClamAV
 * and only forwards the request to the application if ClamAV passes all
 * of these elements as virus-free.
 */
package main

import (
	clamd "github.com/dutchcoders/go-clamd"
	"code.google.com/p/gcfg"
	"clammit/forwarder"
	"net"
	"strings"
	"net/http"
	"net/url"
	"encoding/json"
	"mime"
	"mime/multipart"
	"log"
	"flag"
	"bytes"
	"io"
	"io/ioutil"
	"fmt"
	"os"
	"os/signal"
	"syscall"
)

//
// Configuration structure, designed for gcfg
//
type Config struct {
	App ApplicationConfig      `gcfg:"application"`
}

type ApplicationConfig struct {
	Listen string              `gcfg:"listen"`
	SocketPerms int            `gcfg:"unix-socket-perms"`
	ApplicationURL string      `gcfg:"application-url"`
	ClamdURL string            `gcfg:"clamd-url"`
	Logfile string             `gcfg:"log-file"`
	TestPages bool             `gcfg:"test-pages"`
	Prefix string              `gcfg:"proxy-prefix"`
	Debug bool                 `gcfg:"debug"`
}

//
// The implementation of the ClamAV interceptor
//
type ClamInterceptor struct {
	ClamdURL string
}

//
// Application context
//
type Ctx struct {
	Config Config
	ApplicationURL *url.URL
	ClamInterceptor *ClamInterceptor
	Logger *log.Logger
	Debug bool
	Listener net.Listener
	ActivityChan chan int
	ShuttingDown bool
}

//
// JSON server information response
//
type Info struct {
	ClamdURL string            `json:"clam_server_url"`
	PingResult string          `json:"ping_result"`
	Version string             `json:"version"`
	TestScanVirusResult string `json:"test_scan_virus"`
	TestScanCleanResult string `json:"test_scan_clean"`
}

//
// Global variables and config
//
var ctx *Ctx
var configFile string

func init() {
	flag.StringVar( &configFile, "config", "", "Configuration file" )
}

func main() {
	/*
	 * Construct configuration, set up logging
	 */
	flag.Parse()
	ctx = &Ctx{
		ActivityChan: make(chan int),
		ShuttingDown: false,
    }

	if configFile == "" {
		log.Fatal( "No configuration file specified" )
	}
	if err := gcfg.ReadFileInto( &ctx.Config, configFile ); err != nil {
		log.Fatal( "Configuration read failure:", err )
	}
	if ctx.Config.App.SocketPerms == 0 {
		ctx.Config.App.SocketPerms = 0777;
	}

	startLogging()

	/*
	 * Construct objects, validate the URLs
	 */
	ctx.ApplicationURL = checkURL( ctx.Config.App.ApplicationURL )
	checkURL( ctx.Config.App.ClamdURL )

	ctx.ClamInterceptor =  &ClamInterceptor{ ClamdURL: ctx.Config.App.ClamdURL }

	/*
	 * Set up the HTTP server
	 */
	router := http.NewServeMux()

	router.HandleFunc( "/clammit", infoHandler )
	router.HandleFunc( "/clammit/scan", scanHandler )
	if ctx.Config.App.TestPages {
		fs := http.FileServer( http.Dir( "testfiles" ) )
		router.Handle( "/clammit/test/", http.StripPrefix( "/test/",  fs ) )
	}
	router.HandleFunc( "/", scanForwardHandler )

	log.SetOutput( ioutil.Discard ) // go-clamd has irritating logging, so turn it off

	if listener, err := getListener( ctx.Config.App.Listen, ctx.Config.App.SocketPerms ); err != nil {
		ctx.Logger.Fatal( "Unable to listen on: ", ctx.Config.App.Listen, ", reason: ", err )
	} else {
		ctx.Listener = listener
		beGraceful() // graceful shutdown from here on in
		ctx.Logger.Println( "Listening on", ctx.Config.App.Listen )
		http.Serve( listener, router )
	}
}

/*
 * Starts logging
 */
func startLogging() {
	ctx.Logger = log.New( os.Stdout, "", log.LstdFlags )
	if ctx.Config.App.Logfile != "" {
		w, err := os.OpenFile( ctx.Config.App.Logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660 )
		if err == nil {
			ctx.Logger = log.New( w, "", log.LstdFlags )
		} else {
			log.Fatal( "Failed to open log file", ctx.Config.App.Logfile, ":", err )
		}
	}
}

/*
 * Handles graceful shutdown. Sets ctx.ShuttingDown = true to stop any new
 * requests, then waits for active requests to complete before closing the
 * HTTP listener.
 */
func beGraceful() {
	sigchan := make(chan os.Signal)
	signal.Notify( sigchan, syscall.SIGINT, syscall.SIGTERM )
	go func() {
		activity := 0
		for {
			select {
				case _ = <-sigchan:
					ctx.Logger.Println( "Received termination signal" )
					ctx.ShuttingDown = true
					for activity > 0 {
						ctx.Logger.Printf( "There are %d active requests, waiting", activity )
						i := <-ctx.ActivityChan
						activity += i
					}
					// This will cause main() to continue from http.Serve()
					// it will also clean up the unix socket (if relevant)
					ctx.Listener.Close()
				case i := <-ctx.ActivityChan:
					activity += i
			}
		}
	}()
}

/*
 * Validates the URL is OK (fatal error if not) and returns it
 */
func checkURL( urlString string ) *url.URL {
	parsedURL, err := url.Parse( urlString )
	if err != nil {
		log.Fatal( "Invalid URL:", urlString )
	}
	return parsedURL
}

/*
 * Returns a TCP or Unix socket listener, according to the scheme prefix:
 *
 *   unix:/tmp/foo.sock
 *   tcp::8438
 *   :8438                 - tcp listener
 */
func getListener( address string, socketPerms int ) (listener net.Listener, err error) {
	if address == "" {
		return nil, fmt.Errorf( "No listen address specified" )
	}
	if idx := strings.Index( address, ":" ); idx >= 0 {
		scheme := address[0:idx]
		switch scheme {
			case "tcp", "tcp4" :
				path := address[idx+1:]
				if strings.Index(path,":") == -1 {
					path = ":" + path
				}
				listener, err = net.Listen( scheme, path )
			case "tcp6" : // general form: [host]:port
				path := address[idx+1:]
				if strings.Index(path,"[") != 0 { // port only
					if strings.Index(path,":") != 0 { // no leading :
						path = ":" + path
					}
				}
				listener, err = net.Listen( scheme, path )
			case "unix", "unixpacket" :
				path := address[idx+1:]
				if listener, err = net.Listen( scheme, path ); err == nil {
					os.Chmod( path, os.FileMode(socketPerms) )
				}
			default : // assume TCP4 address
				listener, err = net.Listen( "tcp", address )
		}
	} else { // no scheme, port only specified
		listener, err = net.Listen( "tcp", ":" + address )
	}
	return listener, err
}

/*
 * Handler for /scan
 *
 * Virus checks file and sends response
 */
func scanHandler( w http.ResponseWriter, req *http.Request ) {
	if ctx.ShuttingDown {
		return
	}
	ctx.ActivityChan <- 1
	defer func() { ctx.ActivityChan <- -1 }()

	if ! ctx.ClamInterceptor.Handle( w, req, req.Body ) {
		w.Write( []byte("No virus found") )
	}
}

/*
 * Handler for scan & forward
 *
 * Constructs a forwarder and calls it
 */
func scanForwardHandler( w http.ResponseWriter, req *http.Request ) {
	if ctx.ShuttingDown {
		return
	}
	ctx.ActivityChan <- 1
	defer func() { ctx.ActivityChan <- -1 }()

	fw := forwarder.NewForwarder( ctx.ApplicationURL, ctx.ClamInterceptor )
	fw.SetLogger( ctx.Logger )
	fw.HandleRequest( w, req )
}

/*
 * Handler for /info
 *
 * Validates the Clamd connection
 * Emits the information as a JSON response
 */
func infoHandler( w http.ResponseWriter, req *http.Request ) {
	if ctx.ShuttingDown {
		return
	}
	ctx.ActivityChan <- 1
	defer func() { ctx.ActivityChan <- -1 }()

	c := clamd.NewClamd( ctx.ClamInterceptor.ClamdURL )
	info := &Info{
		ClamdURL: ctx.ClamInterceptor.ClamdURL,
	}
	if err := c.Ping(); err != nil {
		// If we can't ping the Clamd server, no point in making the remaining requests
		info.PingResult = err.Error()
	} else {
		info.PingResult = "Connected to server OK"
		if response, err := c.Version(); err != nil {
			info.Version = err.Error();
		} else {
			for s := range response {
				info.Version += s
			}
		}
		/*
		 * Validate the Clamd response for a viral string
		 */
		reader := bytes.NewReader( clamd.EICAR )
		if response, err := c.ScanStream( reader ); err != nil {
			info.TestScanVirusResult = err.Error()
		} else {
			for s := range response {
				info.TestScanVirusResult += s
			}
		}
		/*
		 * Validate the Clamd response for a non-viral string
		 */
		reader = bytes.NewReader( []byte("foo bar mcgrew") )
		if response, err := c.ScanStream( reader ); err != nil {
			info.TestScanCleanResult = err.Error()
		} else {
			for s := range response {
				info.TestScanCleanResult += s
			}
		}
	}
	// Aaaand return
	s, _ := json.Marshal(info)
	w.Header().Set("Content-Type", "application/json")
	w.Write( []byte(s) )
}

/*
 * Interceptor implementation for Clamd
 *
 * Runs a multi-part parser across the request body and sends all file contents to Clamd
 *
 * returns True if the body contains a virus
 */
func (c *ClamInterceptor) Handle( w http.ResponseWriter, req *http.Request, body io.Reader ) bool {
	//
	// Don't care unless it's a post
	//
	if req.Method != "POST" && req.Method != "PUT" {
		return false
	}

	//
	// Find any attachments
	//
	_, params, err := mime.ParseMediaType( req.Header.Get( "Content-Type" ) )
	if err != nil {
		return false
	}
	boundary := params["boundary"]
	if boundary == "" {
		return false
	}

	reader := multipart.NewReader( body, boundary )

	//
	// Scan them
	//
	var broken_err error

	for {
		if part, err := reader.NextPart(); err != nil {
			break // all done
		} else {
			if part.FileName() != "" {
				defer part.Close()
				ctx.Logger.Println( "Scanning",part.FileName() )
				if hasVirus, err := c.Scan( part ); err != nil {
					broken_err = err
				} else if hasVirus {
					w.WriteHeader( 418 )
					w.Write( []byte(fmt.Sprintf( "File %s has a virus!", part.FileName() ) ) )
					return true
				}
			}
		}
	}

	//
	// If failure, we bomb out here
	//
	if broken_err != nil {
		w.WriteHeader( 500 )
		w.Write( []byte(fmt.Sprintf( "Unable to scan a file: %s", broken_err.Error()) ) )
		return true
	}

	return false
}

/*
 * This function performs the actual virus scan
 */
func (c *ClamInterceptor) Scan( reader io.Reader ) (bool, error) {

	clam := clamd.NewClamd( c.ClamdURL )

	response, err := clam.ScanStream( reader )
	if err != nil {
		return false, err
	}
	hasVirus := false
	for s := range response {
		if s != "stream: OK" {
			ctx.Logger.Printf("%v %v\n", s )
			hasVirus = true
		}
	}

	ctx.Logger.Println( "Result of scan:", hasVirus )

	return hasVirus, nil
}
<commit_msg>A bit of tidying<commit_after>/*
 * The Clammit application intercepts HTTP POST requests with content-type
 * "multipart/form-data", forwards any "file" form-data elements to ClamAV
 * and only forwards the request to the application if ClamAV passes all
 * of these elements as virus-free.
 */
package main

import (
	clamd "github.com/dutchcoders/go-clamd"
	"code.google.com/p/gcfg"
	"clammit/forwarder"
	"net"
	"strings"
	"net/http"
	"net/url"
	"encoding/json"
	"mime"
	"mime/multipart"
	"log"
	"flag"
	"bytes"
	"io"
	"io/ioutil"
	"fmt"
	"os"
	"os/signal"
	"syscall"
)

//
// Configuration structure, designed for gcfg
//
type Config struct {
	App ApplicationConfig      `gcfg:"application"`
}

type ApplicationConfig struct {
	Listen string              `gcfg:"listen"`
	SocketPerms int            `gcfg:"unix-socket-perms"`
	ApplicationURL string      `gcfg:"application-url"`
	ClamdURL string            `gcfg:"clamd-url"`
	Logfile string             `gcfg:"log-file"`
	TestPages bool             `gcfg:"test-pages"`
	Debug bool                 `gcfg:"debug"`
}

//
// The implementation of the ClamAV interceptor
//
type ClamInterceptor struct {
	ClamdURL string
}

//
// Application context
//
type Ctx struct {
	Config Config
	ApplicationURL *url.URL
	ClamInterceptor *ClamInterceptor
	Logger *log.Logger
	Debug bool
	Listener net.Listener
	ActivityChan chan int
	ShuttingDown bool
}

//
// JSON server information response
//
type Info struct {
	ClamdURL string            `json:"clam_server_url"`
	PingResult string          `json:"ping_result"`
	Version string             `json:"version"`
	TestScanVirusResult string `json:"test_scan_virus"`
	TestScanCleanResult string `json:"test_scan_clean"`
}

//
// Global variables and config
//
var ctx *Ctx
var configFile string

func init() {
	flag.StringVar( &configFile, "config", "", "Configuration file" )
}

func main() {
	/*
	 * Construct configuration, set up logging
	 */
	flag.Parse()
	ctx = &Ctx{
		ActivityChan: make(chan int),
		ShuttingDown: false,
    }

	if configFile == "" {
		log.Fatal( "No configuration file specified" )
	}
	if err := gcfg.ReadFileInto( &ctx.Config, configFile ); err != nil {
		log.Fatal( "Configuration read failure:", err )
	}
	if ctx.Config.App.SocketPerms == 0 {
		ctx.Config.App.SocketPerms = 0777;
	}

	startLogging()

	/*
	 * Construct objects, validate the URLs
	 */
	ctx.ApplicationURL = checkURL( ctx.Config.App.ApplicationURL )
	checkURL( ctx.Config.App.ClamdURL )

	ctx.ClamInterceptor =  &ClamInterceptor{ ClamdURL: ctx.Config.App.ClamdURL }

	/*
	 * Set up the HTTP server
	 */
	router := http.NewServeMux()

	router.HandleFunc( "/clammit", infoHandler )
	router.HandleFunc( "/clammit/scan", scanHandler )
	if ctx.Config.App.TestPages {
		fs := http.FileServer( http.Dir( "testfiles" ) )
		router.Handle( "/clammit/test/", http.StripPrefix( "/test/",  fs ) )
	}
	router.HandleFunc( "/", scanForwardHandler )

	log.SetOutput( ioutil.Discard ) // go-clamd has irritating logging, so turn it off

	if listener, err := getListener( ctx.Config.App.Listen, ctx.Config.App.SocketPerms ); err != nil {
		ctx.Logger.Fatal( "Unable to listen on: ", ctx.Config.App.Listen, ", reason: ", err )
	} else {
		ctx.Listener = listener
		beGraceful() // graceful shutdown from here on in
		ctx.Logger.Println( "Listening on", ctx.Config.App.Listen )
		http.Serve( listener, router )
	}
}

/*
 * Starts logging
 */
func startLogging() {
	ctx.Logger = log.New( os.Stdout, "", log.LstdFlags )
	if ctx.Config.App.Logfile != "" {
		w, err := os.OpenFile( ctx.Config.App.Logfile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660 )
		if err == nil {
			ctx.Logger = log.New( w, "", log.LstdFlags )
		} else {
			log.Fatal( "Failed to open log file", ctx.Config.App.Logfile, ":", err )
		}
	}
}

/*
 * Handles graceful shutdown. Sets ctx.ShuttingDown = true to stop any new
 * requests, then waits for active requests to complete before closing the
 * HTTP listener.
 */
func beGraceful() {
	sigchan := make(chan os.Signal)
	signal.Notify( sigchan, syscall.SIGINT, syscall.SIGTERM )
	go func() {
		activity := 0
		for {
			select {
				case _ = <-sigchan:
					ctx.Logger.Println( "Received termination signal" )
					ctx.ShuttingDown = true
					for activity > 0 {
						ctx.Logger.Printf( "There are %d active requests, waiting", activity )
						i := <-ctx.ActivityChan
						activity += i
					}
					// This will cause main() to continue from http.Serve()
					// it will also clean up the unix socket (if relevant)
					ctx.Listener.Close()
				case i := <-ctx.ActivityChan:
					activity += i
			}
		}
	}()
}

/*
 * Validates the URL is OK (fatal error if not) and returns it
 */
func checkURL( urlString string ) *url.URL {
	parsedURL, err := url.Parse( urlString )
	if err != nil {
		log.Fatal( "Invalid URL:", urlString )
	}
	return parsedURL
}

/*
 * Returns a TCP or Unix socket listener, according to the scheme prefix:
 *
 *   unix:/tmp/foo.sock
 *   tcp::8438
 *   :8438                 - tcp listener
 */
func getListener( address string, socketPerms int ) (listener net.Listener, err error) {
	if address == "" {
		return nil, fmt.Errorf( "No listen address specified" )
	}
	if idx := strings.Index( address, ":" ); idx >= 0 {
		scheme := address[0:idx]
		switch scheme {
			case "tcp", "tcp4" :
				path := address[idx+1:]
				if strings.Index(path,":") == -1 {
					path = ":" + path
				}
				listener, err = net.Listen( scheme, path )
			case "tcp6" : // general form: [host]:port
				path := address[idx+1:]
				if strings.Index(path,"[") != 0 { // port only
					if strings.Index(path,":") != 0 { // no leading :
						path = ":" + path
					}
				}
				listener, err = net.Listen( scheme, path )
			case "unix", "unixpacket" :
				path := address[idx+1:]
				if listener, err = net.Listen( scheme, path ); err == nil {
					os.Chmod( path, os.FileMode(socketPerms) )
				}
			default : // assume TCP4 address
				listener, err = net.Listen( "tcp", address )
		}
	} else { // no scheme, port only specified
		listener, err = net.Listen( "tcp", ":" + address )
	}
	return listener, err
}

/*
 * Handler for /scan
 *
 * Virus checks file and sends response
 */
func scanHandler( w http.ResponseWriter, req *http.Request ) {
	if ctx.ShuttingDown {
		return
	}
	ctx.ActivityChan <- 1
	defer func() { ctx.ActivityChan <- -1 }()

	if ! ctx.ClamInterceptor.Handle( w, req, req.Body ) {
		w.Write( []byte("No virus found") )
	}
}

/*
 * Handler for scan & forward
 *
 * Constructs a forwarder and calls it
 */
func scanForwardHandler( w http.ResponseWriter, req *http.Request ) {
	if ctx.ShuttingDown {
		return
	}
	ctx.ActivityChan <- 1
	defer func() { ctx.ActivityChan <- -1 }()

	fw := forwarder.NewForwarder( ctx.ApplicationURL, ctx.ClamInterceptor )
	fw.SetLogger( ctx.Logger )
	fw.HandleRequest( w, req )
}

/*
 * Handler for /info
 *
 * Validates the Clamd connection
 * Emits the information as a JSON response
 */
func infoHandler( w http.ResponseWriter, req *http.Request ) {
	if ctx.ShuttingDown {
		return
	}
	ctx.ActivityChan <- 1
	defer func() { ctx.ActivityChan <- -1 }()

	c := clamd.NewClamd( ctx.ClamInterceptor.ClamdURL )
	info := &Info{
		ClamdURL: ctx.ClamInterceptor.ClamdURL,
	}
	if err := c.Ping(); err != nil {
		// If we can't ping the Clamd server, no point in making the remaining requests
		info.PingResult = err.Error()
	} else {
		info.PingResult = "Connected to server OK"
		if response, err := c.Version(); err != nil {
			info.Version = err.Error();
		} else {
			for s := range response {
				info.Version += s
			}
		}
		/*
		 * Validate the Clamd response for a viral string
		 */
		reader := bytes.NewReader( clamd.EICAR )
		if response, err := c.ScanStream( reader ); err != nil {
			info.TestScanVirusResult = err.Error()
		} else {
			for s := range response {
				info.TestScanVirusResult += s
			}
		}
		/*
		 * Validate the Clamd response for a non-viral string
		 */
		reader = bytes.NewReader( []byte("foo bar mcgrew") )
		if response, err := c.ScanStream( reader ); err != nil {
			info.TestScanCleanResult = err.Error()
		} else {
			for s := range response {
				info.TestScanCleanResult += s
			}
		}
	}
	// Aaaand return
	s, _ := json.Marshal(info)
	w.Header().Set("Content-Type", "application/json")
	w.Write( []byte(s) )
}

/*
 * Interceptor implementation for Clamd
 *
 * Runs a multi-part parser across the request body and sends all file contents to Clamd
 *
 * returns True if the body contains a virus
 */
func (c *ClamInterceptor) Handle( w http.ResponseWriter, req *http.Request, body io.Reader ) bool {
	//
	// Don't care unless it's a post
	//
	if req.Method != "POST" && req.Method != "PUT" {
		return false
	}

	//
	// Find any attachments
	//
	_, params, err := mime.ParseMediaType( req.Header.Get( "Content-Type" ) )
	if err != nil {
		return false
	}
	boundary := params["boundary"]
	if boundary == "" {
		return false
	}

	reader := multipart.NewReader( body, boundary )

	//
	// Scan them
	//
	var broken_err error

	for {
		if part, err := reader.NextPart(); err != nil {
			break // all done
		} else {
			if part.FileName() != "" {
				defer part.Close()
				ctx.Logger.Println( "Scanning",part.FileName() )
				if hasVirus, err := c.Scan( part ); err != nil {
					broken_err = err
				} else if hasVirus {
					w.WriteHeader( 418 )
					w.Write( []byte(fmt.Sprintf( "File %s has a virus!", part.FileName() ) ) )
					return true
				}
			}
		}
	}

	//
	// If failure, we bomb out here
	//
	if broken_err != nil {
		w.WriteHeader( 500 )
		w.Write( []byte(fmt.Sprintf( "Unable to scan a file: %s", broken_err.Error()) ) )
		return true
	}

	return false
}

/*
 * This function performs the actual virus scan
 */
func (c *ClamInterceptor) Scan( reader io.Reader ) (bool, error) {

	clam := clamd.NewClamd( c.ClamdURL )

	response, err := clam.ScanStream( reader )
	if err != nil {
		return false, err
	}
	hasVirus := false
	for s := range response {
		if s != "stream: OK" {
			ctx.Logger.Printf("%v %v\n", s )
			hasVirus = true
		}
	}

	ctx.Logger.Println( "Result of scan:", hasVirus )

	return hasVirus, nil
}
<|endoftext|>


<commit_before>/**
 * Copyright 2015 The AMP HTML Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS-IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Forwards client-side errors to Cloud Error Reporting.

package errortracker

import (
	"fmt"
	"math/rand"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
	"google.golang.org/cloud"
	"google.golang.org/cloud/logging"
)

// Magic fields documented here
// https://cloud.google.com/error-reporting/#error_message_fields
type ErrorRequestMeta struct {
	HTTPReferrer  string `json:"http_referrer,omitempty"`
	HTTPUserAgent string `json:"http_user_agent,omitempty"`
}

type ErrorRequest struct {
	URL    string            `json:"url,omitempty"`
	Method string            `json:"method,omitempty"`
	Meta   *ErrorRequestMeta `json:"meta,omitempty"`
}

type ErrorEvent struct {
	Application string `json:"application,omitempty"`
	AppID       string `json:"app_id,omitempty"`
	Environment string `json:"environment,omitempty"`
	Version     string `json:"version,omitempty"`

	Message   string `json:"message,omitempty"`
	Exception string `json:"exception,omitempty"`

	Request *ErrorRequest `json:"request,omitempty"`

	Filename  string `json:"filename,omitempty"`
	Line      int32  `json:"line,omitempty"`
	Classname string `json:"classname,omitempty"`
	Function  string `json:"function,omitempty"`
	Severity  string `json:"severity,omitempty"`
}

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
	http.HandleFunc("/r", handle)
}

// Get an auth context for logging RPC.
func cloudAuthContext(r *http.Request) (context.Context, error) {
	c := appengine.NewContext(r)
	hc := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.AppEngineTokenSource(c, logging.Scope),
			Base:   &urlfetch.Transport{Context: c},
		},
	}
	return cloud.WithContext(c, appengine.AppID(c), hc), nil
}

func handle(w http.ResponseWriter, r *http.Request) {
	c, _ := cloudAuthContext(r)
	logc, err := logging.NewClient(c, appengine.AppID(c), "javascript.errors")
	if err != nil {
		http.Error(w, "Cannot connect to Google Cloud Logging",
			http.StatusInternalServerError)
		log.Errorf(c, "Cannot connect to Google Cloud Logging: %v", err)
		return
	}
	// Note: Error Reporting currently ignores non-GCE and non-AWS logs.
	logc.ServiceName = "compute.googleapis.com"
	logc.CommonLabels = map[string]string{
		"compute.googleapis.com/resource_type": "logger",
		"compute.googleapis.com/resource_id":   "errors"}

	// Fill query params into JSON struct.
	line, _ := strconv.Atoi(r.URL.Query().Get("l"))
	errorType := "default"
	isUserError := false
	if r.URL.Query().Get("a") == "1" {
		errorType = "assert"
		isUserError = true
	}
	// By default we log as "INFO" severity, because reports are very spammy
	severity := "INFO"
	level := logging.Info
	// But if the request comes from the cache (and thus only from valid AMP
	// docs) we log as "ERROR".
	isCdn := false
	if strings.HasPrefix(r.Referer(), "https://cdn.ampproject.org/") ||
		strings.Contains(r.Referer(), ".cdn.ampproject.org/") ||
		strings.Contains(r.Referer(), ".ampproject.net/") {
		severity = "ERROR"
		level = logging.Error
		errorType += "-cdn"
		isCdn = true
	} else {
		errorType += "-origin"
	}
	is3p := false
	runtime := r.URL.Query().Get("rt")
	if runtime != "" {
		errorType += "-" + runtime
		if runtime == "inabox" {
			severity = "ERROR"
			level = logging.Error
		}
		if runtime == "3p" {
			is3p = true
		}
	} else {
		if r.URL.Query().Get("3p") == "1" {
			is3p = true
			errorType += "-3p"
		} else {
			errorType += "-1p"
		}
	}
	isCanary := false
	if r.URL.Query().Get("ca") == "1" {
		errorType += "-canary"
		isCanary = true
	}
	if r.URL.Query().Get("ex") == "1" {
		errorType += "-expected"
	}
	sample := rand.Float64()
	throttleRate := 0.01

	if isCanary {
		throttleRate = 1.0 // Explicitly log all canary errors.
	} else if is3p {
		throttleRate = 0.1
	} else if isCdn {
		throttleRate = 0.1
	}

	if isUserError {
		throttleRate = throttleRate / 10
	}

	if !(sample <= throttleRate) {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		fmt.Fprintln(w, "THROTTLED\n")
		return
	}
	if rand.Float64() < 0.1 {
		urlString := strings.Replace(r.URL.String(), "amp-error-reporting", "amp-error-reporting-js", 1)
		client := urlfetch.Client(c)
		_, err := client.Get(urlString)
		if err != nil {
			log.Errorf(c, "Error forwarding report: %v", err)
		}
	}

	exception := r.URL.Query().Get("s")
	// If format does not end with :\d+ truncate up to the last newline.
	if !regexp.MustCompile(`:\d+$`).MatchString(exception) {
		exception = string(regexp.MustCompile(`\n.*$`).ReplaceAllString(exception, ""))
	}

	event := &ErrorEvent{
		Message:     r.URL.Query().Get("m"),
		Exception:   exception,
		Version:     errorType + "-" + r.URL.Query().Get("v"),
		Environment: "prod",
		Application: errorType,
		AppID:       appengine.AppID(c),
		Filename:    r.URL.String(),
		Line:        int32(line),
		Classname:   r.URL.Query().Get("el"),
		Severity:    severity,
	}

	if event.Message == "" && event.Exception == "" {
		http.Error(w, "One of 'message' or 'exception' must be present.",
			http.StatusBadRequest)
		log.Errorf(c, "Malformed request: %v", event)
		return
	}

	if IsFilteredMessageOrException(event) {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusNoContent)
		fmt.Fprintln(w, "IGNORE\n")
		return
	}

	// Don't log testing traffic in production
	if event.Version == "$internalRuntimeVersion$" {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	event.Request = &ErrorRequest{
		URL: r.Referer(),
	}
	event.Request.Meta = &ErrorRequestMeta{
		HTTPReferrer:  r.URL.Query().Get("r"),
		HTTPUserAgent: r.UserAgent(),
		// Intentionally not logged.
		// RemoteIP:   r.RemoteAddr,
	}

	err = logc.LogSync(logging.Entry{
		Time:    time.Now().UTC(),
		Payload: event,
		Level:   level,
	})

	if err != nil {
		http.Error(w, "Cannot write to Google Cloud Logging",
			http.StatusInternalServerError)
		log.Errorf(c, "Cannot write to Google Cloud Logging: %v", err)
		return
	}

	// When debug param is present, return a document. This is nicer because
	// browsers otherwise revert the URL during manual testing.
	if r.URL.Query().Get("debug") == "1" {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		fmt.Fprintln(w, "OK\n")
		fmt.Fprintln(w, event)
	} else {
		w.WriteHeader(http.StatusNoContent)
	}
}

func IsFilteredMessageOrException(event *ErrorEvent) bool {
	filteredMessages := [...]string{
		"stop_youtube",
		"null%20is%20not%20an%20object%20(evaluating%20%27elt.parentNode%27)",
	}
	for _, msg := range filteredMessages {
		if strings.Contains(event.Message, msg) ||
			strings.Contains(event.Exception, msg) {
			return true
		}
	}
	return false
}
<commit_msg>Error tracker diversion: Send referrer and user-agent (#10979)<commit_after>/**
 * Copyright 2015 The AMP HTML Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS-IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Forwards client-side errors to Cloud Error Reporting.

package errortracker

import (
	"fmt"
	"math/rand"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/urlfetch"
	"google.golang.org/cloud"
	"google.golang.org/cloud/logging"
)

// Magic fields documented here
// https://cloud.google.com/error-reporting/#error_message_fields
type ErrorRequestMeta struct {
	HTTPReferrer  string `json:"http_referrer,omitempty"`
	HTTPUserAgent string `json:"http_user_agent,omitempty"`
}

type ErrorRequest struct {
	URL    string            `json:"url,omitempty"`
	Method string            `json:"method,omitempty"`
	Meta   *ErrorRequestMeta `json:"meta,omitempty"`
}

type ErrorEvent struct {
	Application string `json:"application,omitempty"`
	AppID       string `json:"app_id,omitempty"`
	Environment string `json:"environment,omitempty"`
	Version     string `json:"version,omitempty"`

	Message   string `json:"message,omitempty"`
	Exception string `json:"exception,omitempty"`

	Request *ErrorRequest `json:"request,omitempty"`

	Filename  string `json:"filename,omitempty"`
	Line      int32  `json:"line,omitempty"`
	Classname string `json:"classname,omitempty"`
	Function  string `json:"function,omitempty"`
	Severity  string `json:"severity,omitempty"`
}

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
	http.HandleFunc("/r", handle)
}

// Get an auth context for logging RPC.
func cloudAuthContext(r *http.Request) (context.Context, error) {
	c := appengine.NewContext(r)
	hc := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.AppEngineTokenSource(c, logging.Scope),
			Base:   &urlfetch.Transport{Context: c},
		},
	}
	return cloud.WithContext(c, appengine.AppID(c), hc), nil
}

func handle(w http.ResponseWriter, r *http.Request) {
	c, _ := cloudAuthContext(r)
	logc, err := logging.NewClient(c, appengine.AppID(c), "javascript.errors")
	if err != nil {
		http.Error(w, "Cannot connect to Google Cloud Logging",
			http.StatusInternalServerError)
		log.Errorf(c, "Cannot connect to Google Cloud Logging: %v", err)
		return
	}
	// Note: Error Reporting currently ignores non-GCE and non-AWS logs.
	logc.ServiceName = "compute.googleapis.com"
	logc.CommonLabels = map[string]string{
		"compute.googleapis.com/resource_type": "logger",
		"compute.googleapis.com/resource_id":   "errors"}

	// Fill query params into JSON struct.
	line, _ := strconv.Atoi(r.URL.Query().Get("l"))
	errorType := "default"
	isUserError := false
	if r.URL.Query().Get("a") == "1" {
		errorType = "assert"
		isUserError = true
	}
	// By default we log as "INFO" severity, because reports are very spammy
	severity := "INFO"
	level := logging.Info
	// But if the request comes from the cache (and thus only from valid AMP
	// docs) we log as "ERROR".
	isCdn := false
	if strings.HasPrefix(r.Referer(), "https://cdn.ampproject.org/") ||
		strings.Contains(r.Referer(), ".cdn.ampproject.org/") ||
		strings.Contains(r.Referer(), ".ampproject.net/") {
		severity = "ERROR"
		level = logging.Error
		errorType += "-cdn"
		isCdn = true
	} else {
		errorType += "-origin"
	}
	is3p := false
	runtime := r.URL.Query().Get("rt")
	if runtime != "" {
		errorType += "-" + runtime
		if runtime == "inabox" {
			severity = "ERROR"
			level = logging.Error
		}
		if runtime == "3p" {
			is3p = true
		}
	} else {
		if r.URL.Query().Get("3p") == "1" {
			is3p = true
			errorType += "-3p"
		} else {
			errorType += "-1p"
		}
	}
	isCanary := false
	if r.URL.Query().Get("ca") == "1" {
		errorType += "-canary"
		isCanary = true
	}
	if r.URL.Query().Get("ex") == "1" {
		errorType += "-expected"
	}
	sample := rand.Float64()
	throttleRate := 0.01

	if isCanary {
		throttleRate = 1.0 // Explicitly log all canary errors.
	} else if is3p {
		throttleRate = 0.1
	} else if isCdn {
		throttleRate = 0.1
	}

	if isUserError {
		throttleRate = throttleRate / 10
	}

	if !(sample <= throttleRate) {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		fmt.Fprintln(w, "THROTTLED\n")
		return
	}
	if rand.Float64() < 0.1 {
		urlString := strings.Replace(r.URL.String(), "amp-error-reporting", "amp-error-reporting-js", 1)
		client := urlfetch.Client(c)
		req, err := http.NewRequest("GET", urlString, nil)
		if err == nil {
			req.Header.Set("User-Agent", r.UserAgent())
			req.Header.Set("Referer", r.Referer())
			_, err := client.Do(req)
			if err != nil {
				log.Errorf(c, "Error forwarding report: %v", err)
			}
		} else {
			log.Errorf(c, "Error making forwarding request: %v", err)
		}
	}

	exception := r.URL.Query().Get("s")
	// If format does not end with :\d+ truncate up to the last newline.
	if !regexp.MustCompile(`:\d+$`).MatchString(exception) {
		exception = string(regexp.MustCompile(`\n.*$`).ReplaceAllString(exception, ""))
	}

	event := &ErrorEvent{
		Message:     r.URL.Query().Get("m"),
		Exception:   exception,
		Version:     errorType + "-" + r.URL.Query().Get("v"),
		Environment: "prod",
		Application: errorType,
		AppID:       appengine.AppID(c),
		Filename:    r.URL.String(),
		Line:        int32(line),
		Classname:   r.URL.Query().Get("el"),
		Severity:    severity,
	}

	if event.Message == "" && event.Exception == "" {
		http.Error(w, "One of 'message' or 'exception' must be present.",
			http.StatusBadRequest)
		log.Errorf(c, "Malformed request: %v", event)
		return
	}

	if IsFilteredMessageOrException(event) {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusNoContent)
		fmt.Fprintln(w, "IGNORE\n")
		return
	}

	// Don't log testing traffic in production
	if event.Version == "$internalRuntimeVersion$" {
		w.WriteHeader(http.StatusNoContent)
		return
	}

	event.Request = &ErrorRequest{
		URL: r.Referer(),
	}
	event.Request.Meta = &ErrorRequestMeta{
		HTTPReferrer:  r.URL.Query().Get("r"),
		HTTPUserAgent: r.UserAgent(),
		// Intentionally not logged.
		// RemoteIP:   r.RemoteAddr,
	}

	err = logc.LogSync(logging.Entry{
		Time:    time.Now().UTC(),
		Payload: event,
		Level:   level,
	})

	if err != nil {
		http.Error(w, "Cannot write to Google Cloud Logging",
			http.StatusInternalServerError)
		log.Errorf(c, "Cannot write to Google Cloud Logging: %v", err)
		return
	}

	// When debug param is present, return a document. This is nicer because
	// browsers otherwise revert the URL during manual testing.
	if r.URL.Query().Get("debug") == "1" {
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		fmt.Fprintln(w, "OK\n")
		fmt.Fprintln(w, event)
	} else {
		w.WriteHeader(http.StatusNoContent)
	}
}

func IsFilteredMessageOrException(event *ErrorEvent) bool {
	filteredMessages := [...]string{
		"stop_youtube",
		"null%20is%20not%20an%20object%20(evaluating%20%27elt.parentNode%27)",
	}
	for _, msg := range filteredMessages {
		if strings.Contains(event.Message, msg) ||
			strings.Contains(event.Exception, msg) {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"regexp"

	valid "gopkg.in/go-playground/validator.v9"
)

var (
	regexDBName  = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\ ]+$`)
	regexFolder  = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,//]+$`)
	regexPGTable = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_]+$`)

	// For input validation
	Validate *valid.Validate
)

func init() {
	// Load validation code
	Validate = valid.New()
	Validate.RegisterValidation("dbname", checkDBName)
	Validate.RegisterValidation("folder", checkFolder)
	Validate.RegisterValidation("pgtable", checkPGTableName)
}

// Custom validation function for SQLite database names.
// At the moment it just allows alphanumeric and ".-_ " chars, though it should probably be extended to cover any
// valid file name
func checkDBName(fl valid.FieldLevel) bool {
	return regexDBName.MatchString(fl.Field().String())
}

// Custom validation function for folder names.
// At the moment it allows alphanumeric and ".-_/" chars.  Will probably need more characters added.
func checkFolder(fl valid.FieldLevel) bool {
	return regexFolder.MatchString(fl.Field().String())
}

// Custom validation function for PostgreSQL table names.
// At the moment it just allows alphanumeric and ".-_" chars (may need to be expanded out at some point).
func checkPGTableName(fl valid.FieldLevel) bool {
	return regexPGTable.MatchString(fl.Field().String())
}

// Checks a username against the list of reserved ones.
func ReservedUsernamesCheck(userName string) error {
	reserved := []string{"about", "admin", "blog", "dbhub", "download", "downloadcsv", "legal", "login", "logout",
		"mail", "news", "pref", "printer", "public", "reference", "register", "root", "star", "stars",
		"system", "table", "upload", "uploaddata", "vis"}
	for _, word := range reserved {
		if userName == word {
			return fmt.Errorf("That username is not available: %s\n", userName)
		}
	}

	return nil
}

// Validate the database name.
func ValidateDB(dbName string) error {
	err := Validate.Var(dbName, "required,dbname,min=1,max=256") // 256 char limit seems reasonable
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided email address.
func ValidateEmail(email string) error {
	err := Validate.Var(email, "required,email")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided folder name.
func ValidateFolder(folder string) error {
	err := Validate.Var(folder, "folder,max=127")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided PostgreSQL table name.
func ValidatePGTable(table string) error {
	// TODO: Improve this to work with all valid SQLite identifiers
	// TODO  Not seeing a definitive reference page for SQLite yet, so using the PostgreSQL one is
	// TODO  probably ok as a fallback:
	// TODO      https://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
	// TODO: Should we exclude SQLite internal tables too? (eg "sqlite_*" https://sqlite.org/lang_createtable.html)
	err := Validate.Var(table, "required,pgtable,max=63")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided username.
func ValidateUser(user string) error {
	err := Validate.Var(user, "required,alphanum,min=3,max=63")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided user and database name.
func ValidateUserDB(user string, db string) error {
	err := ValidateUser(user)
	if err != nil {
		return err
	}

	err = ValidateDB(db)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided user, database, and table name.
func ValidateUserDBTable(user string, db string, table string) error {
	err := ValidateUserDB(user, db)
	if err != nil {
		return err
	}

	err = ValidatePGTable(table)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided username and email address.
func ValidateUserEmail(user string, email string) error {
	err := ValidateUser(user)
	if err != nil {
		return err
	}

	err = ValidateEmail(email)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Allow "-_." characters in usernames<commit_after>package common

import (
	"fmt"
	"regexp"

	valid "gopkg.in/go-playground/validator.v9"
)

var (
	regexDBName   = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,\ ]+$`)
	regexFolder   = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_,//]+$`)
	regexPGTable  = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_]+$`)
	regexUsername = regexp.MustCompile(`^[a-z,A-Z,0-9,\.,\-,\_]+$`)

	// For input validation
	Validate *valid.Validate
)

func init() {
	// Load validation code
	Validate = valid.New()
	Validate.RegisterValidation("dbname", checkDBName)
	Validate.RegisterValidation("folder", checkFolder)
	Validate.RegisterValidation("pgtable", checkPGTableName)
	Validate.RegisterValidation("username", checkUsername)
}

// Custom validation function for SQLite database names.
// At the moment it just allows alphanumeric and ".-_ " chars, though it should probably be extended to cover any
// valid file name
func checkDBName(fl valid.FieldLevel) bool {
	return regexDBName.MatchString(fl.Field().String())
}

// Custom validation function for folder names.
// At the moment it allows alphanumeric and ".-_/" chars.  Will probably need more characters added.
func checkFolder(fl valid.FieldLevel) bool {
	return regexFolder.MatchString(fl.Field().String())
}

// Custom validation function for PostgreSQL table names.
// At the moment it just allows alphanumeric and ".-_" chars (may need to be expanded out at some point).
func checkPGTableName(fl valid.FieldLevel) bool {
	return regexPGTable.MatchString(fl.Field().String())
}

// Custom validation function for Usernames.
// At the moment it just allows alphanumeric and ".-_" chars (may need to be expanded out at some point).
func checkUsername(fl valid.FieldLevel) bool {
	return regexUsername.MatchString(fl.Field().String())
}

// Checks a username against the list of reserved ones.
func ReservedUsernamesCheck(userName string) error {
	reserved := []string{"about", "admin", "blog", "dbhub", "download", "downloadcsv", "legal", "login", "logout",
		"mail", "news", "pref", "printer", "public", "reference", "register", "root", "star", "stars",
		"system", "table", "upload", "uploaddata", "vis"}
	for _, word := range reserved {
		if userName == word {
			return fmt.Errorf("That username is not available: %s\n", userName)
		}
	}

	return nil
}

// Validate the database name.
func ValidateDB(dbName string) error {
	err := Validate.Var(dbName, "required,dbname,min=1,max=256") // 256 char limit seems reasonable
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided email address.
func ValidateEmail(email string) error {
	err := Validate.Var(email, "required,email")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided folder name.
func ValidateFolder(folder string) error {
	err := Validate.Var(folder, "folder,max=127")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided PostgreSQL table name.
func ValidatePGTable(table string) error {
	// TODO: Improve this to work with all valid SQLite identifiers
	// TODO  Not seeing a definitive reference page for SQLite yet, so using the PostgreSQL one is
	// TODO  probably ok as a fallback:
	// TODO      https://www.postgresql.org/docs/current/static/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
	// TODO: Should we exclude SQLite internal tables too? (eg "sqlite_*" https://sqlite.org/lang_createtable.html)
	err := Validate.Var(table, "required,pgtable,max=63")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided username.
func ValidateUser(user string) error {
	err := Validate.Var(user, "required,username,min=2,max=63")
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided user and database name.
func ValidateUserDB(user string, db string) error {
	err := ValidateUser(user)
	if err != nil {
		return err
	}

	err = ValidateDB(db)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided user, database, and table name.
func ValidateUserDBTable(user string, db string, table string) error {
	err := ValidateUserDB(user, db)
	if err != nil {
		return err
	}

	err = ValidatePGTable(table)
	if err != nil {
		return err
	}

	return nil
}

// Validate the provided username and email address.
func ValidateUserEmail(user string, email string) error {
	err := ValidateUser(user)
	if err != nil {
		return err
	}

	err = ValidateEmail(email)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	_ "net/http/pprof"

	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/bitly/go-nsq"
	"github.com/bitly/nsq/internal/app"
	"github.com/raintank/raintank-metric/eventdef"
	"github.com/raintank/raintank-metric/setting"
)

var (
	showVersion = flag.Bool("version", false, "print version string")

	topic         = flag.String("topic", "probe_events", "NSQ topic")
	channel       = flag.String("channel", "elasticsearch", "NSQ channel")
	maxInFlight   = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")
	totalMessages = flag.Int("n", 0, "total messages to process (will wait if starved)")

	consumerOpts     = app.StringArray{}
	nsqdTCPAddrs     = app.StringArray{}
	lookupdHTTPAddrs = app.StringArray{}
)

func init() {
	flag.Var(&consumerOpts, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/bitly/go-nsq#Config)")
	flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
	flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
}

type ESHandler struct {
	totalMessages int
	messagesDone  int
}

func NewESHandler(totalMessages int) (*ESHandler, error) {

	err := eventdef.InitElasticsearch()
	if err != nil {
		return nil, err
	}

	return &ESHandler{
		totalMessages: totalMessages,
	}, nil
}

func (k *ESHandler) HandleMessage(m *nsq.Message) error {
	log.Printf("recieved message.")
	k.messagesDone++
	format := "unknown"
	if m.Body[0] == '\x00' {
		format = "msgFormatJson"
	}

	event := new(eventdef.EventDefinition)
	if err := json.Unmarshal(m.Body[9:], &event); err != nil {
		log.Printf("ERROR: failure to unmarshal message body via format %s: %s. skipping message", format, err)
		return nil
	}
	done := make(chan error, 1)
	go func() {
		if err := event.Save(); err != nil {
			fmt.Printf("ERROR: couldn't process %s: %s\n", event.Id, err)
			done <- err
			return
		}
		done <- nil
	}()

	if err := <-done; err != nil {
		return err
	}

	if k.totalMessages > 0 && k.messagesDone >= k.totalMessages {
		os.Exit(0)
	}
	return nil
}

func main() {
	flag.Parse()

	if *showVersion {
		fmt.Println("nsq_probe_events_to_elasticsearch")
		return
	}

	if *channel == "" {
		rand.Seed(time.Now().UnixNano())
		*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
	}

	if *topic == "" {
		log.Fatal("--topic is required")
	}

	if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
		log.Fatal("--nsqd-tcp-address or --lookupd-http-address required")
	}
	if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
		log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	// Don't ask for more messages than we want
	if *totalMessages > 0 && *totalMessages < *maxInFlight {
		*maxInFlight = *totalMessages
	}

	setting.Config = new(setting.Conf)
	setting.Config.ElasticsearchDomain = "elasticsearch"
	setting.Config.ElasticsearchPort = 9200

	cfg := nsq.NewConfig()
	cfg.UserAgent = "nsq_probe_events_to_elasticsearch"
	err := app.ParseOpts(cfg, consumerOpts)
	if err != nil {
		log.Fatal(err)
	}
	cfg.MaxInFlight = *maxInFlight

	consumer, err := nsq.NewConsumer(*topic, *channel, cfg)
	if err != nil {
		log.Fatal(err)
	}

	handler, err := NewESHandler(*totalMessages)
	if err != nil {
		log.Fatal(err)
	}

	consumer.AddHandler(handler)

	err = consumer.ConnectToNSQDs(nsqdTCPAddrs)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("connected to nsqd")

	err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
	if err != nil {
		log.Fatal(err)
	}
	go func() {
		log.Println("INFO starting listener for http/debug on :6060")
		log.Println(http.ListenAndServe(":6060", nil))
	}()

	for {
		select {
		case <-consumer.StopChan:
			return
		case <-sigChan:
			consumer.Stop()
		}
	}
}
<commit_msg>nsq_probe_events_to_elasticsearch instrumentation<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	_ "net/http/pprof"

	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/bitly/go-nsq"
	"github.com/bitly/nsq/internal/app"
	met "github.com/grafana/grafana/pkg/metric"
	"github.com/grafana/grafana/pkg/metric/helper"
	"github.com/raintank/raintank-metric/instrumented_nsq"

	"github.com/raintank/raintank-metric/eventdef"
	"github.com/raintank/raintank-metric/setting"
)

var (
	showVersion = flag.Bool("version", false, "print version string")

	topic         = flag.String("topic", "probe_events", "NSQ topic")
	channel       = flag.String("channel", "elasticsearch", "NSQ channel")
	maxInFlight   = flag.Int("max-in-flight", 200, "max number of messages to allow in flight")
	totalMessages = flag.Int("n", 0, "total messages to process (will wait if starved)")

	statsdAddr = flag.String("statsd-addr", "localhost:8125", "statsd address (default: localhost:8125)")
	statsdType = flag.String("statsd-type", "standard", "statsd type: standard or datadog (default: standard)")

	consumerOpts     = app.StringArray{}
	nsqdTCPAddrs     = app.StringArray{}
	lookupdHTTPAddrs = app.StringArray{}

	eventsToEsOK   met.Count
	eventsToEsFail met.Count
	messagesSize   met.Meter
	msgsAge        met.Meter // in ms
	esPutDuration  met.Timer
	msgsHandleOK   met.Count
	msgsHandleFail met.Count
)

func init() {
	flag.Var(&consumerOpts, "consumer-opt", "option to passthrough to nsq.Consumer (may be given multiple times, http://godoc.org/github.com/bitly/go-nsq#Config)")
	flag.Var(&nsqdTCPAddrs, "nsqd-tcp-address", "nsqd TCP address (may be given multiple times)")
	flag.Var(&lookupdHTTPAddrs, "lookupd-http-address", "lookupd HTTP address (may be given multiple times)")
}

type ESHandler struct {
	totalMessages int
	messagesDone  int
}

func NewESHandler(totalMessages int) (*ESHandler, error) {

	err := eventdef.InitElasticsearch()
	if err != nil {
		return nil, err
	}

	return &ESHandler{
		totalMessages: totalMessages,
	}, nil
}

func (k *ESHandler) HandleMessage(m *nsq.Message) error {
	log.Printf("recieved message.")
	k.messagesDone++
	format := "unknown"
	if m.Body[0] == '\x00' {
		format = "msgFormatJson"
	}
	var id int64
	buf := bytes.NewReader(m.Body[1:9])
	binary.Read(buf, binary.BigEndian, &id)
	produced := time.Unix(0, id)

	msgsAge.Value(time.Now().Sub(produced).Nanoseconds() / 1000)
	messagesSize.Value(int64(len(m.Body)))

	event := new(eventdef.EventDefinition)
	if err := json.Unmarshal(m.Body[9:], &event); err != nil {
		log.Printf("ERROR: failure to unmarshal message body via format %s: %s. skipping message", format, err)
		return nil
	}
	done := make(chan error, 1)
	go func() {
		pre := time.Now()
		if err := event.Save(); err != nil {
			fmt.Printf("ERROR: couldn't process %s: %s\n", event.Id, err)
			eventsToEsFail.Inc(1)
			done <- err
			return
		}
		esPutDuration.Value(time.Now().Sub(pre))
		eventsToEsOK.Inc(1)
		done <- nil
	}()

	if err := <-done; err != nil {
		msgsHandleFail.Inc(1)
		return err
	}

	msgsHandleOK.Inc(1)

	if k.totalMessages > 0 && k.messagesDone >= k.totalMessages {
		os.Exit(0)
	}
	return nil
}

func main() {
	flag.Parse()

	if *showVersion {
		fmt.Println("nsq_probe_events_to_elasticsearch")
		return
	}

	if *channel == "" {
		rand.Seed(time.Now().UnixNano())
		*channel = fmt.Sprintf("tail%06d#ephemeral", rand.Int()%999999)
	}

	if *topic == "" {
		log.Fatal("--topic is required")
	}

	if len(nsqdTCPAddrs) == 0 && len(lookupdHTTPAddrs) == 0 {
		log.Fatal("--nsqd-tcp-address or --lookupd-http-address required")
	}
	if len(nsqdTCPAddrs) > 0 && len(lookupdHTTPAddrs) > 0 {
		log.Fatal("use --nsqd-tcp-address or --lookupd-http-address not both")
	}
	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}
	metrics, err := helper.New(true, *statsdAddr, *statsdType, "nsq_probe_events_to_elasticsearch", strings.Replace(hostname, ".", "_", -1))
	if err != nil {
		log.Fatal(err)
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

	// Don't ask for more messages than we want
	if *totalMessages > 0 && *totalMessages < *maxInFlight {
		*maxInFlight = *totalMessages
	}

	setting.Config = new(setting.Conf)
	setting.Config.ElasticsearchDomain = "elasticsearch"
	setting.Config.ElasticsearchPort = 9200

	cfg := nsq.NewConfig()
	cfg.UserAgent = "nsq_probe_events_to_elasticsearch"
	err = app.ParseOpts(cfg, consumerOpts)
	if err != nil {
		log.Fatal(err)
	}
	cfg.MaxInFlight = *maxInFlight

	consumer, err := insq.NewConsumer(*topic, *channel, cfg, "%s", metrics)

	if err != nil {
		log.Fatal(err)
	}

	handler, err := NewESHandler(*totalMessages)
	if err != nil {
		log.Fatal(err)
	}

	consumer.AddHandler(handler)

	err = consumer.ConnectToNSQDs(nsqdTCPAddrs)
	if err != nil {
		log.Fatal(err)
	}
	log.Println("connected to nsqd")

	err = consumer.ConnectToNSQLookupds(lookupdHTTPAddrs)
	if err != nil {
		log.Fatal(err)
	}
	go func() {
		log.Println("INFO starting listener for http/debug on :6060")
		log.Println(http.ListenAndServe(":6060", nil))
	}()

	for {
		select {
		case <-consumer.StopChan:
			return
		case <-sigChan:
			consumer.Stop()
		}
	}
}
<|endoftext|>


<commit_before>package main

import "github.com/jangler/edit"

func mustCompile(pattern string, id int) edit.Rule {
	rule, err := edit.NewRule(pattern, id)
	if err != nil {
		panic(err)
	}
	return rule
}

const (
	commentId = iota
	keywordId
	literalId
)

// cRules returns syntax highlighting rules for C.
func cRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`^#(define|undef|include|if|elif|else|endif|ifdef|ifndef|`+
			`line|pragma).*$`, commentId),
		mustCompile(`//.+?$`, commentId),
		mustCompile(`/\*.*?\*/`, commentId),
		mustCompile(`\b(auto|break|case|char|const|continue|default|do|`+
			`double|else|enum|extern|float|for|goto|if|inline|int|long|`+
			`register|restrict|return|short|signed|sizeof|static|struct|`+
			`switch|typedef|union|unsigned|void|volatile|while|_Bool|`+
			`_Complex|_Imaginary)\b`, keywordId),
		mustCompile(`\b(bool|true|false|NULL)\b`, literalId),
		mustCompile(`L?'(\\.|[^'])*?'|"(\\.|[^"])*?"`, literalId),
		mustCompile(`\b(\d+\.\d*|\d*\.\d+|\d+)([eEpP][+-]?\d+)?`+
			`[fFlL]?\b`, literalId),
		mustCompile(`\b(0([bB][01]+|[0-7]+|[xX][0-9a-fA-F]+)|\d+)`+
			`([uU]?[lL]?|[lL]?[uU]?)\b`, literalId),
	}
}

// goRules returns syntax highlighting rules for Go.
func goRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`//.+?$`, commentId),
		mustCompile(`/\*.*?\*/`, commentId),
		mustCompile(`\b(break|case|chan|const|continue|default|defer|else|`+
			`fallthrough|for|func|go|goto|if|import|interface|map|package|`+
			`range|return|select|struct|switch|type|var)\b`, keywordId),
		mustCompile(`\b(append|cap|close|complex|copy|delete|imag|len|make|`+
			`new|panic|print|println|real|recover)\b`, keywordId),
		mustCompile(`\b(bool|byte|complex(64|128)|error|float(32|64)|`+
			`u?int(8|16|32|64)?|rune|string|uintptr)\b`, keywordId),
		mustCompile(`\b(true|false|iota|nil)\b`, literalId),
		mustCompile(`'(\\.|[^'])*?'|"(\\.|[^"])*?"`, literalId),
		mustCompile("`.*?`", literalId),
		mustCompile(`\b0[bB][01]+\b`, literalId),
		mustCompile(`\b0[0-7]+\b`, literalId),
		mustCompile(`\b0[xX][0-9a-fA-F]+\b`, literalId),
		mustCompile(`\b(\d+\.\d*|\d*\.\d+|\d+)([eE][+-]?\d+)?i?\b`, literalId),
		mustCompile(`\b\d+\bi`, literalId),
	}
}

// jsonRules returns syntax highlighting rules for JSON.
func jsonRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`"(\\.|[^"])*?":`, keywordId),
		mustCompile(`"(\\.|[^"])*?"`, literalId),
		mustCompile(`\b(true|false|null)\b`, literalId),
		mustCompile(`\b(\d+\.\d*|\d*\.\d+|\d+)([Ee][+-]?\d+)?\b`, literalId),
	}
}

// makefileRules returns syntax highlighting rules for makefiles.
func makefileRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`#.*$`, commentId),
		mustCompile(`\b(else|end[ei]f|ifn?def|ifn?eq|(-|s)?include|load|`+
			`override|private|(un)?export|(un)?define|vpath)\b`, keywordId),
		mustCompile(`\b(abspath|addprefix|(add)?suffix|and|basename|call|`+
			`error|eval|file|filter(-out)?|findstring|firstword|flavor|`+
			`foreach|guile|if|info|join|lastword|(not)?dir|or(igin)?|`+
			`patsubst|realpath|shell|sort|strip|subst|value|warning|wildcard|`+
			`word(s|list)?)\b`, keywordId),
		mustCompile(`\b\.(DEFAULT|DELETE_ON_ERROR|EXPORT_ALL_VARIABLES|`+
			`IGNORE|INTERMEDIATE|LOW_RESOLUTION_TIME|NOTPARALLEL|ONESHELL|`+
			`PHONY|POSIX|PRECIOUS|SECONDARY|SECONDEXPANSION|SILENT|`+
			`SUFFIXES)\b`, keywordId),
		mustCompile(`\b(DEFAULT_GOAL|\.FEATURES|\.INCLUDE_DIRS|MAKEFILE_LIST|`+
			`MAKE_RESTARTS|MAKE_TERMERR|MAKE_TERMOUT|\.RECIPEPREFIX|`+
			`\.VARIABLES)\b`, keywordId),
		mustCompile(`\b(CURDIR|MAKE(CMDGOALS|FILES|FLAGS|_HOST|LEVEL|`+
			`\.LIBPATTERNS|SHELL|SUFFIXES|_VERSION)?|SHELL|VPATH)\b`,
			keywordId),
		mustCompile(`\$(\(.+\)|\{.+\}|.)`, literalId),
	}
}

// pythonRules returns syntax highlighting rules for Python.
func pythonRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`#.*$`, commentId),
		mustCompile(`\b(and|as|assert|break|class|continue|def|del|elif|else|`+
			`except|finally|for|from|global|if|import|in|is|lambda|nonlocal|`+
			`not|or|pass|raise|return|try|while|with|yield)\b`, keywordId),
		mustCompile(`\b(False|None|True)\b`, literalId),
		mustCompile(`(\b([rR][bB]|[bB][rR]|\b[uUrR]))?('''(\\?.)*?'''|`+
			`"""(\\?.)*?"""|"(\.|[^"])*?"|'(\.|[^'])*?')`, literalId),
		mustCompile(`\b0[bB][01]+\b`, literalId),
		mustCompile(`\b0[oO]?[0-7]+\b`, literalId),
		mustCompile(`\b0[xX][0-9a-fA-F]+\b`, literalId),
		mustCompile(`\b(\d+\.\d*|\d*\.\d+|\d+)([eE][+-]?\d+)?`+
			`([jJ](\d+\.\d*|\d*\.\d+|\d+)([eE][+-]?\d+)?)?\b`, literalId),
	}
}

// bashRules returns syntax highlighting rules for Bash.
func bashRules() []edit.Rule {
	// Not sure how "complete" this is. All the builtins are accounted for, but
	// I might check out how other programs syntax highlight bash to see if
	// other things are usually highlighted.
	return []edit.Rule{
		mustCompile(`\$#`, -1),
		mustCompile(`#.*$`, commentId),
		mustCompile(`[!:.]| \[\[? | \]\]?|\b(alias|bg|bind|break|builtin|`+
			`caller|case|cd|command|compgen|complete|compopt|continue|`+
			`declare|dirs|disown|do|done|echo|elif|else|enable|esac|eval|`+
			`exec|exit|export|false|fc|fg|fi|for|function|getopts|hash|help|`+
			`history|if|in|jobs|kill|let|local|logout|mapfile|popd|printf|`+
			`pushd|pwd|read|readarray|readonly|return|select|set|shift|shopt|`+
			`source|suspend|test|then|time|times|trap|true|type|typeset|`+
			`ulimit|umask|unalias|unset|until|wait|while)\b`, keywordId),
		mustCompile(`\$?("(\\.|[^"])*?"|'(\\.|[^'])*?')`, literalId),
		mustCompile("`(\\.|[^`])*?`", literalId),
	}
}
<commit_msg>Add built-in functions to Python syntax highlighting rules<commit_after>package main

import "github.com/jangler/edit"

func mustCompile(pattern string, id int) edit.Rule {
	rule, err := edit.NewRule(pattern, id)
	if err != nil {
		panic(err)
	}
	return rule
}

const (
	commentId = iota
	keywordId
	literalId
)

// cRules returns syntax highlighting rules for C.
func cRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`^#(define|undef|include|if|elif|else|endif|ifdef|ifndef|`+
			`line|pragma).*$`, commentId),
		mustCompile(`//.+?$`, commentId),
		mustCompile(`/\*.*?\*/`, commentId),
		mustCompile(`\b(auto|break|case|char|const|continue|default|do|`+
			`double|else|enum|extern|float|for|goto|if|inline|int|long|`+
			`register|restrict|return|short|signed|sizeof|static|struct|`+
			`switch|typedef|union|unsigned|void|volatile|while|_Bool|`+
			`_Complex|_Imaginary)\b`, keywordId),
		mustCompile(`\b(bool|true|false|NULL)\b`, literalId),
		mustCompile(`L?'(\\.|[^'])*?'|"(\\.|[^"])*?"`, literalId),
		mustCompile(`\b(\d+\.\d*|\d*\.\d+|\d+)([eEpP][+-]?\d+)?`+
			`[fFlL]?\b`, literalId),
		mustCompile(`\b(0([bB][01]+|[0-7]+|[xX][0-9a-fA-F]+)|\d+)`+
			`([uU]?[lL]?|[lL]?[uU]?)\b`, literalId),
	}
}

// goRules returns syntax highlighting rules for Go.
func goRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`//.+?$`, commentId),
		mustCompile(`/\*.*?\*/`, commentId),
		mustCompile(`\b(break|case|chan|const|continue|default|defer|else|`+
			`fallthrough|for|func|go|goto|if|import|interface|map|package|`+
			`range|return|select|struct|switch|type|var)\b`, keywordId),
		mustCompile(`\b(append|cap|close|complex|copy|delete|imag|len|make|`+
			`new|panic|print|println|real|recover)\b`, keywordId),
		mustCompile(`\b(bool|byte|complex(64|128)|error|float(32|64)|`+
			`u?int(8|16|32|64)?|rune|string|uintptr)\b`, keywordId),
		mustCompile(`\b(true|false|iota|nil)\b`, literalId),
		mustCompile(`'(\\.|[^'])*?'|"(\\.|[^"])*?"`, literalId),
		mustCompile("`.*?`", literalId),
		mustCompile(`\b0[bB][01]+\b`, literalId),
		mustCompile(`\b0[0-7]+\b`, literalId),
		mustCompile(`\b0[xX][0-9a-fA-F]+\b`, literalId),
		mustCompile(`\b(\d+\.\d*|\d*\.\d+|\d+)([eE][+-]?\d+)?i?\b`, literalId),
		mustCompile(`\b\d+\bi`, literalId),
	}
}

// jsonRules returns syntax highlighting rules for JSON.
func jsonRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`"(\\.|[^"])*?":`, keywordId),
		mustCompile(`"(\\.|[^"])*?"`, literalId),
		mustCompile(`\b(true|false|null)\b`, literalId),
		mustCompile(`\b(\d+\.\d*|\d*\.\d+|\d+)([Ee][+-]?\d+)?\b`, literalId),
	}
}

// makefileRules returns syntax highlighting rules for makefiles.
func makefileRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`#.*$`, commentId),
		mustCompile(`\b(else|end[ei]f|ifn?def|ifn?eq|(-|s)?include|load|`+
			`override|private|(un)?export|(un)?define|vpath)\b`, keywordId),
		mustCompile(`\b(abspath|addprefix|(add)?suffix|and|basename|call|`+
			`error|eval|file|filter(-out)?|findstring|firstword|flavor|`+
			`foreach|guile|if|info|join|lastword|(not)?dir|or(igin)?|`+
			`patsubst|realpath|shell|sort|strip|subst|value|warning|wildcard|`+
			`word(s|list)?)\b`, keywordId),
		mustCompile(`\b\.(DEFAULT|DELETE_ON_ERROR|EXPORT_ALL_VARIABLES|`+
			`IGNORE|INTERMEDIATE|LOW_RESOLUTION_TIME|NOTPARALLEL|ONESHELL|`+
			`PHONY|POSIX|PRECIOUS|SECONDARY|SECONDEXPANSION|SILENT|`+
			`SUFFIXES)\b`, keywordId),
		mustCompile(`\b(DEFAULT_GOAL|\.FEATURES|\.INCLUDE_DIRS|MAKEFILE_LIST|`+
			`MAKE_RESTARTS|MAKE_TERMERR|MAKE_TERMOUT|\.RECIPEPREFIX|`+
			`\.VARIABLES)\b`, keywordId),
		mustCompile(`\b(CURDIR|MAKE(CMDGOALS|FILES|FLAGS|_HOST|LEVEL|`+
			`\.LIBPATTERNS|SHELL|SUFFIXES|_VERSION)?|SHELL|VPATH)\b`,
			keywordId),
		mustCompile(`\$(\(.+\)|\{.+\}|.)`, literalId),
	}
}

// pythonRules returns syntax highlighting rules for Python.
func pythonRules() []edit.Rule {
	return []edit.Rule{
		mustCompile(`#.*$`, commentId),
		mustCompile(`\b(and|as|assert|break|class|continue|def|del|elif|else|`+
			`except|finally|for|from|global|if|import|in|is|lambda|nonlocal|`+
			`not|or|pass|raise|return|try|while|with|yield)\b`, keywordId),
		mustCompile(`\b(abs|all|any|ascii|bin|bool|bytearray|bytes|callable|`+
			`chr|classmethod|compile|complex|delattr|dict|dir|divmod|`+
			`enumerate|eval|exec|filter|float|format|frozenset|getattr|`+
			`globals|hasattr|hash|help|hex|id|input|int|isinstance|`+
			`issubclass|iter|len|list|locals|map|max|memoryview|min|next|`+
			`object|oct|open|ord|pow|print|property|range|repr|reversed|`+
			`round|set|setattr|slice|sorted|staticmethod|str|sum|super|`+
			`tuple|type|vars|zip|__import__)\b`, keywordId),
		mustCompile(`\b(False|None|True)\b`, literalId),
		mustCompile(`(\b([rR][bB]|[bB][rR]|\b[uUrR]))?('''(\\?.)*?'''|`+
			`"""(\\?.)*?"""|"(\.|[^"])*?"|'(\.|[^'])*?')`, literalId),
		mustCompile(`\b0[bB][01]+\b`, literalId),
		mustCompile(`\b0[oO]?[0-7]+\b`, literalId),
		mustCompile(`\b0[xX][0-9a-fA-F]+\b`, literalId),
		mustCompile(`\b(\d+\.\d*|\d*\.\d+|\d+)([eE][+-]?\d+)?`+
			`([jJ](\d+\.\d*|\d*\.\d+|\d+)([eE][+-]?\d+)?)?\b`, literalId),
	}
}

// bashRules returns syntax highlighting rules for Bash.
func bashRules() []edit.Rule {
	// Not sure how "complete" this is. All the builtins are accounted for, but
	// I might check out how other programs syntax highlight bash to see if
	// other things are usually highlighted.
	return []edit.Rule{
		mustCompile(`\$#`, -1),
		mustCompile(`#.*$`, commentId),
		mustCompile(`[!:.]| \[\[? | \]\]?|\b(alias|bg|bind|break|builtin|`+
			`caller|case|cd|command|compgen|complete|compopt|continue|`+
			`declare|dirs|disown|do|done|echo|elif|else|enable|esac|eval|`+
			`exec|exit|export|false|fc|fg|fi|for|function|getopts|hash|help|`+
			`history|if|in|jobs|kill|let|local|logout|mapfile|popd|printf|`+
			`pushd|pwd|read|readarray|readonly|return|select|set|shift|shopt|`+
			`source|suspend|test|then|time|times|trap|true|type|typeset|`+
			`ulimit|umask|unalias|unset|until|wait|while)\b`, keywordId),
		mustCompile(`\$?("(\\.|[^"])*?"|'(\\.|[^'])*?')`, literalId),
		mustCompile("`(\\.|[^`])*?`", literalId),
	}
}
<|endoftext|>


<commit_before>package fetcher

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/joneskoo/httpmonitor/checker"
)

// Request configuration for what to check
type Request struct {
	URL      string          // URL to fetch
	Timeout  float32         // Request timeout, seconds
	Interval float32         // Poll interval, seconds
	Checks   []checker.Check // List of checks for status 'pass'
	Error    error
}

// TimeoutDuration returns the request timeout as time.Duration
func (r Request) TimeoutDuration() time.Duration {
	return time.Duration(r.Timeout * 1e9)
}

// PollIntervalDuration returns the poll interval as time.Duration
func (r Request) PollIntervalDuration() time.Duration {
	return time.Duration(r.Interval * 1e9)
}

func (r Request) String() string {
	return fmt.Sprintf("GET '%v' every %v timeout=%v (%v checks)>",
		r.URL, r.PollIntervalDuration(), r.TimeoutDuration(), len(r.Checks))
}

// Result from a HTTP status check
type Result struct {
	URL        string        // URL we fetched
	Dur        time.Duration // Duration it took to fetch it
	Status     bool          // Status check pass (true)/fail (false)
	Error      error         // URL fetching error
	HTTPStatus int           // Response HTTP status code
}

func (r Result) String() string {
	return fmt.Sprintf("%v %v in %v", r.URL, r.StatusText(), r.Dur)
}

// StatusText is the pass/fail/unreachable status for check
func (r Result) StatusText() (status string) {
	if r.Error != nil {
		status = "unreachable"
	} else if r.Status {
		status = "passed"
	} else {
		status = "failed"
	}
	return
}

// StatusEmoji is the pass/fail/unreachable as an emoji symbol
func (r Result) StatusEmoji() string {
	if r.Error != nil {
		return "💤"
	} else if r.Status {
		return "✅"
	} else {
		return "❌"
	}

}

// FetchSingleURL retrieves a single URL based on configuration structure
// Request and returns a response structure Result.
func FetchSingleURL(req Request) (res Result) {
	res = Result{URL: req.URL}
	// Time how long it takes
	requestStartTime := time.Now()

	// Configure timeout
	client := http.Client{
		Timeout: req.TimeoutDuration(),
	}

	// Perform HTTP GET request
	resp, err := client.Get(req.URL)
	if err != nil {
		log.Print("Request failed: ", err)
		res.Dur = time.Since(requestStartTime)
		res.Error = err // Store to result
		return
	}
	res.HTTPStatus = resp.StatusCode
	defer resp.Body.Close() // Close body to free connection after done

	// Run checks
	pass, err := checker.DoCheck(resp, req.Checks)
	if err != nil {
		log.Print("Check failed: ", err)
	}
	res.Status = pass
	res.Dur = time.Since(requestStartTime)
	return
}

// FetchUrls fetches a list of URLs all concurrently in goroutines
// and immediately return a channel streaming Result objects.
func FetchUrls(requests []Request) chan Result {
	c := make(chan Result)
	for _, req := range requests {
		// Start fetch in background and put the result
		// into channel when it is done.
		go func(req Request) {
			for range time.Tick(req.PollIntervalDuration()) {
				res := FetchSingleURL(req)
				c <- res
			}
		}(req)
	}
	// Immediately return the channel where results will arrive
	return c
}
<commit_msg>Rename Result.Status bool to Passed<commit_after>package fetcher

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/joneskoo/httpmonitor/checker"
)

// Request configuration for what to check
type Request struct {
	URL      string          // URL to fetch
	Timeout  float32         // Request timeout, seconds
	Interval float32         // Poll interval, seconds
	Checks   []checker.Check // List of checks for status 'pass'
	Error    error
}

// TimeoutDuration returns the request timeout as time.Duration
func (r Request) TimeoutDuration() time.Duration {
	return time.Duration(r.Timeout * 1e9)
}

// PollIntervalDuration returns the poll interval as time.Duration
func (r Request) PollIntervalDuration() time.Duration {
	return time.Duration(r.Interval * 1e9)
}

func (r Request) String() string {
	return fmt.Sprintf("GET '%v' every %v timeout=%v (%v checks)>",
		r.URL, r.PollIntervalDuration(), r.TimeoutDuration(), len(r.Checks))
}

// Result from a HTTP status check
type Result struct {
	URL        string        // URL we fetched
	Dur        time.Duration // Duration it took to fetch it
	Passed     bool          // Status check pass (true)/fail (false)
	Error      error         // URL fetching error
	HTTPStatus int           // Response HTTP status code
}

func (r Result) String() string {
	return fmt.Sprintf("%v %v in %v", r.URL, r.StatusText(), r.Dur)
}

// StatusText is the pass/fail/unreachable status for check
func (r Result) StatusText() (status string) {
	if r.Error != nil {
		status = "unreachable"
	} else if r.Passed {
		status = "passed"
	} else {
		status = "failed"
	}
	return
}

// StatusEmoji is the pass/fail/unreachable as an emoji symbol
func (r Result) StatusEmoji() string {
	if r.Error != nil {
		return "💤"
	} else if r.Passed {
		return "✅"
	} else {
		return "❌"
	}

}

// FetchSingleURL retrieves a single URL based on configuration structure
// Request and returns a response structure Result.
func FetchSingleURL(req Request) (res Result) {
	res = Result{URL: req.URL}
	// Time how long it takes
	requestStartTime := time.Now()

	// Configure timeout
	client := http.Client{
		Timeout: req.TimeoutDuration(),
	}

	// Perform HTTP GET request
	resp, err := client.Get(req.URL)
	if err != nil {
		log.Print("Request failed: ", err)
		res.Dur = time.Since(requestStartTime)
		res.Error = err // Store to result
		return
	}
	res.HTTPStatus = resp.StatusCode
	defer resp.Body.Close() // Close body to free connection after done

	// Run checks
	pass, err := checker.DoCheck(resp, req.Checks)
	if err != nil {
		log.Print("Check failed: ", err)
	}
	res.Passed = pass
	res.Dur = time.Since(requestStartTime)
	return
}

// FetchUrls fetches a list of URLs all concurrently in goroutines
// and immediately return a channel streaming Result objects.
func FetchUrls(requests []Request) chan Result {
	c := make(chan Result)
	for _, req := range requests {
		// Start fetch in background and put the result
		// into channel when it is done.
		go func(req Request) {
			for range time.Tick(req.PollIntervalDuration()) {
				res := FetchSingleURL(req)
				c <- res
			}
		}(req)
	}
	// Immediately return the channel where results will arrive
	return c
}
<|endoftext|>


<commit_before>package point

import (
	"testing"
	"assert"
)

func TestAsserts(t *testing.T) {
	p1 := Point{1, 1}
	p2 := Point{2, 1}

	assert.Equal(t, p1, p2)
}
<commit_msg>ララララー ラララー ララ<commit_after>package point

import (
	"testing"

	"github.com/bmizerany/assert"
)

func TestAsserts(t *testing.T) {
	p1 := Point{1, 1}
	p2 := Point{2, 1}

	assert.Equal(t, p1, p2)
}
<|endoftext|>