<commit_before>package imgsvr

import (
	"errors"
	"fmt"
	l4g "github.com/alecthomas/log4go"
	"github.com/ctripcorp/cat"
	"github.com/ctripcorp/nephele/imgsvr/img4g"
	"github.com/ctripcorp/nephele/imgsvr/proc"
	"github.com/ctripcorp/nephele/imgsvr/storage"
	"github.com/ctripcorp/nephele/util"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	legalUrl     = util.RegexpExt{regexp.MustCompile("^/images/(.*?)_(R|C|Z|W)_([0-9]+)_([0-9]+)(_R([0-9]+))?(_C([a-zA-Z]+))?(_Q(?P<n0>[0-9]+))?(_M((?P<wn>[a-zA-Z0-9]+)(_(?P<wl>[1-9]))?))?.(?P<ext>jpg|jpeg|gif|png|Jpg)$")}
	forbiddenUrl = util.RegexpExt{regexp.MustCompile("^/images/fd/([a-zA-Z]+)/([a-zA-Z0-9]+)/(.*?)_Source.(?P<ext>jpg|jpeg|gif|png|Jpg)$")}
	proxyPassUrl = util.RegexpExt{regexp.MustCompile("^/images/fd/([a-zA-Z]+)/([a-zA-Z0-9]+)/(.*?).(?P<ext>jpg|jpeg|gif|png|Jpg)$")}
)

type Handler struct {
	ChainBuilder *ProcChainBuilder
}

func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	Cat := cat.Instance()
	handler.ChainBuilder = &ProcChainBuilder{Cat}
	uri := request.URL.String()
	tran := Cat.NewTransaction("URL", getShortUri(uri))
	var (
		err       error
		isSuccess bool = true
	)
	defer func() {
		p := recover()
		if p != nil {
			l4g.Error("%s; rcv(url:%s)", p, uri)
			Cat.LogPanic(p)
			tran.SetStatus(p)
		}

		if isSuccess {
			tran.SetStatus("0")
			tran.Complete()
		} else {
			tran.SetStatus(err)
			tran.Complete()
		}
		if p != nil || err != nil {
			http.Error(writer, http.StatusText(404), 404)
		}
	}()

	LogEvent(Cat, "URL", "URL.client", map[string]string{
		"clientip": GetClientIP(request),
		"serverip": GetIP(),
		"proto":    request.Proto,
		"referer":  request.Referer(),
		"agent":    request.UserAgent(),
	})

	LogEvent(Cat, "URL", "URL.method", map[string]string{
		"Http": request.Method + " " + uri,
	})
	params, ok1 := legalUrl.FindStringSubmatchMap(uri)
	if !ok1 {
		err = errors.New("uri.parseerror")
		l4g.Error("%s; rcv(url:%s)", err, uri)
		LogErrorEvent(Cat, "uri.parseerror", "")
		return
	}
	store, storagetype, err1 := FindStorage(params)
	if err1 != nil {
		err = errors.New("storage.parseerror")
		l4g.Error("%s; rcv(url:%s)", err1, uri)
		LogErrorEvent(Cat, "storage.parseerror", err1.Error())
		return
	}
	chain, err1 := handler.ChainBuilder.Build(params)
	if err1 != nil {
		err = errors.New("procchain.builderror")
		l4g.Error("%s; rcv(url:%s)", err1, uri)
		LogErrorEvent(Cat, "procchain.builderror", err1.Error())
		return
	}

	var bts []byte
	func() {
		var err1 error
		getimagetran := Cat.NewTransaction("Storage", storagetype)
		defer func() {
			if err1 != nil {
				l4g.Error("%s -- %s; rcv(url:%s)", "getimage", err1, uri)
				err = errors.New(storagetype + ".readerror")
				isSuccess = false
				LogErrorEvent(Cat, storagetype+".readerror", err1.Error())
			}
			getimagetran.SetStatus(err)
			getimagetran.Complete()
		}()
		bts, err1 = store.GetImage()
	}()
	if err != nil {
		return
	}
	size := len(bts)
	sizestr := strconv.Itoa(size)
	tran.AddData("size", sizestr)
	Cat.LogEvent("Size", GetImageSizeDistribution(size))

	l4g.Debug("get image length(%d) rcv(url:%s)", size, request.URL.String())
	format, _ := params["ext"]
	img := &img4g.Image{Blob: bts, Format: format}

	status := make(chan bool, 1)
	imgHd := imgHandle{img, chain, status, Cat}
	imgHandleChan <- imgHd

	select {
	case ok := <-status:
		if !ok {
			err = errors.New("processerror")
			isSuccess = false
			l4g.Error("%s; rcv(url:%s)", err, uri)
			return
		}
	case <-time.After(time.Second * 5):
		err = errors.New("processtimeout")
		l4g.Error("%s; rcv(url:%s)", err, uri)
		isSuccess = false
		LogErrorEvent(Cat, "processtimeout", "")
		return
	}
	writer.Header().Set("Content-Type", "image/"+format)
	writer.Header().Set("Content-Length", strconv.Itoa(len(img.Blob)))
	writer.Header().Set("Last-Modified", "2015/1/1 01:01:01")
	l4g.Debug("final size->>>" + strconv.Itoa(len(img.Blob)))
	if _, err1 = writer.Write(img.Blob); err1 != nil {
		l4g.Error(err1)
		err = errors.New("response.writeerror")
		LogErrorEvent(Cat, "response.writeerror", err1.Error())
		isSuccess = false
		return
	}
}

func CycleHandleImage() {
	defer func() {
		if err := recover(); err != nil {
			l4g.Error("%s -- %s", "handleimage.recovererror", err)
			LogErrorEvent(CatInstance, "handleimage.recovererror", fmt.Sprintf("%v", err))
			go CycleHandleImage()
		}
	}()

	for {
		status := true
		imgHd := <-imgHandleChan
		chain := imgHd.chain
		image := imgHd.inImg
		if err := chainProcImg(imgHd.CatInstance, chain, image); err != nil {
			l4g.Error("%s -- %s", "processerror", err)
			LogErrorEvent(imgHd.CatInstance, "processerror", err.Error())
			status = false
		}
		imgHd.status <- status
	}
}

func chainProcImg(catinstance cat.Cat, chain *proc.ProcessorChain, img *img4g.Image) (err error) {
	defer func() {
		if r := recover(); r != nil {
			l4g.Error("%s -- %s", "processimage.recovererror", err)
			LogErrorEvent(catinstance, "processimage.recovererror", fmt.Sprintf("%v", r))
		}
	}()
	defer img.DestoryWand()
	if err = img.CreateWand(); err != nil {
		return err
	}
	if err = chain.Process(img); err != nil {
		return err
	}
	if err = img.WriteImageBlob(); err != nil {
		return err
	}
	return nil
}

func FindStorage(params map[string]string) (storage.Storage, string, error) {
	srcPath, ok := params[":1"]
	if !ok {
		return nil, "", errors.New("Url.UnExpected")
	}
	format, ok := params["ext"]
	if !ok {
		return nil, "", errors.New("Image.Ext.Invalid()")
	}
	sourceType, _, path := ParseUri(srcPath)
	s, err := GetStorage(sourceType, path+"."+format)
	return s, sourceType, err
}

func getShortUri(uri string) string {
	arr := strings.Split(uri, "/")
	if len(arr) < 4 {
		return uri
	}
	if arr[2] == "fd" || arr[2] == "t1" {
		return JoinString("/", arr[1], "/", arr[2], "/", arr[3])
	} else {
		return JoinString("/", arr[1], "/", arr[2])
	}
}
<commit_msg>remove agent<commit_after>package imgsvr

import (
	"errors"
	"fmt"
	l4g "github.com/alecthomas/log4go"
	"github.com/ctripcorp/cat"
	"github.com/ctripcorp/nephele/imgsvr/img4g"
	"github.com/ctripcorp/nephele/imgsvr/proc"
	"github.com/ctripcorp/nephele/imgsvr/storage"
	"github.com/ctripcorp/nephele/util"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	legalUrl     = util.RegexpExt{regexp.MustCompile("^/images/(.*?)_(R|C|Z|W)_([0-9]+)_([0-9]+)(_R([0-9]+))?(_C([a-zA-Z]+))?(_Q(?P<n0>[0-9]+))?(_M((?P<wn>[a-zA-Z0-9]+)(_(?P<wl>[1-9]))?))?.(?P<ext>jpg|jpeg|gif|png|Jpg)$")}
	forbiddenUrl = util.RegexpExt{regexp.MustCompile("^/images/fd/([a-zA-Z]+)/([a-zA-Z0-9]+)/(.*?)_Source.(?P<ext>jpg|jpeg|gif|png|Jpg)$")}
	proxyPassUrl = util.RegexpExt{regexp.MustCompile("^/images/fd/([a-zA-Z]+)/([a-zA-Z0-9]+)/(.*?).(?P<ext>jpg|jpeg|gif|png|Jpg)$")}
)

type Handler struct {
	ChainBuilder *ProcChainBuilder
}

func (handler *Handler) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
	Cat := cat.Instance()
	handler.ChainBuilder = &ProcChainBuilder{Cat}
	uri := request.URL.String()
	tran := Cat.NewTransaction("URL", getShortUri(uri))
	var (
		err       error
		isSuccess bool = true
	)
	defer func() {
		p := recover()
		if p != nil {
			l4g.Error("%s; rcv(url:%s)", p, uri)
			Cat.LogPanic(p)
			tran.SetStatus(p)
		}

		if isSuccess {
			tran.SetStatus("0")
			tran.Complete()
		} else {
			tran.SetStatus(err)
			tran.Complete()
		}
		if p != nil || err != nil {
			http.Error(writer, http.StatusText(404), 404)
		}
	}()

	LogEvent(Cat, "URL", "URL.client", map[string]string{
		"clientip": GetClientIP(request),
		"serverip": GetIP(),
		"proto":    request.Proto,
		"referer":  request.Referer(),
		//"agent":    request.UserAgent(),
	})

	LogEvent(Cat, "URL", "URL.method", map[string]string{
		"Http": request.Method + " " + uri,
	})
	params, ok1 := legalUrl.FindStringSubmatchMap(uri)
	if !ok1 {
		err = errors.New("uri.parseerror")
		l4g.Error("%s; rcv(url:%s)", err, uri)
		LogErrorEvent(Cat, "uri.parseerror", "")
		return
	}
	store, storagetype, err1 := FindStorage(params)
	if err1 != nil {
		err = errors.New("storage.parseerror")
		l4g.Error("%s; rcv(url:%s)", err1, uri)
		LogErrorEvent(Cat, "storage.parseerror", err1.Error())
		return
	}
	chain, err1 := handler.ChainBuilder.Build(params)
	if err1 != nil {
		err = errors.New("procchain.builderror")
		l4g.Error("%s; rcv(url:%s)", err1, uri)
		LogErrorEvent(Cat, "procchain.builderror", err1.Error())
		return
	}

	var bts []byte
	func() {
		var err1 error
		getimagetran := Cat.NewTransaction("Storage", storagetype)
		defer func() {
			if err1 != nil {
				l4g.Error("%s -- %s; rcv(url:%s)", "getimage", err1, uri)
				err = errors.New(storagetype + ".readerror")
				isSuccess = false
				LogErrorEvent(Cat, storagetype+".readerror", err1.Error())
			}
			getimagetran.SetStatus(err)
			getimagetran.Complete()
		}()
		bts, err1 = store.GetImage()
	}()
	if err != nil {
		return
	}
	size := len(bts)
	sizestr := strconv.Itoa(size)
	tran.AddData("size", sizestr)
	Cat.LogEvent("Size", GetImageSizeDistribution(size))

	l4g.Debug("get image length(%d) rcv(url:%s)", size, request.URL.String())
	format, _ := params["ext"]
	img := &img4g.Image{Blob: bts, Format: format}

	status := make(chan bool, 1)
	imgHd := imgHandle{img, chain, status, Cat}
	imgHandleChan <- imgHd

	select {
	case ok := <-status:
		if !ok {
			err = errors.New("processerror")
			isSuccess = false
			l4g.Error("%s; rcv(url:%s)", err, uri)
			return
		}
	case <-time.After(time.Second * 5):
		err = errors.New("processtimeout")
		l4g.Error("%s; rcv(url:%s)", err, uri)
		isSuccess = false
		LogErrorEvent(Cat, "processtimeout", "")
		return
	}
	writer.Header().Set("Content-Type", "image/"+format)
	writer.Header().Set("Content-Length", strconv.Itoa(len(img.Blob)))
	writer.Header().Set("Last-Modified", "2015/1/1 01:01:01")
	l4g.Debug("final size->>>" + strconv.Itoa(len(img.Blob)))
	if _, err1 = writer.Write(img.Blob); err1 != nil {
		l4g.Error(err1)
		err = errors.New("response.writeerror")
		LogErrorEvent(Cat, "response.writeerror", err1.Error())
		isSuccess = false
		return
	}
}

func CycleHandleImage() {
	defer func() {
		if err := recover(); err != nil {
			l4g.Error("%s -- %s", "handleimage.recovererror", err)
			LogErrorEvent(CatInstance, "handleimage.recovererror", fmt.Sprintf("%v", err))
			go CycleHandleImage()
		}
	}()

	for {
		status := true
		imgHd := <-imgHandleChan
		chain := imgHd.chain
		image := imgHd.inImg
		if err := chainProcImg(imgHd.CatInstance, chain, image); err != nil {
			l4g.Error("%s -- %s", "processerror", err)
			LogErrorEvent(imgHd.CatInstance, "processerror", err.Error())
			status = false
		}
		imgHd.status <- status
	}
}

func chainProcImg(catinstance cat.Cat, chain *proc.ProcessorChain, img *img4g.Image) (err error) {
	defer func() {
		if r := recover(); r != nil {
			l4g.Error("%s -- %s", "processimage.recovererror", err)
			LogErrorEvent(catinstance, "processimage.recovererror", fmt.Sprintf("%v", r))
		}
	}()
	defer img.DestoryWand()
	if err = img.CreateWand(); err != nil {
		return err
	}
	if err = chain.Process(img); err != nil {
		return err
	}
	if err = img.WriteImageBlob(); err != nil {
		return err
	}
	return nil
}

func FindStorage(params map[string]string) (storage.Storage, string, error) {
	srcPath, ok := params[":1"]
	if !ok {
		return nil, "", errors.New("Url.UnExpected")
	}
	format, ok := params["ext"]
	if !ok {
		return nil, "", errors.New("Image.Ext.Invalid()")
	}
	sourceType, _, path := ParseUri(srcPath)
	s, err := GetStorage(sourceType, path+"."+format)
	return s, sourceType, err
}

func getShortUri(uri string) string {
	arr := strings.Split(uri, "/")
	if len(arr) < 4 {
		return uri
	}
	if arr[2] == "fd" || arr[2] == "t1" {
		return JoinString("/", arr[1], "/", arr[2], "/", arr[3])
	} else {
		return JoinString("/", arr[1], "/", arr[2])
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net/http"
)

var listenAddr = flag.String("listenAddr", ":8080", "Web server listen address")

func main() {
	flag.Parse()
	log.Println("Web server Listening on " + *listenAddr)
	http.ListenAndServe(*listenAddr, http.FileServer(http.Dir("./public")))
}
<commit_msg>use gorilla/mux so can split off push endpoint<commit_after>package main

import (
	"flag"
	"log"
	"net/http"

	"github.com/gorilla/mux"
)

var listenAddr = flag.String("listenAddr", ":8080", "Web server listen address")

//
// func pushEndpoint(http.ResponseWriter, *http.Request) {
//
// }

func main() {
	flag.Parse()

	router := mux.NewRouter()

	// Push endpoint
	// router.HandleFunc("/push/{userId}", pushEndpoint).Methods("POST")

	// Static public files
	publicFiles := http.FileServer(http.Dir("public"))
	router.Handle("/", publicFiles)

	// Start Web Server
	http.Handle("/", router)
	log.Println("Web server Listening on " + *listenAddr)
	http.ListenAndServe(*listenAddr, nil)
}
<|endoftext|>


<commit_before>package rules

import (
	"fmt"
	"log"

	"github.com/terraform-linters/tflint/rules/awsrules"
	"github.com/terraform-linters/tflint/rules/terraformrules"
	"github.com/terraform-linters/tflint/tflint"
)

// Rule is an implementation that receives a Runner and inspects for resources and modules.
type Rule interface {
	Name() string
	Enabled() bool
	Check(runner *tflint.Runner) error
}

// DefaultRules is rules by default
var DefaultRules = append(manualDefaultRules, modelRules...)
var deepCheckRules = append(manualDeepCheckRules, apiRules...)

var manualDefaultRules = []Rule{
	awsrules.NewAwsDBInstanceDefaultParameterGroupRule(),
	awsrules.NewAwsDBInstanceInvalidTypeRule(),
	awsrules.NewAwsDBInstancePreviousTypeRule(),
	awsrules.NewAwsElastiCacheClusterDefaultParameterGroupRule(),
	awsrules.NewAwsElastiCacheClusterInvalidTypeRule(),
	awsrules.NewAwsElastiCacheClusterPreviousTypeRule(),
	awsrules.NewAwsInstancePreviousTypeRule(),
	awsrules.NewAwsMqBrokerInvalidEngineTypeRule(),
	awsrules.NewAwsMqConfigurationInvalidEngineTypeRule(),
	awsrules.NewAwsRouteNotSpecifiedTargetRule(),
	awsrules.NewAwsRouteSpecifiedMultipleTargetsRule(),
	awsrules.NewAwsS3BucketInvalidACLRule(),
	awsrules.NewAwsS3BucketInvalidRegionRule(),
	awsrules.NewAwsSpotFleetRequestInvalidExcessCapacityTerminationPolicyRule(),
	awsrules.NewAwsResourceMissingTagsRule(),
	awsrules.NewAwsDynamoDBTableInvalidStreamViewTypeRule(),
	terraformrules.NewTerraformDashInResourceNameRule(),
	terraformrules.NewTerraformDashInOutputNameRule(),
	terraformrules.NewTerraformDashInModuleNameRule(),
	terraformrules.NewTerraformDashInDataSourceNameRule(),
	terraformrules.NewTerraformDeprecatedInterpolationRule(),
	terraformrules.NewTerraformDocumentedOutputsRule(),
	terraformrules.NewTerraformDocumentedVariablesRule(),
	terraformrules.NewTerraformModulePinnedSourceRule(),
	terraformrules.NewTerraformTypedVariablesRule(),
	terraformrules.NewTerraformRequiredVersionRule(),
}

var manualDeepCheckRules = []Rule{
	awsrules.NewAwsInstanceInvalidAMIRule(),
	awsrules.NewAwsLaunchConfigurationInvalidImageIDRule(),
}

// CheckRuleNames returns map of rules indexed by name
func CheckRuleNames(ruleNames []string) error {
	log.Print("[INFO] Checking rules")

	rulesMap := map[string]Rule{}
	for _, rule := range append(DefaultRules, deepCheckRules...) {
		rulesMap[rule.Name()] = rule
	}

	totalEnabled := 0
	for _, rule := range rulesMap {
		if rule.Enabled() {
			totalEnabled++
		}
	}
	log.Printf("[INFO]   %d (%d) rules total", len(rulesMap), totalEnabled)
	for _, rule := range ruleNames {
		if _, ok := rulesMap[rule]; !ok {
			return fmt.Errorf("Rule not found: %s", rule)
		}
	}
	return nil
}

// NewRules returns rules according to configuration
func NewRules(c *tflint.Config) []Rule {
	log.Print("[INFO] Prepare rules")

	ret := []Rule{}
	allRules := []Rule{}

	if c.DeepCheck {
		log.Printf("[DEBUG] Deep check mode is enabled. Add deep check rules")
		allRules = append(DefaultRules, deepCheckRules...)
	} else {
		allRules = DefaultRules
	}

	for _, rule := range allRules {
		enabled := rule.Enabled()
		if r := c.Rules[rule.Name()]; r != nil {
			if r.Enabled {
				log.Printf("[DEBUG] `%s` is enabled", rule.Name())
			} else {
				log.Printf("[DEBUG] `%s` is disabled", rule.Name())
			}
			enabled = r.Enabled
		}

		if enabled {
			ret = append(ret, rule)
		}
	}
	log.Printf("[INFO]   %d rules enabled", len(ret))
	return ret
}
<commit_msg>Add naming convention rule<commit_after>package rules

import (
	"fmt"
	"log"

	"github.com/terraform-linters/tflint/rules/awsrules"
	"github.com/terraform-linters/tflint/rules/terraformrules"
	"github.com/terraform-linters/tflint/tflint"
)

// Rule is an implementation that receives a Runner and inspects for resources and modules.
type Rule interface {
	Name() string
	Enabled() bool
	Check(runner *tflint.Runner) error
}

// DefaultRules is rules by default
var DefaultRules = append(manualDefaultRules, modelRules...)
var deepCheckRules = append(manualDeepCheckRules, apiRules...)

var manualDefaultRules = []Rule{
	awsrules.NewAwsDBInstanceDefaultParameterGroupRule(),
	awsrules.NewAwsDBInstanceInvalidTypeRule(),
	awsrules.NewAwsDBInstancePreviousTypeRule(),
	awsrules.NewAwsElastiCacheClusterDefaultParameterGroupRule(),
	awsrules.NewAwsElastiCacheClusterInvalidTypeRule(),
	awsrules.NewAwsElastiCacheClusterPreviousTypeRule(),
	awsrules.NewAwsInstancePreviousTypeRule(),
	awsrules.NewAwsMqBrokerInvalidEngineTypeRule(),
	awsrules.NewAwsMqConfigurationInvalidEngineTypeRule(),
	awsrules.NewAwsRouteNotSpecifiedTargetRule(),
	awsrules.NewAwsRouteSpecifiedMultipleTargetsRule(),
	awsrules.NewAwsS3BucketInvalidACLRule(),
	awsrules.NewAwsS3BucketInvalidRegionRule(),
	awsrules.NewAwsSpotFleetRequestInvalidExcessCapacityTerminationPolicyRule(),
	awsrules.NewAwsResourceMissingTagsRule(),
	awsrules.NewAwsDynamoDBTableInvalidStreamViewTypeRule(),
	terraformrules.NewTerraformDashInResourceNameRule(),
	terraformrules.NewTerraformDashInOutputNameRule(),
	terraformrules.NewTerraformDashInModuleNameRule(),
	terraformrules.NewTerraformDashInDataSourceNameRule(),
	terraformrules.NewTerraformDeprecatedInterpolationRule(),
	terraformrules.NewTerraformDocumentedOutputsRule(),
	terraformrules.NewTerraformDocumentedVariablesRule(),
	terraformrules.NewTerraformModulePinnedSourceRule(),
	terraformrules.NewTerraformNamingConventionRule(),
	terraformrules.NewTerraformTypedVariablesRule(),
	terraformrules.NewTerraformRequiredVersionRule(),
}

var manualDeepCheckRules = []Rule{
	awsrules.NewAwsInstanceInvalidAMIRule(),
	awsrules.NewAwsLaunchConfigurationInvalidImageIDRule(),
}

// CheckRuleNames returns map of rules indexed by name
func CheckRuleNames(ruleNames []string) error {
	log.Print("[INFO] Checking rules")

	rulesMap := map[string]Rule{}
	for _, rule := range append(DefaultRules, deepCheckRules...) {
		rulesMap[rule.Name()] = rule
	}

	totalEnabled := 0
	for _, rule := range rulesMap {
		if rule.Enabled() {
			totalEnabled++
		}
	}
	log.Printf("[INFO]   %d (%d) rules total", len(rulesMap), totalEnabled)
	for _, rule := range ruleNames {
		if _, ok := rulesMap[rule]; !ok {
			return fmt.Errorf("Rule not found: %s", rule)
		}
	}
	return nil
}

// NewRules returns rules according to configuration
func NewRules(c *tflint.Config) []Rule {
	log.Print("[INFO] Prepare rules")

	ret := []Rule{}
	allRules := []Rule{}

	if c.DeepCheck {
		log.Printf("[DEBUG] Deep check mode is enabled. Add deep check rules")
		allRules = append(DefaultRules, deepCheckRules...)
	} else {
		allRules = DefaultRules
	}

	for _, rule := range allRules {
		enabled := rule.Enabled()
		if r := c.Rules[rule.Name()]; r != nil {
			if r.Enabled {
				log.Printf("[DEBUG] `%s` is enabled", rule.Name())
			} else {
				log.Printf("[DEBUG] `%s` is disabled", rule.Name())
			}
			enabled = r.Enabled
		}

		if enabled {
			ret = append(ret, rule)
		}
	}
	log.Printf("[INFO]   %d rules enabled", len(ret))
	return ret
}
<|endoftext|>


<commit_before>package v8js

import (
	"encoding/json"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/ry/v8worker"
	"reflect"
)

type jsCallEnvelope struct {
	Mod  string        `json:"m"`
	Fn   string        `json:"f"`
	Args []interface{} `json:"a"`
}

func (vu *VUContext) BridgeAPI(w *v8worker.Worker) error {
	for modname, mod := range vu.api {
		jsMod := fmt.Sprintf(`
		speedboat._modules["%s"] = {};
		`, modname)
		for name, mem := range mod {
			t := reflect.TypeOf(mem)

			if t.Kind() != reflect.Func {
				return errors.New("Not a function: " + modname + "." + name)
			}

			jsFn := fmt.Sprintf(`speedboat._modules["%s"]["%s"] = function() {
				var args = [];
			`, modname, name)

			numArgs := t.NumIn()
			if !t.IsVariadic() {
				jsFn += fmt.Sprintf(`
					if (arguments.length != %d) {
						throw new Error("wrong number of arguments");
					}
				`, t.NumIn())
			} else {
				numArgs--
			}

			for i := 0; i < numArgs; i++ {
				aT := t.In(i)
				jsFn += fmt.Sprintf("args.push(speedboat._require.%s(arguments[%d]));", aT.Kind().String(), i)
			}
			if t.IsVariadic() {
				varArg := t.In(numArgs)
				eT := varArg.Elem()
				jsFn += fmt.Sprintf(`
					for (var i = %d; i < arguments.length; i++) {
						args.push(speedboat._require.%s(arguments[i]));
					}
				`, numArgs, eT.Kind().String())
			}

			jsFn += fmt.Sprintf(`
				return speedboat._invoke('%s', '%s', args, %v);
			}`, modname, name, false)
			jsMod += "\n\n" + jsFn
		}

		if err := w.Load("module:"+modname, jsMod); err != nil {
			return err
		}
	}

	// Make functions in the "global" module global, preimport console
	makeGlobals := `
	for (key in speedboat._modules['global']) {
		eval(key + " = speedboat._modules['global']['" + key + "'];");
	}
	var console = speedboat._modules['console'];
	`
	if err := w.Load("internal:preload", makeGlobals); err != nil {
		return err
	}

	return nil
}

func (vu *VUContext) Recv(raw string) {
	call := jsCallEnvelope{}
	if err := json.Unmarshal([]byte(raw), &call); err != nil {
		log.WithError(err).Error("Malformed host call")
		return
	}
	log.WithFields(log.Fields{
		"mod":  call.Mod,
		"fn":   call.Fn,
		"args": call.Args,
	}).Debug("Async call")

	if err := vu.invoke(call); err != nil {
		log.WithError(err).Error("Couldn't invoke")
	}
}

func (vu *VUContext) RecvSync(raw string) string {
	call := jsCallEnvelope{}
	if err := json.Unmarshal([]byte(raw), &call); err != nil {
		return jsThrow(fmt.Sprintf("malformed host call: %s", err))
	}
	log.WithFields(log.Fields{
		"mod":  call.Mod,
		"fn":   call.Fn,
		"args": call.Args,
	}).Debug("Sync call")

	if err := vu.invoke(call); err != nil {
		return jsThrow(err.Error())
	}
	return ""
}

func (vu *VUContext) invoke(call jsCallEnvelope) error {
	mod, ok := vu.api[call.Mod]
	if !ok {
		return errors.New(fmt.Sprintf("unknown module '%s'", call.Mod))
	}

	mem, ok := mod[call.Fn]
	if !ok {
		return errors.New(fmt.Sprintf("unrecognized function call: '%s'.'%s'", call.Mod, call.Fn))
	}

	args := make([]reflect.Value, len(call.Args))
	for i, arg := range call.Args {
		args[i] = reflect.ValueOf(arg)
	}

	defer func() {
		if err := recover(); err != nil {
			log.WithField("error", err).Error("Go call panicked")
		}
	}()
	fn := reflect.ValueOf(mem)
	log.WithField("T", fn.Type().String()).Debug("Function")
	fn.Call(args)

	return nil
}
<commit_msg>Yield results from pipelines<commit_after>package v8js

import (
	"encoding/json"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat/runner"
	"github.com/ry/v8worker"
	"reflect"
)

type jsCallEnvelope struct {
	Mod  string        `json:"m"`
	Fn   string        `json:"f"`
	Args []interface{} `json:"a"`
}

func (vu *VUContext) BridgeAPI(w *v8worker.Worker) error {
	for modname, mod := range vu.api {
		jsMod := fmt.Sprintf(`
		speedboat._modules["%s"] = {};
		`, modname)
		for name, mem := range mod {
			t := reflect.TypeOf(mem)

			if t.Kind() != reflect.Func {
				return errors.New("Not a function: " + modname + "." + name)
			}

			jsFn := fmt.Sprintf(`speedboat._modules["%s"]["%s"] = function() {
				var args = [];
			`, modname, name)

			numArgs := t.NumIn()
			if !t.IsVariadic() {
				jsFn += fmt.Sprintf(`
					if (arguments.length != %d) {
						throw new Error("wrong number of arguments");
					}
				`, t.NumIn())
			} else {
				numArgs--
			}

			for i := 0; i < numArgs; i++ {
				aT := t.In(i)
				jsFn += fmt.Sprintf("args.push(speedboat._require.%s(arguments[%d]));", aT.Kind().String(), i)
			}
			if t.IsVariadic() {
				varArg := t.In(numArgs)
				eT := varArg.Elem()
				jsFn += fmt.Sprintf(`
					for (var i = %d; i < arguments.length; i++) {
						args.push(speedboat._require.%s(arguments[i]));
					}
				`, numArgs, eT.Kind().String())
			}

			jsFn += fmt.Sprintf(`
				return speedboat._invoke('%s', '%s', args, %v);
			}`, modname, name, false)
			jsMod += "\n\n" + jsFn
		}

		if err := w.Load("module:"+modname, jsMod); err != nil {
			return err
		}
	}

	// Make functions in the "global" module global, preimport console
	makeGlobals := `
	for (key in speedboat._modules['global']) {
		eval(key + " = speedboat._modules['global']['" + key + "'];");
	}
	var console = speedboat._modules['console'];
	`
	if err := w.Load("internal:preload", makeGlobals); err != nil {
		return err
	}

	return nil
}

func (vu *VUContext) Recv(raw string) {
	call := jsCallEnvelope{}
	if err := json.Unmarshal([]byte(raw), &call); err != nil {
		log.WithError(err).Error("Malformed host call")
		return
	}
	log.WithFields(log.Fields{
		"mod":  call.Mod,
		"fn":   call.Fn,
		"args": call.Args,
	}).Debug("Async call")

	if err := vu.invoke(call); err != nil {
		log.WithError(err).Error("Couldn't invoke")
	}
}

func (vu *VUContext) RecvSync(raw string) string {
	call := jsCallEnvelope{}
	if err := json.Unmarshal([]byte(raw), &call); err != nil {
		return jsThrow(fmt.Sprintf("malformed host call: %s", err))
	}
	log.WithFields(log.Fields{
		"mod":  call.Mod,
		"fn":   call.Fn,
		"args": call.Args,
	}).Debug("Sync call")

	if err := vu.invoke(call); err != nil {
		return jsThrow(err.Error())
	}
	return ""
}

func (vu *VUContext) invoke(call jsCallEnvelope) error {
	mod, ok := vu.api[call.Mod]
	if !ok {
		return errors.New(fmt.Sprintf("unknown module '%s'", call.Mod))
	}

	mem, ok := mod[call.Fn]
	if !ok {
		return errors.New(fmt.Sprintf("unrecognized function call: '%s'.'%s'", call.Mod, call.Fn))
	}

	args := make([]reflect.Value, len(call.Args))
	for i, arg := range call.Args {
		args[i] = reflect.ValueOf(arg)
	}

	defer func() {
		if err := recover(); err != nil {
			log.WithField("error", err).Error("Go call panicked")
		}
	}()
	fn := reflect.ValueOf(mem)
	ret := fn.Call(args)

	for _, val := range ret {
		switch v := val.Interface().(type) {
		case chan runner.Result:
			fallthrough
		case <-chan runner.Result:
		readLoop:
			for {
				select {
				case <-vu.ctx.Done():
					break readLoop
				case r, ok := <-v:
					if !ok {
						break readLoop
					}
					vu.ch <- r
				}
			}
		default:
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	"errors"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/http"
)

// A Signer knows how to create signatures suitable for inclusion in an HTTP
// request to S3.
type Signer interface {
	// Add an appropriate signature header to the supplied HTTP request.
	Sign(r *http.Request) error
}

// NewSigner creates a Signer using the supplied access key.
func NewSigner(key aws.AccessKey) (Signer, error) {
	return nil, errors.New("TODO: Implement NewSigner.")
}
<commit_msg>Split out a helper function.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package auth

import (
	"errors"
	"github.com/jacobsa/aws"
	"github.com/jacobsa/aws/s3/http"
)

// A Signer knows how to create signatures suitable for inclusion in an HTTP
// request to S3.
type Signer interface {
	// Add an appropriate signature header to the supplied HTTP request.
	Sign(r *http.Request) error
}

// NewSigner creates a Signer using the supplied access key.
func NewSigner(key aws.AccessKey) (Signer, error) {
	return newSigner(stringToSign, key)
}

// newSigner is a helper used by NewSigner, split out for testability. It
// allows you to inject the function that is used to determine the string to
// sign for any given request.
func newSigner(
	sts func(*http.Request) (string, error),
	key aws.AccessKey) (Signer, error) {
	return nil, errors.New("TODO: Implement newSigner.")
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package responsewriters

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/apiserver/pkg/util/flushwriter"
	"k8s.io/apiserver/pkg/util/wsstream"
)

// WriteObject renders a returned runtime.Object to the response as a stream or an encoded object. If the object
// returned by the response implements rest.ResourceStreamer that interface will be used to render the
// response. The Accept header and current API version will be passed in, and the output will be copied
// directly to the response body. If content type is returned it is used, otherwise the content type will
// be "application/octet-stream". All other objects are sent to standard JSON serialization.
func WriteObject(statusCode int, gv schema.GroupVersion, s runtime.NegotiatedSerializer, object runtime.Object, w http.ResponseWriter, req *http.Request) {
	stream, ok := object.(rest.ResourceStreamer)
	if !ok {
		WriteObjectNegotiated(s, gv, w, req, statusCode, object)
		return
	}

	out, flush, contentType, err := stream.InputStream(gv.String(), req.Header.Get("Accept"))
	if err != nil {
		ErrorNegotiated(err, s, gv, w, req)
		return
	}
	if out == nil {
		// No output provided - return StatusNoContent
		w.WriteHeader(http.StatusNoContent)
		return
	}
	defer out.Close()

	if wsstream.IsWebSocketRequest(req) {
		r := wsstream.NewReader(out, true, wsstream.NewDefaultReaderProtocols())
		if err := r.Copy(w, req); err != nil {
			utilruntime.HandleError(fmt.Errorf("error encountered while streaming results via websocket: %v", err))
		}
		return
	}

	if len(contentType) == 0 {
		contentType = "application/octet-stream"
	}
	w.Header().Set("Content-Type", contentType)
	w.WriteHeader(statusCode)
	writer := w.(io.Writer)
	if flush {
		writer = flushwriter.Wrap(w)
	}
	io.Copy(writer, out)
}

// WriteObjectNegotiated renders an object in the content type negotiated by the client
func WriteObjectNegotiated(s runtime.NegotiatedSerializer, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
	serializer, err := negotiation.NegotiateOutputSerializer(req, s)
	if err != nil {
		status := apiStatus(err)
		WriteRawJSON(int(status.Code), status, w)
		return
	}

	w.Header().Set("Content-Type", serializer.MediaType)
	w.WriteHeader(statusCode)

	encoder := s.EncoderForVersion(serializer.Serializer, gv)
	if err := encoder.Encode(object, w); err != nil {
		errorJSONFatal(err, encoder, w)
	}
}

// ErrorNegotiated renders an error to the response. Returns the HTTP status code of the error.
func ErrorNegotiated(err error, s runtime.NegotiatedSerializer, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request) int {
	status := apiStatus(err)
	code := int(status.Code)
	// when writing an error, check to see if the status indicates a retry after period
	if status.Details != nil && status.Details.RetryAfterSeconds > 0 {
		delay := strconv.Itoa(int(status.Details.RetryAfterSeconds))
		w.Header().Set("Retry-After", delay)
	}
	WriteObjectNegotiated(s, gv, w, req, code, status)
	return code
}

// errorJSONFatal renders an error to the response, and if codec fails will render plaintext.
// Returns the HTTP status code of the error.
func errorJSONFatal(err error, codec runtime.Encoder, w http.ResponseWriter) int {
	utilruntime.HandleError(fmt.Errorf("apiserver was unable to write a JSON response: %v", err))
	status := apiStatus(err)
	code := int(status.Code)
	output, err := runtime.Encode(codec, status)
	if err != nil {
		w.WriteHeader(code)
		fmt.Fprintf(w, "%s: %s", status.Reason, status.Message)
		return code
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	w.Write(output)
	return code
}

// WriteRawJSON writes a non-API object in JSON.
func WriteRawJSON(statusCode int, object interface{}, w http.ResponseWriter) {
	output, err := json.MarshalIndent(object, "", "  ")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	w.Write(output)
}
<commit_msg>apiserver: no Status in body for http 204<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package responsewriters

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
	"k8s.io/apiserver/pkg/registry/rest"
	"k8s.io/apiserver/pkg/util/flushwriter"
	"k8s.io/apiserver/pkg/util/wsstream"
)

// WriteObject renders a returned runtime.Object to the response as a stream or an encoded object. If the object
// returned by the response implements rest.ResourceStreamer that interface will be used to render the
// response. The Accept header and current API version will be passed in, and the output will be copied
// directly to the response body. If content type is returned it is used, otherwise the content type will
// be "application/octet-stream". All other objects are sent to standard JSON serialization.
func WriteObject(statusCode int, gv schema.GroupVersion, s runtime.NegotiatedSerializer, object runtime.Object, w http.ResponseWriter, req *http.Request) {
	stream, ok := object.(rest.ResourceStreamer)
	if !ok {
		WriteObjectNegotiated(s, gv, w, req, statusCode, object)
		return
	}

	out, flush, contentType, err := stream.InputStream(gv.String(), req.Header.Get("Accept"))
	if err != nil {
		ErrorNegotiated(err, s, gv, w, req)
		return
	}
	if out == nil {
		// No output provided - return StatusNoContent
		w.WriteHeader(http.StatusNoContent)
		return
	}
	defer out.Close()

	if wsstream.IsWebSocketRequest(req) {
		r := wsstream.NewReader(out, true, wsstream.NewDefaultReaderProtocols())
		if err := r.Copy(w, req); err != nil {
			utilruntime.HandleError(fmt.Errorf("error encountered while streaming results via websocket: %v", err))
		}
		return
	}

	if len(contentType) == 0 {
		contentType = "application/octet-stream"
	}
	w.Header().Set("Content-Type", contentType)
	w.WriteHeader(statusCode)
	writer := w.(io.Writer)
	if flush {
		writer = flushwriter.Wrap(w)
	}
	io.Copy(writer, out)
}

// WriteObjectNegotiated renders an object in the content type negotiated by the client
func WriteObjectNegotiated(s runtime.NegotiatedSerializer, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request, statusCode int, object runtime.Object) {
	serializer, err := negotiation.NegotiateOutputSerializer(req, s)
	if err != nil {
		status := apiStatus(err)
		WriteRawJSON(int(status.Code), status, w)
		return
	}

	w.Header().Set("Content-Type", serializer.MediaType)
	w.WriteHeader(statusCode)

	encoder := s.EncoderForVersion(serializer.Serializer, gv)
	if err := encoder.Encode(object, w); err != nil {
		errorJSONFatal(err, encoder, w)
	}
}

// ErrorNegotiated renders an error to the response. Returns the HTTP status code of the error.
func ErrorNegotiated(err error, s runtime.NegotiatedSerializer, gv schema.GroupVersion, w http.ResponseWriter, req *http.Request) int {
	status := apiStatus(err)
	code := int(status.Code)
	// when writing an error, check to see if the status indicates a retry after period
	if status.Details != nil && status.Details.RetryAfterSeconds > 0 {
		delay := strconv.Itoa(int(status.Details.RetryAfterSeconds))
		w.Header().Set("Retry-After", delay)
	}

	if code == http.StatusNoContent {
		w.WriteHeader(code)
		return code
	}

	WriteObjectNegotiated(s, gv, w, req, code, status)
	return code
}

// errorJSONFatal renders an error to the response, and if codec fails will render plaintext.
// Returns the HTTP status code of the error.
func errorJSONFatal(err error, codec runtime.Encoder, w http.ResponseWriter) int {
	utilruntime.HandleError(fmt.Errorf("apiserver was unable to write a JSON response: %v", err))
	status := apiStatus(err)
	code := int(status.Code)
	output, err := runtime.Encode(codec, status)
	if err != nil {
		w.WriteHeader(code)
		fmt.Fprintf(w, "%s: %s", status.Reason, status.Message)
		return code
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	w.Write(output)
	return code
}

// WriteRawJSON writes a non-API object in JSON.
func WriteRawJSON(statusCode int, object interface{}, w http.ResponseWriter) {
	output, err := json.MarshalIndent(object, "", "  ")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(statusCode)
	w.Write(output)
}
<|endoftext|>


<commit_before>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cli is the main entrypoint for runsc.
package cli

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"runtime"
	"time"

	"github.com/google/subcommands"
	"golang.org/x/sys/unix"
	"gvisor.dev/gvisor/pkg/coverage"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/refs"
	"gvisor.dev/gvisor/pkg/refsvfs2"
	"gvisor.dev/gvisor/pkg/sentry/platform"
	"gvisor.dev/gvisor/runsc/cmd"
	"gvisor.dev/gvisor/runsc/config"
	"gvisor.dev/gvisor/runsc/flag"
	"gvisor.dev/gvisor/runsc/specutils"
)

var (
	// Although these flags are not part of the OCI spec, they are used by
	// Docker, and thus should not be changed.
	// TODO(gvisor.dev/issue/193): support systemd cgroups
	systemdCgroup = flag.Bool("systemd-cgroup", false, "Use systemd for cgroups. NOT SUPPORTED.")
	showVersion   = flag.Bool("version", false, "show version and exit.")

	// These flags are unique to runsc, and are used to configure parts of the
	// system that are not covered by the runtime spec.

	// Debugging flags.
	logFD      = flag.Int("log-fd", -1, "file descriptor to log to.  If set, the 'log' flag is ignored.")
	debugLogFD = flag.Int("debug-log-fd", -1, "file descriptor to write debug logs to.  If set, the 'debug-log-dir' flag is ignored.")
	panicLogFD = flag.Int("panic-log-fd", -1, "file descriptor to write Go's runtime messages.")
	coverageFD = flag.Int("coverage-fd", -1, "file descriptor to write Go coverage output.")
)

// Main is the main entrypoint.
func Main(version string) {
	// Help and flags commands are generated automatically.
	help := cmd.NewHelp(subcommands.DefaultCommander)
	help.Register(new(cmd.Syscalls))
	subcommands.Register(help, "")
	subcommands.Register(subcommands.FlagsCommand(), "")

	// Installation helpers.
	const helperGroup = "helpers"
	subcommands.Register(new(cmd.Install), helperGroup)
	subcommands.Register(new(cmd.Uninstall), helperGroup)

	// Register user-facing runsc commands.
	subcommands.Register(new(cmd.Checkpoint), "")
	subcommands.Register(new(cmd.Create), "")
	subcommands.Register(new(cmd.Delete), "")
	subcommands.Register(new(cmd.Do), "")
	subcommands.Register(new(cmd.Events), "")
	subcommands.Register(new(cmd.Exec), "")
	subcommands.Register(new(cmd.Gofer), "")
	subcommands.Register(new(cmd.Kill), "")
	subcommands.Register(new(cmd.List), "")
	subcommands.Register(new(cmd.Pause), "")
	subcommands.Register(new(cmd.PS), "")
	subcommands.Register(new(cmd.Restore), "")
	subcommands.Register(new(cmd.Resume), "")
	subcommands.Register(new(cmd.Run), "")
	subcommands.Register(new(cmd.Spec), "")
	subcommands.Register(new(cmd.State), "")
	subcommands.Register(new(cmd.Start), "")
	subcommands.Register(new(cmd.Symbolize), "")
	subcommands.Register(new(cmd.Wait), "")
	subcommands.Register(new(cmd.Mitigate), "")
	subcommands.Register(new(cmd.VerityPrepare), "")

	// Register internal commands with the internal group name. This causes
	// them to be sorted below the user-facing commands with empty group.
	// The string below will be printed above the commands.
	const internalGroup = "internal use only"
	subcommands.Register(new(cmd.Boot), internalGroup)
	subcommands.Register(new(cmd.Debug), internalGroup)
	subcommands.Register(new(cmd.Gofer), internalGroup)
	subcommands.Register(new(cmd.Statefile), internalGroup)

	config.RegisterFlags()

	// All subcommands must be registered before flag parsing.
	flag.Parse()

	// Are we showing the version?
	if *showVersion {
		// The format here is the same as runc.
		fmt.Fprintf(os.Stdout, "runsc version %s\n", version)
		fmt.Fprintf(os.Stdout, "spec: %s\n", specutils.Version)
		os.Exit(0)
	}

	// Create a new Config from the flags.
	conf, err := config.NewFromFlags()
	if err != nil {
		cmd.Fatalf(err.Error())
	}

	// TODO(gvisor.dev/issue/193): support systemd cgroups
	if *systemdCgroup {
		fmt.Fprintln(os.Stderr, "systemd cgroup flag passed, but systemd cgroups not supported. See gvisor.dev/issue/193")
		os.Exit(1)
	}

	var errorLogger io.Writer
	if *logFD > -1 {
		errorLogger = os.NewFile(uintptr(*logFD), "error log file")

	} else if conf.LogFilename != "" {
		// We must set O_APPEND and not O_TRUNC because Docker passes
		// the same log file for all commands (and also parses these
		// log files), so we can't destroy them on each command.
		var err error
		errorLogger, err = os.OpenFile(conf.LogFilename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			cmd.Fatalf("error opening log file %q: %v", conf.LogFilename, err)
		}
	}
	cmd.ErrorLogger = errorLogger

	if _, err := platform.Lookup(conf.Platform); err != nil {
		cmd.Fatalf("%v", err)
	}

	// Sets the reference leak check mode. Also set it in config below to
	// propagate it to child processes.
	refs.SetLeakMode(conf.ReferenceLeak)

	// Set up logging.
	if conf.Debug {
		log.SetLevel(log.Debug)
	}

	// Logging will include the local date and time via the time package.
	//
	// On first use, time.Local initializes the local time zone, which
	// involves opening tzdata files on the host. Since this requires
	// opening host files, it must be done before syscall filter
	// installation.
	//
	// Generally there will be a log message before filter installation
	// that will force initialization, but force initialization here in
	// case that does not occur.
	_ = time.Local.String()

	subcommand := flag.CommandLine.Arg(0)

	var e log.Emitter
	if *debugLogFD > -1 {
		f := os.NewFile(uintptr(*debugLogFD), "debug log file")

		e = newEmitter(conf.DebugLogFormat, f)

	} else if conf.DebugLog != "" {
		f, err := specutils.DebugLogFile(conf.DebugLog, subcommand, "" /* name */)
		if err != nil {
			cmd.Fatalf("error opening debug log file in %q: %v", conf.DebugLog, err)
		}
		e = newEmitter(conf.DebugLogFormat, f)

	} else {
		// Stderr is reserved for the application, just discard the logs if no debug
		// log is specified.
		e = newEmitter("text", ioutil.Discard)
	}

	if *panicLogFD > -1 || *debugLogFD > -1 {
		fd := *panicLogFD
		if fd < 0 {
			fd = *debugLogFD
		}
		// Quick sanity check to make sure no other commands get passed
		// a log fd (they should use log dir instead).
		if subcommand != "boot" && subcommand != "gofer" {
			cmd.Fatalf("flags --debug-log-fd and --panic-log-fd should only be passed to 'boot' and 'gofer' command, but was passed to %q", subcommand)
		}

		// If we are the boot process, then we own our stdio FDs and can do what we
		// want with them. Since Docker and Containerd both eat boot's stderr, we
		// dup our stderr to the provided log FD so that panics will appear in the
		// logs, rather than just disappear.
		if err := unix.Dup3(fd, int(os.Stderr.Fd()), 0); err != nil {
			cmd.Fatalf("error dup'ing fd %d to stderr: %v", fd, err)
		}
	} else if conf.AlsoLogToStderr {
		e = &log.MultiEmitter{e, newEmitter(conf.DebugLogFormat, os.Stderr)}
	}
	if *coverageFD >= 0 {
		f := os.NewFile(uintptr(*coverageFD), "coverage file")
		coverage.EnableReport(f)
	}

	log.SetTarget(e)

	log.Infof("***************************")
	log.Infof("Args: %s", os.Args)
	log.Infof("Version %s", version)
	log.Infof("GOOS: %s", runtime.GOOS)
	log.Infof("GOARCH: %s", runtime.GOARCH)
	log.Infof("PID: %d", os.Getpid())
	log.Infof("UID: %d, GID: %d", os.Getuid(), os.Getgid())
	log.Infof("Configuration:")
	log.Infof("\t\tRootDir: %s", conf.RootDir)
	log.Infof("\t\tPlatform: %v", conf.Platform)
	log.Infof("\t\tFileAccess: %v, overlay: %t", conf.FileAccess, conf.Overlay)
	log.Infof("\t\tNetwork: %v, logging: %t", conf.Network, conf.LogPackets)
	log.Infof("\t\tStrace: %t, max size: %d, syscalls: %s", conf.Strace, conf.StraceLogSize, conf.StraceSyscalls)
	log.Infof("\t\tVFS2 enabled: %t, LISAFS: %t", conf.VFS2, conf.Lisafs)
	log.Infof("***************************")

	if conf.TestOnlyAllowRunAsCurrentUserWithoutChroot {
		// SIGTERM is sent to all processes if a test exceeds its
		// timeout and this case is handled by syscall_test_runner.
		log.Warningf("Block the TERM signal. This is only safe in tests!")
		signal.Ignore(unix.SIGTERM)
	}

	// Call the subcommand and pass in the configuration.
	var ws unix.WaitStatus
	subcmdCode := subcommands.Execute(context.Background(), conf, &ws)
	// Check for leaks and write coverage report before os.Exit().
	refsvfs2.DoLeakCheck()
	_ = coverage.Report()
	if subcmdCode == subcommands.ExitSuccess {
		log.Infof("Exiting with status: %v", ws)
		if ws.Signaled() {
			// No good way to return it, emulate what the shell does. Maybe raise
			// signal to self?
			os.Exit(128 + int(ws.Signal()))
		}
		os.Exit(ws.ExitStatus())
	}
	// Return an error that is unlikely to be used by the application.
	log.Warningf("Failure to execute command, err: %v", subcmdCode)
	os.Exit(128)
}

func newEmitter(format string, logFile io.Writer) log.Emitter {
	switch format {
	case "text":
		return log.GoogleEmitter{&log.Writer{Next: logFile}}
	case "json":
		return log.JSONEmitter{&log.Writer{Next: logFile}}
	case "json-k8s":
		return log.K8sJSONEmitter{&log.Writer{Next: logFile}}
	}
	cmd.Fatalf("invalid log format %q, must be 'text', 'json', or 'json-k8s'", format)
	panic("unreachable")
}
<commit_msg>Add Debug to log file header<commit_after>// Copyright 2018 The gVisor Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cli is the main entrypoint for runsc.
package cli

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/signal"
	"runtime"
	"time"

	"github.com/google/subcommands"
	"golang.org/x/sys/unix"
	"gvisor.dev/gvisor/pkg/coverage"
	"gvisor.dev/gvisor/pkg/log"
	"gvisor.dev/gvisor/pkg/refs"
	"gvisor.dev/gvisor/pkg/refsvfs2"
	"gvisor.dev/gvisor/pkg/sentry/platform"
	"gvisor.dev/gvisor/runsc/cmd"
	"gvisor.dev/gvisor/runsc/config"
	"gvisor.dev/gvisor/runsc/flag"
	"gvisor.dev/gvisor/runsc/specutils"
)

var (
	// Although these flags are not part of the OCI spec, they are used by
	// Docker, and thus should not be changed.
	// TODO(gvisor.dev/issue/193): support systemd cgroups
	systemdCgroup = flag.Bool("systemd-cgroup", false, "Use systemd for cgroups. NOT SUPPORTED.")
	showVersion   = flag.Bool("version", false, "show version and exit.")

	// These flags are unique to runsc, and are used to configure parts of the
	// system that are not covered by the runtime spec.

	// Debugging flags.
	logFD      = flag.Int("log-fd", -1, "file descriptor to log to.  If set, the 'log' flag is ignored.")
	debugLogFD = flag.Int("debug-log-fd", -1, "file descriptor to write debug logs to.  If set, the 'debug-log-dir' flag is ignored.")
	panicLogFD = flag.Int("panic-log-fd", -1, "file descriptor to write Go's runtime messages.")
	coverageFD = flag.Int("coverage-fd", -1, "file descriptor to write Go coverage output.")
)

// Main is the main entrypoint.
func Main(version string) {
	// Help and flags commands are generated automatically.
	help := cmd.NewHelp(subcommands.DefaultCommander)
	help.Register(new(cmd.Syscalls))
	subcommands.Register(help, "")
	subcommands.Register(subcommands.FlagsCommand(), "")

	// Installation helpers.
	const helperGroup = "helpers"
	subcommands.Register(new(cmd.Install), helperGroup)
	subcommands.Register(new(cmd.Uninstall), helperGroup)

	// Register user-facing runsc commands.
	subcommands.Register(new(cmd.Checkpoint), "")
	subcommands.Register(new(cmd.Create), "")
	subcommands.Register(new(cmd.Delete), "")
	subcommands.Register(new(cmd.Do), "")
	subcommands.Register(new(cmd.Events), "")
	subcommands.Register(new(cmd.Exec), "")
	subcommands.Register(new(cmd.Gofer), "")
	subcommands.Register(new(cmd.Kill), "")
	subcommands.Register(new(cmd.List), "")
	subcommands.Register(new(cmd.Pause), "")
	subcommands.Register(new(cmd.PS), "")
	subcommands.Register(new(cmd.Restore), "")
	subcommands.Register(new(cmd.Resume), "")
	subcommands.Register(new(cmd.Run), "")
	subcommands.Register(new(cmd.Spec), "")
	subcommands.Register(new(cmd.State), "")
	subcommands.Register(new(cmd.Start), "")
	subcommands.Register(new(cmd.Symbolize), "")
	subcommands.Register(new(cmd.Wait), "")
	subcommands.Register(new(cmd.Mitigate), "")
	subcommands.Register(new(cmd.VerityPrepare), "")

	// Register internal commands with the internal group name. This causes
	// them to be sorted below the user-facing commands with empty group.
	// The string below will be printed above the commands.
	const internalGroup = "internal use only"
	subcommands.Register(new(cmd.Boot), internalGroup)
	subcommands.Register(new(cmd.Debug), internalGroup)
	subcommands.Register(new(cmd.Gofer), internalGroup)
	subcommands.Register(new(cmd.Statefile), internalGroup)

	config.RegisterFlags()

	// All subcommands must be registered before flag parsing.
	flag.Parse()

	// Are we showing the version?
	if *showVersion {
		// The format here is the same as runc.
		fmt.Fprintf(os.Stdout, "runsc version %s\n", version)
		fmt.Fprintf(os.Stdout, "spec: %s\n", specutils.Version)
		os.Exit(0)
	}

	// Create a new Config from the flags.
	conf, err := config.NewFromFlags()
	if err != nil {
		cmd.Fatalf(err.Error())
	}

	// TODO(gvisor.dev/issue/193): support systemd cgroups
	if *systemdCgroup {
		fmt.Fprintln(os.Stderr, "systemd cgroup flag passed, but systemd cgroups not supported. See gvisor.dev/issue/193")
		os.Exit(1)
	}

	var errorLogger io.Writer
	if *logFD > -1 {
		errorLogger = os.NewFile(uintptr(*logFD), "error log file")

	} else if conf.LogFilename != "" {
		// We must set O_APPEND and not O_TRUNC because Docker passes
		// the same log file for all commands (and also parses these
		// log files), so we can't destroy them on each command.
		var err error
		errorLogger, err = os.OpenFile(conf.LogFilename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
		if err != nil {
			cmd.Fatalf("error opening log file %q: %v", conf.LogFilename, err)
		}
	}
	cmd.ErrorLogger = errorLogger

	if _, err := platform.Lookup(conf.Platform); err != nil {
		cmd.Fatalf("%v", err)
	}

	// Sets the reference leak check mode. Also set it in config below to
	// propagate it to child processes.
	refs.SetLeakMode(conf.ReferenceLeak)

	// Set up logging.
	if conf.Debug {
		log.SetLevel(log.Debug)
	}

	// Logging will include the local date and time via the time package.
	//
	// On first use, time.Local initializes the local time zone, which
	// involves opening tzdata files on the host. Since this requires
	// opening host files, it must be done before syscall filter
	// installation.
	//
	// Generally there will be a log message before filter installation
	// that will force initialization, but force initialization here in
	// case that does not occur.
	_ = time.Local.String()

	subcommand := flag.CommandLine.Arg(0)

	var e log.Emitter
	if *debugLogFD > -1 {
		f := os.NewFile(uintptr(*debugLogFD), "debug log file")

		e = newEmitter(conf.DebugLogFormat, f)

	} else if conf.DebugLog != "" {
		f, err := specutils.DebugLogFile(conf.DebugLog, subcommand, "" /* name */)
		if err != nil {
			cmd.Fatalf("error opening debug log file in %q: %v", conf.DebugLog, err)
		}
		e = newEmitter(conf.DebugLogFormat, f)

	} else {
		// Stderr is reserved for the application, just discard the logs if no debug
		// log is specified.
		e = newEmitter("text", ioutil.Discard)
	}

	if *panicLogFD > -1 || *debugLogFD > -1 {
		fd := *panicLogFD
		if fd < 0 {
			fd = *debugLogFD
		}
		// Quick sanity check to make sure no other commands get passed
		// a log fd (they should use log dir instead).
		if subcommand != "boot" && subcommand != "gofer" {
			cmd.Fatalf("flags --debug-log-fd and --panic-log-fd should only be passed to 'boot' and 'gofer' command, but was passed to %q", subcommand)
		}

		// If we are the boot process, then we own our stdio FDs and can do what we
		// want with them. Since Docker and Containerd both eat boot's stderr, we
		// dup our stderr to the provided log FD so that panics will appear in the
		// logs, rather than just disappear.
		if err := unix.Dup3(fd, int(os.Stderr.Fd()), 0); err != nil {
			cmd.Fatalf("error dup'ing fd %d to stderr: %v", fd, err)
		}
	} else if conf.AlsoLogToStderr {
		e = &log.MultiEmitter{e, newEmitter(conf.DebugLogFormat, os.Stderr)}
	}
	if *coverageFD >= 0 {
		f := os.NewFile(uintptr(*coverageFD), "coverage file")
		coverage.EnableReport(f)
	}

	log.SetTarget(e)

	log.Infof("***************************")
	log.Infof("Args: %s", os.Args)
	log.Infof("Version %s", version)
	log.Infof("GOOS: %s", runtime.GOOS)
	log.Infof("GOARCH: %s", runtime.GOARCH)
	log.Infof("PID: %d", os.Getpid())
	log.Infof("UID: %d, GID: %d", os.Getuid(), os.Getgid())
	log.Infof("Configuration:")
	log.Infof("\t\tRootDir: %s", conf.RootDir)
	log.Infof("\t\tPlatform: %v", conf.Platform)
	log.Infof("\t\tFileAccess: %v, overlay: %t", conf.FileAccess, conf.Overlay)
	log.Infof("\t\tNetwork: %v, logging: %t", conf.Network, conf.LogPackets)
	log.Infof("\t\tStrace: %t, max size: %d, syscalls: %s", conf.Strace, conf.StraceLogSize, conf.StraceSyscalls)
	log.Infof("\t\tVFS2 enabled: %t, LISAFS: %t", conf.VFS2, conf.Lisafs)
	log.Infof("\t\tDebug: %v", conf.Debug)
	log.Infof("***************************")

	if conf.TestOnlyAllowRunAsCurrentUserWithoutChroot {
		// SIGTERM is sent to all processes if a test exceeds its
		// timeout and this case is handled by syscall_test_runner.
		log.Warningf("Block the TERM signal. This is only safe in tests!")
		signal.Ignore(unix.SIGTERM)
	}

	// Call the subcommand and pass in the configuration.
	var ws unix.WaitStatus
	subcmdCode := subcommands.Execute(context.Background(), conf, &ws)
	// Check for leaks and write coverage report before os.Exit().
	refsvfs2.DoLeakCheck()
	_ = coverage.Report()
	if subcmdCode == subcommands.ExitSuccess {
		log.Infof("Exiting with status: %v", ws)
		if ws.Signaled() {
			// No good way to return it, emulate what the shell does. Maybe raise
			// signal to self?
			os.Exit(128 + int(ws.Signal()))
		}
		os.Exit(ws.ExitStatus())
	}
	// Return an error that is unlikely to be used by the application.
	log.Warningf("Failure to execute command, err: %v", subcmdCode)
	os.Exit(128)
}

func newEmitter(format string, logFile io.Writer) log.Emitter {
	switch format {
	case "text":
		return log.GoogleEmitter{&log.Writer{Next: logFile}}
	case "json":
		return log.JSONEmitter{&log.Writer{Next: logFile}}
	case "json-k8s":
		return log.K8sJSONEmitter{&log.Writer{Next: logFile}}
	}
	cmd.Fatalf("invalid log format %q, must be 'text', 'json', or 'json-k8s'", format)
	panic("unreachable")
}
<|endoftext|>


<commit_before>package oauth2_test

import (
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"
	"github.com/ory-am/fosite"
	foauth2 "github.com/ory-am/fosite/handler/oauth2"
	"github.com/ory-am/hydra/herodot"
	"github.com/ory-am/hydra/oauth2"
	"github.com/ory-am/hydra/pkg"
	"github.com/ory-am/hydra/warden"
	"github.com/ory-am/ladon"
	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
	goauth2 "golang.org/x/oauth2"
)

var (
	introspectors = make(map[string]oauth2.Introspector)
	now           = time.Now().Round(time.Second)
	tokens        = pkg.Tokens(3)
	fositeStore   = pkg.FositeStore()
)

var ladonWarden = pkg.LadonWarden(map[string]ladon.Policy{
	"1": &ladon.DefaultPolicy{
		ID:        "1",
		Subjects:  []string{"alice"},
		Resources: []string{"matrix", "rn:hydra:token<.*>"},
		Actions:   []string{"create", "decide"},
		Effect:    ladon.AllowAccess,
	},
	"2": &ladon.DefaultPolicy{
		ID:        "2",
		Subjects:  []string{"siri"},
		Resources: []string{"<.*>"},
		Actions:   []string{"decide"},
		Effect:    ladon.AllowAccess,
	},
})

var localWarden = &warden.LocalWarden{
	Warden: ladonWarden,
	OAuth2: &fosite.Fosite{
		Store: fositeStore,
		TokenValidators: fosite.TokenValidators{
			0: &foauth2.CoreValidator{
				CoreStrategy:  pkg.HMACStrategy,
				CoreStorage:   fositeStore,
				ScopeStrategy: fosite.HierarchicScopeStrategy,
			},
		},
		ScopeStrategy: fosite.HierarchicScopeStrategy,
	},
	Issuer:              "tests",
	AccessTokenLifespan: time.Hour,
}

func init() {
	introspectors["local"] = &oauth2.LocalIntrospector{
		OAuth2:              localWarden.OAuth2,
		Issuer:              "tests",
		AccessTokenLifespan: time.Hour,
	}

	r := httprouter.New()
	serv := &oauth2.Handler{
		Firewall:     localWarden,
		H:            &herodot.JSON{},
		Introspector: introspectors["local"],
	}
	serv.SetRoutes(r)
	ts = httptest.NewServer(r)

	ar := fosite.NewAccessRequest(oauth2.NewSession("alice"))
	ar.GrantedScopes = fosite.Arguments{0: "core"}
	ar.RequestedAt = now
	ar.Client = &fosite.DefaultClient{ID: "siri"}
	ar.Session.(*oauth2.Session).Extra = map[string]interface{}{"foo": "bar"}
	fositeStore.CreateAccessTokenSession(nil, tokens[0][0], ar)

	ar2 := fosite.NewAccessRequest(oauth2.NewSession("siri"))
	ar2.GrantedScopes = fosite.Arguments{0: "core"}
	ar2.RequestedAt = now
	ar2.Session.(*oauth2.Session).Extra = map[string]interface{}{"foo": "bar"}
	ar2.Client = &fosite.DefaultClient{ID: "siri"}
	fositeStore.CreateAccessTokenSession(nil, tokens[1][0], ar2)

	ar3 := fosite.NewAccessRequest(oauth2.NewSession("siri"))
	ar3.GrantedScopes = fosite.Arguments{0: "core"}
	ar3.RequestedAt = now
	ar2.Session.(*oauth2.Session).Extra = map[string]interface{}{"foo": "bar"}
	ar3.Client = &fosite.DefaultClient{ID: "doesnt-exist"}
	ar3.Session.(*oauth2.Session).AccessTokenExpiry = time.Now().Add(-time.Hour)
	fositeStore.CreateAccessTokenSession(nil, tokens[2][0], ar3)

	conf := &goauth2.Config{
		Scopes:   []string{},
		Endpoint: goauth2.Endpoint{},
	}

	ep, err := url.Parse(ts.URL)
	if err != nil {
		logrus.Fatalf("%s", err)
	}
	introspectors["http"] = &oauth2.HTTPIntrospector{
		Endpoint: ep,
		Client: conf.Client(goauth2.NoContext, &goauth2.Token{
			AccessToken: tokens[1][1],
			Expiry:      time.Now().Add(time.Hour),
			TokenType:   "bearer",
		}),
	}
}

func TestIntrospect(t *testing.T) {
	for _, w := range introspectors {
		for _, c := range []struct {
			token     string
			expectErr bool
			assert    func(*oauth2.Introspection)
		}{
			{
				token:     "invalid",
				expectErr: true,
			},
			{
				token:     tokens[2][1],
				expectErr: true,
			},
			{
				token:     tokens[1][1],
				expectErr: false,
			},
			{
				token:     tokens[0][1],
				expectErr: false,
				assert: func(c *oauth2.Introspection) {
					assert.Equal(t, "alice", c.Subject)
					assert.Equal(t, "tests", c.Issuer)
					assert.Equal(t, now.Add(time.Hour).Unix(), c.ExpiresAt, "expires at")
					assert.Equal(t, now.Unix(), c.IssuedAt, "issued at")
					assert.Equal(t, map[string]interface{}{"foo": "bar"}, c.Extra)
				},
			},
		} {
			ctx, err := w.IntrospectToken(context.Background(), c.token)
			pkg.AssertError(t, c.expectErr, err)
			if err == nil && c.assert != nil {
				c.assert(ctx)
			}
		}
	}
}
<commit_msg>oauth2: Removed vetted 0\'s from tests for better code overview<commit_after>package oauth2_test

import (
	"net/http/httptest"
	"net/url"
	"testing"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"
	"github.com/ory-am/fosite"
	foauth2 "github.com/ory-am/fosite/handler/oauth2"
	"github.com/ory-am/hydra/herodot"
	"github.com/ory-am/hydra/oauth2"
	"github.com/ory-am/hydra/pkg"
	"github.com/ory-am/hydra/warden"
	"github.com/ory-am/ladon"
	"github.com/stretchr/testify/assert"
	"golang.org/x/net/context"
	goauth2 "golang.org/x/oauth2"
)

var (
	introspectors = make(map[string]oauth2.Introspector)
	now           = time.Now().Round(time.Second)
	tokens        = pkg.Tokens(3)
	fositeStore   = pkg.FositeStore()
)

var ladonWarden = pkg.LadonWarden(map[string]ladon.Policy{
	"1": &ladon.DefaultPolicy{
		ID:        "1",
		Subjects:  []string{"alice"},
		Resources: []string{"matrix", "rn:hydra:token<.*>"},
		Actions:   []string{"create", "decide"},
		Effect:    ladon.AllowAccess,
	},
	"2": &ladon.DefaultPolicy{
		ID:        "2",
		Subjects:  []string{"siri"},
		Resources: []string{"<.*>"},
		Actions:   []string{"decide"},
		Effect:    ladon.AllowAccess,
	},
})

var localWarden = &warden.LocalWarden{
	Warden: ladonWarden,
	OAuth2: &fosite.Fosite{
		Store: fositeStore,
		TokenValidators: fosite.TokenValidators{
			&foauth2.CoreValidator{
				CoreStrategy:  pkg.HMACStrategy,
				CoreStorage:   fositeStore,
				ScopeStrategy: fosite.HierarchicScopeStrategy,
			},
		},
		ScopeStrategy: fosite.HierarchicScopeStrategy,
	},
	Issuer:              "tests",
	AccessTokenLifespan: time.Hour,
}

func init() {
	introspectors["local"] = &oauth2.LocalIntrospector{
		OAuth2:              localWarden.OAuth2,
		Issuer:              "tests",
		AccessTokenLifespan: time.Hour,
	}

	r := httprouter.New()
	serv := &oauth2.Handler{
		Firewall:     localWarden,
		H:            &herodot.JSON{},
		Introspector: introspectors["local"],
	}
	serv.SetRoutes(r)
	ts = httptest.NewServer(r)

	ar := fosite.NewAccessRequest(oauth2.NewSession("alice"))
	ar.GrantedScopes = fosite.Arguments{"core"}
	ar.RequestedAt = now
	ar.Client = &fosite.DefaultClient{ID: "siri"}
	ar.Session.(*oauth2.Session).Extra = map[string]interface{}{"foo": "bar"}
	fositeStore.CreateAccessTokenSession(nil, tokens[0][0], ar)

	ar2 := fosite.NewAccessRequest(oauth2.NewSession("siri"))
	ar2.GrantedScopes = fosite.Arguments{"core"}
	ar2.RequestedAt = now
	ar2.Session.(*oauth2.Session).Extra = map[string]interface{}{"foo": "bar"}
	ar2.Client = &fosite.DefaultClient{ID: "siri"}
	fositeStore.CreateAccessTokenSession(nil, tokens[1][0], ar2)

	ar3 := fosite.NewAccessRequest(oauth2.NewSession("siri"))
	ar3.GrantedScopes = fosite.Arguments{"core"}
	ar3.RequestedAt = now
	ar2.Session.(*oauth2.Session).Extra = map[string]interface{}{"foo": "bar"}
	ar3.Client = &fosite.DefaultClient{ID: "doesnt-exist"}
	ar3.Session.(*oauth2.Session).AccessTokenExpiry = time.Now().Add(-time.Hour)
	fositeStore.CreateAccessTokenSession(nil, tokens[2][0], ar3)

	conf := &goauth2.Config{
		Scopes:   []string{},
		Endpoint: goauth2.Endpoint{},
	}

	ep, err := url.Parse(ts.URL)
	if err != nil {
		logrus.Fatalf("%s", err)
	}
	introspectors["http"] = &oauth2.HTTPIntrospector{
		Endpoint: ep,
		Client: conf.Client(goauth2.NoContext, &goauth2.Token{
			AccessToken: tokens[1][1],
			Expiry:      time.Now().Add(time.Hour),
			TokenType:   "bearer",
		}),
	}
}

func TestIntrospect(t *testing.T) {
	for _, w := range introspectors {
		for _, c := range []struct {
			token     string
			expectErr bool
			assert    func(*oauth2.Introspection)
		}{
			{
				token:     "invalid",
				expectErr: true,
			},
			{
				token:     tokens[2][1],
				expectErr: true,
			},
			{
				token:     tokens[1][1],
				expectErr: false,
			},
			{
				token:     tokens[0][1],
				expectErr: false,
				assert: func(c *oauth2.Introspection) {
					assert.Equal(t, "alice", c.Subject)
					assert.Equal(t, "tests", c.Issuer)
					assert.Equal(t, now.Add(time.Hour).Unix(), c.ExpiresAt, "expires at")
					assert.Equal(t, now.Unix(), c.IssuedAt, "issued at")
					assert.Equal(t, map[string]interface{}{"foo": "bar"}, c.Extra)
				},
			},
		} {
			ctx, err := w.IntrospectToken(context.Background(), c.token)
			pkg.AssertError(t, c.expectErr, err)
			if err == nil && c.assert != nil {
				c.assert(ctx)
			}
		}
	}
}
<|endoftext|>


<commit_before>package signalstatus

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"
)

var StatusURL string = "http://status.37signals.com/status.json"

type StatusCheck struct {
	Status StatusBody
}

type StatusBody struct {
	Mood        string
	Description string
	UpdatedAt   time.Time
}

type StatusResponse struct {
	Status StatusBody
}

func Status() (StatusCheck, error) {
	sc := StatusCheck{}

	err := sc.Run()
	if err != nil {
		return StatusCheck{}, err
	}

	return sc, nil
}

func (sc *StatusCheck) Run() error {
	resp, err := http.Get(StatusURL)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	status_response := StatusResponse{}
	err = json.Unmarshal(body, &status_response)
	if err != nil {
		return err
	}

	sc.Status = status_response.Status

	return nil
}

func (sc *StatusCheck) OK() bool {
	return sc.Status.Mood == "good"
}
<commit_msg>Refactor signalstatus slightly<commit_after>package signalstatus

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
)

var StatusURL string = "http://status.37signals.com/status.json"

type StatusCheck struct {
	Status StatusBody
}

type StatusBody struct {
	Mood        string
	Description string
	UpdatedAt   string
}

func Status() (StatusCheck, error) {
	sc := StatusCheck{}

	err := sc.Run()
	if err != nil {
		return StatusCheck{}, err
	}

	return sc, nil
}

func (sc *StatusCheck) Run() error {
	resp, err := http.Get(StatusURL)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	err = json.Unmarshal(body, &sc)
	if err != nil {
		return err
	}

	return nil
}

func (sc *StatusCheck) OK() bool {
	return sc.Status.Mood == "good"
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/stealthly/go-kafka/producer"
	"github.com/stealthly/go-kafka/consumer"
	"time"
	"os"
	"os/signal"
	"github.com/stealthly/go-avro/decoder"
	"math/big"
	"code.google.com/p/go-uuid/uuid"
	"io/ioutil"
)

type PingPong struct {
	Counter int64
	Name    string
	Uuid	string
}

var schemaRegistry = map[int64]string {
	int64(0): "./scalago.avsc",
}

var readTopic string
var writeTopic string
var group = "ping-pong-go-group"

var broker = "localhost:9092"
var zookeeper = "localhost:2181"

var kafkaProducer *producer.KafkaProducer = nil
var kafkaConsumer *consumer.KafkaConsumerGroup = nil

func main() {
	parseArgs()

	kafkaProducer = producer.NewKafkaProducer(writeTopic, []string{broker}, nil)
	kafkaConsumer = consumer.NewKafkaConsumerGroup(readTopic, group, []string{zookeeper}, nil)

	p := &PingPong{}
	pingPongLoop(p)
}

func parseArgs() {
	if len(os.Args) < 3 {
		panic("Usage: go run scala_go_kafka.go $READ_TOPIC $WRITE_TOPIC")
	}

	readTopic = os.Args[1]
	writeTopic = os.Args[2]
}

func pingPongLoop(p *PingPong) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		fmt.Println("\ngolang > Closing consumer")
		kafkaConsumer.Close()
	}()

	fmt.Println("golang > Started!")
	kafkaConsumer.Read(func(bytes []byte) {
		time.Sleep(2 * time.Second)
		decode(p, bytes)
		fmt.Printf("golang > received %#v\n", p)
		modify(p)
		kafkaProducer.SendBytes(encode(p))
	})
}

func modify(obj *PingPong) {
	obj.Counter++
	obj.Uuid = uuid.New()
}

func encode(obj *PingPong) []byte {
	enc := decoder.NewBinaryEncoder()

	data := []byte {CAMUS_MAGIC}
	data = append(data, []byte{0x00, 0x00, 0x00, 0x00}...)
	data = append(data, enc.WriteLong(obj.Counter)...)
	data = append(data, enc.WriteString(obj.Name)...)
	data = append(data, enc.WriteString(obj.Uuid)...)
	return data
}

func decode(obj interface{}, bytes []byte) {
	NewCamusData(bytes).Read(obj)
}

func schemaById(bytes []byte) *decoder.Schema {
	id := new (big.Int)
	id.SetBytes(bytes)
	schemaFile := schemaRegistry[id.Int64()]
	if schemaBytes, err := ioutil.ReadFile(schemaFile); err != nil {
		panic(err)
	} else {
		return decoder.AvroSchema(schemaBytes)
	}
}

var CAMUS_MAGIC byte = byte(0)

type CamusData struct {
	dec *decoder.BinaryDecoder
	datumReader decoder.DatumReader
}

func NewCamusData(data []byte) *CamusData {
	dec := decoder.NewBinaryDecoder(data)
	if magic, err := dec.ReadInt(); err != nil {
		panic(err)
	} else {
		if byte(magic) != CAMUS_MAGIC {
			panic("Wrong Camus magic byte")
		}

		schemaIdArray := make([]byte, 4)
		dec.ReadFixed(schemaIdArray)
		schema := schemaById(schemaIdArray)
		datumReader := decoder.NewGenericDatumReader()
		datumReader.SetSchema(schema)

		return &CamusData{dec, datumReader}
	}
}

func (cd *CamusData) Read(obj interface{}) {
	cd.datumReader.Read(obj, cd.dec)
}
<commit_msg>added better string representation to match scala's version<commit_after>package main

import (
	"fmt"
	"github.com/stealthly/go-kafka/producer"
	"github.com/stealthly/go-kafka/consumer"
	"time"
	"os"
	"os/signal"
	"github.com/stealthly/go-avro/decoder"
	"math/big"
	"code.google.com/p/go-uuid/uuid"
	"io/ioutil"
)

type PingPong struct {
	Counter int64
	Name    string
	Uuid	string
}

//custom string representation to match Scala version. just to simplify reading the console output
func (p *PingPong) String() string {
	return fmt.Sprintf("{\"counter\": %d, \"name\": \"%s\", \"uuid\": \"%s\"}", p.Counter, p.Name, p.Uuid)
}

var schemaRegistry = map[int64]string {
	int64(0): "./scalago.avsc",
}

var readTopic string
var writeTopic string
var group = "ping-pong-go-group"

var broker = "localhost:9092"
var zookeeper = "localhost:2181"

var kafkaProducer *producer.KafkaProducer = nil
var kafkaConsumer *consumer.KafkaConsumerGroup = nil

func main() {
	parseArgs()

	kafkaProducer = producer.NewKafkaProducer(writeTopic, []string{broker}, nil)
	kafkaConsumer = consumer.NewKafkaConsumerGroup(readTopic, group, []string{zookeeper}, nil)

	p := &PingPong{}
	pingPongLoop(p)
}

func parseArgs() {
	if len(os.Args) < 3 {
		panic("Usage: go run scala_go_kafka.go $READ_TOPIC $WRITE_TOPIC")
	}

	readTopic = os.Args[1]
	writeTopic = os.Args[2]
}

func pingPongLoop(p *PingPong) {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		fmt.Println("\ngolang > Closing consumer")
		kafkaConsumer.Close()
	}()

	fmt.Println("golang > Started!")
	kafkaConsumer.Read(func(bytes []byte) {
		time.Sleep(2 * time.Second)
		decode(p, bytes)
		fmt.Printf("golang > received %v\n", p)
		modify(p)
		kafkaProducer.SendBytes(encode(p))
	})
}

func modify(obj *PingPong) {
	obj.Counter++
	obj.Uuid = uuid.New()
}

func encode(obj *PingPong) []byte {
	enc := decoder.NewBinaryEncoder()

	data := []byte {CAMUS_MAGIC}
	data = append(data, []byte{0x00, 0x00, 0x00, 0x00}...)
	data = append(data, enc.WriteLong(obj.Counter)...)
	data = append(data, enc.WriteString(obj.Name)...)
	data = append(data, enc.WriteString(obj.Uuid)...)
	return data
}

func decode(obj interface{}, bytes []byte) {
	NewCamusData(bytes).Read(obj)
}

func schemaById(bytes []byte) *decoder.Schema {
	id := new (big.Int)
	id.SetBytes(bytes)
	schemaFile := schemaRegistry[id.Int64()]
	if schemaBytes, err := ioutil.ReadFile(schemaFile); err != nil {
		panic(err)
	} else {
		return decoder.AvroSchema(schemaBytes)
	}
}

var CAMUS_MAGIC byte = byte(0)

type CamusData struct {
	dec *decoder.BinaryDecoder
	datumReader decoder.DatumReader
}

func NewCamusData(data []byte) *CamusData {
	dec := decoder.NewBinaryDecoder(data)
	if magic, err := dec.ReadInt(); err != nil {
		panic(err)
	} else {
		if byte(magic) != CAMUS_MAGIC {
			panic("Wrong Camus magic byte")
		}

		schemaIdArray := make([]byte, 4)
		dec.ReadFixed(schemaIdArray)
		schema := schemaById(schemaIdArray)
		datumReader := decoder.NewGenericDatumReader()
		datumReader.SetSchema(schema)

		return &CamusData{dec, datumReader}
	}
}

func (cd *CamusData) Read(obj interface{}) {
	cd.datumReader.Read(obj, cd.dec)
}
<|endoftext|>


<commit_before>package recurrent

import (
	"testing"
	"time"

	"github.com/aphistic/sweet"
	"github.com/aphistic/sweet-junit"
	"github.com/efritz/glock"
	. "github.com/onsi/gomega"
)

const (
	TIMEOUT  = time.Millisecond * 1
	INTERVAL = time.Millisecond / 100
)

func TestMain(m *testing.M) {
	RegisterFailHandler(sweet.GomegaFail)

	sweet.Run(m, func(s *sweet.S) {
		s.RegisterPlugin(junit.NewPlugin())
		s.AddSuite(&SchedulerSuite{})
	})
}

type SchedulerSuite struct{}

func (s *SchedulerSuite) TestAutomaticPeriod(t sweet.T) {
	var (
		clock    = glock.NewMockClock()
		sync     = make(chan struct{})
		done     = make(chan struct{})
		attempts = 0
	)

	defer close(sync)

	scheduler := NewScheduler(
		func() {
			attempts++
			sync <- struct{}{}
		},
		WithInterval(time.Second),
		withClock(clock),
	)

	go func() {
		defer close(done)

		for i := 0; i < 25; i++ {
			clock.BlockingAdvance(time.Second)
			<-sync
		}
	}()

	scheduler.Start()
	<-done
	scheduler.Stop()
	Expect(attempts).To(Equal(25))
	Expect(clock.GetAfterArgs()[0]).To(Equal(time.Second))
}

func (s *SchedulerSuite) TestThrottledSchedule(t sweet.T) {
	var (
		clock    = glock.NewMockClock()
		sync     = make(chan struct{})
		done     = make(chan struct{})
		attempts = 0
	)

	scheduler := NewScheduler(
		func() {
			attempts++
			sync <- struct{}{}
		},
		WithInterval(time.Second),
		WithThrottle(time.Millisecond),
		withClock(clock),
	)

	go func() {
		defer close(done)

		for i := 0; i < 25; i++ {
			clock.Advance(time.Second)
			<-sync
		}
	}()

	go func() {
		for {
			select {
			case <-done:
				return

			default:
			}

			clock.Advance(time.Second)
		}
	}()

	scheduler.Start()
	<-done
	scheduler.Stop()
	Expect(attempts).To(Equal(25))

	tickerArgs := clock.GetTickerArgs()
	Expect(tickerArgs).To(HaveLen(1))
	Expect(tickerArgs[0]).To(Equal(time.Millisecond))
}

func (s *SchedulerSuite) TestExplicitFire(t sweet.T) {
	var (
		clock    = glock.NewMockClock()
		sync     = make(chan struct{})
		done     = make(chan struct{})
		attempts = 0
	)

	defer close(sync)

	scheduler := NewScheduler(
		func() {
			attempts++
			sync <- struct{}{}
		},
		WithInterval(time.Second),
		withClock(clock),
	)

	go func() {
		defer close(done)

		for i := 0; i < 25; i++ {
			scheduler.Signal()
			<-sync
		}
	}()

	scheduler.Start()
	<-done
	scheduler.Stop()
	Expect(attempts).To(Equal(25))
}

func (s *SchedulerSuite) TestThrottledExplicitFire(t sweet.T) {
	var (
		clock    = glock.NewMockClock()
		sync     = make(chan struct{})
		done     = make(chan struct{})
		attempts = 0
	)

	defer close(sync)

	scheduler := NewScheduler(
		func() {
			attempts++
			sync <- struct{}{}
		},
		WithInterval(time.Second),
		WithThrottle(time.Millisecond),
		withClock(clock),
	)

	go func() {
		defer close(done)

		for i := 0; i < 100; i++ {
			scheduler.Signal()

			if i%4 == 0 {
				clock.Advance(time.Second)
				<-sync
			}
		}
	}()

	scheduler.Start()
	<-done
	scheduler.Stop()
	Expect(attempts).To(Equal(25))
}

func (s *SchedulerSuite) TestHammer(t sweet.T) {
	var (
		quit = make(chan struct{})
		ch   = hammer(quit)
	)

	for i := 1; i <= 200; i++ {
		<-ch
	}

	close(quit)
	eventually(ch).Should(BeClosed())
}

func (s *SchedulerSuite) TestConvert(t sweet.T) {
	var (
		ch1 = make(chan time.Time)
		ch2 = convert(ch1)
	)

	go func() {
		for i := 1; i <= 200; i++ {
			ch1 <- time.Now().Add(time.Minute * time.Duration(-i))
		}
	}()

	for i := 1; i <= 200; i++ {
		eventually(ch2).Should(Receive(Equal(struct{}{})))
	}

	close(ch1)
	eventually(ch2).Should(BeClosed())
}

func (s *SchedulerSuite) TestThrottle(t sweet.T) {
	var (
		ch1 = make(chan struct{})
		ch2 = make(chan struct{})
		ch3 = throttle(ch1, ch2)
	)

	Consistently(ch2).ShouldNot(Receive())
	ch1 <- struct{}{}

	go func() {
		ch2 <- struct{}{}
	}()

	eventually(ch3).Should(Receive(Equal(struct{}{})))
	close(ch1)
	eventually(ch3).Should(BeClosed())
}

//
// Overwrite gomega's default timeout and interval

func eventually(actual interface{}) GomegaAsyncAssertion {
	return Eventually(actual, TIMEOUT, INTERVAL)
}
<commit_msg>Update test to avoid race condition.<commit_after>package recurrent

import (
	"testing"
	"time"

	"github.com/aphistic/sweet"
	"github.com/aphistic/sweet-junit"
	"github.com/efritz/glock"
	. "github.com/onsi/gomega"
)

const (
	TIMEOUT  = time.Millisecond * 1
	INTERVAL = time.Millisecond / 100
)

func TestMain(m *testing.M) {
	RegisterFailHandler(sweet.GomegaFail)

	sweet.Run(m, func(s *sweet.S) {
		s.RegisterPlugin(junit.NewPlugin())
		s.AddSuite(&SchedulerSuite{})
	})
}

type SchedulerSuite struct{}

func (s *SchedulerSuite) TestAutomaticPeriod(t sweet.T) {
	var (
		clock    = glock.NewMockClock()
		sync     = make(chan struct{})
		done     = make(chan struct{})
		attempts = 0
	)

	defer close(sync)

	scheduler := NewScheduler(
		func() {
			attempts++
			sync <- struct{}{}
		},
		WithInterval(time.Second),
		withClock(clock),
	)

	go func() {
		defer close(done)

		for i := 0; i < 25; i++ {
			clock.BlockingAdvance(time.Second)
			<-sync
		}
	}()

	scheduler.Start()
	<-done
	scheduler.Stop()
	Expect(attempts).To(Equal(25))
	Expect(clock.GetAfterArgs()[0]).To(Equal(time.Second))
}

func (s *SchedulerSuite) TestThrottledSchedule(t sweet.T) {
	var (
		clock    = glock.NewMockClock()
		sync     = make(chan struct{})
		done     = make(chan struct{})
		attempts = 0
	)

	scheduler := NewScheduler(
		func() {
			attempts++
			sync <- struct{}{}
		},
		WithInterval(time.Second),
		WithThrottle(time.Millisecond),
		withClock(clock),
	)

	go func() {
		defer close(done)

		for i := 0; i < 25; i++ {
			<-sync
		}
	}()

	go func() {
		for {
			select {
			case <-done:
				return
			default:
			}

			clock.Advance(time.Second)
		}
	}()

	scheduler.Start()
	<-done
	scheduler.Stop()
	Expect(attempts).To(Equal(25))

	tickerArgs := clock.GetTickerArgs()
	Expect(tickerArgs).To(HaveLen(1))
	Expect(tickerArgs[0]).To(Equal(time.Millisecond))
}

func (s *SchedulerSuite) TestExplicitFire(t sweet.T) {
	var (
		clock    = glock.NewMockClock()
		sync     = make(chan struct{})
		done     = make(chan struct{})
		attempts = 0
	)

	defer close(sync)

	scheduler := NewScheduler(
		func() {
			attempts++
			sync <- struct{}{}
		},
		WithInterval(time.Second),
		withClock(clock),
	)

	go func() {
		defer close(done)

		for i := 0; i < 25; i++ {
			scheduler.Signal()
			<-sync
		}
	}()

	scheduler.Start()
	<-done
	scheduler.Stop()
	Expect(attempts).To(Equal(25))
}

func (s *SchedulerSuite) TestThrottledExplicitFire(t sweet.T) {
	var (
		clock    = glock.NewMockClock()
		sync     = make(chan struct{})
		done     = make(chan struct{})
		attempts = 0
	)

	defer close(sync)

	scheduler := NewScheduler(
		func() {
			attempts++
			sync <- struct{}{}
		},
		WithInterval(time.Second),
		WithThrottle(time.Millisecond),
		withClock(clock),
	)

	go func() {
		defer close(done)

		for i := 0; i < 100; i++ {
			scheduler.Signal()

			if i%4 == 0 {
				clock.BlockingAdvance(time.Second)
				<-sync
			}
		}
	}()

	scheduler.Start()
	<-done
	scheduler.Stop()
	Expect(attempts).To(Equal(25))
}

func (s *SchedulerSuite) TestHammer(t sweet.T) {
	var (
		quit = make(chan struct{})
		ch   = hammer(quit)
	)

	for i := 1; i <= 200; i++ {
		<-ch
	}

	close(quit)
	eventually(ch).Should(BeClosed())
}

func (s *SchedulerSuite) TestConvert(t sweet.T) {
	var (
		ch1 = make(chan time.Time)
		ch2 = convert(ch1)
	)

	go func() {
		for i := 1; i <= 200; i++ {
			ch1 <- time.Now().Add(time.Minute * time.Duration(-i))
		}
	}()

	for i := 1; i <= 200; i++ {
		eventually(ch2).Should(Receive(Equal(struct{}{})))
	}

	close(ch1)
	eventually(ch2).Should(BeClosed())
}

func (s *SchedulerSuite) TestThrottle(t sweet.T) {
	var (
		ch1 = make(chan struct{})
		ch2 = make(chan struct{})
		ch3 = throttle(ch1, ch2)
	)

	Consistently(ch2).ShouldNot(Receive())
	ch1 <- struct{}{}

	go func() {
		ch2 <- struct{}{}
	}()

	eventually(ch3).Should(Receive(Equal(struct{}{})))
	close(ch1)
	eventually(ch3).Should(BeClosed())
}

//
// Overwrite gomega's default timeout and interval

func eventually(actual interface{}) GomegaAsyncAssertion {
	return Eventually(actual, TIMEOUT, INTERVAL)
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package accelerators

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	info "github.com/google/cadvisor/info/v1"

	"github.com/golang/glog"
	"github.com/mindprince/gonvml"
)

type NvidiaManager struct {
	// true if the NVML library (libnvidia-ml.so.1) was loaded successfully
	nvmlInitialized bool

	// nvidiaDevices is a map from device minor number to a handle that can be used to get metrics about the device
	nvidiaDevices map[int]gonvml.Device
}

var sysFsPCIDevicesPath = "/sys/bus/pci/devices/"

const nvidiaVendorId = "0x10de"

// Setup initializes NVML if nvidia devices are present on the node.
func (nm *NvidiaManager) Setup() {
	if !detectDevices(nvidiaVendorId) {
		glog.Info("No NVIDIA devices found.")
		return
	}

	go func() {
		glog.Info("Starting goroutine to initialize NVML")
		nm.initializeNVML()
		if nm.nvmlInitialized {
			return
		}
		// TODO: use globalHousekeepingInterval
		for range time.Tick(time.Minute) {
			nm.initializeNVML()
			if nm.nvmlInitialized {
				return
			}
		}
	}()
}

// detectDevices returns true if a device with given pci id is present on the node.
func detectDevices(vendorId string) bool {
	devices, err := ioutil.ReadDir(sysFsPCIDevicesPath)
	if err != nil {
		glog.Warningf("error reading %q: %v", sysFsPCIDevicesPath, err)
		return false
	}

	for _, device := range devices {
		vendorPath := filepath.Join(sysFsPCIDevicesPath, device.Name(), "vendor")
		content, err := ioutil.ReadFile(vendorPath)
		if err != nil {
			glog.Infof("Error while reading %q: %v", vendorPath, err)
			continue
		}
		if strings.EqualFold(strings.TrimSpace(string(content)), vendorId) {
			glog.Infof("Found device with vendorId %q", vendorId)
			return true
		}
	}
	return false
}

// initializeNVML initializes the NVML library and sets up the nvmlDevices map.
func (nm *NvidiaManager) initializeNVML() {
	if err := gonvml.Initialize(); err != nil {
		// This is under a logging level because otherwise we may cause
		// log spam if the drivers/nvml is not installed on the system.
		glog.V(3).Infof("Could not initialize NVML: %v", err)
		return
	}
	nm.nvmlInitialized = true
	numDevices, err := gonvml.DeviceCount()
	if err != nil {
		glog.Warningf("GPU metrics would not be available. Failed to get the number of nvidia devices: %v", err)
		return
	}
	glog.Infof("NVML initialized. Number of nvidia devices: %v", numDevices)
	nm.nvidiaDevices = make(map[int]gonvml.Device, numDevices)
	for i := 0; i < int(numDevices); i++ {
		device, err := gonvml.DeviceHandleByIndex(uint(i))
		if err != nil {
			glog.Warningf("Failed to get nvidia device handle %d: %v", i, err)
			continue
		}
		minorNumber, err := device.MinorNumber()
		if err != nil {
			glog.Warningf("Failed to get nvidia device minor number: %v", err)
			continue
		}
		nm.nvidiaDevices[int(minorNumber)] = device
	}
}

// Destroy shuts down NVML.
func (nm *NvidiaManager) Destroy() {
	if nm.nvmlInitialized {
		gonvml.Shutdown()
	}
}

// GetCollector returns a collector that can fetch nvidia gpu metrics for nvidia devices
// present in the devices.list file in the given devicesCgroupPath.
func (nm *NvidiaManager) GetCollector(devicesCgroupPath string) (AcceleratorCollector, error) {
	nc := &NvidiaCollector{}
	if !nm.nvmlInitialized || len(nm.nvidiaDevices) == 0 {
		return nc, nil
	}
	nvidiaMinorNumbers, err := parseDevicesCgroup(devicesCgroupPath)
	if err != nil {
		return nc, err
	}
	for _, minor := range nvidiaMinorNumbers {
		device, ok := nm.nvidiaDevices[minor]
		if !ok {
			return nc, fmt.Errorf("nvidia device minor number %d not found in cached devices", minor)
		}
		nc.Devices = append(nc.Devices, device)
	}
	return nc, nil
}

// parseDevicesCgroup parses the devices cgroup devices.list file for the container
// and returns a list of minor numbers corresponding to NVIDIA GPU devices that the
// container is allowed to access. In cases where the container has access to all
// devices or all NVIDIA devices but the devices are not enumerated separately in
// the devices.list file, we return an empty list.
// This is defined as a variable to help in testing.
var parseDevicesCgroup = func(devicesCgroupPath string) ([]int, error) {
	// Always return a non-nil slice
	nvidiaMinorNumbers := []int{}

	devicesList := filepath.Join(devicesCgroupPath, "devices.list")
	f, err := os.Open(devicesList)
	if err != nil {
		return nvidiaMinorNumbers, fmt.Errorf("error while opening devices cgroup file %q: %v", devicesList, err)
	}
	defer f.Close()

	s := bufio.NewScanner(f)

	// See https://www.kernel.org/doc/Documentation/cgroup-v1/devices.txt for the file format
	for s.Scan() {
		text := s.Text()

		fields := strings.Fields(text)
		if len(fields) != 3 {
			return nvidiaMinorNumbers, fmt.Errorf("invalid devices cgroup entry %q: must contain three whitespace-separated fields", text)
		}

		// Split the second field to find out major:minor numbers
		majorMinor := strings.Split(fields[1], ":")
		if len(majorMinor) != 2 {
			return nvidiaMinorNumbers, fmt.Errorf("invalid devices cgroup entry %q: second field should have one colon", text)
		}

		// NVIDIA graphics devices are character devices with major number 195.
		// https://github.com/torvalds/linux/blob/v4.13/Documentation/admin-guide/devices.txt#L2583
		if fields[0] == "c" && majorMinor[0] == "195" {
			minorNumber, err := strconv.Atoi(majorMinor[1])
			if err != nil {
				return nvidiaMinorNumbers, fmt.Errorf("invalid devices cgroup entry %q: minor number is not integer", text)
			}
			// We don't want devices like nvidiactl (195:255) and nvidia-modeset (195:254)
			if minorNumber < 128 {
				nvidiaMinorNumbers = append(nvidiaMinorNumbers, minorNumber)
			}
			// We are ignoring the "195:*" case
			// where the container has access to all NVIDIA devices on the machine.
		}
		// We are ignoring the "*:*" case
		// where the container has access to all devices on the machine.
	}
	return nvidiaMinorNumbers, nil
}

type NvidiaCollector struct {
	// Exposed for testing
	Devices []gonvml.Device
}

// UpdateStats updates the stats for NVIDIA GPUs (if any) attached to the container.
func (nc *NvidiaCollector) UpdateStats(stats *info.ContainerStats) error {
	for _, device := range nc.Devices {
		model, err := device.Name()
		if err != nil {
			return fmt.Errorf("error while getting gpu name: %v", err)
		}
		uuid, err := device.UUID()
		if err != nil {
			return fmt.Errorf("error while getting gpu uuid: %v", err)
		}
		memoryTotal, memoryUsed, err := device.MemoryInfo()
		if err != nil {
			return fmt.Errorf("error while getting gpu memory info: %v", err)
		}
		//TODO: Use housekeepingInterval
		utilizationGPU, err := device.AverageGPUUtilization(10 * time.Second)
		if err != nil {
			return fmt.Errorf("error while getting gpu utilization: %v", err)
		}

		stats.Accelerators = append(stats.Accelerators, info.AcceleratorStats{
			Make:        "nvidia",
			Model:       model,
			ID:          uuid,
			MemoryTotal: memoryTotal,
			MemoryUsed:  memoryUsed,
			DutyCycle:   uint64(utilizationGPU),
		})
	}
	return nil
}
<commit_msg>Avoid race in accessing nvidiaDevices between Setup() and GetCollector()<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package accelerators

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	info "github.com/google/cadvisor/info/v1"

	"github.com/golang/glog"
	"github.com/mindprince/gonvml"
)

type NvidiaManager struct {
	sync.RWMutex

	// true if the NVML library (libnvidia-ml.so.1) was loaded successfully
	nvmlInitialized bool

	// nvidiaDevices is a map from device minor number to a handle that can be used to get metrics about the device
	nvidiaDevices map[int]gonvml.Device
}

var sysFsPCIDevicesPath = "/sys/bus/pci/devices/"

const nvidiaVendorId = "0x10de"

// Setup initializes NVML if nvidia devices are present on the node.
func (nm *NvidiaManager) Setup() {
	if !detectDevices(nvidiaVendorId) {
		glog.Info("No NVIDIA devices found.")
		return
	}

	nm.initializeNVML()
	if nm.nvmlInitialized {
		return
	}
	go func() {
		glog.Info("Starting goroutine to initialize NVML")
		// TODO: use globalHousekeepingInterval
		for range time.Tick(time.Minute) {
			nm.initializeNVML()
			if nm.nvmlInitialized {
				return
			}
		}
	}()
}

// detectDevices returns true if a device with given pci id is present on the node.
func detectDevices(vendorId string) bool {
	devices, err := ioutil.ReadDir(sysFsPCIDevicesPath)
	if err != nil {
		glog.Warningf("error reading %q: %v", sysFsPCIDevicesPath, err)
		return false
	}

	for _, device := range devices {
		vendorPath := filepath.Join(sysFsPCIDevicesPath, device.Name(), "vendor")
		content, err := ioutil.ReadFile(vendorPath)
		if err != nil {
			glog.Infof("Error while reading %q: %v", vendorPath, err)
			continue
		}
		if strings.EqualFold(strings.TrimSpace(string(content)), vendorId) {
			glog.Infof("Found device with vendorId %q", vendorId)
			return true
		}
	}
	return false
}

// initializeNVML initializes the NVML library and sets up the nvmlDevices map.
func (nm *NvidiaManager) initializeNVML() {
	if err := gonvml.Initialize(); err != nil {
		// This is under a logging level because otherwise we may cause
		// log spam if the drivers/nvml is not installed on the system.
		glog.V(3).Infof("Could not initialize NVML: %v", err)
		return
	}
	numDevices, err := gonvml.DeviceCount()
	if err != nil {
		glog.Warningf("GPU metrics would not be available. Failed to get the number of nvidia devices: %v", err)
		nm.Lock()
		// Even though we won't have GPU metrics, the library was initialized and should be shutdown when exiting.
		nm.nvmlInitialized = true
		nm.Unlock()
		return
	}
	glog.Infof("NVML initialized. Number of nvidia devices: %v", numDevices)
	nm.nvidiaDevices = make(map[int]gonvml.Device, numDevices)
	for i := 0; i < int(numDevices); i++ {
		device, err := gonvml.DeviceHandleByIndex(uint(i))
		if err != nil {
			glog.Warningf("Failed to get nvidia device handle %d: %v", i, err)
			continue
		}
		minorNumber, err := device.MinorNumber()
		if err != nil {
			glog.Warningf("Failed to get nvidia device minor number: %v", err)
			continue
		}
		nm.nvidiaDevices[int(minorNumber)] = device
	}
	nm.Lock()
	// Doing this at the end to avoid race in accessing nvidiaDevices in GetCollector.
	nm.nvmlInitialized = true
	nm.Unlock()
}

// Destroy shuts down NVML.
func (nm *NvidiaManager) Destroy() {
	if nm.nvmlInitialized {
		gonvml.Shutdown()
	}
}

// GetCollector returns a collector that can fetch nvidia gpu metrics for nvidia devices
// present in the devices.list file in the given devicesCgroupPath.
func (nm *NvidiaManager) GetCollector(devicesCgroupPath string) (AcceleratorCollector, error) {
	nc := &NvidiaCollector{}
	nm.RLock()
	if !nm.nvmlInitialized || len(nm.nvidiaDevices) == 0 {
		nm.RUnlock()
		return nc, nil
	}
	nm.RUnlock()
	nvidiaMinorNumbers, err := parseDevicesCgroup(devicesCgroupPath)
	if err != nil {
		return nc, err
	}
	for _, minor := range nvidiaMinorNumbers {
		device, ok := nm.nvidiaDevices[minor]
		if !ok {
			return nc, fmt.Errorf("nvidia device minor number %d not found in cached devices", minor)
		}
		nc.Devices = append(nc.Devices, device)
	}
	return nc, nil
}

// parseDevicesCgroup parses the devices cgroup devices.list file for the container
// and returns a list of minor numbers corresponding to NVIDIA GPU devices that the
// container is allowed to access. In cases where the container has access to all
// devices or all NVIDIA devices but the devices are not enumerated separately in
// the devices.list file, we return an empty list.
// This is defined as a variable to help in testing.
var parseDevicesCgroup = func(devicesCgroupPath string) ([]int, error) {
	// Always return a non-nil slice
	nvidiaMinorNumbers := []int{}

	devicesList := filepath.Join(devicesCgroupPath, "devices.list")
	f, err := os.Open(devicesList)
	if err != nil {
		return nvidiaMinorNumbers, fmt.Errorf("error while opening devices cgroup file %q: %v", devicesList, err)
	}
	defer f.Close()

	s := bufio.NewScanner(f)

	// See https://www.kernel.org/doc/Documentation/cgroup-v1/devices.txt for the file format
	for s.Scan() {
		text := s.Text()

		fields := strings.Fields(text)
		if len(fields) != 3 {
			return nvidiaMinorNumbers, fmt.Errorf("invalid devices cgroup entry %q: must contain three whitespace-separated fields", text)
		}

		// Split the second field to find out major:minor numbers
		majorMinor := strings.Split(fields[1], ":")
		if len(majorMinor) != 2 {
			return nvidiaMinorNumbers, fmt.Errorf("invalid devices cgroup entry %q: second field should have one colon", text)
		}

		// NVIDIA graphics devices are character devices with major number 195.
		// https://github.com/torvalds/linux/blob/v4.13/Documentation/admin-guide/devices.txt#L2583
		if fields[0] == "c" && majorMinor[0] == "195" {
			minorNumber, err := strconv.Atoi(majorMinor[1])
			if err != nil {
				return nvidiaMinorNumbers, fmt.Errorf("invalid devices cgroup entry %q: minor number is not integer", text)
			}
			// We don't want devices like nvidiactl (195:255) and nvidia-modeset (195:254)
			if minorNumber < 128 {
				nvidiaMinorNumbers = append(nvidiaMinorNumbers, minorNumber)
			}
			// We are ignoring the "195:*" case
			// where the container has access to all NVIDIA devices on the machine.
		}
		// We are ignoring the "*:*" case
		// where the container has access to all devices on the machine.
	}
	return nvidiaMinorNumbers, nil
}

type NvidiaCollector struct {
	// Exposed for testing
	Devices []gonvml.Device
}

// UpdateStats updates the stats for NVIDIA GPUs (if any) attached to the container.
func (nc *NvidiaCollector) UpdateStats(stats *info.ContainerStats) error {
	for _, device := range nc.Devices {
		model, err := device.Name()
		if err != nil {
			return fmt.Errorf("error while getting gpu name: %v", err)
		}
		uuid, err := device.UUID()
		if err != nil {
			return fmt.Errorf("error while getting gpu uuid: %v", err)
		}
		memoryTotal, memoryUsed, err := device.MemoryInfo()
		if err != nil {
			return fmt.Errorf("error while getting gpu memory info: %v", err)
		}
		//TODO: Use housekeepingInterval
		utilizationGPU, err := device.AverageGPUUtilization(10 * time.Second)
		if err != nil {
			return fmt.Errorf("error while getting gpu utilization: %v", err)
		}

		stats.Accelerators = append(stats.Accelerators, info.AcceleratorStats{
			Make:        "nvidia",
			Model:       model,
			ID:          uuid,
			MemoryTotal: memoryTotal,
			MemoryUsed:  memoryUsed,
			DutyCycle:   uint64(utilizationGPU),
		})
	}
	return nil
}
<|endoftext|>