<commit_before>package p

import (
	. "github.com/alecthomas/chroma" // nolint
	"github.com/alecthomas/chroma/lexers/internal"
)

// Python lexer.
var Python = internal.Register(MustNewLazyLexer(
	&Config{
		Name:      "Python",
		Aliases:   []string{"python", "py", "sage"},
		Filenames: []string{"*.py", "*.pyw", "*.sc", "SConstruct", "SConscript", "*.tac", "*.sage"},
		MimeTypes: []string{"text/x-python", "application/x-python"},
	},
	pythonRules,
))

func pythonRules() Rules {
	return Rules{
		"root": {
			{`\n`, Text, nil},
			{`^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")`, ByGroups(Text, LiteralStringAffix, LiteralStringDoc), nil},
			{`^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')`, ByGroups(Text, LiteralStringAffix, LiteralStringDoc), nil},
			{`[^\S\n]+`, Text, nil},
			{`\A#!.+$`, CommentHashbang, nil},
			{`#.*$`, CommentSingle, nil},
			{`[]{}:(),;[]`, Punctuation, nil},
			{`\\\n`, Text, nil},
			{`\\`, Text, nil},
			{`(in|is|and|or|not)\b`, OperatorWord, nil},
			{`!=|==|<<|>>|[-~+/*%=<>&^|.]`, Operator, nil},
			Include("keywords"),
			{`(def)((?:\s|\\\s)+)`, ByGroups(Keyword, Text), Push("funcname")},
			{`(class)((?:\s|\\\s)+)`, ByGroups(Keyword, Text), Push("classname")},
			{`(from)((?:\s|\\\s)+)`, ByGroups(KeywordNamespace, Text), Push("fromimport")},
			{`(import)((?:\s|\\\s)+)`, ByGroups(KeywordNamespace, Text), Push("import")},
			Include("builtins"),
			Include("magicfuncs"),
			Include("magicvars"),
			Include("backtick"),
			{`([rR]|[uUbB][rR]|[rR][uUbB])(""")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Push("tdqs")},
			{`([rR]|[uUbB][rR]|[rR][uUbB])(''')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Push("tsqs")},
			{`([rR]|[uUbB][rR]|[rR][uUbB])(")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Push("dqs")},
			{`([rR]|[uUbB][rR]|[rR][uUbB])(')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Push("sqs")},
			{`([uUbB]?)(""")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Combined("stringescape", "tdqs")},
			{`([uUbB]?)(''')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Combined("stringescape", "tsqs")},
			{`([uUbB]?)(")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Combined("stringescape", "dqs")},
			{`([uUbB]?)(')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Combined("stringescape", "sqs")},
			Include("name"),
			Include("numbers"),
		},
		"keywords": {
			{Words(``, `\b`, `assert`, `break`, `continue`, `del`, `elif`, `else`, `except`, `exec`, `finally`, `for`, `global`, `if`, `lambda`, `pass`, `print`, `raise`, `return`, `try`, `while`, `yield`, `yield from`, `as`, `with`), Keyword, nil},
		},
		"builtins": {
			{Words(`(?<!\.)`, `\b`, `__import__`, `abs`, `all`, `any`, `apply`, `basestring`, `bin`, `bool`, `buffer`, `bytearray`, `bytes`, `callable`, `chr`, `classmethod`, `cmp`, `coerce`, `compile`, `complex`, `delattr`, `dict`, `dir`, `divmod`, `enumerate`, `eval`, `execfile`, `exit`, `file`, `filter`, `float`, `frozenset`, `getattr`, `globals`, `hasattr`, `hash`, `hex`, `id`, `input`, `int`, `intern`, `isinstance`, `issubclass`, `iter`, `len`, `list`, `locals`, `long`, `map`, `max`, `min`, `next`, `object`, `oct`, `open`, `ord`, `pow`, `property`, `range`, `raw_input`, `reduce`, `reload`, `repr`, `reversed`, `round`, `set`, `setattr`, `slice`, `sorted`, `staticmethod`, `str`, `sum`, `super`, `tuple`, `type`, `unichr`, `unicode`, `vars`, `xrange`, `zip`), NameBuiltin, nil},
			{`(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls)\b`, NameBuiltinPseudo, nil},
			{Words(`(?<!\.)`, `\b`, `ArithmeticError`, `AssertionError`, `AttributeError`, `BaseException`, `DeprecationWarning`, `EOFError`, `EnvironmentError`, `Exception`, `FloatingPointError`, `FutureWarning`, `GeneratorExit`, `IOError`, `ImportError`, `ImportWarning`, `IndentationError`, `IndexError`, `KeyError`, `KeyboardInterrupt`, `LookupError`, `MemoryError`, `NameError`, `NotImplemented`, `NotImplementedError`, `OSError`, `OverflowError`, `OverflowWarning`, `PendingDeprecationWarning`, `ReferenceError`, `RuntimeError`, `RuntimeWarning`, `StandardError`, `StopIteration`, `SyntaxError`, `SyntaxWarning`, `SystemError`, `SystemExit`, `TabError`, `TypeError`, `UnboundLocalError`, `UnicodeDecodeError`, `UnicodeEncodeError`, `UnicodeError`, `UnicodeTranslateError`, `UnicodeWarning`, `UserWarning`, `ValueError`, `VMSError`, `Warning`, `WindowsError`, `ZeroDivisionError`), NameException, nil},
		},
		"magicfuncs": {
			{Words(``, `\b`, `__abs__`, `__add__`, `__and__`, `__call__`, `__cmp__`, `__coerce__`, `__complex__`, `__contains__`, `__del__`, `__delattr__`, `__delete__`, `__delitem__`, `__delslice__`, `__div__`, `__divmod__`, `__enter__`, `__eq__`, `__exit__`, `__float__`, `__floordiv__`, `__ge__`, `__get__`, `__getattr__`, `__getattribute__`, `__getitem__`, `__getslice__`, `__gt__`, `__hash__`, `__hex__`, `__iadd__`, `__iand__`, `__idiv__`, `__ifloordiv__`, `__ilshift__`, `__imod__`, `__imul__`, `__index__`, `__init__`, `__instancecheck__`, `__int__`, `__invert__`, `__iop__`, `__ior__`, `__ipow__`, `__irshift__`, `__isub__`, `__iter__`, `__itruediv__`, `__ixor__`, `__le__`, `__len__`, `__long__`, `__lshift__`, `__lt__`, `__missing__`, `__mod__`, `__mul__`, `__ne__`, `__neg__`, `__new__`, `__nonzero__`, `__oct__`, `__op__`, `__or__`, `__pos__`, `__pow__`, `__radd__`, `__rand__`, `__rcmp__`, `__rdiv__`, `__rdivmod__`, `__repr__`, `__reversed__`, `__rfloordiv__`, `__rlshift__`, `__rmod__`, `__rmul__`, `__rop__`, `__ror__`, `__rpow__`, `__rrshift__`, `__rshift__`, `__rsub__`, `__rtruediv__`, `__rxor__`, `__set__`, `__setattr__`, `__setitem__`, `__setslice__`, `__str__`, `__sub__`, `__subclasscheck__`, `__truediv__`, `__unicode__`, `__xor__`), NameFunctionMagic, nil},
		},
		"magicvars": {
			{Words(``, `\b`, `__bases__`, `__class__`, `__closure__`, `__code__`, `__defaults__`, `__dict__`, `__doc__`, `__file__`, `__func__`, `__globals__`, `__metaclass__`, `__module__`, `__mro__`, `__name__`, `__self__`, `__slots__`, `__weakref__`), NameVariableMagic, nil},
		},
		"numbers": {
			{`(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?`, LiteralNumberFloat, nil},
			{`\d+[eE][+-]?[0-9]+j?`, LiteralNumberFloat, nil},
			{`0[0-7]+j?`, LiteralNumberOct, nil},
			{`0[bB][01]+`, LiteralNumberBin, nil},
			{`0[xX][a-fA-F0-9_]+`, LiteralNumberHex, nil},
			{`\d+L`, LiteralNumberIntegerLong, nil},
			{`[\d_]+j?`, LiteralNumberInteger, nil},
		},
		"backtick": {
			{"`.*?`", LiteralStringBacktick, nil},
		},
		"name": {
			{`@[\w.]+`, NameDecorator, nil},
			{`[a-zA-Z_]\w*`, Name, nil},
		},
		"funcname": {
			Include("magicfuncs"),
			{`[a-zA-Z_]\w*`, NameFunction, Pop(1)},
			Default(Pop(1)),
		},
		"classname": {
			{`[a-zA-Z_]\w*`, NameClass, Pop(1)},
		},
		"import": {
			{`(?:[ \t]|\\\n)+`, Text, nil},
			{`as\b`, KeywordNamespace, nil},
			{`,`, Operator, nil},
			{`[a-zA-Z_][\w.]*`, NameNamespace, nil},
			Default(Pop(1)),
		},
		"fromimport": {
			{`(?:[ \t]|\\\n)+`, Text, nil},
			{`import\b`, KeywordNamespace, Pop(1)},
			{`None\b`, NameBuiltinPseudo, Pop(1)},
			{`[a-zA-Z_.][\w.]*`, NameNamespace, nil},
			Default(Pop(1)),
		},
		"stringescape": {
			{`\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})`, LiteralStringEscape, nil},
		},
		"strings-single": {
			{`%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]`, LiteralStringInterpol, nil},
			{`[^\\\'"%\n]+`, LiteralStringSingle, nil},
			{`[\'"\\]`, LiteralStringSingle, nil},
			{`%`, LiteralStringSingle, nil},
		},
		"strings-double": {
			{`%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]`, LiteralStringInterpol, nil},
			{`[^\\\'"%\n]+`, LiteralStringDouble, nil},
			{`[\'"\\]`, LiteralStringDouble, nil},
			{`%`, LiteralStringDouble, nil},
		},
		"dqs": {
			{`"`, LiteralStringDouble, Pop(1)},
			{`\\\\|\\"|\\\n`, LiteralStringEscape, nil},
			Include("strings-double"),
		},
		"sqs": {
			{`'`, LiteralStringSingle, Pop(1)},
			{`\\\\|\\'|\\\n`, LiteralStringEscape, nil},
			Include("strings-single"),
		},
		"tdqs": {
			{`"""`, LiteralStringDouble, Pop(1)},
			Include("strings-double"),
			{`\n`, LiteralStringDouble, nil},
		},
		"tsqs": {
			{`'''`, LiteralStringSingle, Pop(1)},
			Include("strings-single"),
			{`\n`, LiteralStringSingle, nil},
		},
	}
}
<commit_msg>python: add *.pyi<commit_after>package p

import (
	. "github.com/alecthomas/chroma" // nolint
	"github.com/alecthomas/chroma/lexers/internal"
)

// Python lexer.
var Python = internal.Register(MustNewLazyLexer(
	&Config{
		Name:      "Python",
		Aliases:   []string{"python", "py", "sage"},
		Filenames: []string{"*.py", "*.pyi", "*.pyw", "*.sc", "SConstruct", "SConscript", "*.tac", "*.sage"},
		MimeTypes: []string{"text/x-python", "application/x-python"},
	},
	pythonRules,
))

func pythonRules() Rules {
	return Rules{
		"root": {
			{`\n`, Text, nil},
			{`^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")`, ByGroups(Text, LiteralStringAffix, LiteralStringDoc), nil},
			{`^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')`, ByGroups(Text, LiteralStringAffix, LiteralStringDoc), nil},
			{`[^\S\n]+`, Text, nil},
			{`\A#!.+$`, CommentHashbang, nil},
			{`#.*$`, CommentSingle, nil},
			{`[]{}:(),;[]`, Punctuation, nil},
			{`\\\n`, Text, nil},
			{`\\`, Text, nil},
			{`(in|is|and|or|not)\b`, OperatorWord, nil},
			{`!=|==|<<|>>|[-~+/*%=<>&^|.]`, Operator, nil},
			Include("keywords"),
			{`(def)((?:\s|\\\s)+)`, ByGroups(Keyword, Text), Push("funcname")},
			{`(class)((?:\s|\\\s)+)`, ByGroups(Keyword, Text), Push("classname")},
			{`(from)((?:\s|\\\s)+)`, ByGroups(KeywordNamespace, Text), Push("fromimport")},
			{`(import)((?:\s|\\\s)+)`, ByGroups(KeywordNamespace, Text), Push("import")},
			Include("builtins"),
			Include("magicfuncs"),
			Include("magicvars"),
			Include("backtick"),
			{`([rR]|[uUbB][rR]|[rR][uUbB])(""")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Push("tdqs")},
			{`([rR]|[uUbB][rR]|[rR][uUbB])(''')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Push("tsqs")},
			{`([rR]|[uUbB][rR]|[rR][uUbB])(")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Push("dqs")},
			{`([rR]|[uUbB][rR]|[rR][uUbB])(')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Push("sqs")},
			{`([uUbB]?)(""")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Combined("stringescape", "tdqs")},
			{`([uUbB]?)(''')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Combined("stringescape", "tsqs")},
			{`([uUbB]?)(")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Combined("stringescape", "dqs")},
			{`([uUbB]?)(')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Combined("stringescape", "sqs")},
			Include("name"),
			Include("numbers"),
		},
		"keywords": {
			{Words(``, `\b`, `assert`, `break`, `continue`, `del`, `elif`, `else`, `except`, `exec`, `finally`, `for`, `global`, `if`, `lambda`, `pass`, `print`, `raise`, `return`, `try`, `while`, `yield`, `yield from`, `as`, `with`), Keyword, nil},
		},
		"builtins": {
			{Words(`(?<!\.)`, `\b`, `__import__`, `abs`, `all`, `any`, `apply`, `basestring`, `bin`, `bool`, `buffer`, `bytearray`, `bytes`, `callable`, `chr`, `classmethod`, `cmp`, `coerce`, `compile`, `complex`, `delattr`, `dict`, `dir`, `divmod`, `enumerate`, `eval`, `execfile`, `exit`, `file`, `filter`, `float`, `frozenset`, `getattr`, `globals`, `hasattr`, `hash`, `hex`, `id`, `input`, `int`, `intern`, `isinstance`, `issubclass`, `iter`, `len`, `list`, `locals`, `long`, `map`, `max`, `min`, `next`, `object`, `oct`, `open`, `ord`, `pow`, `property`, `range`, `raw_input`, `reduce`, `reload`, `repr`, `reversed`, `round`, `set`, `setattr`, `slice`, `sorted`, `staticmethod`, `str`, `sum`, `super`, `tuple`, `type`, `unichr`, `unicode`, `vars`, `xrange`, `zip`), NameBuiltin, nil},
			{`(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls)\b`, NameBuiltinPseudo, nil},
			{Words(`(?<!\.)`, `\b`, `ArithmeticError`, `AssertionError`, `AttributeError`, `BaseException`, `DeprecationWarning`, `EOFError`, `EnvironmentError`, `Exception`, `FloatingPointError`, `FutureWarning`, `GeneratorExit`, `IOError`, `ImportError`, `ImportWarning`, `IndentationError`, `IndexError`, `KeyError`, `KeyboardInterrupt`, `LookupError`, `MemoryError`, `NameError`, `NotImplemented`, `NotImplementedError`, `OSError`, `OverflowError`, `OverflowWarning`, `PendingDeprecationWarning`, `ReferenceError`, `RuntimeError`, `RuntimeWarning`, `StandardError`, `StopIteration`, `SyntaxError`, `SyntaxWarning`, `SystemError`, `SystemExit`, `TabError`, `TypeError`, `UnboundLocalError`, `UnicodeDecodeError`, `UnicodeEncodeError`, `UnicodeError`, `UnicodeTranslateError`, `UnicodeWarning`, `UserWarning`, `ValueError`, `VMSError`, `Warning`, `WindowsError`, `ZeroDivisionError`), NameException, nil},
		},
		"magicfuncs": {
			{Words(``, `\b`, `__abs__`, `__add__`, `__and__`, `__call__`, `__cmp__`, `__coerce__`, `__complex__`, `__contains__`, `__del__`, `__delattr__`, `__delete__`, `__delitem__`, `__delslice__`, `__div__`, `__divmod__`, `__enter__`, `__eq__`, `__exit__`, `__float__`, `__floordiv__`, `__ge__`, `__get__`, `__getattr__`, `__getattribute__`, `__getitem__`, `__getslice__`, `__gt__`, `__hash__`, `__hex__`, `__iadd__`, `__iand__`, `__idiv__`, `__ifloordiv__`, `__ilshift__`, `__imod__`, `__imul__`, `__index__`, `__init__`, `__instancecheck__`, `__int__`, `__invert__`, `__iop__`, `__ior__`, `__ipow__`, `__irshift__`, `__isub__`, `__iter__`, `__itruediv__`, `__ixor__`, `__le__`, `__len__`, `__long__`, `__lshift__`, `__lt__`, `__missing__`, `__mod__`, `__mul__`, `__ne__`, `__neg__`, `__new__`, `__nonzero__`, `__oct__`, `__op__`, `__or__`, `__pos__`, `__pow__`, `__radd__`, `__rand__`, `__rcmp__`, `__rdiv__`, `__rdivmod__`, `__repr__`, `__reversed__`, `__rfloordiv__`, `__rlshift__`, `__rmod__`, `__rmul__`, `__rop__`, `__ror__`, `__rpow__`, `__rrshift__`, `__rshift__`, `__rsub__`, `__rtruediv__`, `__rxor__`, `__set__`, `__setattr__`, `__setitem__`, `__setslice__`, `__str__`, `__sub__`, `__subclasscheck__`, `__truediv__`, `__unicode__`, `__xor__`), NameFunctionMagic, nil},
		},
		"magicvars": {
			{Words(``, `\b`, `__bases__`, `__class__`, `__closure__`, `__code__`, `__defaults__`, `__dict__`, `__doc__`, `__file__`, `__func__`, `__globals__`, `__metaclass__`, `__module__`, `__mro__`, `__name__`, `__self__`, `__slots__`, `__weakref__`), NameVariableMagic, nil},
		},
		"numbers": {
			{`(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?`, LiteralNumberFloat, nil},
			{`\d+[eE][+-]?[0-9]+j?`, LiteralNumberFloat, nil},
			{`0[0-7]+j?`, LiteralNumberOct, nil},
			{`0[bB][01]+`, LiteralNumberBin, nil},
			{`0[xX][a-fA-F0-9_]+`, LiteralNumberHex, nil},
			{`\d+L`, LiteralNumberIntegerLong, nil},
			{`[\d_]+j?`, LiteralNumberInteger, nil},
		},
		"backtick": {
			{"`.*?`", LiteralStringBacktick, nil},
		},
		"name": {
			{`@[\w.]+`, NameDecorator, nil},
			{`[a-zA-Z_]\w*`, Name, nil},
		},
		"funcname": {
			Include("magicfuncs"),
			{`[a-zA-Z_]\w*`, NameFunction, Pop(1)},
			Default(Pop(1)),
		},
		"classname": {
			{`[a-zA-Z_]\w*`, NameClass, Pop(1)},
		},
		"import": {
			{`(?:[ \t]|\\\n)+`, Text, nil},
			{`as\b`, KeywordNamespace, nil},
			{`,`, Operator, nil},
			{`[a-zA-Z_][\w.]*`, NameNamespace, nil},
			Default(Pop(1)),
		},
		"fromimport": {
			{`(?:[ \t]|\\\n)+`, Text, nil},
			{`import\b`, KeywordNamespace, Pop(1)},
			{`None\b`, NameBuiltinPseudo, Pop(1)},
			{`[a-zA-Z_.][\w.]*`, NameNamespace, nil},
			Default(Pop(1)),
		},
		"stringescape": {
			{`\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})`, LiteralStringEscape, nil},
		},
		"strings-single": {
			{`%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]`, LiteralStringInterpol, nil},
			{`[^\\\'"%\n]+`, LiteralStringSingle, nil},
			{`[\'"\\]`, LiteralStringSingle, nil},
			{`%`, LiteralStringSingle, nil},
		},
		"strings-double": {
			{`%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]`, LiteralStringInterpol, nil},
			{`[^\\\'"%\n]+`, LiteralStringDouble, nil},
			{`[\'"\\]`, LiteralStringDouble, nil},
			{`%`, LiteralStringDouble, nil},
		},
		"dqs": {
			{`"`, LiteralStringDouble, Pop(1)},
			{`\\\\|\\"|\\\n`, LiteralStringEscape, nil},
			Include("strings-double"),
		},
		"sqs": {
			{`'`, LiteralStringSingle, Pop(1)},
			{`\\\\|\\'|\\\n`, LiteralStringEscape, nil},
			Include("strings-single"),
		},
		"tdqs": {
			{`"""`, LiteralStringDouble, Pop(1)},
			Include("strings-double"),
			{`\n`, LiteralStringDouble, nil},
		},
		"tsqs": {
			{`'''`, LiteralStringSingle, Pop(1)},
			Include("strings-single"),
			{`\n`, LiteralStringSingle, nil},
		},
	}
}
<|endoftext|>


<commit_before>package dropp

import (
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"time"

	gq "github.com/PuerkitoBio/goquery"
	gae "google.golang.org/appengine"
	db "google.golang.org/appengine/datastore"
	tq "google.golang.org/appengine/taskqueue"
	ufe "google.golang.org/appengine/urlfetch"
	usr "google.golang.org/appengine/user"
	tmpl "html/template"
)

// API contains all data needed for the API
type API struct {
	CurrentUser *usr.User
	LogoutURL   string
}

// Item models items sold by Dropp
type Item struct {
	SourceURL string
	EbayID    string
	ItemName  string
	CreatedAt time.Time
	UpdatedAt time.Time
	IsActive  bool
}

// Snapshot contains a snapshot of the current status of an item.
type Snapshot struct {
	Avaliabilty string
	AvCount     int
	OnEbay      bool
	Price       float64
	CreatedAt   time.Time
}

// HomeData contains all data needed by the home page template
type HomeData struct {
	CurrentUser *usr.User
	LogoutURL   string
	Items       *[]Item
}

func (a *API) newHomeData() HomeData {
	return HomeData{
		CurrentUser: a.CurrentUser,
		LogoutURL:   a.LogoutURL,
	}
}

// Allowed users
var allowedUsers map[string]bool
var api *API

func init() {
	allowedUsers = map[string]bool{
		"simone.trubian@gmail.com": true,
		"stoxx84@gmail.com":        true,
		"test@example.com":         true,
	}
	api = newAPI()
	http.Handle("/", api.registerMiddlewares(api.homePage))
	http.Handle("/item", api.registerMiddlewares(api.item))
	http.Handle(
		"/create_snapshots_task", api.registerMiddlewares(api.createSnapshotsTasks))
	http.Handle(
		"/create_snapshots",
		api.recoverMiddleware(http.HandlerFunc(api.createSnapshots)))
}

func newAPI() *API {
	return &API{}
}

func (a *API) item(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)

	item := &Item{
		SourceURL: r.FormValue("item-url-input"),
		EbayID:    r.FormValue("item-ebay-id-input"),
		ItemName:  r.FormValue("item-name-input"),
		CreatedAt: time.Now(),
		UpdatedAt: time.Now(),
	}

	isActiveInput := r.FormValue("item-is-active-input")
	if isActiveInput == "yes" {
		item.IsActive = true
	} else {
		item.IsActive = false
	}

	key := db.NewKey(ctx, "Item", r.FormValue("item-source-name"), 0, nil)
	_, err := db.Put(ctx, key, item)
	if err != nil {
		panic(err.Error())
	}
	http.Redirect(w, r, "/", http.StatusFound)
}

func (a *API) homePage(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)
	homeData := a.newHomeData()
	items := make([]Item, 0, 10)
	_, err := db.NewQuery("Item").GetAll(ctx, &items)
	homeData.Items = &items

	t, err := tmpl.ParseFiles("templates/home.html")
	if err != nil {
		panic(err.Error())
	}
	err = t.Execute(w, homeData)
	if err != nil {
		panic(err.Error())
	}
}

func (a *API) createSnapshotsTasks(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)

	task := tq.NewPOSTTask(
		"/create_snapshot", map[string][]string{})

	registeredTask, err := tq.Add(ctx, task, "default")
	if err != nil {
		log.Printf("Error while trying to add task: %s", err)
	} else {
		log.Printf("Starting new update snapshot task %s", registeredTask.Name)
	}

	if err != nil {
		panic(err.Error())
	}
}

func (a *API) createSnapshots(w http.ResponseWriter, r *http.Request) {

	// Fetch the page
	ctx := gae.NewContext(r)
	items := make([]Item, 0, 10)
	_, err := db.NewQuery("Item").GetAll(ctx, &items)
	for _, item := range items {
		log.Printf("Updating snapshot for item %s", item.SourceURL)
		client := ufe.Client(ctx)
		resp, err := client.Get(item.SourceURL)
		if err != nil {
			panic(err.Error())
		}

		itemKey := db.NewKey(ctx, "Item", item.SourceURL, 0, nil)
		snap := &Snapshot{}
		snap.OnEbay = false
		snap.CreatedAt = time.Now()
		snap.getBGAva(resp)
		snapKey := db.NewIncompleteKey(ctx, "Snapshot", itemKey)
		_, err = db.Put(ctx, snapKey, snap)
	}
	if err != nil {
		panic(err.Error())
	}
}

func (snap *Snapshot) getBGAva(response *http.Response) {
	// Scrape the page and get availability
	doc, err := gq.NewDocumentFromResponse(response)
	if err != nil {
		panic(err.Error())
	}
	ava := doc.Find(".status").Text()
	switch ava {
	case "Currently out of stock":
		snap.Avaliabilty = "Out"
		snap.AvCount = 0
		log.Print("out")
		return
	case "In stock, usually dispatched in 1 business day":
		snap.Avaliabilty = "Available"
		log.Print("available")
		return
	}
	rx, _ := regexp.Compile("[0-9]+")
	avaCount, _ := strconv.Atoi(rx.FindStringSubmatch(ava)[0])
	if avaCount <= 5 {
		snap.Avaliabilty = "Low"
		snap.AvCount = avaCount
		log.Print("low")
		return
	}
	snap.Avaliabilty = "Available"
	snap.AvCount = avaCount
	log.Print("available with number")
	return
}

func (a *API) registerMiddlewares(
	handFunc func(w http.ResponseWriter, r *http.Request)) http.Handler {
	return a.recoverMiddleware(a.authMiddleware(http.HandlerFunc(handFunc)))
}

func (a *API) recoverMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			rec := recover()
			if rec != nil {
				http.Error(w, rec.(string), http.StatusInternalServerError)
			}
		}()
		next.ServeHTTP(w, r)
	})
}

func (a *API) authMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-type", "text/html; charset=utf-8")
		ctx := gae.NewContext(r)
		currentUser := usr.Current(ctx)

		// The user is not signed in, generate a sign-in URL
		if currentUser == nil {
			url, err := usr.LoginURL(ctx, "/")
			if err != nil {
				panic(err.Error())
			}
			fmt.Fprintf(w, `<a href="%s">Sign in or register</a>`, url)
			return
		}

		// The user is signed but not allowed to access the platform, ask them
		// to log out and log in as another user, generate a sign-out URL
		if !allowedUsers[currentUser.Email] {
			url, err := usr.LogoutURL(ctx, "/")
			if err != nil {
				panic(err.Error())
			}
			fmt.Fprintf(
				w,
				`This user is not allowed in the platform, <a href="%s">sign in</a> as a different user`,
				url)
			return
		}

		// The user is signed in and allowed, serve the page and generate a
		// sign-out URL
		if allowedUsers[currentUser.Email] {
			url, err := usr.LogoutURL(ctx, "/")
			if err != nil {
				panic(err.Error())
			}
			a.CurrentUser = currentUser
			a.LogoutURL = url
		}

		next.ServeHTTP(w, r)
	})
}
<commit_msg>[WIP] Stubbed task completion check<commit_after>package dropp

import (
	"fmt"
	"log"
	"net/http"
	"regexp"
	"strconv"
	"time"

	gq "github.com/PuerkitoBio/goquery"
	gae "google.golang.org/appengine"
	db "google.golang.org/appengine/datastore"
	tq "google.golang.org/appengine/taskqueue"
	ufe "google.golang.org/appengine/urlfetch"
	usr "google.golang.org/appengine/user"
	tmpl "html/template"
)

// API contains all data needed for the API
type API struct {
	CurrentUser *usr.User
	LogoutURL   string
}

// Item models items sold by Dropp
type Item struct {
	SourceURL string
	EbayID    string
	ItemName  string
	CreatedAt time.Time
	UpdatedAt time.Time
	IsActive  bool
}

// Snapshot contains a snapshot of the current status of an item.
type Snapshot struct {
	Avaliabilty string
	AvCount     int
	OnEbay      bool
	Price       float64
	CreatedAt   time.Time
}

// HomeData contains all data needed by the home page template
type HomeData struct {
	CurrentUser *usr.User
	LogoutURL   string
	Items       *[]Item
}

func (a *API) newHomeData() HomeData {
	return HomeData{
		CurrentUser: a.CurrentUser,
		LogoutURL:   a.LogoutURL,
	}
}

// Allowed users
var allowedUsers map[string]bool
var api *API

func init() {
	allowedUsers = map[string]bool{
		"simone.trubian@gmail.com": true,
		"stoxx84@gmail.com":        true,
		"test@example.com":         true,
	}
	api = newAPI()
	http.Handle("/", api.registerMiddlewares(api.homePage))
	http.Handle("/item", api.registerMiddlewares(api.item))
	http.Handle(
		"/create_snapshots_task", api.registerMiddlewares(api.createSnapshotsTasks))
	http.Handle(
		"/create_snapshots",
		api.recoverMiddleware(http.HandlerFunc(api.createSnapshots)))
}

func newAPI() *API {
	return &API{}
}

func (a *API) item(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)

	item := &Item{
		SourceURL: r.FormValue("item-url-input"),
		EbayID:    r.FormValue("item-ebay-id-input"),
		ItemName:  r.FormValue("item-name-input"),
		CreatedAt: time.Now(),
		UpdatedAt: time.Now(),
	}

	isActiveInput := r.FormValue("item-is-active-input")
	if isActiveInput == "yes" {
		item.IsActive = true
	} else {
		item.IsActive = false
	}

	key := db.NewKey(ctx, "Item", r.FormValue("item-source-name"), 0, nil)
	_, err := db.Put(ctx, key, item)
	if err != nil {
		panic(err.Error())
	}
	http.Redirect(w, r, "/", http.StatusFound)
}

func (a *API) homePage(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)
	homeData := a.newHomeData()
	items := make([]Item, 0, 10)
	_, err := db.NewQuery("Item").GetAll(ctx, &items)
	homeData.Items = &items

	t, err := tmpl.ParseFiles("templates/home.html")
	if err != nil {
		panic(err.Error())
	}
	err = t.Execute(w, homeData)
	if err != nil {
		panic(err.Error())
	}
}

func (a *API) createSnapshotsTasks(w http.ResponseWriter, r *http.Request) {
	ctx := gae.NewContext(r)

	createSnapshotsTask := tq.NewPOSTTask(
		"/create_snapshot", map[string][]string{})

	checkTaskCompleted := tq.NewPOSTTask(
		"/check_snapshot_task", map[string][]string{"task_name": {"default"}})

	registeredTask, err := tq.AddMulti(
		ctx,
		[]tq.Task{*createSnapshotsTask, *checkTaskCompleted},
		"default")

	if err != nil {
		log.Printf("Error while trying to add task: %s", err)
	} else {
		log.Printf("Starting new update snapshot task %s", registeredTask.Name)
	}

	if err != nil {
		panic(err.Error())
	}
}

func (a *API) createSnapshots(w http.ResponseWriter, r *http.Request) {

	// Fetch the page
	ctx := gae.NewContext(r)
	items := make([]Item, 0, 10)
	_, err := db.NewQuery("Item").GetAll(ctx, &items)
	for _, item := range items {
		log.Printf("Updating snapshot for item %s", item.SourceURL)
		client := ufe.Client(ctx)
		resp, err := client.Get(item.SourceURL)
		if err != nil {
			panic(err.Error())
		}

		itemKey := db.NewKey(ctx, "Item", item.SourceURL, 0, nil)
		snap := &Snapshot{}
		snap.OnEbay = false
		snap.CreatedAt = time.Now()
		snap.getBGAva(resp)
		snapKey := db.NewIncompleteKey(ctx, "Snapshot", itemKey)
		_, err = db.Put(ctx, snapKey, snap)
	}
	if err != nil {
		panic(err.Error())
	}
}

func (snap *Snapshot) getBGAva(response *http.Response) {
	// Scrape the page and get availability
	doc, err := gq.NewDocumentFromResponse(response)
	if err != nil {
		panic(err.Error())
	}
	ava := doc.Find(".status").Text()
	switch ava {
	case "Currently out of stock":
		snap.Avaliabilty = "Out"
		snap.AvCount = 0
		log.Print("out")
		return
	case "In stock, usually dispatched in 1 business day":
		snap.Avaliabilty = "Available"
		log.Print("available")
		return
	}
	rx, _ := regexp.Compile("[0-9]+")
	avaCount, _ := strconv.Atoi(rx.FindStringSubmatch(ava)[0])
	if avaCount <= 5 {
		snap.Avaliabilty = "Low"
		snap.AvCount = avaCount
		log.Print("low")
		return
	}
	snap.Avaliabilty = "Available"
	snap.AvCount = avaCount
	log.Print("available with number")
	return
}

func (a *API) registerMiddlewares(
	handFunc func(w http.ResponseWriter, r *http.Request)) http.Handler {
	return a.recoverMiddleware(a.authMiddleware(http.HandlerFunc(handFunc)))
}

func (a *API) recoverMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer func() {
			rec := recover()
			if rec != nil {
				http.Error(w, rec.(string), http.StatusInternalServerError)
			}
		}()
		next.ServeHTTP(w, r)
	})
}

func (a *API) authMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-type", "text/html; charset=utf-8")
		ctx := gae.NewContext(r)
		currentUser := usr.Current(ctx)

		// The user is not signed in, generate a sign-in URL
		if currentUser == nil {
			url, err := usr.LoginURL(ctx, "/")
			if err != nil {
				panic(err.Error())
			}
			fmt.Fprintf(w, `<a href="%s">Sign in or register</a>`, url)
			return
		}

		// The user is signed but not allowed to access the platform, ask them
		// to log out and log in as another user, generate a sign-out URL
		if !allowedUsers[currentUser.Email] {
			url, err := usr.LogoutURL(ctx, "/")
			if err != nil {
				panic(err.Error())
			}
			fmt.Fprintf(
				w,
				`This user is not allowed in the platform, <a href="%s">sign in</a> as a different user`,
				url)
			return
		}

		// The user is signed in and allowed, serve the page and generate a
		// sign-out URL
		if allowedUsers[currentUser.Email] {
			url, err := usr.LogoutURL(ctx, "/")
			if err != nil {
				panic(err.Error())
			}
			a.CurrentUser = currentUser
			a.LogoutURL = url
		}

		next.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>package lfs

import (
	"errors"
	"testing"
)

func TestChecksHandleGoErrors(t *testing.T) {
	err := errors.New("Go Error")

	if IsFatalError(err) {
		t.Error("go error should not be a fatal error")
	}
}

func TestCheckHandlesWrappedErrors(t *testing.T) {
	err := errors.New("Go error")

	fatal := newFatalError(err)

	if !IsFatalError(fatal) {
		t.Error("expected error to be fatal")
	}
}

func TestBehaviorWraps(t *testing.T) {
	err := errors.New("Go error")

	fatal := newFatalError(err)
	ni := newNotImplementedError(fatal)

	if !IsNotImplementedError(ni) {
		t.Error("expected erro to be not implemeted")
	}

	if !IsFatalError(ni) {
		t.Error("expected wrapped error to also be fatal")
	}

	if IsNotImplementedError(fatal) {
		t.Error("expected fatal error to not be not implemented")
	}
}

func TestContextOnGoErrors(t *testing.T) {
	err := errors.New("Go error")

	ErrorSetContext(err, "foo", "bar")

	v := ErrorGetContext(err, "foo")
	if v == "bar" {
		t.Error("expected empty context on go error")
	}
}

func TestContextOnWrappedErrors(t *testing.T) {
	err := newFatalError(errors.New("Go error"))

	ErrorSetContext(err, "foo", "bar")

	if v := ErrorGetContext(err, "foo"); v != "bar" {
		t.Error("expected to be able to use context on wrapped errors")
	}

	ctxt := ErrorContext(err)
	if ctxt["foo"] != "bar" {
		t.Error("expected to get the context of an error")
	}

	ErrorDelContext(err, "foo")

	if v := ErrorGetContext(err, "foo"); v == "bar" {
		t.Errorf("expected to delete from error context")
	}
}

func TestStack(t *testing.T) {
	s := ErrorStack(errors.New("Go error"))
	if len(s) > 0 {
		t.Error("expected to get no stack from a Go error")
	}

	s = ErrorStack(newFatalError(errors.New("Go error")))
	if len(s) == 0 {
		t.Error("expected to get a stack from a wrapped error")
	}
}
<commit_msg>add tests for GetInnerError()<commit_after>package lfs

import (
	"errors"
	"testing"
)

func TestChecksHandleGoErrors(t *testing.T) {
	err := errors.New("Go Error")

	if IsFatalError(err) {
		t.Error("go error should not be a fatal error")
	}
}

func TestCheckHandlesWrappedErrors(t *testing.T) {
	err := errors.New("Go error")

	fatal := newFatalError(err)

	if !IsFatalError(fatal) {
		t.Error("expected error to be fatal")
	}
}

func TestBehaviorWraps(t *testing.T) {
	err := errors.New("Go error")

	fatal := newFatalError(err)
	ni := newNotImplementedError(fatal)

	if !IsNotImplementedError(ni) {
		t.Error("expected erro to be not implemeted")
	}

	if !IsFatalError(ni) {
		t.Error("expected wrapped error to also be fatal")
	}

	if IsNotImplementedError(fatal) {
		t.Error("expected fatal error to not be not implemented")
	}
}

func TestContextOnGoErrors(t *testing.T) {
	err := errors.New("Go error")

	ErrorSetContext(err, "foo", "bar")

	v := ErrorGetContext(err, "foo")
	if v == "bar" {
		t.Error("expected empty context on go error")
	}
}

func TestContextOnWrappedErrors(t *testing.T) {
	err := newFatalError(errors.New("Go error"))

	ErrorSetContext(err, "foo", "bar")

	if v := ErrorGetContext(err, "foo"); v != "bar" {
		t.Error("expected to be able to use context on wrapped errors")
	}

	ctxt := ErrorContext(err)
	if ctxt["foo"] != "bar" {
		t.Error("expected to get the context of an error")
	}

	ErrorDelContext(err, "foo")

	if v := ErrorGetContext(err, "foo"); v == "bar" {
		t.Errorf("expected to delete from error context")
	}
}

func TestStack(t *testing.T) {
	s := ErrorStack(errors.New("Go error"))
	if len(s) > 0 {
		t.Error("expected to get no stack from a Go error")
	}

	s = ErrorStack(newFatalError(errors.New("Go error")))
	if len(s) == 0 {
		t.Error("expected to get a stack from a wrapped error")
	}
}

func TestGetInnerError(t *testing.T) {
	i := errors.New("inner")
	err := GetInnerError(newWrappedError(i, "wrapped"))
	if err == nil {
		t.Fatal("No inner error found")
	}

	if msg := err.Error(); msg != "inner" {
		t.Errorf("bad inner error: %q", msg)
	}
}

func TestGetNestedInnerError(t *testing.T) {
	i := errors.New("inner")
	err := GetInnerError(newWrappedError(newWrappedError(i, "wrapped 2"), "wrapped"))
	if err == nil {
		t.Fatal("No inner error found")
	}

	if msg := err.Error(); msg != "inner" {
		t.Errorf("bad inner error: %q", msg)
	}
}
<|endoftext|>


<commit_before>package x86

import (
	uc "github.com/lunixbochs/unicorn"

	"../../models"
	"../../syscalls"
)

var darwinSyscalls = map[int]string{
	1:   "exit",
	2:   "fork",
	3:   "read",
	4:   "write",
	5:   "open",
	6:   "close",
	7:   "wait4",
	9:   "link",
	10:  "unlink",
	73:  "munmap",
	197: "mmap",
	199: "lseek",
}

func DarwinSyscall(u models.Usercorn) {
	esp, _ := u.RegRead(uc.UC_X86_REG_ESP)
	eax, _ := u.RegRead(uc.UC_X86_REG_EAX)
	getArgs := syscalls.StackArgs(u)
	ret, _ := u.Syscall(linuxSyscalls, int(eax), getArgs)
	u.RegWrite(uc.UC_X86_REG_EAX, ret)
	u.RegWrite(uc.UC_X86_REG_ESP, esp)
}

func DarwinInterrupt(u models.Usercorn, intno uint32) {
	if intno == 0x80 {
		DarwinSyscall(u)
	}
}

func init() {
	Arch.RegisterOS(&models.OS{Name: "darwin", Syscall: DarwinSyscall, Interrupt: DarwinInterrupt})
}
<commit_msg>fix darwin backend using linux syscalls<commit_after>package x86

import (
	uc "github.com/lunixbochs/unicorn"

	"../../models"
	"../../syscalls"
)

var darwinSyscalls = map[int]string{
	1:   "exit",
	2:   "fork",
	3:   "read",
	4:   "write",
	5:   "open",
	6:   "close",
	7:   "wait4",
	9:   "link",
	10:  "unlink",
	73:  "munmap",
	197: "mmap",
	199: "lseek",
}

func DarwinSyscall(u models.Usercorn) {
	esp, _ := u.RegRead(uc.UC_X86_REG_ESP)
	eax, _ := u.RegRead(uc.UC_X86_REG_EAX)
	getArgs := syscalls.StackArgs(u)
	ret, _ := u.Syscall(darwinSyscalls, int(eax), getArgs)
	u.RegWrite(uc.UC_X86_REG_EAX, ret)
	u.RegWrite(uc.UC_X86_REG_ESP, esp)
}

func DarwinInterrupt(u models.Usercorn, intno uint32) {
	if intno == 0x80 {
		DarwinSyscall(u)
	}
}

func init() {
	Arch.RegisterOS(&models.OS{Name: "darwin", Syscall: DarwinSyscall, Interrupt: DarwinInterrupt})
}
<|endoftext|>


<commit_before>// +build linux,lxd

/*
 * Copyright (C) 2018 Iain Grant
 * Copyright (C) 2018 Red Hat, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy ofthe License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specificlanguage governing permissions and
 * limitations under the License.
 *
 */

package lxd

import (
	"errors"
	"fmt"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	lxd "github.com/lxc/lxd/client"
	"github.com/mitchellh/mapstructure"
	"github.com/vishvananda/netns"

	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/graffiti/graph"
	"github.com/skydive-project/skydive/probe"
	"github.com/skydive-project/skydive/topology"
	tp "github.com/skydive-project/skydive/topology/probes"
	ns "github.com/skydive-project/skydive/topology/probes/netns"
)

type containerInfo struct {
	Pid  int
	Node *graph.Node
}

type loggingEvent struct {
	Type     string `mapstructure:"type"`
	Metadata struct {
		Context struct {
			Action string `mapstructure:"action"`
			Name   string `mapstructure:"name"`
		} `mapstructure:"context"`
		Level   string `mapstructure:"level"`
		Message string `mapstructure:"message"`
	} `mapstructure:"metadata"`
}

// ProbeHandler describes a LXD topology graph that enhance the graph
type ProbeHandler struct {
	common.RWMutex
	*ns.ProbeHandler
	state        int64
	wg           sync.WaitGroup
	connected    atomic.Value
	quit         chan struct{}
	containerMap map[string]containerInfo
	hostNs       netns.NsHandle
	client       lxd.ContainerServer
}

func (p *ProbeHandler) containerNamespace(pid int) string {
	return fmt.Sprintf("/proc/%d/ns/net", pid)
}

func (p *ProbeHandler) registerContainer(id string) {
	p.Ctx.Logger.Debugf("Registering container %s", id)

	p.Lock()
	defer p.Unlock()

	if _, ok := p.containerMap[id]; ok {
		return
	}

	container, _, err := p.client.GetContainer(id)
	if err != nil {
		p.Ctx.Logger.Errorf("Failed to retrieve container %s", id)
		return
	}

	// state.Network[].HostName == host side interface
	// state.Pid for lookup of network namespace
	state, _, _ := p.client.GetContainerState(id)

	if state.Status != "Running" {
		p.Ctx.Logger.Errorf("Container %s is not running", id)
		return
	}

	nsHandle, err := netns.GetFromPid(int(state.Pid))
	if err != nil {
		return
	}
	defer nsHandle.Close()

	namespace := p.containerNamespace(int(state.Pid))

	var n *graph.Node
	if p.hostNs.Equal(nsHandle) {
		n = p.Ctx.RootNode
	} else {
		if n, err = p.Register(namespace, id); err == nil {
			p.Ctx.Graph.Lock()
			p.Ctx.Graph.AddMetadata(n, "Manager", "lxd")
			p.Ctx.Graph.Unlock()
		} else {
			p.Ctx.Logger.Errorf("Error registering probe: %s", err)
		}
	}

	devices := graph.Metadata{}
	for k, v := range container.Devices {
		devices[k] = v
	}

	metadata := Metadata{
		Architecture: container.Architecture,
		CreatedAt:    container.CreatedAt.String(),
		Description:  container.Description,
		Devices:      devices,
		Ephemeral:    strconv.FormatBool(container.Ephemeral),
		Profiles:     container.Profiles,
		Restore:      container.Restore,
		Stateful:     strconv.FormatBool(container.Stateful),
		Status:       container.Status,
	}

	if len(container.Config) != 0 {
		metadata.Config = graph.Metadata(common.NormalizeValue(container.Config).(map[string]interface{}))
	}

	p.Ctx.Graph.Lock()

	containerNode, err := p.Ctx.Graph.NewNode(graph.GenID(), graph.Metadata{
		"Type": "container",
		"Name": id,
		"LXD":  metadata,
	})
	if err != nil {
		p.Ctx.Logger.Error(err)
		return
	}

	topology.AddOwnershipLink(p.Ctx.Graph, n, containerNode, nil)
	p.Ctx.Graph.Unlock()

	p.containerMap[id] = containerInfo{
		Pid:  int(state.Pid),
		Node: containerNode,
	}
}

func (p *ProbeHandler) unregisterContainer(id string) {
	p.Lock()
	defer p.Unlock()

	infos, ok := p.containerMap[id]
	if !ok {
		return
	}

	p.Ctx.Graph.Lock()
	if err := p.Ctx.Graph.DelNode(infos.Node); err != nil {
		p.Ctx.Logger.Error(err)
	}
	p.Ctx.Graph.Unlock()

	namespace := p.containerNamespace(infos.Pid)
	p.Unregister(namespace)

	delete(p.containerMap, id)
}

func (p *ProbeHandler) connect() (err error) {
	if p.hostNs, err = netns.Get(); err != nil {
		return err
	}
	defer p.hostNs.Close()

	p.wg.Add(1)
	defer p.wg.Done()

	p.Ctx.Logger.Debugf("Connecting to LXD")
	client, err := lxd.ConnectLXDUnix("", nil)
	if err != nil {
		return err
	}
	p.client = client

	events, err := client.GetEvents()
	if err != nil {
		return err
	}

	target, err := events.AddHandler(nil, func(obj interface{}) {
		var event loggingEvent
		if err := mapstructure.Decode(obj, &event); err != nil {
			return
		}

		if event.Type == "logging" {
			if event.Metadata.Context.Action == "start" && event.Metadata.Message == "Started container" {
				p.registerContainer(event.Metadata.Context.Name)
			} else if event.Metadata.Message == "Deleted container" {
				p.unregisterContainer(event.Metadata.Context.Name)
			}
		}
	})

	if err != nil {
		return err
	}
	defer events.RemoveHandler(target)

	p.connected.Store(true)
	defer p.connected.Store(false)

	go func() {
		p.wg.Add(1)
		defer p.wg.Done()

		p.Ctx.Logger.Debugf("Listing LXD containers")
		containers, err := p.client.GetContainers()
		if err != nil {
			return
		}

		for _, n := range containers {
			if atomic.LoadInt64(&p.state) == common.RunningState {
				p.registerContainer(n.Name)
			}
		}
	}()

	<-p.quit

	return nil
}

// Start the probe
func (p *ProbeHandler) Start() {
	if !atomic.CompareAndSwapInt64(&p.state, common.StoppedState, common.RunningState) {
		return
	}

	go func() {
		for {
			state := atomic.LoadInt64(&p.state)
			if state == common.StoppingState || state == common.StoppedState {
				break
			}

			if p.connect() != nil {
				time.Sleep(1 * time.Second)
			}

			p.wg.Wait()
		}
	}()
}

// Stop the probe
func (p *ProbeHandler) Stop() {
	if !atomic.CompareAndSwapInt64(&p.state, common.RunningState, common.StoppingState) {
		return
	}

	if p.connected.Load() == true {
		p.quit <- struct{}{}
		p.wg.Wait()
	}

	atomic.StoreInt64(&p.state, common.StoppedState)
}

// NewProbeHandler initializes a new topology Lxd probe
func (p *ProbeHandler) Init(ctx tp.Context, bundle *probe.Bundle) (probe.Handler, error) {
	nsHandler := bundle.GetHandler("netns")
	if nsHandler == nil {
		return nil, errors.New("unable to find the netns handler")
	}

	p.ProbeHandler = nsHandler.(*ns.ProbeHandler)
	p.state = common.StoppedState
	p.containerMap = make(map[string]containerInfo)
	p.quit = make(chan struct{})

	return p, nil
}
<commit_msg>lxd: make use of retry probe wrapper<commit_after>// +build linux,lxd

/*
 * Copyright (C) 2018 Iain Grant
 * Copyright (C) 2018 Red Hat, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy ofthe License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specificlanguage governing permissions and
 * limitations under the License.
 *
 */

package lxd

import (
	"context"
	"errors"
	"fmt"
	"strconv"
	"sync"

	lxd "github.com/lxc/lxd/client"
	"github.com/mitchellh/mapstructure"
	"github.com/vishvananda/netns"

	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/graffiti/graph"
	"github.com/skydive-project/skydive/probe"
	"github.com/skydive-project/skydive/topology"
	"github.com/skydive-project/skydive/topology/probes"
	tp "github.com/skydive-project/skydive/topology/probes"
	ns "github.com/skydive-project/skydive/topology/probes/netns"
)

type containerInfo struct {
	Pid  int
	Node *graph.Node
}

type loggingEvent struct {
	Type     string `mapstructure:"type"`
	Metadata struct {
		Context struct {
			Action string `mapstructure:"action"`
			Name   string `mapstructure:"name"`
		} `mapstructure:"context"`
		Level   string `mapstructure:"level"`
		Message string `mapstructure:"message"`
	} `mapstructure:"metadata"`
}

// ProbeHandler describes a LXD topology graph that enhance the graph
type ProbeHandler struct {
	common.RWMutex
	Ctx          tp.Context
	nsProbe      *ns.ProbeHandler
	hostNs       netns.NsHandle
	containerMap map[string]containerInfo
	client       lxd.ContainerServer
}

func (p *ProbeHandler) containerNamespace(pid int) string {
	return fmt.Sprintf("/proc/%d/ns/net", pid)
}

func (p *ProbeHandler) registerContainer(id string) {
	p.Ctx.Logger.Debugf("Registering container %s", id)

	p.Lock()
	defer p.Unlock()

	if _, ok := p.containerMap[id]; ok {
		return
	}

	container, _, err := p.client.GetContainer(id)
	if err != nil {
		p.Ctx.Logger.Errorf("Failed to retrieve container %s", id)
		return
	}

	// state.Network[].HostName == host side interface
	// state.Pid for lookup of network namespace
	state, _, _ := p.client.GetContainerState(id)

	if state.Status != "Running" {
		p.Ctx.Logger.Errorf("Container %s is not running", id)
		return
	}

	nsHandle, err := netns.GetFromPid(int(state.Pid))
	if err != nil {
		return
	}
	defer nsHandle.Close()

	namespace := p.containerNamespace(int(state.Pid))

	var n *graph.Node
	if p.hostNs.Equal(nsHandle) {
		n = p.Ctx.RootNode
	} else {
		if n, err = p.nsProbe.Register(namespace, id); err == nil {
			p.Ctx.Graph.Lock()
			p.Ctx.Graph.AddMetadata(n, "Manager", "lxd")
			p.Ctx.Graph.Unlock()
		} else {
			p.Ctx.Logger.Errorf("Error registering probe: %s", err)
		}
	}

	devices := graph.Metadata{}
	for k, v := range container.Devices {
		devices[k] = v
	}

	metadata := Metadata{
		Architecture: container.Architecture,
		CreatedAt:    container.CreatedAt.String(),
		Description:  container.Description,
		Devices:      devices,
		Ephemeral:    strconv.FormatBool(container.Ephemeral),
		Profiles:     container.Profiles,
		Restore:      container.Restore,
		Stateful:     strconv.FormatBool(container.Stateful),
		Status:       container.Status,
	}

	if len(container.Config) != 0 {
		metadata.Config = graph.Metadata(common.NormalizeValue(container.Config).(map[string]interface{}))
	}

	p.Ctx.Graph.Lock()

	containerNode, err := p.Ctx.Graph.NewNode(graph.GenID(), graph.Metadata{
		"Type": "container",
		"Name": id,
		"LXD":  metadata,
	})
	if err != nil {
		p.Ctx.Logger.Error(err)
		return
	}

	topology.AddOwnershipLink(p.Ctx.Graph, n, containerNode, nil)
	p.Ctx.Graph.Unlock()

	p.containerMap[id] = containerInfo{
		Pid:  int(state.Pid),
		Node: containerNode,
	}
}

func (p *ProbeHandler) unregisterContainer(id string) {
	p.Lock()
	defer p.Unlock()

	infos, ok := p.containerMap[id]
	if !ok {
		return
	}

	p.Ctx.Graph.Lock()
	if err := p.Ctx.Graph.DelNode(infos.Node); err != nil {
		p.Ctx.Logger.Error(err)
	}
	p.Ctx.Graph.Unlock()

	namespace := p.containerNamespace(infos.Pid)
	p.nsProbe.Unregister(namespace)

	delete(p.containerMap, id)
}

func (p *ProbeHandler) Do(ctx context.Context, wg *sync.WaitGroup) (err error) {
	if p.hostNs, err = netns.Get(); err != nil {
		return err
	}

	p.Ctx.Logger.Debugf("Connecting to LXD")
	client, err := lxd.ConnectLXDUnix("", nil)
	if err != nil {
		return err
	}
	p.client = client

	events, err := client.GetEvents()
	if err != nil {
		return err
	}

	target, err := events.AddHandler(nil, func(obj interface{}) {
		var event loggingEvent
		if err := mapstructure.Decode(obj, &event); err != nil {
			return
		}

		if event.Type == "logging" {
			if event.Metadata.Context.Action == "start" && event.Metadata.Message == "Started container" {
				p.registerContainer(event.Metadata.Context.Name)
			} else if event.Metadata.Message == "Deleted container" {
				p.unregisterContainer(event.Metadata.Context.Name)
			}
		}
	})

	if err != nil {
		return err
	}

	wg.Add(3)
	go func() {
		defer wg.Done()

		p.Ctx.Logger.Debugf("Listing LXD containers")
		containers, err := p.client.GetContainers()
		if err != nil {
			return
		}

		for _, n := range containers {
			p.registerContainer(n.Name)
		}
	}()

	disconnected := make(chan error, 1)
	go func() {
		defer wg.Done()
		defer events.RemoveHandler(target)
		defer p.hostNs.Close()

		disconnected <- events.Wait()
	}()

	go func() {
		defer wg.Done()

		select {
		case <-ctx.Done():
			events.Disconnect()
		case err = <-disconnected:
		}
	}()

	return err
}

// Init initializes a new topology Lxd probe
func (p *ProbeHandler) Init(ctx tp.Context, bundle *probe.Bundle) (probe.Handler, error) {
	nsHandler := bundle.GetHandler("netns")
	if nsHandler == nil {
		return nil, errors.New("unable to find the netns handler")
	}
	p.nsProbe = nsHandler.(*ns.ProbeHandler)

	p.containerMap = make(map[string]containerInfo)
	p.Ctx = ctx

	return probes.NewProbeWrapper(p), nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transport_test

import (
	"context"
	"log"

	"google.golang.org/api/option"
	"google.golang.org/api/transport"
)

func Example_applicationDefaultCredentials() {
	ctx := context.Background()

	// Providing no auth option will cause NewClient to look for Application
	// Default Creds as specified at https://godoc.org/golang.org/x/oauth2/google#FindDefaultCredentials.
	//
	// Note: Given the same set of options, transport.NewHTTPClient and
	// transport.DialGRPC use the same credentials.
	c, _, err := transport.NewHTTPClient(ctx)
	if err != nil {
		log.Fatal(err)
	}
	_ = c // Use authenticated client.
}

func Example_withCredentialsFile() {
	ctx := context.Background()

	// Download service account creds per https://cloud.google.com/docs/authentication/end-user.
	//
	// Note: Given the same set of options, transport.NewHTTPClient and
	// transport.DialGRPC use the same credentials.
	c, _, err := transport.NewHTTPClient(ctx, option.WithCredentialsFile("/path/to/service-account-creds.json"))
	if err != nil {
		log.Fatal(err)
	}
	_ = c // Use authenticated client.
}
<commit_msg>transport: fixed link in the examples_test.go<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package transport_test

import (
	"context"
	"log"

	"google.golang.org/api/option"
	"google.golang.org/api/transport"
)

func Example_applicationDefaultCredentials() {
	ctx := context.Background()

	// Providing no auth option will cause NewClient to look for Application
	// Default Creds as specified at https://godoc.org/golang.org/x/oauth2/google#FindDefaultCredentials.
	//
	// Note: Given the same set of options, transport.NewHTTPClient and
	// transport.DialGRPC use the same credentials.
	c, _, err := transport.NewHTTPClient(ctx)
	if err != nil {
		log.Fatal(err)
	}
	_ = c // Use authenticated client.
}

func Example_withCredentialsFile() {
	ctx := context.Background()

	// Download service account creds per https://cloud.google.com/docs/authentication/production.
	//
	// Note: Given the same set of options, transport.NewHTTPClient and
	// transport.DialGRPC use the same credentials.
	c, _, err := transport.NewHTTPClient(ctx, option.WithCredentialsFile("/path/to/service-account-creds.json"))
	if err != nil {
		log.Fatal(err)
	}
	_ = c // Use authenticated client.
}
<|endoftext|>


<commit_before>package app

import (
	"fmt"
	"github.com/codegangsta/martini"
	"log"
	"net/http"
	"time"
	"os"
	"path/filepath"
	"strings"
	"io/ioutil"
)

type FileCache interface {
	WarmAndQuery(url string, fileAliases []string) *CachedFile
	Query(tokens []string) *CachedFile
	Warm(url string, fileAliases []string)
}

func NewFileCacheWithPath(basePath string) martini.Handler {
	fileCache := NewDiskFileCache(basePath)
	return NewFileCacheMiddleware(fileCache)
}

func NewFileCacheMiddleware(fileCache FileCache) martini.Handler {
	return func(_ http.ResponseWriter, _ *http.Request, c martini.Context, _ *log.Logger) {
		c.MapTo(fileCache, (*FileCache)(nil))
		c.Next()
	}
}

type DiskFileCache struct {
	query             chan QueryCachedFiles
	warm              chan WarmCachedFiles
	warmAndQuery      chan WarmAndQueryCachedFiles
}

func NewDiskFileCache(basePath string) *DiskFileCache {
	diskFileCache := new(DiskFileCache)
	diskFileCache.query = make(chan QueryCachedFiles, 10)
	diskFileCache.warm = make(chan WarmCachedFiles, 10)
	diskFileCache.warmAndQuery = make(chan WarmAndQueryCachedFiles, 10)

	go fileCache(diskFileCache.query, diskFileCache.warm, diskFileCache.warmAndQuery, basePath)

	return diskFileCache
}

func (dfc *DiskFileCache) Close() {
	close(dfc.query)
	close(dfc.warm)
	close(dfc.warmAndQuery)
}

func (dfc *DiskFileCache) WarmAndQuery(url string, fileAliases []string) *CachedFile {
	command := WarmAndQueryCachedFiles{url, fileAliases, make(chan *CachedFile)}
	defer close(command.Response)
	dfc.warmAndQuery <- command

	select {
	case result := <-command.Response:
		return result
	case <- time.After(3 * 1e9):
		return nil
	}
}

func (dfc *DiskFileCache) Query(tokens []string) *CachedFile {
	search := QueryCachedFiles{tokens, make(chan *CachedFile)}
	defer close(search.Response)
	dfc.query <- search

	select {
	case result := <-search.Response:
		return result
	case <- time.After(3 * 1e9):
		return nil
	}
}

func (dfc *DiskFileCache) Warm(url string, fileAliases []string) {
	command := WarmCachedFiles{url, fileAliases}
	dfc.warm <- command
}

func fileCache(query chan QueryCachedFiles, warm chan WarmCachedFiles, warmAndQuery chan WarmAndQueryCachedFiles, cacheDirectory string) {
	cachedFiles := make(map[string]*CachedFile)
	cachedFileAliases := make(map[string]string)
	initCachedFiles(cacheDirectory, cachedFiles, cachedFileAliases)
	for {
		select {
		case command, ok := <-warm:
		{
			if !ok {
				return
			}
			download(command.Url, command.Aliases, cacheDirectory, cachedFiles, cachedFileAliases)
		}
		case command, ok := <-warmAndQuery:
		{
			if !ok {
				return
			}
			command.Response <- download(command.Url, command.Aliases, cacheDirectory, cachedFiles, cachedFileAliases)
		}
		case command, ok := <-query:
		{
			if !ok {
				return
			}
			command.Response <- findCachedFile(command.Query, cacheDirectory, cachedFiles, cachedFileAliases)
		}
		}
	}
}

func initCachedFiles(cacheDirectory string, cachedFiles map[string]*CachedFile, aliases map[string]string) {
	walkFn := func(path string, _ os.FileInfo, err error) error {
		stat, err := os.Stat(path)
		if err != nil {
			return err
		}

		if stat.IsDir() && path != cacheDirectory {
			return filepath.SkipDir
		}

		if err != nil {
			return err
		}
		if strings.HasSuffix(path, ".metadata") {
			content, err := ioutil.ReadFile(path)
			if err == nil {
				dir, fileName := filepath.Split(path)
				fileNameParts := strings.Split(fileName, ".")
				contentHash := fileNameParts[0]
				fileAliases := strings.Split(string(content), "\n")
				cachedFiles[contentHash] = &CachedFile{fileAliases[0], fileAliases, filepath.Join(dir, contentHash)}
				for _, alias := range fileAliases {
					aliases[alias] = contentHash
				}
			}
		}
		return nil
	}
	err := filepath.Walk(cacheDirectory, walkFn)
	if err != nil {
		fmt.Println(err.Error())
	}
}

func findCachedFile(tokens []string, _ string, cachedFiles map[string]*CachedFile, aliases map[string]string) *CachedFile {
	for _, token := range tokens {
		contentHash, hasContentHash := aliases[token]
		if hasContentHash {
			cachedFile, hasCachedFile := cachedFiles[contentHash]
			if hasCachedFile {
				return cachedFile
			}
		}
	}
	return nil
}

func download(url string, urlAliases []string, cacheDirectory string, cachedFiles map[string]*CachedFile, aliases map[string]string) *CachedFile {
	existingCachedFile := findCachedFile(append(urlAliases, url), cacheDirectory, cachedFiles, aliases)
	if existingCachedFile != nil {
		return existingCachedFile
	}

	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}

	urlHash := hash([]byte(url))
	contentHash := hash(body)
	path := filepath.Join(cacheDirectory, contentHash)

	allAliases := make(map[string]bool)
	allAliases[url] = true
	allAliases[urlHash] = true
	allAliases[contentHash] = true
	for _, alias := range urlAliases {
		allAliases[alias] = true
	}

	cachedFile := &CachedFile{url, mapKeys(allAliases), path}
	cachedFile.StoreAsset(body)
	cachedFile.StoreMetadata()

	cachedFiles[contentHash] = cachedFile
	for _, alias := range cachedFile.Aliases {
		aliases[alias] = contentHash
	}

	return cachedFile
}
<commit_msg>Formatting<commit_after>package app

import (
	"fmt"
	"github.com/codegangsta/martini"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"
)

type FileCache interface {
	WarmAndQuery(url string, fileAliases []string) *CachedFile
	Query(tokens []string) *CachedFile
	Warm(url string, fileAliases []string)
}

func NewFileCacheWithPath(basePath string) martini.Handler {
	fileCache := NewDiskFileCache(basePath)
	return NewFileCacheMiddleware(fileCache)
}

func NewFileCacheMiddleware(fileCache FileCache) martini.Handler {
	return func(_ http.ResponseWriter, _ *http.Request, c martini.Context, _ *log.Logger) {
		c.MapTo(fileCache, (*FileCache)(nil))
		c.Next()
	}
}

type DiskFileCache struct {
	query        chan QueryCachedFiles
	warm         chan WarmCachedFiles
	warmAndQuery chan WarmAndQueryCachedFiles
}

func NewDiskFileCache(basePath string) *DiskFileCache {
	diskFileCache := new(DiskFileCache)
	diskFileCache.query = make(chan QueryCachedFiles, 10)
	diskFileCache.warm = make(chan WarmCachedFiles, 10)
	diskFileCache.warmAndQuery = make(chan WarmAndQueryCachedFiles, 10)

	go fileCache(diskFileCache.query, diskFileCache.warm, diskFileCache.warmAndQuery, basePath)

	return diskFileCache
}

func (dfc *DiskFileCache) Close() {
	close(dfc.query)
	close(dfc.warm)
	close(dfc.warmAndQuery)
}

func (dfc *DiskFileCache) WarmAndQuery(url string, fileAliases []string) *CachedFile {
	command := WarmAndQueryCachedFiles{url, fileAliases, make(chan *CachedFile)}
	defer close(command.Response)
	dfc.warmAndQuery <- command

	select {
	case result := <-command.Response:
		return result
	case <-time.After(3 * 1e9):
		return nil
	}
}

func (dfc *DiskFileCache) Query(tokens []string) *CachedFile {
	search := QueryCachedFiles{tokens, make(chan *CachedFile)}
	defer close(search.Response)
	dfc.query <- search

	select {
	case result := <-search.Response:
		return result
	case <-time.After(3 * 1e9):
		return nil
	}
}

func (dfc *DiskFileCache) Warm(url string, fileAliases []string) {
	command := WarmCachedFiles{url, fileAliases}
	dfc.warm <- command
}

func fileCache(query chan QueryCachedFiles, warm chan WarmCachedFiles, warmAndQuery chan WarmAndQueryCachedFiles, cacheDirectory string) {
	cachedFiles := make(map[string]*CachedFile)
	cachedFileAliases := make(map[string]string)
	initCachedFiles(cacheDirectory, cachedFiles, cachedFileAliases)
	for {
		select {
		case command, ok := <-warm:
			{
				if !ok {
					return
				}
				download(command.Url, command.Aliases, cacheDirectory, cachedFiles, cachedFileAliases)
			}
		case command, ok := <-warmAndQuery:
			{
				if !ok {
					return
				}
				command.Response <- download(command.Url, command.Aliases, cacheDirectory, cachedFiles, cachedFileAliases)
			}
		case command, ok := <-query:
			{
				if !ok {
					return
				}
				command.Response <- findCachedFile(command.Query, cacheDirectory, cachedFiles, cachedFileAliases)
			}
		}
	}
}

func initCachedFiles(cacheDirectory string, cachedFiles map[string]*CachedFile, aliases map[string]string) {
	walkFn := func(path string, _ os.FileInfo, err error) error {
		stat, err := os.Stat(path)
		if err != nil {
			return err
		}

		if stat.IsDir() && path != cacheDirectory {
			return filepath.SkipDir
		}

		if err != nil {
			return err
		}
		if strings.HasSuffix(path, ".metadata") {
			content, err := ioutil.ReadFile(path)
			if err == nil {
				dir, fileName := filepath.Split(path)
				fileNameParts := strings.Split(fileName, ".")
				contentHash := fileNameParts[0]
				fileAliases := strings.Split(string(content), "\n")
				cachedFiles[contentHash] = &CachedFile{fileAliases[0], fileAliases, filepath.Join(dir, contentHash)}
				for _, alias := range fileAliases {
					aliases[alias] = contentHash
				}
			}
		}
		return nil
	}
	err := filepath.Walk(cacheDirectory, walkFn)
	if err != nil {
		fmt.Println(err.Error())
	}
}

func findCachedFile(tokens []string, _ string, cachedFiles map[string]*CachedFile, aliases map[string]string) *CachedFile {
	for _, token := range tokens {
		contentHash, hasContentHash := aliases[token]
		if hasContentHash {
			cachedFile, hasCachedFile := cachedFiles[contentHash]
			if hasCachedFile {
				return cachedFile
			}
		}
	}
	return nil
}

func download(url string, urlAliases []string, cacheDirectory string, cachedFiles map[string]*CachedFile, aliases map[string]string) *CachedFile {
	existingCachedFile := findCachedFile(append(urlAliases, url), cacheDirectory, cachedFiles, aliases)
	if existingCachedFile != nil {
		return existingCachedFile
	}

	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err.Error())
		return nil
	}

	urlHash := hash([]byte(url))
	contentHash := hash(body)
	path := filepath.Join(cacheDirectory, contentHash)

	allAliases := make(map[string]bool)
	allAliases[url] = true
	allAliases[urlHash] = true
	allAliases[contentHash] = true
	for _, alias := range urlAliases {
		allAliases[alias] = true
	}

	cachedFile := &CachedFile{url, mapKeys(allAliases), path}
	cachedFile.StoreAsset(body)
	cachedFile.StoreMetadata()

	cachedFiles[contentHash] = cachedFile
	for _, alias := range cachedFile.Aliases {
		aliases[alias] = contentHash
	}

	return cachedFile
}
<|endoftext|>


<commit_before>// Package peppy implements the osu! API as defined on the osu-api repository wiki (https://github.com/ppy/osu-api/wiki).
package peppy

import (
	"database/sql"
	"fmt"

	"github.com/gin-gonic/gin"
	"github.com/thehowl/go-osuapi"
)

// GetUser retrieves general user information.
func GetUser(c *gin.Context, db *sql.DB) {
	if c.Query("u") == "" {
		c.JSON(200, []struct{}{})
		return
	}
	var user osuapi.User
	whereClause, p := genUser(c, db)

	mode := genmode(c.Query("m"))

	fmt.Println(whereClause, p)

	var display bool
	err := db.QueryRow(fmt.Sprintf(
		`SELECT
			users.id, users.username,
			users_stats.playcount_%s, users_stats.ranked_score_%s, users_stats.total_score_%s,
			leaderboard_%s.position, users_stats.pp_%s, users_stats.avg_accuracy_%s,
			users_stats.country, users_stats.show_country
		FROM users
		LEFT JOIN users_stats ON users_stats.id = users.id
		LEFT JOIN leaderboard_%s ON leaderboard_%s.user = users.id
		%s
		LIMIT 1`,
		mode, mode, mode, mode, mode, mode, mode, mode, whereClause,
	), p).Scan(
		&user.UserID, &user.Username,
		&user.Playcount, &user.RankedScore, &user.TotalScore,
		&user.Rank, &user.PP, &user.Accuracy,
		&user.Country, &display,
	)
	if err != nil {
		c.JSON(200, []struct{}{})
		return
	}
	if !display {
		user.Country = "XX"
	}

	c.JSON(200, []osuapi.User{user})
}
<commit_msg>Remove useless println<commit_after>// Package peppy implements the osu! API as defined on the osu-api repository wiki (https://github.com/ppy/osu-api/wiki).
package peppy

import (
	"database/sql"
	"fmt"

	"github.com/gin-gonic/gin"
	"github.com/thehowl/go-osuapi"
)

// GetUser retrieves general user information.
func GetUser(c *gin.Context, db *sql.DB) {
	if c.Query("u") == "" {
		c.JSON(200, []struct{}{})
		return
	}
	var user osuapi.User
	whereClause, p := genUser(c, db)

	mode := genmode(c.Query("m"))

	var display bool
	err := db.QueryRow(fmt.Sprintf(
		`SELECT
			users.id, users.username,
			users_stats.playcount_%s, users_stats.ranked_score_%s, users_stats.total_score_%s,
			leaderboard_%s.position, users_stats.pp_%s, users_stats.avg_accuracy_%s,
			users_stats.country, users_stats.show_country
		FROM users
		LEFT JOIN users_stats ON users_stats.id = users.id
		LEFT JOIN leaderboard_%s ON leaderboard_%s.user = users.id
		%s
		LIMIT 1`,
		mode, mode, mode, mode, mode, mode, mode, mode, whereClause,
	), p).Scan(
		&user.UserID, &user.Username,
		&user.Playcount, &user.RankedScore, &user.TotalScore,
		&user.Rank, &user.PP, &user.Accuracy,
		&user.Country, &display,
	)
	if err != nil {
		c.JSON(200, []struct{}{})
		return
	}
	if !display {
		user.Country = "XX"
	}

	c.JSON(200, []osuapi.User{user})
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"path"
	"testing"

	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/auth"
	"github.com/tsuru/tsuru/auth/native"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/db/dbtest"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/provision/provisiontest"
	"github.com/tsuru/tsuru/queue"
	_ "github.com/tsuru/tsuru/queue/queuetest"
	"github.com/tsuru/tsuru/quota"
	"github.com/tsuru/tsuru/repository"
	"github.com/tsuru/tsuru/repository/repositorytest"
	"github.com/tsuru/tsuru/service"
	"gopkg.in/check.v1"
	"gopkg.in/mgo.v2/bson"
)

func Test(t *testing.T) { check.TestingT(t) }

type S struct {
	conn        *db.Storage
	team        auth.Team
	user        *auth.User
	adminTeam   auth.Team
	admin       *auth.User
	provisioner *provisiontest.FakeProvisioner
	defaultPlan Plan
	Pool        string
	zeroLock map[string]interface{}
}

var _ = check.Suite(&S{})

type greaterChecker struct{}

func (c *greaterChecker) Info() *check.CheckerInfo {
	return &check.CheckerInfo{Name: "Greater", Params: []string{"expected", "obtained"}}
}

func (c *greaterChecker) Check(params []interface{}, names []string) (bool, string) {
	if len(params) != 2 {
		return false, "you should pass two values to compare"
	}
	n1, ok := params[0].(int)
	if !ok {
		return false, "first parameter should be int"
	}
	n2, ok := params[1].(int)
	if !ok {
		return false, "second parameter should be int"
	}
	if n1 > n2 {
		return true, ""
	}
	err := fmt.Sprintf("%d is not greater than %d", params[0], params[1])
	return false, err
}

var Greater check.Checker = &greaterChecker{}

func (s *S) createUserAndTeam(c *check.C) {
	s.user = &auth.User{
		Email: "whydidifall@thewho.com",
		Quota: quota.Unlimited,
	}
	err := s.user.Create()
	c.Assert(err, check.IsNil)
	s.team = auth.Team{Name: "tsuruteam", Users: []string{s.user.Email}}
	err = s.conn.Teams().Insert(s.team)
	c.Assert(err, check.IsNil)
}

var nativeScheme = auth.Scheme(native.NativeScheme{})

func (s *S) SetUpSuite(c *check.C) {
	err := config.ReadConfigFile("testdata/config.yaml")
	c.Assert(err, check.IsNil)
	s.conn, err = db.Conn()
	c.Assert(err, check.IsNil)
	s.createUserAndTeam(c)
	s.provisioner = provisiontest.NewFakeProvisioner()
	Provisioner = s.provisioner
	AuthScheme = nativeScheme
	platform := Platform{Name: "python"}
	s.conn.Platforms().Insert(platform)
	s.defaultPlan = Plan{
		Name:     "default-plan",
		Memory:   1024,
		Swap:     1024,
		CpuShare: 100,
		Default:  true,
	}
	err = s.conn.Plans().Insert(s.defaultPlan)
	c.Assert(err, check.IsNil)
	s.Pool = "pool1"
	err = provision.AddPool(s.Pool)
	c.Assert(err, check.IsNil)
	data, err := json.Marshal(AppLock{})
	c.Assert(err, check.IsNil)
	err = json.Unmarshal(data, &s.zeroLock)
	c.Assert(err, check.IsNil)
}

func (s *S) TearDownSuite(c *check.C) {
	provision.RemovePool(s.Pool)
	dbtest.ClearAllCollections(s.conn.Apps().Database)
}

func (s *S) SetUpTest(c *check.C) {
	repository.Manager().CreateUser(s.user.Email)
	factory, err := queue.Factory()
	c.Assert(err, check.IsNil)
	factory.Reset()
}

func (s *S) TearDownTest(c *check.C) {
	repositorytest.Reset()
	s.provisioner.Reset()
	LogRemove(nil)
	s.conn.Users().Update(
		bson.M{"email": s.user.Email},
		bson.M{"$set": bson.M{"quota": quota.Unlimited}},
	)
	s.conn.AutoScale().RemoveAll(nil)
	s.conn.Deploys().RemoveAll(nil)
}

func (s *S) getTestData(p ...string) io.ReadCloser {
	p = append([]string{}, ".", "testdata")
	fp := path.Join(p...)
	f, _ := os.OpenFile(fp, os.O_RDONLY, 0)
	return f
}

func (s *S) createAdminUserAndTeam(c *check.C) {
	s.admin = &auth.User{Email: "superuser@gmail.com"}
	err := s.admin.Create()
	c.Assert(err, check.IsNil)
	adminTeamName, err := config.GetString("admin-team")
	c.Assert(err, check.IsNil)
	s.adminTeam = auth.Team{Name: adminTeamName, Users: []string{s.admin.Email}}
	err = s.conn.Teams().Insert(&s.adminTeam)
	c.Assert(err, check.IsNil)
}

func (s *S) removeAdminUserAndTeam(c *check.C) {
	err := s.conn.Teams().RemoveId(s.adminTeam.Name)
	c.Assert(err, check.IsNil)
	err = s.conn.Users().Remove(bson.M{"email": s.admin.Email})
	c.Assert(err, check.IsNil)
}

func (s *S) addServiceInstance(c *check.C, appName string, fn http.HandlerFunc) func() {
	ts := httptest.NewServer(fn)
	ret := func() {
		ts.Close()
		s.conn.Services().Remove(bson.M{"_id": "mysql"})
		s.conn.ServiceInstances().Remove(bson.M{"_id": "my-mysql"})
	}
	srvc := service.Service{Name: "mysql", Endpoint: map[string]string{"production": ts.URL}}
	err := srvc.Create()
	c.Assert(err, check.IsNil)
	instance := service.ServiceInstance{Name: "my-mysql", ServiceName: "mysql", Teams: []string{s.team.Name}}
	err = instance.Create()
	c.Assert(err, check.IsNil)
	err = instance.AddApp(appName)
	c.Assert(err, check.IsNil)
	err = s.conn.ServiceInstances().Update(bson.M{"name": instance.Name}, instance)
	c.Assert(err, check.IsNil)
	return ret
}
<commit_msg>app: gofmt -s -w .<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/http/httptest"
	"os"
	"path"
	"testing"

	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/auth"
	"github.com/tsuru/tsuru/auth/native"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/db/dbtest"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/provision/provisiontest"
	"github.com/tsuru/tsuru/queue"
	_ "github.com/tsuru/tsuru/queue/queuetest"
	"github.com/tsuru/tsuru/quota"
	"github.com/tsuru/tsuru/repository"
	"github.com/tsuru/tsuru/repository/repositorytest"
	"github.com/tsuru/tsuru/service"
	"gopkg.in/check.v1"
	"gopkg.in/mgo.v2/bson"
)

func Test(t *testing.T) { check.TestingT(t) }

type S struct {
	conn        *db.Storage
	team        auth.Team
	user        *auth.User
	adminTeam   auth.Team
	admin       *auth.User
	provisioner *provisiontest.FakeProvisioner
	defaultPlan Plan
	Pool        string
	zeroLock    map[string]interface{}
}

var _ = check.Suite(&S{})

type greaterChecker struct{}

func (c *greaterChecker) Info() *check.CheckerInfo {
	return &check.CheckerInfo{Name: "Greater", Params: []string{"expected", "obtained"}}
}

func (c *greaterChecker) Check(params []interface{}, names []string) (bool, string) {
	if len(params) != 2 {
		return false, "you should pass two values to compare"
	}
	n1, ok := params[0].(int)
	if !ok {
		return false, "first parameter should be int"
	}
	n2, ok := params[1].(int)
	if !ok {
		return false, "second parameter should be int"
	}
	if n1 > n2 {
		return true, ""
	}
	err := fmt.Sprintf("%d is not greater than %d", params[0], params[1])
	return false, err
}

var Greater check.Checker = &greaterChecker{}

func (s *S) createUserAndTeam(c *check.C) {
	s.user = &auth.User{
		Email: "whydidifall@thewho.com",
		Quota: quota.Unlimited,
	}
	err := s.user.Create()
	c.Assert(err, check.IsNil)
	s.team = auth.Team{Name: "tsuruteam", Users: []string{s.user.Email}}
	err = s.conn.Teams().Insert(s.team)
	c.Assert(err, check.IsNil)
}

var nativeScheme = auth.Scheme(native.NativeScheme{})

func (s *S) SetUpSuite(c *check.C) {
	err := config.ReadConfigFile("testdata/config.yaml")
	c.Assert(err, check.IsNil)
	s.conn, err = db.Conn()
	c.Assert(err, check.IsNil)
	s.createUserAndTeam(c)
	s.provisioner = provisiontest.NewFakeProvisioner()
	Provisioner = s.provisioner
	AuthScheme = nativeScheme
	platform := Platform{Name: "python"}
	s.conn.Platforms().Insert(platform)
	s.defaultPlan = Plan{
		Name:     "default-plan",
		Memory:   1024,
		Swap:     1024,
		CpuShare: 100,
		Default:  true,
	}
	err = s.conn.Plans().Insert(s.defaultPlan)
	c.Assert(err, check.IsNil)
	s.Pool = "pool1"
	err = provision.AddPool(s.Pool)
	c.Assert(err, check.IsNil)
	data, err := json.Marshal(AppLock{})
	c.Assert(err, check.IsNil)
	err = json.Unmarshal(data, &s.zeroLock)
	c.Assert(err, check.IsNil)
}

func (s *S) TearDownSuite(c *check.C) {
	provision.RemovePool(s.Pool)
	dbtest.ClearAllCollections(s.conn.Apps().Database)
}

func (s *S) SetUpTest(c *check.C) {
	repository.Manager().CreateUser(s.user.Email)
	factory, err := queue.Factory()
	c.Assert(err, check.IsNil)
	factory.Reset()
}

func (s *S) TearDownTest(c *check.C) {
	repositorytest.Reset()
	s.provisioner.Reset()
	LogRemove(nil)
	s.conn.Users().Update(
		bson.M{"email": s.user.Email},
		bson.M{"$set": bson.M{"quota": quota.Unlimited}},
	)
	s.conn.AutoScale().RemoveAll(nil)
	s.conn.Deploys().RemoveAll(nil)
}

func (s *S) getTestData(p ...string) io.ReadCloser {
	p = append([]string{}, ".", "testdata")
	fp := path.Join(p...)
	f, _ := os.OpenFile(fp, os.O_RDONLY, 0)
	return f
}

func (s *S) createAdminUserAndTeam(c *check.C) {
	s.admin = &auth.User{Email: "superuser@gmail.com"}
	err := s.admin.Create()
	c.Assert(err, check.IsNil)
	adminTeamName, err := config.GetString("admin-team")
	c.Assert(err, check.IsNil)
	s.adminTeam = auth.Team{Name: adminTeamName, Users: []string{s.admin.Email}}
	err = s.conn.Teams().Insert(&s.adminTeam)
	c.Assert(err, check.IsNil)
}

func (s *S) removeAdminUserAndTeam(c *check.C) {
	err := s.conn.Teams().RemoveId(s.adminTeam.Name)
	c.Assert(err, check.IsNil)
	err = s.conn.Users().Remove(bson.M{"email": s.admin.Email})
	c.Assert(err, check.IsNil)
}

func (s *S) addServiceInstance(c *check.C, appName string, fn http.HandlerFunc) func() {
	ts := httptest.NewServer(fn)
	ret := func() {
		ts.Close()
		s.conn.Services().Remove(bson.M{"_id": "mysql"})
		s.conn.ServiceInstances().Remove(bson.M{"_id": "my-mysql"})
	}
	srvc := service.Service{Name: "mysql", Endpoint: map[string]string{"production": ts.URL}}
	err := srvc.Create()
	c.Assert(err, check.IsNil)
	instance := service.ServiceInstance{Name: "my-mysql", ServiceName: "mysql", Teams: []string{s.team.Name}}
	err = instance.Create()
	c.Assert(err, check.IsNil)
	err = instance.AddApp(appName)
	c.Assert(err, check.IsNil)
	err = s.conn.ServiceInstances().Update(bson.M{"name": instance.Name}, instance)
	c.Assert(err, check.IsNil)
	return ret
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// request contains logic to make polling HTTP requests against an endpoint with optional host spoofing.

package test

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/knative/pkg/test/logging"
	"github.com/knative/pkg/test/spoof"
)

// Retrying modifies a ResponseChecker to retry certain response codes.
func Retrying(rc spoof.ResponseChecker, codes ...int) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		for _, code := range codes {
			if resp.StatusCode == code {
				// Returning (false, nil) causes SpoofingClient.Poll to retry.
				// sc.logger.Infof("Retrying for code %v", resp.StatusCode)
				return false, nil
			}
		}

		// If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped.
		return rc(resp)
	}
}

// IsOneOfStatusCodes checks that the response code is equal to the given one.
func IsOneOfStatusCodes(codes ...int) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		for _, code := range codes {
			if resp.StatusCode == code {
				return true, nil
			}
		}

		return true, fmt.Errorf("status = %d, want one of: %v", resp.StatusCode, codes)
	}
}

// IsStatusOK checks that the response code is a 200.
func IsStatusOK(resp *spoof.Response) (bool, error) {
	return IsOneOfStatusCodes(http.StatusOK)(resp)
}

// MatchesBody checks that the *first* response body matches the "expected" body, otherwise failing.
func MatchesBody(expected string) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		if !strings.Contains(string(resp.Body), expected) {
			// Returning (true, err) causes SpoofingClient.Poll to fail.
			return true, fmt.Errorf("body = %s, want: %s", string(resp.Body), expected)
		}

		return true, nil
	}
}

// EventuallyMatchesBody checks that the response body *eventually* matches the expected body.
// TODO(#1178): Delete me. We don't want to need this; we should be waiting for an appropriate Status instead.
func EventuallyMatchesBody(expected string) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		if !strings.Contains(string(resp.Body), expected) {
			// Returning (false, nil) causes SpoofingClient.Poll to retry.
			return false, nil
		}

		return true, nil
	}
}

// MatchesAllOf combines multiple ResponseCheckers to one ResponseChecker with a logical AND. The
// checkers are executed in order. The first function to trigger an error or a retry will short-circuit
// the other functions (they will not be executed).
//
// This is useful for combining a body with a status check like:
// MatchesAllOf(IsStatusOK, MatchesBody("test"))
//
// The MatchesBody check will only be executed after the IsStatusOK has passed.
func MatchesAllOf(checkers ...spoof.ResponseChecker) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		for _, checker := range checkers {
			done, err := checker(resp)
			if err != nil || !done {
				return done, err
			}
		}
		return true, nil
	}
}

// WaitForEndpointState will poll an endpoint until inState indicates the state is achieved,
// or default timeout is reached.
// If resolvableDomain is false, it will use kubeClientset to look up the ingress and spoof
// the domain in the request headers, otherwise it will make the request directly to domain.
// desc will be used to name the metric that is emitted to track how long it took for the
// domain to get into the state checked by inState.  Commas in `desc` must be escaped.
func WaitForEndpointState(kubeClient *KubeClient, logf logging.FormatLogger, theURL string, inState spoof.ResponseChecker, desc string, resolvable bool) (*spoof.Response, error) {
	return WaitForEndpointStateWithTimeout(kubeClient, logf, theURL, inState, desc, resolvable, spoof.RequestTimeout)
}

// WaitForEndpointStateWithTimeout will poll an endpoint until inState indicates the state is achieved
// or the provided timeout is achieved.
// If resolvableDomain is false, it will use kubeClientset to look up the ingress and spoof
// the domain in the request headers, otherwise it will make the request directly to domain.
// desc will be used to name the metric that is emitted to track how long it took for the
// domain to get into the state checked by inState.  Commas in `desc` must be escaped.
func WaitForEndpointStateWithTimeout(
	kubeClient *KubeClient, logf logging.FormatLogger, theURL string, inState spoof.ResponseChecker,
	desc string, resolvable bool, timeout time.Duration) (*spoof.Response, error) {
	defer logging.GetEmitableSpan(context.Background(), fmt.Sprintf("WaitForEndpointState/%s", desc)).End()

	// Try parsing the "theURL" with and without a scheme.
	asURL, err := url.Parse(fmt.Sprintf("http://%s", theURL))
	if err != nil {
		asURL, err = url.Parse(theURL)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(http.MethodGet, asURL.String(), nil)
	if err != nil {
		return nil, err
	}

	client, err := NewSpoofingClient(kubeClient, logf, asURL.Hostname(), resolvable)
	if err != nil {
		return nil, err
	}
	client.RequestTimeout = timeout

	return client.Poll(req, inState)
}
<commit_msg>Allow endpoint polling to take request options (#469)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// request contains logic to make polling HTTP requests against an endpoint with optional host spoofing.

package test

import (
	"context"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"

	"github.com/knative/pkg/test/logging"
	"github.com/knative/pkg/test/spoof"
)

// RequestOption enables configuration of requests
// when polling for endpoint states.
type RequestOption func(*http.Request)

// WithHeader will add the provided headers to the request.
func WithHeader(header http.Header) RequestOption {
	return func(r *http.Request) {
		if r.Header == nil {
			r.Header = header
			return
		}
		for key, values := range header {
			for _, value := range values {
				r.Header.Add(key, value)
			}
		}
	}
}

// Retrying modifies a ResponseChecker to retry certain response codes.
func Retrying(rc spoof.ResponseChecker, codes ...int) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		for _, code := range codes {
			if resp.StatusCode == code {
				// Returning (false, nil) causes SpoofingClient.Poll to retry.
				// sc.logger.Infof("Retrying for code %v", resp.StatusCode)
				return false, nil
			}
		}

		// If we didn't match any retryable codes, invoke the ResponseChecker that we wrapped.
		return rc(resp)
	}
}

// IsOneOfStatusCodes checks that the response code is equal to the given one.
func IsOneOfStatusCodes(codes ...int) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		for _, code := range codes {
			if resp.StatusCode == code {
				return true, nil
			}
		}

		return true, fmt.Errorf("status = %d, want one of: %v", resp.StatusCode, codes)
	}
}

// IsStatusOK checks that the response code is a 200.
func IsStatusOK(resp *spoof.Response) (bool, error) {
	return IsOneOfStatusCodes(http.StatusOK)(resp)
}

// MatchesBody checks that the *first* response body matches the "expected" body, otherwise failing.
func MatchesBody(expected string) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		if !strings.Contains(string(resp.Body), expected) {
			// Returning (true, err) causes SpoofingClient.Poll to fail.
			return true, fmt.Errorf("body = %s, want: %s", string(resp.Body), expected)
		}

		return true, nil
	}
}

// EventuallyMatchesBody checks that the response body *eventually* matches the expected body.
// TODO(#1178): Delete me. We don't want to need this; we should be waiting for an appropriate Status instead.
func EventuallyMatchesBody(expected string) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		if !strings.Contains(string(resp.Body), expected) {
			// Returning (false, nil) causes SpoofingClient.Poll to retry.
			return false, nil
		}

		return true, nil
	}
}

// MatchesAllOf combines multiple ResponseCheckers to one ResponseChecker with a logical AND. The
// checkers are executed in order. The first function to trigger an error or a retry will short-circuit
// the other functions (they will not be executed).
//
// This is useful for combining a body with a status check like:
// MatchesAllOf(IsStatusOK, MatchesBody("test"))
//
// The MatchesBody check will only be executed after the IsStatusOK has passed.
func MatchesAllOf(checkers ...spoof.ResponseChecker) spoof.ResponseChecker {
	return func(resp *spoof.Response) (bool, error) {
		for _, checker := range checkers {
			done, err := checker(resp)
			if err != nil || !done {
				return done, err
			}
		}
		return true, nil
	}
}

// WaitForEndpointState will poll an endpoint until inState indicates the state is achieved,
// or default timeout is reached.
// If resolvableDomain is false, it will use kubeClientset to look up the ingress and spoof
// the domain in the request headers, otherwise it will make the request directly to domain.
// desc will be used to name the metric that is emitted to track how long it took for the
// domain to get into the state checked by inState.  Commas in `desc` must be escaped.
func WaitForEndpointState(kubeClient *KubeClient, logf logging.FormatLogger, theURL string, inState spoof.ResponseChecker, desc string, resolvable bool, opts ...RequestOption) (*spoof.Response, error) {
	return WaitForEndpointStateWithTimeout(kubeClient, logf, theURL, inState, desc, resolvable, spoof.RequestTimeout, opts...)
}

// WaitForEndpointStateWithTimeout will poll an endpoint until inState indicates the state is achieved
// or the provided timeout is achieved.
// If resolvableDomain is false, it will use kubeClientset to look up the ingress and spoof
// the domain in the request headers, otherwise it will make the request directly to domain.
// desc will be used to name the metric that is emitted to track how long it took for the
// domain to get into the state checked by inState.  Commas in `desc` must be escaped.
func WaitForEndpointStateWithTimeout(
	kubeClient *KubeClient, logf logging.FormatLogger, theURL string, inState spoof.ResponseChecker,
	desc string, resolvable bool, timeout time.Duration, opts ...RequestOption) (*spoof.Response, error) {
	defer logging.GetEmitableSpan(context.Background(), fmt.Sprintf("WaitForEndpointState/%s", desc)).End()

	// Try parsing the "theURL" with and without a scheme.
	asURL, err := url.Parse(fmt.Sprintf("http://%s", theURL))
	if err != nil {
		asURL, err = url.Parse(theURL)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(http.MethodGet, asURL.String(), nil)
	if err != nil {
		return nil, err
	}

	for _, opt := range opts {
		opt(req)
	}

	client, err := NewSpoofingClient(kubeClient, logf, asURL.Hostname(), resolvable)
	if err != nil {
		return nil, err
	}
	client.RequestTimeout = timeout

	return client.Poll(req, inState)
}
<|endoftext|>


<commit_before>package registry

import (
	"bytes"
	"io/ioutil"
	"net/http"

	"github.com/docker/distribution/manifest"
)

func (registry *Registry) Manifest(repository, reference string) (*manifest.SignedManifest, error) {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)

	resp, err := registry.Client.Get(url)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	signedManifest := &manifest.SignedManifest{}
	err = signedManifest.UnmarshalJSON(body)
	if err != nil {
		return nil, err
	}

	return signedManifest, nil
}

func (registry *Registry) PutManifest(repository, reference string, signedManifest *manifest.SignedManifest) error {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.put url=%s repository=%s reference=%s", url, repository, reference)

	body, err := signedManifest.MarshalJSON()
	if err != nil {
		return err
	}

	buffer := bytes.NewBuffer(body)
	req, err := http.NewRequest("PUT", url, buffer)
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", manifest.ManifestMediaType)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	return err
}
<commit_msg>update manifest import and references to reference schema1<commit_after>package registry

import (
	"bytes"
	"io/ioutil"
	"net/http"

	manifest "github.com/docker/distribution/manifest/schema1"
)

func (registry *Registry) Manifest(repository, reference string) (*manifest.SignedManifest, error) {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.get url=%s repository=%s reference=%s", url, repository, reference)

	resp, err := registry.Client.Get(url)
	if resp != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return nil, err
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	signedManifest := &manifest.SignedManifest{}
	err = signedManifest.UnmarshalJSON(body)
	if err != nil {
		return nil, err
	}

	return signedManifest, nil
}

func (registry *Registry) PutManifest(repository, reference string, signedManifest *manifest.SignedManifest) error {
	url := registry.url("/v2/%s/manifests/%s", repository, reference)
	registry.Logf("registry.manifest.put url=%s repository=%s reference=%s", url, repository, reference)

	body, err := signedManifest.MarshalJSON()
	if err != nil {
		return err
	}

	buffer := bytes.NewBuffer(body)
	req, err := http.NewRequest("PUT", url, buffer)
	if err != nil {
		return err
	}

	req.Header.Set("Content-Type", manifest.MediaTypeManifest)
	resp, err := registry.Client.Do(req)
	if resp != nil {
		defer resp.Body.Close()
	}
	return err
}
<|endoftext|>


<commit_before>package jsonunroller

import (
	"appengine"
	"appengine/urlfetch"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

func dumpobj(prefix string, x interface{}) string {

	s := ""
	switch t := x.(type) {
	case map[string]interface{}:
		for k, v := range t {
			s += dumpobj(prefix+"."+k, v)
		}
	case []interface{}:
		for i, v := range t {
			s += dumpobj(prefix+"["+strconv.Itoa(i)+"]", v)
		}
	case string:
		s += fmt.Sprintf("%s = %q\n", prefix, t)
	default:
		fmt.Printf("Unhandled: %T\n", t)
	}
	return s
}

func init() {
	http.HandleFunc("/", root)
	http.HandleFunc("/unroll", unroll)
}

func root(w http.ResponseWriter, r *http.Request) {
	u := r.FormValue("u")
	log.Println("URL:", u)
	if u == "" {
		fmt.Fprint(w, postForm)
		return
	}

	pu, err := url.ParseRequestURI(u)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if pu.IsAbs() != true {
		err = errors.New("Not absolute URL")
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	c := appengine.NewContext(r)
	client := urlfetch.Client(c)
	res, err := client.Get(pu.String())
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	contenttype := res.Header.Get("Content-Type")
	if strings.Contains(contenttype, "application/json") != true {
		err = errors.New("Not application/json: " + contenttype)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	j, err := ioutil.ReadAll(res.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	// res.Body.Close()

	var pj interface{}
	err = json.Unmarshal([]byte(j), &pj)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	s := dumpobj("this", pj)

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	io.WriteString(w, s)

}

const postForm = `

<form action="/" method="get">

<input type=url name=u size=100></input>

<input type="submit" value="Unroll JSON URL">

</form>
<form action="/unroll" method="post">

<textarea style="box-sizing: border-box; height: 60%; width: 100%;" name="content"></textarea>

<input type="submit">

curl --data-urlencode content@foobar.json http://jsonunroller.appspot.com/unroll

MIT source code

</form>

`

func unroll(w http.ResponseWriter, r *http.Request) {

	f := r.FormValue("content")
	if f == "" {
		err := errors.New("No content!")
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var pj interface{}
	err := json.Unmarshal([]byte(f), &pj)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	s := dumpobj("this", pj)

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	io.WriteString(w, s)

}
<commit_msg>Close the Body like I should http://golang.org/pkg/net/http/#Client.Do<commit_after>package jsonunroller

import (
	"appengine"
	"appengine/urlfetch"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

func dumpobj(prefix string, x interface{}) string {

	s := ""
	switch t := x.(type) {
	case map[string]interface{}:
		for k, v := range t {
			s += dumpobj(prefix+"."+k, v)
		}
	case []interface{}:
		for i, v := range t {
			s += dumpobj(prefix+"["+strconv.Itoa(i)+"]", v)
		}
	case string:
		s += fmt.Sprintf("%s = %q\n", prefix, t)
	default:
		fmt.Printf("Unhandled: %T\n", t)
	}
	return s
}

func init() {
	http.HandleFunc("/", root)
	http.HandleFunc("/unroll", unroll)
}

func root(w http.ResponseWriter, r *http.Request) {
	u := r.FormValue("u")
	log.Println("URL:", u)
	if u == "" {
		fmt.Fprint(w, postForm)
		return
	}

	pu, err := url.ParseRequestURI(u)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if pu.IsAbs() != true {
		err = errors.New("Not absolute URL")
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	c := appengine.NewContext(r)
	client := urlfetch.Client(c)
	res, err := client.Get(pu.String())
	defer res.Body.Close()
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	contenttype := res.Header.Get("Content-Type")
	if strings.Contains(contenttype, "application/json") != true {
		err = errors.New("Not application/json: " + contenttype)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	j, err := ioutil.ReadAll(res.Body)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var pj interface{}
	err = json.Unmarshal([]byte(j), &pj)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	s := dumpobj("this", pj)

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	io.WriteString(w, s)

}

const postForm = `

<form action="/" method="get">

<input type=url name=u size=100></input>

<input type="submit" value="Unroll JSON URL">

</form>
<form action="/unroll" method="post">

<textarea style="box-sizing: border-box; height: 60%; width: 100%;" name="content"></textarea>

<input type="submit">

curl --data-urlencode content@foobar.json http://jsonunroller.appspot.com/unroll

MIT source code

</form>

`

func unroll(w http.ResponseWriter, r *http.Request) {

	f := r.FormValue("content")
	if f == "" {
		err := errors.New("No content!")
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var pj interface{}
	err := json.Unmarshal([]byte(f), &pj)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	s := dumpobj("this", pj)

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	io.WriteString(w, s)

}
<|endoftext|>