<commit_before>package shark

import(
	"strings"
	"os"
	"libxml"
	"fmt"
	xml "libxml/tree"
	tp "tritium/proto"
	"libxml/xpath"
	"rubex"
)

func (ctx *Ctx) runBuiltIn(fun *Function, scope *Scope, ins *tp.Instruction, args []interface{}) (returnValue interface{}) {
	returnValue = ""
	switch fun.Name {
	case "this":
		returnValue = scope.Value
	case "yield":
		myYieldBlock := ctx.yieldBlock()
		ctx.Yields = ctx.Yields[:(len(ctx.Yields)-1)]
		if (ctx.yieldBlock() != nil) {
			returnValue = ctx.runChildren(scope, myYieldBlock.Ins)
		} else {
			ctx.Log.Error("yield() failure")
		}
		ctx.Yields = append(ctx.Yields, myYieldBlock)

	case "var.Text":
		val := ctx.Env[args[0].(string)]
		returnValue = val
		if len(ins.Children) > 0 {
			ts := &Scope{Value: val}
			ctx.runChildren(ts, ins)
			returnValue = ts.Value
			ctx.Env[args[0].(string)] = returnValue.(string)
		}
	case "var.Text.Text":
		ctx.Env[args[0].(string)] = args[1].(string)
		returnValue = args[1].(string)
	case "deprecated.Text":
		ctx.Log.Info(args[0].(string))
	case "match.Text":
		// Setup stacks
		against, ok := args[0].(string)
		if !ok {
			ctx.Log.Error("AH!")
		}
		ctx.MatchStack = append(ctx.MatchStack, against)
		ctx.MatchShouldContinue = append(ctx.MatchShouldContinue, true)

		// Run children
		ctx.runChildren(scope, ins)

		if ctx.matchShouldContinue() {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		// Clear
		ctx.MatchShouldContinue = ctx.MatchShouldContinue[:len(ctx.MatchShouldContinue)-1]
		ctx.MatchStack = ctx.MatchStack[:len(ctx.MatchStack)-1]
	case "with.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) == ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "with.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if (args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) != ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if !(args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "regexp.Text.Text":
		mode := rubex.ONIG_OPTION_DEFAULT
		if strings.Index(args[1].(string), "i") >= 0 {
			mode = rubex.ONIG_OPTION_IGNORECASE
		}
		if strings.Index(args[1].(string), "m") >= 0 {
			mode = rubex.ONIG_OPTION_MULTILINE
		}
		var err os.Error
		returnValue, err = rubex.NewRegexp(args[0].(string), mode)
		if err != nil {
			ctx.Log.Error("Invalid regexp")
		}
	case "export.Text":
		val := make([]string, 2)
		val[0] = args[0].(string)
		ts := &Scope{Value:""}
		ctx.runChildren(ts, ins)
		val[1] = ts.Value.(string)
		ctx.Exports = append(ctx.Exports, val)
	case "log.Text":
		ctx.Logs = append(ctx.Logs, args[0].(string))

	// ATOMIC FUNCTIONS
	case "concat.Text.Text":
		//println("Concat:", args[0].(string), "+", args[1].(string))
		returnValue = args[0].(string) + args[1].(string)
	case "concat.Text.Text.Text": //REMOVE
		returnValue = args[0].(string) + args[1].(string) + args[2].(string)
	case "downcase.Text":
		returnValue = strings.ToLower(args[0].(string))
		return
	case "upcase.Text":
		returnValue = strings.ToUpper(args[0].(string))
		return
	case "index.XMLNode":
		returnValue = fmt.Sprintf("%d", scope.Index + 1)

	// TEXT FUNCTIONS
	case "set.Text":
		scope.Value = args[0]
	case "append.Text":
		scope.Value = scope.Value.(string) + args[0].(string)
	case "prepend.Text":
		scope.Value = args[0].(string) + scope.Value.(string)
	case "replace.Text":
		ts := &Scope{Value:""}
		ctx.runChildren(ts, ins)
		scope.Value = strings.Replace(scope.Value.(string), args[0].(string), ts.Value.(string), -1)
	case "replace.Regexp":
		regexp := args[0].(*rubex.Regexp)
		scope.Value = regexp.GsubFunc(scope.Value.(string), func(match string, captures map[string]string) string {
			usesGlobal := (ctx.Env["use_global_replace_vars"] == "true")

			for name, capture := range captures {
				if usesGlobal {
					//println("setting $", name, "to", capture)
					ctx.Env[name] = capture
				}
				ctx.vars()[name] = capture
			}

			replacementScope := &Scope{Value:match}
			ctx.runChildren(replacementScope, ins)
			//println(ins.String())

			//println("Replacement:", replacementScope.Value.(string))
			innerReplacer := rubex.MustCompile(`[\\$](\d)`)
			return innerReplacer.GsubFunc(replacementScope.Value.(string), func(_ string, numeric_captures map[string]string) string {
				capture := numeric_captures["1"]
				var val string
				if usesGlobal {
					val = ctx.Env[capture]
				} else {
					val = ctx.vars()[capture].(string)
				}
				return val
		    })
		})
		returnValue = scope.Value

	// XML FUNCTIONS
	case "xml":
		doc := libxml.XmlParseString(scope.Value.(string))
		ns := &Scope{Value:doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.String()
		returnValue = scope.Value
		doc.Free()
	case "html":
		doc := libxml.HtmlParseString(scope.Value.(string))
		ns := &Scope{Value:doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.DumpHTML()
		returnValue = scope.Value
		doc.Free()
	case "html_fragment":
		doc := libxml.HtmlParseFragment(scope.Value.(string))
		ns := &Scope{Value: doc.RootElement()}
		ctx.runChildren(ns, ins)
		scope.Value = ns.Value.(xml.Node).Content()
		returnValue = scope.Value
		doc.Free()
	case "select.Text":
		// TODO reuse XPath object
		node := scope.Value.(xml.Node)
		xpCtx := xpath.NewXPath(node.Doc())
		xpath := xpath.CompileXPath(args[0].(string))
		//xpath := ctx.XPath(args[0].(string))
		nodeSet := xpCtx.SearchByCompiledXPath(node, xpath).Slice()
		defer xpCtx.Free()
		if len(nodeSet) == 0 {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		for index, node := range(nodeSet) {
			if (node != nil) && node.IsLinked() && node.IsValid() {
				ns := &Scope{Value: node, Index: index}
				ctx.runChildren(ns, ins)
			}
		}
	case "position.Text":
		returnValue = Positions[args[0].(string)]

	// SHARED NODE FUNCTIONS
	case "remove":
		scope.Value.(xml.Node).Remove()
	case "inner", "inner_text", "text":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Element)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "value":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Attribute)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "name":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Name()}
		ctx.runChildren(ts, ins)
		node.SetName(ts.Value.(string))
		returnValue = ts.Value.(string)
	case "dup":
		node := scope.Value.(xml.Node)
		newNode := node.Duplicate()
		_, isElement := node.(*xml.Element)
		if isElement {
			MoveFunc(newNode, node, AFTER)
		}
		ns := &Scope{Value:newNode}
		ctx.runChildren(ns, ins)
	case "fetch.Text":
		searchNode := scope.Value.(xml.Node)
		xPathObj := xpath.NewXPath(searchNode.Doc())
		nodeSet := xPathObj.Search(searchNode, args[0].(string))
		if nodeSet.Size() > 0 {
			node := nodeSet.First()
			attr, ok := node.(*xml.Attribute)
			if ok {
				returnValue = attr.Content()
			} else {
				returnValue = node.String()
			}
		}
		xPathObj.Free()

	// LIBXML FUNCTIONS
	case "insert_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		tagName := args[1].(string)
		element := node.Doc().NewElement(tagName)
		MoveFunc(element, node, position)
		ns := &Scope{Value: element}
		ctx.runChildren(ns, ins)
	case "inject_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		nodeSet := node.Doc().ParseHtmlFragment(args[1].(string))
		for _, newNode := range(nodeSet) {
			MoveFunc(newNode, node, position)
		}
		if len(nodeSet) > 0 {
			element, ok := nodeSet[0].(*xml.Element)
			if ok {
				// successfully ran scope
				returnValue = "true"
				ns := &Scope{Value: element}
				ctx.runChildren(ns, ins)
			}
		} else {
			returnValue = "false"
		}
	case "cdata.Text":
		elem, ok := scope.Value.(*xml.Element)
		if ok {
			elem.SetCDataContent(args[0].(string))
		}
	case "move.XMLNode.XMLNode.Position", "move.Node.Node.Position":
		//for name, value := range(ctx.LocalVar) {
		//	println(name, ":", value)
		//}
		MoveFunc(args[0].(xml.Node), args[1].(xml.Node), args[2].(Position))
	case "wrap_text_children.Text":
		returnValue = "false"
		child := scope.Value.(xml.Node).First()
		index := 0
		tagName := args[0].(string)
		for child != nil {
			text, ok := child.(*xml.Text)
			childNext := child.Next()
			if ok {
				returnValue = "true"
				wrap := text.Wrap(tagName)
				ns := &Scope{wrap, index}
				ctx.runChildren(ns, ins)
				index++
			}
			child = childNext
		}

	// ATTRIBUTE FUNCTIONS
	case "attribute.Text":
		node := scope.Value.(xml.Node)
		name := args[0].(string)
		_, ok := node.(*xml.Element)
		if ok == true {
			attr, _ := node.Attribute(name)

			as := &Scope{Value:attr}
			ctx.runChildren(as, ins)
			if attr.IsLinked() && (attr.Content() == "") {
				attr.Remove()
			}
			if !attr.IsLinked() {
				attr.Free()
			}
			returnValue = "true"
		}
	case "to_text.XMLNode":
		returnValue = scope.Value.(xml.Node).String()
	default:
		ctx.Log.Error("Must implement " + fun.Name)
	}
	return
}<commit_msg>a node which si linked, is already checked for being valid<commit_after>package shark

import(
	"strings"
	"os"
	"libxml"
	"fmt"
	xml "libxml/tree"
	tp "tritium/proto"
	"libxml/xpath"
	"rubex"
)

func (ctx *Ctx) runBuiltIn(fun *Function, scope *Scope, ins *tp.Instruction, args []interface{}) (returnValue interface{}) {
	returnValue = ""
	switch fun.Name {
	case "this":
		returnValue = scope.Value
	case "yield":
		myYieldBlock := ctx.yieldBlock()
		ctx.Yields = ctx.Yields[:(len(ctx.Yields)-1)]
		if (ctx.yieldBlock() != nil) {
			returnValue = ctx.runChildren(scope, myYieldBlock.Ins)
		} else {
			ctx.Log.Error("yield() failure")
		}
		ctx.Yields = append(ctx.Yields, myYieldBlock)

	case "var.Text":
		val := ctx.Env[args[0].(string)]
		returnValue = val
		if len(ins.Children) > 0 {
			ts := &Scope{Value: val}
			ctx.runChildren(ts, ins)
			returnValue = ts.Value
			ctx.Env[args[0].(string)] = returnValue.(string)
		}
	case "var.Text.Text":
		ctx.Env[args[0].(string)] = args[1].(string)
		returnValue = args[1].(string)
	case "deprecated.Text":
		ctx.Log.Info(args[0].(string))
	case "match.Text":
		// Setup stacks
		against, ok := args[0].(string)
		if !ok {
			ctx.Log.Error("AH!")
		}
		ctx.MatchStack = append(ctx.MatchStack, against)
		ctx.MatchShouldContinue = append(ctx.MatchShouldContinue, true)

		// Run children
		ctx.runChildren(scope, ins)

		if ctx.matchShouldContinue() {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		// Clear
		ctx.MatchShouldContinue = ctx.MatchShouldContinue[:len(ctx.MatchShouldContinue)-1]
		ctx.MatchStack = ctx.MatchStack[:len(ctx.MatchStack)-1]
	case "with.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) == ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "with.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if (args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) != ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if !(args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "regexp.Text.Text":
		mode := rubex.ONIG_OPTION_DEFAULT
		if strings.Index(args[1].(string), "i") >= 0 {
			mode = rubex.ONIG_OPTION_IGNORECASE
		}
		if strings.Index(args[1].(string), "m") >= 0 {
			mode = rubex.ONIG_OPTION_MULTILINE
		}
		var err os.Error
		returnValue, err = rubex.NewRegexp(args[0].(string), mode)
		if err != nil {
			ctx.Log.Error("Invalid regexp")
		}
	case "export.Text":
		val := make([]string, 2)
		val[0] = args[0].(string)
		ts := &Scope{Value:""}
		ctx.runChildren(ts, ins)
		val[1] = ts.Value.(string)
		ctx.Exports = append(ctx.Exports, val)
	case "log.Text":
		ctx.Logs = append(ctx.Logs, args[0].(string))

	// ATOMIC FUNCTIONS
	case "concat.Text.Text":
		//println("Concat:", args[0].(string), "+", args[1].(string))
		returnValue = args[0].(string) + args[1].(string)
	case "concat.Text.Text.Text": //REMOVE
		returnValue = args[0].(string) + args[1].(string) + args[2].(string)
	case "downcase.Text":
		returnValue = strings.ToLower(args[0].(string))
		return
	case "upcase.Text":
		returnValue = strings.ToUpper(args[0].(string))
		return
	case "index.XMLNode":
		returnValue = fmt.Sprintf("%d", scope.Index + 1)

	// TEXT FUNCTIONS
	case "set.Text":
		scope.Value = args[0]
	case "append.Text":
		scope.Value = scope.Value.(string) + args[0].(string)
	case "prepend.Text":
		scope.Value = args[0].(string) + scope.Value.(string)
	case "replace.Text":
		ts := &Scope{Value:""}
		ctx.runChildren(ts, ins)
		scope.Value = strings.Replace(scope.Value.(string), args[0].(string), ts.Value.(string), -1)
	case "replace.Regexp":
		regexp := args[0].(*rubex.Regexp)
		scope.Value = regexp.GsubFunc(scope.Value.(string), func(match string, captures map[string]string) string {
			usesGlobal := (ctx.Env["use_global_replace_vars"] == "true")

			for name, capture := range captures {
				if usesGlobal {
					//println("setting $", name, "to", capture)
					ctx.Env[name] = capture
				}
				ctx.vars()[name] = capture
			}

			replacementScope := &Scope{Value:match}
			ctx.runChildren(replacementScope, ins)
			//println(ins.String())

			//println("Replacement:", replacementScope.Value.(string))
			innerReplacer := rubex.MustCompile(`[\\$](\d)`)
			return innerReplacer.GsubFunc(replacementScope.Value.(string), func(_ string, numeric_captures map[string]string) string {
				capture := numeric_captures["1"]
				var val string
				if usesGlobal {
					val = ctx.Env[capture]
				} else {
					val = ctx.vars()[capture].(string)
				}
				return val
		    })
		})
		returnValue = scope.Value

	// XML FUNCTIONS
	case "xml":
		doc := libxml.XmlParseString(scope.Value.(string))
		ns := &Scope{Value:doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.String()
		returnValue = scope.Value
		doc.Free()
	case "html":
		doc := libxml.HtmlParseString(scope.Value.(string))
		ns := &Scope{Value:doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.DumpHTML()
		returnValue = scope.Value
		doc.Free()
	case "html_fragment":
		doc := libxml.HtmlParseFragment(scope.Value.(string))
		ns := &Scope{Value: doc.RootElement()}
		ctx.runChildren(ns, ins)
		scope.Value = ns.Value.(xml.Node).Content()
		returnValue = scope.Value
		doc.Free()
	case "select.Text":
		// TODO reuse XPath object
		node := scope.Value.(xml.Node)
		xpCtx := xpath.NewXPath(node.Doc())
		xpath := xpath.CompileXPath(args[0].(string))
		//xpath := ctx.XPath(args[0].(string))
		nodeSet := xpCtx.SearchByCompiledXPath(node, xpath).Slice()
		println("Node search for", args[0].(string), "returned this many results", len(nodeSet))
		defer xpCtx.Free()
		if len(nodeSet) == 0 {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		for index, node := range(nodeSet) {
			if (node != nil) && node.IsLinked() {
				ns := &Scope{Value: node, Index: index}
				ctx.runChildren(ns, ins)
			}
		}
	case "position.Text":
		returnValue = Positions[args[0].(string)]

	// SHARED NODE FUNCTIONS
	case "remove":
		scope.Value.(xml.Node).Remove()
	case "inner", "inner_text", "text":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Element)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "value":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Attribute)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "name":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Name()}
		ctx.runChildren(ts, ins)
		node.SetName(ts.Value.(string))
		returnValue = ts.Value.(string)
	case "dup":
		node := scope.Value.(xml.Node)
		newNode := node.Duplicate()
		_, isElement := node.(*xml.Element)
		if isElement {
			MoveFunc(newNode, node, AFTER)
		}
		ns := &Scope{Value:newNode}
		ctx.runChildren(ns, ins)
	case "fetch.Text":
		searchNode := scope.Value.(xml.Node)
		xPathObj := xpath.NewXPath(searchNode.Doc())
		nodeSet := xPathObj.Search(searchNode, args[0].(string))
		if nodeSet.Size() > 0 {
			node := nodeSet.First()
			attr, ok := node.(*xml.Attribute)
			if ok {
				returnValue = attr.Content()
			} else {
				returnValue = node.String()
			}
		}
		xPathObj.Free()

	// LIBXML FUNCTIONS
	case "insert_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		tagName := args[1].(string)
		element := node.Doc().NewElement(tagName)
		MoveFunc(element, node, position)
		ns := &Scope{Value: element}
		ctx.runChildren(ns, ins)
	case "inject_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		nodeSet := node.Doc().ParseHtmlFragment(args[1].(string))
		for _, newNode := range(nodeSet) {
			MoveFunc(newNode, node, position)
		}
		if len(nodeSet) > 0 {
			element, ok := nodeSet[0].(*xml.Element)
			if ok {
				// successfully ran scope
				returnValue = "true"
				ns := &Scope{Value: element}
				ctx.runChildren(ns, ins)
			}
		} else {
			returnValue = "false"
		}
	case "cdata.Text":
		elem, ok := scope.Value.(*xml.Element)
		if ok {
			elem.SetCDataContent(args[0].(string))
		}
	case "move.XMLNode.XMLNode.Position", "move.Node.Node.Position":
		//for name, value := range(ctx.LocalVar) {
		//	println(name, ":", value)
		//}
		MoveFunc(args[0].(xml.Node), args[1].(xml.Node), args[2].(Position))
	case "wrap_text_children.Text":
		returnValue = "false"
		child := scope.Value.(xml.Node).First()
		index := 0
		tagName := args[0].(string)
		for child != nil {
			text, ok := child.(*xml.Text)
			childNext := child.Next()
			if ok {
				returnValue = "true"
				wrap := text.Wrap(tagName)
				ns := &Scope{wrap, index}
				ctx.runChildren(ns, ins)
				index++
			}
			child = childNext
		}

	// ATTRIBUTE FUNCTIONS
	case "attribute.Text":
		node := scope.Value.(xml.Node)
		name := args[0].(string)
		_, ok := node.(*xml.Element)
		if ok == true {
			attr, _ := node.Attribute(name)

			as := &Scope{Value:attr}
			ctx.runChildren(as, ins)
			if attr.IsLinked() && (attr.Content() == "") {
				attr.Remove()
			}
			if !attr.IsLinked() {
				attr.Free()
			}
			returnValue = "true"
		}
	case "to_text.XMLNode":
		returnValue = scope.Value.(xml.Node).String()
	default:
		ctx.Log.Error("Must implement " + fun.Name)
	}
	return
}<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"regexp"
	"net/http"
)

// A comma, as defined in https://tools.ietf.org/html/rfc7230#section-7, with
// OWS defined in https://tools.ietf.org/html/rfc7230#appendix-B. This is
// commonly used as a separator in header field value definitions.
var Comma *regexp.Regexp = regexp.MustCompile(`[ \t]*,[ \t]*`)

// Conditional request headers that ServeHTTP may receive and need to be sent with fetchURL.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Conditional_requests#Conditional_headers
var ConditionalRequestHeaders = map[string]bool{
	"If-Match":            true,
	"If-None-Match":       true,
	"If-Modified-Since":   true,
	"If-Unmodified-Since": true,
	"If-Range":            true,
}

// The following hop-by-hop headers should be removed even when not specified
// in Connection, for backwards compatibility with downstream servers that were
// written against RFC 2616, and expect gateways to behave according to
// https://tools.ietf.org/html/rfc2616#section-13.5.1. (Note: "Trailers" is a
// typo there; should be "Trailer".)
//
// Connection header should also be removed per
// https://tools.ietf.org/html/rfc7230#section-6.1.
//
// Proxy-Connection should also be deleted, per
// https://github.com/WICG/webpackage/pull/339.
var legacyHeaders = map[string]bool{
	"Connection": true,
	"Keep-Alive": true,
	"Proxy-Authenticate": true,
	"Proxy-Authorization": true,
	"Proxy-Connection": true,
	"TE": true,
	"Trailer": true,
	"Transfer-Encoding": true,
	"Upgrade": true,
}

// Via is implicitly forwarded and disallowed to be included in
// config.ForwardedRequestHeaders
var notForwardedRequestHeader = map[string]bool{
	"Via": true,
}

// Remove hop-by-hop headers, per https://tools.ietf.org/html/rfc7230#section-6.1.
func RemoveHopByHopHeaders(h http.Header) {
	if connections, ok := h[http.CanonicalHeaderKey("Connection")]; ok {
		for _, connection := range connections {
			headerNames := Comma.Split(connection, -1)
			for _, headerName := range headerNames {
				h.Del(headerName)
			}
		}
	}

	for headerName, _ := range legacyHeaders {
		h.Del(headerName)
	}
}

func haveInvalidForwardedRequestHeader(h string) string {
	if _, ok := legacyHeaders[http.CanonicalHeaderKey(h)]; ok {
		return fmt.Sprintf("have hop-by-hop header of %s", h)
	}
	if _, ok := ConditionalRequestHeaders[http.CanonicalHeaderKey(h)]; ok {
		return fmt.Sprintf("have conditional request header of %s", h)
	}
	if _, ok := notForwardedRequestHeader[http.CanonicalHeaderKey(h)]; ok {
		return fmt.Sprintf("include request header of %s", h)
	}
	return ""
}
<commit_msg>fix entries of hop-by-hop header (#311)<commit_after>package util

import (
	"fmt"
	"regexp"
	"net/http"
)

// A comma, as defined in https://tools.ietf.org/html/rfc7230#section-7, with
// OWS defined in https://tools.ietf.org/html/rfc7230#appendix-B. This is
// commonly used as a separator in header field value definitions.
var Comma *regexp.Regexp = regexp.MustCompile(`[ \t]*,[ \t]*`)

// Conditional request headers that ServeHTTP may receive and need to be sent with fetchURL.
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Conditional_requests#Conditional_headers
var ConditionalRequestHeaders = map[string]bool{
	"If-Match":            true,
	"If-None-Match":       true,
	"If-Modified-Since":   true,
	"If-Unmodified-Since": true,
	"If-Range":            true,
}

// The following hop-by-hop headers should be removed even when not specified
// in Connection, for backwards compatibility with downstream servers that were
// written against RFC 2616, and expect gateways to behave according to
// https://tools.ietf.org/html/rfc2616#section-13.5.1. (Note: "Trailers" is a
// typo there; should be "Trailer".)
//
// Connection header should also be removed per
// https://tools.ietf.org/html/rfc7230#section-6.1.
//
// Proxy-Connection should also be deleted, per
// https://github.com/WICG/webpackage/pull/339.
var legacyHeaders = map[string]bool{
	"Connection": true,
	"Keep-Alive": true,
	"Proxy-Authenticate": true,
	"Proxy-Connection": true,
	"Trailer": true,
	"Transfer-Encoding": true,
	"Upgrade": true,
}

// Via is implicitly forwarded and disallowed to be included in
// config.ForwardedRequestHeaders
var notForwardedRequestHeader = map[string]bool{
	"Via": true,
}

// Remove hop-by-hop headers, per https://tools.ietf.org/html/rfc7230#section-6.1.
func RemoveHopByHopHeaders(h http.Header) {
	if connections, ok := h[http.CanonicalHeaderKey("Connection")]; ok {
		for _, connection := range connections {
			headerNames := Comma.Split(connection, -1)
			for _, headerName := range headerNames {
				h.Del(headerName)
			}
		}
	}

	for headerName, _ := range legacyHeaders {
		h.Del(headerName)
	}
}

func haveInvalidForwardedRequestHeader(h string) string {
	if _, ok := legacyHeaders[http.CanonicalHeaderKey(h)]; ok {
		return fmt.Sprintf("have hop-by-hop header of %s", h)
	}
	if _, ok := ConditionalRequestHeaders[http.CanonicalHeaderKey(h)]; ok {
		return fmt.Sprintf("have conditional request header of %s", h)
	}
	if _, ok := notForwardedRequestHeader[http.CanonicalHeaderKey(h)]; ok {
		return fmt.Sprintf("include request header of %s", h)
	}
	return ""
}
<|endoftext|>


<commit_before>// +build integration

package integration

import (
	"reflect"
	"testing"

	"github.com/openshift/origin/pkg/dockerregistry"
)

func TestRegistryClientConnect(t *testing.T) {
	c := dockerregistry.NewClient()
	conn, err := c.Connect("docker.io", false)
	if err != nil {
		t.Fatal(err)
	}
	for _, s := range []string{"index.docker.io", "https://docker.io", "https://index.docker.io"} {
		otherConn, err := c.Connect(s, false)
		if err != nil {
			t.Errorf("%s: can't connect: %v", s, err)
			continue
		}
		if !reflect.DeepEqual(otherConn, conn) {
			t.Errorf("%s: did not reuse connection: %#v %#v", s, conn, otherConn)
		}
	}

	otherConn, err := c.Connect("index.docker.io:443", false)
	if err != nil || reflect.DeepEqual(otherConn, conn) {
		t.Errorf("should not have reused index.docker.io:443: %v", err)
	}

	if _, err := c.Connect("http://ba%3/", false); err == nil {
		t.Error("Unexpected non-error")
	}
}

func TestRegistryClientConnectPulpRegistry(t *testing.T) {
	c := dockerregistry.NewClient()
	conn, err := c.Connect("registry.access.redhat.com", false)
	if err != nil {
		t.Fatal(err)
	}
	image, err := conn.ImageByTag("library", "rhel", "latest")
	if err != nil {
		t.Fatalf("unable to retrieve image info: %v", err)
	}
	if len(image.ID) == 0 {
		t.Fatalf("image had no ID: %#v", image)
	}
}

func TestRegistryClientV2DockerHub(t *testing.T) {
	c := dockerregistry.NewClient()
	conn, err := c.Connect("index.docker.io", false)
	if err != nil {
		t.Fatal(err)
	}
	image, err := conn.ImageByTag("kubernetes", "guestbook", "latest")
	// The V2 docker hub registry seems to have a bug for this repo, should eventually get fixed
	if !dockerregistry.IsTagNotFound(err) {
		t.Fatalf("unexpected error: %v", err)
	}

	// a v1 only path
	conn, err = c.Connect("registry.hub.docker.com", false)
	if err != nil {
		t.Fatal(err)
	}
	image, err = conn.ImageByTag("kubernetes", "guestbook", "latest")
	if err != nil {
		t.Fatalf("unable to retrieve image info: %v", err)
	}
	if len(image.ID) == 0 {
		t.Fatalf("image had no ID: %#v", image)
	}
}

func TestRegistryClientRegistryNotFound(t *testing.T) {
	conn, err := dockerregistry.NewClient().Connect("localhost:65000", false)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := conn.ImageByID("foo", "bar", "baz"); !dockerregistry.IsRegistryNotFound(err) {
		t.Error(err)
	}
}

func TestRegistryClientImage(t *testing.T) {
	for _, v2 := range []bool{true, false} {
		host := "index.docker.io"
		if !v2 {
			host = "registry.hub.docker.com"
		}
		conn, err := dockerregistry.NewClient().Connect(host, false)
		if err != nil {
			t.Fatal(err)
		}

		if _, err := conn.ImageByTag("openshift", "origin-not-found", "latest"); !dockerregistry.IsRepositoryNotFound(err) && !dockerregistry.IsTagNotFound(err) {
			t.Errorf("V2=%t: unexpected error: %v", v2, err)
		}

		image, err := conn.ImageByTag("openshift", "origin", "latest")
		if err != nil {
			t.Fatalf("V2=%t: %v", v2, err)
		}
		if len(image.ContainerConfig.Entrypoint) == 0 {
			t.Errorf("V2=%t: unexpected image: %#v", v2, image)
		}
		if v2 && !image.PullByID {
			t.Errorf("V2=%t: should be able to pull by ID %s", v2, image.ID)
		}

		other, err := conn.ImageByID("openshift", "origin", image.ID)
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(other.ContainerConfig.Entrypoint, image.ContainerConfig.Entrypoint) {
			t.Errorf("V2=%t: unexpected image: %#v", v2, other)
		}
	}
}

func TestRegistryClientQuayIOImage(t *testing.T) {
	conn, err := dockerregistry.NewClient().Connect("quay.io", false)
	if err != nil {
		t.Fatal(err)
	}

	_, err = conn.ImageByTag("coreos", "etcd", "latest")
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
}
<commit_msg>Disable quay.io test<commit_after>// +build integration

package integration

import (
	"reflect"
	"testing"

	"github.com/openshift/origin/pkg/dockerregistry"
)

func TestRegistryClientConnect(t *testing.T) {
	c := dockerregistry.NewClient()
	conn, err := c.Connect("docker.io", false)
	if err != nil {
		t.Fatal(err)
	}
	for _, s := range []string{"index.docker.io", "https://docker.io", "https://index.docker.io"} {
		otherConn, err := c.Connect(s, false)
		if err != nil {
			t.Errorf("%s: can't connect: %v", s, err)
			continue
		}
		if !reflect.DeepEqual(otherConn, conn) {
			t.Errorf("%s: did not reuse connection: %#v %#v", s, conn, otherConn)
		}
	}

	otherConn, err := c.Connect("index.docker.io:443", false)
	if err != nil || reflect.DeepEqual(otherConn, conn) {
		t.Errorf("should not have reused index.docker.io:443: %v", err)
	}

	if _, err := c.Connect("http://ba%3/", false); err == nil {
		t.Error("Unexpected non-error")
	}
}

func TestRegistryClientConnectPulpRegistry(t *testing.T) {
	c := dockerregistry.NewClient()
	conn, err := c.Connect("registry.access.redhat.com", false)
	if err != nil {
		t.Fatal(err)
	}
	image, err := conn.ImageByTag("library", "rhel", "latest")
	if err != nil {
		t.Fatalf("unable to retrieve image info: %v", err)
	}
	if len(image.ID) == 0 {
		t.Fatalf("image had no ID: %#v", image)
	}
}

func TestRegistryClientV2DockerHub(t *testing.T) {
	c := dockerregistry.NewClient()
	conn, err := c.Connect("index.docker.io", false)
	if err != nil {
		t.Fatal(err)
	}
	image, err := conn.ImageByTag("kubernetes", "guestbook", "latest")
	// The V2 docker hub registry seems to have a bug for this repo, should eventually get fixed
	if !dockerregistry.IsTagNotFound(err) {
		t.Fatalf("unexpected error: %v", err)
	}

	// a v1 only path
	conn, err = c.Connect("registry.hub.docker.com", false)
	if err != nil {
		t.Fatal(err)
	}
	image, err = conn.ImageByTag("kubernetes", "guestbook", "latest")
	if err != nil {
		t.Fatalf("unable to retrieve image info: %v", err)
	}
	if len(image.ID) == 0 {
		t.Fatalf("image had no ID: %#v", image)
	}
}

func TestRegistryClientRegistryNotFound(t *testing.T) {
	conn, err := dockerregistry.NewClient().Connect("localhost:65000", false)
	if err != nil {
		t.Fatal(err)
	}
	if _, err := conn.ImageByID("foo", "bar", "baz"); !dockerregistry.IsRegistryNotFound(err) {
		t.Error(err)
	}
}

func TestRegistryClientImage(t *testing.T) {
	for _, v2 := range []bool{true, false} {
		host := "index.docker.io"
		if !v2 {
			host = "registry.hub.docker.com"
		}
		conn, err := dockerregistry.NewClient().Connect(host, false)
		if err != nil {
			t.Fatal(err)
		}

		if _, err := conn.ImageByTag("openshift", "origin-not-found", "latest"); !dockerregistry.IsRepositoryNotFound(err) && !dockerregistry.IsTagNotFound(err) {
			t.Errorf("V2=%t: unexpected error: %v", v2, err)
		}

		image, err := conn.ImageByTag("openshift", "origin", "latest")
		if err != nil {
			t.Fatalf("V2=%t: %v", v2, err)
		}
		if len(image.ContainerConfig.Entrypoint) == 0 {
			t.Errorf("V2=%t: unexpected image: %#v", v2, image)
		}
		if v2 && !image.PullByID {
			t.Errorf("V2=%t: should be able to pull by ID %s", v2, image.ID)
		}

		other, err := conn.ImageByID("openshift", "origin", image.ID)
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(other.ContainerConfig.Entrypoint, image.ContainerConfig.Entrypoint) {
			t.Errorf("V2=%t: unexpected image: %#v", v2, other)
		}
	}
}

func TestRegistryClientQuayIOImage(t *testing.T) {
	conn, err := dockerregistry.NewClient().Connect("quay.io", false)
	if err != nil {
		t.Fatal(err)
	}

	_, err = conn.ImageByTag("coreos", "etcd", "latest")
	if err != nil {
		t.Skip("SKIPPING: unexpected error from quay.io: %v", err)
		//t.Errorf("unexpected error: %v", err)
	}
}
<|endoftext|>


<commit_before>package vcsclient

import (
	"bytes"
	"io"
	"net/http"
	"net/url"

	"sourcegraph.com/sourcegraph/vcsstore/git"
)

type gitTransport struct {
	client   *Client
	cloneURL *url.URL
}

var _ git.GitTransport = &gitTransport{}

func (t *gitTransport) InfoRefs(w io.Writer, service string) error {
	rp := &repository{client: t.client, vcsType: "git", cloneURL: t.cloneURL}
	urlQuery := struct {
		Service string `url:"service"`
	}{
		Service: service,
	}
	u, err := rp.url(git.RouteGitInfoRefs, nil, urlQuery)
	if err != nil {
		return err
	}
	u = t.client.BaseURL.ResolveReference(u)

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return err
	}
	var out bytes.Buffer
	_, err = t.client.Do(req, &out)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, &out)
	if err != nil {
		return err
	}
	return nil
}

func (t *gitTransport) ReceivePack(w io.Writer, rdr io.Reader, opt git.GitTransportOpt) error {
	rp := &repository{client: t.client, vcsType: "git", cloneURL: t.cloneURL}
	u, err := rp.url(git.RouteGitReceivePack, nil, nil)
	if err != nil {
		return err
	}
	u = t.client.BaseURL.ResolveReference(u)

	req, err := http.NewRequest("POST", u.String(), rdr)
	if err != nil {
		return err
	}
	req.Header.Set("content-encoding", opt.ContentEncoding)

	var out bytes.Buffer
	_, err = t.client.Do(req, &out)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, &out)

	return nil
}

func (t *gitTransport) UploadPack(w io.Writer, rdr io.Reader, opt git.GitTransportOpt) error {
	rp := &repository{client: t.client, vcsType: "git", cloneURL: t.cloneURL}
	u, err := rp.url(git.RouteGitUploadPack, nil, nil)
	if err != nil {
		return err
	}
	u = t.client.BaseURL.ResolveReference(u)

	req, err := http.NewRequest("POST", u.String(), rdr)
	if err != nil {
		return err
	}
	req.Header.Set("content-encoding", opt.ContentEncoding)

	var out bytes.Buffer
	_, err = t.client.Do(req, &out)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, &out)

	return nil
}
<commit_msg>fix git client<commit_after>package vcsclient

import (
	"bytes"
	"io"
	"net/http"
	"net/url"

	"sourcegraph.com/sourcegraph/vcsstore/git"
)

type gitTransport struct {
	client   *Client
	cloneURL *url.URL
}

var _ git.GitTransport = &gitTransport{}

func (t *gitTransport) InfoRefs(w io.Writer, service string) error {
	rp := &repository{client: t.client, vcsType: "git", cloneURL: t.cloneURL}
	urlQuery := struct {
		Service string `url:"service"`
	}{
		Service: "git-" + service,
	}
	u, err := rp.url(git.RouteGitInfoRefs, nil, urlQuery)
	if err != nil {
		return err
	}
	u = t.client.BaseURL.ResolveReference(u)

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "git/1.9.1") // TODO: kludge
	var out bytes.Buffer
	_, err = t.client.Do(req, &out)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, &out)
	if err != nil {
		return err
	}
	return nil
}

func (t *gitTransport) ReceivePack(w io.Writer, rdr io.Reader, opt git.GitTransportOpt) error {
	rp := &repository{client: t.client, vcsType: "git", cloneURL: t.cloneURL}
	u, err := rp.url(git.RouteGitReceivePack, nil, nil)
	if err != nil {
		return err
	}
	u = t.client.BaseURL.ResolveReference(u)

	req, err := http.NewRequest("POST", u.String(), rdr)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "git/1.9.1") // TODO: kludge
	req.Header.Set("content-encoding", opt.ContentEncoding)

	var out bytes.Buffer
	_, err = t.client.Do(req, &out)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, &out)

	return nil
}

func (t *gitTransport) UploadPack(w io.Writer, rdr io.Reader, opt git.GitTransportOpt) error {
	rp := &repository{client: t.client, vcsType: "git", cloneURL: t.cloneURL}
	u, err := rp.url(git.RouteGitUploadPack, nil, nil)
	if err != nil {
		return err
	}
	u = t.client.BaseURL.ResolveReference(u)

	req, err := http.NewRequest("POST", u.String(), rdr)
	if err != nil {
		return err
	}
	req.Header.Set("User-Agent", "git/1.9.1") // TODO: kludge
	req.Header.Set("content-encoding", opt.ContentEncoding)

	var out bytes.Buffer
	_, err = t.client.Do(req, &out)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, &out)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package handlers

import (
	"net/http"

	"github.com/cihub/seelog"
)

// LoggingHandler is used to log all requests for an endpoint.
type LoggingHandler struct{ h http.Handler }

// NewLoggingHandler creates a new LoggingHandler object.
func NewLoggingHandler(handler http.Handler) LoggingHandler {
	return LoggingHandler{h: handler}
}

// ServeHTTP logs the method and remote address of the request.
func (lh LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	seelog.Info("Handling http request", "method", r.Method, "from", r.RemoteAddr)
	lh.h.ServeHTTP(w, r)
}
<commit_msg>Change log line handling http request to DEBUG<commit_after>// Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package handlers

import (
	"net/http"

	"github.com/cihub/seelog"
)

// LoggingHandler is used to log all requests for an endpoint.
type LoggingHandler struct{ h http.Handler }

// NewLoggingHandler creates a new LoggingHandler object.
func NewLoggingHandler(handler http.Handler) LoggingHandler {
	return LoggingHandler{h: handler}
}

// ServeHTTP logs the method and remote address of the request.
func (lh LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	seelog.Debug("Handling http request", "method", r.Method, "from", r.RemoteAddr)
	lh.h.ServeHTTP(w, r)
}
<|endoftext|>


<commit_before>// +build datasources

package sources

import (
	"log"
	"strings"
	"testing"
	"time"

	"github.com/OWASP/Amass/amass/core"
)

func TestThreaTCrowd(t *testing.T) {
	config := &core.Config{}
	config.AddDomain("google.com")
	buf := new(strings.Builder)
	config.Log = log.New(buf, "", log.Lmicroseconds)

	out := make(chan *core.Request)
	bus := core.NewEventBus()
	bus.Subscribe(core.NewNameTopic, func(req *core.Request) {
		out <- req
	})
	defer bus.Stop()

	srv := NewThreatCrowd(config, bus)
	srv.Start()
	defer srv.Stop()

	count := 0
	expected := 10
	done := time.After(time.Second * 30)

loop:
	for {
		select {
		case <-out:
			count++
			if count == expected {
				return
			}
		case <-done:
			break loop
		}
	}

	if count < expected {
		t.Errorf("Found %d names, expected at least %d instead", count, expected)
	}
}
<commit_msg>fix threadcrowd_test name<commit_after>// +build datasources

package sources

import (
	"log"
	"strings"
	"testing"
	"time"

	"github.com/OWASP/Amass/amass/core"
)

func TestThreatCrowd(t *testing.T) {
	config := &core.Config{}
	config.AddDomain("google.com")
	buf := new(strings.Builder)
	config.Log = log.New(buf, "", log.Lmicroseconds)

	out := make(chan *core.Request)
	bus := core.NewEventBus()
	bus.Subscribe(core.NewNameTopic, func(req *core.Request) {
		out <- req
	})
	defer bus.Stop()

	srv := NewThreatCrowd(config, bus)
	srv.Start()
	defer srv.Stop()

	count := 0
	expected := 10
	done := time.After(time.Second * 30)

loop:
	for {
		select {
		case <-out:
			count++
			if count == expected {
				return
			}
		case <-done:
			break loop
		}
	}

	if count < expected {
		t.Errorf("Found %d names, expected at least %d instead", count, expected)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"
)

var prefixes = "0123456789abcdef"

type CacheEntry struct {
	HTTPCode int
	Expires  time.Time
}

type diskCacheEntry struct {
	Data []byte
	CacheEntry
}

type DiskCache struct {
	cacheRoot  string
	cacheFiles map[string]CacheEntry
	sync.RWMutex
}

func (d *DiskCache) init() {
	d.Lock()
	defer d.Unlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to load uninitialized cache.")
	}

	os.Mkdir(d.cacheRoot, 0770)

	for _, dir := range prefixes {
		dirName := d.cacheRoot + "/" + string(dir)
		err := os.Mkdir(dirName, 0770)
		if err != nil {
			//Either cannot create directory, or directory already exists, let's try opening it to find out.
			dirf, derr := os.Open(dirName)
			if derr != nil {
				// Couldn't open directory, panic.
				log.Fatalf("Couldn't create or open %s: %s/%s", dirName, err, derr)
			}

			files, err := dirf.Readdirnames(0)
			if err != nil {
				log.Fatalf("Couldn't read %s: %s", dirName, err)
			}

			var de diskCacheEntry
			for _, filename := range files {
				fullname := dirName + "/" + filename

				jsondata, err := ioutil.ReadFile(fullname)
				if err != nil {
					log.Fatalf("Failed to read %s: %s", fullname, err)
				}

				err = json.Unmarshal(jsondata, &de)
				if err != nil {
					log.Printf("Recovering from cache consistency error for %s: %s ", fullname, err)
				}

				if err != nil || time.Now().After(de.Expires) {
					err := os.Remove(fullname)
					if err != nil {
						log.Fatalf("Failed to remove expired cache entry %s: %s", fullname, err)
					}
					continue
				}

				d.cacheFiles[filename] = de.CacheEntry
			}
		}
	}
}

var cleanOnce = &sync.Once{}

func (d *DiskCache) clean() {
	log.Printf("Cleaning Up.")
	now := time.Now()

	cleancount := 0
	for tag, ce := range d.cacheFiles {
		if now.After(ce.Expires) {
			os.Remove(d.filename(tag))
			delete(d.cacheFiles, tag)

			cleancount++
		}
	}
	log.Printf("Cleaned up %d entries.", cleancount)
	cleanOnce = &sync.Once{}
}

var storeCount int64

func (d *DiskCache) filename(tag string) string {
	return d.cacheRoot + "/" + string(tag[0]) + "/" + tag
}

func (d *DiskCache) Store(cacheTag string, HTTPCode int, data []byte, Expires time.Time) error {
	d.Lock()
	defer d.Unlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to store to uninitialized cache.")
	}

	storeCount++
	if storeCount%500 == 0 {
		go cleanOnce.Do(func() { d.clean() })
	}

	ce := CacheEntry{HTTPCode, Expires}

	de := diskCacheEntry{data, ce}
	jsondata, err := json.Marshal(&de)
	if err != nil {
		log.Printf("Unknown JSON Marshal Error: %s", err)
		return err
	}

	err = ioutil.WriteFile(d.filename(cacheTag), jsondata, 0660)
	if err != nil {
		log.Printf("Unknown File Error: %s", err)
		return err
	}

	log.Printf("Stored Cache for %s Expires: %s", cacheTag, ce.Expires)
	d.cacheFiles[cacheTag] = ce
	return nil
}

func (d *DiskCache) Get(cacheTag string) (int, []byte, time.Time, error) {
	d.RLock()
	defer d.RUnlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to get from uninitialized cache.")
	}

	ce, exists := d.cacheFiles[cacheTag]
	if !exists || time.Now().After(ce.Expires) {
		return 0, nil, ce.Expires, fmt.Errorf("Not cached.")
	}

	jsondata, err := ioutil.ReadFile(d.filename(cacheTag))
	if err != nil {
		delete(d.cacheFiles, cacheTag)
		return 0, nil, ce.Expires, fmt.Errorf("Cache error - File not found.")
	}

	var de diskCacheEntry
	err = json.Unmarshal(jsondata, &de)
	if err != nil || de.Expires != ce.Expires {
		log.Printf("Cache consistency error: %s (Got: %s Expected: %s)", err, de.Expires, ce.Expires)

		delete(d.cacheFiles, cacheTag)
		return 0, nil, ce.Expires, fmt.Errorf("Cache error - Cache invalid.")
	}

	return ce.HTTPCode, de.Data, ce.Expires, nil
}

func (d *DiskCache) LogStats() {
	d.RLock()
	defer d.RUnlock()

	entries := 0
	expired := 0

	now := time.Now()
	for _, ce := range d.cacheFiles {
		entries++
		if now.After(ce.Expires) {
			expired++
		}
	}

	log.Printf("Cache Entries: %d  Expired Entries: %d", entries, expired)
}

func NewDiskCache(rootDir string) *DiskCache {
	var dc DiskCache

	dc.cacheRoot = rootDir
	dc.cacheFiles = make(map[string]CacheEntry)

	dc.init()

	return &dc
}
<commit_msg>removed debug logging<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sync"
	"time"
)

var prefixes = "0123456789abcdef"

type CacheEntry struct {
	HTTPCode int
	Expires  time.Time
}

type diskCacheEntry struct {
	Data []byte
	CacheEntry
}

type DiskCache struct {
	cacheRoot  string
	cacheFiles map[string]CacheEntry
	sync.RWMutex
}

func (d *DiskCache) init() {
	d.Lock()
	defer d.Unlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to load uninitialized cache.")
	}

	os.Mkdir(d.cacheRoot, 0770)

	for _, dir := range prefixes {
		dirName := d.cacheRoot + "/" + string(dir)
		err := os.Mkdir(dirName, 0770)
		if err != nil {
			//Either cannot create directory, or directory already exists, let's try opening it to find out.
			dirf, derr := os.Open(dirName)
			if derr != nil {
				// Couldn't open directory, panic.
				log.Fatalf("Couldn't create or open %s: %s/%s", dirName, err, derr)
			}

			files, err := dirf.Readdirnames(0)
			if err != nil {
				log.Fatalf("Couldn't read %s: %s", dirName, err)
			}

			var de diskCacheEntry
			for _, filename := range files {
				fullname := dirName + "/" + filename

				jsondata, err := ioutil.ReadFile(fullname)
				if err != nil {
					log.Fatalf("Failed to read %s: %s", fullname, err)
				}

				err = json.Unmarshal(jsondata, &de)
				if err != nil {
					log.Printf("Recovering from cache consistency error for %s: %s ", fullname, err)
				}

				if err != nil || time.Now().After(de.Expires) {
					err := os.Remove(fullname)
					if err != nil {
						log.Fatalf("Failed to remove expired cache entry %s: %s", fullname, err)
					}
					continue
				}

				d.cacheFiles[filename] = de.CacheEntry
			}
		}
	}
}

var cleanOnce = &sync.Once{}

func (d *DiskCache) clean() {
	log.Printf("Cleaning Up.")
	now := time.Now()

	cleancount := 0
	for tag, ce := range d.cacheFiles {
		if now.After(ce.Expires) {
			os.Remove(d.filename(tag))
			delete(d.cacheFiles, tag)

			cleancount++
		}
	}
	log.Printf("Cleaned up %d entries.", cleancount)
	cleanOnce = &sync.Once{}
}

var storeCount int64

func (d *DiskCache) filename(tag string) string {
	return d.cacheRoot + "/" + string(tag[0]) + "/" + tag
}

func (d *DiskCache) Store(cacheTag string, HTTPCode int, data []byte, Expires time.Time) error {
	d.Lock()
	defer d.Unlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to store to uninitialized cache.")
	}

	storeCount++
	if storeCount%500 == 0 {
		go cleanOnce.Do(func() { d.clean() })
	}

	ce := CacheEntry{HTTPCode, Expires}

	de := diskCacheEntry{data, ce}
	jsondata, err := json.Marshal(&de)
	if err != nil {
		log.Printf("Unknown JSON Marshal Error: %s", err)
		return err
	}

	err = ioutil.WriteFile(d.filename(cacheTag), jsondata, 0660)
	if err != nil {
		log.Printf("Unknown File Error: %s", err)
		return err
	}

	d.cacheFiles[cacheTag] = ce
	return nil
}

func (d *DiskCache) Get(cacheTag string) (int, []byte, time.Time, error) {
	d.RLock()
	defer d.RUnlock()

	if d.cacheFiles == nil {
		log.Fatalf("Tried to get from uninitialized cache.")
	}

	ce, exists := d.cacheFiles[cacheTag]
	if !exists || time.Now().After(ce.Expires) {
		return 0, nil, ce.Expires, fmt.Errorf("Not cached.")
	}

	jsondata, err := ioutil.ReadFile(d.filename(cacheTag))
	if err != nil {
		delete(d.cacheFiles, cacheTag)
		return 0, nil, ce.Expires, fmt.Errorf("Cache error - File not found.")
	}

	var de diskCacheEntry
	err = json.Unmarshal(jsondata, &de)
	if err != nil || de.Expires != ce.Expires {
		log.Printf("Cache consistency error: %s (Got: %s Expected: %s)", err, de.Expires, ce.Expires)

		delete(d.cacheFiles, cacheTag)
		return 0, nil, ce.Expires, fmt.Errorf("Cache error - Cache invalid.")
	}

	return ce.HTTPCode, de.Data, ce.Expires, nil
}

func (d *DiskCache) LogStats() {
	d.RLock()
	defer d.RUnlock()

	entries := 0
	expired := 0

	now := time.Now()
	for _, ce := range d.cacheFiles {
		entries++
		if now.After(ce.Expires) {
			expired++
		}
	}

	log.Printf("Cache Entries: %d  Expired Entries: %d", entries, expired)
}

func NewDiskCache(rootDir string) *DiskCache {
	var dc DiskCache

	dc.cacheRoot = rootDir
	dc.cacheFiles = make(map[string]CacheEntry)

	dc.init()

	return &dc
}
<|endoftext|>


<commit_before>// Package graphite implements sending metrics to Graphite via the Carbon text API
//
// This can be done by sending metrics one at a time or by buffering them through a channel.
package graphite

import (
	"fmt"
	"log"
	"net"
	"time"
)

// Graphite is the struct by which we interface with the library. At minimum you must set
// Host and Port fields.
type Graphite struct {
	Host    string
	Port    uint16
	Timeout time.Duration
	conn    net.Conn
}

// Metric is contains fields used for sending metrics to Graphite. If Timestamp is not set,
// one is generated when sent.
type Metric struct {
	Name      string
	Value     string
	Timestamp int64
}

// defaultTimeout is the default connection timeout used by DialTimeout.
const (
	defaultTimeout = 30
)

var doneSending bool

// connect is the unexported function used for connecting
func connect(host string, port uint16, timeout time.Duration) (net.Conn, error) {
	connectAddr := fmt.Sprintf("%s:%d", host, port)

	return net.DialTimeout("tcp", connectAddr, timeout)
}

// sendMetric is the unexported function to send a single metric to Graphite.
func sendMetric(conn net.Conn, metric Metric) {
	log.Printf("sending %s", metric.Name)
	fmt.Fprintf(conn, "%s %s %s", metric.Name, metric.Value, metric.Timestamp)
}

// chanSendMetrics sends a Metric slice to the given channel
func chanSendMetrics(ch chan Metric, buffer []Metric) {
	for _, item := range buffer {
		if len(item.Name) > 0 {
			log.Printf("buffering %s", item.Name)
			ch <- item
		}
	}
}

// chanRecvMetrics reads `bufsz` numbered metrics off of the given channel and
// sends them to Graphite.
func chanRecvMetrics(ch chan Metric, conn net.Conn, bufsz int) {
	for i := 0; i < bufsz; i++ {
		item := <-ch
		sendMetric(conn, item)
	}

	doneSending = true
}

// Connect wraps the unexported connect function.
func (g *Graphite) Connect() {
	var (
		err     error
		timeout time.Duration
	)

	if g.Timeout == 0 {
		g.Timeout = defaultTimeout
	}

	timeout = g.Timeout * time.Second

	g.conn, err = connect(g.Host, g.Port, timeout)
	for err != nil {
		log.Printf(err.Error())
		log.Printf(fmt.Sprintf("error connecting, retrying in %d seconds", int(timeout.Seconds())))
		time.Sleep(timeout)

		timeout = (g.Timeout + 5) * time.Second
		g.conn, err = connect(g.Host, g.Port, timeout)
	}
}

// SendMetric is used to send a single metric to Graphite.
// Sets metric.Timestamp to current Unix time if necessary.
func (g *Graphite) SendMetric(metric Metric) {
	if metric.Timestamp == 0 {
		metric.Timestamp = time.Now().Unix()
	}

	sendMetric(g.conn, metric)
}

// Sendall is used to buffered metrics to Graphite via a channel and go routines.
func (g *Graphite) Sendall(buf []Metric) {
	doneSending = false

	ch := make(chan Metric, len(buf))
	go chanSendMetrics(ch, buf)
	go chanRecvMetrics(ch, g.conn, len(buf))

	for doneSending == false {
		time.Sleep(1 * time.Second)
	}
}
<commit_msg>create an explicit graphite interface<commit_after>// Package graphite implements sending metrics to Graphite via the Carbon text API
//
// This can be done by sending metrics one at a time or by buffering them through a channel.
package graphite

import (
	"fmt"
	"log"
	"net"
	"time"
)

// Interface type for Graphite.
type Graphite interface {
	Connect()
	SendMetric()
	Sendall()
	chanRecv()
	chanSend()
	sendMetric()
}

// Graphite is how we interface with the library. At minimum you must set
// Host and Port fields.
type GraphiteServer struct {
	Host    string
	Port    uint16
	Timeout time.Duration
	conn    net.Conn
}

// Metric is contains fields used for sending metrics to Graphite. If Timestamp is not set,
// one is generated when sent.
type Metric struct {
	Name      string
	Value     string
	Timestamp int64
}

// defaultTimeout is the default connection timeout used by DialTimeout.
const (
	defaultTimeout = 30
)

var doneSending = false

// connect is the unexported function used for connecting
func connect(host string, port uint16, timeout time.Duration) (net.Conn, error) {
	connectAddr := fmt.Sprintf("%s:%d", host, port)

	return net.DialTimeout("tcp", connectAddr, timeout)
}

// Connect wraps the unexported connect function.
// Sets a connection timeout if unset.
func (g *GraphiteServer) Connect() {
	var (
		err     error
		timeout time.Duration
	)

	if g.Timeout == 0 {
		g.Timeout = defaultTimeout
	}

	timeout = g.Timeout * time.Second

	g.conn, err = connect(g.Host, g.Port, timeout)
	for err != nil {
		log.Printf(err.Error())
		log.Printf(fmt.Sprintf("error connecting, retrying in %d seconds", int(timeout.Seconds())))
		time.Sleep(timeout)

		timeout = (g.Timeout + 5) * time.Second
		g.conn, err = connect(g.Host, g.Port, timeout)
	}
}

// SendMetric is used to send a single metric to Graphite.
// Sets metric.Timestamp to current Unix time if necessary.
func (g *GraphiteServer) SendMetric(metric Metric) {
	if metric.Timestamp == 0 {
		metric.Timestamp = time.Now().Unix()
	}

	g.sendMetric(metric)
}

// Sendall is used to buffered metrics to Graphite via a channel and go routines.
func (g *GraphiteServer) Sendall(buf []Metric) {
	doneSending = false

	ch := make(chan Metric, len(buf))
	go g.chanSend(ch, buf)
	go g.chanRecv(ch, len(buf))

	for doneSending == false {
		time.Sleep(1 * time.Second)
	}
}

// chanRecvMetrics reads `bufsz` numbered metrics off of the given channel and
// sends them to Graphite.
func (g *GraphiteServer) chanRecv(ch chan Metric, bufsz int) {
	for i := 0; i < bufsz; i++ {
		item := <-ch
		g.sendMetric(item)
	}

	doneSending = true
}

// chanSendMetrics sends a Metric slice to the given channel
func (g *GraphiteServer) chanSend(ch chan Metric, buffer []Metric) {
	for _, item := range buffer {
		if len(item.Name) > 0 {
			log.Printf("buffering %s", item.Name)
			ch <- item
		}
	}
}

// sendMetric is the unexported function to send a single metric to Graphite.
func (g *GraphiteServer) sendMetric(metric Metric) {
	log.Printf("sending %s", metric.Name)
	fmt.Fprintf(g.conn, "%s %s %d\n", metric.Name, metric.Value, metric.Timestamp)
}
<|endoftext|>


<commit_before>/*
MEL app backend.

Author:		Alastair Hughes
Contact:	<hobbitalastair at yandex dot com>
*/

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"

	"database/sql"
	_ "github.com/mattn/go-sqlite3"
)

// internalError ends the request and logs an internal error.
func internalError(fail func(int), err error) {
	fail(http.StatusInternalServerError)
	log.Printf("%q\n", err)
}

// Authenticate the given HTTP request.
func authenticate(fail func(int), request *http.Request, db *sql.DB) (string, bool) {
	user, password, ok := request.BasicAuth()
	if !ok {
		fail(http.StatusUnauthorized)
		return user, ok
	}

	// dbname and dbpassword are empty values to pass to Scan; we never use them
	// elsewhere.
	dbname := ""
	// FIXME: This is not "best-practice".
	//	We should salt the password (using a locally stored value), and maybe
	//	use encrypt(name+password) to avoid duplicated passwords being obvious?
	err := db.QueryRow("SELECT name FROM users WHERE name=? and password=?", user, password).Scan(&dbname)
	if err == sql.ErrNoRows {
		fail(http.StatusForbidden)
	} else if err != nil {
		internalError(fail, err)
	}

	return user, err == nil
}

// ListProjects responds with the list of projects for the given user.
func ListProjects(fail func(int), encoder *json.Encoder, user string, db *sql.DB) {
	// TODO: This should also return projects which this user owns.
	//	Implement that as a view in the database?
	rows, err := db.Query("SELECT id FROM views WHERE name=?", user)
	if err != nil {
		internalError(fail, err)
		return
	}
	defer rows.Close()

	for rows.Next() {
		id := -1
		err = rows.Scan(&id)
		if err != nil {
			internalError(fail, err)
			return
		}
		err = encoder.Encode(id)
		if err != nil {
			internalError(fail, err)
			return
		}
	}
	if err != nil {
		internalError(fail, err)
		return
	}
}

// Flag responds with the current state of the flag.
func Flag(fail func(int), encoder *json.Encoder, pid int, user string, db *sql.DB) {
	// TODO: We need to authenticate the user here.
	flag := false
	err := db.QueryRow("SELECT flag FROM project WHERE id=?", pid).Scan(&flag)
	if err != nil {
		internalError(fail, err)
		return
	}
	encoder.Encode(flag)
}

// Project responds with the details of the given project.
func Project(fail func(int), encoder *json.Encoder, pid int, user string, db *sql.DB) {
	// TODO: We need to authenticate the user here.
	name, percentage, description := "", "", ""
	err := db.QueryRow("SELECT name, percentage, description FROM project WHERE id=?", pid).Scan(&name, &percentage, &description)
	if err != nil {
		internalError(fail, err)
		return
	}
	encoder.Encode(name)
	encoder.Encode(percentage)
	encoder.Encode(description)
}

// Handle a single HTTP request.
func handle(writer http.ResponseWriter, request *http.Request) {
	// Wrapper for failing functions.
	fail := func(status int) { http.Error(writer, http.StatusText(status), status) }

	// Open the database.
	// FIXME: I'm using sqlite3 here which only seems to report errors when
	//	actually executing a query; I'll need to test this on other systems as well.
	db, err := sql.Open("sqlite3", "test.db") // TODO: Should be the actual db, ...
	if err != nil {
		log.Printf("Error opening DB: %q\n", err)
		fail(http.StatusInternalServerError)
	}

	// Authenticate.
	user, ok := authenticate(fail, request, db)
	if !ok {
		return
	}

	// Parse the URL and return the corresponding value.
	// TODO: This assumes GET requests...
	enc := json.NewEncoder(writer)
	enc.SetEscapeHTML(true)
	paths := strings.Split(strings.TrimPrefix(request.URL.Path, "/"), "/")

	// FIXME: Match using regular expressions instead?
	if len(paths) < 1 || paths[0] != "projects" {
		http.NotFound(writer, request)
	} else if len(paths) == 1 {
		ListProjects(fail, enc, user, db)
	} else {
		// Grab the project ID from the URL.
		pid, err := strconv.Atoi(paths[1])
		if err != nil {
			http.NotFound(writer, request)
			return
		}

		if len(paths) == 2 {
			Project(fail, enc, pid, user, db)
		} else if len(paths) == 3 && paths[2] == "flag" {
			Flag(fail, enc, pid, user, db)
		} else {
			http.NotFound(writer, request)
		}
	}
}

func main() {
	fmt.Printf("Starting server on :8080...\n")
	http.ListenAndServe(":8080", http.HandlerFunc(handle))
}

// vim: sw=4 ts=4 noexpandtab
<commit_msg>Initial implementation with salted passwords<commit_after>/*
MEL app backend.

Author:		Alastair Hughes
Contact:	<hobbitalastair at yandex dot com>
*/

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"strings"

	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"golang.org/x/crypto/scrypt"
)

// internalError ends the request and logs an internal error.
func internalError(fail func(int), err error) {
	fail(http.StatusInternalServerError)
	log.Printf("%q\n", err)
}

// Authenticate the given HTTP request.
func authenticate(fail func(int), request *http.Request, db *sql.DB) (string, bool) {
	user, password, ok := request.BasicAuth()
	if !ok {
		fail(http.StatusUnauthorized)
		return user, false
	}

	// Retrieve the salt and database password.
	salt := []byte("")
	dbpassword := []byte("")
	err := db.QueryRow("SELECT salt, password FROM users WHERE name=?", user).Scan(&salt, &dbpassword)
	if err == sql.ErrNoRows {
		fail(http.StatusForbidden)
		return user, false
	} else if err != nil {
		internalError(fail, err)
		return user, false
	}

	// Check the password. We salt and encrypt it to avoid potential security
	// issues if the db is stolen.
	// This appears to be reasonably close to "best practice", but the 1<<20
	// value probably should be checked for sanity.
	// FIXME: We don't store the 1<<20 value in the db, but it should be
	// increased as compute power grows. Doing so is complicated since some way
	// of migrating users from the old value would also need to be implemented.
	key, err := scrypt.Key([]byte(password), salt, 1<<20, 8, 1, 256)
	if err != nil {
		internalError(fail, err)
		return user, false
	}
	return user, string(key) == string(dbpassword)
}

// ListProjects responds with the list of projects for the given user.
func ListProjects(fail func(int), encoder *json.Encoder, user string, db *sql.DB) {
	// TODO: This should also return projects which this user owns.
	//	Implement that as a view in the database?
	rows, err := db.Query("SELECT id FROM views WHERE name=?", user)
	if err != nil {
		internalError(fail, err)
		return
	}
	defer rows.Close()

	for rows.Next() {
		id := -1
		err = rows.Scan(&id)
		if err != nil {
			internalError(fail, err)
			return
		}
		err = encoder.Encode(id)
		if err != nil {
			internalError(fail, err)
			return
		}
	}
	if err != nil {
		internalError(fail, err)
		return
	}
}

// Flag responds with the current state of the flag.
func Flag(fail func(int), encoder *json.Encoder, pid int, user string, db *sql.DB) {
	// TODO: We need to authenticate the user here.
	flag := false
	err := db.QueryRow("SELECT flag FROM project WHERE id=?", pid).Scan(&flag)
	if err != nil {
		internalError(fail, err)
		return
	}
	encoder.Encode(flag)
}

// Project responds with the details of the given project.
func Project(fail func(int), encoder *json.Encoder, pid int, user string, db *sql.DB) {
	// TODO: We need to authenticate the user here.
	name, percentage, description := "", "", ""
	err := db.QueryRow("SELECT name, percentage, description FROM project WHERE id=?", pid).Scan(&name, &percentage, &description)
	if err != nil {
		internalError(fail, err)
		return
	}
	encoder.Encode(name)
	encoder.Encode(percentage)
	encoder.Encode(description)
}

// Handle a single HTTP request.
func handle(writer http.ResponseWriter, request *http.Request) {
	// Wrapper for failing functions.
	fail := func(status int) { http.Error(writer, http.StatusText(status), status) }

	// Open the database.
	// FIXME: I'm using sqlite3 here which only seems to report errors when
	//	actually executing a query; I'll need to test this on other systems as well.
	db, err := sql.Open("sqlite3", "test.db") // TODO: Should be the actual db, ...
	if err != nil {
		log.Printf("Error opening DB: %q\n", err)
		fail(http.StatusInternalServerError)
	}

	// Authenticate.
	user, ok := authenticate(fail, request, db)
	if !ok {
		return
	}

	// Parse the URL and return the corresponding value.
	// TODO: This assumes GET requests...
	enc := json.NewEncoder(writer)
	enc.SetEscapeHTML(true)
	paths := strings.Split(strings.TrimPrefix(request.URL.Path, "/"), "/")

	// FIXME: Match using regular expressions instead?
	if len(paths) < 1 || paths[0] != "projects" {
		http.NotFound(writer, request)
	} else if len(paths) == 1 {
		ListProjects(fail, enc, user, db)
	} else {
		// Grab the project ID from the URL.
		pid, err := strconv.Atoi(paths[1])
		if err != nil {
			http.NotFound(writer, request)
			return
		}

		if len(paths) == 2 {
			Project(fail, enc, pid, user, db)
		} else if len(paths) == 3 && paths[2] == "flag" {
			Flag(fail, enc, pid, user, db)
		} else {
			http.NotFound(writer, request)
		}
	}
}

func main() {
	fmt.Printf("Starting server on :8080...\n")
	http.ListenAndServe(":8080", http.HandlerFunc(handle))
}

// vim: sw=4 ts=4 noexpandtab
<|endoftext|>


<commit_before>//
//
//
package main

import (
    "fmt"
    "flag"
    "gopkg.in/yaml.v2"
    "io/ioutil"
    "path/filepath"
    "os/exec"
    "os"
    "strings"
    "unsafe"
)

//
// data structures
//
type Target struct {
    Name string
    Type string
}
type StringList struct {
    Type string
    Target string
    Debug []string `yaml:",flow"`
    Release []string `yaml:",flow"`
    List []string `yaml:",flow"`
}
type Variable struct {
    Name string
    Value string
    Type string
    Build string
}
type Build struct {
    Name string
    Command string
    Files []string `yaml:",flow"`
}

type Data struct {
    Target []Target `yaml:",flow"`
    Include []StringList `yaml:",flow"`
    Variable []Variable `yaml:",flow"`
    Define []StringList `yaml:",flow"`
    Option []StringList `yaml:",flow"`
    Archive_Option []StringList `yaml:",flow"`
    Convert_Option []StringList `yaml:",flow"`
    Prebuild []Build `yaml:",flow"`
    Postbuild []Build `yaml:",flow"`
    Source []StringList `yaml:",flow"`
    Convert_List []StringList `yaml:",flow"`
    Subdir []StringList `yaml:",flow"`
}

//
// error
//

type MyError struct {
    str string
}
func (m MyError) Error() string {
    return m.str
}

//
// build information
//

//
type BuildCommand struct {
    cmd string
    args string
    title string
}

//
type BuildResult struct {
    success bool
    create_list []string
}

//
type BuildInfo struct {
    variables map[string] string
    includes string
    defines string
    options string
    archive_options string
    convert_options string
    select_target string
    target string
    subdir []string
    create_list []string
}

//
// global variables
//
var (
    isDebug bool
    isRelease bool
    target_type string
    target_name string
    outputdir string

    need_dir_list []string
    command_list []BuildCommand
)

//
//
// build functions
//
//

//
//
//
func getList(block []StringList,target_name string) []string {
    lists := [] string{}
    for _,i := range block {
        if (i.Type == "" || i.Type == target_type) && (i.Target == "" || i.Target == target_name) {
            for _,l := range i.List {
                lists = append(lists,l)
            }
            if isDebug == true {
                for _,d := range i.Debug {
                    lists = append(lists,d)
                }
            } else {
                for _,r := range i.Release {
                    lists = append(lists,r)
                }
            }
        }
    }
    return lists
}

//
// archive objects
//
func create_archive(info BuildInfo,odir string,create_list []string,target_name string) string {
    arname := odir
    if target_type == "WIN32" {
        arname += target_name + ".lib"
    } else {
        arname += "lib" + target_name + ".a"
    }
    arname = filepath.ToSlash(filepath.Clean(arname))

    archiver := info.variables["archiver"]
    alist := ""
    for _,l := range create_list {
        alist += " " + l
    }

    t := fmt.Sprintf("Library: %s",arname)
    cmd := BuildCommand{
        cmd : archiver,
        args : info.archive_options+arname+alist,
        title : t }
    command_list = append(command_list,cmd)

    return arname
}

//
// link objects
//
func create_link(info BuildInfo,odir string,create_list []string,target_name string) {
    trname := odir
    if target_type == "WIN32" {
        trname += target_name + ".exe"
    } else {
        trname += target_name
    }
    trname = filepath.ToSlash(filepath.Clean(trname))

    linker := info.variables["linker"]

    create_list = append(info.create_list,create_list...)

    flist := ""
    for _,l := range create_list {
        flist += " " + l
    }

    t := fmt.Sprintf("Linking: %s",trname)
    cmd := BuildCommand{
        cmd : linker,
        args : "-o " + trname + flist,
        title : t }
    command_list = append(command_list,cmd)
    //fmt.Println("-o " + NowTarget.Name + flist)
}

//
// convert objects
//
func create_convert(info BuildInfo,loaddir string,odir string,create_list []string,target_name string) {
    cvname := odir + target_name
    cvname = filepath.ToSlash(filepath.Clean(cvname))
    clist := ""
    converter := info.variables["converter"]

    for _,f := range create_list {
        clist += " " + filepath.ToSlash(filepath.Clean(loaddir+f))
    }

    t := fmt.Sprintf("Convert: %s",cvname)
    cmd := BuildCommand{
        cmd : converter,
        args : info.convert_options+"-o "+cvname+clist,
        title : t }
    command_list = append(command_list,cmd)
}

//
// build main
//
func build(info BuildInfo,pathname string) (result BuildResult,err error) {
    loaddir := pathname
    if loaddir == "" {
        loaddir = "./"
    } else {
        loaddir += "/"
    }
    my_yaml := loaddir+"make.yml"
    buf, err := ioutil.ReadFile(my_yaml)
    if err != nil {
        e := MyError{ str : my_yaml + ": " + err.Error() }
        result.success = false
        return result,e
    }

    var d Data
    err = yaml.Unmarshal(buf, &d)
    if err != nil {
        e := MyError { str : my_yaml + ": " + err.Error() }
        result.success = false
        return result,e
    }

    //
    // select target
    //
    var NowTarget Target
    for _,t := range d.Target {
        if info.select_target == "" || t.Name == info.select_target {
            NowTarget = t
            if info.target == "" {
                info.target = t.Name
            }
            break
        }
    }
    if NowTarget.Name == "" {
        e := MyError{ str : "No Target" }
        result.success = false
        return result,e
    }
    info.select_target = ""

    opt_pre := info.variables["option_prefix"]
    //
    // get rules
    //
    for _,v := range d.Variable {
        if v.Type == "" || v.Type == target_type {
            info.variables[v.Name] = v.Value
        }
    }
    for _,i := range getList(d.Include,info.target) {
        abs, err := filepath.Abs(i)
        if err != nil {
            result.success = false
            return result,err
        }
        info.includes += " " + opt_pre + "I" + abs
    }
    for _,d := range getList(d.Define,info.target) {
        info.defines += " " + opt_pre + "D" + d
    }
    for _,o := range getList(d.Option,info.target) {
        info.options += " " + opt_pre + o
    }
    for _,a := range getList(d.Archive_Option,info.target) {
        info.archive_options += " " + opt_pre + a + " "
    }
    for _,c := range getList(d.Convert_Option,info.target) {
        info.convert_options +=  c + " "
    }

    files := getList(d.Source,info.target)
    cvfiles := getList(d.Convert_List,info.target)

    subdirs := getList(d.Subdir,info.target)
    for _,s := range subdirs {
        sd := loaddir+s
        var r,e = build(info,sd)
        if r.success == false {
            return r,e
        }
        info.create_list = append(info.create_list,r.create_list...)
    }

    compiler := info.variables["compiler"]

    arg1 := info.includes + info.defines + info.options
    odir := outputdir + "/" + loaddir
    need_dir_list = append(need_dir_list,filepath.Clean(odir))

    create_list := []string{}
    for _,f := range files {
        sname := filepath.ToSlash(filepath.Clean(loaddir+f))
        oname := filepath.ToSlash(filepath.Clean(odir+f+".o"))
        create_list = append(create_list,oname)

        t := fmt.Sprintf("Compile: %s",sname)
        cmd := BuildCommand{
            cmd : compiler,
            args : arg1+" -o "+oname+" "+sname,
            title : t }
        command_list = append(command_list,cmd)
    }

    if NowTarget.Type == "library" {
        // archive
        if len(create_list) > 0 {
            arname := create_archive(info,odir,create_list,NowTarget.Name)
            result.create_list = append(info.create_list,arname)
            //fmt.Println(info.archive_options+arname+alist)
        } else {
            fmt.Println("There are no files to build.")
        }
    } else if NowTarget.Type == "execute" {
        // link program
        if len(create_list) > 0 && len(info.create_list) > 0 {
            create_link(info,odir,create_list,NowTarget.Name)
        } else {
            fmt.Println("There are no files to build.")
        }
    } else if NowTarget.Type == "convert" {
        if len(cvfiles) > 0 {
            create_convert(info,loaddir,odir,cvfiles,NowTarget.Name)
        } else {
            fmt.Println("There are no files to convert.")
        }
    } else {
        //
        // othre...
        //
        result.create_list = append(info.create_list,create_list...)
    }
    result.success = true
    return result,nil
}

func main() {

    flag.BoolVar(&isRelease,"release",false,"release build")
    flag.BoolVar(&isDebug,"debug",true,"debug build")
    flag.StringVar(&target_type,"type","default","build target type")
    flag.StringVar(&target_name,"t","","build target name")
    flag.StringVar(&outputdir,"o","build","build directory")
    flag.Parse()

    outputdir += "/" + target_type + "/"
    if isRelease {
        isDebug = false
        outputdir += "Release"
    } else {
        outputdir += "Debug"
    }

    build_info := BuildInfo{
        variables : map[string] string{"option_prefix":"-"},
        includes : "",
        defines : "",
        select_target : target_name,
        target: target_name }
    var r,err = build(build_info,"")
    if r.success == false {
        fmt.Println("Error:",err.Error())
        os.Exit(1)
    }

    // setup directories
    for _,nd := range need_dir_list {
        os.MkdirAll(nd,os.ModePerm)
    }

    // execute build
    nlen := len(command_list)
    if nlen > 0 {
        for i,bs := range command_list {
            t := fmt.Sprintf("[%d/%d] %s",i+1,nlen,bs.title)
            fmt.Println(t)
            fmt.Println(bs.cmd + ":"+ bs.args)
            arg_list := strings.Split(bs.args," ")
            c,_ := exec.Command(bs.cmd,arg_list[0:]...).CombinedOutput()
            msg := *(*string)(unsafe.Pointer(&c))
            if msg != "" {
                fmt.Println(msg)
            }
        }
    }
}
//
//
<commit_msg>minimal update.<commit_after>//
//
//
package main

import (
    "fmt"
    "flag"
    "gopkg.in/yaml.v2"
    "io/ioutil"
    "path/filepath"
    "os/exec"
    "os"
    "strings"
    "unsafe"
)

//
// data structures
//
type Target struct {
    Name string
    Type string
}
type StringList struct {
    Type string
    Target string
    Debug []string `yaml:",flow"`
    Release []string `yaml:",flow"`
    List []string `yaml:",flow"`
}
type Variable struct {
    Name string
    Value string
    Type string
    Build string
}
type Build struct {
    Name string
    Command string
    Files []string `yaml:",flow"`
}

type Data struct {
    Target []Target `yaml:",flow"`
    Include []StringList `yaml:",flow"`
    Variable []Variable `yaml:",flow"`
    Define []StringList `yaml:",flow"`
    Option []StringList `yaml:",flow"`
    Archive_Option []StringList `yaml:",flow"`
    Convert_Option []StringList `yaml:",flow"`
    Prebuild []Build `yaml:",flow"`
    Postbuild []Build `yaml:",flow"`
    Source []StringList `yaml:",flow"`
    Convert_List []StringList `yaml:",flow"`
    Subdir []StringList `yaml:",flow"`
}

//
// error
//

type MyError struct {
    str string
}
func (m MyError) Error() string {
    return m.str
}

//
// build information
//

//
type BuildCommand struct {
    cmd string
    args string
    title string
}

//
type BuildResult struct {
    success bool
    create_list []string
}

//
type BuildInfo struct {
    variables map[string] string
    includes string
    defines string
    options string
    archive_options string
    convert_options string
    select_target string
    target string
    subdir []string
    create_list []string
}

//
// global variables
//
var (
    isDebug bool
    isRelease bool
    target_type string
    target_name string
    outputdir string

    need_dir_list []string
    command_list []BuildCommand
)

//
//
// build functions
//
//

//
//
//
func getList(block []StringList,target_name string) []string {
    lists := [] string{}
    for _,i := range block {
        if (i.Type == "" || i.Type == target_type) && (i.Target == "" || i.Target == target_name) {
            for _,l := range i.List {
                lists = append(lists,l)
            }
            if isDebug == true {
                for _,d := range i.Debug {
                    lists = append(lists,d)
                }
            } else {
                for _,r := range i.Release {
                    lists = append(lists,r)
                }
            }
        }
    }
    return lists
}

//
// archive objects
//
func create_archive(info BuildInfo,odir string,create_list []string,target_name string) string {
    arname := odir
    if target_type == "WIN32" {
        arname += target_name + ".lib"
    } else {
        arname += "lib" + target_name + ".a"
    }
    arname = filepath.ToSlash(filepath.Clean(arname))

    archiver := info.variables["archiver"]
    alist := ""
    for _,l := range create_list {
        alist += " " + l
    }

    t := fmt.Sprintf("Library: %s",arname)
    cmd := BuildCommand{
        cmd : archiver,
        args : info.archive_options+arname+alist,
        title : t }
    command_list = append(command_list,cmd)

    return arname
}

//
// link objects
//
func create_link(info BuildInfo,odir string,create_list []string,target_name string) {
    trname := odir
    if target_type == "WIN32" {
        trname += target_name + ".exe"
    } else {
        trname += target_name
    }
    trname = filepath.ToSlash(filepath.Clean(trname))

    linker := info.variables["linker"]

    create_list = append(info.create_list,create_list...)

    flist := ""
    for _,l := range create_list {
        flist += " " + l
    }

    t := fmt.Sprintf("Linking: %s",trname)
    cmd := BuildCommand{
        cmd : linker,
        args : "-o " + trname + flist,
        title : t }
    command_list = append(command_list,cmd)
    //fmt.Println("-o " + NowTarget.Name + flist)
}

//
// convert objects
//
func create_convert(info BuildInfo,loaddir string,odir string,create_list []string,target_name string) {
    cvname := odir + target_name
    cvname = filepath.ToSlash(filepath.Clean(cvname))
    clist := ""
    converter := info.variables["converter"]

    for _,f := range create_list {
        clist += " " + filepath.ToSlash(filepath.Clean(loaddir+f))
    }

    t := fmt.Sprintf("Convert: %s",cvname)
    cmd := BuildCommand{
        cmd : converter,
        args : info.convert_options+"-o "+cvname+clist,
        title : t }
    command_list = append(command_list,cmd)
}

//
// build main
//
func build(info BuildInfo,pathname string) (result BuildResult,err error) {
    loaddir := pathname
    if loaddir == "" {
        loaddir = "./"
    } else {
        loaddir += "/"
    }
    my_yaml := loaddir+"make.yml"
    buf, err := ioutil.ReadFile(my_yaml)
    if err != nil {
        e := MyError{ str : my_yaml + ": " + err.Error() }
        result.success = false
        return result,e
    }

    var d Data
    err = yaml.Unmarshal(buf, &d)
    if err != nil {
        e := MyError { str : my_yaml + ": " + err.Error() }
        result.success = false
        return result,e
    }

    //
    // select target
    //
    var NowTarget Target
    for _,t := range d.Target {
        if info.select_target == "" || t.Name == info.select_target {
            NowTarget = t
            if info.target == "" {
                info.target = t.Name
            }
            break
        }
    }
    if NowTarget.Name == "" {
        e := MyError{ str : "No Target" }
        result.success = false
        return result,e
    }
    info.select_target = ""

    opt_pre := info.variables["option_prefix"]
    //
    // get rules
    //
    for _,v := range d.Variable {
        if v.Type == "" || v.Type == target_type {
            info.variables[v.Name] = v.Value
        }
    }
    for _,i := range getList(d.Include,info.target) {
        abs, err := filepath.Abs(i)
        if err != nil {
            result.success = false
            return result,err
        }
        info.includes += opt_pre + "I" + abs + " "
    }
    for _,d := range getList(d.Define,info.target) {
        info.defines += opt_pre + "D" + d + " "
    }
    for _,o := range getList(d.Option,info.target) {
        info.options += opt_pre + o + " "
    }
    for _,a := range getList(d.Archive_Option,info.target) {
        info.archive_options += opt_pre + a + " "
    }
    for _,c := range getList(d.Convert_Option,info.target) {
        info.convert_options +=  c + " "
    }

    files := getList(d.Source,info.target)
    cvfiles := getList(d.Convert_List,info.target)

    // sub-directories
    subdirs := getList(d.Subdir,info.target)
    for _,s := range subdirs {
        sd := loaddir+s
        var r,e = build(info,sd)
        if r.success == false {
            return r,e
        }
        info.create_list = append(info.create_list,r.create_list...)
    }

    compiler := info.variables["compiler"]

    arg1 := info.includes + info.defines + info.options
    odir := outputdir + "/" + loaddir
    need_dir_list = append(need_dir_list,filepath.Clean(odir))

    create_list := []string{}
    for _,f := range files {
        sname := filepath.ToSlash(filepath.Clean(loaddir+f))
        oname := filepath.ToSlash(filepath.Clean(odir+f+".o"))
        create_list = append(create_list,oname)

        t := fmt.Sprintf("Compile: %s",sname)
        cmd := BuildCommand{
            cmd : compiler,
            args : arg1+"-o "+oname+" "+sname,
            title : t }
        command_list = append(command_list,cmd)
    }

    if NowTarget.Type == "library" {
        // archive
        if len(create_list) > 0 {
            arname := create_archive(info,odir,create_list,NowTarget.Name)
            result.create_list = append(info.create_list,arname)
            //fmt.Println(info.archive_options+arname+alist)
        } else {
            fmt.Println("There are no files to build.")
        }
    } else if NowTarget.Type == "execute" {
        // link program
        if len(create_list) > 0 && len(info.create_list) > 0 {
            create_link(info,odir,create_list,NowTarget.Name)
        } else {
            fmt.Println("There are no files to build.")
        }
    } else if NowTarget.Type == "convert" {
        if len(cvfiles) > 0 {
            create_convert(info,loaddir,odir,cvfiles,NowTarget.Name)
        } else {
            fmt.Println("There are no files to convert.")
        }
    } else {
        //
        // othre...
        //
        result.create_list = append(info.create_list,create_list...)
    }
    result.success = true
    return result,nil
}

func main() {

    flag.BoolVar(&isRelease,"release",false,"release build")
    flag.BoolVar(&isDebug,"debug",true,"debug build")
    flag.StringVar(&target_type,"type","default","build target type")
    flag.StringVar(&target_name,"t","","build target name")
    flag.StringVar(&outputdir,"o","build","build directory")
    flag.Parse()

    outputdir += "/" + target_type + "/"
    if isRelease {
        isDebug = false
        outputdir += "Release"
    } else {
        outputdir += "Debug"
    }

    build_info := BuildInfo{
        variables : map[string] string{"option_prefix":"-"},
        includes : "",
        defines : "",
        select_target : target_name,
        target: target_name }
    var r,err = build(build_info,"")
    if r.success == false {
        fmt.Println("Error:",err.Error())
        os.Exit(1)
    }

    // setup directories
    for _,nd := range need_dir_list {
        os.MkdirAll(nd,os.ModePerm)
    }

    // execute build
    nlen := len(command_list)
    if nlen > 0 {
        for i,bs := range command_list {
            t := fmt.Sprintf("[%d/%d] %s",i+1,nlen,bs.title)
            fmt.Println(t)
            //fmt.Println(bs.cmd + ":"+ bs.args)
            arg_list := strings.Split(bs.args," ")
            c,_ := exec.Command(bs.cmd,arg_list[0:]...).CombinedOutput()
            msg := *(*string)(unsafe.Pointer(&c))
            if msg != "" {
                fmt.Println(msg)
            }
        }
    }
}
//
//
<|endoftext|>