<commit_before>package shark

import (
	"strings"
	"os"
	"libxml"
	"fmt"
	log "log4go"
	xml "libxml/tree"
	tp "athena/proto"
	"libxml/xpath"
	"rubex"
	"css2xpath"
	"goconv"
)

var (
	InnerReplacer = rubex.MustCompile(`[\\$](\d)`)
)

func (ctx *Ctx) runBuiltIn(fun *Function, scope *Scope, ins *tp.Instruction, args []interface{}) (returnValue interface{}) {
	returnValue = ""
	switch fun.Name {
	case "this":
		returnValue = scope.Value
	case "yield":
		myYieldBlock := ctx.yieldBlock()
		ctx.Yields = ctx.Yields[:(len(ctx.Yields) - 1)]
		if ctx.yieldBlock() != nil {
			returnValue = ctx.runChildren(scope, myYieldBlock.Ins)
			if returnValue == nil {
				returnValue = "false"
			}
		} else {
			ctx.Log.Error("yield() failure")
		}
		ctx.Yields = append(ctx.Yields, myYieldBlock)

	case "var.Text", "var.Text.Text":
		val := ctx.Env[args[0].(string)]
		if len(args) > 1 {
			returnValue = args[1].(string)
			ctx.Env[args[0].(string)] = returnValue.(string)
		} else {
			returnValue = val
		}
		if len(ins.Children) > 0 {
			ts := &Scope{Value: returnValue}
			ctx.runChildren(ts, ins)
			returnValue = ts.Value
			ctx.Env[args[0].(string)] = returnValue.(string)
		}
	case "deprecated.Text":
		ctx.Log.Info(args[0].(string))
	case "match.Text":
		// Setup stacks
		against, ok := args[0].(string)
		if !ok {
			ctx.Log.Error("AH!")
		}
		ctx.MatchStack = append(ctx.MatchStack, against)
		ctx.MatchShouldContinue = append(ctx.MatchShouldContinue, true)

		// Run children
		ctx.runChildren(scope, ins)

		if ctx.matchShouldContinue() {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		// Clear
		ctx.MatchShouldContinue = ctx.MatchShouldContinue[:len(ctx.MatchShouldContinue)-1]
		ctx.MatchStack = ctx.MatchStack[:len(ctx.MatchStack)-1]
	case "with.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) == ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "with.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if (args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) != ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if !(args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "else":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
			ctx.runChildren(scope, ins)
			returnValue = "true"
		}
	case "regexp.Text.Text":
		mode := rubex.ONIG_OPTION_DEFAULT
		if strings.Index(args[1].(string), "i") >= 0 {
			mode = rubex.ONIG_OPTION_IGNORECASE
		}
		if strings.Index(args[1].(string), "m") >= 0 {
			mode = rubex.ONIG_OPTION_MULTILINE
		}
		var err os.Error
		returnValue, err = rubex.NewRegexp(args[0].(string), mode)
		if err != nil {
			ctx.Log.Error("Invalid regexp")
		}
	case "export.Text":
		val := make([]string, 2)
		val[0] = args[0].(string)
		ts := &Scope{Value: ""}
		ctx.runChildren(ts, ins)
		val[1] = ts.Value.(string)
		ctx.Exports = append(ctx.Exports, val)
	case "log.Text":
		ctx.Logs = append(ctx.Logs, args[0].(string))
		returnValue = args[0].(string)

	// ATOMIC FUNCTIONS
	case "concat.Text.Text":
		//println("Concat:", args[0].(string), "+", args[1].(string))
		returnValue = args[0].(string) + args[1].(string)
	case "concat.Text.Text.Text": //REMOVE
		returnValue = args[0].(string) + args[1].(string) + args[2].(string)
	case "downcase.Text":
		returnValue = strings.ToLower(args[0].(string))
		return
	case "upcase.Text":
		returnValue = strings.ToUpper(args[0].(string))
		return
	case "index.XMLNode", "index.Node":
		returnValue = fmt.Sprintf("%d", scope.Index+1)

	// TEXT FUNCTIONS
	case "set.Text":
		scope.Value = args[0]
	case "append.Text":
		scope.Value = scope.Value.(string) + args[0].(string)
	case "prepend.Text":
		scope.Value = args[0].(string) + scope.Value.(string)
	case "replace.Text":
		ts := &Scope{Value: ""}
		ctx.runChildren(ts, ins)
		scope.Value = strings.Replace(scope.Value.(string), args[0].(string), ts.Value.(string), -1)
	case "replace.Regexp":
		regexp := args[0].(*rubex.Regexp)
		scope.Value = regexp.GsubFunc(scope.Value.(string), func(match string, captures map[string]string) string {
			usesGlobal := (ctx.Env["use_global_replace_vars"] == "true")

			for name, capture := range captures {
				if usesGlobal {
					//println("setting $", name, "to", capture)
					ctx.Env[name] = capture
				}
				ctx.vars()[name] = capture
			}

			replacementScope := &Scope{Value: match}
			ctx.runChildren(replacementScope, ins)
			//println(ins.String())

			//println("Replacement:", replacementScope.Value.(string))
			return InnerReplacer.GsubFunc(replacementScope.Value.(string), func(_ string, numeric_captures map[string]string) string {
				capture := numeric_captures["1"]
				var val string
				if usesGlobal {
					val = ctx.Env[capture]
				} else {
					val = ctx.vars()[capture].(string)
				}
				return val
			})
		})
		returnValue = scope.Value
	case "convert_encoding.Text.Text":
		input := scope.Value.(string)
		fromCode  := args[0].(string)
		toCode    := args[1].(string)
		ic, err := goconv.OpenWithFallback(fromCode, toCode, goconv.KEEP_UNRECOGNIZED)
		if err == nil {
			outputBytes, _ := ic.Conv([]byte(input))
			scope.Value = string(outputBytes)
			ic.Close()
		} else {
			scope.Value = input
		}
		returnValue = scope.Value
	// XML FUNCTIONS
	case "xml":
		doc := libxml.XmlParseString(scope.Value.(string))
		ns := &Scope{Value: doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.String()
		returnValue = scope.Value
		doc.Free()
	case "html_doc.Text.Text":
		inputEncoding  := args[0].(string)
		outputEncoding    := args[1].(string)
		doc := xml.HtmlParseString(scope.Value.(string), inputEncoding)
		ns := &Scope{Value: doc}
		ctx.runChildren(ns, ins)
		if err := doc.SetMetaEncoding(outputEncoding); err != nil {
			//ctx.Log.Warn("executing html:" + err.String())
		}
		scope.Value = doc.DumpHTML()
		returnValue = scope.Value
		doc.Free()
	case "html_fragment.Text":
		inputEncoding  := args[0].(string)
		doc := xml.HtmlParseFragment(scope.Value.(string), inputEncoding)
		ns := &Scope{Value: doc.RootElement()}
		ctx.runChildren(ns, ins)
		//output is always utf-8 because the content is internal to Doc.
		scope.Value = ns.Value.(xml.Node).Content()
		returnValue = scope.Value
		doc.Free()
	case "select.Text":
		// TODO reuse XPath object
		node := scope.Value.(xml.Node)
		xpCtx := xpath.NewXPath(node.Doc())
		xpath := xpath.CompileXPath(args[0].(string))
		if xpath == nil {
			ctx.Logs = append(ctx.Logs, "Invalid XPath used: " + args[0].(string))
			returnValue = "0"
			return
		}
		nodeSet := xpCtx.SearchByCompiledXPath(node, xpath).Slice()
		defer xpCtx.Free()
		defer xpath.Free()

		if len(nodeSet) == 0 {
			returnValue = "0"
		} else {
			returnValue = fmt.Sprintf("%d", len(nodeSet))
		}

		for index, node := range nodeSet {
			if doc, ok := node.(*xml.Doc); ok {
				node = doc.RootElement()
			}
			if (node != nil) && node.IsLinked() {
				if _, ok := node.(*xml.Element); ok {
					ns := &Scope{Value: node, Index: index}
					ctx.runChildren(ns, ins)
				}
			}
		}
	case "css.Text":
		returnValue = css2xpath.Convert(args[0].(string), css2xpath.LOCAL)
	case "position.Text":
		returnValue = Positions[args[0].(string)]

	// SHARED NODE FUNCTIONS
	case "remove":
		node := scope.Value.(xml.Node)
		node.Remove()
		node.Free()
	case "inner", "inner_text", "text":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value: node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Element)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "value":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value: node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Attribute)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "name":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value: node.Name()}
		ctx.runChildren(ts, ins)
		node.SetName(ts.Value.(string))
		returnValue = ts.Value.(string)
	case "dup":
		node := scope.Value.(xml.Node)
		newNode := node.Duplicate()
		_, isElement := node.(*xml.Element)
		if isElement {
			MoveFunc(newNode, node, AFTER)
		}
		ns := &Scope{Value: newNode}
		ctx.runChildren(ns, ins)
	case "fetch.Text":
		searchNode := scope.Value.(xml.Node)
		xPathObj := xpath.NewXPath(searchNode.Doc())
		nodeSet := xPathObj.Search(searchNode, args[0].(string))
		if nodeSet.Size() > 0 {
			node := nodeSet.First()
			attr, ok := node.(*xml.Attribute)
			if ok {
				returnValue = attr.Content()
			} else {
				returnValue = node.String()
			}
		}
		xPathObj.Free()
	case "path":
		returnValue = scope.Value.(xml.Node).Path()

	// LIBXML FUNCTIONS
	case "insert_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		tagName := args[1].(string)
		element, err := node.Doc().NewElement(tagName)
		if err != nil {
			log.Error("Problem with insert_at(Pos, '" + tagName + "') - " + err.String())
			returnValue = "false"
		} else {
			MoveFunc(element, node, position)
			ns := &Scope{Value: element}
			ctx.runChildren(ns, ins)
			returnValue = "true"
		}
	case "inject_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		nodeSet := node.Doc().ParseHtmlFragment(args[1].(string), "")
		for _, newNode := range nodeSet {
			MoveFunc(newNode, node, position)
		}
		if len(nodeSet) > 0 {
			element, ok := nodeSet[0].(*xml.Element)
			if ok {
				// successfully ran scope
				returnValue = "true"
				ns := &Scope{Value: element}
				ctx.runChildren(ns, ins)
			}
		} else {
			returnValue = "false"
		}
	case "cdata.Text":
		elem, ok := scope.Value.(*xml.Element)
		if ok {
			elem.SetCDataContent(args[0].(string))
		}
	case "move.XMLNode.XMLNode.Position", "move.Node.Node.Position":
		//for name, value := range(ctx.LocalVar) {
		//	println(name, ":", value)
		//}
		MoveFunc(args[0].(xml.Node), args[1].(xml.Node), args[2].(Position))
	case "wrap_text_children.Text":
		returnValue = "false"
		child := scope.Value.(xml.Node).First()
		index := 0
		tagName := args[0].(string)
		for child != nil {
			text, ok := child.(*xml.Text)
			childNext := child.Next()
			if ok {
				returnValue = "true"
				wrap := text.Wrap(tagName)
				ns := &Scope{wrap, index}
				ctx.runChildren(ns, ins)
				index++
			}
			child = childNext
		}

	// ATTRIBUTE FUNCTIONS
	case "attribute.Text":
		node := scope.Value.(xml.Node)
		name := args[0].(string)
		if _, ok := node.(*xml.Element); ok {
			attr, _ := node.Attribute(name)
			as := &Scope{Value: attr}
			ctx.runChildren(as, ins)
			if attr.IsLinked() && (attr.Content() == "") {
				attr.Remove()
			}
			if !attr.IsLinked() {
				attr.Free()
			}
			returnValue = "true"
		}
	case "dump":
		returnValue = scope.Value.(xml.Node).String()

	default:
		ctx.Log.Error("Must implement " + fun.Name)
	}
	return
}
<commit_msg>nil protect for doc<commit_after>package shark

import (
	"strings"
	"os"
	"libxml"
	"fmt"
	log "log4go"
	xml "libxml/tree"
	tp "athena/proto"
	"libxml/xpath"
	"rubex"
	"css2xpath"
	"goconv"
)

var (
	InnerReplacer = rubex.MustCompile(`[\\$](\d)`)
)

func (ctx *Ctx) runBuiltIn(fun *Function, scope *Scope, ins *tp.Instruction, args []interface{}) (returnValue interface{}) {
	returnValue = ""
	switch fun.Name {
	case "this":
		returnValue = scope.Value
	case "yield":
		myYieldBlock := ctx.yieldBlock()
		ctx.Yields = ctx.Yields[:(len(ctx.Yields) - 1)]
		if ctx.yieldBlock() != nil {
			returnValue = ctx.runChildren(scope, myYieldBlock.Ins)
			if returnValue == nil {
				returnValue = "false"
			}
		} else {
			ctx.Log.Error("yield() failure")
		}
		ctx.Yields = append(ctx.Yields, myYieldBlock)

	case "var.Text", "var.Text.Text":
		val := ctx.Env[args[0].(string)]
		if len(args) > 1 {
			returnValue = args[1].(string)
			ctx.Env[args[0].(string)] = returnValue.(string)
		} else {
			returnValue = val
		}
		if len(ins.Children) > 0 {
			ts := &Scope{Value: returnValue}
			ctx.runChildren(ts, ins)
			returnValue = ts.Value
			ctx.Env[args[0].(string)] = returnValue.(string)
		}
	case "deprecated.Text":
		ctx.Log.Info(args[0].(string))
	case "match.Text":
		// Setup stacks
		against, ok := args[0].(string)
		if !ok {
			ctx.Log.Error("AH!")
		}
		ctx.MatchStack = append(ctx.MatchStack, against)
		ctx.MatchShouldContinue = append(ctx.MatchShouldContinue, true)

		// Run children
		ctx.runChildren(scope, ins)

		if ctx.matchShouldContinue() {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		// Clear
		ctx.MatchShouldContinue = ctx.MatchShouldContinue[:len(ctx.MatchShouldContinue)-1]
		ctx.MatchStack = ctx.MatchStack[:len(ctx.MatchStack)-1]
	case "with.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) == ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "with.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if (args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) != ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if !(args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "else":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
			ctx.runChildren(scope, ins)
			returnValue = "true"
		}
	case "regexp.Text.Text":
		mode := rubex.ONIG_OPTION_DEFAULT
		if strings.Index(args[1].(string), "i") >= 0 {
			mode = rubex.ONIG_OPTION_IGNORECASE
		}
		if strings.Index(args[1].(string), "m") >= 0 {
			mode = rubex.ONIG_OPTION_MULTILINE
		}
		var err os.Error
		returnValue, err = rubex.NewRegexp(args[0].(string), mode)
		if err != nil {
			ctx.Log.Error("Invalid regexp")
		}
	case "export.Text":
		val := make([]string, 2)
		val[0] = args[0].(string)
		ts := &Scope{Value: ""}
		ctx.runChildren(ts, ins)
		val[1] = ts.Value.(string)
		ctx.Exports = append(ctx.Exports, val)
	case "log.Text":
		ctx.Logs = append(ctx.Logs, args[0].(string))
		returnValue = args[0].(string)

	// ATOMIC FUNCTIONS
	case "concat.Text.Text":
		//println("Concat:", args[0].(string), "+", args[1].(string))
		returnValue = args[0].(string) + args[1].(string)
	case "concat.Text.Text.Text": //REMOVE
		returnValue = args[0].(string) + args[1].(string) + args[2].(string)
	case "downcase.Text":
		returnValue = strings.ToLower(args[0].(string))
		return
	case "upcase.Text":
		returnValue = strings.ToUpper(args[0].(string))
		return
	case "index.XMLNode", "index.Node":
		returnValue = fmt.Sprintf("%d", scope.Index+1)

	// TEXT FUNCTIONS
	case "set.Text":
		scope.Value = args[0]
	case "append.Text":
		scope.Value = scope.Value.(string) + args[0].(string)
	case "prepend.Text":
		scope.Value = args[0].(string) + scope.Value.(string)
	case "replace.Text":
		ts := &Scope{Value: ""}
		ctx.runChildren(ts, ins)
		scope.Value = strings.Replace(scope.Value.(string), args[0].(string), ts.Value.(string), -1)
	case "replace.Regexp":
		regexp := args[0].(*rubex.Regexp)
		scope.Value = regexp.GsubFunc(scope.Value.(string), func(match string, captures map[string]string) string {
			usesGlobal := (ctx.Env["use_global_replace_vars"] == "true")

			for name, capture := range captures {
				if usesGlobal {
					//println("setting $", name, "to", capture)
					ctx.Env[name] = capture
				}
				ctx.vars()[name] = capture
			}

			replacementScope := &Scope{Value: match}
			ctx.runChildren(replacementScope, ins)
			//println(ins.String())

			//println("Replacement:", replacementScope.Value.(string))
			return InnerReplacer.GsubFunc(replacementScope.Value.(string), func(_ string, numeric_captures map[string]string) string {
				capture := numeric_captures["1"]
				var val string
				if usesGlobal {
					val = ctx.Env[capture]
				} else {
					val = ctx.vars()[capture].(string)
				}
				return val
			})
		})
		returnValue = scope.Value
	case "convert_encoding.Text.Text":
		input := scope.Value.(string)
		fromCode  := args[0].(string)
		toCode    := args[1].(string)
		ic, err := goconv.OpenWithFallback(fromCode, toCode, goconv.KEEP_UNRECOGNIZED)
		if err == nil {
			outputBytes, _ := ic.Conv([]byte(input))
			scope.Value = string(outputBytes)
			ic.Close()
		} else {
			scope.Value = input
		}
		returnValue = scope.Value
	// XML FUNCTIONS
	case "xml":
		doc := libxml.XmlParseString(scope.Value.(string))
		ns := &Scope{Value: doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.String()
		returnValue = scope.Value
		doc.Free()
	case "html_doc.Text.Text":
		inputEncoding  := args[0].(string)
		outputEncoding    := args[1].(string)
		doc := xml.HtmlParseString(scope.Value.(string), inputEncoding)
		ns := &Scope{Value: doc}
		ctx.runChildren(ns, ins)
		if err := doc.SetMetaEncoding(outputEncoding); err != nil {
			//ctx.Log.Warn("executing html:" + err.String())
		}
		scope.Value = doc.DumpHTML()
		returnValue = scope.Value
		doc.Free()
	case "html_fragment.Text":
		inputEncoding  := args[0].(string)
		doc := xml.HtmlParseFragment(scope.Value.(string), inputEncoding)
		ns := &Scope{Value: doc.RootElement()}
		ctx.runChildren(ns, ins)
		//output is always utf-8 because the content is internal to Doc.
		scope.Value = ns.Value.(xml.Node).Content()
		returnValue = scope.Value
		doc.Free()
	case "select.Text":
		// TODO reuse XPath object
		node := scope.Value.(xml.Node)
		xpCtx := xpath.NewXPath(node.Doc())
		xpath := xpath.CompileXPath(args[0].(string))
		if xpath == nil {
			ctx.Logs = append(ctx.Logs, "Invalid XPath used: " + args[0].(string))
			returnValue = "0"
			return
		}
		nodeSet := xpCtx.SearchByCompiledXPath(node, xpath).Slice()
		defer xpCtx.Free()
		defer xpath.Free()

		if len(nodeSet) == 0 {
			returnValue = "0"
		} else {
			returnValue = fmt.Sprintf("%d", len(nodeSet))
		}

		for index, node := range nodeSet {
			if node != nil {
				if doc, ok := node.(*xml.Doc); ok {
					node = doc.RootElement()
				}
				if (node != nil) && node.IsLinked() {
					if _, ok := node.(*xml.Element); ok {
						ns := &Scope{Value: node, Index: index}
						ctx.runChildren(ns, ins)
					}
				}
			}
		}
	case "css.Text":
		returnValue = css2xpath.Convert(args[0].(string), css2xpath.LOCAL)
	case "position.Text":
		returnValue = Positions[args[0].(string)]

	// SHARED NODE FUNCTIONS
	case "remove":
		node := scope.Value.(xml.Node)
		node.Remove()
		node.Free()
	case "inner", "inner_text", "text":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value: node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Element)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "value":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value: node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Attribute)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "name":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value: node.Name()}
		ctx.runChildren(ts, ins)
		node.SetName(ts.Value.(string))
		returnValue = ts.Value.(string)
	case "dup":
		node := scope.Value.(xml.Node)
		newNode := node.Duplicate()
		_, isElement := node.(*xml.Element)
		if isElement {
			MoveFunc(newNode, node, AFTER)
		}
		ns := &Scope{Value: newNode}
		ctx.runChildren(ns, ins)
	case "fetch.Text":
		searchNode := scope.Value.(xml.Node)
		xPathObj := xpath.NewXPath(searchNode.Doc())
		nodeSet := xPathObj.Search(searchNode, args[0].(string))
		if nodeSet.Size() > 0 {
			node := nodeSet.First()
			attr, ok := node.(*xml.Attribute)
			if ok {
				returnValue = attr.Content()
			} else {
				returnValue = node.String()
			}
		}
		xPathObj.Free()
	case "path":
		returnValue = scope.Value.(xml.Node).Path()

	// LIBXML FUNCTIONS
	case "insert_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		tagName := args[1].(string)
		element, err := node.Doc().NewElement(tagName)
		if err != nil {
			log.Error("Problem with insert_at(Pos, '" + tagName + "') - " + err.String())
			returnValue = "false"
		} else {
			MoveFunc(element, node, position)
			ns := &Scope{Value: element}
			ctx.runChildren(ns, ins)
			returnValue = "true"
		}
	case "inject_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		nodeSet := node.Doc().ParseHtmlFragment(args[1].(string), "")
		for _, newNode := range nodeSet {
			MoveFunc(newNode, node, position)
		}
		if len(nodeSet) > 0 {
			element, ok := nodeSet[0].(*xml.Element)
			if ok {
				// successfully ran scope
				returnValue = "true"
				ns := &Scope{Value: element}
				ctx.runChildren(ns, ins)
			}
		} else {
			returnValue = "false"
		}
	case "cdata.Text":
		elem, ok := scope.Value.(*xml.Element)
		if ok {
			elem.SetCDataContent(args[0].(string))
		}
	case "move.XMLNode.XMLNode.Position", "move.Node.Node.Position":
		//for name, value := range(ctx.LocalVar) {
		//	println(name, ":", value)
		//}
		MoveFunc(args[0].(xml.Node), args[1].(xml.Node), args[2].(Position))
	case "wrap_text_children.Text":
		returnValue = "false"
		child := scope.Value.(xml.Node).First()
		index := 0
		tagName := args[0].(string)
		for child != nil {
			text, ok := child.(*xml.Text)
			childNext := child.Next()
			if ok {
				returnValue = "true"
				wrap := text.Wrap(tagName)
				ns := &Scope{wrap, index}
				ctx.runChildren(ns, ins)
				index++
			}
			child = childNext
		}

	// ATTRIBUTE FUNCTIONS
	case "attribute.Text":
		node := scope.Value.(xml.Node)
		name := args[0].(string)
		if _, ok := node.(*xml.Element); ok {
			attr, _ := node.Attribute(name)
			as := &Scope{Value: attr}
			ctx.runChildren(as, ins)
			if attr.IsLinked() && (attr.Content() == "") {
				attr.Remove()
			}
			if !attr.IsLinked() {
				attr.Free()
			}
			returnValue = "true"
		}
	case "dump":
		returnValue = scope.Value.(xml.Node).String()

	default:
		ctx.Log.Error("Must implement " + fun.Name)
	}
	return
}
<|endoftext|>


<commit_before>package api

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"strings"

	"github.com/vmware/harbor/src/chartserver"
	hlog "github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/ui/config"
)

const (
	namespaceParam          = ":repo"
	defaultRepo             = "library"
	rootUploadingEndpoint   = "/api/chartrepo/charts"
	rootIndexEndpoint       = "/chartrepo/index.yaml"
	chartRepoHealthEndpoint = "/api/chartrepo/health"

	accessLevelPublic = iota
	accessLevelRead
	accessLevelWrite
	accessLevelAll
	accessLevelSystem

	formFieldNameForChart = "chart"
	formFiledNameForProv  = "prov"
	headerContentType     = "Content-Type"
	contentTypeMultipart  = "multipart/form-data"
)

//chartController is a singleton instance
var chartController *chartserver.Controller

//ChartRepositoryAPI provides related API handlers for the chart repository APIs
type ChartRepositoryAPI struct {
	//The base controller to provide common utilities
	BaseController

	//Keep the namespace if existing
	namespace string
}

//Prepare something for the following actions
func (cra *ChartRepositoryAPI) Prepare() {
	//Call super prepare method
	cra.BaseController.Prepare()

	//Try to extract namespace for parameter of path
	//It may not exist
	cra.namespace = strings.TrimSpace(cra.GetStringFromPath(namespaceParam))

	//Check the existence of namespace
	//Exclude the following URI
	// -/index.yaml
	// -/api/chartserver/health
	incomingURI := cra.Ctx.Request.RequestURI
	if incomingURI == rootUploadingEndpoint {
		//Forward to the default repository
		cra.namespace = defaultRepo
	}

	if incomingURI != rootIndexEndpoint &&
		incomingURI != chartRepoHealthEndpoint {
		if !cra.requireNamespace(cra.namespace) {
			return
		}
	}

	//Rewrite URL path
	cra.rewriteURLPath(cra.Ctx.Request)
}

//GetHealthStatus handles GET /api/chartserver/health
func (cra *ChartRepositoryAPI) GetHealthStatus() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelSystem) {
		return
	}

	chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//GetIndexByRepo handles GET /:repo/index.yaml
func (cra *ChartRepositoryAPI) GetIndexByRepo() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	chartController.GetRepositoryHandler().GetIndexFileWithNS(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//GetIndex handles GET /index.yaml
func (cra *ChartRepositoryAPI) GetIndex() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelSystem) {
		return
	}

	chartController.GetRepositoryHandler().GetIndexFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//DownloadChart handles GET /:repo/charts/:filename
func (cra *ChartRepositoryAPI) DownloadChart() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	chartController.GetRepositoryHandler().DownloadChartObject(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//ListCharts handles GET /api/:repo/charts
func (cra *ChartRepositoryAPI) ListCharts() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	chartController.GetManipulationHandler().ListCharts(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//ListChartVersions GET /api/:repo/charts/:name
func (cra *ChartRepositoryAPI) ListChartVersions() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	chartController.GetManipulationHandler().GetChart(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//GetChartVersion handles GET /api/:repo/charts/:name/:version
func (cra *ChartRepositoryAPI) GetChartVersion() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	//Let's pass the namespace via the context of request
	req := cra.Ctx.Request
	*req = *(req.WithContext(context.WithValue(req.Context(), chartserver.NamespaceContextKey, cra.namespace)))

	chartController.GetManipulationHandler().GetChartVersion(cra.Ctx.ResponseWriter, req)
}

//DeleteChartVersion handles DELETE /api/:repo/charts/:name/:version
func (cra *ChartRepositoryAPI) DeleteChartVersion() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelAll) {
		return
	}

	chartController.GetManipulationHandler().DeleteChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//UploadChartVersion handles POST /api/:repo/charts
func (cra *ChartRepositoryAPI) UploadChartVersion() {
	hlog.Debugf("Header of request of uploading chart: %#v, content-len=%d", cra.Ctx.Request.Header, cra.Ctx.Request.ContentLength)

	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelWrite) {
		return
	}

	//Rewrite file content if the content type is "multipart/form-data"
	if isMultipartFormData(cra.Ctx.Request) {
		formFiles := make([]formFile, 0)
		formFiles = append(formFiles,
			formFile{
				formField: formFieldNameForChart,
				mustHave:  true,
			},
			formFile{
				formField: formFiledNameForProv,
			})
		if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
			chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
			return
		}
	}

	chartController.GetManipulationHandler().UploadChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//UploadChartProvFile handles POST /api/:repo/prov
func (cra *ChartRepositoryAPI) UploadChartProvFile() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelWrite) {
		return
	}

	//Rewrite file content if the content type is "multipart/form-data"
	if isMultipartFormData(cra.Ctx.Request) {
		formFiles := make([]formFile, 0)
		formFiles = append(formFiles,
			formFile{
				formField: formFiledNameForProv,
				mustHave:  true,
			})
		if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
			chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
			return
		}
	}

	chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//Rewrite the incoming URL with the right backend URL pattern
//Remove 'chartrepo' from the endpoints of manipulation API
//Remove 'chartrepo' from the endpoints of repository services
func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) {
	incomingURLPath := req.RequestURI

	defer func() {
		hlog.Debugf("Incoming URL '%s' is rewritten to '%s'", incomingURLPath, req.URL.String())
	}()

	//Health check endpoint
	if incomingURLPath == chartRepoHealthEndpoint {
		req.URL.Path = "/health"
		return
	}

	//Root uploading endpoint
	if incomingURLPath == rootUploadingEndpoint {
		req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
		return
	}

	//Repository endpoints
	if strings.HasPrefix(incomingURLPath, "/chartrepo") {
		req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
		return
	}

	//API endpoints
	if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
		req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
		return
	}
}

//Check if there exists a valid namespace
//Return true if it does
//Return false if it does not
func (cra *ChartRepositoryAPI) requireNamespace(namespace string) bool {
	//Actually, never should be like this
	if len(namespace) == 0 {
		cra.HandleBadRequest(":repo should be in the request URL")
		return false
	}

	existsing, err := cra.ProjectMgr.Exists(namespace)
	if err != nil {
		//Check failed with error
		cra.RenderError(http.StatusInternalServerError, fmt.Sprintf("failed to check existence of namespace %s with error: %s", namespace, err.Error()))
		return false
	}

	//Not existing
	if !existsing {
		cra.HandleBadRequest(fmt.Sprintf("namespace %s is not existing", namespace))
		return false
	}

	return true
}

//Check if the related access match the expected requirement
//If with right access, return true
//If without right access, return false
func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint) bool {
	if accessLevel == accessLevelPublic {
		return true //do nothing
	}

	//At least, authentication is necessary when level > public
	if !cra.SecurityCtx.IsAuthenticated() {
		cra.HandleUnauthorized()
		return false
	}

	theLevel := accessLevel
	//If repo is empty, system admin role must be required
	if len(namespace) == 0 {
		theLevel = accessLevelSystem
	}

	switch theLevel {
	//Should be system admin role
	case accessLevelSystem:
		if !cra.SecurityCtx.IsSysAdmin() {
			cra.RenderError(http.StatusForbidden, fmt.Sprintf("system admin role is required but user '%s' is not", cra.SecurityCtx.GetUsername()))
			return false
		}
	case accessLevelAll:
		if !cra.SecurityCtx.HasAllPerm(namespace) {
			cra.RenderError(http.StatusForbidden, fmt.Sprintf("project admin role is required but user '%s' does not have", cra.SecurityCtx.GetUsername()))
			return false
		}
	case accessLevelWrite:
		if !cra.SecurityCtx.HasWritePerm(namespace) {
			cra.RenderError(http.StatusForbidden, fmt.Sprintf("developer role is required but user '%s' does not have", cra.SecurityCtx.GetUsername()))
			return false
		}
	case accessLevelRead:
		if !cra.SecurityCtx.HasReadPerm(namespace) {
			cra.RenderError(http.StatusForbidden, fmt.Sprintf("at least a guest role is required for user '%s'", cra.SecurityCtx.GetUsername()))
			return false
		}
	default:
		//access rejected for invalid scope
		cra.RenderError(http.StatusForbidden, "unrecognized access scope")
		return false
	}

	return true
}

//Initialize the chart service controller
func initializeChartController() (*chartserver.Controller, error) {
	addr, err := config.GetChartMuseumEndpoint()
	if err != nil {
		return nil, fmt.Errorf("Failed to get the endpoint URL of chart storage server: %s", err.Error())
	}

	addr = strings.TrimSuffix(addr, "/")
	url, err := url.Parse(addr)
	if err != nil {
		return nil, errors.New("Endpoint URL of chart storage server is malformed")
	}

	controller, err := chartserver.NewController(url)
	if err != nil {
		return nil, errors.New("Failed to initialize chart API controller")
	}

	hlog.Debugf("Chart storage server is set to %s", url.String())
	hlog.Info("API controller for chart repository server is successfully initialized")

	return controller, nil
}

//formFile is used to represent the uploaded files in the form
type formFile struct {
	//form field key contains the form file
	formField string

	//flag to indicate if the file identified by the 'formField'
	//must exist
	mustHave bool
}

//If the files are uploaded with multipart/form-data mimetype, beego will extract the data
//from the request automatically. Then the request passed to the backend server with proxying
//way will have empty content.
//This method will refill the requests with file content.
func (cra *ChartRepositoryAPI) rewriteFileContent(files []formFile, request *http.Request) error {
	if len(files) == 0 {
		return nil //no files, early return
	}

	var body bytes.Buffer
	w := multipart.NewWriter(&body)
	defer func() {
		if err := w.Close(); err != nil {
			//Just log it
			hlog.Errorf("Failed to defer close multipart writer with error: %s", err.Error())
		}
	}()

	//Process files by key one by one
	for _, f := range files {
		mFile, mHeader, err := cra.GetFile(f.formField)
		//Handle error case by case
		if err != nil {
			formatedErr := fmt.Errorf("Get file content with multipart header from key '%s' failed with error: %s", f.formField, err.Error())
			if f.mustHave || err != http.ErrMissingFile {
				return formatedErr
			}

			//Error can be ignored, just log it
			hlog.Warning(formatedErr.Error())
			continue
		}

		fw, err := w.CreateFormFile(f.formField, mHeader.Filename)
		if err != nil {
			return fmt.Errorf("Create form file with multipart header failed with error: %s", err.Error())
		}

		_, err = io.Copy(fw, mFile)
		if err != nil {
			return fmt.Errorf("Copy file stream in multipart form data failed with error: %s", err.Error())
		}
	}

	request.Header.Set(headerContentType, w.FormDataContentType())
	request.ContentLength = int64(body.Len())
	request.Body = ioutil.NopCloser(&body)

	return nil
}

//Check if the request content type is "multipart/form-data"
func isMultipartFormData(req *http.Request) bool {
	return strings.Contains(req.Header.Get(headerContentType), contentTypeMultipart)
}
<commit_msg>Set the content length to -1 to let the reader read all the uploading contents in the request<commit_after>package api

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/url"
	"strings"

	"github.com/vmware/harbor/src/chartserver"
	hlog "github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/ui/config"
)

const (
	namespaceParam          = ":repo"
	defaultRepo             = "library"
	rootUploadingEndpoint   = "/api/chartrepo/charts"
	rootIndexEndpoint       = "/chartrepo/index.yaml"
	chartRepoHealthEndpoint = "/api/chartrepo/health"

	accessLevelPublic = iota
	accessLevelRead
	accessLevelWrite
	accessLevelAll
	accessLevelSystem

	formFieldNameForChart = "chart"
	formFiledNameForProv  = "prov"
	headerContentType     = "Content-Type"
	contentTypeMultipart  = "multipart/form-data"
)

//chartController is a singleton instance
var chartController *chartserver.Controller

//ChartRepositoryAPI provides related API handlers for the chart repository APIs
type ChartRepositoryAPI struct {
	//The base controller to provide common utilities
	BaseController

	//Keep the namespace if existing
	namespace string
}

//Prepare something for the following actions
func (cra *ChartRepositoryAPI) Prepare() {
	//Call super prepare method
	cra.BaseController.Prepare()

	//Try to extract namespace for parameter of path
	//It may not exist
	cra.namespace = strings.TrimSpace(cra.GetStringFromPath(namespaceParam))

	//Check the existence of namespace
	//Exclude the following URI
	// -/index.yaml
	// -/api/chartserver/health
	incomingURI := cra.Ctx.Request.RequestURI
	if incomingURI == rootUploadingEndpoint {
		//Forward to the default repository
		cra.namespace = defaultRepo
	}

	if incomingURI != rootIndexEndpoint &&
		incomingURI != chartRepoHealthEndpoint {
		if !cra.requireNamespace(cra.namespace) {
			return
		}
	}

	//Rewrite URL path
	cra.rewriteURLPath(cra.Ctx.Request)
}

//GetHealthStatus handles GET /api/chartserver/health
func (cra *ChartRepositoryAPI) GetHealthStatus() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelSystem) {
		return
	}

	chartController.GetBaseHandler().GetHealthStatus(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//GetIndexByRepo handles GET /:repo/index.yaml
func (cra *ChartRepositoryAPI) GetIndexByRepo() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	chartController.GetRepositoryHandler().GetIndexFileWithNS(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//GetIndex handles GET /index.yaml
func (cra *ChartRepositoryAPI) GetIndex() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelSystem) {
		return
	}

	chartController.GetRepositoryHandler().GetIndexFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//DownloadChart handles GET /:repo/charts/:filename
func (cra *ChartRepositoryAPI) DownloadChart() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	chartController.GetRepositoryHandler().DownloadChartObject(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//ListCharts handles GET /api/:repo/charts
func (cra *ChartRepositoryAPI) ListCharts() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	chartController.GetManipulationHandler().ListCharts(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//ListChartVersions GET /api/:repo/charts/:name
func (cra *ChartRepositoryAPI) ListChartVersions() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	chartController.GetManipulationHandler().GetChart(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//GetChartVersion handles GET /api/:repo/charts/:name/:version
func (cra *ChartRepositoryAPI) GetChartVersion() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelRead) {
		return
	}

	//Let's pass the namespace via the context of request
	req := cra.Ctx.Request
	*req = *(req.WithContext(context.WithValue(req.Context(), chartserver.NamespaceContextKey, cra.namespace)))

	chartController.GetManipulationHandler().GetChartVersion(cra.Ctx.ResponseWriter, req)
}

//DeleteChartVersion handles DELETE /api/:repo/charts/:name/:version
func (cra *ChartRepositoryAPI) DeleteChartVersion() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelAll) {
		return
	}

	chartController.GetManipulationHandler().DeleteChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//UploadChartVersion handles POST /api/:repo/charts
func (cra *ChartRepositoryAPI) UploadChartVersion() {
	hlog.Debugf("Header of request of uploading chart: %#v, content-len=%d", cra.Ctx.Request.Header, cra.Ctx.Request.ContentLength)

	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelWrite) {
		return
	}

	//Rewrite file content if the content type is "multipart/form-data"
	if isMultipartFormData(cra.Ctx.Request) {
		formFiles := make([]formFile, 0)
		formFiles = append(formFiles,
			formFile{
				formField: formFieldNameForChart,
				mustHave:  true,
			},
			formFile{
				formField: formFiledNameForProv,
			})
		if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
			chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
			return
		}
	}

	chartController.GetManipulationHandler().UploadChartVersion(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//UploadChartProvFile handles POST /api/:repo/prov
func (cra *ChartRepositoryAPI) UploadChartProvFile() {
	//Check access
	if !cra.requireAccess(cra.namespace, accessLevelWrite) {
		return
	}

	//Rewrite file content if the content type is "multipart/form-data"
	if isMultipartFormData(cra.Ctx.Request) {
		formFiles := make([]formFile, 0)
		formFiles = append(formFiles,
			formFile{
				formField: formFiledNameForProv,
				mustHave:  true,
			})
		if err := cra.rewriteFileContent(formFiles, cra.Ctx.Request); err != nil {
			chartserver.WriteInternalError(cra.Ctx.ResponseWriter, err)
			return
		}
	}

	chartController.GetManipulationHandler().UploadProvenanceFile(cra.Ctx.ResponseWriter, cra.Ctx.Request)
}

//Rewrite the incoming URL with the right backend URL pattern
//Remove 'chartrepo' from the endpoints of manipulation API
//Remove 'chartrepo' from the endpoints of repository services
func (cra *ChartRepositoryAPI) rewriteURLPath(req *http.Request) {
	incomingURLPath := req.RequestURI

	defer func() {
		hlog.Debugf("Incoming URL '%s' is rewritten to '%s'", incomingURLPath, req.URL.String())
	}()

	//Health check endpoint
	if incomingURLPath == chartRepoHealthEndpoint {
		req.URL.Path = "/health"
		return
	}

	//Root uploading endpoint
	if incomingURLPath == rootUploadingEndpoint {
		req.URL.Path = strings.Replace(incomingURLPath, "chartrepo", defaultRepo, 1)
		return
	}

	//Repository endpoints
	if strings.HasPrefix(incomingURLPath, "/chartrepo") {
		req.URL.Path = strings.TrimPrefix(incomingURLPath, "/chartrepo")
		return
	}

	//API endpoints
	if strings.HasPrefix(incomingURLPath, "/api/chartrepo") {
		req.URL.Path = strings.Replace(incomingURLPath, "/chartrepo", "", 1)
		return
	}
}

//Check if there exists a valid namespace
//Return true if it does
//Return false if it does not
func (cra *ChartRepositoryAPI) requireNamespace(namespace string) bool {
	//Actually, never should be like this
	if len(namespace) == 0 {
		cra.HandleBadRequest(":repo should be in the request URL")
		return false
	}

	existsing, err := cra.ProjectMgr.Exists(namespace)
	if err != nil {
		//Check failed with error
		cra.RenderError(http.StatusInternalServerError, fmt.Sprintf("failed to check existence of namespace %s with error: %s", namespace, err.Error()))
		return false
	}

	//Not existing
	if !existsing {
		cra.HandleBadRequest(fmt.Sprintf("namespace %s is not existing", namespace))
		return false
	}

	return true
}

//Check if the related access match the expected requirement
//If with right access, return true
//If without right access, return false
func (cra *ChartRepositoryAPI) requireAccess(namespace string, accessLevel uint) bool {
	if accessLevel == accessLevelPublic {
		return true //do nothing
	}

	//At least, authentication is necessary when level > public
	if !cra.SecurityCtx.IsAuthenticated() {
		cra.HandleUnauthorized()
		return false
	}

	theLevel := accessLevel
	//If repo is empty, system admin role must be required
	if len(namespace) == 0 {
		theLevel = accessLevelSystem
	}

	switch theLevel {
	//Should be system admin role
	case accessLevelSystem:
		if !cra.SecurityCtx.IsSysAdmin() {
			cra.RenderError(http.StatusForbidden, fmt.Sprintf("system admin role is required but user '%s' is not", cra.SecurityCtx.GetUsername()))
			return false
		}
	case accessLevelAll:
		if !cra.SecurityCtx.HasAllPerm(namespace) {
			cra.RenderError(http.StatusForbidden, fmt.Sprintf("project admin role is required but user '%s' does not have", cra.SecurityCtx.GetUsername()))
			return false
		}
	case accessLevelWrite:
		if !cra.SecurityCtx.HasWritePerm(namespace) {
			cra.RenderError(http.StatusForbidden, fmt.Sprintf("developer role is required but user '%s' does not have", cra.SecurityCtx.GetUsername()))
			return false
		}
	case accessLevelRead:
		if !cra.SecurityCtx.HasReadPerm(namespace) {
			cra.RenderError(http.StatusForbidden, fmt.Sprintf("at least a guest role is required for user '%s'", cra.SecurityCtx.GetUsername()))
			return false
		}
	default:
		//access rejected for invalid scope
		cra.RenderError(http.StatusForbidden, "unrecognized access scope")
		return false
	}

	return true
}

//Initialize the chart service controller
func initializeChartController() (*chartserver.Controller, error) {
	addr, err := config.GetChartMuseumEndpoint()
	if err != nil {
		return nil, fmt.Errorf("Failed to get the endpoint URL of chart storage server: %s", err.Error())
	}

	addr = strings.TrimSuffix(addr, "/")
	url, err := url.Parse(addr)
	if err != nil {
		return nil, errors.New("Endpoint URL of chart storage server is malformed")
	}

	controller, err := chartserver.NewController(url)
	if err != nil {
		return nil, errors.New("Failed to initialize chart API controller")
	}

	hlog.Debugf("Chart storage server is set to %s", url.String())
	hlog.Info("API controller for chart repository server is successfully initialized")

	return controller, nil
}

//formFile is used to represent the uploaded files in the form
type formFile struct {
	//form field key contains the form file
	formField string

	//flag to indicate if the file identified by the 'formField'
	//must exist
	mustHave bool
}

//If the files are uploaded with multipart/form-data mimetype, beego will extract the data
//from the request automatically. Then the request passed to the backend server with proxying
//way will have empty content.
//This method will refill the requests with file content.
func (cra *ChartRepositoryAPI) rewriteFileContent(files []formFile, request *http.Request) error {
	if len(files) == 0 {
		return nil //no files, early return
	}

	var body bytes.Buffer
	w := multipart.NewWriter(&body)
	defer func() {
		if err := w.Close(); err != nil {
			//Just log it
			hlog.Errorf("Failed to defer close multipart writer with error: %s", err.Error())
		}
	}()

	//Process files by key one by one
	for _, f := range files {
		mFile, mHeader, err := cra.GetFile(f.formField)
		//Handle error case by case
		if err != nil {
			formatedErr := fmt.Errorf("Get file content with multipart header from key '%s' failed with error: %s", f.formField, err.Error())
			if f.mustHave || err != http.ErrMissingFile {
				return formatedErr
			}

			//Error can be ignored, just log it
			hlog.Warning(formatedErr.Error())
			continue
		}

		fw, err := w.CreateFormFile(f.formField, mHeader.Filename)
		if err != nil {
			return fmt.Errorf("Create form file with multipart header failed with error: %s", err.Error())
		}

		_, err = io.Copy(fw, mFile)
		if err != nil {
			return fmt.Errorf("Copy file stream in multipart form data failed with error: %s", err.Error())
		}
	}

	request.Header.Set(headerContentType, w.FormDataContentType())
	request.ContentLength = -1
	request.Body = ioutil.NopCloser(&body)

	return nil
}

//Check if the request content type is "multipart/form-data"
func isMultipartFormData(req *http.Request) bool {
	return strings.Contains(req.Header.Get(headerContentType), contentTypeMultipart)
}
<|endoftext|>


<commit_before>/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package service

import (
	"encoding/json"
	"regexp"
	"strings"

	"github.com/vmware/harbor/src/ui/api"
	"github.com/vmware/harbor/src/common/dao"
	"github.com/vmware/harbor/src/common/models"
	"github.com/vmware/harbor/src/ui/service/cache"
	"github.com/vmware/harbor/src/common/utils"
	"github.com/vmware/harbor/src/common/utils/log"

	"github.com/astaxie/beego"
)

// NotificationHandler handles request on /service/notifications/, which listens to registry's events.
type NotificationHandler struct {
	beego.Controller
}

const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+(json|prettyjws)`

// Post handles POST request, and records audit log or refreshes cache based on event.
func (n *NotificationHandler) Post() {
	var notification models.Notification
	err := json.Unmarshal(n.Ctx.Input.CopyBody(1<<32), &notification)

	if err != nil {
		log.Errorf("failed to decode notification: %v", err)
		return
	}

	events, err := filterEvents(&notification)
	if err != nil {
		log.Errorf("failed to filter events: %v", err)
		return
	}

	for _, event := range events {
		repository := event.Target.Repository

		project, _ := utils.ParseRepository(repository)
		tag := event.Target.Tag
		action := event.Action

		user := event.Actor.Name
		if len(user) == 0 {
			user = "anonymous"
		}

		go func() {
			if err := dao.AccessLog(user, project, repository, tag, action); err != nil {
				log.Errorf("failed to add access log: %v", err)
			}
		}()
		if action == "push" {
			go func() {
				exist := dao.RepositoryExists(repository)
				if exist {
					return
				}
				log.Debugf("Add repository %s into DB.", repository)
				repoRecord := models.RepoRecord{Name: repository, OwnerName: user, ProjectName: project}
				if err := dao.AddRepository(repoRecord); err != nil {
					log.Errorf("Error happens when adding repository: %v", err)
				}
				if err := cache.RefreshCatalogCache(); err != nil {
					log.Errorf("failed to refresh cache: %v", err)
				}
			}()
			go api.TriggerReplicationByRepository(repository, []string{tag}, models.RepOpTransfer)
		}
		if action == "pull" {
			go func() {
				log.Debugf("Increase the repository %s pull count.", repository)
				if err := dao.IncreasePullCount(repository); err != nil {
					log.Errorf("Error happens when increasing pull count: %v", repository)
				}
			}()
		}
	}
}

func filterEvents(notification *models.Notification) ([]*models.Event, error) {
	events := []*models.Event{}

	for _, event := range notification.Events {
		log.Debugf("receive an event: ID-%s, target-%s:%s, digest-%s, action-%s", event.ID, event.Target.Repository, event.Target.Tag,
			event.Target.Digest, event.Action)

		isManifest, err := regexp.MatchString(manifestPattern, event.Target.MediaType)
		if err != nil {
			log.Errorf("failed to match the media type against pattern: %v", err)
			continue
		}

		if !isManifest {
			continue
		}

		//pull and push manifest by docker-client
		if strings.HasPrefix(event.Request.UserAgent, "docker") && (event.Action == "pull" || event.Action == "push") {
			events = append(events, &event)
			log.Debugf("add event to collect: %s", event.ID)
			continue
		}

		//push manifest by docker-client or job-service
		if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) == "harbor-registry-client" && event.Action == "push" {
			events = append(events, &event)
			log.Debugf("add event to collect: %s", event.ID)
			continue
		}
	}

	return events, nil
}

// Render returns nil as it won't render any template.
func (n *NotificationHandler) Render() error {
	return nil
}
<commit_msg>recognize vic in notification handler<commit_after>/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package service

import (
	"encoding/json"
	"regexp"
	"strings"

	"github.com/vmware/harbor/src/common/dao"
	"github.com/vmware/harbor/src/common/models"
	"github.com/vmware/harbor/src/common/utils"
	"github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/ui/api"
	"github.com/vmware/harbor/src/ui/service/cache"

	"github.com/astaxie/beego"
)

// NotificationHandler handles request on /service/notifications/, which listens to registry's events.
type NotificationHandler struct {
	beego.Controller
}

const manifestPattern = `^application/vnd.docker.distribution.manifest.v\d\+(json|prettyjws)`
const vicPrefix = "vic/"

// Post handles POST request, and records audit log or refreshes cache based on event.
func (n *NotificationHandler) Post() {
	var notification models.Notification
	err := json.Unmarshal(n.Ctx.Input.CopyBody(1<<32), &notification)

	if err != nil {
		log.Errorf("failed to decode notification: %v", err)
		return
	}

	events, err := filterEvents(&notification)
	if err != nil {
		log.Errorf("failed to filter events: %v", err)
		return
	}

	for _, event := range events {
		repository := event.Target.Repository

		project, _ := utils.ParseRepository(repository)
		tag := event.Target.Tag
		action := event.Action

		user := event.Actor.Name
		if len(user) == 0 {
			user = "anonymous"
		}

		go func() {
			if err := dao.AccessLog(user, project, repository, tag, action); err != nil {
				log.Errorf("failed to add access log: %v", err)
			}
		}()
		if action == "push" {
			go func() {
				exist := dao.RepositoryExists(repository)
				if exist {
					return
				}
				log.Debugf("Add repository %s into DB.", repository)
				repoRecord := models.RepoRecord{Name: repository, OwnerName: user, ProjectName: project}
				if err := dao.AddRepository(repoRecord); err != nil {
					log.Errorf("Error happens when adding repository: %v", err)
				}
				if err := cache.RefreshCatalogCache(); err != nil {
					log.Errorf("failed to refresh cache: %v", err)
				}
			}()
			go api.TriggerReplicationByRepository(repository, []string{tag}, models.RepOpTransfer)
		}
		if action == "pull" {
			go func() {
				log.Debugf("Increase the repository %s pull count.", repository)
				if err := dao.IncreasePullCount(repository); err != nil {
					log.Errorf("Error happens when increasing pull count: %v", repository)
				}
			}()
		}
	}
}

func filterEvents(notification *models.Notification) ([]*models.Event, error) {
	events := []*models.Event{}

	for _, event := range notification.Events {
		log.Debugf("receive an event: \n----ID: %s \n----target: %s:%s \n----digest: %s \n----action: %s \n----mediatype: %s \n----user-agent: %s", event.ID, event.Target.Repository,
			event.Target.Tag, event.Target.Digest, event.Action, event.Target.MediaType, event.Request.UserAgent)

		isManifest, err := regexp.MatchString(manifestPattern, event.Target.MediaType)
		if err != nil {
			log.Errorf("failed to match the media type against pattern: %v", err)
			continue
		}

		if !isManifest {
			continue
		}

		//pull and push manifest by docker-client or vic
		if (strings.HasPrefix(event.Request.UserAgent, "docker") || strings.HasPrefix(event.Request.UserAgent, vicPrefix)) &&
			(event.Action == "pull" || event.Action == "push") {
			events = append(events, &event)
			log.Debugf("add event to collect: %s", event.ID)
			continue
		}

		//push manifest by docker-client or job-service
		if strings.ToLower(strings.TrimSpace(event.Request.UserAgent)) == "harbor-registry-client" && event.Action == "push" {
			events = append(events, &event)
			log.Debugf("add event to collect: %s", event.ID)
			continue
		}
	}

	return events, nil
}

// Render returns nil as it won't render any template.
func (n *NotificationHandler) Render() error {
	return nil
}
<|endoftext|>


<commit_before>package ionic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"

	"github.com/hashicorp/go-version"
	"github.com/ion-channel/ionic/pagination"
	"github.com/ion-channel/ionic/responses"
	"github.com/ion-channel/ionic/scans"

	"github.com/ion-channel/ionic/dependencies"
)

const (
	// RubyEcosystem represents the ruby ecosystem for resolving dependencies
	RubyEcosystem = "ruby"
)

var (
	ecosystems = map[string]bool{"maven": true}
)

// ResolveDependenciesInFile takes a dependency file location and token to send
// the specified file to the API. All dependencies that are able to be resolved will
// be with their info returned, and a list of any errors encountered during the
// process.
func (ic *IonClient) ResolveDependenciesInFile(o dependencies.DependencyResolutionRequest, token string) (*dependencies.DependencyResolutionResponse, error) {
	params := url.Values{}
	params.Set("type", o.Ecosystem)
	if o.Flatten {
		params.Set("flatten", "true")
	}

	var buf bytes.Buffer
	w := multipart.NewWriter(&buf)

	fw, err := w.CreateFormFile("file", o.File)
	if err != nil {
		return nil, fmt.Errorf("failed to create form file: %v", err.Error())
	}

	fh, err := os.Open(o.File)
	if err != nil {
		return nil, fmt.Errorf("failed to open file: %v", err.Error())
	}

	_, err = io.Copy(fw, fh)
	if err != nil {
		return nil, fmt.Errorf("failed to copy file contents: %v", err.Error())
	}

	w.Close()

	var endpoint string
	switch {
	case strings.Contains(path.Base(o.File), "Gemfile.lock") ||
		strings.Contains(path.Base(o.File), "go.mod") ||
		strings.Contains(path.Base(o.File), "package.json") ||
		strings.Contains(path.Base(o.File), "package-lock.json") ||
		strings.Contains(path.Base(o.File), "Pipfile") ||
		strings.Contains(path.Base(o.File), "requirements.txt") ||
		strings.Contains(path.Base(o.File), "yarn.lock") ||
		strings.Contains(path.Base(o.File), "nuget") ||
		ecosystems[o.Ecosystem]:
		endpoint = dependencies.ResolveFromFileEndpoint
	default:
		endpoint = dependencies.ResolveDependenciesInFileEndpoint
	}

	h := http.Header{}
	h.Set("Content-Type", w.FormDataContentType())

	b, err := ic.Post(endpoint, token, params, buf, h)
	if err != nil {
		return nil, fmt.Errorf("failed to resolve dependencies: %v", err.Error())
	}

	var resp dependencies.DependencyResolutionResponse
	err = json.Unmarshal(b, &resp)
	if err != nil {
		return nil, fmt.Errorf("failed to read response: %v", err.Error())
	}

	return &resp, nil
}

// GetLatestVersionForDependency takes a package name, an ecosystem to find the
// package in, and a token for accessing the API. It returns a dependency
// representation of the latest version and any errors it encounters with the
// API.
func (ic *IonClient) GetLatestVersionForDependency(packageName, ecosystem, token string) (*dependencies.Dependency, error) {
	params := url.Values{}
	params.Set("name", packageName)
	params.Set("type", ecosystem)

	b, _, err := ic.Get(dependencies.GetLatestVersionForDependencyEndpoint, token, params, nil, pagination.Pagination{})
	if err != nil {
		return nil, fmt.Errorf("failed to get latest version for dependency: %v", err.Error())
	}

	var dep dependencies.Dependency
	err = json.Unmarshal(b, &dep)
	if err != nil {
		return nil, fmt.Errorf("cannot parse dependency: %v", err.Error())
	}

	dep.Name = packageName
	return &dep, nil
}

// GetVersionsForDependency takes a package name, an ecosystem to find the
// package in, and a token for accessing the API. It returns a dependency
// representation of the latest versions and any errors it encounters with the
// API.
func (ic *IonClient) GetVersionsForDependency(packageName, ecosystem, token string) ([]dependencies.Dependency, error) {
	params := url.Values{}
	params.Set("name", packageName)
	params.Set("type", ecosystem)

	b, _, err := ic.Get(dependencies.GetVersionsForDependencyEndpoint, token, params, nil, pagination.Pagination{})
	if err != nil {
		return nil, fmt.Errorf("failed to get latest version for dependency: %v", err.Error())
	}

	var vs []string
	err = json.Unmarshal(b, &vs)
	if err != nil {
		return nil, fmt.Errorf("cannot parse dependency: %v", err.Error())
	}

	deps := []dependencies.Dependency{}
	for index := range vs {
		dep := dependencies.Dependency{
			Name:    packageName,
			Version: vs[index],
		}
		deps = append(deps, dep)
	}

	return deps, nil
}

// SearchDependencies takes a query `org AND name` and
// calls the Ion API to retrieve the information, then forms a slice of
// Ionic dependencies.Dependency objects
func (ic *IonClient) SearchDependencies(q string, page pagination.Pagination, token string) ([]dependencies.Dependency, *responses.Meta, error) {
	params := url.Values{}
	params.Set("q", q)

	b, m, err := ic.Get(dependencies.ResolveDependencySearchEndpoint, token, params, nil, page)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get dependencies: %v", err.Error())
	}
	var results []dependencies.Dependency
	err = json.Unmarshal(b, &results)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to unmarshal search results: %v (%v)", err.Error(), string(b))
	}
	return results, m, nil
}

// GetDependencyVersions takes a package name, an ecosystem to find the
// package in, optional version, and a token for accessing the API.
// If version is supplied, it will return all known versions greater than what was given
// It returns a slice of Ionic dependencies.Dependency objects
func (ic *IonClient) GetDependencyVersions(packageName, ecosystem, version, token string) ([]dependencies.Dependency, error) {
	params := url.Values{}
	params.Set("name", packageName)
	params.Set("type", ecosystem)
	if version != "" {
		params.Set("version", version)
	}

	b, _, err := ic.Get(dependencies.GetDependencyVersions, token, params, nil, pagination.Pagination{})
	if err != nil {
		return nil, fmt.Errorf("failed to get dependency versions: %v", err.Error())
	}

	var deps []dependencies.Dependency
	err = json.Unmarshal(b, &deps)
	if err != nil {
		return nil, fmt.Errorf("cannot parse dependency: %v", err.Error())
	}

	return deps, nil
}

// GetDifferenceBetweenVersions calculates the difference between two version strings, returning an OutdatedMeta
// object, or an error.
func GetDifferenceBetweenVersions(newerVersion, olderVersion string) (outdatedMeta scans.OutdatedMeta, err error) {
	ver, err := version.NewVersion(olderVersion)
	if err != nil {
		return outdatedMeta, err
	}

	latestVersion, err := version.NewVersion(newerVersion)
	if err != nil {
		return outdatedMeta, err
	}

	// for major.minor.patch
	var versionsBehind [3]int
	// check if our version is out of date, and calculate its versions behind if so
	if ver.LessThan(latestVersion) {
		versions := ver.Segments()
		latestVer := latestVersion.Segments()
		for i := 0; i < len(versionsBehind); i++ {
			if versions[i] <= latestVer[i] {
				versionsBehind[i] = latestVer[i] - versions[i]
			} else {
				versionsBehind[i] = 0
			}
		}

		outdatedMeta = scans.OutdatedMeta{
			MajorBehind: versionsBehind[0],
			MinorBehind: versionsBehind[1],
			PatchBehind: versionsBehind[2],
		}
	}

	return outdatedMeta, err
}
<commit_msg>refactor endpoint to use ecosystem type<commit_after>package ionic

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"

	"github.com/hashicorp/go-version"
	"github.com/ion-channel/ionic/pagination"
	"github.com/ion-channel/ionic/responses"
	"github.com/ion-channel/ionic/scans"

	"github.com/ion-channel/ionic/dependencies"
)

const (
	// RubyEcosystem represents the ruby ecosystem for resolving dependencies
	RubyEcosystem = "ruby"
)

var (
	ecosystems = map[string]bool{"maven": true}
)

// ResolveDependenciesInFile takes a dependency file location and token to send
// the specified file to the API. All dependencies that are able to be resolved will
// be with their info returned, and a list of any errors encountered during the
// process.
func (ic *IonClient) ResolveDependenciesInFile(o dependencies.DependencyResolutionRequest, token string) (*dependencies.DependencyResolutionResponse, error) {
	params := url.Values{}
	params.Set("type", o.Ecosystem)
	if o.Flatten {
		params.Set("flatten", "true")
	}

	var buf bytes.Buffer
	w := multipart.NewWriter(&buf)

	fw, err := w.CreateFormFile("file", o.File)
	if err != nil {
		return nil, fmt.Errorf("failed to create form file: %v", err.Error())
	}

	fh, err := os.Open(o.File)
	if err != nil {
		return nil, fmt.Errorf("failed to open file: %v", err.Error())
	}

	_, err = io.Copy(fw, fh)
	if err != nil {
		return nil, fmt.Errorf("failed to copy file contents: %v", err.Error())
	}

	w.Close()

	var endpoint string
	switch o.Ecosystem {
	// case strings.Contains(path.Base(o.E), "Gemfile.lock") ||
	// 	strings.Contains(path.Base(o.File), "go.mod") ||
	// 	strings.Contains(path.Base(o.File), "package.json") ||
	// 	strings.Contains(path.Base(o.File), "package-lock.json") ||
	// 	strings.Contains(path.Base(o.File), "Pipfile") ||
	// 	strings.Contains(path.Base(o.File), "requirements.txt") ||
	// 	strings.Contains(path.Base(o.File), "yarn.lock") ||
	// 	strings.Contains(path.Base(o.File), "nuget") ||
	case "gomod", "package-json", "package", "package-lock", "pipfile", "maven", "requirements", "yarnlock", "nuget":
		endpoint = dependencies.ResolveFromFileEndpoint
	default:
		endpoint = dependencies.ResolveDependenciesInFileEndpoint
	}

	h := http.Header{}
	h.Set("Content-Type", w.FormDataContentType())

	b, err := ic.Post(endpoint, token, params, buf, h)
	if err != nil {
		return nil, fmt.Errorf("failed to resolve dependencies: %v", err.Error())
	}

	var resp dependencies.DependencyResolutionResponse
	err = json.Unmarshal(b, &resp)
	if err != nil {
		return nil, fmt.Errorf("failed to read response: %v", err.Error())
	}

	return &resp, nil
}

// GetLatestVersionForDependency takes a package name, an ecosystem to find the
// package in, and a token for accessing the API. It returns a dependency
// representation of the latest version and any errors it encounters with the
// API.
func (ic *IonClient) GetLatestVersionForDependency(packageName, ecosystem, token string) (*dependencies.Dependency, error) {
	params := url.Values{}
	params.Set("name", packageName)
	params.Set("type", ecosystem)

	b, _, err := ic.Get(dependencies.GetLatestVersionForDependencyEndpoint, token, params, nil, pagination.Pagination{})
	if err != nil {
		return nil, fmt.Errorf("failed to get latest version for dependency: %v", err.Error())
	}

	var dep dependencies.Dependency
	err = json.Unmarshal(b, &dep)
	if err != nil {
		return nil, fmt.Errorf("cannot parse dependency: %v", err.Error())
	}

	dep.Name = packageName
	return &dep, nil
}

// GetVersionsForDependency takes a package name, an ecosystem to find the
// package in, and a token for accessing the API. It returns a dependency
// representation of the latest versions and any errors it encounters with the
// API.
func (ic *IonClient) GetVersionsForDependency(packageName, ecosystem, token string) ([]dependencies.Dependency, error) {
	params := url.Values{}
	params.Set("name", packageName)
	params.Set("type", ecosystem)

	b, _, err := ic.Get(dependencies.GetVersionsForDependencyEndpoint, token, params, nil, pagination.Pagination{})
	if err != nil {
		return nil, fmt.Errorf("failed to get latest version for dependency: %v", err.Error())
	}

	var vs []string
	err = json.Unmarshal(b, &vs)
	if err != nil {
		return nil, fmt.Errorf("cannot parse dependency: %v", err.Error())
	}

	deps := []dependencies.Dependency{}
	for index := range vs {
		dep := dependencies.Dependency{
			Name:    packageName,
			Version: vs[index],
		}
		deps = append(deps, dep)
	}

	return deps, nil
}

// SearchDependencies takes a query `org AND name` and
// calls the Ion API to retrieve the information, then forms a slice of
// Ionic dependencies.Dependency objects
func (ic *IonClient) SearchDependencies(q string, page pagination.Pagination, token string) ([]dependencies.Dependency, *responses.Meta, error) {
	params := url.Values{}
	params.Set("q", q)

	b, m, err := ic.Get(dependencies.ResolveDependencySearchEndpoint, token, params, nil, page)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to get dependencies: %v", err.Error())
	}
	var results []dependencies.Dependency
	err = json.Unmarshal(b, &results)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to unmarshal search results: %v (%v)", err.Error(), string(b))
	}
	return results, m, nil
}

// GetDependencyVersions takes a package name, an ecosystem to find the
// package in, optional version, and a token for accessing the API.
// If version is supplied, it will return all known versions greater than what was given
// It returns a slice of Ionic dependencies.Dependency objects
func (ic *IonClient) GetDependencyVersions(packageName, ecosystem, version, token string) ([]dependencies.Dependency, error) {
	params := url.Values{}
	params.Set("name", packageName)
	params.Set("type", ecosystem)
	if version != "" {
		params.Set("version", version)
	}

	b, _, err := ic.Get(dependencies.GetDependencyVersions, token, params, nil, pagination.Pagination{})
	if err != nil {
		return nil, fmt.Errorf("failed to get dependency versions: %v", err.Error())
	}

	var deps []dependencies.Dependency
	err = json.Unmarshal(b, &deps)
	if err != nil {
		return nil, fmt.Errorf("cannot parse dependency: %v", err.Error())
	}

	return deps, nil
}

// GetDifferenceBetweenVersions calculates the difference between two version strings, returning an OutdatedMeta
// object, or an error.
func GetDifferenceBetweenVersions(newerVersion, olderVersion string) (outdatedMeta scans.OutdatedMeta, err error) {
	ver, err := version.NewVersion(olderVersion)
	if err != nil {
		return outdatedMeta, err
	}

	latestVersion, err := version.NewVersion(newerVersion)
	if err != nil {
		return outdatedMeta, err
	}

	// for major.minor.patch
	var versionsBehind [3]int
	// check if our version is out of date, and calculate its versions behind if so
	if ver.LessThan(latestVersion) {
		versions := ver.Segments()
		latestVer := latestVersion.Segments()
		for i := 0; i < len(versionsBehind); i++ {
			if versions[i] <= latestVer[i] {
				versionsBehind[i] = latestVer[i] - versions[i]
			} else {
				versionsBehind[i] = 0
			}
		}

		outdatedMeta = scans.OutdatedMeta{
			MajorBehind: versionsBehind[0],
			MinorBehind: versionsBehind[1],
			PatchBehind: versionsBehind[2],
		}
	}

	return outdatedMeta, err
}
<|endoftext|>


<commit_before>package health

import (
	"errors"
	"time"

	. "github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-check"
	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/random"
	"github.com/flynn/flynn/pkg/stream"
)

type RegisterSuite struct{}

var _ = Suite(&RegisterSuite{})

type RegistrarFunc func(service string, inst *discoverd.Instance) (discoverd.Heartbeater, error)

func (f RegistrarFunc) RegisterInstance(service string, inst *discoverd.Instance) (discoverd.Heartbeater, error) {
	return f(service, inst)
}

type FakeHeartbeat struct {
	closeFn   func() error
	setMetaFn func(map[string]string) error
	addrFn    func() string
}

func (f FakeHeartbeat) SetMeta(meta map[string]string) error { return f.setMetaFn(meta) }
func (f FakeHeartbeat) Close() error                         { return f.closeFn() }
func (f FakeHeartbeat) Addr() string                         { return f.addrFn() }
func (f FakeHeartbeat) SetClient(*discoverd.Client)          {}

func init() {
	registerErrWait = time.Millisecond
}

func (RegisterSuite) TestRegister(c *C) {
	type step struct {
		event      bool // send an event
		up         bool // event type
		register   bool // event should trigger register
		unregister bool // event should unregister service
		setMeta    bool // attempt SetMeta
		success    bool // true if SetMeta or Register should succeed
	}

	type called struct {
		args      map[string]interface{}
		returnVal chan bool
	}

	run := func(c *C, steps []step) {
		check := CheckFunc(func() error { return nil })

		metaChan := make(chan called)
		unregisterChan := make(chan called)
		heartbeater := FakeHeartbeat{
			addrFn: func() string {
				return "notnil"
			},
			closeFn: func() error {
				unregisterChan <- called{}
				return nil
			},
			setMetaFn: func(meta map[string]string) error {
				success := make(chan bool)
				metaChan <- called{
					args: map[string]interface{}{
						"meta": meta,
					},
					returnVal: success,
				}
				if !<-success {
					return errors.New("SetMeta failed")
				}
				return nil
			},
		}

		registrarChan := make(chan called)
		registrar := RegistrarFunc(func(service string, inst *discoverd.Instance) (discoverd.Heartbeater, error) {
			success := make(chan bool)
			registrarChan <- called{
				args: map[string]interface{}{
					"service": service,
					"inst":    inst,
				},
				returnVal: success,
			}
			defer func() { registrarChan <- called{} }()
			if <-success {
				return heartbeater, nil
			}
			return nil, errors.New("registrar failure")
		})

		monitorChan := make(chan bool)
		monitor := func(c Check, ch chan MonitorEvent) stream.Stream {
			stream := stream.New()
			go func() {
				defer close(ch)
			outer:
				for {
					select {
					case up, ok := <-monitorChan:
						if !ok {
							break outer
						}
						if up {
							ch <- MonitorEvent{
								Check:  check,
								Status: MonitorStatusUp,
							}
						} else {
							ch <- MonitorEvent{
								Check:  check,
								Status: MonitorStatusDown,
							}
						}
					case <-stream.StopCh:
						break outer
					}
				}
			}()
			return stream
		}

		reg := Registration{
			Service: "test",
			Instance: &discoverd.Instance{
				Meta: make(map[string]string),
			},
			Registrar: registrar,
			Monitor:   monitor,
			Check:     check,
			Events:    make(chan MonitorEvent),
		}
		hb := reg.Register()
		defer func() {
			go func() { <-unregisterChan }()
			hb.Close()
			close(unregisterChan)
		}()

		errCh := make(chan bool)
		errCheck := func(ch chan called, stop chan bool) {
			go func() {
				select {
				case call, ok := <-ch:
					if !ok {
						return
					}
					if call.returnVal != nil {
						call.returnVal <- false
					}
					errCh <- true
				case <-stop:
					errCh <- false
				}
			}()
		}

		var stop chan bool
		currentMeta := make(map[string]string)
		for _, step := range steps {
			stop = make(chan bool)
			var wait int

			if step.event {
				monitorChan <- step.up
			}
			if step.register {
				call := <-registrarChan
				c.Assert(call.args["inst"].(*discoverd.Instance).Meta, DeepEquals, currentMeta)
				call.returnVal <- step.success
				<-registrarChan
			} else {
				wait++
				errCheck(registrarChan, stop)
			}
			if step.unregister {
				// before unregistering, Addr should not be nil
				c.Assert(hb.Addr(), Not(Equals), "")
				select {
				case <-unregisterChan:
				case <-time.After(3 * time.Second):
					c.Error("Timed out waiting for unregistration")
				}
				// Addr should be nil now
				c.Assert(hb.Addr(), Equals, "")
			} else {
				wait++
				errCheck(unregisterChan, stop)
			}
			if step.setMeta {
				go func() {
					call := <-metaChan
					if call.returnVal != nil {
						call.returnVal <- step.success
					}
				}()
				// SetMeta needs to succeed every time, regardless of the situation
				currentMeta["TEST"] = random.UUID()
				c.Assert(hb.SetMeta(currentMeta), IsNil)
			} else {
				wait++
				errCheck(metaChan, stop)
			}

			// check event forwarding
			if step.event {
				select {
				case val := <-reg.Events:
					if step.up {
						c.Assert(val.Status, Equals, MonitorStatusUp)
					} else {
						c.Assert(val.Status, Equals, MonitorStatusDown)
					}
				case <-time.After(3 * time.Second):
					c.Error("Timed out waiting for Registration.Events")
				}
			}

			// check whether functions were called on this step that should not
			// have ran
			close(stop)
			for i := 0; i < wait; i++ {
				select {
				case err := <-errCh:
					if err {
						c.Error("Received data on a channel that should not be recieving on this step")
					}
				}
			}
		}
		close(monitorChan)
		close(registrarChan)
		close(metaChan)
	}

	for _, t := range []struct {
		name  string
		steps []step
	}{
		{
			name: "register success up/down/up",
			steps: []step{
				{event: true, up: true, register: true, success: true},
				{event: true, unregister: true},
				{event: true, up: true, register: true, success: true},
			},
		},
		{
			name: "register fails then succeeds",
			steps: []step{
				{event: true, up: true, register: true, success: false},
				{register: true, success: true},
			},
		},
		{
			name: "register is called only once if we get two up events",
			steps: []step{
				{event: true, up: true, register: true, success: true},
				{event: true, up: true, register: false},
			},
		},
		{
			name: "setmeta while registered",
			steps: []step{
				{event: true, up: true, register: true, success: true},
				{setMeta: true, success: true},
			},
		},
		{
			name: "setmeta while offline",
			steps: []step{
				{setMeta: true, success: false},
				// confirm that the right meta is sent when the process does
				// come up
				{event: true, up: true, register: true, success: true},
			},
		},
		{
			name: "setmeta while erroring registration",
			steps: []step{
				{event: true, up: true, register: true, success: false},
				{register: true, success: false},
				{setMeta: true, success: false},
				{register: true, success: true},
			},
		},
		{
			name: "register failing then offline",
			steps: []step{
				{event: true, up: true, register: true, success: false},
				{register: true, success: false},
				{event: true},
				{}, // make sure register does not run
				{},
			},
		},
	} {
		c.Log("--- TEST:", t.name)
		run(c, t.steps)
	}
}
<commit_msg>discoverd/health: Delete register tests<commit_after><|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/beatrichartz/martini-sockets"
	log "github.com/cihub/seelog"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/encoder"
	"github.com/stampzilla/stampzilla-go/stampzilla-server/logic"
	serverprotocol "github.com/stampzilla/stampzilla-go/stampzilla-server/protocol"
)

// Webserver that serves static files

type WebServer struct {
	Config     *ServerConfig         `inject:""`
	Logic      *logic.Logic          `inject:""`
	Nodes      *serverprotocol.Nodes `inject:""`
	WsClients  *Clients              `inject:""`
	WebHandler *WebHandler           `inject:""`
}

func NewWebServer() *WebServer {
	return &WebServer{}
}

func (ws *WebServer) Start() {
	log.Info("Starting WEB (:" + ws.Config.WebPort + " in " + ws.Config.WebRoot + ")")

	//m := martini.Classic()
	r := martini.NewRouter()
	ma := martini.New()
	ma.Use(martini.Logger())
	ma.Use(martini.Recovery())
	ma.Use(martini.Static(ws.Config.WebRoot))
	ma.MapTo(r, (*martini.Routes)(nil))
	ma.Action(r.Handle)
	m := &martini.ClassicMartini{ma, r}

	m.Use(func(c martini.Context, w http.ResponseWriter) {
		c.MapTo(encoder.JsonEncoder{}, (*encoder.Encoder)(nil))
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
	})

	m.Get("/socket", sockets.JSON(Message{}), ws.WsClients.websocketRoute)

	//Nodes
	m.Get("/api/nodes", ws.WebHandler.GetNodes)
	m.Get("/api/node/:id", ws.WebHandler.GetNode)
	m.Put("/api/node/:id/cmd", ws.WebHandler.CommandToNode)

	//Rules
	m.Get("/api/rules", ws.WebHandler.GetRules)

	log.Critical(http.ListenAndServe(":"+ws.Config.WebPort, m))
}
<commit_msg>removed unnedded inject deps<commit_after>package main

import (
	"net/http"

	"github.com/beatrichartz/martini-sockets"
	log "github.com/cihub/seelog"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/encoder"
)

// Webserver that serves static files

type WebServer struct {
	Config     *ServerConfig `inject:""`
	WsClients  *Clients      `inject:""`
	WebHandler *WebHandler   `inject:""`
}

func NewWebServer() *WebServer {
	return &WebServer{}
}

func (ws *WebServer) Start() {
	log.Info("Starting WEB (:" + ws.Config.WebPort + " in " + ws.Config.WebRoot + ")")

	//m := martini.Classic()
	r := martini.NewRouter()
	ma := martini.New()
	ma.Use(martini.Logger())
	ma.Use(martini.Recovery())
	ma.Use(martini.Static(ws.Config.WebRoot))
	ma.MapTo(r, (*martini.Routes)(nil))
	ma.Action(r.Handle)
	m := &martini.ClassicMartini{ma, r}

	m.Use(func(c martini.Context, w http.ResponseWriter) {
		c.MapTo(encoder.JsonEncoder{}, (*encoder.Encoder)(nil))
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
	})

	m.Get("/socket", sockets.JSON(Message{}), ws.WsClients.websocketRoute)

	//Nodes
	m.Get("/api/nodes", ws.WebHandler.GetNodes)
	m.Get("/api/node/:id", ws.WebHandler.GetNode)
	m.Put("/api/node/:id/cmd", ws.WebHandler.CommandToNode)

	//Rules
	m.Get("/api/rules", ws.WebHandler.GetRules)

	log.Critical(http.ListenAndServe(":"+ws.Config.WebPort, m))
}
<|endoftext|>