<commit_before>// Resolve function calls and variable types

package parser

import (
	"fmt"
	"sort"

	. "github.com/benhoyt/goawk/internal/ast"
	. "github.com/benhoyt/goawk/lexer"
)

type varType int

const (
	typeUnknown varType = iota
	typeScalar
	typeArray
)

// typeInfo records type information for a single variable
type typeInfo struct {
	typ      varType
	ref      *VarExpr
	scope    VarScope
	index    int
	callName string
	argIndex int
}

// Used by printVarTypes when debugTypes is turned on
func (t typeInfo) String() string {
	var typ string
	switch t.typ {
	case typeScalar:
		typ = "Scalar"
	case typeArray:
		typ = "Array"
	default:
		typ = "Unknown"
	}
	var scope string
	switch t.scope {
	case ScopeGlobal:
		scope = "Global"
	case ScopeLocal:
		scope = "Local"
	default:
		scope = "Special"
	}
	return fmt.Sprintf("typ=%s ref=%p scope=%s index=%d callName=%q argIndex=%d",
		typ, t.ref, scope, t.index, t.callName, t.argIndex)
}

// A single variable reference (normally scalar)
type varRef struct {
	funcName string
	ref      *VarExpr
	isArg    bool
	pos      Position
}

// A single array reference
type arrayRef struct {
	funcName string
	ref      *ArrayExpr
	pos      Position
}

// Initialize the resolver
func (p *parser) initResolve() {
	p.varTypes = make(map[string]map[string]typeInfo)
	p.varTypes[""] = make(map[string]typeInfo) // globals
	p.functions = make(map[string]int)
	p.arrayRef("ARGV", Position{1, 1}) // interpreter relies on ARGV being present
	p.multiExprs = make(map[*MultiExpr]Position, 3)
}

// Signal the start of a function: records the function name and
// local variables so variable references can determine scope
func (p *parser) startFunction(name string, params []string) {
	p.funcName = name
	p.varTypes[name] = make(map[string]typeInfo)
	p.locals = make(map[string]bool, len(params))
	for _, param := range params {
		p.locals[param] = true
	}
}

// Signal the end of a function
func (p *parser) stopFunction() {
	p.funcName = ""
	p.locals = nil
}

// Add function by name with given index
func (p *parser) addFunction(name string, index int) {
	p.functions[name] = index
}

// Records a call to a user function (for resolving indexes later)
type userCall struct {
	call   *UserCallExpr
	pos    Position
	inFunc string
}

// Record a user call site
func (p *parser) recordUserCall(call *UserCallExpr, pos Position) {
	p.userCalls = append(p.userCalls, userCall{call, pos, p.funcName})
}

// After parsing, resolve all user calls to their indexes. Also
// ensures functions called have actually been defined, and that
// they're not being called with too many arguments.
func (p *parser) resolveUserCalls(prog *Program) {
	for _, c := range p.userCalls {
		index, ok := p.functions[c.call.Name]
		if !ok {
			panic(&ParseError{c.pos, fmt.Sprintf("undefined function %q", c.call.Name)})
		}
		function := prog.Functions[index]
		if len(c.call.Args) > len(function.Params) {
			panic(&ParseError{c.pos, fmt.Sprintf("%q called with more arguments than declared", c.call.Name)})
		}
		c.call.Index = index
	}
}

// For arguments that are variable references, we don't know the
// type based on context, so mark the types for these as unknown.
func (p *parser) processUserCallArg(funcName string, arg Expr, index int) {
	if varExpr, ok := arg.(*VarExpr); ok {
		// TODO: It think p.funcName here is wrong! because it may be
		// a global referenced inside a function
		ref := p.varTypes[p.funcName][varExpr.Name].ref
		if ref == varExpr {
			// Only applies if this is the first reference to this
			// variable (otherwise we know the type already)
			scope := p.varTypes[p.funcName][varExpr.Name].scope
			p.varTypes[p.funcName][varExpr.Name] = typeInfo{typeUnknown, ref, scope, 0, funcName, index}
		}
		// Mark the last related varRef (the most recent one) as a
		// call argument for later error handling
		p.varRefs[len(p.varRefs)-1].isArg = true
	}
}

// Determine scope of given variable reference (and funcName if it's
// a local, otherwise empty string)
func (p *parser) getScope(name string) (VarScope, string) {
	switch {
	case p.funcName != "" && p.locals[name]:
		return ScopeLocal, p.funcName
	case SpecialVarIndex(name) > 0:
		return ScopeSpecial, ""
	default:
		return ScopeGlobal, ""
	}
}

// Record a variable (scalar) reference and return the *VarExpr (but
// VarExpr.Index won't be set till later)
func (p *parser) varRef(name string, pos Position) *VarExpr {
	scope, funcName := p.getScope(name)
	expr := &VarExpr{scope, 0, name}
	p.varRefs = append(p.varRefs, varRef{funcName, expr, false, pos})
	typ := p.varTypes[funcName][name].typ
	if typ == typeUnknown {
		p.varTypes[funcName][name] = typeInfo{typeScalar, expr, scope, 0, "", 0}
	}
	return expr
}

// Record an array reference and return the *ArrayExpr (but
// ArrayExpr.Index won't be set till later)
func (p *parser) arrayRef(name string, pos Position) *ArrayExpr {
	scope, funcName := p.getScope(name)
	if scope == ScopeSpecial {
		panic(p.error("can't use scalar %q as array", name))
	}
	expr := &ArrayExpr{scope, 0, name}
	p.arrayRefs = append(p.arrayRefs, arrayRef{funcName, expr, pos})
	typ := p.varTypes[funcName][name].typ
	if typ == typeUnknown {
		p.varTypes[funcName][name] = typeInfo{typeArray, nil, scope, 0, "", 0}
	}
	return expr
}

// Print variable type information (for debugging) on p.debugWriter
func (p *parser) printVarTypes(prog *Program) {
	fmt.Fprintf(p.debugWriter, "scalars: %v\n", prog.Scalars)
	fmt.Fprintf(p.debugWriter, "arrays: %v\n", prog.Arrays)
	funcNames := []string{}
	for funcName := range p.varTypes {
		funcNames = append(funcNames, funcName)
	}
	sort.Strings(funcNames)
	for _, funcName := range funcNames {
		if funcName != "" {
			fmt.Fprintf(p.debugWriter, "function %s\n", funcName)
		} else {
			fmt.Fprintf(p.debugWriter, "globals\n")
		}
		varNames := []string{}
		for name := range p.varTypes[funcName] {
			varNames = append(varNames, name)
		}
		sort.Strings(varNames)
		for _, name := range varNames {
			info := p.varTypes[funcName][name]
			fmt.Fprintf(p.debugWriter, "  %s: %s\n", name, info)
		}
	}
}

// If we can't finish resolving after this many iterations, give up
const maxResolveIterations = 10000

// Resolve unknown variables types and generate variable indexes and
// name-to-index mappings for interpreter
func (p *parser) resolveVars(prog *Program) {
	// First go through all unknown types and try to determine the
	// type from the parameter type in that function definition. May
	// need multiple passes depending on the order of functions. This
	// is not particularly efficient, but on realistic programs it's
	// not an issue.
	for i := 0; ; i++ {
		progressed := false
		for funcName, infos := range p.varTypes {
			for name, info := range infos {
				if info.scope == ScopeSpecial {
					continue
				}
				if info.typ == typeUnknown {
					paramName := prog.Functions[p.functions[info.callName]].Params[info.argIndex]
					typ := p.varTypes[info.callName][paramName].typ
					if typ != typeUnknown {
						info.typ = typ
						p.varTypes[funcName][name] = info
						progressed = true
					}
				}
			}
		}
		if !progressed {
			// If we didn't progress we're done (or trying again is
			// not going to help)
			break
		}
		if i >= maxResolveIterations {
			panic(p.error("too many iterations trying to resolve variable types"))
		}
	}

	// Resolve global variables (iteration order is undefined, so
	// assign indexes basically randomly)
	prog.Scalars = make(map[string]int)
	prog.Arrays = make(map[string]int)
	for name, info := range p.varTypes[""] {
		var index int
		if info.scope == ScopeSpecial {
			index = SpecialVarIndex(name)
		} else if info.typ == typeArray {
			index = len(prog.Arrays)
			prog.Arrays[name] = index
		} else {
			index = len(prog.Scalars)
			prog.Scalars[name] = index
		}
		info.index = index
		p.varTypes[""][name] = info
	}

	// Resolve local variables (assign indexes in order of params).
	// Also patch up Function.Arrays (tells interpreter which args
	// are arrays).
	for funcName, infos := range p.varTypes {
		if funcName == "" {
			continue
		}
		scalarIndex := 0
		arrayIndex := 0
		functionIndex := p.functions[funcName]
		function := prog.Functions[functionIndex]
		arrays := make([]bool, len(function.Params))
		for i, name := range function.Params {
			info := infos[name]
			var index int
			if info.typ == typeArray {
				index = arrayIndex
				arrayIndex++
				arrays[i] = true
			} else {
				// typeScalar or typeUnknown: variables may still be
				// of unknown type if they've never been referenced --
				// default to scalar in that case
				index = scalarIndex
				scalarIndex++
			}
			info.index = index
			p.varTypes[funcName][name] = info
		}
		prog.Functions[functionIndex].Arrays = arrays
	}

	// Check that variables passed to functions are the correct type
	for _, c := range p.userCalls {
		function := prog.Functions[c.call.Index]
		for i, arg := range c.call.Args {
			varExpr, ok := arg.(*VarExpr)
			if !ok {
				if function.Arrays[i] {
					message := fmt.Sprintf("can't pass scalar %s as array param", arg)
					panic(&ParseError{c.pos, message})
				}
				continue
			}
			// If this VarExpr refers to a local, use caller name to
			// look up in varTypes, otherwise use "" (meaning global).
			funcName := ""
			for _, p := range prog.Functions[p.functions[c.inFunc]].Params {
				if varExpr.Name == p {
					funcName = c.inFunc
					break
				}
			}
			info := p.varTypes[funcName][varExpr.Name]
			if info.typ == typeArray && !function.Arrays[i] {
				message := fmt.Sprintf("can't pass array %q as scalar param", varExpr.Name)
				panic(&ParseError{c.pos, message})
			}
			if info.typ != typeArray && function.Arrays[i] {
				message := fmt.Sprintf("can't pass scalar %q as array param", varExpr.Name)
				panic(&ParseError{c.pos, message})
			}
		}
	}

	if p.debugTypes {
		p.printVarTypes(prog)
	}

	// Patch up variable indexes (interpreter uses an index instead
	// the name for more efficient lookups)
	for _, varRef := range p.varRefs {
		info := p.varTypes[varRef.funcName][varRef.ref.Name]
		if info.typ == typeArray && !varRef.isArg {
			message := fmt.Sprintf("can't use array %q as scalar", varRef.ref.Name)
			panic(&ParseError{varRef.pos, message})
		}
		varRef.ref.Index = info.index
	}
	for _, arrayRef := range p.arrayRefs {
		info := p.varTypes[arrayRef.funcName][arrayRef.ref.Name]
		if info.typ == typeScalar {
			message := fmt.Sprintf("can't use scalar %q as array", arrayRef.ref.Name)
			panic(&ParseError{arrayRef.pos, message})
		}
		arrayRef.ref.Index = info.index
	}
}

func (p *parser) multiExpr(exprs []Expr, pos Position) Expr {
	expr := &MultiExpr{exprs}
	p.multiExprs[expr] = pos
	return expr
}

func (p *parser) useMultiExpr(expr *MultiExpr) {
	delete(p.multiExprs, expr)
}

func (p *parser) checkMultiExprs() {
	if len(p.multiExprs) == 0 {
		return
	}
	// Show error on first comma-separated expression
	min := Position{1000000000, 1000000000}
	for _, pos := range p.multiExprs {
		if pos.Line < min.Line || (pos.Line == min.Line && pos.Column < min.Column) {
			min = pos
		}
	}
	message := fmt.Sprintf("unexpected comma-separated expression")
	panic(&ParseError{min, message})
}
<commit_msg>Remove incorrect comment<commit_after>// Resolve function calls and variable types

package parser

import (
	"fmt"
	"sort"

	. "github.com/benhoyt/goawk/internal/ast"
	. "github.com/benhoyt/goawk/lexer"
)

type varType int

const (
	typeUnknown varType = iota
	typeScalar
	typeArray
)

// typeInfo records type information for a single variable
type typeInfo struct {
	typ      varType
	ref      *VarExpr
	scope    VarScope
	index    int
	callName string
	argIndex int
}

// Used by printVarTypes when debugTypes is turned on
func (t typeInfo) String() string {
	var typ string
	switch t.typ {
	case typeScalar:
		typ = "Scalar"
	case typeArray:
		typ = "Array"
	default:
		typ = "Unknown"
	}
	var scope string
	switch t.scope {
	case ScopeGlobal:
		scope = "Global"
	case ScopeLocal:
		scope = "Local"
	default:
		scope = "Special"
	}
	return fmt.Sprintf("typ=%s ref=%p scope=%s index=%d callName=%q argIndex=%d",
		typ, t.ref, scope, t.index, t.callName, t.argIndex)
}

// A single variable reference (normally scalar)
type varRef struct {
	funcName string
	ref      *VarExpr
	isArg    bool
	pos      Position
}

// A single array reference
type arrayRef struct {
	funcName string
	ref      *ArrayExpr
	pos      Position
}

// Initialize the resolver
func (p *parser) initResolve() {
	p.varTypes = make(map[string]map[string]typeInfo)
	p.varTypes[""] = make(map[string]typeInfo) // globals
	p.functions = make(map[string]int)
	p.arrayRef("ARGV", Position{1, 1}) // interpreter relies on ARGV being present
	p.multiExprs = make(map[*MultiExpr]Position, 3)
}

// Signal the start of a function: records the function name and
// local variables so variable references can determine scope
func (p *parser) startFunction(name string, params []string) {
	p.funcName = name
	p.varTypes[name] = make(map[string]typeInfo)
	p.locals = make(map[string]bool, len(params))
	for _, param := range params {
		p.locals[param] = true
	}
}

// Signal the end of a function
func (p *parser) stopFunction() {
	p.funcName = ""
	p.locals = nil
}

// Add function by name with given index
func (p *parser) addFunction(name string, index int) {
	p.functions[name] = index
}

// Records a call to a user function (for resolving indexes later)
type userCall struct {
	call   *UserCallExpr
	pos    Position
	inFunc string
}

// Record a user call site
func (p *parser) recordUserCall(call *UserCallExpr, pos Position) {
	p.userCalls = append(p.userCalls, userCall{call, pos, p.funcName})
}

// After parsing, resolve all user calls to their indexes. Also
// ensures functions called have actually been defined, and that
// they're not being called with too many arguments.
func (p *parser) resolveUserCalls(prog *Program) {
	for _, c := range p.userCalls {
		index, ok := p.functions[c.call.Name]
		if !ok {
			panic(&ParseError{c.pos, fmt.Sprintf("undefined function %q", c.call.Name)})
		}
		function := prog.Functions[index]
		if len(c.call.Args) > len(function.Params) {
			panic(&ParseError{c.pos, fmt.Sprintf("%q called with more arguments than declared", c.call.Name)})
		}
		c.call.Index = index
	}
}

// For arguments that are variable references, we don't know the
// type based on context, so mark the types for these as unknown.
func (p *parser) processUserCallArg(funcName string, arg Expr, index int) {
	if varExpr, ok := arg.(*VarExpr); ok {
		ref := p.varTypes[p.funcName][varExpr.Name].ref
		if ref == varExpr {
			// Only applies if this is the first reference to this
			// variable (otherwise we know the type already)
			scope := p.varTypes[p.funcName][varExpr.Name].scope
			p.varTypes[p.funcName][varExpr.Name] = typeInfo{typeUnknown, ref, scope, 0, funcName, index}
		}
		// Mark the last related varRef (the most recent one) as a
		// call argument for later error handling
		p.varRefs[len(p.varRefs)-1].isArg = true
	}
}

// Determine scope of given variable reference (and funcName if it's
// a local, otherwise empty string)
func (p *parser) getScope(name string) (VarScope, string) {
	switch {
	case p.funcName != "" && p.locals[name]:
		return ScopeLocal, p.funcName
	case SpecialVarIndex(name) > 0:
		return ScopeSpecial, ""
	default:
		return ScopeGlobal, ""
	}
}

// Record a variable (scalar) reference and return the *VarExpr (but
// VarExpr.Index won't be set till later)
func (p *parser) varRef(name string, pos Position) *VarExpr {
	scope, funcName := p.getScope(name)
	expr := &VarExpr{scope, 0, name}
	p.varRefs = append(p.varRefs, varRef{funcName, expr, false, pos})
	typ := p.varTypes[funcName][name].typ
	if typ == typeUnknown {
		p.varTypes[funcName][name] = typeInfo{typeScalar, expr, scope, 0, "", 0}
	}
	return expr
}

// Record an array reference and return the *ArrayExpr (but
// ArrayExpr.Index won't be set till later)
func (p *parser) arrayRef(name string, pos Position) *ArrayExpr {
	scope, funcName := p.getScope(name)
	if scope == ScopeSpecial {
		panic(p.error("can't use scalar %q as array", name))
	}
	expr := &ArrayExpr{scope, 0, name}
	p.arrayRefs = append(p.arrayRefs, arrayRef{funcName, expr, pos})
	typ := p.varTypes[funcName][name].typ
	if typ == typeUnknown {
		p.varTypes[funcName][name] = typeInfo{typeArray, nil, scope, 0, "", 0}
	}
	return expr
}

// Print variable type information (for debugging) on p.debugWriter
func (p *parser) printVarTypes(prog *Program) {
	fmt.Fprintf(p.debugWriter, "scalars: %v\n", prog.Scalars)
	fmt.Fprintf(p.debugWriter, "arrays: %v\n", prog.Arrays)
	funcNames := []string{}
	for funcName := range p.varTypes {
		funcNames = append(funcNames, funcName)
	}
	sort.Strings(funcNames)
	for _, funcName := range funcNames {
		if funcName != "" {
			fmt.Fprintf(p.debugWriter, "function %s\n", funcName)
		} else {
			fmt.Fprintf(p.debugWriter, "globals\n")
		}
		varNames := []string{}
		for name := range p.varTypes[funcName] {
			varNames = append(varNames, name)
		}
		sort.Strings(varNames)
		for _, name := range varNames {
			info := p.varTypes[funcName][name]
			fmt.Fprintf(p.debugWriter, "  %s: %s\n", name, info)
		}
	}
}

// If we can't finish resolving after this many iterations, give up
const maxResolveIterations = 10000

// Resolve unknown variables types and generate variable indexes and
// name-to-index mappings for interpreter
func (p *parser) resolveVars(prog *Program) {
	// First go through all unknown types and try to determine the
	// type from the parameter type in that function definition. May
	// need multiple passes depending on the order of functions. This
	// is not particularly efficient, but on realistic programs it's
	// not an issue.
	for i := 0; ; i++ {
		progressed := false
		for funcName, infos := range p.varTypes {
			for name, info := range infos {
				if info.scope == ScopeSpecial {
					continue
				}
				if info.typ == typeUnknown {
					paramName := prog.Functions[p.functions[info.callName]].Params[info.argIndex]
					typ := p.varTypes[info.callName][paramName].typ
					if typ != typeUnknown {
						info.typ = typ
						p.varTypes[funcName][name] = info
						progressed = true
					}
				}
			}
		}
		if !progressed {
			// If we didn't progress we're done (or trying again is
			// not going to help)
			break
		}
		if i >= maxResolveIterations {
			panic(p.error("too many iterations trying to resolve variable types"))
		}
	}

	// Resolve global variables (iteration order is undefined, so
	// assign indexes basically randomly)
	prog.Scalars = make(map[string]int)
	prog.Arrays = make(map[string]int)
	for name, info := range p.varTypes[""] {
		var index int
		if info.scope == ScopeSpecial {
			index = SpecialVarIndex(name)
		} else if info.typ == typeArray {
			index = len(prog.Arrays)
			prog.Arrays[name] = index
		} else {
			index = len(prog.Scalars)
			prog.Scalars[name] = index
		}
		info.index = index
		p.varTypes[""][name] = info
	}

	// Resolve local variables (assign indexes in order of params).
	// Also patch up Function.Arrays (tells interpreter which args
	// are arrays).
	for funcName, infos := range p.varTypes {
		if funcName == "" {
			continue
		}
		scalarIndex := 0
		arrayIndex := 0
		functionIndex := p.functions[funcName]
		function := prog.Functions[functionIndex]
		arrays := make([]bool, len(function.Params))
		for i, name := range function.Params {
			info := infos[name]
			var index int
			if info.typ == typeArray {
				index = arrayIndex
				arrayIndex++
				arrays[i] = true
			} else {
				// typeScalar or typeUnknown: variables may still be
				// of unknown type if they've never been referenced --
				// default to scalar in that case
				index = scalarIndex
				scalarIndex++
			}
			info.index = index
			p.varTypes[funcName][name] = info
		}
		prog.Functions[functionIndex].Arrays = arrays
	}

	// Check that variables passed to functions are the correct type
	for _, c := range p.userCalls {
		function := prog.Functions[c.call.Index]
		for i, arg := range c.call.Args {
			varExpr, ok := arg.(*VarExpr)
			if !ok {
				if function.Arrays[i] {
					message := fmt.Sprintf("can't pass scalar %s as array param", arg)
					panic(&ParseError{c.pos, message})
				}
				continue
			}
			// If this VarExpr refers to a local, use caller name to
			// look up in varTypes, otherwise use "" (meaning global).
			funcName := ""
			for _, p := range prog.Functions[p.functions[c.inFunc]].Params {
				if varExpr.Name == p {
					funcName = c.inFunc
					break
				}
			}
			info := p.varTypes[funcName][varExpr.Name]
			if info.typ == typeArray && !function.Arrays[i] {
				message := fmt.Sprintf("can't pass array %q as scalar param", varExpr.Name)
				panic(&ParseError{c.pos, message})
			}
			if info.typ != typeArray && function.Arrays[i] {
				message := fmt.Sprintf("can't pass scalar %q as array param", varExpr.Name)
				panic(&ParseError{c.pos, message})
			}
		}
	}

	if p.debugTypes {
		p.printVarTypes(prog)
	}

	// Patch up variable indexes (interpreter uses an index instead
	// the name for more efficient lookups)
	for _, varRef := range p.varRefs {
		info := p.varTypes[varRef.funcName][varRef.ref.Name]
		if info.typ == typeArray && !varRef.isArg {
			message := fmt.Sprintf("can't use array %q as scalar", varRef.ref.Name)
			panic(&ParseError{varRef.pos, message})
		}
		varRef.ref.Index = info.index
	}
	for _, arrayRef := range p.arrayRefs {
		info := p.varTypes[arrayRef.funcName][arrayRef.ref.Name]
		if info.typ == typeScalar {
			message := fmt.Sprintf("can't use scalar %q as array", arrayRef.ref.Name)
			panic(&ParseError{arrayRef.pos, message})
		}
		arrayRef.ref.Index = info.index
	}
}

func (p *parser) multiExpr(exprs []Expr, pos Position) Expr {
	expr := &MultiExpr{exprs}
	p.multiExprs[expr] = pos
	return expr
}

func (p *parser) useMultiExpr(expr *MultiExpr) {
	delete(p.multiExprs, expr)
}

func (p *parser) checkMultiExprs() {
	if len(p.multiExprs) == 0 {
		return
	}
	// Show error on first comma-separated expression
	min := Position{1000000000, 1000000000}
	for _, pos := range p.multiExprs {
		if pos.Line < min.Line || (pos.Line == min.Line && pos.Column < min.Column) {
			min = pos
		}
	}
	message := fmt.Sprintf("unexpected comma-separated expression")
	panic(&ParseError{min, message})
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pathtools

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/google/blueprint/deptools"
)

var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **")
var GlobLastRecursiveErr = errors.New("pattern ** as last path element")

// Glob returns the list of files that match the given pattern but do not match
// the given exclude patterns, along with the list of directories and other
// dependencies that were searched to construct the file list.  The supported
// glob and exclude patterns are equivalent to filepath.Glob, with an extension
// that recursive glob (** matching zero or more complete path entries) is
// supported.  Glob also returns a list of directories that were searched.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func Glob(pattern string, excludes []string) (matches, deps []string, err error) {
	return startGlob(OsFs, pattern, excludes)
}

func startGlob(fs FileSystem, pattern string, excludes []string) (matches, deps []string, err error) {
	if filepath.Base(pattern) == "**" {
		return nil, nil, GlobLastRecursiveErr
	} else {
		matches, deps, err = glob(fs, pattern, false)
	}

	if err != nil {
		return nil, nil, err
	}

	matches, err = filterExcludes(matches, excludes)
	if err != nil {
		return nil, nil, err
	}

	// If the pattern has wildcards, we added dependencies on the
	// containing directories to know about changes.
	//
	// If the pattern didn't have wildcards, and didn't find matches, the
	// most specific found directories were added.
	//
	// But if it didn't have wildcards, and did find a match, no
	// dependencies were added, so add the match itself to detect when it
	// is removed.
	if !isWild(pattern) {
		deps = append(deps, matches...)
	}

	return matches, deps, nil
}

// glob is a recursive helper function to handle globbing each level of the pattern individually,
// allowing searched directories to be tracked.  Also handles the recursive glob pattern, **.
func glob(fs FileSystem, pattern string, hasRecursive bool) (matches, dirs []string, err error) {
	if !isWild(pattern) {
		// If there are no wilds in the pattern, check whether the file exists or not.
		// Uses filepath.Glob instead of manually statting to get consistent results.
		pattern = filepath.Clean(pattern)
		matches, err = fs.glob(pattern)
		if err != nil {
			return matches, dirs, err
		}

		if len(matches) == 0 {
			// Some part of the non-wild pattern didn't exist.  Add the last existing directory
			// as a dependency.
			var matchDirs []string
			for len(matchDirs) == 0 {
				pattern, _ = saneSplit(pattern)
				matchDirs, err = fs.glob(pattern)
				if err != nil {
					return matches, dirs, err
				}
			}
			dirs = append(dirs, matchDirs...)
		}
		return matches, dirs, err
	}

	dir, file := saneSplit(pattern)

	if file == "**" {
		if hasRecursive {
			return matches, dirs, GlobMultipleRecursiveErr
		}
		hasRecursive = true
	}

	dirMatches, dirs, err := glob(fs, dir, hasRecursive)
	if err != nil {
		return nil, nil, err
	}

	for _, m := range dirMatches {
		if isDir, err := fs.IsDir(m); err != nil {
			return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
		} else if isDir {
			if file == "**" {
				recurseDirs, err := fs.ListDirsRecursive(m)
				if err != nil {
					return nil, nil, err
				}
				matches = append(matches, recurseDirs...)
			} else {
				dirs = append(dirs, m)
				newMatches, err := fs.glob(filepath.Join(m, file))
				if err != nil {
					return nil, nil, err
				}
				if file[0] != '.' {
					newMatches = filterDotFiles(newMatches)
				}
				matches = append(matches, newMatches...)
			}
		}
	}

	return matches, dirs, nil
}

// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations
// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is
// not "/".  Returns ".", "" if path is "."
func saneSplit(path string) (dir, file string) {
	if path == "." {
		return ".", ""
	}
	dir, file = filepath.Split(path)
	switch dir {
	case "":
		dir = "."
	case "/":
		// Nothing
	default:
		dir = dir[:len(dir)-1]
	}
	return dir, file
}

func isWild(pattern string) bool {
	return strings.ContainsAny(pattern, "*?[")
}

// Filters the strings in matches based on the glob patterns in excludes.  Hierarchical (a/*) and
// recursive (**) glob patterns are supported.
func filterExcludes(matches []string, excludes []string) ([]string, error) {
	if len(excludes) == 0 {
		return matches, nil
	}

	var ret []string
matchLoop:
	for _, m := range matches {
		for _, e := range excludes {
			exclude, err := match(e, m)
			if err != nil {
				return nil, err
			}
			if exclude {
				continue matchLoop
			}
		}
		ret = append(ret, m)
	}

	return ret, nil
}

// filterDotFiles filters out files that start with '.'
func filterDotFiles(matches []string) []string {
	ret := make([]string, 0, len(matches))

	for _, match := range matches {
		_, name := filepath.Split(match)
		if name[0] == '.' {
			continue
		}
		ret = append(ret, match)
	}

	return ret
}

// match returns true if name matches pattern using the same rules as filepath.Match, but supporting
// hierarchical patterns (a/*) and recursive globs (**).
func match(pattern, name string) (bool, error) {
	if filepath.Base(pattern) == "**" {
		return false, GlobLastRecursiveErr
	}

	for {
		var patternFile, nameFile string
		pattern, patternFile = saneSplit(pattern)
		name, nameFile = saneSplit(name)

		if patternFile == "**" {
			return matchPrefix(pattern, filepath.Join(name, nameFile))
		}

		if nameFile == "" && patternFile == "" {
			return true, nil
		} else if nameFile == "" || patternFile == "" {
			return false, nil
		}

		match, err := filepath.Match(patternFile, nameFile)
		if err != nil || !match {
			return match, err
		}
	}
}

// matchPrefix returns true if the beginning of name matches pattern using the same rules as
// filepath.Match, but supporting hierarchical patterns (a/*).  Recursive globs (**) are not
// supported, they should have been handled in match().
func matchPrefix(pattern, name string) (bool, error) {
	if len(pattern) > 0 && pattern[0] == '/' {
		if len(name) > 0 && name[0] == '/' {
			pattern = pattern[1:]
			name = name[1:]
		} else {
			return false, nil
		}
	}

	for {
		var patternElem, nameElem string
		patternElem, pattern = saneSplitFirst(pattern)
		nameElem, name = saneSplitFirst(name)

		if patternElem == "." {
			patternElem = ""
		}
		if nameElem == "." {
			nameElem = ""
		}

		if patternElem == "**" {
			return false, GlobMultipleRecursiveErr
		}

		if patternElem == "" {
			return true, nil
		} else if nameElem == "" {
			return false, nil
		}

		match, err := filepath.Match(patternElem, nameElem)
		if err != nil || !match {
			return match, err
		}
	}
}

func saneSplitFirst(path string) (string, string) {
	i := strings.IndexRune(path, filepath.Separator)
	if i < 0 {
		return path, ""
	}
	return path[:i], path[i+1:]
}

func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) {
	var (
		matches []string
		deps    []string
	)

	globedList = make([]string, 0)
	depDirs = make([]string, 0)

	for _, pattern := range patterns {
		if isWild(pattern) {
			matches, deps, err = Glob(filepath.Join(prefix, pattern), nil)
			if err != nil {
				return nil, nil, err
			}
			globedList = append(globedList, matches...)
			depDirs = append(depDirs, deps...)
		} else {
			globedList = append(globedList, filepath.Join(prefix, pattern))
		}
	}
	return globedList, depDirs, nil
}

// IsGlob returns true if the pattern contains any glob characters (*, ?, or [).
func IsGlob(pattern string) bool {
	return strings.IndexAny(pattern, "*?[") >= 0
}

// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [).
func HasGlob(in []string) bool {
	for _, s := range in {
		if IsGlob(s) {
			return true
		}
	}

	return false
}

// GlobWithDepFile finds all files that match glob.  It compares the list of files
// against the contents of fileListFile, and rewrites fileListFile if it has changed.  It also
// writes all of the the directories it traversed as a depenencies on fileListFile to depFile.
//
// The format of glob is either path/*.ext for a single directory glob, or path/**/*.ext
// for a recursive glob.
//
// Returns a list of file paths, and an error.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) {
	files, deps, err := Glob(glob, excludes)
	if err != nil {
		return nil, err
	}

	fileList := strings.Join(files, "\n") + "\n"

	WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
	deptools.WriteDepFile(depFile, fileListFile, deps)

	return
}

// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if
// the files does not already exist with identical contents.  This can be used
// along with ninja restat rules to skip rebuilding downstream rules if no
// changes were made by a rule.
func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error {
	var isChanged bool

	dir := filepath.Dir(filename)
	err := os.MkdirAll(dir, 0777)
	if err != nil {
		return err
	}

	info, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			// The file does not exist yet.
			isChanged = true
		} else {
			return err
		}
	} else {
		if info.Size() != int64(len(data)) {
			isChanged = true
		} else {
			oldData, err := ioutil.ReadFile(filename)
			if err != nil {
				return err
			}

			if len(oldData) != len(data) {
				isChanged = true
			} else {
				for i := range data {
					if oldData[i] != data[i] {
						isChanged = true
						break
					}
				}
			}
		}
	}

	if isChanged {
		err = ioutil.WriteFile(filename, data, perm)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Export pathtools.Match<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pathtools

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/google/blueprint/deptools"
)

var GlobMultipleRecursiveErr = errors.New("pattern contains multiple **")
var GlobLastRecursiveErr = errors.New("pattern ** as last path element")

// Glob returns the list of files that match the given pattern but do not match
// the given exclude patterns, along with the list of directories and other
// dependencies that were searched to construct the file list.  The supported
// glob and exclude patterns are equivalent to filepath.Glob, with an extension
// that recursive glob (** matching zero or more complete path entries) is
// supported.  Glob also returns a list of directories that were searched.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func Glob(pattern string, excludes []string) (matches, deps []string, err error) {
	return startGlob(OsFs, pattern, excludes)
}

func startGlob(fs FileSystem, pattern string, excludes []string) (matches, deps []string, err error) {
	if filepath.Base(pattern) == "**" {
		return nil, nil, GlobLastRecursiveErr
	} else {
		matches, deps, err = glob(fs, pattern, false)
	}

	if err != nil {
		return nil, nil, err
	}

	matches, err = filterExcludes(matches, excludes)
	if err != nil {
		return nil, nil, err
	}

	// If the pattern has wildcards, we added dependencies on the
	// containing directories to know about changes.
	//
	// If the pattern didn't have wildcards, and didn't find matches, the
	// most specific found directories were added.
	//
	// But if it didn't have wildcards, and did find a match, no
	// dependencies were added, so add the match itself to detect when it
	// is removed.
	if !isWild(pattern) {
		deps = append(deps, matches...)
	}

	return matches, deps, nil
}

// glob is a recursive helper function to handle globbing each level of the pattern individually,
// allowing searched directories to be tracked.  Also handles the recursive glob pattern, **.
func glob(fs FileSystem, pattern string, hasRecursive bool) (matches, dirs []string, err error) {
	if !isWild(pattern) {
		// If there are no wilds in the pattern, check whether the file exists or not.
		// Uses filepath.Glob instead of manually statting to get consistent results.
		pattern = filepath.Clean(pattern)
		matches, err = fs.glob(pattern)
		if err != nil {
			return matches, dirs, err
		}

		if len(matches) == 0 {
			// Some part of the non-wild pattern didn't exist.  Add the last existing directory
			// as a dependency.
			var matchDirs []string
			for len(matchDirs) == 0 {
				pattern, _ = saneSplit(pattern)
				matchDirs, err = fs.glob(pattern)
				if err != nil {
					return matches, dirs, err
				}
			}
			dirs = append(dirs, matchDirs...)
		}
		return matches, dirs, err
	}

	dir, file := saneSplit(pattern)

	if file == "**" {
		if hasRecursive {
			return matches, dirs, GlobMultipleRecursiveErr
		}
		hasRecursive = true
	}

	dirMatches, dirs, err := glob(fs, dir, hasRecursive)
	if err != nil {
		return nil, nil, err
	}

	for _, m := range dirMatches {
		if isDir, err := fs.IsDir(m); err != nil {
			return nil, nil, fmt.Errorf("unexpected error after glob: %s", err)
		} else if isDir {
			if file == "**" {
				recurseDirs, err := fs.ListDirsRecursive(m)
				if err != nil {
					return nil, nil, err
				}
				matches = append(matches, recurseDirs...)
			} else {
				dirs = append(dirs, m)
				newMatches, err := fs.glob(filepath.Join(m, file))
				if err != nil {
					return nil, nil, err
				}
				if file[0] != '.' {
					newMatches = filterDotFiles(newMatches)
				}
				matches = append(matches, newMatches...)
			}
		}
	}

	return matches, dirs, nil
}

// Faster version of dir, file := filepath.Dir(path), filepath.File(path) with no allocations
// Similar to filepath.Split, but returns "." if dir is empty and trims trailing slash if dir is
// not "/".  Returns ".", "" if path is "."
func saneSplit(path string) (dir, file string) {
	if path == "." {
		return ".", ""
	}
	dir, file = filepath.Split(path)
	switch dir {
	case "":
		dir = "."
	case "/":
		// Nothing
	default:
		dir = dir[:len(dir)-1]
	}
	return dir, file
}

func isWild(pattern string) bool {
	return strings.ContainsAny(pattern, "*?[")
}

// Filters the strings in matches based on the glob patterns in excludes.  Hierarchical (a/*) and
// recursive (**) glob patterns are supported.
func filterExcludes(matches []string, excludes []string) ([]string, error) {
	if len(excludes) == 0 {
		return matches, nil
	}

	var ret []string
matchLoop:
	for _, m := range matches {
		for _, e := range excludes {
			exclude, err := Match(e, m)
			if err != nil {
				return nil, err
			}
			if exclude {
				continue matchLoop
			}
		}
		ret = append(ret, m)
	}

	return ret, nil
}

// filterDotFiles filters out files that start with '.'
func filterDotFiles(matches []string) []string {
	ret := make([]string, 0, len(matches))

	for _, match := range matches {
		_, name := filepath.Split(match)
		if name[0] == '.' {
			continue
		}
		ret = append(ret, match)
	}

	return ret
}

// Match returns true if name matches pattern using the same rules as filepath.Match, but supporting
// hierarchical patterns (a/*) and recursive globs (**).
func Match(pattern, name string) (bool, error) {
	if filepath.Base(pattern) == "**" {
		return false, GlobLastRecursiveErr
	}

	for {
		var patternFile, nameFile string
		pattern, patternFile = saneSplit(pattern)
		name, nameFile = saneSplit(name)

		if patternFile == "**" {
			return matchPrefix(pattern, filepath.Join(name, nameFile))
		}

		if nameFile == "" && patternFile == "" {
			return true, nil
		} else if nameFile == "" || patternFile == "" {
			return false, nil
		}

		match, err := filepath.Match(patternFile, nameFile)
		if err != nil || !match {
			return match, err
		}
	}
}

// matchPrefix returns true if the beginning of name matches pattern using the same rules as
// filepath.Match, but supporting hierarchical patterns (a/*).  Recursive globs (**) are not
// supported, they should have been handled in Match().
func matchPrefix(pattern, name string) (bool, error) {
	if len(pattern) > 0 && pattern[0] == '/' {
		if len(name) > 0 && name[0] == '/' {
			pattern = pattern[1:]
			name = name[1:]
		} else {
			return false, nil
		}
	}

	for {
		var patternElem, nameElem string
		patternElem, pattern = saneSplitFirst(pattern)
		nameElem, name = saneSplitFirst(name)

		if patternElem == "." {
			patternElem = ""
		}
		if nameElem == "." {
			nameElem = ""
		}

		if patternElem == "**" {
			return false, GlobMultipleRecursiveErr
		}

		if patternElem == "" {
			return true, nil
		} else if nameElem == "" {
			return false, nil
		}

		match, err := filepath.Match(patternElem, nameElem)
		if err != nil || !match {
			return match, err
		}
	}
}

func saneSplitFirst(path string) (string, string) {
	i := strings.IndexRune(path, filepath.Separator)
	if i < 0 {
		return path, ""
	}
	return path[:i], path[i+1:]
}

func GlobPatternList(patterns []string, prefix string) (globedList []string, depDirs []string, err error) {
	var (
		matches []string
		deps    []string
	)

	globedList = make([]string, 0)
	depDirs = make([]string, 0)

	for _, pattern := range patterns {
		if isWild(pattern) {
			matches, deps, err = Glob(filepath.Join(prefix, pattern), nil)
			if err != nil {
				return nil, nil, err
			}
			globedList = append(globedList, matches...)
			depDirs = append(depDirs, deps...)
		} else {
			globedList = append(globedList, filepath.Join(prefix, pattern))
		}
	}
	return globedList, depDirs, nil
}

// IsGlob returns true if the pattern contains any glob characters (*, ?, or [).
func IsGlob(pattern string) bool {
	return strings.IndexAny(pattern, "*?[") >= 0
}

// HasGlob returns true if any string in the list contains any glob characters (*, ?, or [).
func HasGlob(in []string) bool {
	for _, s := range in {
		if IsGlob(s) {
			return true
		}
	}

	return false
}

// GlobWithDepFile finds all files that match glob.  It compares the list of files
// against the contents of fileListFile, and rewrites fileListFile if it has changed.  It also
// writes all of the the directories it traversed as a depenencies on fileListFile to depFile.
//
// The format of glob is either path/*.ext for a single directory glob, or path/**/*.ext
// for a recursive glob.
//
// Returns a list of file paths, and an error.
//
// In general ModuleContext.GlobWithDeps or SingletonContext.GlobWithDeps
// should be used instead, as they will automatically set up dependencies
// to rerun the primary builder when the list of matching files changes.
func GlobWithDepFile(glob, fileListFile, depFile string, excludes []string) (files []string, err error) {
	files, deps, err := Glob(glob, excludes)
	if err != nil {
		return nil, err
	}

	fileList := strings.Join(files, "\n") + "\n"

	WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
	deptools.WriteDepFile(depFile, fileListFile, deps)

	return
}

// WriteFileIfChanged wraps ioutil.WriteFile, but only writes the file if
// the files does not already exist with identical contents.  This can be used
// along with ninja restat rules to skip rebuilding downstream rules if no
// changes were made by a rule.
func WriteFileIfChanged(filename string, data []byte, perm os.FileMode) error {
	var isChanged bool

	dir := filepath.Dir(filename)
	err := os.MkdirAll(dir, 0777)
	if err != nil {
		return err
	}

	info, err := os.Stat(filename)
	if err != nil {
		if os.IsNotExist(err) {
			// The file does not exist yet.
			isChanged = true
		} else {
			return err
		}
	} else {
		if info.Size() != int64(len(data)) {
			isChanged = true
		} else {
			oldData, err := ioutil.ReadFile(filename)
			if err != nil {
				return err
			}

			if len(oldData) != len(data) {
				isChanged = true
			} else {
				for i := range data {
					if oldData[i] != data[i] {
						isChanged = true
						break
					}
				}
			}
		}
	}

	if isChanged {
		err = ioutil.WriteFile(filename, data, perm)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package plugin

import (
	"github.com/cloudfoundry/cli/cf/command_metadata"
	"github.com/cloudfoundry/cli/cf/configuration/plugin_config"
	. "github.com/cloudfoundry/cli/cf/i18n"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/cloudfoundry/cli/plugin/rpc"
	"github.com/codegangsta/cli"
)

type Plugins struct {
	ui     terminal.UI
	config plugin_config.PluginConfiguration
}

func NewPlugins(ui terminal.UI, config plugin_config.PluginConfiguration) *Plugins {
	return &Plugins{
		ui:     ui,
		config: config,
	}
}

func (cmd *Plugins) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "plugins",
		Description: T("list all available plugin commands"),
		Usage:       T("CF_NAME plugins"),
	}
}

func (cmd *Plugins) GetRequirements(_ requirements.Factory, _ *cli.Context) (req []requirements.Requirement, err error) {
	return
}

func (cmd *Plugins) Run(c *cli.Context) {
	cmd.ui.Say(T("Listing Installed Plugins..."))

	plugins := cmd.config.Plugins()

	table := terminal.NewTable(cmd.ui, []string{T("Plugin name"), T("Command name")})

	service, err := rpc.NewRpcService()
	if err != nil {
		cmd.ui.Failed(err.Error())
	}

	err = service.Start()
	if err != nil {
		cmd.ui.Failed(err.Error())
	}
	defer service.Stop()

	for pluginName, metadata := range plugins {
		for _, command := range metadata.Commands {
			table.Add(pluginName, command.Name)
		}
	}

	cmd.ui.Ok()
	cmd.ui.Say("")

	table.Print()
}
<commit_msg>Don't use rpc to list plugins<commit_after>package plugin

import (
	"github.com/cloudfoundry/cli/cf/command_metadata"
	"github.com/cloudfoundry/cli/cf/configuration/plugin_config"
	. "github.com/cloudfoundry/cli/cf/i18n"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/codegangsta/cli"
)

type Plugins struct {
	ui     terminal.UI
	config plugin_config.PluginConfiguration
}

func NewPlugins(ui terminal.UI, config plugin_config.PluginConfiguration) *Plugins {
	return &Plugins{
		ui:     ui,
		config: config,
	}
}

func (cmd *Plugins) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "plugins",
		Description: T("list all available plugin commands"),
		Usage:       T("CF_NAME plugins"),
	}
}

func (cmd *Plugins) GetRequirements(_ requirements.Factory, _ *cli.Context) (req []requirements.Requirement, err error) {
	return
}

func (cmd *Plugins) Run(c *cli.Context) {
	cmd.ui.Say(T("Listing Installed Plugins..."))

	plugins := cmd.config.Plugins()

	table := terminal.NewTable(cmd.ui, []string{T("Plugin name"), T("Command name")})

	for pluginName, metadata := range plugins {
		for _, command := range metadata.Commands {
			table.Add(pluginName, command.Name)
		}
	}

	cmd.ui.Ok()
	cmd.ui.Say("")

	table.Print()
}
<|endoftext|>


<commit_before>package acl

import (
	"fmt"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// LoadByName loads and initialises a Network ACL from the database by project and name.
func LoadByName(s *state.State, projectName string, name string) (NetworkACL, error) {
	id, aclInfo, err := s.Cluster.GetNetworkACL(projectName, name)
	if err != nil {
		return nil, err
	}

	var acl NetworkACL = &common{} // Only a single driver currently.
	acl.init(s, id, projectName, aclInfo)

	return acl, nil
}

// Create validates supplied record and creates new Network ACL record in the database.
func Create(s *state.State, projectName string, aclInfo *api.NetworkACLsPost) error {
	var acl NetworkACL = &common{} // Only a single driver currently.
	acl.init(s, -1, projectName, nil)

	err := acl.validateName(aclInfo.Name)
	if err != nil {
		return err
	}

	err = acl.validateConfig(&aclInfo.NetworkACLPut)
	if err != nil {
		return err
	}

	// Insert DB record.
	_, err = s.Cluster.CreateNetworkACL(projectName, aclInfo)
	if err != nil {
		return err
	}

	return nil
}

// Exists checks the ACL name(s) provided exists in the project.
// If multiple names are provided, also checks that duplicate names aren't specified in the list.
func Exists(s *state.State, projectName string, name ...string) error {
	existingACLNames, err := s.Cluster.GetNetworkACLs(projectName)
	if err != nil {
		return err
	}

	checkedACLNames := make(map[string]struct{}, len(name))

	for _, aclName := range name {
		if !shared.StringInSlice(aclName, existingACLNames) {
			return fmt.Errorf("Network ACL %q does not exist", aclName)
		}

		_, found := checkedACLNames[aclName]
		if found {
			return fmt.Errorf("Network ACL %q specified multiple times", aclName)
		}

		checkedACLNames[aclName] = struct{}{}
	}

	return nil
}

// UsedBy finds all networks, profiles and instance NICs that use any of the specified ACLs and executes usageFunc
// once for each resource using one or more of the ACLs with info about the resource and matched ACLs being used.
func UsedBy(s *state.State, aclProjectName string, usageFunc func(matchedACLNames []string, usageType interface{}, nicName string, nicConfig map[string]string) error, matchACLNames ...string) error {
	if len(matchACLNames) <= 0 {
		return nil
	}

	// Find networks using the ACLs. Cheapest to do.
	networkNames, err := s.Cluster.GetCreatedNetworks(aclProjectName)
	if err != nil && err != db.ErrNoSuchObject {
		return errors.Wrapf(err, "Failed loading networks for project %q", aclProjectName)
	}

	for _, networkName := range networkNames {
		_, network, _, err := s.Cluster.GetNetworkInAnyState(aclProjectName, networkName)
		if err != nil {
			return errors.Wrapf(err, "Failed to get network config for %q", networkName)
		}

		netACLNames := util.SplitNTrimSpace(network.Config["security.acls"], ",", -1, true)
		matchedACLNames := []string{}
		for _, netACLName := range netACLNames {
			if shared.StringInSlice(netACLName, matchACLNames) {
				matchedACLNames = append(matchedACLNames, netACLName)
			}
		}

		if len(matchedACLNames) > 0 {
			// Call usageFunc with a list of matched ACLs and info about the network.
			err := usageFunc(matchedACLNames, network, "", nil)
			if err != nil {
				return err
			}
		}
	}

	// Look for profiles. Next cheapest to do.
	var profiles []db.Profile
	err = s.Cluster.Transaction(func(tx *db.ClusterTx) error {
		profiles, err = tx.GetProfiles(db.ProfileFilter{})
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	for _, profile := range profiles {
		// Get the profiles's effective network project name.
		profileNetworkProjectName, _, err := project.NetworkProject(s.Cluster, profile.Project)
		if err != nil {
			return err
		}

		// Skip profiles who's effective network project doesn't match this Network ACL's project.
		if profileNetworkProjectName != aclProjectName {
			continue
		}

		// Iterate through each of the instance's devices, looking for NICs that are using any of the ACLs.
		for devName, devConfig := range deviceConfig.NewDevices(profile.Devices) {
			matchedACLNames := isInUseByDevice(devConfig, matchACLNames...)
			if len(matchedACLNames) > 0 {
				// Call usageFunc with a list of matched ACLs and info about the instance NIC.
				err := usageFunc(matchedACLNames, profile, devName, devConfig)
				if err != nil {
					return err
				}
			}
		}
	}

	// Find ACLs that have rules that reference the ACLs.
	aclNames, err := s.Cluster.GetNetworkACLs(aclProjectName)
	if err != nil {
		return err
	}

	for _, aclName := range aclNames {
		_, aclInfo, err := s.Cluster.GetNetworkACL(aclProjectName, aclName)
		if err != nil {
			return err
		}

		matchedACLNames := []string{}

		// Ingress rules can specify ACL names in their Source subjects.
		for _, rule := range aclInfo.Ingress {
			for _, subject := range util.SplitNTrimSpace(rule.Source, ",", -1, true) {

				// Look for new matching ACLs, but ignore our own ACL reference in our own rules.
				if shared.StringInSlice(subject, matchACLNames) && !shared.StringInSlice(subject, matchedACLNames) && subject != aclInfo.Name {
					matchedACLNames = append(matchedACLNames, subject)
				}
			}
		}

		// Egress rules can specify ACL names in their Destination subjects.
		for _, rule := range aclInfo.Egress {
			for _, subject := range util.SplitNTrimSpace(rule.Destination, ",", -1, true) {

				// Look for new matching ACLs, but ignore our own ACL reference in our own rules.
				if shared.StringInSlice(subject, matchACLNames) && !shared.StringInSlice(subject, matchedACLNames) && subject != aclInfo.Name {
					matchedACLNames = append(matchedACLNames, subject)
				}
			}
		}

		if len(matchedACLNames) > 0 {
			// Call usageFunc with a list of matched ACLs and info about the ACL.
			err := usageFunc(matchedACLNames, aclInfo, "", nil)
			if err != nil {
				return err
			}
		}
	}

	// Find instances using the ACLs. Most expensive to do.
	err = s.Cluster.InstanceList(nil, func(inst db.Instance, p api.Project, profiles []api.Profile) error {
		// Get the instance's effective network project name.
		instNetworkProject := project.NetworkProjectFromRecord(&p)

		// Skip instances who's effective network project doesn't match this Network ACL's project.
		if instNetworkProject != aclProjectName {
			return nil
		}

		devices := db.ExpandInstanceDevices(deviceConfig.NewDevices(inst.Devices), profiles)

		// Iterate through each of the instance's devices, looking for NICs that are using any of the ACLs.
		for devName, devConfig := range devices {
			matchedACLNames := isInUseByDevice(devConfig, matchACLNames...)
			if len(matchedACLNames) > 0 {
				// Call usageFunc with a list of matched ACLs and info about the instance NIC.
				err := usageFunc(matchedACLNames, inst, devName, devConfig)
				if err != nil {
					return err
				}
			}
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

// isInUseByDevice returns any of the supplied matching ACL names found referenced by the NIC device.
func isInUseByDevice(d deviceConfig.Device, matchACLNames ...string) []string {
	matchedACLNames := []string{}

	// Only NICs linked to managed networks can use network ACLs.
	if d["type"] != "nic" || d["network"] == "" {
		return matchedACLNames
	}

	for _, nicACLName := range util.SplitNTrimSpace(d["security.acls"], ",", -1, true) {
		if shared.StringInSlice(nicACLName, matchACLNames) {
			matchedACLNames = append(matchedACLNames, nicACLName)
		}
	}

	return matchedACLNames
}

// NetworkACLUsage info about a network and what ACL it uses.
type NetworkACLUsage struct {
	ID     int64
	Name   string
	Type   string
	Config map[string]string
}

// NetworkUsage populates the provided aclNets map with networks that are using any of the specified ACLs.
func NetworkUsage(s *state.State, aclProjectName string, aclNames []string, aclNets map[string]NetworkACLUsage) error {
	// Find all networks and instance/profile NICs that use any of the specified Network ACLs.
	err := UsedBy(s, aclProjectName, func(matchedACLNames []string, usageType interface{}, _ string, nicConfig map[string]string) error {
		switch u := usageType.(type) {
		case db.Instance, db.Profile:
			networkID, network, _, err := s.Cluster.GetNetworkInAnyState(aclProjectName, nicConfig["network"])
			if err != nil {
				return errors.Wrapf(err, "Failed to load network %q", nicConfig["network"])
			}

			if network.Type == "ovn" {
				if _, found := aclNets[network.Name]; !found {
					aclNets[network.Name] = NetworkACLUsage{
						ID:   networkID,
						Name: network.Name,
						Type: network.Type,
					}
				}
			}
		case *api.Network:
			if u.Type == "ovn" {
				if _, found := aclNets[u.Name]; !found {
					networkID, network, _, err := s.Cluster.GetNetworkInAnyState(aclProjectName, u.Name)
					if err != nil {
						return errors.Wrapf(err, "Failed to load network %q", u.Name)
					}

					aclNets[u.Name] = NetworkACLUsage{
						ID:   networkID,
						Name: network.Name,
						Type: network.Type,
					}
				}
			}
		case *api.NetworkACL:
			return nil // Nothing to do for ACL rules referencing us.
		default:
			return fmt.Errorf("Unrecognised usage type %T", u)
		}

		return nil
	}, aclNames...)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/network/acl/acl/load: Update NetworkUsage to return bridge and ovn networks<commit_after>package acl

import (
	"fmt"

	"github.com/pkg/errors"

	"github.com/lxc/lxd/lxd/db"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

// LoadByName loads and initialises a Network ACL from the database by project and name.
func LoadByName(s *state.State, projectName string, name string) (NetworkACL, error) {
	id, aclInfo, err := s.Cluster.GetNetworkACL(projectName, name)
	if err != nil {
		return nil, err
	}

	var acl NetworkACL = &common{} // Only a single driver currently.
	acl.init(s, id, projectName, aclInfo)

	return acl, nil
}

// Create validates supplied record and creates new Network ACL record in the database.
func Create(s *state.State, projectName string, aclInfo *api.NetworkACLsPost) error {
	var acl NetworkACL = &common{} // Only a single driver currently.
	acl.init(s, -1, projectName, nil)

	err := acl.validateName(aclInfo.Name)
	if err != nil {
		return err
	}

	err = acl.validateConfig(&aclInfo.NetworkACLPut)
	if err != nil {
		return err
	}

	// Insert DB record.
	_, err = s.Cluster.CreateNetworkACL(projectName, aclInfo)
	if err != nil {
		return err
	}

	return nil
}

// Exists checks the ACL name(s) provided exists in the project.
// If multiple names are provided, also checks that duplicate names aren't specified in the list.
func Exists(s *state.State, projectName string, name ...string) error {
	existingACLNames, err := s.Cluster.GetNetworkACLs(projectName)
	if err != nil {
		return err
	}

	checkedACLNames := make(map[string]struct{}, len(name))

	for _, aclName := range name {
		if !shared.StringInSlice(aclName, existingACLNames) {
			return fmt.Errorf("Network ACL %q does not exist", aclName)
		}

		_, found := checkedACLNames[aclName]
		if found {
			return fmt.Errorf("Network ACL %q specified multiple times", aclName)
		}

		checkedACLNames[aclName] = struct{}{}
	}

	return nil
}

// UsedBy finds all networks, profiles and instance NICs that use any of the specified ACLs and executes usageFunc
// once for each resource using one or more of the ACLs with info about the resource and matched ACLs being used.
func UsedBy(s *state.State, aclProjectName string, usageFunc func(matchedACLNames []string, usageType interface{}, nicName string, nicConfig map[string]string) error, matchACLNames ...string) error {
	if len(matchACLNames) <= 0 {
		return nil
	}

	// Find networks using the ACLs. Cheapest to do.
	networkNames, err := s.Cluster.GetCreatedNetworks(aclProjectName)
	if err != nil && err != db.ErrNoSuchObject {
		return errors.Wrapf(err, "Failed loading networks for project %q", aclProjectName)
	}

	for _, networkName := range networkNames {
		_, network, _, err := s.Cluster.GetNetworkInAnyState(aclProjectName, networkName)
		if err != nil {
			return errors.Wrapf(err, "Failed to get network config for %q", networkName)
		}

		netACLNames := util.SplitNTrimSpace(network.Config["security.acls"], ",", -1, true)
		matchedACLNames := []string{}
		for _, netACLName := range netACLNames {
			if shared.StringInSlice(netACLName, matchACLNames) {
				matchedACLNames = append(matchedACLNames, netACLName)
			}
		}

		if len(matchedACLNames) > 0 {
			// Call usageFunc with a list of matched ACLs and info about the network.
			err := usageFunc(matchedACLNames, network, "", nil)
			if err != nil {
				return err
			}
		}
	}

	// Look for profiles. Next cheapest to do.
	var profiles []db.Profile
	err = s.Cluster.Transaction(func(tx *db.ClusterTx) error {
		profiles, err = tx.GetProfiles(db.ProfileFilter{})
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return err
	}

	for _, profile := range profiles {
		// Get the profiles's effective network project name.
		profileNetworkProjectName, _, err := project.NetworkProject(s.Cluster, profile.Project)
		if err != nil {
			return err
		}

		// Skip profiles who's effective network project doesn't match this Network ACL's project.
		if profileNetworkProjectName != aclProjectName {
			continue
		}

		// Iterate through each of the instance's devices, looking for NICs that are using any of the ACLs.
		for devName, devConfig := range deviceConfig.NewDevices(profile.Devices) {
			matchedACLNames := isInUseByDevice(devConfig, matchACLNames...)
			if len(matchedACLNames) > 0 {
				// Call usageFunc with a list of matched ACLs and info about the instance NIC.
				err := usageFunc(matchedACLNames, profile, devName, devConfig)
				if err != nil {
					return err
				}
			}
		}
	}

	// Find ACLs that have rules that reference the ACLs.
	aclNames, err := s.Cluster.GetNetworkACLs(aclProjectName)
	if err != nil {
		return err
	}

	for _, aclName := range aclNames {
		_, aclInfo, err := s.Cluster.GetNetworkACL(aclProjectName, aclName)
		if err != nil {
			return err
		}

		matchedACLNames := []string{}

		// Ingress rules can specify ACL names in their Source subjects.
		for _, rule := range aclInfo.Ingress {
			for _, subject := range util.SplitNTrimSpace(rule.Source, ",", -1, true) {

				// Look for new matching ACLs, but ignore our own ACL reference in our own rules.
				if shared.StringInSlice(subject, matchACLNames) && !shared.StringInSlice(subject, matchedACLNames) && subject != aclInfo.Name {
					matchedACLNames = append(matchedACLNames, subject)
				}
			}
		}

		// Egress rules can specify ACL names in their Destination subjects.
		for _, rule := range aclInfo.Egress {
			for _, subject := range util.SplitNTrimSpace(rule.Destination, ",", -1, true) {

				// Look for new matching ACLs, but ignore our own ACL reference in our own rules.
				if shared.StringInSlice(subject, matchACLNames) && !shared.StringInSlice(subject, matchedACLNames) && subject != aclInfo.Name {
					matchedACLNames = append(matchedACLNames, subject)
				}
			}
		}

		if len(matchedACLNames) > 0 {
			// Call usageFunc with a list of matched ACLs and info about the ACL.
			err := usageFunc(matchedACLNames, aclInfo, "", nil)
			if err != nil {
				return err
			}
		}
	}

	// Find instances using the ACLs. Most expensive to do.
	err = s.Cluster.InstanceList(nil, func(inst db.Instance, p api.Project, profiles []api.Profile) error {
		// Get the instance's effective network project name.
		instNetworkProject := project.NetworkProjectFromRecord(&p)

		// Skip instances who's effective network project doesn't match this Network ACL's project.
		if instNetworkProject != aclProjectName {
			return nil
		}

		devices := db.ExpandInstanceDevices(deviceConfig.NewDevices(inst.Devices), profiles)

		// Iterate through each of the instance's devices, looking for NICs that are using any of the ACLs.
		for devName, devConfig := range devices {
			matchedACLNames := isInUseByDevice(devConfig, matchACLNames...)
			if len(matchedACLNames) > 0 {
				// Call usageFunc with a list of matched ACLs and info about the instance NIC.
				err := usageFunc(matchedACLNames, inst, devName, devConfig)
				if err != nil {
					return err
				}
			}
		}

		return nil
	})
	if err != nil {
		return err
	}

	return nil
}

// isInUseByDevice returns any of the supplied matching ACL names found referenced by the NIC device.
func isInUseByDevice(d deviceConfig.Device, matchACLNames ...string) []string {
	matchedACLNames := []string{}

	// Only NICs linked to managed networks can use network ACLs.
	if d["type"] != "nic" || d["network"] == "" {
		return matchedACLNames
	}

	for _, nicACLName := range util.SplitNTrimSpace(d["security.acls"], ",", -1, true) {
		if shared.StringInSlice(nicACLName, matchACLNames) {
			matchedACLNames = append(matchedACLNames, nicACLName)
		}
	}

	return matchedACLNames
}

// NetworkACLUsage info about a network and what ACL it uses.
type NetworkACLUsage struct {
	ID     int64
	Name   string
	Type   string
	Config map[string]string
}

// NetworkUsage populates the provided aclNets map with networks that are using any of the specified ACLs.
func NetworkUsage(s *state.State, aclProjectName string, aclNames []string, aclNets map[string]NetworkACLUsage) error {
	supportedNetTypes := []string{"bridge", "ovn"}

	// Find all networks and instance/profile NICs that use any of the specified Network ACLs.
	err := UsedBy(s, aclProjectName, func(matchedACLNames []string, usageType interface{}, _ string, nicConfig map[string]string) error {
		switch u := usageType.(type) {
		case db.Instance, db.Profile:
			networkID, network, _, err := s.Cluster.GetNetworkInAnyState(aclProjectName, nicConfig["network"])
			if err != nil {
				return errors.Wrapf(err, "Failed to load network %q", nicConfig["network"])
			}

			if shared.StringInSlice(network.Type, supportedNetTypes) {
				if _, found := aclNets[network.Name]; !found {
					aclNets[network.Name] = NetworkACLUsage{
						ID:     networkID,
						Name:   network.Name,
						Type:   network.Type,
						Config: network.Config,
					}
				}
			}
		case *api.Network:
			if shared.StringInSlice(u.Type, supportedNetTypes) {
				if _, found := aclNets[u.Name]; !found {
					networkID, network, _, err := s.Cluster.GetNetworkInAnyState(aclProjectName, u.Name)
					if err != nil {
						return errors.Wrapf(err, "Failed to load network %q", u.Name)
					}

					aclNets[u.Name] = NetworkACLUsage{
						ID:     networkID,
						Name:   network.Name,
						Type:   network.Type,
						Config: network.Config,
					}
				}
			}
		case *api.NetworkACL:
			return nil // Nothing to do for ACL rules referencing us.
		default:
			return fmt.Errorf("Unrecognised usage type %T", u)
		}

		return nil
	}, aclNames...)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package transactions

import (
	"errors"
	"time"

	"github.com/joshheinrichs/geosource/server/types"
)

type PostQuery struct {
	Flags struct {
		// If true, all posts will be searched, regardless of the channels and
		// other flags that were set in the query.
		All bool `url:"all"`
		// If true, posts that were created by the user will be included in
		// the search results.
		Mine bool `url:"mine"`
		// If true, posts that were favorited by the user will be included in
		// the search results.
		Favorites bool `url:"favorites"`
	} `url:"flags"`
	Time struct {
		Start time.Time `url:"start"`
		End   time.Time `url:"end"`
	} `url:"time"`
	Location struct {
		// The upper left bound of the region
		Start types.Location `url:"start"`
		// The lower right bound of the region
		End types.Location `url:"end"`
	}
	// The set of channels to query. If the all flag is true, this field is
	// ignored.
	Channels []string `url:"channels"`
}

func IsPostCreator(requester, userID, postID string) (bool, error) {
	return false, errors.New("function has not yet been implemented.")
}

func AddPost(requester string, post *types.Post) error {
	return db.Create(post).Error
}

func GetPosts(requester string) ([]*types.PersonalizedPostInfo, error) {
	var posts []*types.PersonalizedPostInfo
	err := db.Table("posts").
		Joins("LEFT JOIN user_favorites ON (p_postid = uf_postid)").
		Joins("LEFT JOIN users ON (u_userid = p_userid_creator)").
		Select("*, (uf_postid IS NOT NULL) AS favorited").
		Order("p_time desc").Find(&posts).Error
	if err != nil {
		return nil, err
	}
	return posts, nil
}

func GetPost(requester, postID string) (*types.Post, error) {
	var post types.Post
	err := db.Where("p_postid = ?", postID).First(&post).Error
	if err != nil {
		return nil, err
	}
	return &post, nil
}

func RemovePost(requester, postID string) error {
	return errors.New("function has not yet been implemented.")
}
<commit_msg>Fix posts query to properly personalize for requester<commit_after>package transactions

import (
	"errors"
	"time"

	"github.com/joshheinrichs/geosource/server/types"
)

type PostQuery struct {
	Flags struct {
		// If true, all posts will be searched, regardless of the channels and
		// other flags that were set in the query.
		All bool `url:"all"`
		// If true, posts that were created by the user will be included in
		// the search results.
		Mine bool `url:"mine"`
		// If true, posts that were favorited by the user will be included in
		// the search results.
		Favorites bool `url:"favorites"`
	} `url:"flags"`
	Time struct {
		Start time.Time `url:"start"`
		End   time.Time `url:"end"`
	} `url:"time"`
	Location struct {
		// The upper left bound of the region
		Start types.Location `url:"start"`
		// The lower right bound of the region
		End types.Location `url:"end"`
	}
	// The set of channels to query. If the all flag is true, this field is
	// ignored.
	Channels []string `url:"channels"`
}

func IsPostCreator(requester, userID, postID string) (bool, error) {
	return false, errors.New("function has not yet been implemented.")
}

func AddPost(requester string, post *types.Post) error {
	return db.Create(post).Error
}

func GetPosts(requester string) ([]*types.PersonalizedPostInfo, error) {
	var posts []*types.PersonalizedPostInfo
	err := db.Table("posts").
		Joins("LEFT JOIN user_favorites ON (p_postid = uf_postid AND uf_userid = ?)", requester).
		Joins("LEFT JOIN users ON (u_userid = p_userid_creator)").
		Select("*, (uf_postid IS NOT NULL) AS favorited").
		Order("p_time desc").Find(&posts).Error
	if err != nil {
		return nil, err
	}
	return posts, nil
}

func GetPost(requester, postID string) (*types.Post, error) {
	var post types.Post
	err := db.Where("p_postid = ?", postID).First(&post).Error
	if err != nil {
		return nil, err
	}
	return &post, nil
}

func RemovePost(requester, postID string) error {
	return errors.New("function has not yet been implemented.")
}
<|endoftext|>


<commit_before>package workflow

import (
	"archive/tar"
	"bytes"
	"context"
	"fmt"
	"net/http"
	"path/filepath"
	"time"

	"github.com/fsamin/go-dump"
	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/keys"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/services"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// WorkflowAsCodePattern is the default code pattern to find cds files
const WorkflowAsCodePattern = ".cds/**/*.yml"

// PushOption is the set of options for workflow push
type PushOption struct {
	VCSServer          string
	FromRepository     string
	Branch             string
	IsDefaultBranch    bool
	RepositoryName     string
	RepositoryStrategy sdk.RepositoryStrategy
	DryRun             bool
	HookUUID           string
}

// CreateFromRepository a workflow from a repository
func CreateFromRepository(ctx context.Context, db *gorp.DbMap, store cache.Store, p *sdk.Project, w *sdk.Workflow, opts sdk.WorkflowRunPostHandlerOption, u *sdk.User, decryptFunc keys.DecryptFunc) ([]sdk.Message, error) {
	ctx, end := observability.Span(ctx, "workflow.CreateFromRepository")
	defer end()

	ope, err := createOperationRequest(*w, opts)
	if err != nil {
		return nil, sdk.WrapError(err, "Unable to create operation request")
	}

	// update user permission if  we come from hook
	if opts.Hook != nil {
		u.Groups = make([]sdk.Group, len(p.ProjectGroups))
		for i, gp := range p.ProjectGroups {
			u.Groups[i] = gp.Group
		}
	}

	if err := PostRepositoryOperation(ctx, db, *p, &ope, nil); err != nil {
		return nil, sdk.WrapError(err, "Unable to post repository operation")
	}

	if err := pollRepositoryOperation(ctx, db, store, &ope); err != nil {
		return nil, sdk.WrapError(err, "Cannot analyse repository")
	}

	var uuid string
	if opts.Hook != nil {
		uuid = opts.Hook.WorkflowNodeHookUUID
	} else {
		// Search for repo web hook uuid
		for _, h := range w.WorkflowData.Node.Hooks {
			if h.HookModelName == sdk.RepositoryWebHookModelName {
				uuid = h.UUID
				break
			}
		}
	}
	return extractWorkflow(ctx, db, store, p, w, ope, u, decryptFunc, uuid)
}

func extractWorkflow(ctx context.Context, db *gorp.DbMap, store cache.Store, p *sdk.Project, w *sdk.Workflow, ope sdk.Operation, u *sdk.User, decryptFunc keys.DecryptFunc, hookUUID string) ([]sdk.Message, error) {
	ctx, end := observability.Span(ctx, "workflow.extractWorkflow")
	defer end()

	// Read files
	tr, err := ReadCDSFiles(ope.LoadFiles.Results)
	if err != nil {
		return nil, sdk.WrapError(err, "Unable to read cds files")
	}
	ope.RepositoryStrategy.SSHKeyContent = ""
	opt := &PushOption{
		VCSServer:          ope.VCSServer,
		RepositoryName:     ope.RepoFullName,
		RepositoryStrategy: ope.RepositoryStrategy,
		Branch:             ope.Setup.Checkout.Branch,
		FromRepository:     ope.RepositoryInfo.FetchURL,
		IsDefaultBranch:    ope.Setup.Checkout.Branch == ope.RepositoryInfo.DefaultBranch,
		DryRun:             true,
		HookUUID:           hookUUID,
	}

	allMsg, workflowPushed, errP := Push(ctx, db, store, p, tr, opt, u, decryptFunc)
	if errP != nil {
		return nil, sdk.WrapError(errP, "extractWorkflow> Unable to get workflow from file")
	}
	*w = *workflowPushed
	return allMsg, nil
}

// ReadCDSFiles reads CDS files
func ReadCDSFiles(files map[string][]byte) (*tar.Reader, error) {
	// Create a buffer to write our archive to.
	buf := new(bytes.Buffer)
	// Create a new tar archive.
	tw := tar.NewWriter(buf)
	// Add some files to the archive.
	for fname, fcontent := range files {
		log.Debug("ReadCDSFiles> Reading %s", fname)
		hdr := &tar.Header{
			Name: filepath.Base(fname),
			Mode: 0600,
			Size: int64(len(fcontent)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return nil, sdk.WrapError(err, "Cannot write header")
		}
		if n, err := tw.Write(fcontent); err != nil {
			return nil, sdk.WrapError(err, "Cannot write content")
		} else if n == 0 {
			return nil, fmt.Errorf("nothing to write")
		}
	}
	// Make sure to check the error on Close.
	if err := tw.Close(); err != nil {
		return nil, err
	}

	return tar.NewReader(buf), nil
}

func pollRepositoryOperation(c context.Context, db gorp.SqlExecutor, store cache.Store, ope *sdk.Operation) error {
	tickTimeout := time.NewTicker(10 * time.Minute)
	tickPoll := time.NewTicker(2 * time.Second)
	defer tickTimeout.Stop()
	for {
		select {
		case <-c.Done():
			if c.Err() != nil {
				return sdk.WrapError(c.Err(), "pollRepositoryOperation> Exiting")
			}
		case <-tickTimeout.C:
			return sdk.WrapError(sdk.ErrRepoOperationTimeout, "pollRepositoryOperation> Timeout analyzing repository")
		case <-tickPoll.C:
			if err := GetRepositoryOperation(c, db, ope); err != nil {
				return sdk.WrapError(err, "Cannot get repository operation status")
			}
			switch ope.Status {
			case sdk.OperationStatusError:
				return sdk.WrapError(fmt.Errorf("%s", ope.Error), "getImportAsCodeHandler> Operation in error. %+v", ope)
			case sdk.OperationStatusDone:
				return nil
			}
			continue
		}
	}
}

func createOperationRequest(w sdk.Workflow, opts sdk.WorkflowRunPostHandlerOption) (sdk.Operation, error) {
	ope := sdk.Operation{}
	if w.Root.Context.Application == nil {
		return ope, sdk.WrapError(sdk.ErrApplicationNotFound, "CreateFromRepository> Workflow node root does not have a application context")
	}
	ope = sdk.Operation{
		VCSServer:          w.Root.Context.Application.VCSServer,
		RepoFullName:       w.Root.Context.Application.RepositoryFullname,
		URL:                w.FromRepository,
		RepositoryStrategy: w.Root.Context.Application.RepositoryStrategy,
		Setup: sdk.OperationSetup{
			Checkout: sdk.OperationCheckout{
				Branch: "",
				Commit: "",
			},
		},
		LoadFiles: sdk.OperationLoadFiles{
			Pattern: WorkflowAsCodePattern,
		},
	}

	var branch, commit string
	if opts.Hook != nil {
		branch = opts.Hook.Payload[tagGitBranch]
		commit = opts.Hook.Payload[tagGitHash]
	}
	if opts.Manual != nil {
		e := dump.NewDefaultEncoder(new(bytes.Buffer))
		e.Formatters = []dump.KeyFormatterFunc{dump.WithDefaultLowerCaseFormatter()}
		e.ExtraFields.DetailedMap = false
		e.ExtraFields.DetailedStruct = false
		e.ExtraFields.Len = false
		e.ExtraFields.Type = false
		m1, errm1 := e.ToStringMap(opts.Manual.Payload)
		if errm1 != nil {
			return ope, sdk.WrapError(errm1, "CreateFromRepository> Unable to compute payload")
		}
		branch = m1[tagGitBranch]
		commit = m1[tagGitHash]
	}
	ope.Setup.Checkout.Commit = commit
	ope.Setup.Checkout.Branch = branch

	// This should not append because the hook must set a default payload with git.branch
	if ope.Setup.Checkout.Branch == "" {
		return ope, sdk.WrapError(sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("branch parameter is mandatory")), "createOperationRequest")
	}

	return ope, nil
}

// PostRepositoryOperation creates a new repository operation
func PostRepositoryOperation(ctx context.Context, db gorp.SqlExecutor, prj sdk.Project, ope *sdk.Operation, multipartData *services.MultiPartData) error {
	srvs, err := services.FindByType(db, services.TypeRepositories)
	if err != nil {
		return sdk.WrapError(err, "Unable to found repositories service")
	}

	if ope.RepositoryStrategy.ConnectionType == "ssh" {
		for _, k := range prj.Keys {
			if k.Name == ope.RepositoryStrategy.SSHKey {
				ope.RepositoryStrategy.SSHKeyContent = k.Private
				break
			}
		}
	}

	if multipartData == nil {
		if _, err := services.DoJSONRequest(ctx, srvs, http.MethodPost, "/operations", ope, ope); err != nil {
			return sdk.WrapError(err, "Unable to perform operation")
		}
		return nil
	}
	if _, err := services.DoMultiPartRequest(ctx, srvs, http.MethodPost, "/operations", multipartData, ope, ope); err != nil {
		return sdk.WrapError(err, "Unable to perform multipart operation")
	}
	return nil
}

// GetRepositoryOperation get repository operation status
func GetRepositoryOperation(ctx context.Context, db gorp.SqlExecutor, ope *sdk.Operation) error {
	srvs, err := services.FindByType(db, services.TypeRepositories)
	if err != nil {
		return sdk.WrapError(err, "Unable to found repositories service")
	}

	if _, err := services.DoJSONRequest(ctx, srvs, http.MethodGet, "/operations/"+ope.UUID, nil, ope); err != nil {
		return sdk.WrapError(err, "Unable to get operation")
	}
	return nil
}
<commit_msg>fix(repository): display trusted operation in error logs (#3916)<commit_after>package workflow

import (
	"archive/tar"
	"bytes"
	"context"
	"fmt"
	"net/http"
	"path/filepath"
	"time"

	"github.com/fsamin/go-dump"
	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/keys"
	"github.com/ovh/cds/engine/api/observability"
	"github.com/ovh/cds/engine/api/services"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// WorkflowAsCodePattern is the default code pattern to find cds files
const WorkflowAsCodePattern = ".cds/**/*.yml"

// PushOption is the set of options for workflow push
type PushOption struct {
	VCSServer          string
	FromRepository     string
	Branch             string
	IsDefaultBranch    bool
	RepositoryName     string
	RepositoryStrategy sdk.RepositoryStrategy
	DryRun             bool
	HookUUID           string
}

// CreateFromRepository a workflow from a repository
func CreateFromRepository(ctx context.Context, db *gorp.DbMap, store cache.Store, p *sdk.Project, w *sdk.Workflow, opts sdk.WorkflowRunPostHandlerOption, u *sdk.User, decryptFunc keys.DecryptFunc) ([]sdk.Message, error) {
	ctx, end := observability.Span(ctx, "workflow.CreateFromRepository")
	defer end()

	ope, err := createOperationRequest(*w, opts)
	if err != nil {
		return nil, sdk.WrapError(err, "Unable to create operation request")
	}

	// update user permission if  we come from hook
	if opts.Hook != nil {
		u.Groups = make([]sdk.Group, len(p.ProjectGroups))
		for i, gp := range p.ProjectGroups {
			u.Groups[i] = gp.Group
		}
	}

	if err := PostRepositoryOperation(ctx, db, *p, &ope, nil); err != nil {
		return nil, sdk.WrapError(err, "Unable to post repository operation")
	}

	if err := pollRepositoryOperation(ctx, db, store, &ope); err != nil {
		return nil, sdk.WrapError(err, "Cannot analyse repository")
	}

	var uuid string
	if opts.Hook != nil {
		uuid = opts.Hook.WorkflowNodeHookUUID
	} else {
		// Search for repo web hook uuid
		for _, h := range w.WorkflowData.Node.Hooks {
			if h.HookModelName == sdk.RepositoryWebHookModelName {
				uuid = h.UUID
				break
			}
		}
	}
	return extractWorkflow(ctx, db, store, p, w, ope, u, decryptFunc, uuid)
}

func extractWorkflow(ctx context.Context, db *gorp.DbMap, store cache.Store, p *sdk.Project, w *sdk.Workflow, ope sdk.Operation, u *sdk.User, decryptFunc keys.DecryptFunc, hookUUID string) ([]sdk.Message, error) {
	ctx, end := observability.Span(ctx, "workflow.extractWorkflow")
	defer end()

	// Read files
	tr, err := ReadCDSFiles(ope.LoadFiles.Results)
	if err != nil {
		return nil, sdk.WrapError(err, "Unable to read cds files")
	}
	ope.RepositoryStrategy.SSHKeyContent = ""
	opt := &PushOption{
		VCSServer:          ope.VCSServer,
		RepositoryName:     ope.RepoFullName,
		RepositoryStrategy: ope.RepositoryStrategy,
		Branch:             ope.Setup.Checkout.Branch,
		FromRepository:     ope.RepositoryInfo.FetchURL,
		IsDefaultBranch:    ope.Setup.Checkout.Branch == ope.RepositoryInfo.DefaultBranch,
		DryRun:             true,
		HookUUID:           hookUUID,
	}

	allMsg, workflowPushed, errP := Push(ctx, db, store, p, tr, opt, u, decryptFunc)
	if errP != nil {
		return nil, sdk.WrapError(errP, "extractWorkflow> Unable to get workflow from file")
	}
	*w = *workflowPushed
	return allMsg, nil
}

// ReadCDSFiles reads CDS files
func ReadCDSFiles(files map[string][]byte) (*tar.Reader, error) {
	// Create a buffer to write our archive to.
	buf := new(bytes.Buffer)
	// Create a new tar archive.
	tw := tar.NewWriter(buf)
	// Add some files to the archive.
	for fname, fcontent := range files {
		log.Debug("ReadCDSFiles> Reading %s", fname)
		hdr := &tar.Header{
			Name: filepath.Base(fname),
			Mode: 0600,
			Size: int64(len(fcontent)),
		}
		if err := tw.WriteHeader(hdr); err != nil {
			return nil, sdk.WrapError(err, "Cannot write header")
		}
		if n, err := tw.Write(fcontent); err != nil {
			return nil, sdk.WrapError(err, "Cannot write content")
		} else if n == 0 {
			return nil, fmt.Errorf("nothing to write")
		}
	}
	// Make sure to check the error on Close.
	if err := tw.Close(); err != nil {
		return nil, err
	}

	return tar.NewReader(buf), nil
}

func pollRepositoryOperation(c context.Context, db gorp.SqlExecutor, store cache.Store, ope *sdk.Operation) error {
	tickTimeout := time.NewTicker(10 * time.Minute)
	tickPoll := time.NewTicker(2 * time.Second)
	defer tickTimeout.Stop()
	for {
		select {
		case <-c.Done():
			if c.Err() != nil {
				return sdk.WrapError(c.Err(), "pollRepositoryOperation> Exiting")
			}
		case <-tickTimeout.C:
			return sdk.WrapError(sdk.ErrRepoOperationTimeout, "pollRepositoryOperation> Timeout analyzing repository")
		case <-tickPoll.C:
			if err := GetRepositoryOperation(c, db, ope); err != nil {
				return sdk.WrapError(err, "Cannot get repository operation status")
			}
			switch ope.Status {
			case sdk.OperationStatusError:
				opeTrusted := *ope
				opeTrusted.RepositoryStrategy.SSHKeyContent = "***"
				opeTrusted.RepositoryStrategy.Password = "***"
				return sdk.WrapError(fmt.Errorf("%s", ope.Error), "getImportAsCodeHandler> Operation in error. %+v", opeTrusted)
			case sdk.OperationStatusDone:
				return nil
			}
			continue
		}
	}
}

func createOperationRequest(w sdk.Workflow, opts sdk.WorkflowRunPostHandlerOption) (sdk.Operation, error) {
	ope := sdk.Operation{}
	if w.Root.Context.Application == nil {
		return ope, sdk.WrapError(sdk.ErrApplicationNotFound, "CreateFromRepository> Workflow node root does not have a application context")
	}
	ope = sdk.Operation{
		VCSServer:          w.Root.Context.Application.VCSServer,
		RepoFullName:       w.Root.Context.Application.RepositoryFullname,
		URL:                w.FromRepository,
		RepositoryStrategy: w.Root.Context.Application.RepositoryStrategy,
		Setup: sdk.OperationSetup{
			Checkout: sdk.OperationCheckout{
				Branch: "",
				Commit: "",
			},
		},
		LoadFiles: sdk.OperationLoadFiles{
			Pattern: WorkflowAsCodePattern,
		},
	}

	var branch, commit string
	if opts.Hook != nil {
		branch = opts.Hook.Payload[tagGitBranch]
		commit = opts.Hook.Payload[tagGitHash]
	}
	if opts.Manual != nil {
		e := dump.NewDefaultEncoder(new(bytes.Buffer))
		e.Formatters = []dump.KeyFormatterFunc{dump.WithDefaultLowerCaseFormatter()}
		e.ExtraFields.DetailedMap = false
		e.ExtraFields.DetailedStruct = false
		e.ExtraFields.Len = false
		e.ExtraFields.Type = false
		m1, errm1 := e.ToStringMap(opts.Manual.Payload)
		if errm1 != nil {
			return ope, sdk.WrapError(errm1, "CreateFromRepository> Unable to compute payload")
		}
		branch = m1[tagGitBranch]
		commit = m1[tagGitHash]
	}
	ope.Setup.Checkout.Commit = commit
	ope.Setup.Checkout.Branch = branch

	// This should not append because the hook must set a default payload with git.branch
	if ope.Setup.Checkout.Branch == "" {
		return ope, sdk.WrapError(sdk.NewError(sdk.ErrWrongRequest, fmt.Errorf("branch parameter is mandatory")), "createOperationRequest")
	}

	return ope, nil
}

// PostRepositoryOperation creates a new repository operation
func PostRepositoryOperation(ctx context.Context, db gorp.SqlExecutor, prj sdk.Project, ope *sdk.Operation, multipartData *services.MultiPartData) error {
	srvs, err := services.FindByType(db, services.TypeRepositories)
	if err != nil {
		return sdk.WrapError(err, "Unable to found repositories service")
	}

	if ope.RepositoryStrategy.ConnectionType == "ssh" {
		for _, k := range prj.Keys {
			if k.Name == ope.RepositoryStrategy.SSHKey {
				ope.RepositoryStrategy.SSHKeyContent = k.Private
				break
			}
		}
	}

	if multipartData == nil {
		if _, err := services.DoJSONRequest(ctx, srvs, http.MethodPost, "/operations", ope, ope); err != nil {
			return sdk.WrapError(err, "Unable to perform operation")
		}
		return nil
	}
	if _, err := services.DoMultiPartRequest(ctx, srvs, http.MethodPost, "/operations", multipartData, ope, ope); err != nil {
		return sdk.WrapError(err, "Unable to perform multipart operation")
	}
	return nil
}

// GetRepositoryOperation get repository operation status
func GetRepositoryOperation(ctx context.Context, db gorp.SqlExecutor, ope *sdk.Operation) error {
	srvs, err := services.FindByType(db, services.TypeRepositories)
	if err != nil {
		return sdk.WrapError(err, "Unable to found repositories service")
	}

	if _, err := services.DoJSONRequest(ctx, srvs, http.MethodGet, "/operations/"+ope.UUID, nil, ope); err != nil {
		return sdk.WrapError(err, "Unable to get operation")
	}
	return nil
}
<|endoftext|>


<commit_before>package ping

import (
	"testing"
	"time"

	"github.com/hsheth2/logs"

	"network/ipv4"
)

func ping_tester(t *testing.T, ip *ipv4.Address, num uint16) {
	err := GlobalPingManager.SendPing(ip, time.Second, time.Second, num)
	if err != nil {
		logs.Error.Println(err)
		t.Error(err)
	} else {
		t.Log("Success")
	}
	time.Sleep(500 * time.Millisecond)
}

func TestLocalPing(t *testing.T) {
	ping_tester(t, ipv4.MakeIP("127.0.0.1"), 5)
}

func TestTapPing(t *testing.T) {
	ping_tester(t, ipv4.MakeIP("10.0.0.2"), 5)
}

func TestExternalPing(t *testing.T) {
	ping_tester(t, ipv4.MakeIP("192.168.1.2"), 5) // TODO decide dynamically based on ip address
}
<commit_msg>Added a new ping test for completely external networks<commit_after>package ping

import (
	"testing"
	"time"

	"github.com/hsheth2/logs"

	"network/ipv4"
)

func ping_tester(t *testing.T, ip *ipv4.Address, num uint16) {
	err := GlobalPingManager.SendPing(ip, time.Second, time.Second, num)
	if err != nil {
		logs.Error.Println(err)
		t.Error(err)
	} else {
		t.Log("Success")
	}
	time.Sleep(500 * time.Millisecond)
}

func TestLocalPing(t *testing.T) {
	ping_tester(t, ipv4.MakeIP("127.0.0.1"), 5)
}

func TestTapPing(t *testing.T) {
	ping_tester(t, ipv4.MakeIP("10.0.0.2"), 5)
}

func TestExternalPing(t *testing.T) {
	ping_tester(t, ipv4.MakeIP("192.168.1.2"), 5) // TODO decide dynamically based on ip address
}

func TestPingBing(t *testing.T) {
	t.Skip("Pinging externally does not work yet")
	ping_tester(t, ipv4.MakeIP("204.79.197.200"), 5) // TODO use DNS to determine this IP
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"strings"
)

type PathwarGenerateAToken struct {
	ID      string `json:"_id"`
	Created string `json:"_created"`
	Etag    string `json:"_etag"`
	Status  string `json:"_status"`
}

func (p *APIPathwar) GenerateAToken(login, password string, tmp bool) (*PathwarGenerateAToken, error) {
	request := p.client.Post(strings.Join([]string{APIUrl, "user-tokens"}, "/"))
	request = request.SetBasicAuth(login, password)

	request = request.Send(fmt.Sprintf("{\"is_session\": %v}", tmp))
	if p.debug {
		request = request.SetDebug(true)
	}
	resp, body, errs := request.EndBytes()

	if len(errs) != 0 {
		return nil, printErrors(errs)
	}
	if err := httpHandleError([]int{201}, resp.StatusCode, body); err != nil {
		return nil, err
	}
	ret := &PathwarGenerateAToken{}

	if err := json.Unmarshal(body, ret); err != nil {
		return nil, err
	}
	return ret, nil
}

type PathwarToken struct {
	Token string `json:"token"`
}

func (p *APIPathwar) GetToken(login, password, id string) (*PathwarToken, error) {
	request := p.client.Get(fmt.Sprintf("%s/user-tokens/%s", APIUrl, id))
	request = request.SetBasicAuth(login, password)
	if p.debug {
		request = request.SetDebug(true)
	}
	resp, body, errs := request.EndBytes()

	if len(errs) != 0 {
		return nil, printErrors(errs)
	}
	if err := httpHandleError([]int{200}, resp.StatusCode, body); err != nil {
		return nil, err
	}
	ret := &PathwarToken{}
	if err := json.Unmarshal(body, ret); err != nil {
		return nil, err
	}
	return ret, nil
}
<commit_msg>Remove Token method from API structure<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/parnurzeal/gorequest"
)

type PathwarGenerateAToken struct {
	ID      string `json:"_id"`
	Created string `json:"_created"`
	Etag    string `json:"_etag"`
	Status  string `json:"_status"`
}

func GenerateAToken(login, password string, tmp bool) (*PathwarGenerateAToken, error) {
	request := gorequest.New().Post(fmt.Sprintf("%s/user-tokens/", APIUrl))
	request = request.SetBasicAuth(login, password)
	request = request.Send(fmt.Sprintf("{\"is_session\": %v}", tmp))
	if os.Getenv("PATHWAR_DEBUG") != "" {
		request = request.SetDebug(true)
	}
	resp, body, errs := request.EndBytes()

	if len(errs) != 0 {
		return nil, printErrors(errs)
	}
	if err := httpHandleError([]int{201}, resp.StatusCode, body); err != nil {
		return nil, err
	}
	ret := &PathwarGenerateAToken{}

	if err := json.Unmarshal(body, ret); err != nil {
		return nil, err
	}
	return ret, nil
}

type PathwarToken struct {
	Token string `json:"token"`
}

func GetToken(login, password, id string) (*PathwarToken, error) {
	request := gorequest.New().Get(fmt.Sprintf("%s/user-tokens/%s", APIUrl, id))
	request = request.SetBasicAuth(login, password)
	if os.Getenv("PATHWAR_DEBUG") != "" {
		request = request.SetDebug(true)
	}
	resp, body, errs := request.EndBytes()

	if len(errs) != 0 {
		return nil, printErrors(errs)
	}
	if err := httpHandleError([]int{200}, resp.StatusCode, body); err != nil {
		return nil, err
	}
	ret := &PathwarToken{}
	if err := json.Unmarshal(body, ret); err != nil {
		return nil, err
	}
	return ret, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"fmt"
	"path"

	"github.com/GoogleContainerTools/kaniko/pkg/config"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/google/go-containerregistry/pkg/v1/tarball"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func WarmCache(opts *config.WarmerOptions) error {
	cacheDir := opts.CacheDir
	images := opts.Images
	logrus.Debugf("%s\n", cacheDir)
	logrus.Debugf("%s\n", images)

	for _, image := range images {
		cacheRef, err := name.NewTag(image, name.WeakValidation)
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image))
		}
		img, err := remote.Image(cacheRef)
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image))
		}

		digest, err := img.Digest()
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image))
		}
		cachePath := path.Join(cacheDir, digest.String())
		err = tarball.WriteToFile(cachePath, cacheRef, img)
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image))
		}
		logrus.Debugf("Wrote %s to cache", image)
	}
	return nil
}
<commit_msg>Update the cache warmer to also save manifests. (#576)<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"fmt"
	"io/ioutil"
	"path"

	"github.com/GoogleContainerTools/kaniko/pkg/config"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1/remote"
	"github.com/google/go-containerregistry/pkg/v1/tarball"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

func WarmCache(opts *config.WarmerOptions) error {
	cacheDir := opts.CacheDir
	images := opts.Images
	logrus.Debugf("%s\n", cacheDir)
	logrus.Debugf("%s\n", images)

	for _, image := range images {
		cacheRef, err := name.NewTag(image, name.WeakValidation)
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to verify image name: %s", image))
		}
		img, err := remote.Image(cacheRef)
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve image: %s", image))
		}

		digest, err := img.Digest()
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve digest: %s", image))
		}
		cachePath := path.Join(cacheDir, digest.String())
		err = tarball.WriteToFile(cachePath, cacheRef, img)
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to write %s to cache", image))
		}

		mfst, err := img.RawManifest()
		if err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to retrieve manifest for %s", image))
		}
		mfstPath := cachePath + ".json"
		if err := ioutil.WriteFile(mfstPath, mfst, 0666); err != nil {
			return errors.Wrap(err, fmt.Sprintf("Failed to save manifest for %s", image))
		}
		logrus.Debugf("Wrote %s to cache", image)
	}
	return nil
}
<|endoftext|>