<commit_before>// Package for building an execution pipeline and executing it.
package pipeline

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"

	"gopkg.in/yaml.v2"

	"github.com/fjukstad/kvik/kompute"
)

// A pipeline stage.
// Name: A unique name for each stage.
// Function: Function name, e.g. plot.
// Package: Which statistical package the function comes from, e.g. graphics for plot().
// Arguments: A name:value map for arguments to the function.
// Depends: List of stages it depends on.
// Session: Reference to the OpenCPU session used to compute the results from this stage.
// Output: String for storing output from pipeline stage execution.
type Stage struct {
	Name      string            "name,omitempty"
	Package   string            "package,omitempty"
	Function  string            "function,omitempty"
	Arguments map[string]string "arguments,omitempty"
	Depends   []string          "depends,omitempty"
	Session   *kompute.Session  "session,omitempty"
	Output    string            "output,omitempty"
}

// The pipeline. A name, reference to an opencpu server and list of pipeline
// stages.
type Pipeline struct {
	Name    string           "name,omitempty"
	Kompute *kompute.Kompute "kompute,omitempty"
	Stages  []*Stage         "stages,omitempty"
}

// Set up new pipeline with the given name and reference to kompute server
func NewPipeline(name string, k *kompute.Kompute) Pipeline {
	p := Pipeline{name, k, nil}
	return p
}

// Appends a pipeline stage to to the end of the pipeline.
func (p *Pipeline) AddStage(s Stage) {
	p.Stages = append(p.Stages, &s)
}

// Creates a new pipeline stage. Argnames is an array of argument names, args
// are the values for each of the arguments.  Assumes that argnames and args are
// of the same length.
func NewStage(name, function, pkg string, argnames, args []string) Stage {

	argmap := make(map[string]string, 0)

	for i, argname := range argnames {
		arg := args[i]
		argmap[argname] = arg
	}

	s := Stage{name, pkg, function, argmap, []string{}, nil, ""}

	return s
}

// Imports a pipeline from a pipeline description in yaml. It should follow this
// format:
//
//	 name: PIPELINE NAME
//	 stages:
//	 - name: STAGE NAME
//	   package: PACKAGE NAME
//	   function: FUNCTION NAME
//	   arguments:
//	     ARGUMENT NAME: ARGUMENT VALUE
//	     ...

func ImportPipeline(filename string) (Pipeline, error) {
	p := Pipeline{}

	in, err := ioutil.ReadFile(filename)
	if err != nil {
		return Pipeline{}, err
	}

	err = yaml.Unmarshal(in, &p)
	if err != nil {
		return Pipeline{}, err
	}
	return p, nil

}

// For storing the result of a pipeline stage. Key can be used to retrieve
// plots/results later using the kompute package.
type Result struct {
	m     *sync.Mutex
	c     *sync.Cond
	Key   string
	Error error
}

// Exectues a pipeline. Uses a go routine per pipeline stage making it possible
// to execute multiple stages simultaneously. Returns a list of results, one per
// stage.
func (p *Pipeline) Run() ([]*Result, error) {

	resultMap := make(map[string]*Result, 0)
	done := make(chan bool, len(p.Stages))

	for _, stage := range p.Stages {
		m := sync.Mutex{}
		c := sync.NewCond(&m)
		resultMap[stage.Name] = &Result{&m, c, "", nil}

		deps := stage.GetDependencies()
		r := resultMap[stage.Name]
		stage.Depends = deps

		go func(r *Result, stage *Stage, deps []string) {
			if len(deps) != 0 {
				for _, dep := range deps {
					r := resultMap[dep]
					r.m.Lock()

					for r.Key == "" {
						r.c.Wait()
					}

					stage.ReplaceArg(dep, r.Key)
					r.m.Unlock()
				}
			}
			r.m.Lock()
			for r.Key == "" {
				r.Key, r.Error = p.ExecuteStage(stage)
			}
			r.c.Broadcast()
			r.m.Unlock()

			done <- true
		}(r, stage, deps)
	}

	for i := 0; i < len(p.Stages); i++ {
		<-done
	}

	results := []*Result{}
	for _, stage := range p.Stages {
		r := resultMap[stage.Name]
		results = append(results, r)
	}

	return results, nil
}

// Replace any "from:stage-name" argument values into opencpu references
func (s *Stage) ReplaceArg(oldarg string, newarg string) {
	for i, arg := range s.Arguments {
		if strings.Contains(arg, oldarg) {
			s.Arguments[i] = strings.Replace(s.Arguments[i], "from:"+oldarg, newarg, -1)
			return
		}
	}
}

// Executes a pipeline stage.
func (p *Pipeline) ExecuteStage(stage *Stage) (string, error) {

	args := stage.GetArguments()
	fun := stage.GetFullFunctionName()

	s, err := p.Kompute.Call(fun, args)

	if err != nil {
		s, err = p.Kompute.Call(fun, args)

		if err != nil {
			fmt.Println("ERROR", err)
			return "", err
		}
	}

	fmt.Println("Stage", stage.Name, "completed")

	stage.Session = s
	return s.Key, nil
}

// Get the final result from the last stage in a pipeline.
func (p *Pipeline) GetResult(format string) string {
	lastStage := p.Stages[len(p.Stages)-1]
	res, _ := lastStage.Session.GetResult(p.Kompute, format)
	return res
}

// Stores a pipeline as a pipeline description yaml file.
func (p *Pipeline) Save() error {
	file, err := os.OpenFile(p.Name+".yaml", os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return err
	}

	b, err := yaml.Marshal(p)
	if err != nil {
		return err
	}

	_, err = file.Write(b)
	if err != nil {
		return err
	}
	return nil
}

// Returns the full function name to be used with the kompute package.
func (s *Stage) GetFullFunctionName() string {
	return s.Package + "/R/" + s.Function
}

// Create a string of arguments for usage by the Call method in the kompute
// package
func (s *Stage) GetArguments() string {
	str := ""
	i := 0
	for k, v := range s.Arguments {
		if strings.Contains(v, "/") || strings.Contains(v, ".") {
			v = "\"" + v + "\""
		}
		str = str + k + "=" + v
		if i < len(s.Arguments)-1 {
			str = str + "&"
		}
		i++
	}

	return str
}

// Returns all dependencies for a stage.
func (s Stage) GetDependencies() []string {
	deps := []string{}
	for _, arg := range s.Arguments {
		if strings.Contains(arg, "from:") {
			args := strings.Split(arg, "from:")
			var argname string

			// if argument is list of from: s
			if len(args) > 2 {
				for _, a := range args {
					if len(a) > 1 {
						a = strings.TrimRight(a, ",")
						a = strings.TrimRight(a, "]")
						deps = append(deps, a)
					}
				}
			} else {
				argname = strings.Split(arg, "from:")[1]
				argname = strings.TrimRight(argname, "]")
				deps = append(deps, argname)
			}
		}
	}

	return deps
}

// Print a pipeline stage.
func (p *Pipeline) Print() {
	for _, stage := range p.Stages {
		if stage.Session != nil {
			res, _ := p.Kompute.Get(stage.Session.Key, "")
			stage.Output = string(res)
		}
		stage.Print()
		fmt.Println()
	}
}

// Return pipeline results, i.e. result of last pipeline stage.
func (p *Pipeline) Results(resultType string) (string, error) {
	stage := p.Stages[len(p.Stages)-1]

	if resultType == "png" || resultType == "pdf" {

		a := "/Users/bjorn/Dropbox/go/src/github.com/fjukstad/kvik/pipeline/example/x-y/"

		return "", stage.Session.DownloadPlot(resultType, a+p.Name+"."+resultType)
	}

	return stage.Session.GetResult(p.Kompute, resultType)
}

// Print a pipeline stage.
func (s *Stage) Print() {
	fmt.Println("\tName:", s.Name)
	fmt.Println("\tPackage", s.Package)
	fmt.Println("\tFunction", s.Function)
	fmt.Println("\tDepends on:", s.Depends)
	fmt.Println("\tArguments:")
	for k, v := range s.Arguments {
		fmt.Println("\t\t", k, "\t", v)
	}
	if s.Session != nil {
		fmt.Println("\tSession: ", s.Session.Key)
		fmt.Println("\tURL: /ocpu/tmp/" + s.Session.Key + "/R/")
		fmt.Println("\tOutput:\n", s.Output)
	}
}
<commit_msg>use renamed ogopencpu pkg<commit_after>// Package for building an execution pipeline and executing it.
package pipeline

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"

	"gopkg.in/yaml.v2"

	"github.com/fjukstad/kvik/gopencpu"
)

// A pipeline stage.
// Name: A unique name for each stage.
// Function: Function name, e.g. plot.
// Package: Which statistical package the function comes from, e.g. graphics for plot().
// Arguments: A name:value map for arguments to the function.
// Depends: List of stages it depends on.
// Session: Reference to the OpenCPU session used to compute the results from this stage.
// Output: String for storing output from pipeline stage execution.
type Stage struct {
	Name      string            "name,omitempty"
	Package   string            "package,omitempty"
	Function  string            "function,omitempty"
	Arguments map[string]string "arguments,omitempty"
	Depends   []string          "depends,omitempty"
	Session   *gopencpu.Session "session,omitempty"
	Output    string            "output,omitempty"
}

// The pipeline. A name, reference to an opencpu server and list of pipeline
// stages.
type Pipeline struct {
	Name      string              "name,omitempty"
	GoOpenCPU *gopencpu.GoOpenCPU "gopencpu,omitempty"
	Stages    []*Stage            "stages,omitempty"
}

// Set up new pipeline with the given name and reference to gopencpu server
func NewPipeline(name string, k *gopencpu.GoOpenCPU) Pipeline {
	p := Pipeline{name, k, nil}
	return p
}

// Appends a pipeline stage to to the end of the pipeline.
func (p *Pipeline) AddStage(s Stage) {
	p.Stages = append(p.Stages, &s)
}

// Creates a new pipeline stage. Argnames is an array of argument names, args
// are the values for each of the arguments.  Assumes that argnames and args are
// of the same length.
func NewStage(name, function, pkg string, argnames, args []string) Stage {

	argmap := make(map[string]string, 0)

	for i, argname := range argnames {
		arg := args[i]
		argmap[argname] = arg
	}

	s := Stage{name, pkg, function, argmap, []string{}, nil, ""}

	return s
}

// Imports a pipeline from a pipeline description in yaml. It should follow this
// format:
//
//	 name: PIPELINE NAME
//	 stages:
//	 - name: STAGE NAME
//	   package: PACKAGE NAME
//	   function: FUNCTION NAME
//	   arguments:
//	     ARGUMENT NAME: ARGUMENT VALUE
//	     ...

func ImportPipeline(filename string) (Pipeline, error) {
	p := Pipeline{}

	in, err := ioutil.ReadFile(filename)
	if err != nil {
		return Pipeline{}, err
	}

	err = yaml.Unmarshal(in, &p)
	if err != nil {
		return Pipeline{}, err
	}
	return p, nil

}

// For storing the result of a pipeline stage. Key can be used to retrieve
// plots/results later using the gopencpu package.
type Result struct {
	m     *sync.Mutex
	c     *sync.Cond
	Key   string
	Error error
}

// Exectues a pipeline. Uses a go routine per pipeline stage making it possible
// to execute multiple stages simultaneously. Returns a list of results, one per
// stage.
func (p *Pipeline) Run() ([]*Result, error) {

	resultMap := make(map[string]*Result, 0)
	done := make(chan bool, len(p.Stages))

	for _, stage := range p.Stages {
		m := sync.Mutex{}
		c := sync.NewCond(&m)
		resultMap[stage.Name] = &Result{&m, c, "", nil}

		deps := stage.GetDependencies()
		r := resultMap[stage.Name]
		stage.Depends = deps

		go func(r *Result, stage *Stage, deps []string) {
			if len(deps) != 0 {
				for _, dep := range deps {
					r := resultMap[dep]
					r.m.Lock()

					for r.Key == "" {
						r.c.Wait()
					}

					stage.ReplaceArg(dep, r.Key)
					r.m.Unlock()
				}
			}
			r.m.Lock()
			for r.Key == "" {
				r.Key, r.Error = p.ExecuteStage(stage)
			}
			r.c.Broadcast()
			r.m.Unlock()

			done <- true
		}(r, stage, deps)
	}

	for i := 0; i < len(p.Stages); i++ {
		<-done
	}

	results := []*Result{}
	for _, stage := range p.Stages {
		r := resultMap[stage.Name]
		results = append(results, r)
	}

	return results, nil
}

func (p *Pipeline) RunSequential() ([]*Result, error) {
	resultMap := make(map[string]*Result, 0)

	for _, stage := range p.Stages {
		resultMap[stage.Name] = &Result{nil, nil, "", nil}

		deps := stage.GetDependencies()
		r := resultMap[stage.Name]

		for _, dep := range deps {
			r := resultMap[dep]
			stage.ReplaceArg(dep, r.Key)
		}

		for r.Key == "" {
			r.Key, r.Error = p.ExecuteStage(stage)
		}

	}

	results := []*Result{}
	for _, stage := range p.Stages {
		r := resultMap[stage.Name]
		results = append(results, r)
	}

	return results, nil

}

// Replace any "from:stage-name" argument values into opencpu references
func (s *Stage) ReplaceArg(oldarg string, newarg string) {
	for i, arg := range s.Arguments {
		if strings.Contains(arg, oldarg) {
			s.Arguments[i] = strings.Replace(s.Arguments[i], "from:"+oldarg, newarg, -1)
			return
		}
	}
}

// Executes a pipeline stage.
func (p *Pipeline) ExecuteStage(stage *Stage) (string, error) {

	args := stage.GetArguments()
	fun := stage.GetFullFunctionName()

	s, err := p.GoOpenCPU.Call(fun, args)

	if err != nil {
		s, err = p.GoOpenCPU.Call(fun, args)

		if err != nil {
			fmt.Println("ERROR", err)
			return "", err
		}
	}

	fmt.Println("Stage", stage.Name, "completed")

	stage.Session = s
	return s.Key, nil
}

// Get the final result from the last stage in a pipeline.
func (p *Pipeline) GetResult(format string) string {
	lastStage := p.Stages[len(p.Stages)-1]
	res, _ := lastStage.Session.GetResult(p.GoOpenCPU, format)
	return res
}

// Stores a pipeline as a pipeline description yaml file.
func (p *Pipeline) Save() error {
	file, err := os.OpenFile(p.Name+".yaml", os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return err
	}

	b, err := yaml.Marshal(p)
	if err != nil {
		return err
	}

	_, err = file.Write(b)
	if err != nil {
		return err
	}
	return nil
}

// Returns the full function name to be used with the gopencpu package.
func (s *Stage) GetFullFunctionName() string {
	return s.Package + "/R/" + s.Function
}

// Create a string of arguments for usage by the Call method in the gopencpu
// package
func (s *Stage) GetArguments() string {
	str := ""
	i := 0
	for k, v := range s.Arguments {
		if strings.Contains(v, "/") || strings.Contains(v, ".") {
			v = "\"" + v + "\""
		}
		str = str + k + "=" + v
		if i < len(s.Arguments)-1 {
			str = str + "&"
		}
		i++
	}

	return str
}

// Returns all dependencies for a stage.
func (s Stage) GetDependencies() []string {
	deps := []string{}
	for _, arg := range s.Arguments {
		if strings.Contains(arg, "from:") {
			args := strings.Split(arg, "from:")
			var argname string

			// if argument is list of from: s
			if len(args) > 2 {
				for _, a := range args {
					if len(a) > 1 {
						a = strings.TrimRight(a, ",")
						a = strings.TrimRight(a, "]")
						deps = append(deps, a)
					}
				}
			} else {
				argname = strings.Split(arg, "from:")[1]
				argname = strings.TrimRight(argname, "]")
				deps = append(deps, argname)
			}
		}
	}

	return deps
}

// Print a pipeline stage.
func (p *Pipeline) Print() {
	for _, stage := range p.Stages {
		if stage.Session != nil {
			res, _ := p.GoOpenCPU.Get(stage.Session.Key, "")
			stage.Output = string(res)
		}
		stage.Print()
		fmt.Println()
	}
}

// Return pipeline results, i.e. result of last pipeline stage.
func (p *Pipeline) Results(resultType string) (string, error) {
	stage := p.Stages[len(p.Stages)-1]

	if resultType == "png" || resultType == "pdf" {

		a := "/Users/bjorn/Dropbox/go/src/github.com/fjukstad/kvik/pipeline/example/x-y/"

		return "", stage.Session.DownloadPlot(resultType, a+p.Name+"."+resultType)
	}

	return stage.Session.GetResult(p.GoOpenCPU, resultType)
}

// Print a pipeline stage.
func (s *Stage) Print() {
	fmt.Println("\tName:", s.Name)
	fmt.Println("\tPackage", s.Package)
	fmt.Println("\tFunction", s.Function)
	fmt.Println("\tDepends on:", s.Depends)
	fmt.Println("\tArguments:")
	for k, v := range s.Arguments {
		fmt.Println("\t\t", k, "\t", v)
	}
	if s.Session != nil {
		fmt.Println("\tSession: ", s.Session.Key)
		fmt.Println("\tURL: /ocpu/tmp/" + s.Session.Key + "/R/")
		fmt.Println("\tOutput:\n", s.Output)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package audit

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"reflect"
	"time"

	authnv1 "k8s.io/api/authentication/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/apiserver/pkg/authorization/authorizer"
	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/klog/v2"

	"github.com/google/uuid"
)

const (
	maxUserAgentLength      = 1024
	userAgentTruncateSuffix = "...TRUNCATED"
)

func NewEventFromRequest(req *http.Request, requestReceivedTimestamp time.Time, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
	ev := &auditinternal.Event{
		RequestReceivedTimestamp: metav1.NewMicroTime(requestReceivedTimestamp),
		Verb:                     attribs.GetVerb(),
		RequestURI:               req.URL.RequestURI(),
		UserAgent:                maybeTruncateUserAgent(req),
		Level:                    level,
	}

	auditID, found := request.AuditIDFrom(req.Context())
	if !found {
		auditID = types.UID(uuid.New().String())
	}
	ev.AuditID = auditID

	ips := utilnet.SourceIPs(req)
	ev.SourceIPs = make([]string, len(ips))
	for i := range ips {
		ev.SourceIPs[i] = ips[i].String()
	}

	if user := attribs.GetUser(); user != nil {
		ev.User.Username = user.GetName()
		ev.User.Extra = map[string]authnv1.ExtraValue{}
		for k, v := range user.GetExtra() {
			ev.User.Extra[k] = authnv1.ExtraValue(v)
		}
		ev.User.Groups = user.GetGroups()
		ev.User.UID = user.GetUID()
	}

	if attribs.IsResourceRequest() {
		ev.ObjectRef = &auditinternal.ObjectReference{
			Namespace:   attribs.GetNamespace(),
			Name:        attribs.GetName(),
			Resource:    attribs.GetResource(),
			Subresource: attribs.GetSubresource(),
			APIGroup:    attribs.GetAPIGroup(),
			APIVersion:  attribs.GetAPIVersion(),
		}
	}

	for _, kv := range auditAnnotationsFrom(req.Context()) {
		LogAnnotation(ev, kv.key, kv.value)
	}

	return ev, nil
}

// LogImpersonatedUser fills in the impersonated user attributes into an audit event.
func LogImpersonatedUser(ae *auditinternal.Event, user user.Info) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	ae.ImpersonatedUser = &authnv1.UserInfo{
		Username: user.GetName(),
	}
	ae.ImpersonatedUser.Groups = user.GetGroups()
	ae.ImpersonatedUser.UID = user.GetUID()
	ae.ImpersonatedUser.Extra = map[string]authnv1.ExtraValue{}
	for k, v := range user.GetExtra() {
		ae.ImpersonatedUser.Extra[k] = authnv1.ExtraValue(v)
	}
}

// LogRequestObject fills in the request object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogRequestObject(ctx context.Context, obj runtime.Object, objGV schema.GroupVersion, gvr schema.GroupVersionResource, subresource string, s runtime.NegotiatedSerializer) {
	ae := AuditEventFrom(ctx)
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}

	// complete ObjectRef
	if ae.ObjectRef == nil {
		ae.ObjectRef = &auditinternal.ObjectReference{}
	}

	// meta.Accessor is more general than ObjectMetaAccessor, but if it fails, we can just skip setting these bits
	if meta, err := meta.Accessor(obj); err == nil {
		if len(ae.ObjectRef.Namespace) == 0 {
			ae.ObjectRef.Namespace = meta.GetNamespace()
		}
		if len(ae.ObjectRef.Name) == 0 {
			ae.ObjectRef.Name = meta.GetName()
		}
		if len(ae.ObjectRef.UID) == 0 {
			ae.ObjectRef.UID = meta.GetUID()
		}
		if len(ae.ObjectRef.ResourceVersion) == 0 {
			ae.ObjectRef.ResourceVersion = meta.GetResourceVersion()
		}
	}
	if len(ae.ObjectRef.APIVersion) == 0 {
		ae.ObjectRef.APIGroup = gvr.Group
		ae.ObjectRef.APIVersion = gvr.Version
	}
	if len(ae.ObjectRef.Resource) == 0 {
		ae.ObjectRef.Resource = gvr.Resource
	}
	if len(ae.ObjectRef.Subresource) == 0 {
		ae.ObjectRef.Subresource = subresource
	}

	if ae.Level.Less(auditinternal.LevelRequest) {
		return
	}

	if shouldOmitManagedFields(ctx) {
		copy, ok, err := copyWithoutManagedFields(obj)
		if err != nil {
			klog.Warningf("error while dropping managed fields from the request for %q error: %v", reflect.TypeOf(obj).Name(), err)
		}
		if ok {
			obj = copy
		}
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.RequestObject, err = encodeObject(obj, objGV, s)
	if err != nil {
		// TODO(audit): add error slice to audit event struct
		klog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err)
		return
	}
}

// LogRequestPatch fills in the given patch as the request object into an audit event.
func LogRequestPatch(ctx context.Context, patch []byte) {
	ae := AuditEventFrom(ctx)
	if ae == nil || ae.Level.Less(auditinternal.LevelRequest) {
		return
	}

	ae.RequestObject = &runtime.Unknown{
		Raw:         patch,
		ContentType: runtime.ContentTypeJSON,
	}
}

// LogResponseObject fills in the response object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogResponseObject(ctx context.Context, obj runtime.Object, gv schema.GroupVersion, s runtime.NegotiatedSerializer) {
	ae := AuditEventFrom(ctx)
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	if status, ok := obj.(*metav1.Status); ok {
		// selectively copy the bounded fields.
		ae.ResponseStatus = &metav1.Status{
			Status: status.Status,
			Reason: status.Reason,
			Code:   status.Code,
		}
	}

	if ae.Level.Less(auditinternal.LevelRequestResponse) {
		return
	}

	if shouldOmitManagedFields(ctx) {
		copy, ok, err := copyWithoutManagedFields(obj)
		if err != nil {
			klog.Warningf("error while dropping managed fields from the response for %q error: %v", reflect.TypeOf(obj).Name(), err)
		}
		if ok {
			obj = copy
		}
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.ResponseObject, err = encodeObject(obj, gv, s)
	if err != nil {
		klog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err)
	}
}

func encodeObject(obj runtime.Object, gv schema.GroupVersion, serializer runtime.NegotiatedSerializer) (*runtime.Unknown, error) {
	const mediaType = runtime.ContentTypeJSON
	info, ok := runtime.SerializerInfoForMediaType(serializer.SupportedMediaTypes(), mediaType)
	if !ok {
		return nil, fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType)
	}

	enc := serializer.EncoderForVersion(info.Serializer, gv)
	var buf bytes.Buffer
	if err := enc.Encode(obj, &buf); err != nil {
		return nil, fmt.Errorf("encoding failed: %v", err)
	}

	return &runtime.Unknown{
		Raw:         buf.Bytes(),
		ContentType: runtime.ContentTypeJSON,
	}, nil
}

// LogAnnotation fills in the Annotations according to the key value pair.
func LogAnnotation(ae *auditinternal.Event, key, value string) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	if ae.Annotations == nil {
		ae.Annotations = make(map[string]string)
	}
	if v, ok := ae.Annotations[key]; ok && v != value {
		klog.Warningf("Failed to set annotations[%q] to %q for audit:%q, it has already been set to %q", key, value, ae.AuditID, ae.Annotations[key])
		return
	}
	ae.Annotations[key] = value
}

// truncate User-Agent if too long, otherwise return it directly.
func maybeTruncateUserAgent(req *http.Request) string {
	ua := req.UserAgent()
	if len(ua) > maxUserAgentLength {
		ua = ua[:maxUserAgentLength] + userAgentTruncateSuffix
	}

	return ua
}

// copyWithoutManagedFields will make a deep copy of the specified object and
// will discard the managed fields from the copy.
// The specified object is expected to be a meta.Object or a "list".
// The specified object obj is treated as readonly and hence not mutated.
// On return, an error is set if the function runs into any error while
// removing the managed fields, the boolean value is true if the copy has
// been made successfully, otherwise false.
func copyWithoutManagedFields(obj runtime.Object) (runtime.Object, bool, error) {
	isAccessor := true
	if _, err := meta.Accessor(obj); err != nil {
		isAccessor = false
	}
	isList := meta.IsListType(obj)
	_, isTable := obj.(*metav1.Table)
	if !isAccessor && !isList && !isTable {
		return nil, false, nil
	}

	// TODO a deep copy isn't really needed here, figure out how we can reliably
	//  use shallow copy here to omit the manageFields.
	copy := obj.DeepCopyObject()

	if isAccessor {
		if err := removeManagedFields(copy); err != nil {
			return nil, false, err
		}
	}

	if isList {
		if err := meta.EachListItem(copy, removeManagedFields); err != nil {
			return nil, false, err
		}
	}

	if isTable {
		table := copy.(*metav1.Table)
		for i := range table.Rows {
			rowObj := table.Rows[i].Object
			if err := removeManagedFields(rowObj.Object); err != nil {
				return nil, false, err
			}
		}
	}

	return copy, true, nil
}

func removeManagedFields(obj runtime.Object) error {
	if obj == nil {
		return nil
	}
	accessor, err := meta.Accessor(obj)
	if err != nil {
		return err
	}
	accessor.SetManagedFields(nil)
	return nil
}

func shouldOmitManagedFields(ctx context.Context) bool {
	if auditContext := AuditContextFrom(ctx); auditContext != nil {
		return auditContext.RequestAuditConfig.OmitManagedFields
	}

	// If we can't decide, return false to maintain current behavior which is
	// to retain the manage fields in the audit.
	return false
}
<commit_msg>Issue 105353: Add messages+details to audit logs response (#105415)<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package audit

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"reflect"
	"time"

	authnv1 "k8s.io/api/authentication/v1"
	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/apiserver/pkg/authorization/authorizer"
	"k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/klog/v2"

	"github.com/google/uuid"
)

const (
	maxUserAgentLength      = 1024
	userAgentTruncateSuffix = "...TRUNCATED"
)

func NewEventFromRequest(req *http.Request, requestReceivedTimestamp time.Time, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
	ev := &auditinternal.Event{
		RequestReceivedTimestamp: metav1.NewMicroTime(requestReceivedTimestamp),
		Verb:                     attribs.GetVerb(),
		RequestURI:               req.URL.RequestURI(),
		UserAgent:                maybeTruncateUserAgent(req),
		Level:                    level,
	}

	auditID, found := request.AuditIDFrom(req.Context())
	if !found {
		auditID = types.UID(uuid.New().String())
	}
	ev.AuditID = auditID

	ips := utilnet.SourceIPs(req)
	ev.SourceIPs = make([]string, len(ips))
	for i := range ips {
		ev.SourceIPs[i] = ips[i].String()
	}

	if user := attribs.GetUser(); user != nil {
		ev.User.Username = user.GetName()
		ev.User.Extra = map[string]authnv1.ExtraValue{}
		for k, v := range user.GetExtra() {
			ev.User.Extra[k] = authnv1.ExtraValue(v)
		}
		ev.User.Groups = user.GetGroups()
		ev.User.UID = user.GetUID()
	}

	if attribs.IsResourceRequest() {
		ev.ObjectRef = &auditinternal.ObjectReference{
			Namespace:   attribs.GetNamespace(),
			Name:        attribs.GetName(),
			Resource:    attribs.GetResource(),
			Subresource: attribs.GetSubresource(),
			APIGroup:    attribs.GetAPIGroup(),
			APIVersion:  attribs.GetAPIVersion(),
		}
	}

	for _, kv := range auditAnnotationsFrom(req.Context()) {
		LogAnnotation(ev, kv.key, kv.value)
	}

	return ev, nil
}

// LogImpersonatedUser fills in the impersonated user attributes into an audit event.
func LogImpersonatedUser(ae *auditinternal.Event, user user.Info) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	ae.ImpersonatedUser = &authnv1.UserInfo{
		Username: user.GetName(),
	}
	ae.ImpersonatedUser.Groups = user.GetGroups()
	ae.ImpersonatedUser.UID = user.GetUID()
	ae.ImpersonatedUser.Extra = map[string]authnv1.ExtraValue{}
	for k, v := range user.GetExtra() {
		ae.ImpersonatedUser.Extra[k] = authnv1.ExtraValue(v)
	}
}

// LogRequestObject fills in the request object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogRequestObject(ctx context.Context, obj runtime.Object, objGV schema.GroupVersion, gvr schema.GroupVersionResource, subresource string, s runtime.NegotiatedSerializer) {
	ae := AuditEventFrom(ctx)
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}

	// complete ObjectRef
	if ae.ObjectRef == nil {
		ae.ObjectRef = &auditinternal.ObjectReference{}
	}

	// meta.Accessor is more general than ObjectMetaAccessor, but if it fails, we can just skip setting these bits
	if meta, err := meta.Accessor(obj); err == nil {
		if len(ae.ObjectRef.Namespace) == 0 {
			ae.ObjectRef.Namespace = meta.GetNamespace()
		}
		if len(ae.ObjectRef.Name) == 0 {
			ae.ObjectRef.Name = meta.GetName()
		}
		if len(ae.ObjectRef.UID) == 0 {
			ae.ObjectRef.UID = meta.GetUID()
		}
		if len(ae.ObjectRef.ResourceVersion) == 0 {
			ae.ObjectRef.ResourceVersion = meta.GetResourceVersion()
		}
	}
	if len(ae.ObjectRef.APIVersion) == 0 {
		ae.ObjectRef.APIGroup = gvr.Group
		ae.ObjectRef.APIVersion = gvr.Version
	}
	if len(ae.ObjectRef.Resource) == 0 {
		ae.ObjectRef.Resource = gvr.Resource
	}
	if len(ae.ObjectRef.Subresource) == 0 {
		ae.ObjectRef.Subresource = subresource
	}

	if ae.Level.Less(auditinternal.LevelRequest) {
		return
	}

	if shouldOmitManagedFields(ctx) {
		copy, ok, err := copyWithoutManagedFields(obj)
		if err != nil {
			klog.Warningf("error while dropping managed fields from the request for %q error: %v", reflect.TypeOf(obj).Name(), err)
		}
		if ok {
			obj = copy
		}
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.RequestObject, err = encodeObject(obj, objGV, s)
	if err != nil {
		// TODO(audit): add error slice to audit event struct
		klog.Warningf("Auditing failed of %v request: %v", reflect.TypeOf(obj).Name(), err)
		return
	}
}

// LogRequestPatch fills in the given patch as the request object into an audit event.
func LogRequestPatch(ctx context.Context, patch []byte) {
	ae := AuditEventFrom(ctx)
	if ae == nil || ae.Level.Less(auditinternal.LevelRequest) {
		return
	}

	ae.RequestObject = &runtime.Unknown{
		Raw:         patch,
		ContentType: runtime.ContentTypeJSON,
	}
}

// LogResponseObject fills in the response object into an audit event. The passed runtime.Object
// will be converted to the given gv.
func LogResponseObject(ctx context.Context, obj runtime.Object, gv schema.GroupVersion, s runtime.NegotiatedSerializer) {
	ae := AuditEventFrom(ctx)
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	if status, ok := obj.(*metav1.Status); ok {
		// selectively copy the bounded fields.
		ae.ResponseStatus = &metav1.Status{
			Status:  status.Status,
			Message: status.Message,
			Reason:  status.Reason,
			Details: status.Details,
			Code:    status.Code,
		}
	}

	if ae.Level.Less(auditinternal.LevelRequestResponse) {
		return
	}

	if shouldOmitManagedFields(ctx) {
		copy, ok, err := copyWithoutManagedFields(obj)
		if err != nil {
			klog.Warningf("error while dropping managed fields from the response for %q error: %v", reflect.TypeOf(obj).Name(), err)
		}
		if ok {
			obj = copy
		}
	}

	// TODO(audit): hook into the serializer to avoid double conversion
	var err error
	ae.ResponseObject, err = encodeObject(obj, gv, s)
	if err != nil {
		klog.Warningf("Audit failed for %q response: %v", reflect.TypeOf(obj).Name(), err)
	}
}

func encodeObject(obj runtime.Object, gv schema.GroupVersion, serializer runtime.NegotiatedSerializer) (*runtime.Unknown, error) {
	const mediaType = runtime.ContentTypeJSON
	info, ok := runtime.SerializerInfoForMediaType(serializer.SupportedMediaTypes(), mediaType)
	if !ok {
		return nil, fmt.Errorf("unable to locate encoder -- %q is not a supported media type", mediaType)
	}

	enc := serializer.EncoderForVersion(info.Serializer, gv)
	var buf bytes.Buffer
	if err := enc.Encode(obj, &buf); err != nil {
		return nil, fmt.Errorf("encoding failed: %v", err)
	}

	return &runtime.Unknown{
		Raw:         buf.Bytes(),
		ContentType: runtime.ContentTypeJSON,
	}, nil
}

// LogAnnotation fills in the Annotations according to the key value pair.
func LogAnnotation(ae *auditinternal.Event, key, value string) {
	if ae == nil || ae.Level.Less(auditinternal.LevelMetadata) {
		return
	}
	if ae.Annotations == nil {
		ae.Annotations = make(map[string]string)
	}
	if v, ok := ae.Annotations[key]; ok && v != value {
		klog.Warningf("Failed to set annotations[%q] to %q for audit:%q, it has already been set to %q", key, value, ae.AuditID, ae.Annotations[key])
		return
	}
	ae.Annotations[key] = value
}

// truncate User-Agent if too long, otherwise return it directly.
func maybeTruncateUserAgent(req *http.Request) string {
	ua := req.UserAgent()
	if len(ua) > maxUserAgentLength {
		ua = ua[:maxUserAgentLength] + userAgentTruncateSuffix
	}

	return ua
}

// copyWithoutManagedFields will make a deep copy of the specified object and
// will discard the managed fields from the copy.
// The specified object is expected to be a meta.Object or a "list".
// The specified object obj is treated as readonly and hence not mutated.
// On return, an error is set if the function runs into any error while
// removing the managed fields, the boolean value is true if the copy has
// been made successfully, otherwise false.
func copyWithoutManagedFields(obj runtime.Object) (runtime.Object, bool, error) {
	isAccessor := true
	if _, err := meta.Accessor(obj); err != nil {
		isAccessor = false
	}
	isList := meta.IsListType(obj)
	_, isTable := obj.(*metav1.Table)
	if !isAccessor && !isList && !isTable {
		return nil, false, nil
	}

	// TODO a deep copy isn't really needed here, figure out how we can reliably
	//  use shallow copy here to omit the manageFields.
	copy := obj.DeepCopyObject()

	if isAccessor {
		if err := removeManagedFields(copy); err != nil {
			return nil, false, err
		}
	}

	if isList {
		if err := meta.EachListItem(copy, removeManagedFields); err != nil {
			return nil, false, err
		}
	}

	if isTable {
		table := copy.(*metav1.Table)
		for i := range table.Rows {
			rowObj := table.Rows[i].Object
			if err := removeManagedFields(rowObj.Object); err != nil {
				return nil, false, err
			}
		}
	}

	return copy, true, nil
}

func removeManagedFields(obj runtime.Object) error {
	if obj == nil {
		return nil
	}
	accessor, err := meta.Accessor(obj)
	if err != nil {
		return err
	}
	accessor.SetManagedFields(nil)
	return nil
}

func shouldOmitManagedFields(ctx context.Context) bool {
	if auditContext := AuditContextFrom(ctx); auditContext != nil {
		return auditContext.RequestAuditConfig.OmitManagedFields
	}

	// If we can't decide, return false to maintain current behavior which is
	// to retain the manage fields in the audit.
	return false
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"time"

	"github.com/ash2k/smith"
)

const (
	// maxResponseSize is the maximum response size we are willing to read.
	maxResponseSize = 1 * 1024 * 1024
	// maxStatusResponseSize is the maximum status response size we are willing to read.
	maxStatusResponseSize = 10 * 1024
)

type StreamHandler func(io.Reader) error

type ResponseHandler func(*http.Response) error

// ResultFactory creates new instances of an object, that is used as JSON deserialization target.
// Must be safe for concurrent use.
type ResultFactory func() interface{}

type ResourceClient struct {
	Scheme      string
	HostPort    string
	BearerToken string
	Agent       string
	Client      http.Client
}

type StatusError struct {
	msg    string
	status smith.Status
}

func (se *StatusError) Error() string {
	return se.msg
}

func (se *StatusError) Status() smith.Status {
	return se.status
}

func (c *ResourceClient) List(ctx context.Context, groupVersion, namespace, resource string, args url.Values, request interface{}, into interface{}) error {
	return c.Do(ctx, "GET", groupVersion, namespace, resource, "", args, http.StatusOK, request, into)
}

func (c *ResourceClient) Create(ctx context.Context, groupVersion, namespace, resource string, request interface{}, response interface{}) error {
	return c.Do(ctx, "POST", groupVersion, namespace, resource, "", nil, http.StatusCreated, request, response)
}

func (c *ResourceClient) Update(ctx context.Context, groupVersion, namespace, resource string, request interface{}, response interface{}) error {
	return c.Do(ctx, "PUT", groupVersion, namespace, resource, "", nil, http.StatusOK, request, response)
}

func (c *ResourceClient) Delete(ctx context.Context, groupVersion, namespace, resource, name string, request interface{}, response interface{}) error {
	return c.Do(ctx, "DELETE", groupVersion, namespace, resource, name, nil, http.StatusOK, request, response)
}

func (c *ResourceClient) Watch(ctx context.Context, groupVersion, namespace, resource string, request interface{}, args url.Values, rf ResultFactory) <-chan interface{} {
	type hasResourceVersion interface {
		ResourceVersion() string
	}

	results := make(chan interface{})
	if args == nil {
		args = url.Values{}
	} else {
		// TODO clone args
	}
	go func() {
		defer close(results)
		for {
			var rv string
			err := c.DoCheckResponse(ctx, "GET", groupVersion, "watch", namespace, resource, "", args, http.StatusOK, request, func(r io.Reader) error {
				decoder := json.NewDecoder(r)
				for {
					res := rf()
					if err := decoder.Decode(res); err != nil {
						return err
					}
					select {
					case <-ctx.Done():
						return ctx.Err()
					case results <- res:
					}
					if rvResource, ok := res.(hasResourceVersion); ok {
						rv = rvResource.ResourceVersion()
					}
				}
			})
			if err != nil {
				//results <- err
				if err == context.Canceled || err == context.DeadlineExceeded {
					return
				}
			}
			// Delay after failed/closed connection
			select {
			case <-ctx.Done():
				//results <- ctx.Err()
				return
			case <-time.After(1 * time.Second):
			}
			if rv == "" {
				// It should never happen but just in case - the watch will restart from the
				// current "event horison" rather than from the original resource version which may no
				// longer be available.
				args.Del("resourceVersion")
			} else {
				args.Set("resourceVersion", rv)
			}
		}
	}()
	return results
}

func (c *ResourceClient) Do(ctx context.Context, verb, groupVersion, namespace, resource, name string, args url.Values, expectedStatus int, request interface{}, response interface{}) error {
	return c.DoCheckResponse(ctx, verb, groupVersion, "", namespace, resource, name, args, expectedStatus, request, func(r io.Reader) error {
		// Consume body even if "response" is nil to enable connection reuse
		b, err := ioutil.ReadAll(io.LimitReader(r, maxResponseSize))
		if err != nil {
			return err
		}
		log.Printf("Server response:\n%s", b)
		if response == nil {
			return nil
		}
		return json.Unmarshal(b, response)
	})
}

func (c *ResourceClient) DoCheckResponse(ctx context.Context, verb, prefix, groupVersion, namespace, resource, name string, args url.Values, expectedStatus int, request interface{}, f StreamHandler) error {
	return c.DoRequest(ctx, verb, prefix, groupVersion, namespace, resource, name, args, request, func(resp *http.Response) error {
		if resp.StatusCode != expectedStatus {
			msg := fmt.Sprintf("received bad status code %d", resp.StatusCode)
			b, err := ioutil.ReadAll(io.LimitReader(resp.Body, maxStatusResponseSize))
			if err != nil {
				return errors.New(msg)
			}
			se := StatusError{
				msg: msg,
			}
			log.Printf("Unexpected server response: %d\n%s", resp.StatusCode, b)
			if json.Unmarshal(b, &se.status) != nil {
				return errors.New(msg)
			}
			return &se
		}
		return f(resp.Body)
	})
}

func (c *ResourceClient) DoRequest(ctx context.Context, verb, prefix, groupVersion, namespace, resource, name string, args url.Values, request interface{}, f ResponseHandler) error {
	var body []byte
	if request != nil {
		var err error
		body, err = json.Marshal(request)
		if err != nil {
			return err
		}
	}
	p := []string{smith.DefaultAPIPath, groupVersion, prefix}
	if namespace != "" {
		p = append(p, "namespaces", namespace)
	}
	p = append(p, resource, name)
	reqUrl := url.URL{
		Scheme:   c.Scheme,
		Host:     c.HostPort,
		Path:     path.Join(p...),
		RawQuery: args.Encode(),
	}
	req, err := http.NewRequest(verb, reqUrl.String(), bytes.NewReader(body))
	if err != nil {
		return fmt.Errorf("unable to create http.Request: %v", err)
	}
	req = req.WithContext(ctx)
	if request != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("User-Agent", c.Agent)
	req.Header.Set("Authorization", "Bearer "+c.BearerToken)
	resp, err := c.Client.Do(req)
	if err != nil {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
			return fmt.Errorf("request failed: %v", err)
		}
	}
	defer resp.Body.Close()
	return f(resp)
}

func NewInCluster() (*ResourceClient, error) {
	host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
	if len(host) == 0 || len(port) == 0 {
		return nil, errors.New("unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined")
	}
	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/" + smith.ServiceAccountTokenKey)
	if err != nil {
		return nil, err
	}
	rootCA := "/var/run/secrets/kubernetes.io/serviceaccount/" + smith.ServiceAccountRootCAKey
	CAData, err := ioutil.ReadFile(rootCA)
	if err != nil {
		return nil, err
	}
	certPool := x509.NewCertPool()
	if !certPool.AppendCertsFromPEM(CAData) {
		log.Printf("Failed to add certificate from %s", rootCA)
	}
	return &ResourceClient{
		Scheme:      "https",
		HostPort:    net.JoinHostPort(host, port),
		BearerToken: string(token),
		Client: http.Client{
			Timeout: 10 * time.Minute,
			Transport: &http.Transport{
				Proxy:               http.ProxyFromEnvironment,
				TLSHandshakeTimeout: 10 * time.Second,
				TLSClientConfig: &tls.Config{
					// Can't use SSLv3 because of POODLE and BEAST
					// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
					// Can't use TLSv1.1 because of RC4 cipher usage
					MinVersion: tls.VersionTLS12,
					RootCAs:    certPool,
					//InsecureSkipVerify: true,
				},
				Dial: (&net.Dialer{
					Timeout:   30 * time.Second,
					KeepAlive: 30 * time.Second,
				}).Dial,
			},
		},
	}, nil
}
<commit_msg>Use DialContext to make dials cancellable<commit_after>package client

import (
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"time"

	"github.com/ash2k/smith"
)

const (
	// maxResponseSize is the maximum response size we are willing to read.
	maxResponseSize = 1 * 1024 * 1024
	// maxStatusResponseSize is the maximum status response size we are willing to read.
	maxStatusResponseSize = 10 * 1024
)

type StreamHandler func(io.Reader) error

type ResponseHandler func(*http.Response) error

// ResultFactory creates new instances of an object, that is used as JSON deserialization target.
// Must be safe for concurrent use.
type ResultFactory func() interface{}

type ResourceClient struct {
	Scheme      string
	HostPort    string
	BearerToken string
	Agent       string
	Client      http.Client
}

type StatusError struct {
	msg    string
	status smith.Status
}

func (se *StatusError) Error() string {
	return se.msg
}

func (se *StatusError) Status() smith.Status {
	return se.status
}

func (c *ResourceClient) List(ctx context.Context, groupVersion, namespace, resource string, args url.Values, request interface{}, into interface{}) error {
	return c.Do(ctx, "GET", groupVersion, namespace, resource, "", args, http.StatusOK, request, into)
}

func (c *ResourceClient) Create(ctx context.Context, groupVersion, namespace, resource string, request interface{}, response interface{}) error {
	return c.Do(ctx, "POST", groupVersion, namespace, resource, "", nil, http.StatusCreated, request, response)
}

func (c *ResourceClient) Update(ctx context.Context, groupVersion, namespace, resource string, request interface{}, response interface{}) error {
	return c.Do(ctx, "PUT", groupVersion, namespace, resource, "", nil, http.StatusOK, request, response)
}

func (c *ResourceClient) Delete(ctx context.Context, groupVersion, namespace, resource, name string, request interface{}, response interface{}) error {
	return c.Do(ctx, "DELETE", groupVersion, namespace, resource, name, nil, http.StatusOK, request, response)
}

func (c *ResourceClient) Watch(ctx context.Context, groupVersion, namespace, resource string, request interface{}, args url.Values, rf ResultFactory) <-chan interface{} {
	type hasResourceVersion interface {
		ResourceVersion() string
	}

	results := make(chan interface{})
	if args == nil {
		args = url.Values{}
	} else {
		// TODO clone args
	}
	go func() {
		defer close(results)
		for {
			var rv string
			err := c.DoCheckResponse(ctx, "GET", groupVersion, "watch", namespace, resource, "", args, http.StatusOK, request, func(r io.Reader) error {
				decoder := json.NewDecoder(r)
				for {
					res := rf()
					if err := decoder.Decode(res); err != nil {
						return err
					}
					select {
					case <-ctx.Done():
						return ctx.Err()
					case results <- res:
					}
					if rvResource, ok := res.(hasResourceVersion); ok {
						rv = rvResource.ResourceVersion()
					}
				}
			})
			if err != nil {
				//results <- err
				if err == context.Canceled || err == context.DeadlineExceeded {
					return
				}
			}
			// Delay after failed/closed connection
			select {
			case <-ctx.Done():
				//results <- ctx.Err()
				return
			case <-time.After(1 * time.Second):
			}
			if rv == "" {
				// It should never happen but just in case - the watch will restart from the
				// current "event horison" rather than from the original resource version which may no
				// longer be available.
				args.Del("resourceVersion")
			} else {
				args.Set("resourceVersion", rv)
			}
		}
	}()
	return results
}

func (c *ResourceClient) Do(ctx context.Context, verb, groupVersion, namespace, resource, name string, args url.Values, expectedStatus int, request interface{}, response interface{}) error {
	return c.DoCheckResponse(ctx, verb, groupVersion, "", namespace, resource, name, args, expectedStatus, request, func(r io.Reader) error {
		// Consume body even if "response" is nil to enable connection reuse
		b, err := ioutil.ReadAll(io.LimitReader(r, maxResponseSize))
		if err != nil {
			return err
		}
		log.Printf("Server response:\n%s", b)
		if response == nil {
			return nil
		}
		return json.Unmarshal(b, response)
	})
}

func (c *ResourceClient) DoCheckResponse(ctx context.Context, verb, prefix, groupVersion, namespace, resource, name string, args url.Values, expectedStatus int, request interface{}, f StreamHandler) error {
	return c.DoRequest(ctx, verb, prefix, groupVersion, namespace, resource, name, args, request, func(resp *http.Response) error {
		if resp.StatusCode != expectedStatus {
			msg := fmt.Sprintf("received bad status code %d", resp.StatusCode)
			b, err := ioutil.ReadAll(io.LimitReader(resp.Body, maxStatusResponseSize))
			if err != nil {
				return errors.New(msg)
			}
			se := StatusError{
				msg: msg,
			}
			log.Printf("Unexpected server response: %d\n%s", resp.StatusCode, b)
			if json.Unmarshal(b, &se.status) != nil {
				return errors.New(msg)
			}
			return &se
		}
		return f(resp.Body)
	})
}

func (c *ResourceClient) DoRequest(ctx context.Context, verb, prefix, groupVersion, namespace, resource, name string, args url.Values, request interface{}, f ResponseHandler) error {
	var body []byte
	if request != nil {
		var err error
		body, err = json.Marshal(request)
		if err != nil {
			return err
		}
	}
	p := []string{smith.DefaultAPIPath, groupVersion, prefix}
	if namespace != "" {
		p = append(p, "namespaces", namespace)
	}
	p = append(p, resource, name)
	reqUrl := url.URL{
		Scheme:   c.Scheme,
		Host:     c.HostPort,
		Path:     path.Join(p...),
		RawQuery: args.Encode(),
	}
	req, err := http.NewRequest(verb, reqUrl.String(), bytes.NewReader(body))
	if err != nil {
		return fmt.Errorf("unable to create http.Request: %v", err)
	}
	req = req.WithContext(ctx)
	if request != nil {
		req.Header.Set("Content-Type", "application/json")
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("User-Agent", c.Agent)
	req.Header.Set("Authorization", "Bearer "+c.BearerToken)
	resp, err := c.Client.Do(req)
	if err != nil {
		select {
		case <-ctx.Done():
			return ctx.Err()
		default:
			return fmt.Errorf("request failed: %v", err)
		}
	}
	defer resp.Body.Close()
	return f(resp)
}

func NewInCluster() (*ResourceClient, error) {
	host, port := os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")
	if len(host) == 0 || len(port) == 0 {
		return nil, errors.New("unable to load in-cluster configuration, KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT must be defined")
	}
	token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/" + smith.ServiceAccountTokenKey)
	if err != nil {
		return nil, err
	}
	rootCA := "/var/run/secrets/kubernetes.io/serviceaccount/" + smith.ServiceAccountRootCAKey
	CAData, err := ioutil.ReadFile(rootCA)
	if err != nil {
		return nil, err
	}
	certPool := x509.NewCertPool()
	if !certPool.AppendCertsFromPEM(CAData) {
		log.Printf("Failed to add certificate from %s", rootCA)
	}
	return &ResourceClient{
		Scheme:      "https",
		HostPort:    net.JoinHostPort(host, port),
		BearerToken: string(token),
		Client: http.Client{
			Timeout: 10 * time.Minute,
			Transport: &http.Transport{
				Proxy:               http.ProxyFromEnvironment,
				TLSHandshakeTimeout: 10 * time.Second,
				TLSClientConfig: &tls.Config{
					// Can't use SSLv3 because of POODLE and BEAST
					// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
					// Can't use TLSv1.1 because of RC4 cipher usage
					MinVersion: tls.VersionTLS12,
					RootCAs:    certPool,
					//InsecureSkipVerify: true,
				},
				DialContext: (&net.Dialer{
					Timeout:   30 * time.Second,
					KeepAlive: 30 * time.Second,
				}).DialContext,
			},
		},
	}, nil
}
<|endoftext|>


<commit_before>package config

import "runtime/debug"

const (
	_defaultSemVer = "v0.0.0-dev"
)

// SemVer is the version of mockery at build time.
var SemVer = ""

func GetSemverInfo() string {
	if SemVer != "" {
		return SemVer
	}
	version, ok := debug.ReadBuildInfo()
	if ok && version.Main.Version != "(devel)" && version.Main.Version != "" {
		return version.Main.Version
	}
	return _defaultSemVer
}

type Config struct {
	All                  bool
	BuildTags            string `mapstructure:"tags"`
	Case                 string
	Config               string
	Cpuprofile           string
	Dir                  string
	DisableVersionString bool `mapstructure:"disable-version-string"`
	DryRun               bool `mapstructure:"dry-run"`
	Exported             bool `mapstructure:"exported"`
	FileName             string
	InPackage            bool
	KeepTree             bool
	LogLevel             string `mapstructure:"log-level"`
	Name                 string
	Note                 string
	Outpkg               string
	Packageprefix        string
	Output               string
	Print                bool
	Profile              string
	Quiet                bool
	Recursive            bool
	SrcPkg               string
	BoilerplateFile      string `mapstructure:"boilerplate-file"`
	// StructName overrides the name given to the mock struct and should only be nonempty
	// when generating for an exact match (non regex expression in -name).
	StructName     string
	Tags           string
	TestOnly       bool
	UnrollVariadic bool `mapstructure:"unroll-variadic"`
	Version        bool
	WithExpecter   bool `mapstructure:"with-expecter"`
}
<commit_msg>fix: unused config field `Tags`<commit_after>package config

import "runtime/debug"

const (
	_defaultSemVer = "v0.0.0-dev"
)

// SemVer is the version of mockery at build time.
var SemVer = ""

func GetSemverInfo() string {
	if SemVer != "" {
		return SemVer
	}
	version, ok := debug.ReadBuildInfo()
	if ok && version.Main.Version != "(devel)" && version.Main.Version != "" {
		return version.Main.Version
	}
	return _defaultSemVer
}

type Config struct {
	All                  bool
	BuildTags            string `mapstructure:"tags"`
	Case                 string
	Config               string
	Cpuprofile           string
	Dir                  string
	DisableVersionString bool `mapstructure:"disable-version-string"`
	DryRun               bool `mapstructure:"dry-run"`
	Exported             bool `mapstructure:"exported"`
	FileName             string
	InPackage            bool
	KeepTree             bool
	LogLevel             string `mapstructure:"log-level"`
	Name                 string
	Note                 string
	Outpkg               string
	Packageprefix        string
	Output               string
	Print                bool
	Profile              string
	Quiet                bool
	Recursive            bool
	SrcPkg               string
	BoilerplateFile      string `mapstructure:"boilerplate-file"`
	// StructName overrides the name given to the mock struct and should only be nonempty
	// when generating for an exact match (non regex expression in -name).
	StructName     string
	TestOnly       bool
	UnrollVariadic bool `mapstructure:"unroll-variadic"`
	Version        bool
	WithExpecter   bool `mapstructure:"with-expecter"`
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package config manages and updates the ghpc input config
package config

import (
	"io/ioutil"
	"log"

	"gopkg.in/yaml.v2"

	"hpc-toolkit/pkg/resreader"
)

const expectedVarFormat = "$(vars.var_name) or $(resource_id.var_name)"

var errorMessages = map[string]string{
	// general
	"notImplemented": "not yet implemented",
	// config
	"fileLoadError":      "failed to read the input yaml",
	"yamlUnmarshalError": "failed to unmarshal the yaml config",
	"yamlMarshalError":   "failed to marshal the yaml config",
	"fileSaveError":      "failed to write the expanded yaml",
	// expand
	"missingSetting":    "a required setting is missing from a resource",
	"globalLabelType":   "global labels are not a map",
	"settingsLabelType": "labels in resources settings are not a map",
	"invalidVar":        "invalid variable definition in",
	"varNotFound":       "Could not find source of variable",
	"noOutput":          "Output not found for a variable",
	// validator
	"emptyID":         "a resource id cannot be empty",
	"emptySource":     "a resource source cannot be empty",
	"wrongKind":       "a resource kind is invalid",
	"extraSetting":    "a setting was added that is not found in the resource",
	"mixedResourcees": "mixing resources of differing kinds in a resource group is not supported",
	"duplicateGroup":  "group names must be unique",
	"duplicateID":     "resource IDs must be unique",
	"emptyGroupName":  "group name must be set for each resource group",
	"illegalChars":    "invalid character(s) found in group name",
}

// ResourceGroup defines a group of Resource that are all executed together
type ResourceGroup struct {
	Name             string           `yaml:"group"`
	TerraformBackend TerraformBackend `yaml:"terraform_backend"`
	Resources        []Resource
}

// TerraformBackend defines the configuration for the terraform state backend
type TerraformBackend struct {
	Type          string
	Configuration map[string]interface{}
}

// HasKind checks to see if a resource group contains any resources of the given
// kind. Note that a resourceGroup should never have more than one kind, this
// function is used in the validation step to ensure that is true.
func (g ResourceGroup) HasKind(kind string) bool {
	for _, res := range g.Resources {
		if res.Kind == kind {
			return true
		}
	}
	return false
}

// Resource stores YAML definition of a resource
type Resource struct {
	Source       string
	Kind         string
	ID           string
	ResourceName string
	Use          []string
	Settings     map[string]interface{}
}

// YamlConfig stores the contents on the User YAML
type YamlConfig struct {
	BlueprintName            string `yaml:"blueprint_name"`
	Vars                     map[string]interface{}
	ResourceGroups           []ResourceGroup  `yaml:"resource_groups"`
	TerraformBackendDefaults TerraformBackend `yaml:"terraform_backend_defaults"`
}

// BlueprintConfig is a container for the imported YAML data and supporting data for
// creating the blueprint from it
type BlueprintConfig struct {
	Config YamlConfig
	// Indexed by Resource Group name and Resource Source
	ResourcesInfo map[string]map[string]resreader.ResourceInfo
	// Maps resource ID to group index
	ResourceToGroup map[string]int
	expanded        bool
}

// ExpandConfig expands the yaml config in place
func (bc *BlueprintConfig) ExpandConfig() {
	bc.setResourcesInfo()
	bc.checkResourceAndGroupNames()
	bc.expand()
	bc.validate()
	bc.expanded = true
}

// NewBlueprintConfig is a constructor for BlueprintConfig
func NewBlueprintConfig(configFilename string) BlueprintConfig {
	newBlueprintConfig := BlueprintConfig{
		Config: importYamlConfig(configFilename),
	}
	return newBlueprintConfig
}

// ImportYamlConfig imports the blueprint configuration provided.
func importYamlConfig(yamlConfigFilename string) YamlConfig {
	yamlConfigText, err := ioutil.ReadFile(yamlConfigFilename)
	if err != nil {
		log.Fatalf("%s, filename=%s: %v",
			errorMessages["fileLoadError"], yamlConfigFilename, err)
	}

	var yamlConfig YamlConfig
	err = yaml.UnmarshalStrict(yamlConfigText, &yamlConfig)

	if err != nil {
		log.Fatalf("%s filename=%s: %v",
			errorMessages["yamlUnmarshalError"], yamlConfigFilename, err)
	}

	// Ensure Vars is not a nil map if not set by the user
	if len(yamlConfig.Vars) == 0 {
		yamlConfig.Vars = make(map[string]interface{})
	}

	// 1. DEFAULT: use TerraformBackend configuration (if supplied) in each
	//    resource group
	// 2. If top-level TerraformBackendDefaults is defined, insert that
	//    backend into resource groups which have no explicit
	//    TerraformBackend
	// 3. In all cases, add a prefix for GCS backends if one is not defined
	if yamlConfig.TerraformBackendDefaults.Type != "" {
		for i, grp := range yamlConfig.ResourceGroups {
			if grp.TerraformBackend.Type == "" {
				grp.TerraformBackend = yamlConfig.TerraformBackendDefaults
			}
			if grp.TerraformBackend.Type == "gcs" && grp.TerraformBackend.Configuration["prefix"] == nil {
				DeploymentName := yamlConfig.Vars["deployment_name"]
				prefix := yamlConfig.BlueprintName
				if DeploymentName != nil {
					prefix += "/" + DeploymentName.(string)
				}
				prefix += "/" + grp.Name
				grp.TerraformBackend.Configuration["prefix"] = prefix
			}
			yamlConfig.ResourceGroups[i].TerraformBackend = grp.TerraformBackend
		}
	}

	return yamlConfig
}

// ExportYamlConfig exports the internal representation of a blueprint config
func (bc BlueprintConfig) ExportYamlConfig(outputFilename string) []byte {
	d, err := yaml.Marshal(&bc.Config)
	if err != nil {
		log.Fatalf("%s: %v", errorMessages["yamlMarshalError"], err)
	}
	if outputFilename == "" {
		return d
	}
	err = ioutil.WriteFile(outputFilename, d, 0644)
	if err != nil {
		log.Fatalf("%s, Filename: %s",
			errorMessages["fileSaveError"], outputFilename)
	}
	return nil
}

func createResourceInfo(
	resourceGroup ResourceGroup) map[string]resreader.ResourceInfo {
	resInfo := make(map[string]resreader.ResourceInfo)
	for _, res := range resourceGroup.Resources {
		if _, exists := resInfo[res.Source]; !exists {
			reader := resreader.Factory(res.Kind)
			ri, err := reader.GetInfo(res.Source)
			if err != nil {
				log.Fatalf(
					"failed to get info for resource at %s while setting bc.ResourcesInfo: %e",
					res.Source, err)
			}
			resInfo[res.Source] = ri
		}
	}
	return resInfo
}

// setResourcesInfo populates needed information from resources.
func (bc *BlueprintConfig) setResourcesInfo() {
	bc.ResourcesInfo = make(map[string]map[string]resreader.ResourceInfo)
	for _, grp := range bc.Config.ResourceGroups {
		bc.ResourcesInfo[grp.Name] = createResourceInfo(grp)
	}
}

func validateGroupName(name string, usedNames map[string]bool) {
	if name == "" {
		log.Fatal(errorMessages["emptyGroupName"])
	}
	if hasIllegalChars(name) {
		log.Fatalf("%s %s", errorMessages["illegalChars"], name)
	}
	if _, ok := usedNames[name]; ok {
		log.Fatalf(
			"%s: %s used more than once", errorMessages["duplicateGroup"], name)
	}
	usedNames[name] = true
}

// checkResourceAndGroupNames checks and imports resource and resource group IDs
// and names respectively.
func (bc *BlueprintConfig) checkResourceAndGroupNames() {
	bc.ResourceToGroup = make(map[string]int)
	groupNames := make(map[string]bool)
	for iGrp, grp := range bc.Config.ResourceGroups {
		validateGroupName(grp.Name, groupNames)
		var groupKind string
		for _, res := range grp.Resources {
			// Verify no duplicate resource names
			if _, ok := bc.ResourceToGroup[res.ID]; ok {
				log.Fatalf(
					"%s: %s used more than once", errorMessages["duplicateID"], res.ID)
			}
			bc.ResourceToGroup[res.ID] = iGrp

			// Verify Resource Kind matches group Kind
			if groupKind == "" {
				groupKind = res.Kind
			} else if groupKind != res.Kind {
				log.Fatalf("%s: resource group %s, got: %s, wanted: %s",
					errorMessages["mixedResources"],
					grp.Name, groupKind, res.Kind)
			}
		}
	}
}

// expand expands variables and strings in the yaml config
func (bc BlueprintConfig) expand() {
	bc.addSettingsToResources()
	if err := bc.combineLabels(); err != nil {
		log.Fatalf(
			"failed to update resources labels when expanding the config: %v", err)
	}

	if err := bc.applyGlobalVariables(); err != nil {
		log.Fatalf(
			"failed to apply global variables in resources when expanding the config: %v",
			err)
	}
	bc.expandVariables()
}

func (bc BlueprintConfig) validate() {
	if err := bc.validateVars(); err != nil {
		log.Fatal(err)
	}
	if err := bc.validateResources(); err != nil {
		log.Fatal(err)
	}
	if err := bc.validateResourceSettings(); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Add check for use resources IDs<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package config manages and updates the ghpc input config
package config

import (
	"fmt"
	"io/ioutil"
	"log"

	"gopkg.in/yaml.v2"

	"hpc-toolkit/pkg/resreader"
)

const expectedVarFormat = "$(vars.var_name) or $(resource_id.var_name)"

var errorMessages = map[string]string{
	// general
	"notImplemented": "not yet implemented",
	// config
	"fileLoadError":      "failed to read the input yaml",
	"yamlUnmarshalError": "failed to unmarshal the yaml config",
	"yamlMarshalError":   "failed to marshal the yaml config",
	"fileSaveError":      "failed to write the expanded yaml",
	// expand
	"missingSetting":    "a required setting is missing from a resource",
	"globalLabelType":   "global labels are not a map",
	"settingsLabelType": "labels in resources settings are not a map",
	"invalidVar":        "invalid variable definition in",
	"varNotFound":       "Could not find source of variable",
	"noOutput":          "Output not found for a variable",
	// validator
	"emptyID":         "a resource id cannot be empty",
	"emptySource":     "a resource source cannot be empty",
	"wrongKind":       "a resource kind is invalid",
	"extraSetting":    "a setting was added that is not found in the resource",
	"mixedResourcees": "mixing resources of differing kinds in a resource group is not supported",
	"duplicateGroup":  "group names must be unique",
	"duplicateID":     "resource IDs must be unique",
	"emptyGroupName":  "group name must be set for each resource group",
	"illegalChars":    "invalid character(s) found in group name",
}

// ResourceGroup defines a group of Resource that are all executed together
type ResourceGroup struct {
	Name             string           `yaml:"group"`
	TerraformBackend TerraformBackend `yaml:"terraform_backend"`
	Resources        []Resource
}

// TerraformBackend defines the configuration for the terraform state backend
type TerraformBackend struct {
	Type          string
	Configuration map[string]interface{}
}

// HasKind checks to see if a resource group contains any resources of the given
// kind. Note that a resourceGroup should never have more than one kind, this
// function is used in the validation step to ensure that is true.
func (g ResourceGroup) HasKind(kind string) bool {
	for _, res := range g.Resources {
		if res.Kind == kind {
			return true
		}
	}
	return false
}

// Resource stores YAML definition of a resource
type Resource struct {
	Source       string
	Kind         string
	ID           string
	ResourceName string
	Use          []string
	Settings     map[string]interface{}
}

// YamlConfig stores the contents on the User YAML
type YamlConfig struct {
	BlueprintName            string `yaml:"blueprint_name"`
	Vars                     map[string]interface{}
	ResourceGroups           []ResourceGroup  `yaml:"resource_groups"`
	TerraformBackendDefaults TerraformBackend `yaml:"terraform_backend_defaults"`
}

// BlueprintConfig is a container for the imported YAML data and supporting data for
// creating the blueprint from it
type BlueprintConfig struct {
	Config YamlConfig
	// Indexed by Resource Group name and Resource Source
	ResourcesInfo map[string]map[string]resreader.ResourceInfo
	// Maps resource ID to group index
	ResourceToGroup map[string]int
	expanded        bool
}

// ExpandConfig expands the yaml config in place
func (bc *BlueprintConfig) ExpandConfig() {
	bc.setResourcesInfo()
	bc.validateConfig()
	bc.expand()
	bc.validate()
	bc.expanded = true
}

// NewBlueprintConfig is a constructor for BlueprintConfig
func NewBlueprintConfig(configFilename string) BlueprintConfig {
	newBlueprintConfig := BlueprintConfig{
		Config: importYamlConfig(configFilename),
	}
	return newBlueprintConfig
}

// ImportYamlConfig imports the blueprint configuration provided.
func importYamlConfig(yamlConfigFilename string) YamlConfig {
	yamlConfigText, err := ioutil.ReadFile(yamlConfigFilename)
	if err != nil {
		log.Fatalf("%s, filename=%s: %v",
			errorMessages["fileLoadError"], yamlConfigFilename, err)
	}

	var yamlConfig YamlConfig
	err = yaml.UnmarshalStrict(yamlConfigText, &yamlConfig)

	if err != nil {
		log.Fatalf("%s filename=%s: %v",
			errorMessages["yamlUnmarshalError"], yamlConfigFilename, err)
	}

	// Ensure Vars is not a nil map if not set by the user
	if len(yamlConfig.Vars) == 0 {
		yamlConfig.Vars = make(map[string]interface{})
	}

	// 1. DEFAULT: use TerraformBackend configuration (if supplied) in each
	//    resource group
	// 2. If top-level TerraformBackendDefaults is defined, insert that
	//    backend into resource groups which have no explicit
	//    TerraformBackend
	// 3. In all cases, add a prefix for GCS backends if one is not defined
	if yamlConfig.TerraformBackendDefaults.Type != "" {
		for i, grp := range yamlConfig.ResourceGroups {
			if grp.TerraformBackend.Type == "" {
				grp.TerraformBackend = yamlConfig.TerraformBackendDefaults
			}
			if grp.TerraformBackend.Type == "gcs" && grp.TerraformBackend.Configuration["prefix"] == nil {
				DeploymentName := yamlConfig.Vars["deployment_name"]
				prefix := yamlConfig.BlueprintName
				if DeploymentName != nil {
					prefix += "/" + DeploymentName.(string)
				}
				prefix += "/" + grp.Name
				grp.TerraformBackend.Configuration["prefix"] = prefix
			}
			yamlConfig.ResourceGroups[i].TerraformBackend = grp.TerraformBackend
		}
	}

	return yamlConfig
}

// ExportYamlConfig exports the internal representation of a blueprint config
func (bc BlueprintConfig) ExportYamlConfig(outputFilename string) []byte {
	d, err := yaml.Marshal(&bc.Config)
	if err != nil {
		log.Fatalf("%s: %v", errorMessages["yamlMarshalError"], err)
	}
	if outputFilename == "" {
		return d
	}
	err = ioutil.WriteFile(outputFilename, d, 0644)
	if err != nil {
		log.Fatalf("%s, Filename: %s",
			errorMessages["fileSaveError"], outputFilename)
	}
	return nil
}

func createResourceInfo(
	resourceGroup ResourceGroup) map[string]resreader.ResourceInfo {
	resInfo := make(map[string]resreader.ResourceInfo)
	for _, res := range resourceGroup.Resources {
		if _, exists := resInfo[res.Source]; !exists {
			reader := resreader.Factory(res.Kind)
			ri, err := reader.GetInfo(res.Source)
			if err != nil {
				log.Fatalf(
					"failed to get info for resource at %s while setting bc.ResourcesInfo: %e",
					res.Source, err)
			}
			resInfo[res.Source] = ri
		}
	}
	return resInfo
}

// setResourcesInfo populates needed information from resources.
func (bc *BlueprintConfig) setResourcesInfo() {
	bc.ResourcesInfo = make(map[string]map[string]resreader.ResourceInfo)
	for _, grp := range bc.Config.ResourceGroups {
		bc.ResourcesInfo[grp.Name] = createResourceInfo(grp)
	}
}

func validateGroupName(name string, usedNames map[string]bool) {
	if name == "" {
		log.Fatal(errorMessages["emptyGroupName"])
	}
	if hasIllegalChars(name) {
		log.Fatalf("%s %s", errorMessages["illegalChars"], name)
	}
	if _, ok := usedNames[name]; ok {
		log.Fatalf(
			"%s: %s used more than once", errorMessages["duplicateGroup"], name)
	}
	usedNames[name] = true
}

// checkResourceAndGroupNames checks and imports resource and resource group IDs
// and names respectively.
func (bc *BlueprintConfig) checkResourceAndGroupNames() {
	bc.ResourceToGroup = make(map[string]int)
	groupNames := make(map[string]bool)
	for iGrp, grp := range bc.Config.ResourceGroups {
		validateGroupName(grp.Name, groupNames)
		var groupKind string
		for _, res := range grp.Resources {
			// Verify no duplicate resource names
			if _, ok := bc.ResourceToGroup[res.ID]; ok {
				log.Fatalf(
					"%s: %s used more than once", errorMessages["duplicateID"], res.ID)
			}
			bc.ResourceToGroup[res.ID] = iGrp

			// Verify Resource Kind matches group Kind
			if groupKind == "" {
				groupKind = res.Kind
			} else if groupKind != res.Kind {
				log.Fatalf("%s: resource group %s, got: %s, wanted: %s",
					errorMessages["mixedResources"],
					grp.Name, groupKind, res.Kind)
			}
		}
	}
}

func checkUsedResourceNames(
	resGroups []ResourceGroup, idToGroup map[string]int) error {
	for iGrp, grp := range resGroups {
		for _, res := range grp.Resources {
			for _, usedRes := range res.Use {
				// Check if resource even exists
				if _, ok := idToGroup[usedRes]; !ok {
					return fmt.Errorf("used resource ID %s does not exist", usedRes)
				}
				// Ensure resource is from the correct group
				if idToGroup[usedRes] != iGrp {
					return fmt.Errorf(
						"used resource ID %s not found in this Resource Group", usedRes)
				}
			}
		}
	}
	return nil
}

func (bc *BlueprintConfig) validateConfig() {
	bc.checkResourceAndGroupNames()
	if err := checkUsedResourceNames(
		bc.Config.ResourceGroups, bc.ResourceToGroup); err != nil {
		log.Fatal(err)
	}
}

// expand expands variables and strings in the yaml config
func (bc BlueprintConfig) expand() {
	bc.addSettingsToResources()
	if err := bc.combineLabels(); err != nil {
		log.Fatalf(
			"failed to update resources labels when expanding the config: %v", err)
	}

	if err := bc.applyGlobalVariables(); err != nil {
		log.Fatalf(
			"failed to apply global variables in resources when expanding the config: %v",
			err)
	}
	bc.expandVariables()
}

func (bc BlueprintConfig) validate() {
	if err := bc.validateVars(); err != nil {
		log.Fatal(err)
	}
	if err := bc.validateResources(); err != nil {
		log.Fatal(err)
	}
	if err := bc.validateResourceSettings(); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>