<commit_before>package copy

import (
	"strings"
)

// File Meta data representation
type Meta struct {
	Id                      string                  `json:"id,omitempty"`
	Path                    string                  `json:"path,omitempty"`
	Name                    string                  `json:"name,omitempty"`
	LinkName                string                  `json:"link_name,omitempty"`
	Token                   string                  `json:"token,omitempty"`
	Permissions             string                  `json:"permissions,omitempty"`
	Public                  bool                    `json:"public,omitempty"`
	Type                    string                  `json:"type,omitempty"`
	Size                    int                     `json:"size,omitempty"`
	DateLastSynced          int                     `json:"date_last_synced,omitempty"`
	Stub                    bool                    `json:"stub,omitempty"`
	Share                   bool                    `json:"share,omitempty"`
	Children                []Meta                  `json:"children,omitempty"` // Inception :D
	Counts                  Count                   `json:"counts,omitempty"`   // Array? (sometimes? ask copy.com)
	RecipientConfirmed      bool                    `json:""recipient_confirmed",omitempty"`
	ObjectAvailable         bool                    `json:"object_available,omitempty"`
	Links                   []Link                  `json:"links,omitempty"`
	Revisions               []Revision              `json:"revisions,omitempty"`
	Url                     string                  `json:"url,omitempty"`
	RevisionId              int                     `json:"revision_id,omitempty"`
	Thumb                   string                  `json:"thumb,omitempty"`
	ThumbOriginalDimensions ThumbOriginalDimensions `json:"thumb_original_dimensions,omitempty"`
	ChildrenCount           int                     `json:"children_count",omitempty"`
}

type Count struct {
	New    int `json:"new,omitempty"`
	Viewed int `json:"viewed,omitempty"`
	Hidden int `json:"hidden,omitempty"`
}

type Link struct {
	Id                    string      `json:"id,omitempty"`
	Public                bool        `json:"public,omitempty"`
	Expires               bool        `json:"expires,omitempty"`
	Expired               bool        `json:"expired,omitempty"`
	Url                   string      `json:"url,omitempty"`
	UrlShort              string      `json:"url_short,omitempty"`
	Recipients            []Recipient `json:"recipients,omitempty"`
	CreatorId             string      `json:"creator_id,omitempty"`
	Confirmation_required bool        `json:"confirmation_required,omitempty"`
}
type Recipient struct {
	ContactType   string  `json:"contact_type,omitempty"`
	ContactId     string  `json:"contact_id,omitempty"`
	ContactSource string  `json:"contact_source,omitempty"`
	UserId        string  `json:"user_id,omitempty"`
	FirstName     string  `json:"first_name,omitempty"`
	LastName      string  `json:"last_name,omitempty"`
	Email         string  `json:"email,omitempty"`
	Permissions   string  `json:"permissions,omitempty"`
	Emails        []Email `json:"emails,omitempty"` // In users.go
}

type ThumbOriginalDimensions struct {
	Width  int `json:"width,omitempty"`
	Height int `json:"Height,omitempty"`
}

type Revision struct {
	RevisionId   string  `json:"revision_id,omitempty"`
	ModifiedTime string  `json:"modified_time,omitempty"`
	Size         int     `json:"size,omitempty"`
	Latest       bool    `json:"latest,omitempty"`
	Conflict     int     `json:"conflict,omitempty"`
	Id           string  `json:"id,omitempty"`
	Type         string  `json:"type,omitempty"`
	Creator      Creator `json:"creator,omitempty"`
}

type Creator struct {
	UserId      string `json:"user_id,omitempty"`
	CreatedTime int    `json:"created_time,omitempty"`
	Email       string `json:"email,omitempty"`
	FirstName   string `json:"first_name,omitempty"`
	LastName    string `json:"last_name,omitempty"`
	Confirmed   bool   `json:"confirmed,omitempty"`
}

type FileService struct {
	client *Client
}

var (
	metaTopLevelSuffix  = "meta"
	firstLevelSuffix    = strings.Join([]string{metaTopLevelSuffix, "copy"}, "/")
	filesTopLevelSuffix = "files"
)

func NewFileService(client *Client) *FileService {
	fs := new(FileService)
	fs.client = client
	return fs
}

// Returns the top level metadata (this is root folder, cannot change, see docs)
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetTopLevelMeta() (*Meta, error) {
	meta := new(Meta)
	fs.client.Do("GET", metaTopLevelSuffix, nil, meta)
	return meta, nil
}
<commit_msg>Fixed some fields in the file object<commit_after>package copy

import (
	"strings"
)

// File Meta data representation
type Meta struct {
	Id                      string                  `json:"id,omitempty"`
	Path                    string                  `json:"path,omitempty"`
	Name                    string                  `json:"name,omitempty"`
	LinkName                string                  `json:"link_name,omitempty"`
	Token                   string                  `json:"token,omitempty"`
	Permissions             string                  `json:"permissions,omitempty"`
	Public                  bool                    `json:"public,omitempty"`
	Type                    string                  `json:"type,omitempty"`
	Size                    int                     `json:"size,omitempty"`
	DateLastSynced          int                     `json:"date_last_synced,omitempty"`
	Stub                    bool                    `json:"stub,omitempty"`
	Share                   bool                    `json:"share,omitempty"`
	Children                []Meta                  `json:"children,omitempty"` // Inception :D
	Counts                  Count                   `json:"counts,omitempty"`   // Array? (sometimes? ask copy.com)
	RecipientConfirmed      bool                    `json:"recipient_confirmed",omitempty"`
	ObjectAvailable         bool                    `json:"object_available,omitempty"`
	Links                   []Link                  `json:"links,omitempty"`
	Revisions               []Revision              `json:"revisions,omitempty"`
	Url                     string                  `json:"url,omitempty"`
	RevisionId              int                     `json:"revision_id,omitempty"`
	Thumb                   string                  `json:"thumb,omitempty"`
	ThumbOriginalDimensions ThumbOriginalDimensions `json:"thumb_original_dimensions,omitempty"`
	ChildrenCount           int                     `json:"children_count",omitempty"`
}

type Count struct {
	New    int `json:"new,omitempty"`
	Viewed int `json:"viewed,omitempty"`
	Hidden int `json:"hidden,omitempty"`
}

type Link struct {
	Id                   string      `json:"id,omitempty"`
	Public               bool        `json:"public,omitempty"`
	Expires              bool        `json:"expires,omitempty"`
	Expired              bool        `json:"expired,omitempty"`
	Url                  string      `json:"url,omitempty"`
	UrlShort             string      `json:"url_short,omitempty"`
	Recipients           []Recipient `json:"recipients,omitempty"`
	CreatorId            string      `json:"creator_id,omitempty"`
	ConfirmationRequired bool        `json:"confirmation_required,omitempty"`
}
type Recipient struct {
	ContactType   string  `json:"contact_type,omitempty"`
	ContactId     string  `json:"contact_id,omitempty"`
	ContactSource string  `json:"contact_source,omitempty"`
	UserId        string  `json:"user_id,omitempty"`
	FirstName     string  `json:"first_name,omitempty"`
	LastName      string  `json:"last_name,omitempty"`
	Email         string  `json:"email,omitempty"`
	Permissions   string  `json:"permissions,omitempty"`
	Emails        []Email `json:"emails,omitempty"` // In users.go
}

type ThumbOriginalDimensions struct {
	Width  int `json:"width,omitempty"`
	Height int `json:"Height,omitempty"`
}

type Revision struct {
	RevisionId   string  `json:"revision_id,omitempty"`
	ModifiedTime string  `json:"modified_time,omitempty"`
	Size         int     `json:"size,omitempty"`
	Latest       bool    `json:"latest,omitempty"`
	Conflict     int     `json:"conflict,omitempty"`
	Id           string  `json:"id,omitempty"`
	Type         string  `json:"type,omitempty"`
	Creator      Creator `json:"creator,omitempty"`
}

type Creator struct {
	UserId      string `json:"user_id,omitempty"`
	CreatedTime int    `json:"created_time,omitempty"`
	Email       string `json:"email,omitempty"`
	FirstName   string `json:"first_name,omitempty"`
	LastName    string `json:"last_name,omitempty"`
	Confirmed   bool   `json:"confirmed,omitempty"`
}

type FileService struct {
	client *Client
}

var (
	metaTopLevelSuffix  = "meta"
	firstLevelSuffix    = strings.Join([]string{metaTopLevelSuffix, "copy"}, "/")
	filesTopLevelSuffix = "files"
)

func NewFileService(client *Client) *FileService {
	fs := new(FileService)
	fs.client = client
	return fs
}

// Returns the top level metadata (this is root folder, cannot change, see docs)
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) GetTopLevelMeta() (*Meta, error) {
	meta := new(Meta)
	fs.client.Do("GET", metaTopLevelSuffix, nil, meta)
	return meta, nil
}

// Returns the metadata of the provided path as String
//
// https://www.copy.com/developer/documentation#api-calls/filesystem
func (fs *FileService) Get(path string) (*Meta, error) {
	meta := new(Meta)
	fs.client.Do("GET", strings.Join([]string{firstLevelSuffix, path}, "/"), nil, meta)
	return meta, nil
}
<|endoftext|>


<commit_before>// Package core is the main package contents of the GoCore application collection of packages and utilities
// Also the root contains some debugging/dumping variable functions
package core

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"reflect"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"github.com/DanielRenne/GoCore/core/extensions"
	"github.com/DanielRenne/GoCore/core/serverSettings"
	"github.com/davidrenne/reflections"
	"github.com/go-errors/errors"
)

type core_debug struct{}

var core_logger = log.New(os.Stdout, "", 0)

// TransactionLog provides a thread-safe buffer/string if you have serverSettings.WebConfig.Application.ReleaseMode == "development" you can call something like core.Debug.GetDump in many places and then read the TransactionLog when you need to (note you must manually clear it as it will just increase your memory usage the more logs are sent)
var TransactionLog string

// Debug is a base struct for all debug functions.
var Debug = core_debug{}

// Logger can be overridden with log.New(os.Stdout, "", 0) to log to stdout or some other writer
var Logger = core_logger

// TransactionLogMutex is a mutex for the TransactionLog which should be used on your end to clear the value safely
var TransactionLogMutex *sync.RWMutex

func init() {
	TransactionLogMutex = &sync.RWMutex{}
}

// Nop is a dummy function that can be called in source files where
// other debug functions are constantly added and removed.
// That way import "github.com/ungerik/go-start/debug" won't cause an error when
// no other debug function is currently used.
// Arbitrary objects can be passed as arguments to avoid "declared and not used"
// error messages when commenting code out and in.
// The result is a nil interface{} dummy value.
func (self *core_debug) Nop(dummiesIn ...interface{}) (dummyOut interface{}) {
	return nil
}

// CallStackInfo returns a string with the call stack info.
func (self *core_debug) CallStackInfo(skip int) (info string) {
	pc, file, line, ok := runtime.Caller(skip)
	if ok {
		funcName := runtime.FuncForPC(pc).Name()
		info += fmt.Sprintf("In function %s()", funcName)
	}
	for i := 0; ok; i++ {
		info += fmt.Sprintf("\n%s:%d", file, line)
		_, file, line, ok = runtime.Caller(skip + i)
	}
	return info
}

// PrintCallStack prints the call stack info.
func (self *core_debug) PrintCallStack() {
	debug.PrintStack()
}

// LogCallStack logs the call stack info.
func (self *core_debug) LogCallStack() {
	log.Print(self.Stack())
}

func (self *core_debug) Stack() string {
	return string(debug.Stack())
}

func (self *core_debug) formatValue(value interface{}) string {
	return fmt.Sprintf("\n     Type: %T\n    Value: %v\nGo Syntax: %#v", value, value, value)
}

func (self *core_debug) formatCallstack(skip int) string {
	return fmt.Sprintf("\nCallstack: %s", self.CallStackInfo(skip+1))
}

// FormatSkip formats a value with callstack info.
func (self *core_debug) FormatSkip(skip int, value interface{}) string {
	return self.formatValue(value) + self.formatCallstack(skip+1)
}

// Format formats a value with callstack info.
func (self *core_debug) Format(value interface{}) string {
	return self.FormatSkip(2, value)
}

// IsZeroOfUnderlyingType returns true if the value is the zero value (nil) for its type.
func IsZeroOfUnderlyingType(x interface{}) bool {
	return reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface())
}

// IsZeroOfUnderlyingType2 returns true if the value is the zero value (nil) for its type.
func IsZeroOfUnderlyingType2(x interface{}) bool {
	return x == reflect.Zero(reflect.TypeOf(x)).Interface()
}

// HandleError is a helper function that will log an error and return it with the callers line and file.
func (self *core_debug) HandleError(err error) (s string) {
	if err != nil {
		// notice that we're using 1, so it will actually log the where
		// the error happened, 0 = this function, we don't want that.
		_, fn, line, _ := runtime.Caller(1)
		fileNameParts := strings.Split(fn, "/")
		return fmt.Sprintf("  Error Info: %s Line %d. ErrorType: %v", fileNameParts[len(fileNameParts)-1], line, err)
	}
	return ""
}

// ErrLineAndFile returns the line and file of the error.
func (self *core_debug) ErrLineAndFile(err error) (s string) {
	if err != nil {
		// notice that we're using 1, so it will actually log the where
		// the error happened, 0 = this function, we don't want that.
		_, fn, line, _ := runtime.Caller(1)
		fileNameParts := strings.Split(fn, "/")
		return fmt.Sprintf("%s Line %d", fileNameParts[len(fileNameParts)-1], line)
	}
	return ""
}

// Dump is a helper function that will log unlimited values to print to stdout or however you have log setup if you overload core/Logger
func (self *core_debug) Dump(valuesOriginal ...interface{}) {
	t := time.Now()
	l := "!!!!!!!!!!!!! DEBUG " + t.Format("2006-01-02 15:04:05.000000") + "!!!!!!!!!!!!!\n\n"
	Logger.Println(l)

	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
	for _, value := range valuesOriginal {
		l := self.dumpBase(value)
		Logger.Print(l)
		serverSettings.WebConfigMutex.RLock()
		if serverSettings.WebConfig.Application.ReleaseMode == "development" {
			TransactionLogMutex.Lock()
			TransactionLog += l
			TransactionLogMutex.Unlock()
		}
		serverSettings.WebConfigMutex.RUnlock()
	}
	l = self.ThrowAndPrintError()
	Logger.Print(l)

	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
	l = "!!!!!!!!!!!!! ENDDEBUG " + t.Format("2006-01-02 15:04:05.000000") + "!!!!!!!!!!!!!"
	Logger.Println(l)
	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
}

// GetDump is a helper function that will log unlimited values which will return a string representation of what was logged
func (self *core_debug) GetDump(valuesOriginal ...interface{}) (output string) {
	for _, value := range valuesOriginal {
		output += self.dumpBase(value)
	}
	//output += self.ThrowAndPrintError()
	return output
}

func (self *core_debug) GetDumpWithInfo(valuesOriginal ...interface{}) (output string) {
	t := time.Now()
	return self.GetDumpWithInfoAndTimeString(t.String(), valuesOriginal...)
}

// GetDumpWithInfoAndTimeString is a helper function that will log unlimited values which will return a string representation of what was logged but allows you to pass your own time string in a case of timezone offsets
func (self *core_debug) GetDumpWithInfoAndTimeString(timeStr string, valuesOriginal ...interface{}) (output string) {
	l := "\n!!!!!!!!!!!!! DEBUG " + timeStr + "!!!!!!!!!!!!!\n\n"
	output += l

	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()

	for _, value := range valuesOriginal {
		output += self.dumpBase(value) + "\n"
	}

	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += output
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()

	l = self.ThrowAndPrintError()
	output += l
	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
	l = "!!!!!!!!!!!!! ENDDEBUG " + timeStr + "!!!!!!!!!!!!!\n"
	output += l
	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
	return output
}

func (self *core_debug) dumpBase(values ...interface{}) (output string) {
	var jsonString string
	var err error
	var structKeys []string
	if Logger != nil {
		for _, value := range values {
			isAllJSON := true
			var kind string
			kind = strings.TrimSpace(fmt.Sprintf("%T", value))
			// var pieces = strings.Split(kind, " ")
			// if pieces[0] == "struct" || strings.Index(pieces[0], "model.") != -1 || strings.Index(pieces[0], "viewModel.") != -1 {
			if !IsZeroOfUnderlyingType(value) {
				kind = reflections.ReflectKind(value)
				structKeys, err = reflections.FieldsDeep(value)
				if err == nil {
					for _, field := range structKeys {
						jsonString, err = reflections.GetFieldTag(value, field, "json")
						if err != nil {
							isAllJSON = false
						}
						if jsonString == "" {
							isAllJSON = false
						}
					}
				} else {
					isAllJSON = false
				}
			} else {
				isAllJSON = false
			}

			if isAllJSON || kind == "map" || kind == "bson.M" || kind == "slice" {
				var rawBytes []byte
				rawBytes, err = json.MarshalIndent(value, "", "\t")
				if err == nil {
					if kind == "slice" || kind[:2] == "[]" {
						valReflected := reflect.ValueOf(value)
						output += fmt.Sprintf("#### %-39s [len:%s]####\n%+v", kind, extensions.IntToString(valReflected.Len()), string(rawBytes[:]))
					} else {
						output += fmt.Sprintf("#### %-39s ####\n%+v", kind, string(rawBytes[:]))
					}
				}
			} else {
				if strings.TrimSpace(kind) == "string" {
					var stringVal = fmt.Sprintf("%+v", value)
					position := strings.Index(stringVal, "Desc->")
					if position == -1 {
						if !extensions.IsPrintable(stringVal) {
							kind += " (non printables -> dump hex)"
							stringVal = hex.Dump([]byte(stringVal))
						}
						valReflected := reflect.ValueOf(value)
						output += fmt.Sprintf("#### %-39s [len:%s]####\n%s", kind, extensions.IntToString(valReflected.Len()), stringVal)
					} else {
						output += stringVal[6:] + " --> "
					}
				} else if kind[:2] == "[]" || strings.TrimSpace(kind) == "array" {
					valReflected := reflect.ValueOf(value)
					output += fmt.Sprintf("#### %-39s [len:%s]####\n%+v", kind, extensions.IntToString(valReflected.Len()), value)
				} else {
					output += fmt.Sprintf("#### %-39s ####\n%+v", kind, value)
				}
			}
		}
	}
	return output
}

// ThrowAndPrintError is a helper function that will throw a fake error and get the callstack and return it as a string (you probably shouldnt use this)
func (self *core_debug) ThrowAndPrintError() (output string) {

	serverSettings.WebConfigMutex.RLock()
	ok := serverSettings.WebConfig.Application.CoreDebugStackTrace
	serverSettings.WebConfigMutex.RUnlock()
	if ok {
		output += "\n"
		errorInfo := self.ThrowError()
		stack := strings.Split(errorInfo.ErrorStack(), "\n")
		if len(stack) >= 8 {
			output += "\nDump Caller:"
			output += "\n---------------"
			//output += strings.Join(stack, ",")
			output += "\n golines ==> " + strings.TrimSpace(stack[6])
			output += "\n         ==> " + strings.TrimSpace(stack[7])
			output += "\n         ==> " + strings.TrimSpace(stack[8])
			output += "\n         ==> " + strings.TrimSpace(stack[9])
			output += "\n         ==> " + strings.TrimSpace(stack[10])
			if len(stack) >= 12 {
				output += "\n         ==> " + strings.TrimSpace(stack[11])
			}
			if len(stack) >= 13 {
				output += "\n         ==> " + strings.TrimSpace(stack[12])
			}
			if len(stack) >= 14 {
				output += "\n         ==> " + strings.TrimSpace(stack[13])
			}
			if len(stack) >= 15 {
				output += "\n         ==> " + strings.TrimSpace(stack[14])
			}
			if len(stack) >= 16 {
				output += "\n         ==> " + strings.TrimSpace(stack[15])
			}
			output += "\n---------------"
			output += "\n"
			output += "\n"
		}
	}
	return output
}

// ThrowError is a helper function that will throw a fake error and get the callstack and return it as an error (you probably shouldnt use this)
func (self *core_debug) ThrowError() *errors.Error {
	return errors.Errorf("Debug Dump")
}

// GetDump is a helper function that will return a string of the dump of the values passed in
func GetDump(valuesOriginal ...interface{}) string {
	return Debug.GetDump(valuesOriginal...)
}

// Dump is a helper function that will dump the values passed to it
func Dump(valuesOriginal ...interface{}) {
	Debug.Dump(valuesOriginal...)
}
<commit_msg>Russ Cox's Emailed me and his 2012 talk showed me: %#v<commit_after>// Package core is the main package contents of the GoCore application collection of packages and utilities
// Also the root contains some debugging/dumping variable functions
package core

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"log"
	"os"
	"reflect"
	"runtime"
	"runtime/debug"
	"strings"
	"sync"
	"time"

	"github.com/DanielRenne/GoCore/core/extensions"
	"github.com/DanielRenne/GoCore/core/serverSettings"
	"github.com/davidrenne/reflections"
	"github.com/go-errors/errors"
)

type core_debug struct{}

var core_logger = log.New(os.Stdout, "", 0)

// TransactionLog provides a thread-safe buffer/string if you have serverSettings.WebConfig.Application.ReleaseMode == "development" you can call something like core.Debug.GetDump in many places and then read the TransactionLog when you need to (note you must manually clear it as it will just increase your memory usage the more logs are sent)
var TransactionLog string

// Debug is a base struct for all debug functions.
var Debug = core_debug{}

// Logger can be overridden with log.New(os.Stdout, "", 0) to log to stdout or some other writer
var Logger = core_logger

// TransactionLogMutex is a mutex for the TransactionLog which should be used on your end to clear the value safely
var TransactionLogMutex *sync.RWMutex

func init() {
	TransactionLogMutex = &sync.RWMutex{}
}

// Nop is a dummy function that can be called in source files where
// other debug functions are constantly added and removed.
// That way import "github.com/ungerik/go-start/debug" won't cause an error when
// no other debug function is currently used.
// Arbitrary objects can be passed as arguments to avoid "declared and not used"
// error messages when commenting code out and in.
// The result is a nil interface{} dummy value.
func (self *core_debug) Nop(dummiesIn ...interface{}) (dummyOut interface{}) {
	return nil
}

// CallStackInfo returns a string with the call stack info.
func (self *core_debug) CallStackInfo(skip int) (info string) {
	pc, file, line, ok := runtime.Caller(skip)
	if ok {
		funcName := runtime.FuncForPC(pc).Name()
		info += fmt.Sprintf("In function %s()", funcName)
	}
	for i := 0; ok; i++ {
		info += fmt.Sprintf("\n%s:%d", file, line)
		_, file, line, ok = runtime.Caller(skip + i)
	}
	return info
}

// PrintCallStack prints the call stack info.
func (self *core_debug) PrintCallStack() {
	debug.PrintStack()
}

// LogCallStack logs the call stack info.
func (self *core_debug) LogCallStack() {
	log.Print(self.Stack())
}

func (self *core_debug) Stack() string {
	return string(debug.Stack())
}

func (self *core_debug) formatValue(value interface{}) string {
	return fmt.Sprintf("\n     Type: %T\n    Value: %v\nGo Syntax: %#v", value, value, value)
}

func (self *core_debug) formatCallstack(skip int) string {
	return fmt.Sprintf("\nCallstack: %s", self.CallStackInfo(skip+1))
}

// FormatSkip formats a value with callstack info.
func (self *core_debug) FormatSkip(skip int, value interface{}) string {
	return self.formatValue(value) + self.formatCallstack(skip+1)
}

// Format formats a value with callstack info.
func (self *core_debug) Format(value interface{}) string {
	return self.FormatSkip(2, value)
}

// IsZeroOfUnderlyingType returns true if the value is the zero value (nil) for its type.
func IsZeroOfUnderlyingType(x interface{}) bool {
	return reflect.DeepEqual(x, reflect.Zero(reflect.TypeOf(x)).Interface())
}

// IsZeroOfUnderlyingType2 returns true if the value is the zero value (nil) for its type.
func IsZeroOfUnderlyingType2(x interface{}) bool {
	return x == reflect.Zero(reflect.TypeOf(x)).Interface()
}

// HandleError is a helper function that will log an error and return it with the callers line and file.
func (self *core_debug) HandleError(err error) (s string) {
	if err != nil {
		// notice that we're using 1, so it will actually log the where
		// the error happened, 0 = this function, we don't want that.
		_, fn, line, _ := runtime.Caller(1)
		fileNameParts := strings.Split(fn, "/")
		return fmt.Sprintf("  Error Info: %s Line %d. ErrorType: %v", fileNameParts[len(fileNameParts)-1], line, err)
	}
	return ""
}

// ErrLineAndFile returns the line and file of the error.
func (self *core_debug) ErrLineAndFile(err error) (s string) {
	if err != nil {
		// notice that we're using 1, so it will actually log the where
		// the error happened, 0 = this function, we don't want that.
		_, fn, line, _ := runtime.Caller(1)
		fileNameParts := strings.Split(fn, "/")
		return fmt.Sprintf("%s Line %d", fileNameParts[len(fileNameParts)-1], line)
	}
	return ""
}

// Dump is a helper function that will log unlimited values to print to stdout or however you have log setup if you overload core/Logger
func (self *core_debug) Dump(valuesOriginal ...interface{}) {
	t := time.Now()
	l := "!!!!!!!!!!!!! DEBUG " + t.Format("2006-01-02 15:04:05.000000") + "!!!!!!!!!!!!!\n\n"
	Logger.Println(l)

	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
	for _, value := range valuesOriginal {
		l := self.dumpBase(value)
		Logger.Print(l)
		serverSettings.WebConfigMutex.RLock()
		if serverSettings.WebConfig.Application.ReleaseMode == "development" {
			TransactionLogMutex.Lock()
			TransactionLog += l
			TransactionLogMutex.Unlock()
		}
		serverSettings.WebConfigMutex.RUnlock()
	}
	l = self.ThrowAndPrintError()
	Logger.Print(l)

	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
	l = "!!!!!!!!!!!!! ENDDEBUG " + t.Format("2006-01-02 15:04:05.000000") + "!!!!!!!!!!!!!"
	Logger.Println(l)
	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
}

// GetDump is a helper function that will log unlimited values which will return a string representation of what was logged
func (self *core_debug) GetDump(valuesOriginal ...interface{}) (output string) {
	for _, value := range valuesOriginal {
		output += self.dumpBase(value)
	}
	//output += self.ThrowAndPrintError()
	return output
}

func (self *core_debug) GetDumpWithInfo(valuesOriginal ...interface{}) (output string) {
	t := time.Now()
	return self.GetDumpWithInfoAndTimeString(t.String(), valuesOriginal...)
}

// GetDumpWithInfoAndTimeString is a helper function that will log unlimited values which will return a string representation of what was logged but allows you to pass your own time string in a case of timezone offsets
func (self *core_debug) GetDumpWithInfoAndTimeString(timeStr string, valuesOriginal ...interface{}) (output string) {
	l := "\n!!!!!!!!!!!!! DEBUG " + timeStr + "!!!!!!!!!!!!!\n\n"
	output += l

	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()

	for _, value := range valuesOriginal {
		output += self.dumpBase(value) + "\n"
	}

	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += output
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()

	l = self.ThrowAndPrintError()
	output += l
	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
	l = "!!!!!!!!!!!!! ENDDEBUG " + timeStr + "!!!!!!!!!!!!!\n"
	output += l
	serverSettings.WebConfigMutex.RLock()
	if serverSettings.WebConfig.Application.ReleaseMode == "development" {
		TransactionLogMutex.Lock()
		TransactionLog += l
		TransactionLogMutex.Unlock()
	}
	serverSettings.WebConfigMutex.RUnlock()
	return output
}

func (self *core_debug) dumpBase(values ...interface{}) (output string) {
	var jsonString string
	var err error
	var structKeys []string
	if Logger != nil {
		for _, value := range values {
			isAllJSON := true
			var kind string
			kind = strings.TrimSpace(fmt.Sprintf("%T", value))
			// var pieces = strings.Split(kind, " ")
			// if pieces[0] == "struct" || strings.Index(pieces[0], "model.") != -1 || strings.Index(pieces[0], "viewModel.") != -1 {
			if !IsZeroOfUnderlyingType(value) {
				kind = reflections.ReflectKind(value)
				structKeys, err = reflections.FieldsDeep(value)
				if err == nil {
					for _, field := range structKeys {
						jsonString, err = reflections.GetFieldTag(value, field, "json")
						if err != nil {
							isAllJSON = false
						}
						if jsonString == "" {
							isAllJSON = false
						}
					}
				} else {
					isAllJSON = false
				}
			} else {
				isAllJSON = false
			}

			if isAllJSON || kind == "map" || kind == "bson.M" || kind == "slice" {
				var rawBytes []byte
				rawBytes, err = json.MarshalIndent(value, "", "\t")
				if err == nil {
					if kind == "slice" || kind[:2] == "[]" {
						valReflected := reflect.ValueOf(value)
						output += fmt.Sprintf("#### %-39s [len:%s]####\n%+v", kind, extensions.IntToString(valReflected.Len()), string(rawBytes[:]))
					} else {
						output += fmt.Sprintf("#### %-39s ####\n%+v", kind, string(rawBytes[:]))
					}
				}
			} else {
				if strings.TrimSpace(kind) == "string" {
					var stringVal = fmt.Sprintf("%+v", value)
					position := strings.Index(stringVal, "Desc->")
					if position == -1 {
						if !extensions.IsPrintable(stringVal) {
							kind += " (non printables -> dump hex)"
							stringVal = hex.Dump([]byte(stringVal))
						}
						valReflected := reflect.ValueOf(value)
						output += fmt.Sprintf("#### %-39s [len:%s]####\n%s", kind, extensions.IntToString(valReflected.Len()), stringVal)
					} else {
						output += stringVal[6:] + " --> "
					}
				} else if kind[:2] == "[]" || strings.TrimSpace(kind) == "array" {
					valReflected := reflect.ValueOf(value)
					output += fmt.Sprintf("#### %-39s [len:%s]####\n%#v", kind, extensions.IntToString(valReflected.Len()), value)
				} else {
					output += fmt.Sprintf("#### %-39s ####\n%#v", kind, value)
				}
			}
		}
	}
	return output
}

// ThrowAndPrintError is a helper function that will throw a fake error and get the callstack and return it as a string (you probably shouldnt use this)
func (self *core_debug) ThrowAndPrintError() (output string) {

	serverSettings.WebConfigMutex.RLock()
	ok := serverSettings.WebConfig.Application.CoreDebugStackTrace
	serverSettings.WebConfigMutex.RUnlock()
	if ok {
		output += "\n"
		errorInfo := self.ThrowError()
		stack := strings.Split(errorInfo.ErrorStack(), "\n")
		if len(stack) >= 8 {
			output += "\nDump Caller:"
			output += "\n---------------"
			//output += strings.Join(stack, ",")
			output += "\n golines ==> " + strings.TrimSpace(stack[6])
			output += "\n         ==> " + strings.TrimSpace(stack[7])
			output += "\n         ==> " + strings.TrimSpace(stack[8])
			output += "\n         ==> " + strings.TrimSpace(stack[9])
			output += "\n         ==> " + strings.TrimSpace(stack[10])
			if len(stack) >= 12 {
				output += "\n         ==> " + strings.TrimSpace(stack[11])
			}
			if len(stack) >= 13 {
				output += "\n         ==> " + strings.TrimSpace(stack[12])
			}
			if len(stack) >= 14 {
				output += "\n         ==> " + strings.TrimSpace(stack[13])
			}
			if len(stack) >= 15 {
				output += "\n         ==> " + strings.TrimSpace(stack[14])
			}
			if len(stack) >= 16 {
				output += "\n         ==> " + strings.TrimSpace(stack[15])
			}
			output += "\n---------------"
			output += "\n"
			output += "\n"
		}
	}
	return output
}

// ThrowError is a helper function that will throw a fake error and get the callstack and return it as an error (you probably shouldnt use this)
func (self *core_debug) ThrowError() *errors.Error {
	return errors.Errorf("Debug Dump")
}

// GetDump is a helper function that will return a string of the dump of the values passed in
func GetDump(valuesOriginal ...interface{}) string {
	return Debug.GetDump(valuesOriginal...)
}

// Dump is a helper function that will dump the values passed to it
func Dump(valuesOriginal ...interface{}) {
	Debug.Dump(valuesOriginal...)
}
<|endoftext|>


<commit_before>package task_test

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/lxc/lxd/lxd/task"
	"github.com/stretchr/testify/assert"
)

// The given task is executed immediately by the scheduler.
func TestTask_ExecuteImmediately(t *testing.T) {
	f, wait := newFunc(t, 1)
	defer startTask(t, f, task.Every(time.Second))()
	wait(100 * time.Millisecond)
}

// The given task is executed again after the specified time interval has
// elapsed.
func TestTask_ExecutePeriodically(t *testing.T) {
	f, wait := newFunc(t, 2)
	defer startTask(t, f, task.Every(250*time.Millisecond))()
	wait(100 * time.Millisecond)
	wait(400 * time.Millisecond)
}

// If the scheduler is reset, the task is re-executed immediately and then
// again after the interval.
func TestTask_Reset(t *testing.T) {
	f, wait := newFunc(t, 3)
	stop, reset := task.Start(context.Background(), f, task.Every(250*time.Millisecond))
	defer stop(time.Second)

	wait(50 * time.Millisecond)  // First execution, immediately
	reset()                      // Trigger a reset
	wait(50 * time.Millisecond)  // Second execution, immediately after reset
	wait(400 * time.Millisecond) // Third execution, after the timeout
}

// If the interval is zero, the task function is never run.
func TestTask_ZeroInterval(t *testing.T) {
	f, _ := newFunc(t, 0)
	defer startTask(t, f, task.Every(0*time.Millisecond))()

	// Sleep a little bit to prove that the task function does not get run.
	time.Sleep(100 * time.Millisecond)
}

// If the schedule returns an error, the task is aborted.
func TestTask_ScheduleError(t *testing.T) {
	schedule := func() (time.Duration, error) {
		return 0, fmt.Errorf("boom")
	}
	f, _ := newFunc(t, 0)
	defer startTask(t, f, schedule)()

	// Sleep a little bit to prove that the task function does not get run.
	time.Sleep(100 * time.Millisecond)
}

// If the schedule returns an error, but its interval is positive, the task will
// try again to invoke the schedule function after that interval.
func TestTask_ScheduleTemporaryError(t *testing.T) {
	errored := false
	schedule := func() (time.Duration, error) {
		if !errored {
			errored = true
			return time.Millisecond, fmt.Errorf("boom")
		}
		return time.Second, nil
	}
	f, wait := newFunc(t, 1)
	defer startTask(t, f, schedule)()

	// The task gets executed since the schedule error is temporary and gets
	// resolved.
	wait(50 * time.Millisecond)
}

// If SkipFirst is passed, the given task is only executed at the second round.
func TestTask_SkipFirst(t *testing.T) {
	i := 0
	f := func(context.Context) {
		i++
	}
	defer startTask(t, f, task.Every(250*time.Millisecond, task.SkipFirst))()
	time.Sleep(400 * time.Millisecond)
	assert.Equal(t, 1, i) // The function got executed only once, not twice.
}

// Create a new task function that sends a notification to a channel every time
// it's run.
//
// Return the task function, along with a "wait" function which will block
// until one notification is received through such channel, or fails the test
// if no notification is received within the given timeout.
//
// The n parameter can be used to limit the number of times the task function
// is allowed run: when that number is reached the task function will trigger a
// test failure (zero means that the task function will make the test fail as
// soon as it is invoked).
func newFunc(t *testing.T, n int) (task.Func, func(time.Duration)) {
	i := 0
	notifications := make(chan struct{})
	f := func(context.Context) {
		if i == n {
			t.Errorf("task was supposed to be called at most %d times", n)
		}
		notifications <- struct{}{}
		i++
	}
	wait := func(timeout time.Duration) {
		select {
		case <-notifications:
		case <-time.After(timeout):
			t.Errorf("no notification received in %s", timeout)
		}
	}
	return f, wait
}

// Convenience around task.Start which also makes sure that the stop function
// of the task actually terminates.
func startTask(t *testing.T, f task.Func, schedule task.Schedule) func() {
	stop, _ := task.Start(context.Background(), f, schedule)
	return func() {
		assert.NoError(t, stop(time.Second))
	}
}
<commit_msg>lxd/task: Checks or explicitly ignores errors.<commit_after>package task_test

import (
	"context"
	"fmt"
	"testing"
	"time"

	"github.com/lxc/lxd/lxd/task"
	"github.com/stretchr/testify/assert"
)

// The given task is executed immediately by the scheduler.
func TestTask_ExecuteImmediately(t *testing.T) {
	f, wait := newFunc(t, 1)
	defer startTask(t, f, task.Every(time.Second))()
	wait(100 * time.Millisecond)
}

// The given task is executed again after the specified time interval has
// elapsed.
func TestTask_ExecutePeriodically(t *testing.T) {
	f, wait := newFunc(t, 2)
	defer startTask(t, f, task.Every(250*time.Millisecond))()
	wait(100 * time.Millisecond)
	wait(400 * time.Millisecond)
}

// If the scheduler is reset, the task is re-executed immediately and then
// again after the interval.
func TestTask_Reset(t *testing.T) {
	f, wait := newFunc(t, 3)
	stop, reset := task.Start(context.Background(), f, task.Every(250*time.Millisecond))
	defer func() { _ = stop(time.Second) }()

	wait(50 * time.Millisecond)  // First execution, immediately
	reset()                      // Trigger a reset
	wait(50 * time.Millisecond)  // Second execution, immediately after reset
	wait(400 * time.Millisecond) // Third execution, after the timeout
}

// If the interval is zero, the task function is never run.
func TestTask_ZeroInterval(t *testing.T) {
	f, _ := newFunc(t, 0)
	defer startTask(t, f, task.Every(0*time.Millisecond))()

	// Sleep a little bit to prove that the task function does not get run.
	time.Sleep(100 * time.Millisecond)
}

// If the schedule returns an error, the task is aborted.
func TestTask_ScheduleError(t *testing.T) {
	schedule := func() (time.Duration, error) {
		return 0, fmt.Errorf("boom")
	}
	f, _ := newFunc(t, 0)
	defer startTask(t, f, schedule)()

	// Sleep a little bit to prove that the task function does not get run.
	time.Sleep(100 * time.Millisecond)
}

// If the schedule returns an error, but its interval is positive, the task will
// try again to invoke the schedule function after that interval.
func TestTask_ScheduleTemporaryError(t *testing.T) {
	errored := false
	schedule := func() (time.Duration, error) {
		if !errored {
			errored = true
			return time.Millisecond, fmt.Errorf("boom")
		}
		return time.Second, nil
	}
	f, wait := newFunc(t, 1)
	defer startTask(t, f, schedule)()

	// The task gets executed since the schedule error is temporary and gets
	// resolved.
	wait(50 * time.Millisecond)
}

// If SkipFirst is passed, the given task is only executed at the second round.
func TestTask_SkipFirst(t *testing.T) {
	i := 0
	f := func(context.Context) {
		i++
	}
	defer startTask(t, f, task.Every(250*time.Millisecond, task.SkipFirst))()
	time.Sleep(400 * time.Millisecond)
	assert.Equal(t, 1, i) // The function got executed only once, not twice.
}

// Create a new task function that sends a notification to a channel every time
// it's run.
//
// Return the task function, along with a "wait" function which will block
// until one notification is received through such channel, or fails the test
// if no notification is received within the given timeout.
//
// The n parameter can be used to limit the number of times the task function
// is allowed run: when that number is reached the task function will trigger a
// test failure (zero means that the task function will make the test fail as
// soon as it is invoked).
func newFunc(t *testing.T, n int) (task.Func, func(time.Duration)) {
	i := 0
	notifications := make(chan struct{})
	f := func(context.Context) {
		if i == n {
			t.Errorf("task was supposed to be called at most %d times", n)
		}
		notifications <- struct{}{}
		i++
	}
	wait := func(timeout time.Duration) {
		select {
		case <-notifications:
		case <-time.After(timeout):
			t.Errorf("no notification received in %s", timeout)
		}
	}
	return f, wait
}

// Convenience around task.Start which also makes sure that the stop function
// of the task actually terminates.
func startTask(t *testing.T, f task.Func, schedule task.Schedule) func() {
	stop, _ := task.Start(context.Background(), f, schedule)
	return func() {
		assert.NoError(t, stop(time.Second))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/olekukonko/tablewriter"
	"github.com/pinzolo/dbmodel"
)

type markdownPublisher struct {
	cfg    *Config
	cnv    Converter
	errors []error
}

func newMarkdownPublisher(config *Config, converter Converter) markdownPublisher {
	return markdownPublisher{
		cfg:    config,
		cnv:    converter,
		errors: make([]error, 0, 0),
	}
}

func (p markdownPublisher) Publish(tables []*dbmodel.Table) {
	path, err := resolvePath(p.cfg.Out)
	if err != nil {
		p.errors = append(p.errors, err)
		return
	}
	if err := cleanDir(path); err != nil {
		p.errors = append(p.errors, err)
		return
	}

	for _, tbl := range tables {
		md := p.convertToMarkdown(tbl)
		if f, err := os.Create(filepath.Join(path, tbl.Name()+".md")); err != nil {
			p.errors = append(p.errors, err)
		} else {
			f.Write([]byte(md))
			err = f.Close()
			if err != nil {
				p.errors = append(p.errors, err)
			}
		}
	}

	idxMd := p.convertToIndexMarkdown(tables)
	if f, err := os.Create(filepath.Join(path, "00_index.md")); err != nil {
		p.errors = append(p.errors, err)
	} else {
		f.Write([]byte(idxMd))
		err = f.Close()
		if err != nil {
			p.errors = append(p.errors, err)
		}
	}
}

func (p markdownPublisher) Errors() []error {
	return p.errors
}

func (p markdownPublisher) convertToIndexMarkdown(tables []*dbmodel.Table) string {
	buf := &bytes.Buffer{}
	nl := fmt.Sprintln()

	fmt.Fprintln(buf, "# Index"+nl)
	for _, tbl := range tables {
		if tbl.Comment() == "" {
			fmt.Fprintf(buf, "* [%s](%s.md)%s", tbl.Name(), tbl.Name(), nl)
		} else {
			fmt.Fprintf(buf, "* [%s](%s.md) : %s%s", tbl.Name(), tbl.Name(), tbl.Comment(), nl)
		}
	}
	return buf.String()
}

func (p markdownPublisher) convertToMarkdown(table *dbmodel.Table) string {
	buf := &bytes.Buffer{}
	nl := fmt.Sprintln()

	fmt.Fprintf(buf, "# %s%s", table.Name(), nl)
	if table.Comment() != "" {
		fmt.Fprintln(buf, nl+table.Comment())
	}

	fmt.Fprintf(buf, "%s## Columns%s%s", nl, nl, nl)
	w := newMdTableWriter(buf)
	w.SetHeader([]string{"PK", "NAME", "TYPE", "SIZE", "NULL", "DEFAULT", "COMMENT"})
	for _, col := range table.Columns() {
		w.Append(p.cnv.ConvertColumn(col))
	}
	w.Render()

	if len(table.Indices()) > 0 {
		fmt.Fprintf(buf, "%s## Indices%s%s", nl, nl, nl)
		w = newMdTableWriter(buf)
		w.SetHeader([]string{"NAME", "COLUMNS", "UNIQUE"})
		for _, idx := range table.Indices() {
			w.Append(p.cnv.ConvertIndex(idx))
		}
		w.Render()
	}

	if len(table.Constraints()) > 0 {
		fmt.Fprintf(buf, "%s## Constants%s%s", nl, nl, nl)
		w = newMdTableWriter(buf)
		w.SetHeader([]string{"NAME", "KIND", "CONTENT"})
		for _, con := range table.Constraints() {
			w.Append(p.cnv.ConvertConstraint(con))
		}
		w.Render()
	}

	if len(table.ForeignKeys()) > 0 {
		fmt.Fprintf(buf, "%s## Foreign keys%s%s", nl, nl, nl)
		w = newMdTableWriter(buf)
		w.SetHeader([]string{"NAME", "COLUMNS", "FOREIGN TABLE", "FOREIGN COLUMNS"})
		for _, fk := range table.ForeignKeys() {
			w.Append(p.cnv.ConvertForeignKey(fk))
		}
		w.Render()
	}

	if len(table.ReferencedKeys()) > 0 {
		fmt.Fprintf(buf, "%s## Referenced keys%s%s", nl, nl, nl)
		w = newMdTableWriter(buf)
		w.SetHeader([]string{"NAME", "SOURCE TABLE", "SOURCE COLUMNS", "COLUMNS"})
		for _, rk := range table.ReferencedKeys() {
			w.Append(p.cnv.ConvertReferencedKey(rk))
		}
		w.Render()
	}

	return buf.String()
}

func newMdTableWriter(w io.Writer) *tablewriter.Table {
	tw := tablewriter.NewWriter(w)
	tw.SetAutoWrapText(false)
	tw.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
	tw.SetCenterSeparator("|")
	return tw
}

func cleanDir(path string) error {
	if err := os.RemoveAll(path); err != nil {
		return err
	}
	if err := os.MkdirAll(path, 0777); err != nil {
		return err
	}
	return nil
}
<commit_msg>Refactoring<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/olekukonko/tablewriter"
	"github.com/pinzolo/dbmodel"
)

type markdownPublisher struct {
	cfg    *Config
	conv   Converter
	errors []error
}

func newMarkdownPublisher(config *Config, converter Converter) markdownPublisher {
	return markdownPublisher{
		cfg:    config,
		conv:   converter,
		errors: make([]error, 0, 0),
	}
}

func (p markdownPublisher) Publish(tables []*dbmodel.Table) {
	path, err := resolvePath(p.cfg.Out)
	if err != nil {
		p.errors = append(p.errors, err)
		return
	}
	if err := cleanDir(path); err != nil {
		p.errors = append(p.errors, err)
		return
	}

	for _, tbl := range tables {
		md := convertToMarkdown(tbl, p.conv)
		if f, err := os.Create(filepath.Join(path, tbl.Name()+".md")); err != nil {
			p.errors = append(p.errors, err)
		} else {
			f.Write(md)
			err = f.Close()
			if err != nil {
				p.errors = append(p.errors, err)
			}
		}
	}

	idxMd := convertToIndexMarkdown(tables)
	if f, err := os.Create(filepath.Join(path, "00_index.md")); err != nil {
		p.errors = append(p.errors, err)
	} else {
		f.Write(idxMd)
		err = f.Close()
		if err != nil {
			p.errors = append(p.errors, err)
		}
	}
}

func (p markdownPublisher) Errors() []error {
	return p.errors
}

func convertToMarkdown(table *dbmodel.Table, conv Converter) []byte {
	buf := &bytes.Buffer{}
	nl := fmt.Sprintln()

	fmt.Fprintf(buf, "# %s%s", table.Name(), nl)
	if table.Comment() != "" {
		fmt.Fprintln(buf, nl+table.Comment())
	}

	fmt.Fprintf(buf, "%s## Columns%s%s", nl, nl, nl)
	w := newMdTableWriter(buf)
	w.SetHeader([]string{"PK", "NAME", "TYPE", "SIZE", "NULL", "DEFAULT", "COMMENT"})
	for _, col := range table.Columns() {
		w.Append(conv.ConvertColumn(col))
	}
	w.Render()

	if len(table.Indices()) > 0 {
		fmt.Fprintf(buf, "%s## Indices%s%s", nl, nl, nl)
		w = newMdTableWriter(buf)
		w.SetHeader([]string{"NAME", "COLUMNS", "UNIQUE"})
		for _, idx := range table.Indices() {
			w.Append(conv.ConvertIndex(idx))
		}
		w.Render()
	}

	if len(table.Constraints()) > 0 {
		fmt.Fprintf(buf, "%s## Constants%s%s", nl, nl, nl)
		w = newMdTableWriter(buf)
		w.SetHeader([]string{"NAME", "KIND", "CONTENT"})
		for _, con := range table.Constraints() {
			w.Append(conv.ConvertConstraint(con))
		}
		w.Render()
	}

	if len(table.ForeignKeys()) > 0 {
		fmt.Fprintf(buf, "%s## Foreign keys%s%s", nl, nl, nl)
		w = newMdTableWriter(buf)
		w.SetHeader([]string{"NAME", "COLUMNS", "FOREIGN TABLE", "FOREIGN COLUMNS"})
		for _, fk := range table.ForeignKeys() {
			w.Append(conv.ConvertForeignKey(fk))
		}
		w.Render()
	}

	if len(table.ReferencedKeys()) > 0 {
		fmt.Fprintf(buf, "%s## Referenced keys%s%s", nl, nl, nl)
		w = newMdTableWriter(buf)
		w.SetHeader([]string{"NAME", "SOURCE TABLE", "SOURCE COLUMNS", "COLUMNS"})
		for _, rk := range table.ReferencedKeys() {
			w.Append(conv.ConvertReferencedKey(rk))
		}
		w.Render()
	}

	return buf.Bytes()
}

func convertToIndexMarkdown(tables []*dbmodel.Table) []byte {
	buf := &bytes.Buffer{}
	nl := fmt.Sprintln()

	fmt.Fprintln(buf, "# Index"+nl)
	for _, tbl := range tables {
		if tbl.Comment() == "" {
			fmt.Fprintf(buf, "* [%s](%s.md)%s", tbl.Name(), tbl.Name(), nl)
		} else {
			fmt.Fprintf(buf, "* [%s](%s.md) : %s%s", tbl.Name(), tbl.Name(), tbl.Comment(), nl)
		}
	}
	return buf.Bytes()
}

func newMdTableWriter(w io.Writer) *tablewriter.Table {
	tw := tablewriter.NewWriter(w)
	tw.SetAutoWrapText(false)
	tw.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
	tw.SetCenterSeparator("|")
	return tw
}

func cleanDir(path string) error {
	// TODO: check dir exists
	if err := os.RemoveAll(path); err != nil {
		return err
	}
	if err := os.MkdirAll(path, 0777); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package matutil

import (

//	"fmt"
	"testing"
	"math"
	"code.google.com/p/gomatrix/matrix"
)

func TestColSliceValid(t *testing.T) {
	rows := 3
	columns := 2
	mat := matrix.MakeDenseMatrix([]float64{1, 2, 3, 4, 5, 6}, rows, columns)
	c := ColSlice(mat, 1)
	if len(c) != rows {
		t.Errorf("Returned slice has len=%d instead of %d.", len(c), rows)
	}
}

func TestAppendColInvalid(t *testing.T) {
	rows := 3
	columns := 2
	mat := matrix.MakeDenseMatrix([]float64{1, 2, 3, 4, 5, 6}, rows, columns)
	col := []float64{1.1, 2.2, 3.3, 4.4}
	mat, err := AppendCol(mat, col)
	if err == nil {
		t.Errorf("AppendCol err=%v", err)
	}
}

func TestAppendColValid(t *testing.T) {
	rows := 3
	columns := 2
	mat := matrix.MakeDenseMatrix([]float64{1, 2, 3, 4, 5, 6}, rows, columns)
	col := []float64{1.1, 2.2, 3.3}
	mat, err := AppendCol(mat, col)
	if err != nil {
		t.Errorf("AppendCol err=%v", err)
	}
}

func TestPow(t *testing.T) {
	p00 := float64(3)
	p01 := float64(4)
	mat := matrix.MakeDenseMatrix([]float64{p00, p01}, 1, 2)
	raised := Pow(mat, 2)

	r00 := raised.Get(0, 0)
	if r00 != 9 {
		t.Errorf("TestPow r00 should be 9, but is %f", r00)
	}
	r01 := raised.Get(0, 1)
	if r01 != 16 {
		t.Errorf("TestPow r01 should be 16, but is %f", r01)
	}
}

func TestSumRows(t *testing.T) {
	p00 := 3.0
	p01 := 4.0
	p10 := 3.5
	p11 := 4.6
	mat := matrix.MakeDenseMatrix([]float64{p00, p01, p10, p11}, 2, 2)
	sums := SumRows(mat)

	numRows, numCols := sums.GetSize()
	if numRows != 2 || numCols != 1 {
		t.Errorf("SumRows returned a %dx%d matrix.  It should be 2x1.", numRows, numCols)
	}
	s00 := sums.Get(0, 0)
	if s00 != (p00 + p01) {
		t.Errorf("SumRows row 0 col 0 is %d.  It should be %d.", s00, p00+p01)
	}
	s10 := sums.Get(1, 0)
	if s10 != (p10 + p11) {
		t.Errorf("SumRows row 1 col 2 is %d.  It should be %d.", s10, p10+p11)
	}
}

func TestSumCols(t *testing.T) {
	p00 := 3.0
	p01 := 4.0
	p10 := 3.5
	p11 := 4.6
	mat := matrix.MakeDenseMatrix([]float64{p00, p01, p10, p11}, 2, 2)
	sums := SumCols(mat)

	numRows, numCols := sums.GetSize()
	if numRows != 1 || numCols != 2 {
		t.Errorf("SumCols returned a %dx%d matrix.  It should be 1x2.", numRows, numCols)
	}
	s00 := sums.Get(0, 0)
	if s00 != (p00 + p10) {
		t.Errorf("SumCols row 0 col 0 is %d.  It should be %d.", s00, p00+p10)
	}
	s10 := sums.Get(0, 1)
	if s10 != (p01 + p11) {
		t.Errorf("SumCols row 0 col 1 is %d.  It should be %d.", s10, p01+p11)
	}
}

func TestFiltCol(t *testing.T) {
	mat := matrix.MakeDenseMatrix([]float64{2, 1, 4, 2, 6, 3, 8, 4, 10, 5, 1, 1}, 5, 2)
	// mat, max, min, column
	matches, err := FiltCol(mat, 2.0, 4.0, 1)
	if err != nil {
		t.Errorf("FiltCol returned error: %v", err)
		return
	}
	r, _ := matches.GetSize()
	if r != 3 {
		t.Errorf("FiltCol: expected 3 rows and got %d", r)
	}

	m0 := matches.Get(0,1)
	if m0 != float64(2) {
		t.Errorf("FiltCol: expected row 0 col 1 to be 2, but got %f",m0)
	}

	m1 := matches.Get(1, 1)
	if m1 != 3 {
		t.Errorf("FiltCol: expected row 1 col 1 to be 3, but got %f",m1)
	}

	m2 := matches.Get(2, 1)
	if m2 !=4 {
		t.Errorf("FiltCol: expected row 1 col 1 to be 3, but got %f",m2)
	}
}

func TestFiltColMap(t *testing.T) {
	mat := matrix.MakeDenseMatrix([]float64{2, 1, 4, 2, 6, 3,8, 4, 10, 5, 1, 1}, 5, 2)
	matches, err := FiltColMap(mat, 2.0, 4.0, 1)
	if err != nil {
		t.Errorf("FiltColMap returned error: %v", err)
		return
	}

	if len(matches) != 3 {
		t.Errorf("FiltColMap expecte a map of len 3, but got len %d", len(matches))
	}

	if matches[1] != 2 || matches[2] != 3 || matches[3] != 4 {
		t.Errorf("FiltColMap expected a map with vals 2, 3, 4 but got %v", matches)
	}
}

func TestEuclidDist(t *testing.T) {
	var ed EuclidDist
	rows := 1
	columns := 2

	centroid := matrix.MakeDenseMatrix([]float64{4.6, 9.5}, rows, columns)
	point := matrix.MakeDenseMatrix([]float64{3.0, 4.1}, rows, columns)
	calcEd, err := ed.CalcDist(centroid, point)
	if err != nil {
		t.Errorf("EuclidDist: returned an error.  err=%v", err)
	}

	expectedEd := 5.632051 //expected value
	epsilon := .000001

	na := math.Nextafter(expectedEd, expectedEd + 1)
	diff := math.Abs(calcEd - na)

	if diff > epsilon {
		t.Errorf("EuclidDist: excpected %f but received %f.  The difference %f exceeds epsilon %f", expectedEd, calcEd, diff, epsilon)
	}
}

func BenchmarkEuclidDist(b *testing.B) {
	var ed EuclidDist
	rows := 1
	columns := 2

	centroid := matrix.MakeDenseMatrix([]float64{4.6, 9.5}, rows, columns)
	point := matrix.MakeDenseMatrix([]float64{3.0, 4.1}, rows, columns)
    for i := 0; i < b.N; i++ {
		_, _ = ed.CalcDist(centroid, point)
    }
}

func TestManhattanDist(t *testing.T) {
	var md ManhattanDist
	rows := 1
	columns := 2

	a := matrix.MakeDenseMatrix([]float64{4.6, 9.5}, rows, columns)
	b := matrix.MakeDenseMatrix([]float64{3.0, 4.1}, rows, columns)

	calcMd, err := md.CalcDist(a, b)
	if err != nil {
		t.Errorf("ManhattandDist: returned an error.  err=%v", err)
	}

	// 1.6 + 5.4 = 7.0
	if calcMd != float64(7.0) {
		t.Errorf("ManhattanDist: should be 7.0, but returned %f", calcMd)
	}
}
//TODO: test for MeanCols<commit_msg>Added test for no match error condition in FiltCol.<commit_after>package matutil

import (

//	"fmt"
	"testing"
	"math"
	"code.google.com/p/gomatrix/matrix"
)

func TestColSliceValid(t *testing.T) {
	rows := 3
	columns := 2
	mat := matrix.MakeDenseMatrix([]float64{1, 2, 3, 4, 5, 6}, rows, columns)
	c := ColSlice(mat, 1)
	if len(c) != rows {
		t.Errorf("Returned slice has len=%d instead of %d.", len(c), rows)
	}
}

func TestAppendColInvalid(t *testing.T) {
	rows := 3
	columns := 2
	mat := matrix.MakeDenseMatrix([]float64{1, 2, 3, 4, 5, 6}, rows, columns)
	col := []float64{1.1, 2.2, 3.3, 4.4}
	mat, err := AppendCol(mat, col)
	if err == nil {
		t.Errorf("AppendCol err=%v", err)
	}
}

func TestAppendColValid(t *testing.T) {
	rows := 3
	columns := 2
	mat := matrix.MakeDenseMatrix([]float64{1, 2, 3, 4, 5, 6}, rows, columns)
	col := []float64{1.1, 2.2, 3.3}
	mat, err := AppendCol(mat, col)
	if err != nil {
		t.Errorf("AppendCol err=%v", err)
	}
}

func TestPow(t *testing.T) {
	p00 := float64(3)
	p01 := float64(4)
	mat := matrix.MakeDenseMatrix([]float64{p00, p01}, 1, 2)
	raised := Pow(mat, 2)

	r00 := raised.Get(0, 0)
	if r00 != 9 {
		t.Errorf("TestPow r00 should be 9, but is %f", r00)
	}
	r01 := raised.Get(0, 1)
	if r01 != 16 {
		t.Errorf("TestPow r01 should be 16, but is %f", r01)
	}
}

func TestSumRows(t *testing.T) {
	p00 := 3.0
	p01 := 4.0
	p10 := 3.5
	p11 := 4.6
	mat := matrix.MakeDenseMatrix([]float64{p00, p01, p10, p11}, 2, 2)
	sums := SumRows(mat)

	numRows, numCols := sums.GetSize()
	if numRows != 2 || numCols != 1 {
		t.Errorf("SumRows returned a %dx%d matrix.  It should be 2x1.", numRows, numCols)
	}
	s00 := sums.Get(0, 0)
	if s00 != (p00 + p01) {
		t.Errorf("SumRows row 0 col 0 is %d.  It should be %d.", s00, p00+p01)
	}
	s10 := sums.Get(1, 0)
	if s10 != (p10 + p11) {
		t.Errorf("SumRows row 1 col 2 is %d.  It should be %d.", s10, p10+p11)
	}
}

func TestSumCols(t *testing.T) {
	p00 := 3.0
	p01 := 4.0
	p10 := 3.5
	p11 := 4.6
	mat := matrix.MakeDenseMatrix([]float64{p00, p01, p10, p11}, 2, 2)
	sums := SumCols(mat)

	numRows, numCols := sums.GetSize()
	if numRows != 1 || numCols != 2 {
		t.Errorf("SumCols returned a %dx%d matrix.  It should be 1x2.", numRows, numCols)
	}
	s00 := sums.Get(0, 0)
	if s00 != (p00 + p10) {
		t.Errorf("SumCols row 0 col 0 is %d.  It should be %d.", s00, p00+p10)
	}
	s10 := sums.Get(0, 1)
	if s10 != (p01 + p11) {
		t.Errorf("SumCols row 0 col 1 is %d.  It should be %d.", s10, p01+p11)
	}
}

func TestFiltCol(t *testing.T) {
	mat := matrix.MakeDenseMatrix([]float64{2, 1, 4, 2, 6, 3, 8, 4, 10, 5, 1, 1}, 5, 2)
	// mat, max, min, column
	matches, err := FiltCol(mat, 2.0, 4.0, 1)
	if err != nil {
		t.Errorf("FiltCol returned error: %v", err)
		return
	}
	r, _ := matches.GetSize()
	if r != 3 {
		t.Errorf("FiltCol: expected 3 rows and got %d", r)
	}

	m0 := matches.Get(0,1)
	if m0 != float64(2) {
		t.Errorf("FiltCol: expected row 0 col 1 to be 2, but got %f",m0)
	}

	m1 := matches.Get(1, 1)
	if m1 != 3 {
		t.Errorf("FiltCol: expected row 1 col 1 to be 3, but got %f",m1)
	}

	m2 := matches.Get(2, 1)
	if m2 !=4 {
		t.Errorf("FiltCol: expected row 1 col 1 to be 3, but got %f",m2)
	}
	matches, err = FiltCol(mat, 100.0, 200.00, 1)
	if err == nil {
		t.Errorf("FiltCol did not return err on no match condition.")
	}
}

func TestFiltColMap(t *testing.T) {
	mat := matrix.MakeDenseMatrix([]float64{2, 1, 4, 2, 6, 3,8, 4, 10, 5, 1, 1}, 5, 2)
	matches, err := FiltColMap(mat, 2.0, 4.0, 1)
	if err != nil {
		t.Errorf("FiltColMap returned error: %v", err)
		return
	}

	if len(matches) != 3 {
		t.Errorf("FiltColMap expecte a map of len 3, but got len %d", len(matches))
	}

	if matches[1] != 2 || matches[2] != 3 || matches[3] != 4 {
		t.Errorf("FiltColMap expected a map with vals 2, 3, 4 but got %v", matches)
	}
}

func TestEuclidDist(t *testing.T) {
	var ed EuclidDist
	rows := 1
	columns := 2

	centroid := matrix.MakeDenseMatrix([]float64{4.6, 9.5}, rows, columns)
	point := matrix.MakeDenseMatrix([]float64{3.0, 4.1}, rows, columns)
	calcEd, err := ed.CalcDist(centroid, point)
	if err != nil {
		t.Errorf("EuclidDist: returned an error.  err=%v", err)
	}

	expectedEd := 5.632051 //expected value
	epsilon := .000001

	na := math.Nextafter(expectedEd, expectedEd + 1)
	diff := math.Abs(calcEd - na)

	if diff > epsilon {
		t.Errorf("EuclidDist: excpected %f but received %f.  The difference %f exceeds epsilon %f", expectedEd, calcEd, diff, epsilon)
	}
}

func BenchmarkEuclidDist(b *testing.B) {
	var ed EuclidDist
	rows := 1
	columns := 2

	centroid := matrix.MakeDenseMatrix([]float64{4.6, 9.5}, rows, columns)
	point := matrix.MakeDenseMatrix([]float64{3.0, 4.1}, rows, columns)
    for i := 0; i < b.N; i++ {
		_, _ = ed.CalcDist(centroid, point)
    }
}

func TestManhattanDist(t *testing.T) {
	var md ManhattanDist
	rows := 1
	columns := 2

	a := matrix.MakeDenseMatrix([]float64{4.6, 9.5}, rows, columns)
	b := matrix.MakeDenseMatrix([]float64{3.0, 4.1}, rows, columns)

	calcMd, err := md.CalcDist(a, b)
	if err != nil {
		t.Errorf("ManhattandDist: returned an error.  err=%v", err)
	}

	// 1.6 + 5.4 = 7.0
	if calcMd != float64(7.0) {
		t.Errorf("ManhattanDist: should be 7.0, but returned %f", calcMd)
	}
}
//TODO: test for MeanCols<|endoftext|>


<commit_before>package k8s_test

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"testing"
	"time"

	"github.com/caarlos0/env"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"

	. "github.com/concourse/concourse/topgun"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	_ "k8s.io/client-go/plugin/pkg/client/auth"
)

func TestK8s(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "K8s Suite")
}

type environment struct {
	HelmChartsDir        string `env:"HELM_CHARTS_DIR,required"`
	ConcourseChartDir    string `env:"CONCOURSE_CHART_DIR,required"`
	ConcourseImageDigest string `env:"CONCOURSE_IMAGE_DIGEST"`
	ConcourseImageName   string `env:"CONCOURSE_IMAGE_NAME,required"`
	ConcourseImageTag    string `env:"CONCOURSE_IMAGE_TAG"`
	FlyPath              string `env:"FLY_PATH"`
	K8sEngine            string `env:"K8S_ENGINE" envDefault:"GKE"`
	InCluster            bool   `env:"IN_CLUSTER" envDefault:"false"`
}

var (
	Environment            environment
	endpointFactory        EndpointFactory
	fly                    FlyCli
	releaseName, namespace string
	kubeClient             *kubernetes.Clientset
	deployedReleases       map[string]string // map[releaseName] = namespace
)

var _ = SynchronizedBeforeSuite(func() []byte {
	var parsedEnv environment

	err := env.Parse(&parsedEnv)
	Expect(err).ToNot(HaveOccurred())

	if parsedEnv.FlyPath == "" {
		parsedEnv.FlyPath = BuildBinary()
	}

	By("Checking if kubectl has a context set for port forwarding later")
	Wait(Start(nil, "kubectl", "config", "current-context"))

	By("Updating the dependencies of the Concourse chart locally")
	Wait(Start(nil, "helm", "dependency", "update", parsedEnv.ConcourseChartDir))

	envBytes, err := json.Marshal(parsedEnv)
	Expect(err).ToNot(HaveOccurred())

	return envBytes
}, func(data []byte) {
	err := json.Unmarshal(data, &Environment)
	Expect(err).ToNot(HaveOccurred())

	deployedReleases = make(map[string]string)
})

var _ = BeforeEach(func() {
	SetDefaultEventuallyTimeout(90 * time.Second)
	SetDefaultConsistentlyDuration(30 * time.Second)

	tmp, err := ioutil.TempDir("", "topgun-tmp")
	Expect(err).ToNot(HaveOccurred())

	fly = FlyCli{
		Bin:    Environment.FlyPath,
		Target: "concourse-topgun-k8s-" + strconv.Itoa(GinkgoParallelNode()),
		Home:   filepath.Join(tmp, "fly-home-"+strconv.Itoa(GinkgoParallelNode())),
	}

	endpointFactory = PortForwardingEndpointFactory{}
	if Environment.InCluster {
		endpointFactory = AddressEndpointFactory{}
	}

	err = os.Mkdir(fly.Home, 0755)
	Expect(err).ToNot(HaveOccurred())

	By("Checking if kubeconfig exists")
	kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config")
	_, err = os.Stat(kubeconfig)
	Expect(err).ToNot(HaveOccurred(), "kubeconfig should exist")

	By("Creating a kubernetes client")
	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
	Expect(err).ToNot(HaveOccurred())

	kubeClient, err = kubernetes.NewForConfig(config)
	Expect(err).ToNot(HaveOccurred())
})

// In case one of the tests exited before entering the It block, make sure to cleanup
// any releases that might've already been deployed
var _ = AfterSuite(func() {
	cleanupReleases()
})

func setReleaseNameAndNamespace(description string) {
	rand.Seed(time.Now().UTC().UnixNano())
	releaseName = fmt.Sprintf("topgun-"+description+"-%d", rand.Int63n(100000000))
	namespace = releaseName
}

// pod corresponds to the Json object that represents a Kuberneted pod from the
// apiserver perspective.
//
type pod struct {
	Status struct {
		Conditions []struct {
			Type   string `json:"type"`
			Status string `json:"status"`
		} `json:"conditions"`
		ContainerStatuses []struct {
			Name  string `json:"name"`
			Ready bool   `json:"ready"`
		} `json:"containerStatuses"`
		Ip string `json:"podIP"`
	} `json:"status"`
	Metadata struct {
		Name string `json:"name"`
	} `json:"metadata"`
}

// Endpoint represents a service that can be reached from a given address.
//
type Endpoint interface {
	Address() (addr string)
	Close() (err error)
}

// EndpointFactory represents those entities able to generate Endpoints for
// both services and pods.
//
type EndpointFactory interface {
	NewServiceEndpoint(namespace, service, port string) (endpoint Endpoint)
	NewPodEndpoint(namespace, pod, port string) (endpoint Endpoint)
}

// PortForwardingEndpoint is a service that can be reached through a local
// address, having connections port forwarded to entities in a cluster.
//
type PortForwardingEndpoint struct {
	session *gexec.Session
	address string
}

func (p PortForwardingEndpoint) Address() string {
	return p.address
}

func (p PortForwardingEndpoint) Close() error {
	p.session.Interrupt()
	return nil
}

// AddressEndpoint represents a direct address without any underlying session.
//
type AddressEndpoint struct {
	address string
}

func (p AddressEndpoint) Address() string {
	return p.address
}

func (p AddressEndpoint) Close() error {
	return nil
}

// PortForwardingFactory deals with creating endpoints that reach the targets
// through port-forwarding.
//
type PortForwardingEndpointFactory struct{}

func (f PortForwardingEndpointFactory) NewServiceEndpoint(namespace, service, port string) Endpoint {
	session, address := portForward(namespace, "service/"+service, port)

	return PortForwardingEndpoint{
		session: session,
		address: address,
	}
}

func (f PortForwardingEndpointFactory) NewPodEndpoint(namespace, pod, port string) Endpoint {
	session, address := portForward(namespace, "pod/"+pod, port)

	return PortForwardingEndpoint{
		session: session,
		address: address,
	}
}

// AddressFactory deals with creating endpoints that reach the targets
// through port-forwarding.
//
type AddressEndpointFactory struct{}

func (f AddressEndpointFactory) NewServiceEndpoint(namespace, service, port string) Endpoint {
	address := serviceAddress(namespace, service)

	return AddressEndpoint{
		address: address + ":" + port,
	}
}

func (f AddressEndpointFactory) NewPodEndpoint(namespace, pod, port string) Endpoint {
	address := podAddress(namespace, pod)

	return AddressEndpoint{
		address: address + ":" + port,
	}
}

func podAddress(namespace, pod string) string {
	pods := getPods(namespace, metav1.ListOptions{FieldSelector: "metadata.name=" + pod})
	Expect(pods).To(HaveLen(1))

	return pods[0].Status.PodIP
}

// serviceAddress retrieves the ClusterIP address of a service on a given
// namespace.
//
func serviceAddress(namespace, serviceName string) (address string) {
	return serviceName + "." + namespace
}

// portForward establishes a port-forwarding session against a given kubernetes
// resource, for a particular port.
//
func portForward(namespace, resource, port string) (*gexec.Session, string) {
	sess := Start(nil,
		"kubectl", "port-forward",
		"--namespace="+namespace,
		resource,
		":"+port,
	)

	Eventually(sess.Out).Should(gbytes.Say("Forwarding"))

	address := regexp.MustCompile(`127\.0\.0\.1:[0-9]+`).
		FindStringSubmatch(string(sess.Out.Contents()))
	Expect(address).NotTo(BeEmpty())

	return sess, address[0]
}

func helmDeploy(releaseName, namespace, chartDir string, args ...string) *gexec.Session {

	helmArgs := []string{
		"upgrade",
		"--install",
		"--force",
		"--namespace", namespace,
		"--create-namespace",
	}

	helmArgs = append(helmArgs, args...)
	helmArgs = append(helmArgs, releaseName, chartDir)

	sess := Start(nil, "helm", helmArgs...)
	<-sess.Exited

	if sess.ExitCode() == 0 {
		deployedReleases[releaseName] = namespace
	}
	return sess
}

func helmInstallArgs(args ...string) []string {
	helmArgs := []string{
		"--set=concourse.web.kubernetes.keepNamespaces=false",
		"--set=concourse.worker.bindIp=0.0.0.0",
		"--set=postgresql.persistence.enabled=false",
		"--set=web.resources.requests.cpu=500m",
		"--set=worker.readinessProbe.httpGet.path=/",
		"--set=worker.readinessProbe.httpGet.port=worker-hc",
		"--set=worker.resources.requests.cpu=500m",
		"--set=image=" + Environment.ConcourseImageName}

	if Environment.ConcourseImageTag != "" {
		helmArgs = append(helmArgs, "--set=imageTag="+Environment.ConcourseImageTag)
	}

	if Environment.ConcourseImageDigest != "" {
		helmArgs = append(helmArgs, "--set=imageDigest="+Environment.ConcourseImageDigest)
	}

	return append(helmArgs, args...)
}

func deployFailingConcourseChart(releaseName string, expectedErr string, args ...string) {
	helmArgs := helmInstallArgs(args...)
	sess := helmDeploy(releaseName, releaseName, Environment.ConcourseChartDir, helmArgs...)
	Expect(sess.ExitCode()).ToNot(Equal(0))
	Expect(sess.Err).To(gbytes.Say(expectedErr))
}

func deployConcourseChart(releaseName string, args ...string) {
	helmArgs := helmInstallArgs(args...)
	sess := helmDeploy(releaseName, releaseName, Environment.ConcourseChartDir, helmArgs...)
	Expect(sess.ExitCode()).To(Equal(0))
}

func helmDestroy(releaseName, namespace string) {
	helmArgs := []string{
		"delete",
		"--namespace",
		namespace,
		releaseName,
	}

	Wait(Start(nil, "helm", helmArgs...))
}

func getPods(namespace string, listOptions metav1.ListOptions) []corev1.Pod {
	pods, err := kubeClient.CoreV1().Pods(namespace).List(context.TODO(), listOptions)
	Expect(err).ToNot(HaveOccurred())

	return pods.Items
}

func isPodReady(p corev1.Pod) bool {
	for _, condition := range p.Status.Conditions {
		if condition.Type != corev1.ContainersReady {
			continue
		}

		return condition.Status == "True"
	}

	return false
}

func waitAllPodsInNamespaceToBeReady(namespace string) {
	Eventually(func() bool {
		expectedPods := getPods(namespace, metav1.ListOptions{})
		actualPods := getPods(namespace, metav1.ListOptions{FieldSelector: "status.phase=Running"})

		if len(expectedPods) != len(actualPods) {
			return false
		}

		podsReady := 0
		for _, pod := range actualPods {
			if isPodReady(pod) {
				podsReady++
			}
		}

		return podsReady == len(expectedPods)
	}, 15*time.Minute, 10*time.Second).Should(BeTrue(), "expected all pods to be running")
}

func deletePods(namespace string, flags ...string) []string {
	var (
		podNames []string
		args     = append([]string{"delete", "pod",
			"--namespace=" + namespace,
		}, flags...)
		session = Start(nil, "kubectl", args...)
	)

	Wait(session)

	scanner := bufio.NewScanner(bytes.NewBuffer(session.Out.Contents()))
	for scanner.Scan() {
		podNames = append(podNames, scanner.Text())
	}

	return podNames
}

func getRunningWorkers(workers []Worker) (running []Worker) {
	for _, w := range workers {
		if w.State == "running" {
			running = append(running, w)
		}
	}
	return
}

func waitAndLogin(namespace, service string) Endpoint {
	waitAllPodsInNamespaceToBeReady(namespace)

	atc := endpointFactory.NewServiceEndpoint(
		namespace,
		service,
		"8080",
	)

	fly.Login("test", "test", "http://"+atc.Address())

	Eventually(func() []Worker {
		return getRunningWorkers(fly.GetWorkers())
	}, 2*time.Minute, 10*time.Second).
		ShouldNot(HaveLen(0))

	return atc
}

func cleanupReleases() {
	for releaseName, namespace := range deployedReleases {
		helmDestroy(releaseName, namespace)
		kubeClient.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{})
	}

	deployedReleases = make(map[string]string)
}

func onPks(f func()) {
	Context("PKS", func() {

		BeforeEach(func() {
			if Environment.K8sEngine != "PKS" {
				Skip("not running on PKS")
			}
		})

		f()
	})
}

func onGke(f func()) {
	Context("GKE", func() {

		BeforeEach(func() {
			if Environment.K8sEngine != "GKE" {
				Skip("not running on GKE")
			}
		})

		f()
	})
}
<commit_msg>With commit in concourse charts https://github.com/concourse/concourse-chart/pull/163<commit_after>package k8s_test

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"testing"
	"time"

	"github.com/caarlos0/env"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"

	. "github.com/concourse/concourse/topgun"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	_ "k8s.io/client-go/plugin/pkg/client/auth"
)

func TestK8s(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "K8s Suite")
}

type environment struct {
	HelmChartsDir        string `env:"HELM_CHARTS_DIR,required"`
	ConcourseChartDir    string `env:"CONCOURSE_CHART_DIR,required"`
	ConcourseImageDigest string `env:"CONCOURSE_IMAGE_DIGEST"`
	ConcourseImageName   string `env:"CONCOURSE_IMAGE_NAME,required"`
	ConcourseImageTag    string `env:"CONCOURSE_IMAGE_TAG"`
	FlyPath              string `env:"FLY_PATH"`
	K8sEngine            string `env:"K8S_ENGINE" envDefault:"GKE"`
	InCluster            bool   `env:"IN_CLUSTER" envDefault:"false"`
}

var (
	Environment            environment
	endpointFactory        EndpointFactory
	fly                    FlyCli
	releaseName, namespace string
	kubeClient             *kubernetes.Clientset
	deployedReleases       map[string]string // map[releaseName] = namespace
)

var _ = SynchronizedBeforeSuite(func() []byte {
	var parsedEnv environment

	err := env.Parse(&parsedEnv)
	Expect(err).ToNot(HaveOccurred())

	if parsedEnv.FlyPath == "" {
		parsedEnv.FlyPath = BuildBinary()
	}

	By("Checking if kubectl has a context set for port forwarding later")
	Wait(Start(nil, "kubectl", "config", "current-context"))

	By("Updating the dependencies of the Concourse chart locally")
	Wait(Start(nil, "helm", "dependency", "update", parsedEnv.ConcourseChartDir))

	envBytes, err := json.Marshal(parsedEnv)
	Expect(err).ToNot(HaveOccurred())

	return envBytes
}, func(data []byte) {
	err := json.Unmarshal(data, &Environment)
	Expect(err).ToNot(HaveOccurred())

	deployedReleases = make(map[string]string)
})

var _ = BeforeEach(func() {
	SetDefaultEventuallyTimeout(90 * time.Second)
	SetDefaultConsistentlyDuration(30 * time.Second)

	tmp, err := ioutil.TempDir("", "topgun-tmp")
	Expect(err).ToNot(HaveOccurred())

	fly = FlyCli{
		Bin:    Environment.FlyPath,
		Target: "concourse-topgun-k8s-" + strconv.Itoa(GinkgoParallelNode()),
		Home:   filepath.Join(tmp, "fly-home-"+strconv.Itoa(GinkgoParallelNode())),
	}

	endpointFactory = PortForwardingEndpointFactory{}
	if Environment.InCluster {
		endpointFactory = AddressEndpointFactory{}
	}

	err = os.Mkdir(fly.Home, 0755)
	Expect(err).ToNot(HaveOccurred())

	By("Checking if kubeconfig exists")
	kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config")
	_, err = os.Stat(kubeconfig)
	Expect(err).ToNot(HaveOccurred(), "kubeconfig should exist")

	By("Creating a kubernetes client")
	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
	Expect(err).ToNot(HaveOccurred())

	kubeClient, err = kubernetes.NewForConfig(config)
	Expect(err).ToNot(HaveOccurred())
})

// In case one of the tests exited before entering the It block, make sure to cleanup
// any releases that might've already been deployed
var _ = AfterSuite(func() {
	cleanupReleases()
})

func setReleaseNameAndNamespace(description string) {
	rand.Seed(time.Now().UTC().UnixNano())
	releaseName = fmt.Sprintf("topgun-"+description+"-%d", rand.Int63n(100000000))
	namespace = releaseName
}

// pod corresponds to the Json object that represents a Kuberneted pod from the
// apiserver perspective.
//
type pod struct {
	Status struct {
		Conditions []struct {
			Type   string `json:"type"`
			Status string `json:"status"`
		} `json:"conditions"`
		ContainerStatuses []struct {
			Name  string `json:"name"`
			Ready bool   `json:"ready"`
		} `json:"containerStatuses"`
		Ip string `json:"podIP"`
	} `json:"status"`
	Metadata struct {
		Name string `json:"name"`
	} `json:"metadata"`
}

// Endpoint represents a service that can be reached from a given address.
//
type Endpoint interface {
	Address() (addr string)
	Close() (err error)
}

// EndpointFactory represents those entities able to generate Endpoints for
// both services and pods.
//
type EndpointFactory interface {
	NewServiceEndpoint(namespace, service, port string) (endpoint Endpoint)
	NewPodEndpoint(namespace, pod, port string) (endpoint Endpoint)
}

// PortForwardingEndpoint is a service that can be reached through a local
// address, having connections port forwarded to entities in a cluster.
//
type PortForwardingEndpoint struct {
	session *gexec.Session
	address string
}

func (p PortForwardingEndpoint) Address() string {
	return p.address
}

func (p PortForwardingEndpoint) Close() error {
	p.session.Interrupt()
	return nil
}

// AddressEndpoint represents a direct address without any underlying session.
//
type AddressEndpoint struct {
	address string
}

func (p AddressEndpoint) Address() string {
	return p.address
}

func (p AddressEndpoint) Close() error {
	return nil
}

// PortForwardingFactory deals with creating endpoints that reach the targets
// through port-forwarding.
//
type PortForwardingEndpointFactory struct{}

func (f PortForwardingEndpointFactory) NewServiceEndpoint(namespace, service, port string) Endpoint {
	session, address := portForward(namespace, "service/"+service, port)

	return PortForwardingEndpoint{
		session: session,
		address: address,
	}
}

func (f PortForwardingEndpointFactory) NewPodEndpoint(namespace, pod, port string) Endpoint {
	session, address := portForward(namespace, "pod/"+pod, port)

	return PortForwardingEndpoint{
		session: session,
		address: address,
	}
}

// AddressFactory deals with creating endpoints that reach the targets
// through port-forwarding.
//
type AddressEndpointFactory struct{}

func (f AddressEndpointFactory) NewServiceEndpoint(namespace, service, port string) Endpoint {
	address := serviceAddress(namespace, service)

	return AddressEndpoint{
		address: address + ":" + port,
	}
}

func (f AddressEndpointFactory) NewPodEndpoint(namespace, pod, port string) Endpoint {
	address := podAddress(namespace, pod)

	return AddressEndpoint{
		address: address + ":" + port,
	}
}

func podAddress(namespace, pod string) string {
	pods := getPods(namespace, metav1.ListOptions{FieldSelector: "metadata.name=" + pod})
	Expect(pods).To(HaveLen(1))

	return pods[0].Status.PodIP
}

// serviceAddress retrieves the ClusterIP address of a service on a given
// namespace.
//
func serviceAddress(namespace, serviceName string) (address string) {
	return serviceName + "." + namespace
}

// portForward establishes a port-forwarding session against a given kubernetes
// resource, for a particular port.
//
func portForward(namespace, resource, port string) (*gexec.Session, string) {
	sess := Start(nil,
		"kubectl", "port-forward",
		"--namespace="+namespace,
		resource,
		":"+port,
	)

	Eventually(sess.Out).Should(gbytes.Say("Forwarding"))

	address := regexp.MustCompile(`127\.0\.0\.1:[0-9]+`).
		FindStringSubmatch(string(sess.Out.Contents()))
	Expect(address).NotTo(BeEmpty())

	return sess, address[0]
}

func helmDeploy(releaseName, namespace, chartDir string, args ...string) *gexec.Session {

	helmArgs := []string{
		"upgrade",
		"--install",
		"--namespace", namespace,
		"--create-namespace",
	}

	helmArgs = append(helmArgs, args...)
	helmArgs = append(helmArgs, releaseName, chartDir)

	sess := Start(nil, "helm", helmArgs...)
	<-sess.Exited

	if sess.ExitCode() == 0 {
		deployedReleases[releaseName] = namespace
	}
	return sess
}

func helmInstallArgs(args ...string) []string {
	helmArgs := []string{
		"--set=concourse.web.kubernetes.keepNamespaces=false",
		"--set=concourse.worker.bindIp=0.0.0.0",
		"--set=postgresql.persistence.enabled=false",
		"--set=web.resources.requests.cpu=500m",
		"--set=worker.readinessProbe.httpGet.path=/",
		"--set=worker.readinessProbe.httpGet.port=worker-hc",
		"--set=worker.resources.requests.cpu=500m",
		"--set=image=" + Environment.ConcourseImageName}

	if Environment.ConcourseImageTag != "" {
		helmArgs = append(helmArgs, "--set=imageTag="+Environment.ConcourseImageTag)
	}

	if Environment.ConcourseImageDigest != "" {
		helmArgs = append(helmArgs, "--set=imageDigest="+Environment.ConcourseImageDigest)
	}

	return append(helmArgs, args...)
}

func deployFailingConcourseChart(releaseName string, expectedErr string, args ...string) {
	helmArgs := helmInstallArgs(args...)
	sess := helmDeploy(releaseName, releaseName, Environment.ConcourseChartDir, helmArgs...)
	Expect(sess.ExitCode()).ToNot(Equal(0))
	Expect(sess.Err).To(gbytes.Say(expectedErr))
}

func deployConcourseChart(releaseName string, args ...string) {
	helmArgs := helmInstallArgs(args...)
	sess := helmDeploy(releaseName, releaseName, Environment.ConcourseChartDir, helmArgs...)
	Expect(sess.ExitCode()).To(Equal(0))
}

func helmDestroy(releaseName, namespace string) {
	helmArgs := []string{
		"delete",
		"--namespace",
		namespace,
		releaseName,
	}

	Wait(Start(nil, "helm", helmArgs...))
}

func getPods(namespace string, listOptions metav1.ListOptions) []corev1.Pod {
	pods, err := kubeClient.CoreV1().Pods(namespace).List(context.TODO(), listOptions)
	Expect(err).ToNot(HaveOccurred())

	return pods.Items
}

func isPodReady(p corev1.Pod) bool {
	for _, condition := range p.Status.Conditions {
		if condition.Type != corev1.ContainersReady {
			continue
		}

		return condition.Status == "True"
	}

	return false
}

func waitAllPodsInNamespaceToBeReady(namespace string) {
	Eventually(func() bool {
		expectedPods := getPods(namespace, metav1.ListOptions{})
		actualPods := getPods(namespace, metav1.ListOptions{FieldSelector: "status.phase=Running"})

		if len(expectedPods) != len(actualPods) {
			return false
		}

		podsReady := 0
		for _, pod := range actualPods {
			if isPodReady(pod) {
				podsReady++
			}
		}

		return podsReady == len(expectedPods)
	}, 15*time.Minute, 10*time.Second).Should(BeTrue(), "expected all pods to be running")
}

func deletePods(namespace string, flags ...string) []string {
	var (
		podNames []string
		args     = append([]string{"delete", "pod",
			"--namespace=" + namespace,
		}, flags...)
		session = Start(nil, "kubectl", args...)
	)

	Wait(session)

	scanner := bufio.NewScanner(bytes.NewBuffer(session.Out.Contents()))
	for scanner.Scan() {
		podNames = append(podNames, scanner.Text())
	}

	return podNames
}

func getRunningWorkers(workers []Worker) (running []Worker) {
	for _, w := range workers {
		if w.State == "running" {
			running = append(running, w)
		}
	}
	return
}

func waitAndLogin(namespace, service string) Endpoint {
	waitAllPodsInNamespaceToBeReady(namespace)

	atc := endpointFactory.NewServiceEndpoint(
		namespace,
		service,
		"8080",
	)

	fly.Login("test", "test", "http://"+atc.Address())

	Eventually(func() []Worker {
		return getRunningWorkers(fly.GetWorkers())
	}, 2*time.Minute, 10*time.Second).
		ShouldNot(HaveLen(0))

	return atc
}

func cleanupReleases() {
	for releaseName, namespace := range deployedReleases {
		helmDestroy(releaseName, namespace)
		kubeClient.CoreV1().Namespaces().Delete(context.TODO(), namespace, metav1.DeleteOptions{})
	}

	deployedReleases = make(map[string]string)
}

func onPks(f func()) {
	Context("PKS", func() {

		BeforeEach(func() {
			if Environment.K8sEngine != "PKS" {
				Skip("not running on PKS")
			}
		})

		f()
	})
}

func onGke(f func()) {
	Context("GKE", func() {

		BeforeEach(func() {
			if Environment.K8sEngine != "GKE" {
				Skip("not running on GKE")
			}
		})

		f()
	})
}
<|endoftext|>


<commit_before>
package main

import (
  "utf8"
)

type altpos struct {
  alt int               // alt index
  is_end bool           // end (true) or begin (false)
  pos int               // character pos
  prev *altpos          // previous in stack
}

type pair struct {
  state int
  alt *altpos
}

type m_submatch struct {
  next []pair
  npos int              // next string index
}

func (m *m_submatch) addstate(st *instr, a *altpos) {
  if st == nil {
    return // invalid
  }
  if st.mode == kSplit {
    m.addstate(st.out, a)
    m.addstate(st.out1, a)
  } else if st.mode == kAltBegin {
    a = &altpos{st.alt, false, m.npos, a}
    m.addstate(st.out, a)
  } else if st.mode == kAltEnd {
    a = &altpos{st.alt, true, m.npos, a}
    m.addstate(st.out, a)
  } else {
    // terminal, store (s.idx, altpos) in state
    // note that s.idx won't always be unique (but if both are equal, we could use this)
    pos := len(m.next)
    m.next = m.next[0:pos+1]
    m.next[pos] = pair{st.idx, a}
  }
}

/**
 * Submatch regexp matcher entry point. Must return both true/false as well
 * as information on all submatches.
 */
func (r *sregexp) RunSubMatch(src string) (bool, []int) {
  states := len(r.prog)*4 // maximum number of states; should go away in optimisation
  m := &m_submatch{make([]pair, 0, states), 0}
  m.addstate(r.prog[0], nil)
  curr := m.next
  m.next = make([]pair, 0, states)

  var ch int
  for _, ch = range src {
    m.npos += utf8.RuneLen(ch)

    // move along rune paths
    for _, p := range curr {
      st := r.prog[p.state]
      if st.match(ch) {
        m.addstate(st.out, p.alt)
      }
    }

    curr, m.next = m.next, curr
    m.next = m.next[0:0]
  }

  for _, p := range curr { // ??? just to how many are here.
    if r.prog[p.state].mode == kMatch {
      alt := make([]int, r.alts*2)
      for i := 0; i < len(alt); i++ {
        alt[i] = -1
      }

      a := p.alt
      for a != nil {
        pos := (a.alt * 2)
        if a.is_end {
          pos += 1
        }
        if alt[pos] == -1 {
          alt[pos] = a.pos
        }
        a = a.prev
      }
      return true, alt
    }
  }

  return false, make([]int, 0)
}
<commit_msg>comments on submatch<commit_after>
package main

import (
  "utf8"
)

type altpos struct {
  alt int               // alt index
  is_end bool           // end (true) or begin (false)
  pos int               // character pos
  prev *altpos          // previous in stack
}

type pair struct {
  state int
  alt *altpos
}

type m_submatch struct {
  next []pair
  npos int              // next string index
}

func (m *m_submatch) addstate(st *instr, a *altpos) {
  if st == nil {
    return // invalid
  }
  if st.mode == kSplit {
    m.addstate(st.out, a)
    m.addstate(st.out1, a)
  } else if st.mode == kAltBegin {
    a = &altpos{st.alt, false, m.npos, a}
    m.addstate(st.out, a)
  } else if st.mode == kAltEnd {
    a = &altpos{st.alt, true, m.npos, a}
    m.addstate(st.out, a)
  } else {
    // terminal, store (s.idx, altpos) in state
    // note that s.idx won't always be unique (but if both are equal, we could use this)
    pos := len(m.next)
    m.next = m.next[0:pos+1]
    m.next[pos] = pair{st.idx, a}
  }
}

/**
 * Submatch regexp matcher entry point. Must return both true/false as well
 * as information on all submatches.
 */
func (r *sregexp) RunSubMatch(src string) (bool, []int) {
  states := len(r.prog)*4 // maximum number of states; should go away in optimisation
  m := &m_submatch{make([]pair, 0, states), 0}
  m.addstate(r.prog[0], nil)
  curr := m.next
  m.next = make([]pair, 0, states)

  var ch int
  for _, ch = range src {
    m.npos += utf8.RuneLen(ch)

    // move along rune paths
    for _, p := range curr {
      st := r.prog[p.state]
      if st.match(ch) {
        m.addstate(st.out, p.alt)
      }
    }

    curr, m.next = m.next, curr
    m.next = m.next[0:0]
  }

  // Search for a terminal state (in current states). If one is found, allocate
  // and return submatch information for those encountered.
  for _, p := range curr {
    if r.prog[p.state].mode == kMatch {
      alt := make([]int, r.alts*2)
      for i := 0; i < len(alt); i++ {
        // if a particular submatch is not encountered, return -1.
        alt[i] = -1
      }

      a := p.alt
      for a != nil {
        pos := (a.alt * 2)
        if a.is_end {
          pos += 1
        }
        if alt[pos] == -1 {
          alt[pos] = a.pos
        }
        a = a.prev
      }
      return true, alt
    }
  }

  return false, make([]int, 0)
}
<|endoftext|>


<commit_before>package aws_helper

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"time"
)

// Returns an AWS session object for the given region (required), profile name (optional), and IAM role to assume
// (optional), ensuring that the credentials are available
func CreateAwsSession(awsRegion, awsEndpoint string, awsAccessKey string, awsSecretKey string, awsProfile string, iamRoleArn string, terragruntOptions *options.TerragruntOptions) (*session.Session, error) {
	var awsConfig aws.Config
	if awsAccessKey != "" && awsSecretKey != "" {
		awsConfig = aws.Config{
			Region:      aws.String(awsRegion),
			Endpoint:    aws.String(awsEndpoint),
			Credentials: credentials.NewStaticCredentials(awsAccessKey, awsSecretKey, ""),
		}
	} else {
		awsConfig = aws.Config{
			Region:   aws.String(awsRegion),
			Endpoint: aws.String(awsEndpoint),
		}
	}

	sess, err := session.NewSessionWithOptions(session.Options{
		Config:            awsConfig,
		Profile:           awsProfile,
		SharedConfigState: session.SharedConfigEnable,
	})
	if err != nil {
		return nil, errors.WithStackTraceAndPrefix(err, "Error initializing session")
	}

	if iamRoleArn != "" {
		sess.Config.Credentials = stscreds.NewCredentials(sess, iamRoleArn)
	} else if terragruntOptions.IamRole != "" {
		sess.Config.Credentials = stscreds.NewCredentials(sess, terragruntOptions.IamRole)
	}

	_, err = sess.Config.Credentials.Get()
	if err != nil {
		return nil, errors.WithStackTraceAndPrefix(err, "Error finding AWS credentials (did you set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables?)")
	}

	return sess, nil
}

// Make API calls to AWS to assume the IAM role specified and return the temporary AWS credentials to use that role
func AssumeIamRole(iamRoleArn string) (*sts.Credentials, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	_, err = sess.Config.Credentials.Get()
	if err != nil {
		return nil, errors.WithStackTraceAndPrefix(err, "Error finding AWS credentials (did you set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables?)")
	}

	stsClient := sts.New(sess)

	input := sts.AssumeRoleInput{
		RoleArn:         aws.String(iamRoleArn),
		RoleSessionName: aws.String(fmt.Sprintf("terragrunt-%d", time.Now().UTC().UnixNano())),
	}

	output, err := stsClient.AssumeRole(&input)
	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return output.Credentials, nil
}
<commit_msg>Added custom endpoint resolver only for S3. BTW regression tests run successfully<commit_after>package aws_helper

import (
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
	"github.com/aws/aws-sdk-go/aws/endpoints"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/sts"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"time"
)

// Returns an AWS session object for the given region (required), profile name (optional), and IAM role to assume
// (optional), ensuring that the credentials are available
func CreateAwsSession(awsRegion, awsEndpoint string, awsAccessKey string, awsSecretKey string, awsProfile string, iamRoleArn string, terragruntOptions *options.TerragruntOptions) (*session.Session, error) {
	defaultResolver := endpoints.DefaultResolver()
	s3CustResolverFn := func(service, region string, optFns ...func(*endpoints.Options)) (endpoints.ResolvedEndpoint, error) {
		if service == "s3" {
			return endpoints.ResolvedEndpoint{
				URL:           awsEndpoint,
				SigningRegion: "custom-signing-region",
			}, nil
		}

		return defaultResolver.EndpointFor(service, region, optFns...)
	}

	var awsConfig aws.Config
	if awsAccessKey != "" && awsSecretKey != "" {
		awsConfig = aws.Config{
			Region:           aws.String(awsRegion),
			EndpointResolver: endpoints.ResolverFunc(s3CustResolverFn),
			Credentials:      credentials.NewStaticCredentials(awsAccessKey, awsSecretKey, ""),
		}
	} else {
		awsConfig = aws.Config{
			Region:           aws.String(awsRegion),
			EndpointResolver: endpoints.ResolverFunc(s3CustResolverFn),
		}
	}

	sess, err := session.NewSessionWithOptions(session.Options{
		Config:            awsConfig,
		Profile:           awsProfile,
		SharedConfigState: session.SharedConfigEnable,
	})
	if err != nil {
		return nil, errors.WithStackTraceAndPrefix(err, "Error initializing session")
	}

	if iamRoleArn != "" {
		sess.Config.Credentials = stscreds.NewCredentials(sess, iamRoleArn)
	} else if terragruntOptions.IamRole != "" {
		sess.Config.Credentials = stscreds.NewCredentials(sess, terragruntOptions.IamRole)
	}

	_, err = sess.Config.Credentials.Get()
	if err != nil {
		return nil, errors.WithStackTraceAndPrefix(err, "Error finding AWS credentials (did you set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables?)")
	}

	return sess, nil
}

// Make API calls to AWS to assume the IAM role specified and return the temporary AWS credentials to use that role
func AssumeIamRole(iamRoleArn string) (*sts.Credentials, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	_, err = sess.Config.Credentials.Get()
	if err != nil {
		return nil, errors.WithStackTraceAndPrefix(err, "Error finding AWS credentials (did you set the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables?)")
	}

	stsClient := sts.New(sess)

	input := sts.AssumeRoleInput{
		RoleArn:         aws.String(iamRoleArn),
		RoleSessionName: aws.String(fmt.Sprintf("terragrunt-%d", time.Now().UTC().UnixNano())),
	}

	output, err := stsClient.AssumeRole(&input)
	if err != nil {
		return nil, errors.WithStackTrace(err)
	}

	return output.Credentials, nil
}
<|endoftext|>