<commit_before>// Copyright 2014 beego Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Usage:
//
// import "github.com/astaxie/beego/logs"
//
//	log := NewLogger(10000)
//	log.SetLogger("console", "")
//
//	> the first params stand for how many channel
//
// Use it like this:
//
//	log.Trace("trace")
//	log.Info("info")
//	log.Warn("warning")
//	log.Debug("debug")
//	log.Critical("critical")
//
//  more docs http://beego.me/docs/module/logs.md
package logs

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"path"
	"reflect"
	"runtime"
	"strings"
	"sync"

	"github.com/gogap/errors"
)

// RFC5424 log message levels.
const (
	LevelError = iota
	LevelWarn
	LevelInfo
	LevelDebug
)

// Legacy loglevel constants to ensure backwards compatibility.
//
// Deprecated: will be removed in 1.5.0.

type loggerType func() LoggerInterface

// LoggerInterface defines the behavior of a log provider.
type LoggerInterface interface {
	Init(config string) error
	WriteMsg(msg string, level int) error
	Destroy()
	Flush()
}

var adapters = make(map[string]loggerType)

// Register makes a log provide available by the provided name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func Register(name string, log loggerType) {
	if log == nil {
		panic("logs: Register provide is nil")
	}
	if _, dup := adapters[name]; dup {
		panic("logs: Register called twice for provider " + name)
	}
	adapters[name] = log
}

// Logger is default logger in beego application.
// it can contain several providers and log message into all providers.
type Logger struct {
	lock                sync.Mutex
	level               int
	enableFuncCallDepth bool
	loggerFuncCallDepth int
	msg                 chan *logMsg
	outputs             map[string]LoggerInterface
}

type logMsg struct {
	level int
	msg   string
}

// NewLogger returns a new Logger.
// channellen means the number of messages in chan.
// if the buffering chan is full, logger adapters write to file or other way.
func NewLogger(channellen int64) *Logger {
	bl := new(Logger)
	bl.level = LevelDebug
	bl.loggerFuncCallDepth = 2
	bl.EnableFuncCallDepth(true)
	bl.msg = make(chan *logMsg, channellen)
	bl.outputs = make(map[string]LoggerInterface)
	bl.SetLogger("console", "") // default output to console
	go bl.startLogger()
	return bl
}

func NewFileLogger(file string) *Logger {
	l := NewLogger(1024)
	path := strings.Split(file, "/")
	if len(path) > 1 {
		exec.Command("mkdir", path[0]).Run()
	}
	l.SetLogger("file", fmt.Sprintf(`{"filename":"%s","maxdays":7}`, file))
	l.EnableFuncCallDepth(true)
	l.SetLogFuncCallDepth(3)
	return l
}

// SetLogger provides a given logger adapter into Logger with config string.
// config need to be correct JSON as string: {"interval":360}.
func (bl *Logger) SetLogger(adaptername string, config string) error {
	bl.lock.Lock()
	defer bl.lock.Unlock()
	if log, ok := adapters[adaptername]; ok {
		lg := log()
		err := lg.Init(config)
		bl.outputs[adaptername] = lg
		if err != nil {
			fmt.Println("logs.Logger.SetLogger: " + err.Error())
			return err
		}
	} else {
		return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adaptername)
	}
	return nil
}

// remove a logger adapter in Logger.
func (bl *Logger) DelLogger(adaptername string) error {
	bl.lock.Lock()
	defer bl.lock.Unlock()
	if lg, ok := bl.outputs[adaptername]; ok {
		lg.Destroy()
		delete(bl.outputs, adaptername)
		return nil
	} else {
		return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adaptername)
	}
}

func (bl *Logger) writerMsg(loglevel int, msg string) error {
	if loglevel > bl.level {
		return nil
	}
	lm := new(logMsg)
	lm.level = loglevel
	if bl.enableFuncCallDepth {
		_, file, line, ok := runtime.Caller(bl.loggerFuncCallDepth)
		if ok {
			_, filename := path.Split(file)
			lm.msg = fmt.Sprintf("[%s:%d] %s", filename, line, msg)
		} else {
			lm.msg = msg
		}
	} else {
		lm.msg = msg
	}
	bl.msg <- lm
	return nil
}

// Set log message level.
//
// If message level (such as LevelDebug) is higher than logger level (such as LevelWarning),
// log providers will not even be sent the message.
func (bl *Logger) SetLevel(l int) {
	bl.level = l
}

// set log funcCallDepth
func (bl *Logger) SetLogFuncCallDepth(d int) {
	bl.loggerFuncCallDepth = d
}

// enable log funcCallDepth
func (bl *Logger) EnableFuncCallDepth(b bool) {
	bl.enableFuncCallDepth = b
}

// start logger chan reading.
// when chan is not empty, write logs.
func (bl *Logger) startLogger() {
	for {
		select {
		case bm := <-bl.msg:
			for _, l := range bl.outputs {
				err := l.WriteMsg(bm.msg, bm.level)
				if err != nil {
					fmt.Println("ERROR, unable to WriteMsg:", err)
				}
			}
		}
	}
}

// Log ERROR level message.
func (bl *Logger) Error(v ...interface{}) {
	bl.log("Error", LevelError, v)
}

// Log WARNING level message.
func (bl *Logger) Warn(v ...interface{}) {
	bl.log("Warn", LevelWarn, v)
}

// Log INFORMATIONAL level message.
func (bl *Logger) Info(v ...interface{}) {
	bl.log("Info", LevelInfo, v)
}

// Log DEBUG level message.
func (bl *Logger) Debug(v ...interface{}) {
	bl.log("Debug", LevelDebug, v)
}

func (bl *Logger) log(tp string, level int, v ...interface{}) {
	msg := fmt.Sprintf("["+tp+"] "+generateFmtStr(len(v)), v...)

	stack := handleError(rotate(v))
	if stack != "" {
		msg = msg + "\n" + stack
	}
	bl.writerMsg(level, msg)
}

//simple and strong, niu b !
func rotate(item interface{}) interface{} {
	if items, ok := item.([]interface{}); ok {
		if len(items) > 0 {
			for _, v := range items {
				item = rotate(v)
				if errcode, ok := item.(errors.ErrCode); ok {
					return errcode
				}
			}
		}
	}
	return item
}

func (bl *Logger) Pretty(v interface{}, message string) {
	bl.pretty(v, message)
}
func (bl *Logger) pretty(v interface{}, message string) {
	b, _ := json.MarshalIndent(v, " ", "  ")
	if message == "" {
		message = reflect.TypeOf(v).String()
	}
	bl.writerMsg(LevelDebug, fmt.Sprintf("[Pretty]\n%s\n%s", message, string(b)))
}

// flush all chan data.
func (bl *Logger) Flush() {
	for _, l := range bl.outputs {
		l.Flush()
	}
}

// close logger, flush all chan data and destroy all adapters in Logger.
func (bl *Logger) Close() {
	for {
		if len(bl.msg) > 0 {
			bm := <-bl.msg
			for _, l := range bl.outputs {
				err := l.WriteMsg(bm.msg, bm.level)
				if err != nil {
					fmt.Println("ERROR, unable to WriteMsg (while closing logger):", err)
				}
			}
		} else {
			break
		}
	}
	for _, l := range bl.outputs {
		l.Flush()
		l.Destroy()
	}
}

func generateFmtStr(n int) string {
	return strings.Repeat("%v ", n)
}

func handleError(v interface{}) (msg string) {
	if err, ok := v.(errors.ErrCode); ok {
		msg = msg + err.StackTrace()
	}
	return
}
<commit_msg>make rotate better<commit_after>// Copyright 2014 beego Author. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Usage:
//
// import "github.com/astaxie/beego/logs"
//
//	log := NewLogger(10000)
//	log.SetLogger("console", "")
//
//	> the first params stand for how many channel
//
// Use it like this:
//
//	log.Trace("trace")
//	log.Info("info")
//	log.Warn("warning")
//	log.Debug("debug")
//	log.Critical("critical")
//
//  more docs http://beego.me/docs/module/logs.md
package logs

import (
	"encoding/json"
	"fmt"
	"os/exec"
	"path"
	"reflect"
	"runtime"
	"strings"
	"sync"

	"github.com/gogap/errors"
)

// RFC5424 log message levels.
const (
	LevelError = iota
	LevelWarn
	LevelInfo
	LevelDebug
)

// Legacy loglevel constants to ensure backwards compatibility.
//
// Deprecated: will be removed in 1.5.0.

type loggerType func() LoggerInterface

// LoggerInterface defines the behavior of a log provider.
type LoggerInterface interface {
	Init(config string) error
	WriteMsg(msg string, level int) error
	Destroy()
	Flush()
}

var adapters = make(map[string]loggerType)

// Register makes a log provide available by the provided name.
// If Register is called twice with the same name or if driver is nil,
// it panics.
func Register(name string, log loggerType) {
	if log == nil {
		panic("logs: Register provide is nil")
	}
	if _, dup := adapters[name]; dup {
		panic("logs: Register called twice for provider " + name)
	}
	adapters[name] = log
}

// Logger is default logger in beego application.
// it can contain several providers and log message into all providers.
type Logger struct {
	lock                sync.Mutex
	level               int
	enableFuncCallDepth bool
	loggerFuncCallDepth int
	msg                 chan *logMsg
	outputs             map[string]LoggerInterface
}

type logMsg struct {
	level int
	msg   string
}

// NewLogger returns a new Logger.
// channellen means the number of messages in chan.
// if the buffering chan is full, logger adapters write to file or other way.
func NewLogger(channellen int64) *Logger {
	bl := new(Logger)
	bl.level = LevelDebug
	bl.loggerFuncCallDepth = 2
	bl.EnableFuncCallDepth(true)
	bl.msg = make(chan *logMsg, channellen)
	bl.outputs = make(map[string]LoggerInterface)
	bl.SetLogger("console", "") // default output to console
	go bl.startLogger()
	return bl
}

func NewFileLogger(file string) *Logger {
	l := NewLogger(1024)
	path := strings.Split(file, "/")
	if len(path) > 1 {
		exec.Command("mkdir", path[0]).Run()
	}
	l.SetLogger("file", fmt.Sprintf(`{"filename":"%s","maxdays":7}`, file))
	l.EnableFuncCallDepth(true)
	l.SetLogFuncCallDepth(3)
	return l
}

// SetLogger provides a given logger adapter into Logger with config string.
// config need to be correct JSON as string: {"interval":360}.
func (bl *Logger) SetLogger(adaptername string, config string) error {
	bl.lock.Lock()
	defer bl.lock.Unlock()
	if log, ok := adapters[adaptername]; ok {
		lg := log()
		err := lg.Init(config)
		bl.outputs[adaptername] = lg
		if err != nil {
			fmt.Println("logs.Logger.SetLogger: " + err.Error())
			return err
		}
	} else {
		return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adaptername)
	}
	return nil
}

// remove a logger adapter in Logger.
func (bl *Logger) DelLogger(adaptername string) error {
	bl.lock.Lock()
	defer bl.lock.Unlock()
	if lg, ok := bl.outputs[adaptername]; ok {
		lg.Destroy()
		delete(bl.outputs, adaptername)
		return nil
	} else {
		return fmt.Errorf("logs: unknown adaptername %q (forgotten Register?)", adaptername)
	}
}

func (bl *Logger) writerMsg(loglevel int, msg string) error {
	if loglevel > bl.level {
		return nil
	}
	lm := new(logMsg)
	lm.level = loglevel
	if bl.enableFuncCallDepth {
		_, file, line, ok := runtime.Caller(bl.loggerFuncCallDepth)
		if ok {
			_, filename := path.Split(file)
			lm.msg = fmt.Sprintf("[%s:%d] %s", filename, line, msg)
		} else {
			lm.msg = msg
		}
	} else {
		lm.msg = msg
	}
	bl.msg <- lm
	return nil
}

// Set log message level.
//
// If message level (such as LevelDebug) is higher than logger level (such as LevelWarning),
// log providers will not even be sent the message.
func (bl *Logger) SetLevel(l int) {
	bl.level = l
}

// set log funcCallDepth
func (bl *Logger) SetLogFuncCallDepth(d int) {
	bl.loggerFuncCallDepth = d
}

// enable log funcCallDepth
func (bl *Logger) EnableFuncCallDepth(b bool) {
	bl.enableFuncCallDepth = b
}

// start logger chan reading.
// when chan is not empty, write logs.
func (bl *Logger) startLogger() {
	for {
		select {
		case bm := <-bl.msg:
			for _, l := range bl.outputs {
				err := l.WriteMsg(bm.msg, bm.level)
				if err != nil {
					fmt.Println("ERROR, unable to WriteMsg:", err)
				}
			}
		}
	}
}

// Log ERROR level message.
func (bl *Logger) Error(v ...interface{}) {
	bl.log("Error", LevelError, v)
}

// Log WARNING level message.
func (bl *Logger) Warn(v ...interface{}) {
	bl.log("Warn", LevelWarn, v)
}

// Log INFORMATIONAL level message.
func (bl *Logger) Info(v ...interface{}) {
	bl.log("Info", LevelInfo, v)
}

// Log DEBUG level message.
func (bl *Logger) Debug(v ...interface{}) {
	bl.log("Debug", LevelDebug, v)
}

func (bl *Logger) log(tp string, level int, v ...interface{}) {
	msg := fmt.Sprintf("["+tp+"] "+generateFmtStr(len(v)), v...)

	stack := handleError(rotate(v))
	if stack != "" {
		msg = msg + "\n" + stack
	}
	bl.writerMsg(level, msg)
}

//simple and strong, niu b !
func rotate(item interface{}) interface{} {
	if items, ok := item.([]interface{}); ok {
		for _, item = range items {
			if errcode, ok := rotate(item).(errors.ErrCode); ok {
				return errcode
			}
		}
	}
	return item
}

func (bl *Logger) Pretty(v interface{}, message string) {
	bl.pretty(v, message)
}
func (bl *Logger) pretty(v interface{}, message string) {
	b, _ := json.MarshalIndent(v, " ", "  ")
	if message == "" {
		message = reflect.TypeOf(v).String()
	}
	bl.writerMsg(LevelDebug, fmt.Sprintf("[Pretty]\n%s\n%s", message, string(b)))
}

// flush all chan data.
func (bl *Logger) Flush() {
	for _, l := range bl.outputs {
		l.Flush()
	}
}

// close logger, flush all chan data and destroy all adapters in Logger.
func (bl *Logger) Close() {
	for {
		if len(bl.msg) > 0 {
			bm := <-bl.msg
			for _, l := range bl.outputs {
				err := l.WriteMsg(bm.msg, bm.level)
				if err != nil {
					fmt.Println("ERROR, unable to WriteMsg (while closing logger):", err)
				}
			}
		} else {
			break
		}
	}
	for _, l := range bl.outputs {
		l.Flush()
		l.Destroy()
	}
}

func generateFmtStr(n int) string {
	return strings.Repeat("%v ", n)
}

func handleError(v interface{}) (msg string) {
	if err, ok := v.(errors.ErrCode); ok {
		msg = msg + err.StackTrace()
	}
	return
}
<|endoftext|>


<commit_before>package gleam

import (
	"log"
	"net/http"
	"os"
	"path"

	"github.com/cjoudrey/gluahttp"
	"github.com/cjoudrey/gluaurl"
	mqtt "github.com/eclipse/paho.mqtt.golang"
	"github.com/yuin/gluare"
	lua "github.com/yuin/gopher-lua"
	"layeh.com/gopher-json"
	"layeh.com/gopher-lfs"
	"layeh.com/gopher-luar"
)

type luaEnv struct {
	root  string
	state *lua.LState
}

func newLuaEnv(root string) *luaEnv {
	return &luaEnv{
		root: root,
	}
}

func (l *luaEnv) Init(config *Config) error {
	if err := os.Chdir(l.root); err != nil {
		return err
	}
	opt := lua.Options{
		CallStackSize:       1024,
		IncludeGoStackTrace: true,
		RegistrySize:        1024 * 64,
		SkipOpenLibs:        false,
	}
	l.state = lua.NewState(opt)
	json.Preload(l.state)
	lfs.Preload(l.state)
	l.state.PreloadModule("http", gluahttp.NewHttpModule(&http.Client{}).Loader)
	l.state.PreloadModule("re", gluare.Loader)
	l.state.PreloadModule("url", gluaurl.Loader)
	l.state.SetGlobal("Log", l.state.NewFunction(func(L *lua.LState) int {
		argc := L.GetTop()
		argv := make([]interface{}, argc)
		for i := 1; i <= argc; i++ {
			argv[i-1] = L.Get(i)
		}
		log.Print(argv...)
		return 0
	}))
	l.state.SetGlobal("Logf", l.state.NewFunction(func(L *lua.LState) int {
		argc := L.GetTop()
		format := L.Get(1).String()
		argv := make([]interface{}, argc-1)
		for i := 2; i <= argc; i++ {
			argv[i-2] = L.Get(i)
		}
		log.Printf(format, argv...)
		return 0
	}))
	l.state.SetGlobal("config", luar.New(l.state, config))
	return l.mustDoScript("init")
}

func (l *luaEnv) Final() error {
	err := l.mustDoScript("final")
	l.state.Close()
	return err
}

func (l *luaEnv) mustDoScript(name string) error {
	script := path.Join(l.root, name+".lua")
	if _, err := os.Stat(script); err != nil {
		return err
	}
	if err := l.state.DoFile(script); err != nil {
		return err
	}
	return nil
}

func (l *luaEnv) newHandler(name string) mqtt.MessageHandler {
	script := path.Join(l.root, name+".lua")
	if _, err := os.Stat(script); err != nil {
		return nil
	}
	return func(client mqtt.Client, msg mqtt.Message) {
		state, cancel := l.state.NewThread()
		defer func() {
			if cancel != nil {
				cancel()
			}
		}()

		clientL := luar.New(l.state, client)
		state.SetGlobal("Client", clientL)
		msgL := messageToLua(l.state, msg)
		state.SetGlobal("Message", msgL)

		if err := state.DoFile(script); err != nil {
			log.Printf("MsgErr[%X]: %s", msg.MessageID(), err)
		}
		defer state.Close()
	}
}

func (l *luaEnv) defaultHandler(client mqtt.Client, msg mqtt.Message) {
	p := lua.P{
		Fn:      l.state.GetGlobal("DefaultPublishHandler"),
		NRet:    0,
		Protect: true,
	}

	clientL := luar.New(l.state, client)
	msgL := messageToLua(l.state, msg)

	if err := l.state.CallByParam(p, clientL, msgL); err != nil {
		log.Printf("DefMsgErr[%s-%X]: %s", msg.Topic(), msg.MessageID(), err)
	}
}

func messageToLua(L *lua.LState, msg mqtt.Message) *lua.LTable {
	msgL := L.CreateTable(0, 6)
	msgL.RawSetString("Duplicate", lua.LBool(msg.Duplicate()))
	msgL.RawSetString("MessageID", lua.LNumber(msg.MessageID()))
	msgL.RawSetString("Payload", lua.LString(msg.Payload()))
	msgL.RawSetString("Qos", lua.LNumber(msg.Qos()))
	msgL.RawSetString("Retained", lua.LBool(msg.Retained()))
	msgL.RawSetString("Topic", lua.LString(msg.Topic()))
	return msgL
}
<commit_msg>fixed pwd & stat file<commit_after>package gleam

import (
	"log"
	"net/http"
	"os"
	"path"

	"github.com/cjoudrey/gluahttp"
	"github.com/cjoudrey/gluaurl"
	mqtt "github.com/eclipse/paho.mqtt.golang"
	"github.com/yuin/gluare"
	lua "github.com/yuin/gopher-lua"
	"layeh.com/gopher-json"
	"layeh.com/gopher-lfs"
	"layeh.com/gopher-luar"
)

type luaEnv struct {
	root  string
	state *lua.LState
}

func newLuaEnv(root string) *luaEnv {
	return &luaEnv{
		root: root,
	}
}

func (l *luaEnv) Init(config *Config) error {
	if err := os.Chdir(l.root); err != nil {
		return err
	}
	opt := lua.Options{
		CallStackSize:       1024,
		IncludeGoStackTrace: true,
		RegistrySize:        1024 * 64,
		SkipOpenLibs:        false,
	}
	l.state = lua.NewState(opt)
	json.Preload(l.state)
	lfs.Preload(l.state)
	l.state.PreloadModule("http", gluahttp.NewHttpModule(&http.Client{}).Loader)
	l.state.PreloadModule("re", gluare.Loader)
	l.state.PreloadModule("url", gluaurl.Loader)
	l.state.SetGlobal("Log", l.state.NewFunction(func(L *lua.LState) int {
		argc := L.GetTop()
		argv := make([]interface{}, argc)
		for i := 1; i <= argc; i++ {
			argv[i-1] = L.Get(i)
		}
		log.Print(argv...)
		return 0
	}))
	l.state.SetGlobal("Logf", l.state.NewFunction(func(L *lua.LState) int {
		argc := L.GetTop()
		format := L.Get(1).String()
		argv := make([]interface{}, argc-1)
		for i := 2; i <= argc; i++ {
			argv[i-2] = L.Get(i)
		}
		log.Printf(format, argv...)
		return 0
	}))
	l.state.SetGlobal("config", luar.New(l.state, config))
	return l.mustDoScript("init")
}

func (l *luaEnv) Final() error {
	err := l.mustDoScript("final")
	l.state.Close()
	return err
}

func (l *luaEnv) mustDoScript(name string) error {
	script := name + ".lua"
	if _, err := os.Stat(script); err != nil {
		return err
	}
	if err := l.state.DoFile(script); err != nil {
		return err
	}
	return nil
}

func (l *luaEnv) newHandler(name string) mqtt.MessageHandler {
	script := path.Join(l.root, name+".lua")
	if _, err := os.Stat(script); err != nil {
		return nil
	}
	return func(client mqtt.Client, msg mqtt.Message) {
		state, cancel := l.state.NewThread()
		defer func() {
			if cancel != nil {
				cancel()
			}
		}()

		clientL := luar.New(l.state, client)
		state.SetGlobal("Client", clientL)
		msgL := messageToLua(l.state, msg)
		state.SetGlobal("Message", msgL)

		if err := state.DoFile(script); err != nil {
			log.Printf("MsgErr[%X]: %s", msg.MessageID(), err)
		}
		defer state.Close()
	}
}

func (l *luaEnv) defaultHandler(client mqtt.Client, msg mqtt.Message) {
	p := lua.P{
		Fn:      l.state.GetGlobal("DefaultPublishHandler"),
		NRet:    0,
		Protect: true,
	}

	clientL := luar.New(l.state, client)
	msgL := messageToLua(l.state, msg)

	if err := l.state.CallByParam(p, clientL, msgL); err != nil {
		log.Printf("DefMsgErr[%s-%X]: %s", msg.Topic(), msg.MessageID(), err)
	}
}

func messageToLua(L *lua.LState, msg mqtt.Message) *lua.LTable {
	msgL := L.CreateTable(0, 6)
	msgL.RawSetString("Duplicate", lua.LBool(msg.Duplicate()))
	msgL.RawSetString("MessageID", lua.LNumber(msg.MessageID()))
	msgL.RawSetString("Payload", lua.LString(msg.Payload()))
	msgL.RawSetString("Qos", lua.LNumber(msg.Qos()))
	msgL.RawSetString("Retained", lua.LBool(msg.Retained()))
	msgL.RawSetString("Topic", lua.LString(msg.Topic()))
	return msgL
}
<|endoftext|>


<commit_before>package ping

import (
	"bytes"
	"fmt"
	"html/template"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// Protocol ...
type Protocol int

func (protocol Protocol) String() string {
	switch protocol {
	case TCP:
		return "tcp"
	case HTTP:
		return "http"
	case HTTPS:
		return "https"
	}
	return "unkown"
}

const (
	// TCP is tcp protocol
	TCP Protocol = iota
	// HTTP is http protocol
	HTTP
	// HTTPS is https protocol
	HTTPS
)

// NewProtocol convert protocol stirng to Protocol
func NewProtocol(protocol string) (Protocol, error) {
	switch strings.ToLower(protocol) {
	case TCP.String():
		return TCP, nil
	case HTTP.String():
		return HTTP, nil
	case HTTPS.String():
		return HTTPS, nil
	}
	return 0, fmt.Errorf("protocol %s not support", protocol)
}

// Target is a ping
type Target struct {
	Protocol Protocol
	Host     string
	Port     int

	Counter  int
	Interval time.Duration
	Timeout  time.Duration
}

func (target Target) String() string {
	return fmt.Sprintf("%s://%s:%d", target.Protocol, target.Host, target.Port)
}

// Pinger is a ping interface
type Pinger interface {
	Start() <-chan struct{}
	Stop()
	Result() *Result
	SetTarget(target *Target)
}

// Ping is a ping interface
type Ping interface {
	Start() <-chan struct{}

	Host() string
	Port() int
	Protocol() Protocol
	Counter() int

	Stop()

	Result() Result
}

// Result ...
type Result struct {
	Counter        int
	SuccessCounter int
	Target         *Target

	MinDuration   time.Duration
	MaxDuration   time.Duration
	TotalDuration time.Duration
}

// Avg return the average time of ping
func (result Result) Avg() time.Duration {
	if result.SuccessCounter == 0 {
		return 0
	}
	return result.TotalDuration / time.Duration(result.SuccessCounter)
}

// Failed return failed counter
func (result Result) Failed() int {
	return result.Counter - result.SuccessCounter
}

func (result Result) String() string {
	const resultTpl = `
Ping statistics {{.Target}}
	{{.Counter}} probes sent.
	{{.SuccessCounter}} successful, {{.Failed}} failed.
Approximate trip times:
	Minimum = {{.MinDuration}}, Maximum = {{.MaxDuration}}, Average = {{.Avg}}`
	t := template.Must(template.New("result").Parse(resultTpl))
	res := bytes.NewBufferString("")
	t.Execute(res, result)
	return res.String()
}

// CheckURI check uri
func CheckURI(uri string) (schema, host string, port int, matched bool) {
	const reExp = `^((?P<schema>((ht|f)tp(s?))|tcp)\://)?((([a-zA-Z0-9_\-]{2,}\.)+[a-zA-Z]{2,})|((?:(?:25[0-5]|2[0-4]\d|[01]\d\d|\d?\d)((\.?\d)\.)){4})|(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9]))(:([0-9]+))?(/[a-zA-Z0-9\-\._\?\,\'/\\\+&%\$#\=~]*)?$`
	pattern := regexp.MustCompile(reExp)
	res := pattern.FindStringSubmatch(uri)
	if len(res) == 0 {
		return
	}
	matched = true
	schema = res[2]
	if schema == "" {
		schema = "tcp"
	}
	host = res[6]
	if res[17] == "" {
		if schema == HTTPS.String() {
			port = 443
		} else {
			port = 80
		}
	} else {
		port, _ = strconv.Atoi(res[17])
	}

	return
}
<commit_msg>fix CheckURI<commit_after>package ping

import (
	"bytes"
	"fmt"
	"html/template"
	"regexp"
	"strconv"
	"strings"
	"time"
)

// Protocol ...
type Protocol int

func (protocol Protocol) String() string {
	switch protocol {
	case TCP:
		return "tcp"
	case HTTP:
		return "http"
	case HTTPS:
		return "https"
	}
	return "unkown"
}

const (
	// TCP is tcp protocol
	TCP Protocol = iota
	// HTTP is http protocol
	HTTP
	// HTTPS is https protocol
	HTTPS
)

// NewProtocol convert protocol stirng to Protocol
func NewProtocol(protocol string) (Protocol, error) {
	switch strings.ToLower(protocol) {
	case TCP.String():
		return TCP, nil
	case HTTP.String():
		return HTTP, nil
	case HTTPS.String():
		return HTTPS, nil
	}
	return 0, fmt.Errorf("protocol %s not support", protocol)
}

// Target is a ping
type Target struct {
	Protocol Protocol
	Host     string
	Port     int

	Counter  int
	Interval time.Duration
	Timeout  time.Duration
}

func (target Target) String() string {
	return fmt.Sprintf("%s://%s:%d", target.Protocol, target.Host, target.Port)
}

// Pinger is a ping interface
type Pinger interface {
	Start() <-chan struct{}
	Stop()
	Result() *Result
	SetTarget(target *Target)
}

// Ping is a ping interface
type Ping interface {
	Start() <-chan struct{}

	Host() string
	Port() int
	Protocol() Protocol
	Counter() int

	Stop()

	Result() Result
}

// Result ...
type Result struct {
	Counter        int
	SuccessCounter int
	Target         *Target

	MinDuration   time.Duration
	MaxDuration   time.Duration
	TotalDuration time.Duration
}

// Avg return the average time of ping
func (result Result) Avg() time.Duration {
	if result.SuccessCounter == 0 {
		return 0
	}
	return result.TotalDuration / time.Duration(result.SuccessCounter)
}

// Failed return failed counter
func (result Result) Failed() int {
	return result.Counter - result.SuccessCounter
}

func (result Result) String() string {
	const resultTpl = `
Ping statistics {{.Target}}
	{{.Counter}} probes sent.
	{{.SuccessCounter}} successful, {{.Failed}} failed.
Approximate trip times:
	Minimum = {{.MinDuration}}, Maximum = {{.MaxDuration}}, Average = {{.Avg}}`
	t := template.Must(template.New("result").Parse(resultTpl))
	res := bytes.NewBufferString("")
	t.Execute(res, result)
	return res.String()
}

// CheckURI check uri
func CheckURI(uri string) (schema, host string, port int, matched bool) {
	const reExp = `^((?P<schema>((ht|f)tp(s?))|tcp)\://)?((([a-zA-Z0-9_\-]+\.)+[a-zA-Z]{2,})|((?:(?:25[0-5]|2[0-4]\d|[01]\d\d|\d?\d)((\.?\d)\.)){4})|(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9]))(:([0-9]+))?(/[a-zA-Z0-9\-\._\?\,\'/\\\+&%\$#\=~]*)?$`
	pattern := regexp.MustCompile(reExp)
	res := pattern.FindStringSubmatch(uri)
	if len(res) == 0 {
		return
	}
	matched = true
	schema = res[2]
	if schema == "" {
		schema = "tcp"
	}
	host = res[6]
	if res[17] == "" {
		if schema == HTTPS.String() {
			port = 443
		} else {
			port = 80
		}
	} else {
		port, _ = strconv.Atoi(res[17])
	}

	return
}
<|endoftext|>


<commit_before>// Package global provides an algorithm for globally adaptive hierarchical
// interpolation.
package global

import (
	"fmt"
	"math"

	"github.com/ready-steady/adapt/algorithm/internal"
)

var (
	infinity = math.Inf(1.0)
	none     = ^uint(0)
)

// Basis is a functional basis.
type Basis interface {
	// Compute evaluates the value of a basis function.
	Compute([]uint64, []float64) float64
}

// Grid is a sparse grid.
type Grid interface {
	// Compute returns the nodes corresponding to a set of indices.
	Compute([]uint64) []float64

	// Index returns the indices of a set of levels.
	Index([]uint8) []uint64
}

// Interpolator is an instance of the algorithm.
type Interpolator struct {
	config Config
	basis  Basis
	grid   Grid
}

// Progress contains information about the interpolation process.
type Progress struct {
	Level       uint8 // Reached level
	Active      uint  // The number of active indices
	Passive     uint  // The number of passive indices
	Evaluations uint  // The number of function evaluations
}

// New creates an interpolator.
func New(grid Grid, basis Basis, config *Config) *Interpolator {
	return &Interpolator{
		config: *config,
		basis:  basis,
		grid:   grid,
	}
}

// Compute constructs an interpolant for a function.
func (self *Interpolator) Compute(target Target) *Surrogate {
	config := &self.config

	ni, no := target.Dimensions()
	nw := config.Workers

	surrogate := newSurrogate(ni, no)
	progress := Progress{}

	lindices := repeatUint8(0, 1*ni)
	active := []bool{true}
	depths := []uint{0}
	forward := repeatUint(none, 1*ni)
	backward := repeatUint(none, 1*ni)
	progress.Active++

	indices := self.grid.Index(lindices)

	nn := uint(len(indices)) / ni
	nodes := self.grid.Compute(indices)
	counts := []uint{nn}

	values := internal.Invoke(target.Compute, nodes, ni, no, nw)
	progress.Evaluations += nn

	surrogate.push(indices, values)

	lower, upper := updateBounds(nil, nil, values, no)
	scores, errors := updateScores(nil, nil, counts, values, no)
	for {
		target.Monitor(&progress)

		cursor := find(active)

		terminate := true
		δ := threshold(lower, upper, config.AbsTolerance, config.RelTolerance)

	accuracy:
		for _, i := range cursor {
			for j := uint(0); j < no; j++ {
				if errors[i*no+j] > δ[j] {
					terminate = false
					break accuracy
				}
			}
		}
		if terminate {
			break
		}

		min, current := minUint(depths, cursor...)
		max, _ := maxUint(depths)
		if float64(min) > (1.0-config.Adaptivity)*float64(max) {
			_, current = maxFloat64(scores, cursor...)
		}

		active[current] = false
		progress.Active--
		progress.Passive++

		lindex := lindices[current*ni : (current+1)*ni]

		indices := make([]uint64, 0)
		counts := make([]uint, 0)
		total := progress.Active + progress.Passive

	admissibility:
		for i := uint(0); i < ni && total < config.MaxIndices; i++ {
			if lindex[i] >= config.MaxLevel {
				continue
			}

			newBackward := repeatUint(none, ni)
			newBackward[i] = current
			for j := uint(0); j < ni; j++ {
				if i == j || lindex[j] == 0 {
					continue
				}
				l := forward[backward[current*ni+j]*ni+i]
				if l == none || active[l] {
					continue admissibility
				}
				newBackward[j] = l
			}

			lindices = append(lindices, lindex...)
			lindex := lindices[total*ni:]
			lindex[i]++

			if lindex[i] > progress.Level {
				progress.Level = lindex[i]
			}

			newIndices := self.grid.Index(lindex)
			indices = append(indices, newIndices...)
			counts = append(counts, uint(len(newIndices))/ni)

			active = append(active, true)
			depths = append(depths, depths[current]+1)

			for j := uint(0); j < ni; j++ {
				if newBackward[j] == none {
					continue
				}
				forward[newBackward[j]*ni+j] = total
			}

			forward = append(forward, repeatUint(none, ni)...)
			backward = append(backward, newBackward...)

			progress.Active++
			total++
		}

		nn := uint(len(indices)) / ni
		if progress.Evaluations+nn > config.MaxEvaluations {
			break
		}

		nodes := self.grid.Compute(indices)

		values := internal.Invoke(target.Compute, nodes, ni, no, nw)
		progress.Evaluations += nn

		surpluses := internal.Subtract(values, internal.Approximate(self.basis,
			surrogate.Indices, surrogate.Surpluses, nodes, ni, no, nw))

		surrogate.push(indices, surpluses)

		lower, upper = updateBounds(lower, upper, values, no)
		scores, errors = updateScores(scores, errors, counts, surpluses, no)
	}

	return surrogate
}

// Evaluate computes the values of an interpolant at a set of points.
func (self *Interpolator) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	return internal.Approximate(self.basis, surrogate.Indices, surrogate.Surpluses, points,
		surrogate.Inputs, surrogate.Outputs, self.config.Workers)
}

// String returns a human-friendly representation.
func (self *Progress) String() string {
	phantom := struct {
		level       uint8
		active      uint
		passive     uint
		evaluations uint
	}{
		level:       self.Level,
		active:      self.Active,
		passive:     self.Passive,
		evaluations: self.Evaluations,
	}
	return fmt.Sprintf("%+v", phantom)
}

func threshold(lower, upper []float64, absolute, relative float64) []float64 {
	no := uint(len(lower))
	threshold := make([]float64, no)
	for i := uint(0); i < no; i++ {
		threshold[i] = relative * (upper[i] - lower[i])
		if threshold[i] < absolute {
			threshold[i] = absolute
		}
	}
	return threshold
}

func updateBounds(lower, upper []float64, data []float64, no uint) ([]float64, []float64) {
	if lower == nil {
		lower = repeatFloat64(infinity, no)
	}
	if upper == nil {
		upper = repeatFloat64(-infinity, no)
	}
	nn := uint(len(data)) / no
	for i := uint(0); i < nn; i++ {
		for j := uint(0); j < no; j++ {
			point := data[i*no+j]
			if lower[j] > point {
				lower[j] = point
			}
			if upper[j] < point {
				upper[j] = point
			}
		}
	}
	return lower, upper
}

func updateScores(scores, errors []float64, counts []uint, surpluses []float64,
	no uint) ([]float64, []float64) {

	offset := uint(0)
	for _, count := range counts {
		score := 0.0
		error := repeatFloat64(-infinity, no)
		for j := uint(0); j < count; j++ {
			for l := uint(0); l < no; l++ {
				Δ := math.Abs(surpluses[(offset+j)*no+l])
				error[l] = math.Max(error[l], Δ)
				score += Δ
			}
		}
		score /= float64(count)
		scores = append(scores, score)
		errors = append(errors, error...)
		offset += count
	}
	return scores, errors
}
<commit_msg>a/global: a cosmetic change<commit_after>// Package global provides an algorithm for globally adaptive hierarchical
// interpolation.
package global

import (
	"fmt"
	"math"

	"github.com/ready-steady/adapt/algorithm/internal"
)

var (
	infinity = math.Inf(1.0)
	none     = ^uint(0)
)

// Basis is a functional basis.
type Basis interface {
	// Compute evaluates the value of a basis function.
	Compute([]uint64, []float64) float64
}

// Grid is a sparse grid.
type Grid interface {
	// Compute returns the nodes corresponding to a set of indices.
	Compute([]uint64) []float64

	// Index returns the indices of a set of levels.
	Index([]uint8) []uint64
}

// Interpolator is an instance of the algorithm.
type Interpolator struct {
	config Config
	basis  Basis
	grid   Grid
}

// Progress contains information about the interpolation process.
type Progress struct {
	Level       uint8 // Reached level
	Active      uint  // The number of active indices
	Passive     uint  // The number of passive indices
	Evaluations uint  // The number of function evaluations
}

// New creates an interpolator.
func New(grid Grid, basis Basis, config *Config) *Interpolator {
	return &Interpolator{
		config: *config,
		basis:  basis,
		grid:   grid,
	}
}

// Compute constructs an interpolant for a function.
func (self *Interpolator) Compute(target Target) *Surrogate {
	config := &self.config

	ni, no := target.Dimensions()
	nw := config.Workers

	surrogate := newSurrogate(ni, no)
	progress := Progress{}

	lindices := repeatUint8(0, 1*ni)
	active := []bool{true}
	depths := []uint{0}
	forward := repeatUint(none, 1*ni)
	backward := repeatUint(none, 1*ni)
	progress.Active++

	indices := self.grid.Index(lindices)

	nn := uint(len(indices)) / ni
	nodes := self.grid.Compute(indices)
	counts := []uint{nn}

	values := internal.Invoke(target.Compute, nodes, ni, no, nw)
	progress.Evaluations += nn

	surrogate.push(indices, values)

	lower, upper := updateBounds(nil, nil, values, no)
	scores, errors := updateScores(nil, nil, counts, values, no)
	for {
		target.Monitor(&progress)

		cursor := find(active)

		terminate := true
		δ := threshold(lower, upper, config.AbsTolerance, config.RelTolerance)

	accuracy:
		for _, i := range cursor {
			for j := uint(0); j < no; j++ {
				if errors[i*no+j] > δ[j] {
					terminate = false
					break accuracy
				}
			}
		}
		if terminate {
			break
		}

		min, current := minUint(depths, cursor...)
		max, _ := maxUint(depths)
		if float64(min) > (1.0-config.Adaptivity)*float64(max) {
			_, current = maxFloat64(scores, cursor...)
		}

		active[current] = false
		progress.Active--
		progress.Passive++

		lindex := lindices[current*ni : (current+1)*ni]

		indices := make([]uint64, 0)
		counts := make([]uint, 0)
		total := progress.Active + progress.Passive

	admissibility:
		for i := uint(0); i < ni && total < config.MaxIndices; i++ {
			if lindex[i] >= config.MaxLevel {
				continue
			}

			newBackward := repeatUint(none, ni)
			newBackward[i] = current
			for j := uint(0); j < ni; j++ {
				if i == j || lindex[j] == 0 {
					continue
				}
				l := forward[backward[current*ni+j]*ni+i]
				if l == none || active[l] {
					continue admissibility
				}
				newBackward[j] = l
			}

			lindices = append(lindices, lindex...)
			lindex := lindices[total*ni:]
			lindex[i]++

			newIndices := self.grid.Index(lindex)
			indices = append(indices, newIndices...)
			counts = append(counts, uint(len(newIndices))/ni)

			for j := uint(0); j < ni; j++ {
				if newBackward[j] != none {
					forward[newBackward[j]*ni+j] = total
				}
			}

			active = append(active, true)
			depths = append(depths, depths[current]+1)
			forward = append(forward, repeatUint(none, ni)...)
			backward = append(backward, newBackward...)

			if lindex[i] > progress.Level {
				progress.Level = lindex[i]
			}

			progress.Active++
			total++
		}

		nn := uint(len(indices)) / ni
		if progress.Evaluations+nn > config.MaxEvaluations {
			break
		}

		nodes := self.grid.Compute(indices)

		values := internal.Invoke(target.Compute, nodes, ni, no, nw)
		progress.Evaluations += nn

		surpluses := internal.Subtract(values, internal.Approximate(self.basis,
			surrogate.Indices, surrogate.Surpluses, nodes, ni, no, nw))

		surrogate.push(indices, surpluses)

		lower, upper = updateBounds(lower, upper, values, no)
		scores, errors = updateScores(scores, errors, counts, surpluses, no)
	}

	return surrogate
}

// Evaluate computes the values of an interpolant at a set of points.
func (self *Interpolator) Evaluate(surrogate *Surrogate, points []float64) []float64 {
	return internal.Approximate(self.basis, surrogate.Indices, surrogate.Surpluses, points,
		surrogate.Inputs, surrogate.Outputs, self.config.Workers)
}

// String returns a human-friendly representation.
func (self *Progress) String() string {
	phantom := struct {
		level       uint8
		active      uint
		passive     uint
		evaluations uint
	}{
		level:       self.Level,
		active:      self.Active,
		passive:     self.Passive,
		evaluations: self.Evaluations,
	}
	return fmt.Sprintf("%+v", phantom)
}

func threshold(lower, upper []float64, absolute, relative float64) []float64 {
	no := uint(len(lower))
	threshold := make([]float64, no)
	for i := uint(0); i < no; i++ {
		threshold[i] = relative * (upper[i] - lower[i])
		if threshold[i] < absolute {
			threshold[i] = absolute
		}
	}
	return threshold
}

func updateBounds(lower, upper []float64, data []float64, no uint) ([]float64, []float64) {
	if lower == nil {
		lower = repeatFloat64(infinity, no)
	}
	if upper == nil {
		upper = repeatFloat64(-infinity, no)
	}
	nn := uint(len(data)) / no
	for i := uint(0); i < nn; i++ {
		for j := uint(0); j < no; j++ {
			point := data[i*no+j]
			if lower[j] > point {
				lower[j] = point
			}
			if upper[j] < point {
				upper[j] = point
			}
		}
	}
	return lower, upper
}

func updateScores(scores, errors []float64, counts []uint, surpluses []float64,
	no uint) ([]float64, []float64) {

	offset := uint(0)
	for _, count := range counts {
		score := 0.0
		error := repeatFloat64(-infinity, no)
		for j := uint(0); j < count; j++ {
			for l := uint(0); l < no; l++ {
				Δ := math.Abs(surpluses[(offset+j)*no+l])
				error[l] = math.Max(error[l], Δ)
				score += Δ
			}
		}
		score /= float64(count)
		scores = append(scores, score)
		errors = append(errors, error...)
		offset += count
	}
	return scores, errors
}
<|endoftext|>


<commit_before>package catalog

import (
	"fmt"

	"github.com/dnaeon/gru/module"
	"github.com/dnaeon/gru/resource"
)

// Catalog type represents a collection of modules and resources
type Catalog struct {
	// Loaded modules after topological sorting
	Modules []*module.Module

	// Instantiated resources from the loaded modules
	Resources []resource.Resource

	// Catalog configuration
	Config *Config
}

// Config type represents a set of settings to use when
// creating and processing the catalog
type Config struct {
	// Name of main module to load
	Main string

	// Do not take any actions, just report what would be done
	DryRun bool

	// Module configuration settings to use
	ModuleConfig *module.Config
}

// Run processes the catalog
func (c *Catalog) Run() error {
	// Use the same writer as the one used by the resources
	w := c.Config.ModuleConfig.ResourceConfig.Writer

	fmt.Fprintf(w, "Loaded %d resources from %d modules\n", len(c.Resources), len(c.Modules))
	for _, r := range c.Resources {
		id := r.ResourceID()

		state, err := r.Evaluate()
		if err != nil {
			fmt.Fprintf(w, "%s %s\n", id, err)
			continue
		}

		if c.Config.DryRun {
			continue
		}

		// TODO: Skip resources which have failed dependencies

		var resourceErr error
		switch {
		case state.Want == state.Current:
			// Resource is in the desired state
			break
		case state.Want == resource.StatePresent || state.Want == resource.StateRunning:
			// Resource is absent, should be present
			if state.Current == resource.StateAbsent || state.Current == resource.StateStopped {
				fmt.Fprintf(w, "%s is %s, should be %s\n", id, state.Current, state.Want)
				resourceErr = r.Create()
			}
		case state.Want == resource.StateAbsent || state.Want == resource.StateStopped:
			// Resource is present, should be absent
			if state.Current == resource.StatePresent || state.Current == resource.StateRunning {
				fmt.Fprintf(w, "%s is %s, should be %s\n", id, state.Current, state.Want)
				resourceErr = r.Delete()
			}
		default:
			fmt.Fprintf(w, "%s unknown state(s): want %s, current %s\n", id, state.Want, state.Current)
			continue
		}

		if resourceErr != nil {
			fmt.Fprintf(w, "%s %s\n", id, resourceErr)
		}

		// Update resource if needed
		if state.Update {
			fmt.Fprintf(w, "%s resource is out of date, will be updated\n", id)
			if err := r.Update(); err != nil {
				fmt.Fprintf(w, "%s %s\n", id, err)
			}
		}
	}

	return nil
}

// Load creates a new catalog from the provided configuration
func Load(config *Config) (*Catalog, error) {
	c := &Catalog{
		Modules:   make([]*module.Module, 0),
		Resources: make([]resource.Resource, 0),
		Config:    config,
	}

	// Discover and load the modules from the provided
	// module path, sort the import graph and
	// finally add the sorted modules to the catalog
	modules, err := module.DiscoverAndLoad(config.ModuleConfig)
	if err != nil {
		return c, err
	}

	modulesGraph, err := module.ImportGraph(config.Main, config.ModuleConfig.Path)
	if err != nil {
		return c, err
	}

	modulesSorted, err := modulesGraph.Sort()
	if err != nil {
		return c, err
	}

	for _, node := range modulesSorted {
		c.Modules = append(c.Modules, modules[node.Name])
	}

	// Build the dependency graph for the resources from the
	// loaded modules and sort them
	collection, err := module.ResourceCollection(c.Modules)
	if err != nil {
		return c, err
	}

	collectionGraph, err := collection.DependencyGraph()
	if err != nil {
		return c, err
	}

	collectionSorted, err := collectionGraph.Sort()
	if err != nil {
		return c, err
	}

	for _, node := range collectionSorted {
		c.Resources = append(c.Resources, collection[node.Name])
	}

	return c, nil
}
<commit_msg>catalog: Register builtins and catalog in Lua<commit_after>package catalog

import (
	"fmt"
	"io"

	"github.com/dnaeon/gru/resource"
	"github.com/layeh/gopher-luar"
	"github.com/yuin/gopher-lua"
)

// Catalog type contains a collection of resources
type Catalog struct {
	// Unsorted contains the list of resources created by Lua
	unsorted []resource.Resource

	// Sorted contains the list of resources after a topological sort
	sorted []resource.Resource

	// Configuration settings
	config *Config
}

// Config type represents a set of settings to use when
// creating and processing the catalog
type Config struct {
	// Name of the Lua module to load and execute
	Module string

	// Do not take any actions, just report what would be done
	DryRun bool

	// Writer used to log events
	Writer io.Writer

	// Path to the site repo containing module and data files
	SiteRepo string

	// The Lua state
	L *lua.LState
}

// Run processes the catalog
func (c *Catalog) Run() error {
	// Use the same writer as the one used by the resources
	w := c.Config.ModuleConfig.ResourceConfig.Writer

	fmt.Fprintf(w, "Loaded %d resources from %d modules\n", len(c.Resources), len(c.Modules))
	for _, r := range c.Resources {
		id := r.ResourceID()

		state, err := r.Evaluate()
		if err != nil {
			fmt.Fprintf(w, "%s %s\n", id, err)
			continue
		}

		if c.Config.DryRun {
			continue
		}

		// TODO: Skip resources which have failed dependencies

		var resourceErr error
		switch {
		case state.Want == state.Current:
			// Resource is in the desired state
			break
		case state.Want == resource.StatePresent || state.Want == resource.StateRunning:
			// Resource is absent, should be present
			if state.Current == resource.StateAbsent || state.Current == resource.StateStopped {
				fmt.Fprintf(w, "%s is %s, should be %s\n", id, state.Current, state.Want)
				resourceErr = r.Create()
			}
		case state.Want == resource.StateAbsent || state.Want == resource.StateStopped:
			// Resource is present, should be absent
			if state.Current == resource.StatePresent || state.Current == resource.StateRunning {
				fmt.Fprintf(w, "%s is %s, should be %s\n", id, state.Current, state.Want)
				resourceErr = r.Delete()
			}
		default:
			fmt.Fprintf(w, "%s unknown state(s): want %s, current %s\n", id, state.Want, state.Current)
			continue
		}

		if resourceErr != nil {
			fmt.Fprintf(w, "%s %s\n", id, resourceErr)
		}

		// Update resource if needed
		if state.Update {
			fmt.Fprintf(w, "%s resource is out of date, will be updated\n", id)
			if err := r.Update(); err != nil {
				fmt.Fprintf(w, "%s %s\n", id, err)
			}
		}
	}

	return nil
}

// Load creates a new catalog from the provided configuration
func Load(config *Config) (*Catalog, error) {
	c := &Catalog{
		config:   config,
		sorted:   make([]resource.Resource, 0),
		unsorted: make([]resource.Resource, 0),
	}

	// Inject the configuration for resources
	resource.DefaultConfig = &resource.Config{
		Writer:   config.Writer,
		SiteRepo: config.SiteRepo,
	}

	// Register the resources and catalog in Lua
	resource.LuaRegisterBuiltin(config.L)
	config.L.SetGlobal("catalog", luar.New(config.L, c.unsorted))
	if err := L.DoFile(config.Module); err != nil {
		return c, err
	}

	// Perform a topological sort of the resources
	collection, err := resource.CreateCollection(c.unsorted)
	if err != nil {
		return c, err
	}

	collectionGraph, err := collection.DependencyGraph()
	if err != nil {
		return c, err
	}

	collectionSorted, err := collectionGraph.Sort()
	if err != nil {
		return c, err
	}

	for _, node := range collectionSorted {
		c.sorted = append(c.sorted, collection[node.Name])
	}

	return c, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package catalog

type Error error

type Store interface {
	Catalog() (*Catalog, Error)
}

type Catalog interface {
	Buckets() (map[string]*Bucket, Error)
}

type Bucket interface {
	Name() string
	Count() (int64, Error) // why is this needed?
	AccessPaths() ([]*Scanner, Error)
	Fetch(id string) (*Item, Error)
}

type IndexStatistics interface {
	Count() (int64, Error)
	Min() (*Item, Error)
	Max() (*Item, Error)
	DistinctCount(int64, Error)
	Bins() ([]*Bin, Error)
}

type Bin interface {
	Count() (int64, Error)
	Min() (*Item, Error)
	Max() (*Item, Error)
	DistinctCount(int64, Error)
}

type ItemChannel chan *Item

type Scanner interface {
	Channel() (ItemChannel, Error)
	SetChannel(ItemChannel)
}

type FullScanner interface {
	Scanner
}

// in rdbms, value -> id mappings are called inverted entries.
type InvertedScanner interface {
	Scanner
	Key() []string
	Direction() Direction // enum for ASC/DESC
	Statistics() (*IndexStatistics, Error)
}

// Couchbase view indexes
type ViewScanner interface {
	InvertedScanner
}

// Real indexes, a la SQL (not Couchbase view indexes)
type IndexScanner interface {
	InvertedScanner
}

// Full text search
type SearchScanner interface {
	Scanner
}
<commit_msg>Renamed InvertedScanner to RangeScanner.  Ran go build.<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package catalog

// Will be moved to a shared package
type Error error

// Will be moved to a shared package
type Item interface {
}

type Store interface {
	Catalog() (*Catalog, Error)
}

type Catalog interface {
	Buckets() (map[string]*Bucket, Error)
}

type Bucket interface {
	Name() string
	Count() (int64, Error) // why is this needed?
	AccessPaths() ([]*Scanner, Error)
	Fetch(id string) (*Item, Error)
}

type IndexStatistics interface {
	Count() (int64, Error)
	Min() (*Item, Error)
	Max() (*Item, Error)
	DistinctCount(int64, Error)
	Bins() ([]*Bin, Error)
}

type Bin interface {
	Count() (int64, Error)
	Min() (*Item, Error)
	Max() (*Item, Error)
	DistinctCount(int64, Error)
}

type ItemChannel chan *Item

type Scanner interface {
	Channel() (ItemChannel, Error)
	SetChannel(ItemChannel)
}

type FullScanner interface {
	Scanner
}

type Direction int

const (
	ASC  Direction = 1
	DESC           = 2
)

type RangeScanner interface {
	Scanner
	Key() []string
	Direction() Direction
	Statistics() (*IndexStatistics, Error)
}

// Couchbase view indexes
type ViewScanner interface {
	RangeScanner
}

// Declarative btree indexes
type IndexScanner interface {
	RangeScanner
}

// Full text search
type SearchScanner interface {
	Scanner
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"context"
	"fmt"
	"io"

	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
)

const initContainer = "kaniko-init-container"

func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace string, artifact *latest.KanikoArtifact, tag string) (string, error) {
	client, err := kubernetes.Client()
	if err != nil {
		return "", fmt.Errorf("getting Kubernetes client: %w", err)
	}
	pods := client.CoreV1().Pods(b.Namespace)

	podSpec, err := b.kanikoPodSpec(artifact, tag)
	if err != nil {
		return "", err
	}

	pod, err := pods.Create(podSpec)
	if err != nil {
		return "", fmt.Errorf("creating kaniko pod: %w", err)
	}
	defer func() {
		if err := pods.Delete(pod.Name, &metav1.DeleteOptions{
			GracePeriodSeconds: new(int64),
		}); err != nil {
			logrus.Fatalf("deleting pod: %s", err)
		}
	}()

	if err := b.copyKanikoBuildContext(ctx, workspace, artifact, pods, pod.Name); err != nil {
		return "", fmt.Errorf("copying sources: %w", err)
	}

	// Wait for the pods to succeed while streaming the logs
	waitForLogs := streamLogs(ctx, out, pod.Name, pods)

	if err := kubernetes.WaitForPodSucceeded(ctx, pods, pod.Name, b.timeout); err != nil {
		waitForLogs()
		return "", err
	}

	waitForLogs()

	return docker.RemoteDigest(tag, b.insecureRegistries)
}

// first copy over the buildcontext tarball into the init container tmp dir via kubectl cp
// Via kubectl exec, we extract the tarball to the empty dir
// Then, via kubectl exec, create the /tmp/complete file via kubectl exec to complete the init container
func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string, artifact *latest.KanikoArtifact, pods corev1.PodInterface, podName string) error {
	if err := kubernetes.WaitForPodInitialized(ctx, pods, podName); err != nil {
		return fmt.Errorf("waiting for pod to initialize: %w", err)
	}

	buildCtx, buildCtxWriter := io.Pipe()
	go func() {
		err := docker.CreateDockerTarContext(ctx, buildCtxWriter, workspace, &latest.DockerArtifact{
			BuildArgs:      artifact.BuildArgs,
			DockerfilePath: artifact.DockerfilePath,
		}, b.insecureRegistries)
		if err != nil {
			buildCtxWriter.CloseWithError(fmt.Errorf("creating docker context: %w", err))
			return
		}
		buildCtxWriter.Close()
	}()

	if err := b.kubectlcli.Run(ctx, buildCtx, nil, "exec", "-i", podName, "-c", initContainer, "-n", b.Namespace, "--", "tar", "-xf", "-", "-C", constants.DefaultKanikoEmptyDirMountPath); err != nil {
		return fmt.Errorf("uploading build context: %w", err)
	}

	// Generate a file to successfully terminate the init container
	return b.kubectlcli.Run(ctx, nil, nil, "exec", podName, "-c", initContainer, "-n", b.Namespace, "--", "touch", "/tmp/complete")
}
<commit_msg>[kaniko] Better error message when upload fails (#4023)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cluster

import (
	"bytes"
	"context"
	"fmt"
	"io"

	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
)

const initContainer = "kaniko-init-container"

func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace string, artifact *latest.KanikoArtifact, tag string) (string, error) {
	client, err := kubernetes.Client()
	if err != nil {
		return "", fmt.Errorf("getting Kubernetes client: %w", err)
	}
	pods := client.CoreV1().Pods(b.Namespace)

	podSpec, err := b.kanikoPodSpec(artifact, tag)
	if err != nil {
		return "", err
	}

	pod, err := pods.Create(podSpec)
	if err != nil {
		return "", fmt.Errorf("creating kaniko pod: %w", err)
	}
	defer func() {
		if err := pods.Delete(pod.Name, &metav1.DeleteOptions{
			GracePeriodSeconds: new(int64),
		}); err != nil {
			logrus.Fatalf("deleting pod: %s", err)
		}
	}()

	if err := b.copyKanikoBuildContext(ctx, workspace, artifact, pods, pod.Name); err != nil {
		return "", fmt.Errorf("copying sources: %w", err)
	}

	// Wait for the pods to succeed while streaming the logs
	waitForLogs := streamLogs(ctx, out, pod.Name, pods)

	if err := kubernetes.WaitForPodSucceeded(ctx, pods, pod.Name, b.timeout); err != nil {
		waitForLogs()
		return "", err
	}

	waitForLogs()

	return docker.RemoteDigest(tag, b.insecureRegistries)
}

// first copy over the buildcontext tarball into the init container tmp dir via kubectl cp
// Via kubectl exec, we extract the tarball to the empty dir
// Then, via kubectl exec, create the /tmp/complete file via kubectl exec to complete the init container
func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string, artifact *latest.KanikoArtifact, pods corev1.PodInterface, podName string) error {
	if err := kubernetes.WaitForPodInitialized(ctx, pods, podName); err != nil {
		return fmt.Errorf("waiting for pod to initialize: %w", err)
	}

	buildCtx, buildCtxWriter := io.Pipe()
	go func() {
		err := docker.CreateDockerTarContext(ctx, buildCtxWriter, workspace, &latest.DockerArtifact{
			BuildArgs:      artifact.BuildArgs,
			DockerfilePath: artifact.DockerfilePath,
		}, b.insecureRegistries)
		if err != nil {
			buildCtxWriter.CloseWithError(fmt.Errorf("creating docker context: %w", err))
			return
		}
		buildCtxWriter.Close()
	}()

	// Send context by piping into `tar`.
	// In case of an error, print the command's output. (The `err` itself is useless: exit status 1).
	var out bytes.Buffer
	if err := b.kubectlcli.Run(ctx, buildCtx, &out, "exec", "-i", podName, "-c", initContainer, "-n", b.Namespace, "--", "tar", "-xf", "-", "-C", constants.DefaultKanikoEmptyDirMountPath); err != nil {
		return fmt.Errorf("uploading build context: %s", out.String())
	}

	// Generate a file to successfully terminate the init container.
	if out, err := b.kubectlcli.RunOut(ctx, "exec", podName, "-c", initContainer, "-n", b.Namespace, "--", "touch", "/tmp/complete"); err != nil {
		return fmt.Errorf("finishing upload of the build context: %s", out)
	}

	return nil
}
<|endoftext|>


<commit_before>package in

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strconv"

	"terraform-resource/encoder"
	"terraform-resource/logger"
	"terraform-resource/models"
	"terraform-resource/storage"
	"terraform-resource/terraform"
)

type Runner struct {
	OutputDir string
	LogWriter io.Writer
}

type EnvNotFoundError error

var ErrOutputModule error = errors.New("the `output_module` feature was removed in Terraform 0.12.0, you must now explicitly declare all outputs in the root module")

func (r Runner) Run(req models.InRequest) (models.InResponse, error) {
	if err := req.Version.Validate(); err != nil {
		return models.InResponse{}, fmt.Errorf("Invalid Version request: %s", err)
	}

	envName := req.Version.EnvName
	nameFilepath := path.Join(r.OutputDir, "name")
	if err := ioutil.WriteFile(nameFilepath, []byte(envName), 0644); err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to create name file at path '%s': %s", nameFilepath, err)
	}

	if req.Params.Action == models.DestroyAction {
		resp := models.InResponse{
			Version: req.Version,
		}
		return resp, nil
	}

	if err := req.Source.Validate(); err != nil {
		return models.InResponse{}, err
	}

	tmpDir, err := ioutil.TempDir(os.TempDir(), "terraform-resource-in")
	if err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to create tmp dir at '%s'", os.TempDir())
	}
	defer os.RemoveAll(tmpDir)

	var resp models.InResponse
	if req.Source.BackendType != "" && req.Source.MigratedFromStorage != (storage.Model{}) {
		resp, err = r.inWithMigratedFromStorage(req, tmpDir)
	} else if req.Source.BackendType != "" {
		resp, err = r.inWithBackend(req, tmpDir)
	} else {
		resp, err = r.inWithLegacyStorage(req, tmpDir)
	}
	if err != nil {
		return models.InResponse{}, err
	}

	if err = r.writeNameToFile(req.Version.EnvName); err != nil {
		return models.InResponse{}, err
	}

	return resp, nil
}

func (r Runner) inWithMigratedFromStorage(req models.InRequest, tmpDir string) (models.InResponse, error) {
	resp, err := r.inWithBackend(req, tmpDir)
	if err == nil {
		return resp, nil
	}

	if _, ok := err.(EnvNotFoundError); ok {
		req.Source.Storage = req.Source.MigratedFromStorage
		return r.inWithLegacyStorage(req, tmpDir)
	}

	return models.InResponse{}, err
}

func (r Runner) inWithBackend(req models.InRequest, tmpDir string) (models.InResponse, error) {
	if req.Version.IsPlan() && req.Params.OutputJSONPlanfile == false {
		resp := models.InResponse{
			Version: req.Version,
		}
		return resp, nil
	}

	terraformModel := req.Source.Terraform
	if err := terraformModel.Validate(); err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to validate terraform Model: %s", err)
	}
	terraformModel.Source = "."
	if req.Params.OutputModule != "" {
		return models.InResponse{}, ErrOutputModule
	}

	targetEnvName := req.Version.EnvName

	client := terraform.NewClient(
		terraformModel,
		r.LogWriter,
	)

	if err := client.InitWithBackend(); err != nil {
		return models.InResponse{}, err
	}

	if err := r.ensureEnvExistsInBackend(targetEnvName, client); err != nil {
		return models.InResponse{}, err
	}

	tfOutput, err := client.Output(targetEnvName)
	if err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to parse terraform output.\nError: %s", err)
	}
	result := terraform.Result{
		Output: tfOutput,
	}

	if err = r.writeRawOutputToFile(result); err != nil {
		return models.InResponse{}, err
	}

	if req.Params.OutputStatefile {
		if err = r.writeBackendStateToFile(targetEnvName, client); err != nil {
			return models.InResponse{}, err
		}
	}
	stateVersion, err := client.CurrentStateVersion(targetEnvName)
	if err != nil {
		return models.InResponse{}, err
	}

	metadata, err := r.sanitizedOutput(result, client)
	if err != nil {
		return models.InResponse{}, err
	}

	if req.Params.OutputJSONPlanfile {
		if err = r.writeJSONPlanToFile(targetEnvName+"-plan", client); err != nil {
			return models.InResponse{}, err
		}
		metadata = nil
	}

	resp := models.InResponse{
		Version: models.Version{
			EnvName: targetEnvName,
			Serial:  strconv.Itoa(stateVersion.Serial),
			Lineage: stateVersion.Lineage,
		},
		Metadata: metadata,
	}
	return resp, nil
}

func (r Runner) ensureEnvExistsInBackend(envName string, client terraform.Client) error {
	spaces, err := client.WorkspaceList()
	if err != nil {
		return err
	}
	foundEnv := false
	for _, space := range spaces {
		if space == envName {
			foundEnv = true
		}
	}
	if !foundEnv {
		return EnvNotFoundError(fmt.Errorf(
			"Workspace '%s' does not exist in backend."+
				"\nIf you intended to run the `destroy` action, add `put.get_params.action: destroy`."+
				"\nThis is a temporary requirement until Concourse supports a `delete` step.",
			envName,
		))
	}

	return nil
}

func (r Runner) writeNameToFile(envName string) error {
	nameFilepath := path.Join(r.OutputDir, "name")
	return ioutil.WriteFile(nameFilepath, []byte(envName), 0644)
}

func (r Runner) writeRawOutputToFile(result terraform.Result) error {
	outputFilepath := path.Join(r.OutputDir, "metadata")
	outputFile, err := os.Create(outputFilepath)
	if err != nil {
		return fmt.Errorf("Failed to create output file at path '%s': %s", outputFilepath, err)
	}

	if err = encoder.NewJSONEncoder(outputFile).Encode(result.RawOutput()); err != nil {
		return fmt.Errorf("Failed to write output file: %s", err)
	}

	return nil
}

func (r Runner) writeBackendStateToFile(envName string, client terraform.Client) error {
	stateFilePath := path.Join(r.OutputDir, "terraform.tfstate")
	stateContents, err := client.StatePull(envName)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(stateFilePath, stateContents, 0777)
}

func (r Runner) writeJSONPlanToFile(envName string, client terraform.Client) error {

	stateContents, err := client.StatePull(envName)
	if err != nil {
		return err
	}

	tfOutput := models.TfState{}
	if err = json.Unmarshal(stateContents, &tfOutput); err != nil {
		return fmt.Errorf("Failed to unmarshal JSON output.\nError: %s\nOutput: %s", err, stateContents)
	}

	planFilePath := path.Join(r.OutputDir, "plan.json")

	var encodedPlan string
	if val, ok := tfOutput.Outputs[models.PlanContentJSON]; ok {
		encodedPlan = val["value"].(string)
	} else {
		return fmt.Errorf("state has no output for key %s", models.PlanContentJSON)
	}
	decodedPlan, err := base64.StdEncoding.DecodeString(encodedPlan)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(planFilePath, decodedPlan, 0644)
}

func (r Runner) writeLegacyStateToFile(localStatefilePath string) error {
	stateFilePath := path.Join(r.OutputDir, "terraform.tfstate")
	stateContents, err := ioutil.ReadFile(localStatefilePath)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(stateFilePath, stateContents, 0777)
}

func (r Runner) sanitizedOutput(result terraform.Result, client terraform.Client) ([]models.MetadataField, error) {
	metadata := []models.MetadataField{}
	for key, value := range result.SanitizedOutput() {
		metadata = append(metadata, models.MetadataField{
			Name:  key,
			Value: value,
		})
	}

	tfVersion, err := client.Version()
	if err != nil {
		return nil, err
	}
	return append(metadata, models.MetadataField{
		Name:  "terraform_version",
		Value: tfVersion,
	}), nil
}

func (r Runner) inWithLegacyStorage(req models.InRequest, tmpDir string) (models.InResponse, error) {
	logger := logger.Logger{
		Sink: r.LogWriter,
	}
	logger.Warn(fmt.Sprintf("%s\n", storage.DeprecationWarning))

	if req.Version.IsPlan() {
		resp := models.InResponse{
			Version: req.Version,
		}
		return resp, nil
	}

	stateFile, err := r.stateFileFromLegacyStorage(req, tmpDir)
	if err != nil {
		return models.InResponse{}, err
	}

	terraformModel := models.Terraform{
		StateFileLocalPath:  stateFile.LocalPath,
		StateFileRemotePath: stateFile.RemotePath,
	}

	if req.Params.OutputModule != "" {
		return models.InResponse{}, ErrOutputModule
	}

	if err := terraformModel.Validate(); err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to validate terraform Model: %s", err)
	}

	client := terraform.NewClient(
		terraformModel,
		r.LogWriter,
	)

	storageVersion, err := stateFile.Download()
	if err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to download state file from storage backend: %s", err)
	}
	version := models.NewVersionFromLegacyStorage(storageVersion)

	if err = client.InitWithoutBackend(); err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to initialize terraform.\nError: %s", err)
	}

	tfOutput, err := client.OutputWithLegacyStorage()
	if err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to parse terraform output.\nError: %s", err)
	}
	result := terraform.Result{
		Output: tfOutput,
	}

	if err = r.writeRawOutputToFile(result); err != nil {
		return models.InResponse{}, err
	}

	if req.Params.OutputStatefile {
		if err = r.writeLegacyStateToFile(terraformModel.StateFileLocalPath); err != nil {
			return models.InResponse{}, err
		}
	}

	metadata, err := r.sanitizedOutput(result, client)
	if err != nil {
		return models.InResponse{}, err
	}

	resp := models.InResponse{
		Version:  version,
		Metadata: metadata,
	}
	return resp, nil

}

func (r Runner) stateFileFromLegacyStorage(req models.InRequest, tmpDir string) (storage.StateFile, error) {
	storageModel := req.Source.Storage
	if err := storageModel.Validate(); err != nil {
		return storage.StateFile{}, fmt.Errorf("Failed to validate storage Model: %s", err)
	}
	storageDriver := storage.BuildDriver(storageModel)

	stateFile := storage.StateFile{
		LocalPath:     path.Join(tmpDir, "terraform.tfstate"),
		RemotePath:    fmt.Sprintf("%s.tfstate", req.Version.EnvName),
		StorageDriver: storageDriver,
	}

	existsAsTainted, err := stateFile.ExistsAsTainted()
	if err != nil {
		return storage.StateFile{}, fmt.Errorf("Failed to check for tainted state file: %s", err)
	}
	if existsAsTainted {
		stateFile = stateFile.ConvertToTainted()
	}

	exists, err := stateFile.Exists()
	if err != nil {
		return storage.StateFile{}, fmt.Errorf("Failed to check for existing state file: %s", err)
	}
	if !exists {
		return storage.StateFile{}, EnvNotFoundError(fmt.Errorf(
			"State file does not exist with key '%s'."+
				"\nIf you intended to run the `destroy` action, add `put.get_params.action: destroy`."+
				"\nThis is a temporary requirement until Concourse supports a `delete` step.",
			stateFile.RemotePath,
		))
	}
	return stateFile, nil
}
<commit_msg>moved planfile creation after init<commit_after>package in

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strconv"

	"terraform-resource/encoder"
	"terraform-resource/logger"
	"terraform-resource/models"
	"terraform-resource/storage"
	"terraform-resource/terraform"
)

type Runner struct {
	OutputDir string
	LogWriter io.Writer
}

type EnvNotFoundError error

var ErrOutputModule error = errors.New("the `output_module` feature was removed in Terraform 0.12.0, you must now explicitly declare all outputs in the root module")

func (r Runner) Run(req models.InRequest) (models.InResponse, error) {
	if err := req.Version.Validate(); err != nil {
		return models.InResponse{}, fmt.Errorf("Invalid Version request: %s", err)
	}

	envName := req.Version.EnvName
	nameFilepath := path.Join(r.OutputDir, "name")
	if err := ioutil.WriteFile(nameFilepath, []byte(envName), 0644); err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to create name file at path '%s': %s", nameFilepath, err)
	}

	if req.Params.Action == models.DestroyAction {
		resp := models.InResponse{
			Version: req.Version,
		}
		return resp, nil
	}

	if err := req.Source.Validate(); err != nil {
		return models.InResponse{}, err
	}

	tmpDir, err := ioutil.TempDir(os.TempDir(), "terraform-resource-in")
	if err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to create tmp dir at '%s'", os.TempDir())
	}
	defer os.RemoveAll(tmpDir)

	var resp models.InResponse
	if req.Source.BackendType != "" && req.Source.MigratedFromStorage != (storage.Model{}) {
		resp, err = r.inWithMigratedFromStorage(req, tmpDir)
	} else if req.Source.BackendType != "" {
		resp, err = r.inWithBackend(req, tmpDir)
	} else {
		resp, err = r.inWithLegacyStorage(req, tmpDir)
	}
	if err != nil {
		return models.InResponse{}, err
	}

	if err = r.writeNameToFile(req.Version.EnvName); err != nil {
		return models.InResponse{}, err
	}

	return resp, nil
}

func (r Runner) inWithMigratedFromStorage(req models.InRequest, tmpDir string) (models.InResponse, error) {
	resp, err := r.inWithBackend(req, tmpDir)
	if err == nil {
		return resp, nil
	}

	if _, ok := err.(EnvNotFoundError); ok {
		req.Source.Storage = req.Source.MigratedFromStorage
		return r.inWithLegacyStorage(req, tmpDir)
	}

	return models.InResponse{}, err
}

func (r Runner) inWithBackend(req models.InRequest, tmpDir string) (models.InResponse, error) {
	if req.Version.IsPlan() && req.Params.OutputJSONPlanfile == false {
		resp := models.InResponse{
			Version: req.Version,
		}
		return resp, nil
	}

	terraformModel := req.Source.Terraform
	if err := terraformModel.Validate(); err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to validate terraform Model: %s", err)
	}
	terraformModel.Source = "."
	if req.Params.OutputModule != "" {
		return models.InResponse{}, ErrOutputModule
	}

	targetEnvName := req.Version.EnvName

	client := terraform.NewClient(
		terraformModel,
		r.LogWriter,
	)

	if err := client.InitWithBackend(); err != nil {
		return models.InResponse{}, err
	}

	if req.Params.OutputJSONPlanfile {
		if err := r.writeJSONPlanToFile(targetEnvName+"-plan", client); err != nil {
			return models.InResponse{}, err
		}
		if req.Version.IsPlan() {
			resp := models.InResponse{
				Version: req.Version,
			}
			return resp, nil
		}
	}

	if err := r.ensureEnvExistsInBackend(targetEnvName, client); err != nil {
		return models.InResponse{}, err
	}

	tfOutput, err := client.Output(targetEnvName)
	if err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to parse terraform output.\nError: %s", err)
	}
	result := terraform.Result{
		Output: tfOutput,
	}

	if err = r.writeRawOutputToFile(result); err != nil {
		return models.InResponse{}, err
	}

	if req.Params.OutputStatefile {
		if err = r.writeBackendStateToFile(targetEnvName, client); err != nil {
			return models.InResponse{}, err
		}
	}
	stateVersion, err := client.CurrentStateVersion(targetEnvName)
	if err != nil {
		return models.InResponse{}, err
	}

	metadata, err := r.sanitizedOutput(result, client)
	if err != nil {
		return models.InResponse{}, err
	}

	resp := models.InResponse{
		Version: models.Version{
			EnvName: targetEnvName,
			Serial:  strconv.Itoa(stateVersion.Serial),
			Lineage: stateVersion.Lineage,
		},
		Metadata: metadata,
	}
	return resp, nil
}

func (r Runner) ensureEnvExistsInBackend(envName string, client terraform.Client) error {
	spaces, err := client.WorkspaceList()
	if err != nil {
		return err
	}
	foundEnv := false
	for _, space := range spaces {
		if space == envName {
			foundEnv = true
		}
	}
	if !foundEnv {
		return EnvNotFoundError(fmt.Errorf(
			"Workspace '%s' does not exist in backend."+
				"\nIf you intended to run the `destroy` action, add `put.get_params.action: destroy`."+
				"\nThis is a temporary requirement until Concourse supports a `delete` step.",
			envName,
		))
	}

	return nil
}

func (r Runner) writeNameToFile(envName string) error {
	nameFilepath := path.Join(r.OutputDir, "name")
	return ioutil.WriteFile(nameFilepath, []byte(envName), 0644)
}

func (r Runner) writeRawOutputToFile(result terraform.Result) error {
	outputFilepath := path.Join(r.OutputDir, "metadata")
	outputFile, err := os.Create(outputFilepath)
	if err != nil {
		return fmt.Errorf("Failed to create output file at path '%s': %s", outputFilepath, err)
	}

	if err = encoder.NewJSONEncoder(outputFile).Encode(result.RawOutput()); err != nil {
		return fmt.Errorf("Failed to write output file: %s", err)
	}

	return nil
}

func (r Runner) writeBackendStateToFile(envName string, client terraform.Client) error {
	stateFilePath := path.Join(r.OutputDir, "terraform.tfstate")
	stateContents, err := client.StatePull(envName)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(stateFilePath, stateContents, 0777)
}

func (r Runner) writeJSONPlanToFile(envName string, client terraform.Client) error {

	stateContents, err := client.StatePull(envName)
	if err != nil {
		return err
	}

	tfOutput := models.TfState{}
	if err = json.Unmarshal(stateContents, &tfOutput); err != nil {
		return fmt.Errorf("Failed to unmarshal JSON output.\nError: %s\nOutput: %s", err, stateContents)
	}

	planFilePath := path.Join(r.OutputDir, "plan.json")

	var encodedPlan string
	if val, ok := tfOutput.Outputs[models.PlanContentJSON]; ok {
		encodedPlan = val["value"].(string)
	} else {
		return fmt.Errorf("state has no output for key %s", models.PlanContentJSON)
	}
	decodedPlan, err := base64.StdEncoding.DecodeString(encodedPlan)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(planFilePath, decodedPlan, 0644)
}

func (r Runner) writeLegacyStateToFile(localStatefilePath string) error {
	stateFilePath := path.Join(r.OutputDir, "terraform.tfstate")
	stateContents, err := ioutil.ReadFile(localStatefilePath)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(stateFilePath, stateContents, 0777)
}

func (r Runner) sanitizedOutput(result terraform.Result, client terraform.Client) ([]models.MetadataField, error) {
	metadata := []models.MetadataField{}
	for key, value := range result.SanitizedOutput() {
		metadata = append(metadata, models.MetadataField{
			Name:  key,
			Value: value,
		})
	}

	tfVersion, err := client.Version()
	if err != nil {
		return nil, err
	}
	return append(metadata, models.MetadataField{
		Name:  "terraform_version",
		Value: tfVersion,
	}), nil
}

func (r Runner) inWithLegacyStorage(req models.InRequest, tmpDir string) (models.InResponse, error) {
	logger := logger.Logger{
		Sink: r.LogWriter,
	}
	logger.Warn(fmt.Sprintf("%s\n", storage.DeprecationWarning))

	if req.Version.IsPlan() {
		resp := models.InResponse{
			Version: req.Version,
		}
		return resp, nil
	}

	stateFile, err := r.stateFileFromLegacyStorage(req, tmpDir)
	if err != nil {
		return models.InResponse{}, err
	}

	terraformModel := models.Terraform{
		StateFileLocalPath:  stateFile.LocalPath,
		StateFileRemotePath: stateFile.RemotePath,
	}

	if req.Params.OutputModule != "" {
		return models.InResponse{}, ErrOutputModule
	}

	if err := terraformModel.Validate(); err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to validate terraform Model: %s", err)
	}

	client := terraform.NewClient(
		terraformModel,
		r.LogWriter,
	)

	storageVersion, err := stateFile.Download()
	if err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to download state file from storage backend: %s", err)
	}
	version := models.NewVersionFromLegacyStorage(storageVersion)

	if err = client.InitWithoutBackend(); err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to initialize terraform.\nError: %s", err)
	}

	tfOutput, err := client.OutputWithLegacyStorage()
	if err != nil {
		return models.InResponse{}, fmt.Errorf("Failed to parse terraform output.\nError: %s", err)
	}
	result := terraform.Result{
		Output: tfOutput,
	}

	if err = r.writeRawOutputToFile(result); err != nil {
		return models.InResponse{}, err
	}

	if req.Params.OutputStatefile {
		if err = r.writeLegacyStateToFile(terraformModel.StateFileLocalPath); err != nil {
			return models.InResponse{}, err
		}
	}

	metadata, err := r.sanitizedOutput(result, client)
	if err != nil {
		return models.InResponse{}, err
	}

	resp := models.InResponse{
		Version:  version,
		Metadata: metadata,
	}
	return resp, nil

}

func (r Runner) stateFileFromLegacyStorage(req models.InRequest, tmpDir string) (storage.StateFile, error) {
	storageModel := req.Source.Storage
	if err := storageModel.Validate(); err != nil {
		return storage.StateFile{}, fmt.Errorf("Failed to validate storage Model: %s", err)
	}
	storageDriver := storage.BuildDriver(storageModel)

	stateFile := storage.StateFile{
		LocalPath:     path.Join(tmpDir, "terraform.tfstate"),
		RemotePath:    fmt.Sprintf("%s.tfstate", req.Version.EnvName),
		StorageDriver: storageDriver,
	}

	existsAsTainted, err := stateFile.ExistsAsTainted()
	if err != nil {
		return storage.StateFile{}, fmt.Errorf("Failed to check for tainted state file: %s", err)
	}
	if existsAsTainted {
		stateFile = stateFile.ConvertToTainted()
	}

	exists, err := stateFile.Exists()
	if err != nil {
		return storage.StateFile{}, fmt.Errorf("Failed to check for existing state file: %s", err)
	}
	if !exists {
		return storage.StateFile{}, EnvNotFoundError(fmt.Errorf(
			"State file does not exist with key '%s'."+
				"\nIf you intended to run the `destroy` action, add `put.get_params.action: destroy`."+
				"\nThis is a temporary requirement until Concourse supports a `delete` step.",
			stateFile.RemotePath,
		))
	}
	return stateFile, nil
}
<|endoftext|>


<commit_before>package rwvfs

import (
	"bytes"
	"io"
	"os"
	pathpkg "path"
	"path/filepath"
	"strings"

	"golang.org/x/tools/godoc/vfs"
	"golang.org/x/tools/godoc/vfs/mapfs"
)

// Map returns a new FileSystem from the provided map. Map keys should be
// forward slash-separated pathnames and not contain a leading slash.
func Map(m map[string]string) FileSystem {
	fs := mapFS{
		m:          m,
		dirs:       map[string]struct{}{"/": struct{}{}},
		FileSystem: mapfs.New(m),
	}

	// Create initial dirs.
	for path := range m {
		if err := MkdirAll(fs, filepath.Dir(path)); err != nil {
			panic(err.Error())
		}
	}

	return fs
}

type mapFS struct {
	m    map[string]string
	dirs map[string]struct{}
	vfs.FileSystem
}

func (mfs mapFS) Open(path string) (vfs.ReadSeekCloser, error) {
	path = slash(path)
	f, err := mfs.FileSystem.Open(path)
	if err != nil {
		return nil, &os.PathError{Op: "open", Path: path, Err: err}
	}
	return f, nil
}

func (mfs mapFS) Create(path string) (io.WriteCloser, error) {
	// Mimic behavior of OS filesystem: truncate to empty string upon creation;
	// immediately update string values with writes.
	path = slash(path)
	mfs.m[noslash(path)] = ""
	return &mapFile{m: mfs.m, path: noslash(path)}, nil
}

func noslash(p string) string {
	p = slash(p)
	if p == "/" {
		return "."
	}
	return strings.TrimPrefix(p, "/")
}

// slashdir returns path.Dir(p), but special-cases paths not beginning
// with a slash to be in the root.
func slashdir(p string) string {
	d := pathpkg.Dir(p)
	if d == "." {
		return "/"
	}
	if strings.HasPrefix(p, "/") {
		return d
	}
	return "/" + d
}

func slash(p string) string {
	if p == "." {
		return "/"
	}
	return "/" + strings.TrimPrefix(p, "/")
}

type mapFile struct {
	buf  bytes.Buffer
	m    map[string]string
	path string
}

func (f *mapFile) Write(p []byte) (int, error) {
	return f.buf.Write(p)
}

func (f *mapFile) Close() error {
	if f.m == nil {
		// duplicate closes are noop
		return nil
	}
	f.m[f.path] = f.buf.String()
	f.buf.Reset()
	f.m = nil
	return nil
}

func (mfs mapFS) lstat(p string) (os.FileInfo, error) {
	// proxy mapfs.mapFS.Lstat to not return errors for empty directories
	// created with Mkdir
	p = slash(p)
	fi, err := mfs.FileSystem.Lstat(p)
	if os.IsNotExist(err) {
		_, ok := mfs.dirs[p]
		if ok {
			return fileInfo{name: pathpkg.Base(p), dir: true}, nil
		}
	}
	return fi, err
}

func (mfs mapFS) Lstat(p string) (os.FileInfo, error) {
	fi, err := mfs.lstat(p)
	if err != nil {
		err = &os.PathError{Op: "lstat", Path: p, Err: err}
	}
	return fi, err
}

func (mfs mapFS) Stat(p string) (os.FileInfo, error) {
	fi, err := mfs.lstat(p)
	if err != nil {
		err = &os.PathError{Op: "stat", Path: p, Err: err}
	}
	return fi, err
}

func (mfs mapFS) ReadDir(p string) ([]os.FileInfo, error) {
	// proxy mapfs.mapFS.ReadDir to not return errors for empty directories
	// created with Mkdir
	p = slash(p)
	fis, err := mfs.FileSystem.ReadDir(p)
	if os.IsNotExist(err) {
		_, ok := mfs.dirs[p]
		if ok {
			// return a list of subdirs and files (the underlying ReadDir impl
			// fails here because it thinks the directories don't exist).
			fis = nil
			for dir, _ := range mfs.dirs {
				if (p != "/" && filepath.Dir(dir) == p) || (p == "/" && filepath.Dir(dir) == "." && dir != "." && dir != "") {
					fis = append(fis, newDirInfo(dir))
				}
			}
			for fn, b := range mfs.m {
				if slashdir(fn) == "/"+p {
					fis = append(fis, newFileInfo(fn, b))
				}
			}
			return fis, nil
		}
	}
	return fis, err
}

func fileInfoNames(fis []os.FileInfo) []string {
	names := make([]string, len(fis))
	for i, fi := range fis {
		names[i] = fi.Name()
	}
	return names
}

func (mfs mapFS) Mkdir(name string) error {
	name = slash(name)
	if slashdir(name) != slash(name) { // don't check for root dir's parent
		if _, err := mfs.Stat(slashdir(name)); err != nil {
			if osErr, ok := err.(*os.PathError); ok && osErr != nil {
				osErr.Op = "mkdir"
				osErr.Path = name
			}
			return err
		}
	}
	fi, _ := mfs.Stat(name)
	if fi != nil {
		return &os.PathError{Op: "mkdir", Path: name, Err: os.ErrExist}
	}
	mfs.dirs[slash(name)] = struct{}{}
	return nil
}

func (mfs mapFS) Remove(name string) error {
	name = slash(name)
	delete(mfs.dirs, name)
	delete(mfs.m, noslash(name))
	return nil
}
<commit_msg>synchronize mapFS<commit_after>package rwvfs

import (
	"bytes"
	"io"
	"os"
	pathpkg "path"
	"path/filepath"
	"strings"
	"sync"

	"golang.org/x/tools/godoc/vfs"
	"golang.org/x/tools/godoc/vfs/mapfs"
)

// Map returns a new FileSystem from the provided map. Map keys should be
// forward slash-separated pathnames and not contain a leading slash.
func Map(m map[string]string) FileSystem {
	fs := mapFS{
		m:          m,
		dirs:       map[string]struct{}{"/": struct{}{}},
		mu:         new(sync.RWMutex),
		FileSystem: mapfs.New(m),
	}

	// Create initial dirs.
	for path := range m {
		if err := MkdirAll(fs, filepath.Dir(path)); err != nil {
			panic(err.Error())
		}
	}

	return fs
}

type mapFS struct {
	m    map[string]string
	dirs map[string]struct{}
	mu   *sync.RWMutex
	vfs.FileSystem
}

func (mfs mapFS) Open(path string) (vfs.ReadSeekCloser, error) {
	mfs.mu.RLock()
	defer mfs.mu.RUnlock()

	path = slash(path)
	f, err := mfs.FileSystem.Open(path)
	if err != nil {
		return nil, &os.PathError{Op: "open", Path: path, Err: err}
	}
	return f, nil
}

func (mfs mapFS) Create(path string) (io.WriteCloser, error) {
	mfs.mu.Lock()
	defer mfs.mu.Unlock()

	// Mimic behavior of OS filesystem: truncate to empty string upon creation;
	// immediately update string values with writes.
	path = slash(path)
	mfs.m[noslash(path)] = ""
	return &mapFile{m: mfs.m, path: noslash(path), fsMu: mfs.mu}, nil
}

func noslash(p string) string {
	p = slash(p)
	if p == "/" {
		return "."
	}
	return strings.TrimPrefix(p, "/")
}

// slashdir returns path.Dir(p), but special-cases paths not beginning
// with a slash to be in the root.
func slashdir(p string) string {
	d := pathpkg.Dir(p)
	if d == "." {
		return "/"
	}
	if strings.HasPrefix(p, "/") {
		return d
	}
	return "/" + d
}

func slash(p string) string {
	if p == "." {
		return "/"
	}
	return "/" + strings.TrimPrefix(p, "/")
}

type mapFile struct {
	buf  bytes.Buffer
	m    map[string]string
	path string
	fsMu *sync.RWMutex
}

func (f *mapFile) Write(p []byte) (int, error) {
	return f.buf.Write(p)
}

func (f *mapFile) Close() error {
	if f.m == nil {
		// duplicate closes are noop
		return nil
	}

	f.fsMu.Lock()
	defer f.fsMu.Unlock()

	f.m[f.path] = f.buf.String()
	f.buf.Reset()
	f.m = nil
	return nil
}

func (mfs mapFS) lstat(p string) (os.FileInfo, error) {
	mfs.mu.RLock()
	defer mfs.mu.RUnlock()

	// proxy mapfs.mapFS.Lstat to not return errors for empty directories
	// created with Mkdir
	p = slash(p)
	fi, err := mfs.FileSystem.Lstat(p)
	if os.IsNotExist(err) {
		_, ok := mfs.dirs[p]
		if ok {
			return fileInfo{name: pathpkg.Base(p), dir: true}, nil
		}
	}
	return fi, err
}

func (mfs mapFS) Lstat(p string) (os.FileInfo, error) {
	fi, err := mfs.lstat(p)
	if err != nil {
		err = &os.PathError{Op: "lstat", Path: p, Err: err}
	}
	return fi, err
}

func (mfs mapFS) Stat(p string) (os.FileInfo, error) {
	fi, err := mfs.lstat(p)
	if err != nil {
		err = &os.PathError{Op: "stat", Path: p, Err: err}
	}
	return fi, err
}

func (mfs mapFS) ReadDir(p string) ([]os.FileInfo, error) {
	mfs.mu.RLock()
	defer mfs.mu.RUnlock()

	// proxy mapfs.mapFS.ReadDir to not return errors for empty directories
	// created with Mkdir
	p = slash(p)
	fis, err := mfs.FileSystem.ReadDir(p)
	if os.IsNotExist(err) {
		_, ok := mfs.dirs[p]
		if ok {
			// return a list of subdirs and files (the underlying ReadDir impl
			// fails here because it thinks the directories don't exist).
			fis = nil
			for dir, _ := range mfs.dirs {
				if (p != "/" && filepath.Dir(dir) == p) || (p == "/" && filepath.Dir(dir) == "." && dir != "." && dir != "") {
					fis = append(fis, newDirInfo(dir))
				}
			}
			for fn, b := range mfs.m {
				if slashdir(fn) == "/"+p {
					fis = append(fis, newFileInfo(fn, b))
				}
			}
			return fis, nil
		}
	}
	return fis, err
}

func fileInfoNames(fis []os.FileInfo) []string {
	names := make([]string, len(fis))
	for i, fi := range fis {
		names[i] = fi.Name()
	}
	return names
}

func (mfs mapFS) Mkdir(name string) error {
	name = slash(name)
	if slashdir(name) != slash(name) { // don't check for root dir's parent
		if _, err := mfs.Stat(slashdir(name)); err != nil {
			if osErr, ok := err.(*os.PathError); ok && osErr != nil {
				osErr.Op = "mkdir"
				osErr.Path = name
			}
			return err
		}
	}
	fi, _ := mfs.Stat(name)
	if fi != nil {
		return &os.PathError{Op: "mkdir", Path: name, Err: os.ErrExist}
	}

	mfs.mu.Lock()
	defer mfs.mu.Unlock()

	mfs.dirs[slash(name)] = struct{}{}
	return nil
}

func (mfs mapFS) Remove(name string) error {
	mfs.mu.Lock()
	defer mfs.mu.Unlock()

	name = slash(name)
	delete(mfs.dirs, name)
	delete(mfs.m, noslash(name))
	return nil
}
<|endoftext|>