<commit_before>package grunway

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/amattn/deeperror"
)

type Context struct {
	Req *http.Request // original http request
	End Endpoint      // parsed endpoint information

	// only populated after a call to ctx.RequestBody()
	cachedRequestBody      []byte
	cachedRequestBodyError error

	// exposing the responseWriter tends to induce bugs.  We keep this internal for now.
	w http.ResponseWriter

	// router
	router *Router

	// only populated after auth
	PublicKey string // for Auth'd requests, will be set to public key if Auth was successful, "" otherwise

	// generic maps for middleware to stuff arbitrary data
	middleware map[string]interface{}
	postware   map[string]interface{}

	// only populated after a write

	written       bool // true after a write, false before.  Used to prevent "double writes".
	StatusCode    int  // The http status code written out. Only populated after a write.
	ContentLength int  // The number of bytes written out. Only populated after a write.
}

func (ctx *Context) AddHeader(key, value string) {
	ctx.w.Header().Add(key, value)
}
func (ctx *Context) DelHeader(key string) {
	ctx.w.Header().Del(key)
}
func (ctx *Context) GetHeader(key string) string {
	return ctx.w.Header().Get(key)
}
func (ctx *Context) SetHeader(key, value string) {
	ctx.w.Header().Set(key, value)
}

func (ctx *Context) RequestBody() ([]byte, error) {
	if ctx.cachedRequestBody != nil {
		return ctx.cachedRequestBody, nil
	}
	if ctx.cachedRequestBodyError != nil {
		return nil, ctx.cachedRequestBodyError
	}

	ctx.cachedRequestBody, ctx.cachedRequestBodyError = ioutil.ReadAll(ctx.Req.Body)
	return ctx.cachedRequestBody, ctx.cachedRequestBodyError
}

//  #####
// #     # #####   ##   #    # #####    ##   #####  #####
// #         #    #  #  ##   # #    #  #  #  #    # #    #
//  #####    #   #    # # #  # #    # #    # #    # #    #
//       #   #   ###### #  # # #    # ###### #####  #    #
// #     #   #   #    # #   ## #    # #    # #   #  #    #
//  #####    #   #    # #    # #####  #    # #    # #####
//
// ######
// #     # ######  ####  #    # #      #####  ####
// #     # #      #      #    # #        #   #
// ######  #####   ####  #    # #        #    ####
// #   #   #           # #    # #        #        #
// #    #  #      #    # #    # #        #   #    #
// #     # ######  ####   ####  ######   #    ####
//

func (ctx *Context) MakeRouteHandlerResultError(code int, errNo int64, errMsg string) RouteHandlerResult {
	errInfo := ErrorInfo{
		ErrorNumber:  errNo,
		ErrorMessage: errMsg,
	}
	return ctx.MakeRouteHandlerResultErrorInfo(code, errInfo)
}
func (ctx *Context) MakeRouteHandlerResultDebugError(code int, errNo int64, errMsg string, debugNo int64, debugMsg string) RouteHandlerResult {
	errInfo := ErrorInfo{
		ErrorNumber:  errNo,
		ErrorMessage: errMsg,
		DebugNumber:  debugNo,
		DebugMessage: debugMsg,
	}
	return ctx.MakeRouteHandlerResultErrorInfo(code, errInfo)
}
func (ctx *Context) MakeRouteHandlerResultErrorInfo(code int, errInfo ErrorInfo) RouteHandlerResult {
	rerr := NewRouteError(code, errInfo)
	return RouteHandlerResult{rerr, nil, nil}
}

func (ctx *Context) MakeRouteHandlerResultAlert(code int, errNo int64, alert string) RouteHandlerResult {
	return ctx.MakeRouteHandlerResultCustom(func(innerCtx *Context) {
		sendErrorPayload(innerCtx, code, ErrorInfo{ErrorNumber: errNo}, alert)
	})
}
func (ctx *Context) MakeRouteHandlerResultPayloads(payloads ...Payload) RouteHandlerResult {
	return RouteHandlerResult{nil, MakePayloadMapFromPayloads(payloads...), nil}
}
func (ctx *Context) MakeRouteHandlerResultGenericJSON(v interface{}) RouteHandlerResult {
	return ctx.MakeRouteHandlerResultStatusGenericJSON(http.StatusOK, v)
}
func (ctx *Context) MakeRouteHandlerResultStatusGenericJSON(statusCode int, v interface{}) RouteHandlerResult {
	jsonBytes, err := json.Marshal(v)
	if err != nil {
		rerr := NewRouteError(
			http.StatusInternalServerError,
			ErrorInfo{
				ErrorNumber:  3913952842,
				ErrorMessage: "Internal Server Error",
			})
		return RouteHandlerResult{rerr, nil, nil}
	} else {
		return RouteHandlerResult{nil, nil, func(innerCtx *Context) {
			if rw, isResponseWriter := innerCtx.w.(http.ResponseWriter); isResponseWriter {
				rw.WriteHeader(statusCode)
				if len(jsonBytes) == 0 {
					log.Println("jsonBytes", jsonBytes, innerCtx.Req.URL)
				}
				bytesWritten, err := rw.Write(jsonBytes)
				if err != nil {
					log.Println("3952513088 WRITE ERROR", err)
				}
				innerCtx.ContentLength = bytesWritten
			}
		}}
	}
}
func (ctx *Context) MakeRouteHandlerResultCustom(crr CustomRouteResponse) RouteHandlerResult {
	return RouteHandlerResult{nil, nil, crr}
}
func (ctx *Context) MakeRouteHandlerResultOk() RouteHandlerResult {
	return ctx.MakeRouteHandlerResultCustom(func(innerCtx *Context) {
		sendOkPayload(innerCtx)
	})
}
func (ctx *Context) MakeRouteHandlerResultNotFound(errNo int64) RouteHandlerResult {
	return ctx.MakeRouteHandlerResultCustom(func(innerCtx *Context) {
		sendNotFoundPayload(innerCtx, errNo)
	})
}

//  #####
// #     # #    #  ####  #####  ####  #    #
// #       #    # #        #   #    # ##  ##
// #       #    #  ####    #   #    # # ## #
// #       #    #      #   #   #    # #    #
// #     # #    # #    #   #   #    # #    #
//  #####   ####   ####    #    ####  #    #
//
// ######
// #     # ######  ####  #    # #      #####  ####
// #     # #      #      #    # #        #   #
// ######  #####   ####  #    # #        #    ####
// #   #   #           # #    # #        #        #
// #    #  #      #    # #    # #        #   #    #
// #     # ######  ####   ####  ######   #    ####
//

func (ctx *Context) WrapAndSendPayload(payload Payload) {
	if payload == nil {
		ctx.SendSimpleErrorPayload(http.StatusInternalServerError, 388359273, "")
		return
	}
	wrapAndSendPayload(ctx, payload)
}

// for a slice of Entities
func (ctx *Context) WrapAndSendPayloadsList(payloadList []Payload) {
	wrapAndSendPayloadsList(ctx, payloadList)
}

// for a map of slices of Entities
func (ctx *Context) WrapAndSendPayloadsMap(payloads PayloadsMap) {
	wrapAndSendPayloadsMap(ctx, payloads)
}

// Error
func (ctx *Context) SendSimpleErrorPayload(code int, errNo int64, errorMsg string) {
	ctx.SendErrorInfoPayload(code, ErrorInfo{
		ErrorNumber:  errNo,
		ErrorMessage: errorMsg,
	})
}
func (ctx *Context) SendErrorInfoPayload(code int, errInfo ErrorInfo) {
	enduserErrMsg := errInfo.ErrorMessage
	if len(enduserErrMsg) == 0 {
		enduserErrMsg = http.StatusText(code)
	}
	sendErrorPayload(ctx, code, errInfo, "")
}

// Alert payloads are designed as a general notification service for clients (ie client must upgrade, server is in maint mode, etc.)
func (ctx *Context) SendSimpleAlertPayload(code int, errNo int64, errMsg, alert string) {
	sendErrorPayload(ctx, code, ErrorInfo{ErrorNumber: errNo, ErrorMessage: errMsg}, alert)
}

func (ctx *Context) DecodeResponseBodyOrSendError(pc PayloadController, payloadReference interface{}) interface{} {
	requestBody := ctx.Req.Body
	if requestBody == nil {
		errMsg := BadRequestPrefix + ": Expected non-empty body"
		ctx.SendSimpleErrorPayload(http.StatusBadRequest, 3003399819, errMsg)
		return nil
	}
	defer requestBody.Close()

	decoder := json.NewDecoder(requestBody)
	err := decoder.Decode(payloadReference)

	if err != nil {
		errMsg := BadRequestPrefix + ": Cannot parse body"
		derr := deeperror.New(3005488054, errMsg, err)
		log.Println("derr", derr)
		ctx.SendSimpleErrorPayload(http.StatusBadRequest, derr.Num, errMsg)
		return nil
	}

	return payloadReference
}
<commit_msg>set proper content-type application/json for generic json handler<commit_after>package grunway

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"

	"github.com/amattn/deeperror"
)

type Context struct {
	Req *http.Request // original http request
	End Endpoint      // parsed endpoint information

	// only populated after a call to ctx.RequestBody()
	cachedRequestBody      []byte
	cachedRequestBodyError error

	// exposing the responseWriter tends to induce bugs.  We keep this internal for now.
	w http.ResponseWriter

	// router
	router *Router

	// only populated after auth
	PublicKey string // for Auth'd requests, will be set to public key if Auth was successful, "" otherwise

	// generic maps for middleware to stuff arbitrary data
	middleware map[string]interface{}
	postware   map[string]interface{}

	// only populated after a write

	written       bool // true after a write, false before.  Used to prevent "double writes".
	StatusCode    int  // The http status code written out. Only populated after a write.
	ContentLength int  // The number of bytes written out. Only populated after a write.
}

func (ctx *Context) AddHeader(key, value string) {
	ctx.w.Header().Add(key, value)
}
func (ctx *Context) DelHeader(key string) {
	ctx.w.Header().Del(key)
}
func (ctx *Context) GetHeader(key string) string {
	return ctx.w.Header().Get(key)
}
func (ctx *Context) SetHeader(key, value string) {
	ctx.w.Header().Set(key, value)
}

func (ctx *Context) RequestBody() ([]byte, error) {
	if ctx.cachedRequestBody != nil {
		return ctx.cachedRequestBody, nil
	}
	if ctx.cachedRequestBodyError != nil {
		return nil, ctx.cachedRequestBodyError
	}

	ctx.cachedRequestBody, ctx.cachedRequestBodyError = ioutil.ReadAll(ctx.Req.Body)
	return ctx.cachedRequestBody, ctx.cachedRequestBodyError
}

//  #####
// #     # #####   ##   #    # #####    ##   #####  #####
// #         #    #  #  ##   # #    #  #  #  #    # #    #
//  #####    #   #    # # #  # #    # #    # #    # #    #
//       #   #   ###### #  # # #    # ###### #####  #    #
// #     #   #   #    # #   ## #    # #    # #   #  #    #
//  #####    #   #    # #    # #####  #    # #    # #####
//
// ######
// #     # ######  ####  #    # #      #####  ####
// #     # #      #      #    # #        #   #
// ######  #####   ####  #    # #        #    ####
// #   #   #           # #    # #        #        #
// #    #  #      #    # #    # #        #   #    #
// #     # ######  ####   ####  ######   #    ####
//

func (ctx *Context) MakeRouteHandlerResultError(code int, errNo int64, errMsg string) RouteHandlerResult {
	errInfo := ErrorInfo{
		ErrorNumber:  errNo,
		ErrorMessage: errMsg,
	}
	return ctx.MakeRouteHandlerResultErrorInfo(code, errInfo)
}
func (ctx *Context) MakeRouteHandlerResultDebugError(code int, errNo int64, errMsg string, debugNo int64, debugMsg string) RouteHandlerResult {
	errInfo := ErrorInfo{
		ErrorNumber:  errNo,
		ErrorMessage: errMsg,
		DebugNumber:  debugNo,
		DebugMessage: debugMsg,
	}
	return ctx.MakeRouteHandlerResultErrorInfo(code, errInfo)
}
func (ctx *Context) MakeRouteHandlerResultErrorInfo(code int, errInfo ErrorInfo) RouteHandlerResult {
	rerr := NewRouteError(code, errInfo)
	return RouteHandlerResult{rerr, nil, nil}
}

func (ctx *Context) MakeRouteHandlerResultAlert(code int, errNo int64, alert string) RouteHandlerResult {
	return ctx.MakeRouteHandlerResultCustom(func(innerCtx *Context) {
		sendErrorPayload(innerCtx, code, ErrorInfo{ErrorNumber: errNo}, alert)
	})
}
func (ctx *Context) MakeRouteHandlerResultPayloads(payloads ...Payload) RouteHandlerResult {
	return RouteHandlerResult{nil, MakePayloadMapFromPayloads(payloads...), nil}
}
func (ctx *Context) MakeRouteHandlerResultGenericJSON(v interface{}) RouteHandlerResult {
	return ctx.MakeRouteHandlerResultStatusGenericJSON(http.StatusOK, v)
}
func (ctx *Context) MakeRouteHandlerResultStatusGenericJSON(statusCode int, v interface{}) RouteHandlerResult {
	ctx.SetHeader(httpHeaderContentType, httpHeaderContentTypeJSON)
	jsonBytes, err := json.Marshal(v)
	if err != nil {
		rerr := NewRouteError(
			http.StatusInternalServerError,
			ErrorInfo{
				ErrorNumber:  3913952842,
				ErrorMessage: "Internal Server Error",
			})
		return RouteHandlerResult{rerr, nil, nil}
	} else {
		return RouteHandlerResult{nil, nil, func(innerCtx *Context) {
			if rw, isResponseWriter := innerCtx.w.(http.ResponseWriter); isResponseWriter {
				rw.WriteHeader(statusCode)
				if len(jsonBytes) == 0 {
					log.Println("jsonBytes", jsonBytes, innerCtx.Req.URL)
				}
				bytesWritten, err := rw.Write(jsonBytes)
				if err != nil {
					log.Println("3952513088 WRITE ERROR", err)
				}
				innerCtx.ContentLength = bytesWritten
			}
		}}
	}
}
func (ctx *Context) MakeRouteHandlerResultCustom(crr CustomRouteResponse) RouteHandlerResult {
	return RouteHandlerResult{nil, nil, crr}
}
func (ctx *Context) MakeRouteHandlerResultOk() RouteHandlerResult {
	return ctx.MakeRouteHandlerResultCustom(func(innerCtx *Context) {
		sendOkPayload(innerCtx)
	})
}
func (ctx *Context) MakeRouteHandlerResultNotFound(errNo int64) RouteHandlerResult {
	return ctx.MakeRouteHandlerResultCustom(func(innerCtx *Context) {
		sendNotFoundPayload(innerCtx, errNo)
	})
}

//  #####
// #     # #    #  ####  #####  ####  #    #
// #       #    # #        #   #    # ##  ##
// #       #    #  ####    #   #    # # ## #
// #       #    #      #   #   #    # #    #
// #     # #    # #    #   #   #    # #    #
//  #####   ####   ####    #    ####  #    #
//
// ######
// #     # ######  ####  #    # #      #####  ####
// #     # #      #      #    # #        #   #
// ######  #####   ####  #    # #        #    ####
// #   #   #           # #    # #        #        #
// #    #  #      #    # #    # #        #   #    #
// #     # ######  ####   ####  ######   #    ####
//

func (ctx *Context) WrapAndSendPayload(payload Payload) {
	if payload == nil {
		ctx.SendSimpleErrorPayload(http.StatusInternalServerError, 388359273, "")
		return
	}
	wrapAndSendPayload(ctx, payload)
}

// for a slice of Entities
func (ctx *Context) WrapAndSendPayloadsList(payloadList []Payload) {
	wrapAndSendPayloadsList(ctx, payloadList)
}

// for a map of slices of Entities
func (ctx *Context) WrapAndSendPayloadsMap(payloads PayloadsMap) {
	wrapAndSendPayloadsMap(ctx, payloads)
}

// Error
func (ctx *Context) SendSimpleErrorPayload(code int, errNo int64, errorMsg string) {
	ctx.SendErrorInfoPayload(code, ErrorInfo{
		ErrorNumber:  errNo,
		ErrorMessage: errorMsg,
	})
}
func (ctx *Context) SendErrorInfoPayload(code int, errInfo ErrorInfo) {
	enduserErrMsg := errInfo.ErrorMessage
	if len(enduserErrMsg) == 0 {
		enduserErrMsg = http.StatusText(code)
	}
	sendErrorPayload(ctx, code, errInfo, "")
}

// Alert payloads are designed as a general notification service for clients (ie client must upgrade, server is in maint mode, etc.)
func (ctx *Context) SendSimpleAlertPayload(code int, errNo int64, errMsg, alert string) {
	sendErrorPayload(ctx, code, ErrorInfo{ErrorNumber: errNo, ErrorMessage: errMsg}, alert)
}

func (ctx *Context) DecodeResponseBodyOrSendError(pc PayloadController, payloadReference interface{}) interface{} {
	requestBody := ctx.Req.Body
	if requestBody == nil {
		errMsg := BadRequestPrefix + ": Expected non-empty body"
		ctx.SendSimpleErrorPayload(http.StatusBadRequest, 3003399819, errMsg)
		return nil
	}
	defer requestBody.Close()

	decoder := json.NewDecoder(requestBody)
	err := decoder.Decode(payloadReference)

	if err != nil {
		errMsg := BadRequestPrefix + ": Cannot parse body"
		derr := deeperror.New(3005488054, errMsg, err)
		log.Println("derr", derr)
		ctx.SendSimpleErrorPayload(http.StatusBadRequest, derr.Num, errMsg)
		return nil
	}

	return payloadReference
}
<|endoftext|>


<commit_before>package endly

import (
	"fmt"
	uuid "github.com/satori/go.uuid"
	"github.com/viant/endly/msg"
	"github.com/viant/neatly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/data"
	"github.com/viant/toolbox/secret"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

var serviceManagerKey = (*manager)(nil)
var deferFunctionsKey = (*[]func())(nil)

//Context represents a workflow session context/state
type Context struct {
	SessionID       string
	CLIEnabled      bool
	HasLogger       bool
	AsyncUnsafeKeys map[interface{}]bool
	Secrets         *secret.Service
	Wait            *sync.WaitGroup
	Listener        msg.Listener
	Source          *url.Resource
	state           data.Map
	toolbox.Context
	cloned []*Context
	closed int32
}

//Publish publishes event to listeners, it updates current run details like activity workflow name etc ...
func (c *Context) Publish(value interface{}) msg.Event {
	event, ok := value.(msg.Event)
	if !ok {
		event = msg.NewEvent(value)
	}
	if c.Listener != nil {
		c.Listener(event)
	}
	return event
}

//PublishWithStartEvent publishes event to listeners, it updates current run details like activity workflow name etc ...
func (c *Context) PublishWithStartEvent(value interface{}, init msg.Event) msg.Event {
	event := msg.NewEventWithInit(value, init)
	if c.Listener != nil {
		c.Listener(event)
	}
	return event
}

//SetListener sets context event Listener
func (c *Context) SetListener(listener msg.Listener) {
	c.Listener = listener
}

//IsClosed returns true if it is closed.
func (c *Context) IsClosed() bool {
	return atomic.LoadInt32(&c.closed) == 1
}

//Clone clones the context.
func (c *Context) Clone() *Context {
	if len(c.cloned) == 0 {
		c.cloned = make([]*Context, 0)
	}
	result := &Context{}
	result.Wait = &sync.WaitGroup{}
	result.Context = c.Context.Clone()
	result.state = NewDefaultState()
	result.state.Apply(c.state)
	result.SessionID = c.SessionID
	result.Listener = c.Listener
	result.CLIEnabled = c.CLIEnabled
	result.Secrets = c.Secrets
	result.AsyncUnsafeKeys = make(map[interface{}]bool)
	c.cloned = append(c.cloned, result)
	return result
}

func (c *Context) parentURLCandidates() []string {
	var result = make([]string, 0)
	if c.Source != nil {
		baseURL, _ := toolbox.URLSplit(c.Source.URL)
		result = append(result, baseURL)
	}
	currentDirectory, err := os.Getwd()
	if err == nil {
		result = append(result, toolbox.FileSchema+currentDirectory)
	}

	return result
}

//ExpandResource substitutes any $ expression with the key value from the state map if it is present.
func (c *Context) ExpandResource(resource *url.Resource) (*url.Resource, error) {
	if resource == nil {
		return nil, msg.ReportError(fmt.Errorf("resource  was empty"))
	}
	if resource.URL == "" {
		return nil, msg.ReportError(fmt.Errorf("url was empty"))
	}

	if !strings.Contains(resource.URL, "://") {
		for _, parentCandidate := range c.parentURLCandidates() {
			service, err := storage.NewServiceForURL(parentCandidate, "")
			if err != nil {
				continue
			}
			var candidateURL = toolbox.URLPathJoin(parentCandidate, resource.URL)
			if exists, err := service.Exists(candidateURL); exists && err == nil {
				resource.URL = candidateURL
			}
		}
	}
	var result = url.NewResource(c.Expand(resource.URL), c.Expand(resource.Credentials))
	if result.ParsedURL == nil {
		return nil, fmt.Errorf("failed to parse URL %v", result.URL)
	}
	result.Cache = c.Expand(resource.Cache)
	result.CacheExpiryMs = resource.CacheExpiryMs
	return result, nil
}

//Manager returns workflow manager or error
func (c *Context) Manager() (Manager, error) {
	if c == nil {
		return nil, fmt.Errorf("context was nil")
	}
	var manager = &manager{}
	if !c.GetInto(serviceManagerKey, &manager) {
		return nil, msg.ReportError(fmt.Errorf("failed to lookup Manager"))
	}
	return manager, nil
}

//Service returns a service fo provided id or error.
func (c *Context) Service(name string) (Service, error) {
	manager, err := c.Manager()
	if err != nil {
		return nil, err
	}
	return manager.Service(name)
}

//Deffer add function to be executed if context closes. If returns currently registered functions.
func (c *Context) Deffer(functions ...func()) []func() {
	var result *[]func()
	if !c.Contains(deferFunctionsKey) {
		var functions = make([]func(), 0)
		result = &functions
		_ = c.Put(deferFunctionsKey, result)
	} else {
		c.GetInto(deferFunctionsKey, &result)
	}

	*result = append(*result, functions...)
	_ = c.Put(deferFunctionsKey, &result)
	return *result
}

//State returns a context state map.
func (c *Context) State() data.Map {
	if c.state == nil {
		c.state = NewDefaultState()
	}
	return c.state
}

//SetState sets a new state map
func (c *Context) SetState(state data.Map) {
	c.state = state
}

//Expand substitute $ expression if present in the text and state map.
func (c *Context) Expand(text string) string {
	state := c.State()
	return state.ExpandAsText(text)
}

//PublishAndRestore sets supplied value and returns func restoring original values
func (s *Context) PublishAndRestore(values map[string]interface{}) func() {
	var backup = map[string]interface{}{}
	for k, v := range values {
		if value, has := s.state.GetValue(k); has {
			backup[k] = value
		}
		s.state.SetValue(k, v)
	}
	return func() {
		for k, v := range backup {
			s.state.SetValue(k, v)
		}
	}
}

//NewRequest creates a new request for service and action
func (c *Context) NewRequest(serviceName, action string) (interface{}, error) {
	service, err := c.Service(serviceName)
	if err != nil {
		return nil, err
	}
	route, err := service.Route(action)
	if err != nil {
		return nil, err
	}
	return route.RequestProvider(), nil
}

//AsRequest converts a source map into request for provided service and action.
func (c *Context) AsRequest(serviceName, action string, source map[string]interface{}) (request interface{}, err error) {
	if request, err = c.NewRequest(serviceName, action); err != nil {
		return request, err
	}
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("failed to create request, unable to cast %v into %T, %v", source, request, r)
		}
	}()
	expanded := c.state.Expand(source)
	source = toolbox.AsMap(expanded)
	err = toolbox.DefaultConverter.AssignConverted(request, source)
	return request, err
}

//Close closes this context, it executes all deferred function and set closed flag.
func (c *Context) Close() {
	atomic.StoreInt32(&c.closed, 1)
	for _, context := range c.cloned {
		context.Close()
	}
	for _, function := range c.Deffer() {
		function()
	}
}

//MakeAsyncSafe makes this contex async safe

func (c *Context) MakeAsyncSafe() *msg.Events {
	for k := range c.AsyncUnsafeKeys {
		c.Context.Remove(k)
	}
	result := msg.NewEvents()
	c.Listener = result.AsListener()
	return result
}

var yyyyMMDDLayout = toolbox.DateFormatToLayout("yyyy-MM-dd")
var yyyMMDDHHMMSSLayout = toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss")
var numberDateLayout = toolbox.DateFormatToLayout("yyyyMMddhhmmSSS")

/*
NewDefaultState returns a new default state.
It comes with the following registered keys:
	* rand - random int64
	* date -  current date formatted as yyyy-MM-dd
	* time - current time formatted as yyyy-MM-dd hh:mm:ss
	* ts - current timestamp formatted  as yyyyMMddhhmmSSS
	* timestamp.XXX - timestamp in ms where XXX is time diff expression i.e 3DaysAgo, tomorrow, hourAhead
	* unix.XXX - timestamp in sec where XXX is time diff expression i.e 3DaysAgo, tomorrow, hourAhead
	* tzTime.XXX - RFC3339 formatted time where XXX is time diff expression i.e 3DaysAgo, tomorrow, hourAhead
	* tmpDir - temp directory
	* uuid.next - generate unique id
	* uuid.Get - returns previously generated unique id, or generate new
	*.env.XXX where XXX is the ID of the env variable to return
	* all UFD registry functions
*/
func NewDefaultState() data.Map {
	var result = data.NewMap()
	var now = time.Now()
	source := rand.NewSource(now.UnixNano())
	result.Put("rand", source.Int63())
	result.Put("date", now.Format(yyyyMMDDLayout))
	result.Put("time", now.Format(yyyMMDDHHMMSSLayout))
	result.Put("ts", now.Format(numberDateLayout))

	result.Put("tmpDir", func(key string) interface{} {
		tempPath := path.Join(os.TempDir(), key)
		exec.Command("mkdir -p " + tempPath)
		return tempPath
	})

	var cachedUUID uuid.UUID
	result.Put("uuid", func(key string) interface{} {
		if key == "next" {
			var err error
			cachedUUID, err = uuid.NewV4()
			if err != nil {
				return ""
			}
		}
		if len(cachedUUID) > 0 {
			return cachedUUID.String()
		}
		return ""
	})

	//returns time in ms
	result.Put("timestamp", func(key string) interface{} {
		timeAt, err := toolbox.TimeAt(key)
		if err != nil {
			return nil
		}
		return int(timeAt.Unix()+timeAt.UnixNano()) / 1000000
	})
	//return time in sec
	result.Put("unix", func(key string) interface{} {
		timeAt, err := toolbox.TimeAt(key)
		if err != nil {
			return nil
		}
		return int(timeAt.Unix()+timeAt.UnixNano()) / 1000000000
	})

	//return formatted time with time.RFC3339 yyyy-MM-ddThh:mm:ss.SSS Z  i.e ${tzTime.4daysAgoInUTC}

	result.Put("tzTime", func(key string) interface{} {
		timeAt, err := toolbox.TimeAt(key)
		if err != nil {
			return nil
		}
		return timeAt.Format(time.RFC3339)
	})

	result.Put("env", func(key string) interface{} {
		return os.Getenv(key)
	})

	neatly.AddStandardUdf(result)
	for k, v := range UdfRegistry {
		result.Put(k, v)
	}
	return result
}
<commit_msg>change  variable to current unix timestamp<commit_after>package endly

import (
	"fmt"
	uuid "github.com/satori/go.uuid"
	"github.com/viant/endly/msg"
	"github.com/viant/neatly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/data"
	"github.com/viant/toolbox/secret"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

var serviceManagerKey = (*manager)(nil)
var deferFunctionsKey = (*[]func())(nil)

//Context represents a workflow session context/state
type Context struct {
	SessionID       string
	CLIEnabled      bool
	HasLogger       bool
	AsyncUnsafeKeys map[interface{}]bool
	Secrets         *secret.Service
	Wait            *sync.WaitGroup
	Listener        msg.Listener
	Source          *url.Resource
	state           data.Map
	toolbox.Context
	cloned []*Context
	closed int32
}

//Publish publishes event to listeners, it updates current run details like activity workflow name etc ...
func (c *Context) Publish(value interface{}) msg.Event {
	event, ok := value.(msg.Event)
	if !ok {
		event = msg.NewEvent(value)
	}
	if c.Listener != nil {
		c.Listener(event)
	}
	return event
}

//PublishWithStartEvent publishes event to listeners, it updates current run details like activity workflow name etc ...
func (c *Context) PublishWithStartEvent(value interface{}, init msg.Event) msg.Event {
	event := msg.NewEventWithInit(value, init)
	if c.Listener != nil {
		c.Listener(event)
	}
	return event
}

//SetListener sets context event Listener
func (c *Context) SetListener(listener msg.Listener) {
	c.Listener = listener
}

//IsClosed returns true if it is closed.
func (c *Context) IsClosed() bool {
	return atomic.LoadInt32(&c.closed) == 1
}

//Clone clones the context.
func (c *Context) Clone() *Context {
	if len(c.cloned) == 0 {
		c.cloned = make([]*Context, 0)
	}
	result := &Context{}
	result.Wait = &sync.WaitGroup{}
	result.Context = c.Context.Clone()
	result.state = NewDefaultState()
	result.state.Apply(c.state)
	result.SessionID = c.SessionID
	result.Listener = c.Listener
	result.CLIEnabled = c.CLIEnabled
	result.Secrets = c.Secrets
	result.AsyncUnsafeKeys = make(map[interface{}]bool)
	c.cloned = append(c.cloned, result)
	return result
}

func (c *Context) parentURLCandidates() []string {
	var result = make([]string, 0)
	if c.Source != nil {
		baseURL, _ := toolbox.URLSplit(c.Source.URL)
		result = append(result, baseURL)
	}
	currentDirectory, err := os.Getwd()
	if err == nil {
		result = append(result, toolbox.FileSchema+currentDirectory)
	}

	return result
}

//ExpandResource substitutes any $ expression with the key value from the state map if it is present.
func (c *Context) ExpandResource(resource *url.Resource) (*url.Resource, error) {
	if resource == nil {
		return nil, msg.ReportError(fmt.Errorf("resource  was empty"))
	}
	if resource.URL == "" {
		return nil, msg.ReportError(fmt.Errorf("url was empty"))
	}

	if !strings.Contains(resource.URL, "://") {
		for _, parentCandidate := range c.parentURLCandidates() {
			service, err := storage.NewServiceForURL(parentCandidate, "")
			if err != nil {
				continue
			}
			var candidateURL = toolbox.URLPathJoin(parentCandidate, resource.URL)
			if exists, err := service.Exists(candidateURL); exists && err == nil {
				resource.URL = candidateURL
			}
		}
	}
	var result = url.NewResource(c.Expand(resource.URL), c.Expand(resource.Credentials))
	if result.ParsedURL == nil {
		return nil, fmt.Errorf("failed to parse URL %v", result.URL)
	}
	result.Cache = c.Expand(resource.Cache)
	result.CacheExpiryMs = resource.CacheExpiryMs
	return result, nil
}

//Manager returns workflow manager or error
func (c *Context) Manager() (Manager, error) {
	if c == nil {
		return nil, fmt.Errorf("context was nil")
	}
	var manager = &manager{}
	if !c.GetInto(serviceManagerKey, &manager) {
		return nil, msg.ReportError(fmt.Errorf("failed to lookup Manager"))
	}
	return manager, nil
}

//Service returns a service fo provided id or error.
func (c *Context) Service(name string) (Service, error) {
	manager, err := c.Manager()
	if err != nil {
		return nil, err
	}
	return manager.Service(name)
}

//Deffer add function to be executed if context closes. If returns currently registered functions.
func (c *Context) Deffer(functions ...func()) []func() {
	var result *[]func()
	if !c.Contains(deferFunctionsKey) {
		var functions = make([]func(), 0)
		result = &functions
		_ = c.Put(deferFunctionsKey, result)
	} else {
		c.GetInto(deferFunctionsKey, &result)
	}

	*result = append(*result, functions...)
	_ = c.Put(deferFunctionsKey, &result)
	return *result
}

//State returns a context state map.
func (c *Context) State() data.Map {
	if c.state == nil {
		c.state = NewDefaultState()
	}
	return c.state
}

//SetState sets a new state map
func (c *Context) SetState(state data.Map) {
	c.state = state
}

//Expand substitute $ expression if present in the text and state map.
func (c *Context) Expand(text string) string {
	state := c.State()
	return state.ExpandAsText(text)
}

//PublishAndRestore sets supplied value and returns func restoring original values
func (s *Context) PublishAndRestore(values map[string]interface{}) func() {
	var backup = map[string]interface{}{}
	for k, v := range values {
		if value, has := s.state.GetValue(k); has {
			backup[k] = value
		}
		s.state.SetValue(k, v)
	}
	return func() {
		for k, v := range backup {
			s.state.SetValue(k, v)
		}
	}
}

//NewRequest creates a new request for service and action
func (c *Context) NewRequest(serviceName, action string) (interface{}, error) {
	service, err := c.Service(serviceName)
	if err != nil {
		return nil, err
	}
	route, err := service.Route(action)
	if err != nil {
		return nil, err
	}
	return route.RequestProvider(), nil
}

//AsRequest converts a source map into request for provided service and action.
func (c *Context) AsRequest(serviceName, action string, source map[string]interface{}) (request interface{}, err error) {
	if request, err = c.NewRequest(serviceName, action); err != nil {
		return request, err
	}
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("failed to create request, unable to cast %v into %T, %v", source, request, r)
		}
	}()
	expanded := c.state.Expand(source)
	source = toolbox.AsMap(expanded)
	err = toolbox.DefaultConverter.AssignConverted(request, source)
	return request, err
}

//Close closes this context, it executes all deferred function and set closed flag.
func (c *Context) Close() {
	atomic.StoreInt32(&c.closed, 1)
	for _, context := range c.cloned {
		context.Close()
	}
	for _, function := range c.Deffer() {
		function()
	}
}

//MakeAsyncSafe makes this contex async safe

func (c *Context) MakeAsyncSafe() *msg.Events {
	for k := range c.AsyncUnsafeKeys {
		c.Context.Remove(k)
	}
	result := msg.NewEvents()
	c.Listener = result.AsListener()
	return result
}

var yyyyMMDDLayout = toolbox.DateFormatToLayout("yyyy-MM-dd")
var yyyMMDDHHMMSSLayout = toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss")
var numberDateLayout = toolbox.DateFormatToLayout("yyyyMMddhhmmSSS")

/*
NewDefaultState returns a new default state.
It comes with the following registered keys:
	* rand - random int64
	* date -  current date formatted as yyyy-MM-dd
	* time - current time formatted as yyyy-MM-dd hh:mm:ss
	* ts - current timestamp formatted  as yyyyMMddhhmmSSS
	* timestamp.XXX - timestamp in ms where XXX is time diff expression i.e 3DaysAgo, tomorrow, hourAhead
	* unix.XXX - timestamp in sec where XXX is time diff expression i.e 3DaysAgo, tomorrow, hourAhead
	* tzTime.XXX - RFC3339 formatted time where XXX is time diff expression i.e 3DaysAgo, tomorrow, hourAhead
	* tmpDir - temp directory
	* uuid.next - generate unique id
	* uuid.Get - returns previously generated unique id, or generate new
	*.env.XXX where XXX is the ID of the env variable to return
	* all UFD registry functions
*/
func NewDefaultState() data.Map {
	var result = data.NewMap()
	var now = time.Now()
	source := rand.NewSource(now.UnixNano())
	result.Put("rand", source.Int63())
	result.Put("date", now.Format(yyyyMMDDLayout))
	result.Put("time", now.Format(yyyMMDDHHMMSSLayout))
	result.Put("ts", now.Unix())

	result.Put("tmpDir", func(key string) interface{} {
		tempPath := path.Join(os.TempDir(), key)
		exec.Command("mkdir -p " + tempPath)
		return tempPath
	})

	var cachedUUID uuid.UUID
	result.Put("uuid", func(key string) interface{} {
		if key == "next" {
			var err error
			cachedUUID, err = uuid.NewV4()
			if err != nil {
				return ""
			}
		}
		if len(cachedUUID) > 0 {
			return cachedUUID.String()
		}
		return ""
	})

	//returns time in ms
	result.Put("timestamp", func(key string) interface{} {
		timeAt, err := toolbox.TimeAt(key)
		if err != nil {
			return nil
		}
		return int(timeAt.Unix()+timeAt.UnixNano()) / 1000000
	})
	//return time in sec
	result.Put("unix", func(key string) interface{} {
		timeAt, err := toolbox.TimeAt(key)
		if err != nil {
			return nil
		}
		return int(timeAt.Unix()+timeAt.UnixNano()) / 1000000000
	})

	//return formatted time with time.RFC3339 yyyy-MM-ddThh:mm:ss.SSS Z  i.e ${tzTime.4daysAgoInUTC}

	result.Put("tzTime", func(key string) interface{} {
		timeAt, err := toolbox.TimeAt(key)
		if err != nil {
			return nil
		}
		return timeAt.Format(time.RFC3339)
	})

	result.Put("env", func(key string) interface{} {
		return os.Getenv(key)
	})

	neatly.AddStandardUdf(result)
	for k, v := range UdfRegistry {
		result.Put(k, v)
	}
	return result
}
<|endoftext|>


<commit_before>package goldb

import (
	"bytes"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/iterator"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

// Context is context of reading data via get or fetch-methods.
// Context is implemented by Transaction and Storage
type Context struct {
	qCtx         queryContext
	ReadOptions  *opt.ReadOptions
	WriteOptions *opt.WriteOptions
}

type queryContext interface {
	Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
	NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator
}

// Get returns raw data by key
func (c *Context) Get(key []byte) ([]byte, error) {
	data, err := c.qCtx.Get(key, c.ReadOptions)
	if err != nil && err != leveldb.ErrNotFound {
		return nil, nil
	}
	return data, err
}

// GetNum returns uint64-data by key
func (c *Context) GetInt(key []byte) (num int64, err error) {
	_, err = c.GetVar(key, &num)
	return
}

// GetID returns uint64-data by key
func (c *Context) GetID(key []byte) (id uint64, err error) {
	data, err := c.Get(key)
	if err == nil {
		id, err = DecodeID(data)
	}
	return
}

// GetVar get data by key and unmarshal to to variable;
// Returns true when data by key existed
func (c *Context) GetVar(key []byte, v interface{}) (bool, error) {
	data, err := c.Get(key)
	if err == leveldb.ErrNotFound {
		return false, nil
	}
	if err == nil {
		err = DecodeData(data, v)
	}
	if err != nil {
		return false, err
	}
	return true, nil
}

// GetNumRows fetches data by query and calculates count rows
func (c *Context) GetNumRows(q *Query) (numRows uint64, err error) {
	err = c.execute(q, nil)
	numRows = q.NumRows
	return
}

// Fetch fetches raw-data by query
func (c *Context) Fetch(q *Query, fnRow func(value []byte) error) error {
	return c.execute(q, fnRow)
}

// FetchID fetches uint64-ID by query
func (c *Context) FetchID(q *Query, fnRow func(id uint64) error) error {
	return c.execute(q, func(v []byte) error {
		if id, err := DecodeID(v); err != nil {
			return err
		} else {
			return fnRow(id)
		}
	})
}

// FetchObject fetches object by query
func (c *Context) FetchObject(q *Query, obj interface{}, fnRow func() error) error {
	return c.execute(q, func(data []byte) error {
		if err := DecodeData(data, obj); err != nil {
			return err
		} else {
			return fnRow()
		}
	})
}

// QueryIDs returns slice of row-id by query
func (c *Context) QueryIDs(q *Query) (ids []uint64, err error) {
	err = c.FetchID(q, func(id uint64) error {
		ids = append(ids, id)
		return nil
	})
	return
}

// QueryID returns first row-id by query
func (c *Context) QueryID(q *Query) (id uint64, err error) {
	q.Limit(1)
	ids, err := c.QueryIDs(q)
	if len(ids) > 0 {
		return ids[0], err
	}
	return 0, err
}

//------ private ------
var tail1024 = bytes.Repeat([]byte{255}, 1024)

func (c *Context) execute(q *Query, fnRow func([]byte) error) (err error) {
	q.NumRows = 0
	pfx := q.filter
	pfxLen := len(pfx)
	start := append(pfx, q.offset...)
	nStart := len(start)
	limit := q.limit
	skipFirst := len(q.offset) > 0

	var iter iterator.Iterator
	var iterNext func() bool
	var rowID uint64
	fnRecordFilter := q.recFilter

	if !q.desc { // ask
		iter = c.qCtx.NewIterator(&util.Range{Start: start}, nil)
		iterNext = func() bool { return iter.Next() }

	} else { // desc
		iter = c.qCtx.NewIterator(nil, nil)
		iter.Seek(append(start, tail1024...))
		iterNext = func() bool { return iter.Prev() }
	}

	defer func() {
		iter.Release()
		if err == nil {
			err = iter.Error()
		}
	}()

	for limit > 0 && iterNext() {
		key := iter.Key()
		if !bytes.HasPrefix(key, pfx) {
			break
		}
		if skipFirst { // skip first record if record.key == startOffset
			//if q.strongOffset { // todo: use only strong compare (returns actual offset after each query)
			//	skipFirst = false
			//	if bytes.Equal(key, start) {
			//		continue
			//	}
			//} else {
			if len(key) >= nStart && bytes.Equal(key[:nStart], start) {
				continue
			}
			skipFirst = false
			//}
		}
		val := iter.Value()
		if fnRecordFilter != nil {
			if rowID, err = DecodeID(val); err != nil {
				break
			} else if !fnRecordFilter(rowID) {
				continue
			}
		}
		q.offset = key[pfxLen:]
		limit--
		if fnRow != nil {
			if err = fnRow(val); err != nil {
				break
			}
		}
		q.NumRows++
	}
	return
}
<commit_msg>add Exists method<commit_after>package goldb

import (
	"bytes"

	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/iterator"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/util"
)

// Context is context of reading data via get or fetch-methods.
// Context is implemented by Transaction and Storage
type Context struct {
	qCtx         queryContext
	ReadOptions  *opt.ReadOptions
	WriteOptions *opt.WriteOptions
}

type queryContext interface {
	Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
	NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator
}

// Get returns raw data by key
func (c *Context) Get(key []byte) ([]byte, error) {
	data, err := c.qCtx.Get(key, c.ReadOptions)
	if err != nil && err != leveldb.ErrNotFound {
		return nil, nil
	}
	return data, err
}

// GetNum returns uint64-data by key
func (c *Context) GetInt(key []byte) (num int64, err error) {
	_, err = c.GetVar(key, &num)
	return
}

// GetID returns uint64-data by key
func (c *Context) GetID(key []byte) (id uint64, err error) {
	data, err := c.Get(key)
	if err == nil {
		id, err = DecodeID(data)
	}
	return
}

// GetVar get data by key and unmarshal to to variable;
// Returns true when data by key existed
func (c *Context) GetVar(key []byte, v interface{}) (bool, error) {
	data, err := c.Get(key)
	if err == leveldb.ErrNotFound {
		return false, nil
	}
	if err == nil {
		err = DecodeData(data, v)
	}
	if err != nil {
		return false, err
	}
	return true, nil
}

// GetNumRows fetches data by query and calculates count rows
func (c *Context) GetNumRows(q *Query) (numRows uint64, err error) {
	err = c.execute(q, nil)
	numRows = q.NumRows
	return
}

// Exists returns true when exists results by query
func (c *Context) Exists(q *Query) (ok bool, err error) {
	var qCopy = *q
	qCopy.Limit(1)
	err = c.execute(&qCopy, nil)
	ok = qCopy.NumRows > 0
	return
}

// Fetch fetches raw-data by query
func (c *Context) Fetch(q *Query, fnRow func(value []byte) error) error {
	return c.execute(q, fnRow)
}

// FetchID fetches uint64-ID by query
func (c *Context) FetchID(q *Query, fnRow func(id uint64) error) error {
	return c.execute(q, func(v []byte) error {
		if id, err := DecodeID(v); err != nil {
			return err
		} else {
			return fnRow(id)
		}
	})
}

// FetchObject fetches object by query
func (c *Context) FetchObject(q *Query, obj interface{}, fnRow func() error) error {
	return c.execute(q, func(data []byte) error {
		if err := DecodeData(data, obj); err != nil {
			return err
		} else {
			return fnRow()
		}
	})
}

// QueryIDs returns slice of row-id by query
func (c *Context) QueryIDs(q *Query) (ids []uint64, err error) {
	err = c.FetchID(q, func(id uint64) error {
		ids = append(ids, id)
		return nil
	})
	return
}

// QueryID returns first row-id by query
func (c *Context) QueryID(q *Query) (id uint64, err error) {
	q.Limit(1)
	ids, err := c.QueryIDs(q)
	if len(ids) > 0 {
		return ids[0], err
	}
	return 0, err
}

//------ private ------
var tail1024 = bytes.Repeat([]byte{255}, 1024)

func (c *Context) execute(q *Query, fnRow func([]byte) error) (err error) {
	q.NumRows = 0
	pfx := q.filter
	pfxLen := len(pfx)
	start := append(pfx, q.offset...)
	nStart := len(start)
	limit := q.limit
	skipFirst := len(q.offset) > 0

	var iter iterator.Iterator
	var iterNext func() bool
	var rowID uint64
	fnRecordFilter := q.recFilter

	if !q.desc { // ask
		iter = c.qCtx.NewIterator(&util.Range{Start: start}, nil)
		iterNext = func() bool { return iter.Next() }

	} else { // desc
		iter = c.qCtx.NewIterator(nil, nil)
		iter.Seek(append(start, tail1024...))
		iterNext = func() bool { return iter.Prev() }
	}

	defer func() {
		iter.Release()
		if err == nil {
			err = iter.Error()
		}
	}()

	for limit > 0 && iterNext() {
		key := iter.Key()
		if !bytes.HasPrefix(key, pfx) {
			break
		}
		if skipFirst { // skip first record if record.key == startOffset
			//if q.strongOffset { // todo: use only strong compare (returns actual offset after each query)
			//	skipFirst = false
			//	if bytes.Equal(key, start) {
			//		continue
			//	}
			//} else {
			if len(key) >= nStart && bytes.Equal(key[:nStart], start) {
				continue
			}
			skipFirst = false
			//}
		}
		val := iter.Value()
		if fnRecordFilter != nil {
			if rowID, err = DecodeID(val); err != nil {
				break
			} else if !fnRecordFilter(rowID) {
				continue
			}
		}
		q.offset = key[pfxLen:]
		limit--
		if fnRow != nil {
			if err = fnRow(val); err != nil {
				break
			}
		}
		q.NumRows++
	}
	return
}
<|endoftext|>


<commit_before>package endly

import (
	"fmt"
	uuid "github.com/satori/go.uuid"
	"github.com/viant/endly/msg"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/data"
	"github.com/viant/toolbox/secret"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

var serviceManagerKey = (*manager)(nil)
var deferFunctionsKey = (*[]func())(nil)

//Context represents a workflow session context/state
type Context struct {
	SessionID       string
	CLIEnabled      bool
	HasLogger       bool
	AsyncUnsafeKeys map[interface{}]bool
	Secrets         *secret.Service
	Wait            *sync.WaitGroup
	Listener        msg.Listener
	Source          *url.Resource
	state           data.Map
	toolbox.Context
	cloned []*Context
	closed int32
}

//Publish publishes event to listeners, it updates current run details like activity workflow name etc ...
func (c *Context) Publish(value interface{}) msg.Event {
	event, ok := value.(msg.Event)
	if !ok {
		event = msg.NewEvent(value)
	}
	if c.Listener != nil {
		c.Listener(event)
	}
	return event
}

//PublishWithStartEvent publishes event to listeners, it updates current run details like activity workflow name etc ...
func (c *Context) PublishWithStartEvent(value interface{}, init msg.Event) msg.Event {
	event := msg.NewEventWithInit(value, init)
	if c.Listener != nil {
		c.Listener(event)
	}
	return event
}

//SetListener sets context event Listener
func (c *Context) SetListener(listener msg.Listener) {
	c.Listener = listener
}

//IsClosed returns true if it is closed.
func (c *Context) IsClosed() bool {
	return atomic.LoadInt32(&c.closed) == 1
}

//Clone clones the context.
func (c *Context) Clone() *Context {
	if len(c.cloned) == 0 {
		c.cloned = make([]*Context, 0)
	}
	result := &Context{}
	result.Wait = &sync.WaitGroup{}
	result.Context = c.Context.Clone()
	result.state = NewDefaultState()
	result.state.Apply(c.state)
	result.SessionID = c.SessionID
	result.Listener = c.Listener
	result.CLIEnabled = c.CLIEnabled
	result.Secrets = c.Secrets
	result.AsyncUnsafeKeys = make(map[interface{}]bool)
	c.cloned = append(c.cloned, result)
	return result
}

func (c *Context) parentURLCandidates() []string {
	var result = make([]string, 0)
	if c.Source != nil {
		baseURL, _ := toolbox.URLSplit(c.Source.URL)
		result = append(result, baseURL)
	}
	currentDirectory, err := os.Getwd()
	if err == nil {
		result = append(result, toolbox.FileSchema+currentDirectory)
	}

	return result
}

//ExpandResource substitutes any $ expression with the key value from the state map if it is present.
func (c *Context) ExpandResource(resource *url.Resource) (*url.Resource, error) {
	if resource == nil {
		return nil, msg.ReportError(fmt.Errorf("resource  was empty"))
	}
	if resource.URL == "" {
		return nil, msg.ReportError(fmt.Errorf("url was empty"))
	}

	if !strings.Contains(resource.URL, "://") {
		for _, parentCandidate := range c.parentURLCandidates() {
			service, err := storage.NewServiceForURL(parentCandidate, "")
			if err != nil {
				continue
			}
			var candidateURL = toolbox.URLPathJoin(parentCandidate, resource.URL)
			if exists, err := service.Exists(candidateURL); exists && err == nil {
				resource.URL = candidateURL
			}
		}
	}
	var result = url.NewResource(c.Expand(resource.URL), c.Expand(resource.Credentials))
	if result.ParsedURL == nil {
		return nil, fmt.Errorf("failed to parse URL %v", result.URL)
	}
	result.Cache = c.Expand(resource.Cache)
	result.CacheExpiryMs = resource.CacheExpiryMs
	return result, nil
}

//Manager returns workflow manager or error
func (c *Context) Manager() (Manager, error) {
	if c == nil {
		return nil, fmt.Errorf("context was nil")
	}
	var manager = &manager{}
	if !c.GetInto(serviceManagerKey, &manager) {
		return nil, msg.ReportError(fmt.Errorf("failed to lookup Manager"))
	}
	return manager, nil
}

//Service returns a service fo provided id or error.
func (c *Context) Service(name string) (Service, error) {
	manager, err := c.Manager()
	if err != nil {
		return nil, err
	}
	return manager.Service(name)
}

//Deffer add function to be executed if context closes. If returns currently registered functions.
func (c *Context) Deffer(functions ...func()) []func() {
	var result *[]func()
	if !c.Contains(deferFunctionsKey) {
		var functions = make([]func(), 0)
		result = &functions
		_ = c.Put(deferFunctionsKey, result)
	} else {
		c.GetInto(deferFunctionsKey, &result)
	}

	*result = append(*result, functions...)
	_ = c.Put(deferFunctionsKey, &result)
	return *result
}

//State returns a context state map.
func (c *Context) State() data.Map {
	if c.state == nil {
		c.state = NewDefaultState()
	}
	return c.state
}

//SetState sets a new state map
func (c *Context) SetState(state data.Map) {
	c.state = state
}

//Expand substitute $ expression if present in the text and state map.
func (c *Context) Expand(text string) string {
	state := c.State()
	return state.ExpandAsText(text)
}

//PublishAndRestore sets supplied value and returns func restoring original values
func (s *Context) PublishAndRestore(values map[string]interface{}) func() {
	var backup = map[string]interface{}{}
	for k, v := range values {
		if value, has := s.state.GetValue(k); has {
			backup[k] = value
		}
		s.state.SetValue(k, v)
	}
	return func() {
		for k, v := range backup {
			s.state.SetValue(k, v)
		}
	}
}

//NewRequest creates a new request for service and action
func (c *Context) NewRequest(serviceName, action string) (interface{}, error) {
	service, err := c.Service(serviceName)
	if err != nil {
		return nil, err
	}
	route, err := service.Route(action)
	if err != nil {
		return nil, err
	}
	return route.RequestProvider(), nil
}

//AsRequest converts a source map into request for provided service and action.
func (c *Context) AsRequest(serviceName, action string, source map[string]interface{}) (request interface{}, err error) {
	if request, err = c.NewRequest(serviceName, action); err != nil {
		return request, err
	}
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("failed to create request, unable to cast %v into %T, %v", source, request, r)
		}
	}()
	expanded := c.state.Expand(source)
	source = toolbox.AsMap(expanded)
	err = toolbox.DefaultConverter.AssignConverted(request, source)
	return request, err
}

//Close closes this context, it executes all deferred function and set closed flag.
func (c *Context) Close() {
	atomic.StoreInt32(&c.closed, 1)
	for _, context := range c.cloned {
		context.Close()
	}
	for _, function := range c.Deffer() {
		function()
	}
}

//MakeAsyncSafe makes this contex async safe

func (c *Context) MakeAsyncSafe() *msg.Events {
	for k := range c.AsyncUnsafeKeys {
		c.Context.Remove(k)
	}
	result := msg.NewEvents()
	c.Listener = result.AsListener()
	return result
}

/*
NewDefaultState returns a new default state.
It comes with the following registered keys:
	* rand - random int64
	*  date -  current date formatted as yyyy-MM-dd
	* time - current time formatted as yyyy-MM-dd hh:mm:ss
	* ts - current timestamp formatted  as yyyyMMddhhmmSSS
	* timestamp.yesterday - timestamp in ms
	* timestamp.now - timestamp in ms
	* timestamp.tomorrow - timestamp in ms
	* tmpDir - temp directory
	* uuid.next - generate unique id
	* uuid.Get - returns previously generated unique id, or generate new
	*.end.XXX where XXX is the ID of the env variable to return
	* all UFD registry functions
*/
func NewDefaultState() data.Map {
	var result = data.NewMap()
	var now = time.Now()
	source := rand.NewSource(now.UnixNano())
	result.Put("rand", source.Int63())
	result.Put("date", now.Format(toolbox.DateFormatToLayout("yyyy-MM-dd")))
	result.Put("time", now.Format(toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss")))
	result.Put("ts", now.Format(toolbox.DateFormatToLayout("yyyyMMddhhmmSSS")))

	result.Put("tmpDir", func(key string) interface{} {
		tempPath := path.Join(os.TempDir(), key)
		exec.Command("mkdir -p " + tempPath)
		return tempPath
	})

	var cachedUUID uuid.UUID
	result.Put("uuid", func(key string) interface{} {
		if key == "next" {
			var err error
			cachedUUID, err = uuid.NewV4()
			if err != nil {
				return ""
			}
		}
		if len(cachedUUID) > 0 {
			return cachedUUID.String()
		}
		return ""
	})

	result.Put("timestamp", func(key string) interface{} {
		var timeDiffProvider = toolbox.NewTimeDiffProvider()
		switch key {
		case "now":
			result, _ := timeDiffProvider.Get(nil, "now", 0, "day", "timestamp")
			return result
		case "tomorrow":
			result, _ := timeDiffProvider.Get(nil, "now", 1, "day", "timestamp")
			return result
		case "yesterday":
			result, _ := timeDiffProvider.Get(nil, "now", -1, "day", "timestamp")
			return result
		}
		return nil
	})

	result.Put("unix", func(key string) interface{} {
		var timeDiffProvider = toolbox.NewTimeDiffProvider()
		switch key {
		case "now":
			result, _ := timeDiffProvider.Get(nil, "now", 0, "day", "unix")
			return result
		case "tomorrow":
			result, _ := timeDiffProvider.Get(nil, "now", 1, "day", "unix")
			return result
		case "yesterday":
			result, _ := timeDiffProvider.Get(nil, "now", -1, "day", "unix")
			return result
		case "hourbefore":
			result, _ := timeDiffProvider.Get(nil, "now", -1, "hour", "unix")
			return result
		case "2hourbefore":
			result, _ := timeDiffProvider.Get(nil, "now", -2, "hour", "unix")
			return result
		case "hourlater":
			result, _ := timeDiffProvider.Get(nil, "now", 1, "hour", "unix")
			return result
		case "2hourlater":
			result, _ := timeDiffProvider.Get(nil, "now", 2, "hour", "unix")
			return result
		}
		return nil
	})

	result.Put("env", func(key string) interface{} {
		return os.Getenv(key)
	})

	for k, v := range UdfRegistry {
		result.Put(k, v)
	}
	return result
}
<commit_msg>reformatted<commit_after>package endly

import (
	"fmt"
	uuid "github.com/satori/go.uuid"
	"github.com/viant/endly/msg"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/data"
	"github.com/viant/toolbox/secret"
	"github.com/viant/toolbox/storage"
	"github.com/viant/toolbox/url"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"strings"
	"sync"
	"sync/atomic"
	"time"
)

var serviceManagerKey = (*manager)(nil)
var deferFunctionsKey = (*[]func())(nil)

//Context represents a workflow session context/state
type Context struct {
	SessionID       string
	CLIEnabled      bool
	HasLogger       bool
	AsyncUnsafeKeys map[interface{}]bool
	Secrets         *secret.Service
	Wait            *sync.WaitGroup
	Listener        msg.Listener
	Source          *url.Resource
	state           data.Map
	toolbox.Context
	cloned []*Context
	closed int32
}

//Publish publishes event to listeners, it updates current run details like activity workflow name etc ...
func (c *Context) Publish(value interface{}) msg.Event {
	event, ok := value.(msg.Event)
	if !ok {
		event = msg.NewEvent(value)
	}
	if c.Listener != nil {
		c.Listener(event)
	}
	return event
}

//PublishWithStartEvent publishes event to listeners, it updates current run details like activity workflow name etc ...
func (c *Context) PublishWithStartEvent(value interface{}, init msg.Event) msg.Event {
	event := msg.NewEventWithInit(value, init)
	if c.Listener != nil {
		c.Listener(event)
	}
	return event
}

//SetListener sets context event Listener
func (c *Context) SetListener(listener msg.Listener) {
	c.Listener = listener
}

//IsClosed returns true if it is closed.
func (c *Context) IsClosed() bool {
	return atomic.LoadInt32(&c.closed) == 1
}

//Clone clones the context.
func (c *Context) Clone() *Context {
	if len(c.cloned) == 0 {
		c.cloned = make([]*Context, 0)
	}
	result := &Context{}
	result.Wait = &sync.WaitGroup{}
	result.Context = c.Context.Clone()
	result.state = NewDefaultState()
	result.state.Apply(c.state)
	result.SessionID = c.SessionID
	result.Listener = c.Listener
	result.CLIEnabled = c.CLIEnabled
	result.Secrets = c.Secrets
	result.AsyncUnsafeKeys = make(map[interface{}]bool)
	c.cloned = append(c.cloned, result)
	return result
}

func (c *Context) parentURLCandidates() []string {
	var result = make([]string, 0)
	if c.Source != nil {
		baseURL, _ := toolbox.URLSplit(c.Source.URL)
		result = append(result, baseURL)
	}
	currentDirectory, err := os.Getwd()
	if err == nil {
		result = append(result, toolbox.FileSchema+currentDirectory)
	}

	return result
}

//ExpandResource substitutes any $ expression with the key value from the state map if it is present.
func (c *Context) ExpandResource(resource *url.Resource) (*url.Resource, error) {
	if resource == nil {
		return nil, msg.ReportError(fmt.Errorf("resource  was empty"))
	}
	if resource.URL == "" {
		return nil, msg.ReportError(fmt.Errorf("url was empty"))
	}

	if !strings.Contains(resource.URL, "://") {
		for _, parentCandidate := range c.parentURLCandidates() {
			service, err := storage.NewServiceForURL(parentCandidate, "")
			if err != nil {
				continue
			}
			var candidateURL = toolbox.URLPathJoin(parentCandidate, resource.URL)
			if exists, err := service.Exists(candidateURL); exists && err == nil {
				resource.URL = candidateURL
			}
		}
	}
	var result = url.NewResource(c.Expand(resource.URL), c.Expand(resource.Credentials))
	if result.ParsedURL == nil {
		return nil, fmt.Errorf("failed to parse URL %v", result.URL)
	}
	result.Cache = c.Expand(resource.Cache)
	result.CacheExpiryMs = resource.CacheExpiryMs
	return result, nil
}

//Manager returns workflow manager or error
func (c *Context) Manager() (Manager, error) {
	if c == nil {
		return nil, fmt.Errorf("context was nil")
	}
	var manager = &manager{}
	if !c.GetInto(serviceManagerKey, &manager) {
		return nil, msg.ReportError(fmt.Errorf("failed to lookup Manager"))
	}
	return manager, nil
}

//Service returns a service fo provided id or error.
func (c *Context) Service(name string) (Service, error) {
	manager, err := c.Manager()
	if err != nil {
		return nil, err
	}
	return manager.Service(name)
}

//Deffer add function to be executed if context closes. If returns currently registered functions.
func (c *Context) Deffer(functions ...func()) []func() {
	var result *[]func()
	if !c.Contains(deferFunctionsKey) {
		var functions = make([]func(), 0)
		result = &functions
		_ = c.Put(deferFunctionsKey, result)
	} else {
		c.GetInto(deferFunctionsKey, &result)
	}

	*result = append(*result, functions...)
	_ = c.Put(deferFunctionsKey, &result)
	return *result
}

//State returns a context state map.
func (c *Context) State() data.Map {
	if c.state == nil {
		c.state = NewDefaultState()
	}
	return c.state
}

//SetState sets a new state map
func (c *Context) SetState(state data.Map) {
	c.state = state
}

//Clone substitute $ expression if present in the text and state map.
func (c *Context) Expand(text string) string {
	state := c.State()
	return state.ExpandAsText(text)
}

//PublishAndRestore sets supplied value and returns func restoring original values
func (s *Context) PublishAndRestore(values map[string]interface{}) func() {
	var backup = map[string]interface{}{}
	for k, v := range values {
		if value, has := s.state.GetValue(k); has {
			backup[k] = value
		}
		s.state.SetValue(k, v)
	}
	return func() {
		for k, v := range backup {
			s.state.SetValue(k, v)
		}
	}
}

//NewRequest creates a new request for service and action
func (c *Context) NewRequest(serviceName, action string) (interface{}, error) {
	service, err := c.Service(serviceName)
	if err != nil {
		return nil, err
	}
	route, err := service.Route(action)
	if err != nil {
		return nil, err
	}
	return route.RequestProvider(), nil
}

//AsRequest converts a source map into request for provided service and action.
func (c *Context) AsRequest(serviceName, action string, source map[string]interface{}) (request interface{}, err error) {
	if request, err = c.NewRequest(serviceName, action); err != nil {
		return request, err
	}
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("failed to create request, unable to cast %v into %T, %v", source, request, r)
		}
	}()
	expanded := c.state.Expand(source)
	source = toolbox.AsMap(expanded)
	err = toolbox.DefaultConverter.AssignConverted(request, source)
	return request, err
}

//Close closes this context, it executes all deferred function and set closed flag.
func (c *Context) Close() {
	atomic.StoreInt32(&c.closed, 1)
	for _, context := range c.cloned {
		context.Close()
	}
	for _, function := range c.Deffer() {
		function()
	}
}

//MakeAsyncSafe makes this contex async safe

func (c *Context) MakeAsyncSafe() *msg.Events {
	for k := range c.AsyncUnsafeKeys {
		c.Context.Remove(k)
	}
	result := msg.NewEvents()
	c.Listener = result.AsListener()
	return result
}

/*
NewDefaultState returns a new default state.
It comes with the following registered keys:
	* rand - random int64
	*  date -  current date formatted as yyyy-MM-dd
	* time - current time formatted as yyyy-MM-dd hh:mm:ss
	* ts - current timestamp formatted  as yyyyMMddhhmmSSS
	* timestamp.yesterday - timestamp in ms
	* timestamp.now - timestamp in ms
	* timestamp.tomorrow - timestamp in ms
	* tmpDir - temp directory
	* uuid.next - generate unique id
	* uuid.Get - returns previously generated unique id, or generate new
	*.end.XXX where XXX is the ID of the env variable to return
	* all UFD registry functions
*/
func NewDefaultState() data.Map {
	var result = data.NewMap()
	var now = time.Now()
	source := rand.NewSource(now.UnixNano())
	result.Put("rand", source.Int63())
	result.Put("date", now.Format(toolbox.DateFormatToLayout("yyyy-MM-dd")))
	result.Put("time", now.Format(toolbox.DateFormatToLayout("yyyy-MM-dd hh:mm:ss")))
	result.Put("ts", now.Format(toolbox.DateFormatToLayout("yyyyMMddhhmmSSS")))

	result.Put("tmpDir", func(key string) interface{} {
		tempPath := path.Join(os.TempDir(), key)
		exec.Command("mkdir -p " + tempPath)
		return tempPath
	})

	var cachedUUID uuid.UUID
	result.Put("uuid", func(key string) interface{} {
		if key == "next" {
			var err error
			cachedUUID, err = uuid.NewV4()
			if err != nil {
				return ""
			}
		}
		if len(cachedUUID) > 0 {
			return cachedUUID.String()
		}
		return ""
	})

	result.Put("timestamp", func(key string) interface{} {
		var timeDiffProvider = toolbox.NewTimeDiffProvider()
		switch key {
		case "now":
			result, _ := timeDiffProvider.Get(nil, "now", 0, "day", "timestamp")
			return result
		case "tomorrow":
			result, _ := timeDiffProvider.Get(nil, "now", 1, "day", "timestamp")
			return result
		case "yesterday":
			result, _ := timeDiffProvider.Get(nil, "now", -1, "day", "timestamp")
			return result
		}
		return nil
	})

	result.Put("unix", func(key string) interface{} {
		var timeDiffProvider = toolbox.NewTimeDiffProvider()
		switch key {
		case "now":
			result, _ := timeDiffProvider.Get(nil, "now", 0, "day", "unix")
			return result
		case "tomorrow":
			result, _ := timeDiffProvider.Get(nil, "now", 1, "day", "unix")
			return result
		case "yesterday":
			result, _ := timeDiffProvider.Get(nil, "now", -1, "day", "unix")
			return result
		case "hourbefore":
			result, _ := timeDiffProvider.Get(nil, "now", -1, "hour", "unix")
			return result
		case "2hourbefore":
			result, _ := timeDiffProvider.Get(nil, "now", -2, "hour", "unix")
			return result
		case "hourlater":
			result, _ := timeDiffProvider.Get(nil, "now", 1, "hour", "unix")
			return result
		case "2hourlater":
			result, _ := timeDiffProvider.Get(nil, "now", 2, "hour", "unix")
			return result
		}
		return nil
	})

	result.Put("env", func(key string) interface{} {
		return os.Getenv(key)
	})

	for k, v := range UdfRegistry {
		result.Put(k, v)
	}
	return result
}
<|endoftext|>


<commit_before>package plugin

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os/exec"
	"time"

	"github.com/intelsdilabs/pulse/pkg/logger"
)

const (
	// enums for different waiting states
	pluginKilled      waitSignal = iota // plugin was killed
	pluginTimeout                       // plugin timed out
	pluginResponseOk                    // plugin response received (valid)
	pluginResponseBad                   // plugin response received (invalid)
)

// A plugin that is executable as a forked process on *Linux.
type ExecutablePlugin struct {
	cmd    *exec.Cmd
	stdout io.Reader
	args   Arg
}

// A interface representing an executable plugin.
type pluginExecutor interface {
	Kill() error
	WaitForExit() error
	ResponseReader() io.Reader
}

type waitSignal int

type waitSignalValue struct {
	Signal   waitSignal
	Response *Response
	Error    *error
}

// Starts the plugin and returns error if one occurred. This is non blocking.
func (e *ExecutablePlugin) Start() error {
	e.connectStdErrToLogs()
	return e.cmd.Start()
}

// connectStdErrToLogs connects plugin stderr
// and sends them to logs
// it's not required to stop this goroutine explicilty because it stops when cmd stops
// on the plugin side you should redirect all logs (warnings, errors) to stderr - panic goes there by default
func (e *ExecutablePlugin) connectStdErrToLogs() {

	// ignore the error because we
	stderr, err := e.cmd.StderrPipe()
	if err != nil {
		logger.Errorf("execution.stderr - error when connecting to executablePlugin %s: %s", e.cmd.Path, err)
	}
	go func() {
		scanner := bufio.NewScanner(stderr)
		// this automaticly ends when EOR received eg. process is killed/stoped
		for scanner.Scan() {
			logger.Debug("execution.stderr", scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			logger.Debug("execution.stderr", "got error when reading:", err.Error())
		}
	}()
}

// Kills the plugin and returns error if one occurred. This is blocking.
func (e *ExecutablePlugin) Kill() error {
	return e.cmd.Process.Kill()
}

// Waits for plugin to halt. If error is returned then plugin stopped with error. If not plugin stopped safely.
func (e *ExecutablePlugin) WaitForExit() error {
	return e.cmd.Wait()
}

// The STDOUT pipe for the plugin as io.Reader. Use to read from plugin process STDOUT.
func (e *ExecutablePlugin) ResponseReader() io.Reader {
	return e.stdout
}

// Initialize a new ExecutablePlugin from path to executable and daemon mode (true or false)
func NewExecutablePlugin(a Arg, path string) (*ExecutablePlugin, error) {
	jsonArgs, err := json.Marshal(a)
	if err != nil {
		return nil, err
	}
	// Init the cmd
	cmd := new(exec.Cmd)
	cmd.Path = path
	cmd.Args = []string{path, string(jsonArgs)}
	// Link the stdout for response reading
	stdout, err2 := cmd.StdoutPipe()
	if err2 != nil {
		return nil, err2
	}
	// Init the ExecutablePlugin and return
	ePlugin := new(ExecutablePlugin)
	ePlugin.cmd = cmd
	ePlugin.stdout = stdout
	ePlugin.args = a

	return ePlugin, nil
}

// Waits for a plugin response from a started plugin
func (e *ExecutablePlugin) WaitForResponse(timeout time.Duration) (*Response, error) {
	r, err := waitHandling(e, timeout)
	return r, err
}

// Private method which handles behvaior for wait for response for daemon and non-daemon modes.
func waitHandling(p pluginExecutor, timeout time.Duration) (*Response, error) {
	// disable to turn debug logging back on
	log.SetOutput(ioutil.Discard)
	/*
		Bit of complex behavior so some notes:
			A. We need to wait for three scenarios depending on the daemon setting
					1)	plugin is killed (like a safe exit in non-daemon)
						causing WaitForExit to fire
					2) 	plugin timeout fires calling Kill() and causing
						WaitForExit to fire
					3)	A response is returned before either 1 or 2 occur

				notes:
					*	In daemon mode (daemon == true) we want to wait until (1) or
						(2 then 1) or (3) occurs and stop waiting right after.
					*	In non-daemon mode (daemon == false) we want to return on (1)
						or (2 then 1) regardless of whether (3) occurs before or after.

			B. We will start three go routines to handle
					1)	waiting for timeout, on timeout we signal timeout and then
						kill plugin
					2)	wait for exit, also known as wait for kill, on kill we fire
						proper code to waitChannel
					3)	wait for response, on response we fire proper code to waitChannel

			C. The wait behavior loops collecting
					1)	timeout signal, this is used to mark exit by timeout
					2)	killed signal, signal the plugin has stopped - this exits
						the loop for all scenarios
					3)	response received, signal the plugin has responded - this exits
						the loop if daemon == true, otherwise waits for (2)
					4)	response received but corrupt
	*/

	// wait channel
	waitChannel := make(chan waitSignalValue, 3)

	// send timeout signal to our channel on timeout
	log.Println("timeout chan start")
	go waitForPluginTimeout(timeout, p, waitChannel)

	// send response received signal to our channel on response
	log.Println("response chan start")
	go waitForResponseFromPlugin(p.ResponseReader(), waitChannel)

	// send killed plugin signal to our channel on kill
	log.Println("kill chan start")
	go waitForKilledPlugin(p, waitChannel)

	// flag to indicate a timeout occurred
	var timeoutFlag bool
	// error value indicating a bad response was found
	var errResponse *error
	// var holding a good response (or nil if none was returned)
	var response *Response
	// Loop to wait for signals and return
	for {
		w := <-waitChannel
		switch w.Signal {
		case pluginTimeout: // plugin timeout signal received
			log.Println("plugin timeout signal received")
			// If timeout received after response we are ok with it and
			// don't need to flip the timeout flag.
			fmt.Println(response)
			if response == nil {
				log.Println("timeout flag flipped")
				// We got a timeout without getting a response
				// set the flag
				timeoutFlag = true
				// Kill the plugin. This will eventually trigger a kill signal.
				p.Kill()
				break
			}
			log.Println("timeout flag ignored because of response")

		case pluginKilled: // plugin killed signal received
			log.Println("plugin kill signal received")
			// We check a few scenarios and return based on how things worked out to this point
			// 1) If a bad response was received we return signalling this with an error (fail)
			if errResponse != nil {
				log.Println("returning with error (bad response)")
				return nil, *errResponse
			}
			// 2) If a timeout occurred we return that as error (fail)
			if timeoutFlag {
				log.Println("returning with error (timeout)")
				return nil, errors.New("timeout waiting for response")
			}
			// 3) If a good response was returned we return that with no error (success)
			if response != nil {
				log.Println("returning with response (after wait for kill)")
				return response, nil
			}
			// 4) otherwise we return no response and an error that no response was received (fail)
			log.Println("returning with error (killed without response)")
			// The kill could have been without error so we check if ExitError was returned and return
			// our own if not.
			if *w.Error != nil {
				return nil, *w.Error
			} else {
				return nil, errors.New("plugin died without sending response")
			}

		case pluginResponseOk: // plugin response (valid) signal received
			log.Println("plugin response (ok) received")
			// If in daemon mode we can return now (succes) since the plugin will continue to run
			// if not we let the loop continue (to wait for kill)
			response = w.Response
			return response, nil

		case pluginResponseBad: // plugin response (invalid) signal received
			log.Println("plugin response (bad) received")
			// A bad response is end of game in all scerarios and indictive of an unhealthy or unsupported plugin
			// We save the response bad error var (for handling later on plugin kill)
			errResponse = w.Error
		}
	}
}

func waitForPluginTimeout(timeout time.Duration, p pluginExecutor, waitChannel chan waitSignalValue) {
	// sleep for timeout duration
	time.Sleep(timeout)
	// Check if waitChannel is closed. If it is we exit now.
	// Send out timeout signal, waiting method will still wait for exit caused by p.Kill
	// Because this channel is shared this ensures that the resulting kill signals the channel after
	// the response has already queued across it.
	waitChannel <- waitSignalValue{Signal: pluginTimeout}
}

func waitForResponseFromPlugin(r io.Reader, waitChannel chan waitSignalValue) {
	scanner := bufio.NewScanner(r)
	resp := new(Response)
	// scan until we get a response or reader is closed
	for scanner.Scan() {
		// Get bytes
		b := scanner.Bytes()
		// attempt to unmarshall into struct
		err := json.Unmarshal(b, resp)
		if err != nil {
			log.Println("JSON error in response: " + err.Error())
			log.Printf("response: \"%s\"\n", string(b))
			e := errors.New("JSONError - " + err.Error())
			// send plugin response received but bad
			waitChannel <- waitSignalValue{Signal: pluginResponseBad, Error: &e}
			// exit function
			return
		}
		// send plugin response received (valid)
		waitChannel <- waitSignalValue{Signal: pluginResponseOk, Response: resp}
		// exit function
		return
	}
}

func waitForKilledPlugin(p pluginExecutor, waitChannel chan waitSignalValue) {
	// simply wait for this to return (blocking method)
	// TODO, refactor not to block. In daemon mode this would hang for the life of process.
	// ideally this should check if running or waitChannel closed and then exit on either.
	e := p.WaitForExit()
	time.Sleep(time.Millisecond * 100)
	// send signal
	waitChannel <- waitSignalValue{Signal: pluginKilled, Error: &e}
}
<commit_msg>This commit adding some logging redirection that was not part of the design. Rolling it back because this is just spamming false debug messages and not providing value.<commit_after>package plugin

import (
	"bufio"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os/exec"
	"time"
)

const (
	// enums for different waiting states
	pluginKilled      waitSignal = iota // plugin was killed
	pluginTimeout                       // plugin timed out
	pluginResponseOk                    // plugin response received (valid)
	pluginResponseBad                   // plugin response received (invalid)
)

// A plugin that is executable as a forked process on *Linux.
type ExecutablePlugin struct {
	cmd    *exec.Cmd
	stdout io.Reader
	args   Arg
}

// A interface representing an executable plugin.
type pluginExecutor interface {
	Kill() error
	WaitForExit() error
	ResponseReader() io.Reader
}

type waitSignal int

type waitSignalValue struct {
	Signal   waitSignal
	Response *Response
	Error    *error
}

// Starts the plugin and returns error if one occurred. This is non blocking.
func (e *ExecutablePlugin) Start() error {
	return e.cmd.Start()
}

// Kills the plugin and returns error if one occurred. This is blocking.
func (e *ExecutablePlugin) Kill() error {
	return e.cmd.Process.Kill()
}

// Waits for plugin to halt. If error is returned then plugin stopped with error. If not plugin stopped safely.
func (e *ExecutablePlugin) WaitForExit() error {
	return e.cmd.Wait()
}

// The STDOUT pipe for the plugin as io.Reader. Use to read from plugin process STDOUT.
func (e *ExecutablePlugin) ResponseReader() io.Reader {
	return e.stdout
}

// Initialize a new ExecutablePlugin from path to executable and daemon mode (true or false)
func NewExecutablePlugin(a Arg, path string) (*ExecutablePlugin, error) {
	jsonArgs, err := json.Marshal(a)
	if err != nil {
		return nil, err
	}
	// Init the cmd
	cmd := new(exec.Cmd)
	cmd.Path = path
	cmd.Args = []string{path, string(jsonArgs)}
	// Link the stdout for response reading
	stdout, err2 := cmd.StdoutPipe()
	if err2 != nil {
		return nil, err2
	}
	// Init the ExecutablePlugin and return
	ePlugin := new(ExecutablePlugin)
	ePlugin.cmd = cmd
	ePlugin.stdout = stdout
	ePlugin.args = a

	return ePlugin, nil
}

// Waits for a plugin response from a started plugin
func (e *ExecutablePlugin) WaitForResponse(timeout time.Duration) (*Response, error) {
	r, err := waitHandling(e, timeout)
	return r, err
}

// Private method which handles behvaior for wait for response for daemon and non-daemon modes.
func waitHandling(p pluginExecutor, timeout time.Duration) (*Response, error) {
	// disable to turn debug logging back on
	log.SetOutput(ioutil.Discard)
	/*
		Bit of complex behavior so some notes:
			A. We need to wait for three scenarios depending on the daemon setting
					1)	plugin is killed (like a safe exit in non-daemon)
						causing WaitForExit to fire
					2) 	plugin timeout fires calling Kill() and causing
						WaitForExit to fire
					3)	A response is returned before either 1 or 2 occur

				notes:
					*	In daemon mode (daemon == true) we want to wait until (1) or
						(2 then 1) or (3) occurs and stop waiting right after.
					*	In non-daemon mode (daemon == false) we want to return on (1)
						or (2 then 1) regardless of whether (3) occurs before or after.

			B. We will start three go routines to handle
					1)	waiting for timeout, on timeout we signal timeout and then
						kill plugin
					2)	wait for exit, also known as wait for kill, on kill we fire
						proper code to waitChannel
					3)	wait for response, on response we fire proper code to waitChannel

			C. The wait behavior loops collecting
					1)	timeout signal, this is used to mark exit by timeout
					2)	killed signal, signal the plugin has stopped - this exits
						the loop for all scenarios
					3)	response received, signal the plugin has responded - this exits
						the loop if daemon == true, otherwise waits for (2)
					4)	response received but corrupt
	*/

	// wait channel
	waitChannel := make(chan waitSignalValue, 3)

	// send timeout signal to our channel on timeout
	log.Println("timeout chan start")
	go waitForPluginTimeout(timeout, p, waitChannel)

	// send response received signal to our channel on response
	log.Println("response chan start")
	go waitForResponseFromPlugin(p.ResponseReader(), waitChannel)

	// send killed plugin signal to our channel on kill
	log.Println("kill chan start")
	go waitForKilledPlugin(p, waitChannel)

	// flag to indicate a timeout occurred
	var timeoutFlag bool
	// error value indicating a bad response was found
	var errResponse *error
	// var holding a good response (or nil if none was returned)
	var response *Response
	// Loop to wait for signals and return
	for {
		w := <-waitChannel
		switch w.Signal {
		case pluginTimeout: // plugin timeout signal received
			log.Println("plugin timeout signal received")
			// If timeout received after response we are ok with it and
			// don't need to flip the timeout flag.
			fmt.Println(response)
			if response == nil {
				log.Println("timeout flag flipped")
				// We got a timeout without getting a response
				// set the flag
				timeoutFlag = true
				// Kill the plugin. This will eventually trigger a kill signal.
				p.Kill()
				break
			}
			log.Println("timeout flag ignored because of response")

		case pluginKilled: // plugin killed signal received
			log.Println("plugin kill signal received")
			// We check a few scenarios and return based on how things worked out to this point
			// 1) If a bad response was received we return signalling this with an error (fail)
			if errResponse != nil {
				log.Println("returning with error (bad response)")
				return nil, *errResponse
			}
			// 2) If a timeout occurred we return that as error (fail)
			if timeoutFlag {
				log.Println("returning with error (timeout)")
				return nil, errors.New("timeout waiting for response")
			}
			// 3) If a good response was returned we return that with no error (success)
			if response != nil {
				log.Println("returning with response (after wait for kill)")
				return response, nil
			}
			// 4) otherwise we return no response and an error that no response was received (fail)
			log.Println("returning with error (killed without response)")
			// The kill could have been without error so we check if ExitError was returned and return
			// our own if not.
			if *w.Error != nil {
				return nil, *w.Error
			} else {
				return nil, errors.New("plugin died without sending response")
			}

		case pluginResponseOk: // plugin response (valid) signal received
			log.Println("plugin response (ok) received")
			// If in daemon mode we can return now (succes) since the plugin will continue to run
			// if not we let the loop continue (to wait for kill)
			response = w.Response
			return response, nil

		case pluginResponseBad: // plugin response (invalid) signal received
			log.Println("plugin response (bad) received")
			// A bad response is end of game in all scerarios and indictive of an unhealthy or unsupported plugin
			// We save the response bad error var (for handling later on plugin kill)
			errResponse = w.Error
		}
	}
}

func waitForPluginTimeout(timeout time.Duration, p pluginExecutor, waitChannel chan waitSignalValue) {
	// sleep for timeout duration
	time.Sleep(timeout)
	// Check if waitChannel is closed. If it is we exit now.
	// Send out timeout signal, waiting method will still wait for exit caused by p.Kill
	// Because this channel is shared this ensures that the resulting kill signals the channel after
	// the response has already queued across it.
	waitChannel <- waitSignalValue{Signal: pluginTimeout}
}

func waitForResponseFromPlugin(r io.Reader, waitChannel chan waitSignalValue) {
	scanner := bufio.NewScanner(r)
	resp := new(Response)
	// scan until we get a response or reader is closed
	for scanner.Scan() {
		// Get bytes
		b := scanner.Bytes()
		// attempt to unmarshall into struct
		err := json.Unmarshal(b, resp)
		if err != nil {
			log.Println("JSON error in response: " + err.Error())
			log.Printf("response: \"%s\"\n", string(b))
			e := errors.New("JSONError - " + err.Error())
			// send plugin response received but bad
			waitChannel <- waitSignalValue{Signal: pluginResponseBad, Error: &e}
			// exit function
			return
		}
		// send plugin response received (valid)
		waitChannel <- waitSignalValue{Signal: pluginResponseOk, Response: resp}
		// exit function
		return
	}
}

func waitForKilledPlugin(p pluginExecutor, waitChannel chan waitSignalValue) {
	// simply wait for this to return (blocking method)
	// TODO, refactor not to block. In daemon mode this would hang for the life of process.
	// ideally this should check if running or waitChannel closed and then exit on either.
	e := p.WaitForExit()
	time.Sleep(time.Millisecond * 100)
	// send signal
	waitChannel <- waitSignalValue{Signal: pluginKilled, Error: &e}
}
<|endoftext|>


<commit_before>package hop

import (
	"context"
)

type contextKey string

const (
	keyPathParams  contextKey = "hop.pathparams"
	keyPathInt                = "hop.path.int."
	keyPathString             = "hop.path.str."
	keyContentType contextKey = "hop.content-type"
	keyCredential  contextKey = "hop.credential"
	keyBodyJSON    contextKey = "hop.body.json"
)

func setPathParams(c context.Context, params map[string]string) context.Context {
	return context.WithValue(c, keyPathParams, params)
}

// PathParams returns Path parameters
func PathParams(c context.Context) map[string]string {
	if params, ok := c.Value(keyPathParams).(map[string]string); ok {
		return params
	}
	return map[string]string{}
}

func setPathInt(c context.Context, key string, value int) context.Context {
	return context.WithValue(c, contextKey(keyPathInt+key), value)
}

// PathInt returns path parameter as int value. If value is not int,
// returns 0 instread.
func PathInt(c context.Context, key string) int {
	if v, ok := c.Value(contextKey(keyPathInt + key)).(int); ok {
		return v
	}
	return 0
}

func setPathString(c context.Context, key, value string) context.Context {
	return context.WithValue(c, contextKey(keyPathString+key), value)
}

// PathString returns path parameter as int value. If value is not string,
// returns "" instread.
func PathString(c context.Context, key string) string {
	if v, ok := c.Value(contextKey(keyPathString + key)).(string); ok {
		return v
	}
	return ""
}

func setContentType(c context.Context, contentType string) context.Context {
	return context.WithValue(c, keyContentType, contentType)
}

// ContentType returns Content-Type header
func ContentType(c context.Context) string {
	if v, ok := c.Value(keyContentType).(string); ok {
		return v
	}
	return ""
}

func setCredential(c context.Context, cred Cred) context.Context {
	return context.WithValue(c, keyCredential, cred)
}

// Credential returns TokenType and Token
func Credential(c context.Context) Cred {
	if v, ok := c.Value(keyCredential).(Cred); ok {
		return v
	}
	return Cred{}
}

func setBodyJSON(c context.Context, body map[string]interface{}) context.Context {
	return context.WithValue(c, keyBodyJSON, body)
}

// BodyJSON returns request body as JSON(map[string]interface{}) format
func BodyJSON(c context.Context) map[string]interface{} {
	if v, ok := c.Value(keyBodyJSON).(map[string]interface{}); ok {
		return v
	}
	return map[string]interface{}{}
}
<commit_msg>Gets value from PathParams<commit_after>package hop

import (
	"context"
	"strconv"
)

type contextKey string

const (
	keyPathParams  contextKey = "hop.pathparams"
	keyPathInt                = "hop.path.int."
	keyPathString             = "hop.path.str."
	keyContentType contextKey = "hop.content-type"
	keyCredential  contextKey = "hop.credential"
	keyBodyJSON    contextKey = "hop.body.json"
)

func setPathParams(c context.Context, params map[string]string) context.Context {
	return context.WithValue(c, keyPathParams, params)
}

// PathParams returns Path parameters
func PathParams(c context.Context) map[string]string {
	if params, ok := c.Value(keyPathParams).(map[string]string); ok {
		return params
	}
	return map[string]string{}
}

func setPathInt(c context.Context, key string, value int) context.Context {
	return context.WithValue(c, contextKey(keyPathInt+key), value)
}

// PathInt returns path parameter as int value. If value is not int,
// returns 0 instread.
func PathInt(c context.Context, key string) int {
	valStr := PathString(c, key)
	val, err := strconv.Atoi(valStr)
	if err != nil {
		return 0
	}
	return val
}

func setPathString(c context.Context, key, value string) context.Context {
	return context.WithValue(c, contextKey(keyPathString+key), value)
}

// PathString returns path parameter as int value. If value is not string,
// returns "" instread.
func PathString(c context.Context, key string) string {
	params := PathParams(c)
	if v, ok := params[key]; ok {
		return v
	}
	return ""
}

func setContentType(c context.Context, contentType string) context.Context {
	return context.WithValue(c, keyContentType, contentType)
}

// ContentType returns Content-Type header
func ContentType(c context.Context) string {
	if v, ok := c.Value(keyContentType).(string); ok {
		return v
	}
	return ""
}

func setCredential(c context.Context, cred Cred) context.Context {
	return context.WithValue(c, keyCredential, cred)
}

// Credential returns TokenType and Token
func Credential(c context.Context) Cred {
	if v, ok := c.Value(keyCredential).(Cred); ok {
		return v
	}
	return Cred{}
}

func setBodyJSON(c context.Context, body map[string]interface{}) context.Context {
	return context.WithValue(c, keyBodyJSON, body)
}

// BodyJSON returns request body as JSON(map[string]interface{}) format
func BodyJSON(c context.Context) map[string]interface{} {
	if v, ok := c.Value(keyBodyJSON).(map[string]interface{}); ok {
		return v
	}
	return map[string]interface{}{}
}
<|endoftext|>