<commit_before>package lib

import (
	"context"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat/stats"
	"github.com/robertkrimen/otto"
	"gopkg.in/guregu/null.v3"
	"strconv"
	"sync"
	"time"
)

const (
	TickRate          = 1 * time.Millisecond
	ThresholdTickRate = 2 * time.Second
	ShutdownTimeout   = 10 * time.Second
)

var (
	MetricVUs    = &stats.Metric{Name: "vus", Type: stats.Gauge}
	MetricVUsMax = &stats.Metric{Name: "vus_max", Type: stats.Gauge}
	MetricErrors = &stats.Metric{Name: "errors", Type: stats.Counter}

	ErrTooManyVUs = errors.New("More VUs than the maximum requested")
	ErrMaxTooLow  = errors.New("Can't lower max below current VU count")

	// Special error used to taint a test, without printing an error.
	ErrVUWantsTaint = errors.New("[ErrVUWantsTaint is never logged]")
)

type vuEntry struct {
	VU     VU
	Buffer []stats.Sample
	Cancel context.CancelFunc
}

type Engine struct {
	Runner    Runner
	Status    Status
	Stages    []Stage
	Collector stats.Collector
	Pause     sync.WaitGroup
	Metrics   map[*stats.Metric]stats.Sink

	thresholds  map[string][]Threshold
	thresholdVM *otto.Otto

	ctx    context.Context
	vus    []*vuEntry
	nextID int64

	vuMutex   sync.Mutex
	waitGroup sync.WaitGroup
}

func NewEngine(r Runner) (*Engine, error) {
	e := &Engine{
		Runner: r,
		Status: Status{
			Running: null.BoolFrom(false),
			Tainted: null.BoolFrom(false),
			VUs:     null.IntFrom(0),
			VUsMax:  null.IntFrom(0),
			AtTime:  null.IntFrom(0),
		},
		Metrics:     make(map[*stats.Metric]stats.Sink),
		thresholds:  make(map[string][]Threshold),
		thresholdVM: otto.New(),
	}
	e.Pause.Add(1)

	return e, nil
}

func (e *Engine) Apply(opts Options) error {
	if opts.Run.Valid {
		e.SetRunning(opts.Run.Bool)
	}
	if opts.VUsMax.Valid {
		if err := e.SetMaxVUs(opts.VUs.Int64); err != nil {
			return err
		}
	}
	if opts.VUs.Valid {
		if !opts.VUsMax.Valid {
			if err := e.SetMaxVUs(opts.VUs.Int64); err != nil {
				return err
			}
		}
		if err := e.SetVUs(opts.VUs.Int64); err != nil {
			return err
		}
	}
	if opts.Duration.Valid {
		duration, err := time.ParseDuration(opts.Duration.String)
		if err != nil {
			return err
		}
		e.Stages = []Stage{Stage{Duration: null.IntFrom(int64(duration))}}
	}

	if opts.Quit.Valid {
		e.Status.Quit = opts.Quit
	}
	if opts.QuitOnTaint.Valid {
		e.Status.QuitOnTaint = opts.QuitOnTaint
	}

	if opts.Thresholds != nil {
		e.thresholds = opts.Thresholds

		// Make sure all scripts are compiled!
		for m, scripts := range e.thresholds {
			for i, script := range scripts {
				if script.Script != nil {
					continue
				}

				s, err := e.thresholdVM.Compile(fmt.Sprintf("threshold$%s:%i", m, i), script.Source)
				if err != nil {
					return err
				}
				script.Script = s
				scripts[i] = script
			}
		}
	}

	return nil
}

func (e *Engine) Run(ctx context.Context, opts Options) error {
	subctx, cancel := context.WithCancel(context.Background())
	e.ctx = subctx
	e.nextID = 1

	e.Apply(opts)

	if e.Collector != nil {
		e.waitGroup.Add(1)
		go func() {
			e.Collector.Run(subctx)
			log.Debug("Engine: Collector shut down")
			e.waitGroup.Done()
		}()
	} else {
		log.Debug("Engine: No Collector")
	}

	e.waitGroup.Add(1)
	go func() {
		e.runThresholds(subctx)
		log.Debug("Engine: Thresholds shut down")
		e.waitGroup.Done()
	}()

	e.consumeEngineStats()

	ticker := time.NewTicker(TickRate)
	lastTick := time.Now()

loop:
	for {
		select {
		case now := <-ticker.C:
			timeDelta := now.Sub(lastTick)
			e.Status.AtTime.Int64 += int64(timeDelta)
			lastTick = now

			stage, left, ok := StageAt(e.Stages, time.Duration(e.Status.AtTime.Int64))
			if stage.StartVUs.Valid && stage.EndVUs.Valid {
				progress := (float64(stage.Duration.Int64-int64(left)) / float64(stage.Duration.Int64))
				vus := Lerp(stage.StartVUs.Int64, stage.EndVUs.Int64, progress)
				e.SetVUs(vus)
			}

			for _, vu := range e.vus {
				buffer := vu.Buffer
				if buffer == nil {
					continue
				}
				vu.Buffer = nil
				e.consumeBuffer(buffer)
			}

			if !ok {
				e.SetRunning(false)

				if e.Status.Quit.Bool {
					break loop
				} else {
					log.Info("Test finished, press Ctrl+C to exit")
					<-ctx.Done()
					break loop
				}
			}

			if e.Status.QuitOnTaint.Bool && e.Status.Tainted.Bool {
				log.Warn("Test tainted, ending early...")
				break loop
			}

			e.consumeEngineStats()
		case <-ctx.Done():
			break loop
		}
	}

	e.SetRunning(false)
	e.SetVUs(0)
	e.SetMaxVUs(0)
	e.consumeEngineStats()

	cancel()

	log.Debug("Engine: Waiting for subsystem shutdown...")

	done := make(chan interface{})
	go func() {
		e.waitGroup.Wait()
		close(done)
	}()
	timeout := time.After(ShutdownTimeout)
	select {
	case <-done:
	case <-timeout:
		log.Warn("VUs took too long to finish, shutting down anyways")
	}

	return nil
}

func (e *Engine) IsRunning() bool {
	return e.ctx != nil
}

func (e *Engine) TotalTime() (total time.Duration, finite bool) {
	for _, stage := range e.Stages {
		if stage.Duration.Valid {
			total += time.Duration(stage.Duration.Int64)
			finite = true
		}
	}
	return total, finite
}

func (e *Engine) SetRunning(running bool) {
	if running && !e.Status.Running.Bool {
		e.Pause.Done()
		log.Debug("Engine Unpaused")
	} else if !running && e.Status.Running.Bool {
		e.Pause.Add(1)
		log.Debug("Engine Paused")
	}
	e.Status.Running.Bool = running
}

func (e *Engine) SetVUs(v int64) error {
	if e.Status.VUs.Int64 == v {
		return nil
	}

	log.WithFields(log.Fields{"from": e.Status.VUs.Int64, "to": v}).Debug("Setting VUs")

	e.vuMutex.Lock()
	defer e.vuMutex.Unlock()

	if v > e.Status.VUsMax.Int64 {
		return ErrTooManyVUs
	}

	current := e.Status.VUs.Int64
	for i := current; i < v; i++ {
		entry := e.vus[i]
		if entry.Cancel != nil {
			panic(errors.New("ATTEMPTED TO RESCHEDULE RUNNING VU"))
		}

		ctx, cancel := context.WithCancel(e.ctx)
		entry.Cancel = cancel

		if err := entry.VU.Reconfigure(e.nextID); err != nil {
			return err
		}
		e.waitGroup.Add(1)
		go func() {
			id := e.nextID
			e.runVU(ctx, id, entry)
			log.WithField("id", id).Debug("Engine: VU terminated")
			e.waitGroup.Done()
		}()
		e.nextID++
	}
	for i := current - 1; i >= v; i-- {
		entry := e.vus[i]
		entry.Cancel()
		entry.Cancel = nil
	}

	e.Status.VUs.Int64 = v
	return nil
}

func (e *Engine) SetMaxVUs(v int64) error {
	if e.Status.VUsMax.Int64 == v {
		return nil
	}

	log.WithFields(log.Fields{"from": e.Status.VUsMax.Int64, "to": v}).Debug("Setting Max VUs")

	e.vuMutex.Lock()
	defer e.vuMutex.Unlock()

	if v < e.Status.VUs.Int64 {
		return ErrMaxTooLow
	}

	current := e.Status.VUsMax.Int64
	if v > current {
		vus := e.vus
		for i := current; i < v; i++ {
			vu, err := e.Runner.NewVU()
			if err != nil {
				return err
			}
			entry := &vuEntry{VU: vu}
			vus = append(vus, entry)
		}
		e.vus = vus
	} else if v < current {
		e.vus = e.vus[:v]
	}

	e.Status.VUsMax.Int64 = v
	return nil
}

func (e *Engine) runVU(ctx context.Context, id int64, vu *vuEntry) {
	idString := strconv.FormatInt(id, 10)

waitForPause:
	e.Pause.Wait()

	for {
		samples, err := vu.VU.RunOnce(ctx)

		// If the context is cancelled, the iteration is likely to produce erroneous output
		// due to cancelled HTTP requests and whatnot. Discard output from such runs.
		select {
		case <-ctx.Done():
			return
		default:
		}

		if err != nil {
			e.Status.Tainted.Bool = true

			if err != ErrVUWantsTaint {
				if s, ok := err.(fmt.Stringer); ok {
					log.Error(s.String())
				} else {
					log.WithError(err).Error("Runtime Error")
				}

				samples = append(samples, stats.Sample{
					Metric: MetricErrors,
					Time:   time.Now(),
					Tags:   map[string]string{"vu": idString, "error": err.Error()},
					Value:  float64(1),
				})
			}
		}

		if samples != nil {
			buffer := vu.Buffer
			if buffer == nil {
				buffer = samples
			} else {
				buffer = append(buffer, samples...)
			}
			vu.Buffer = buffer
		}

		if !e.Status.Running.Bool {
			goto waitForPause
		}
	}
}

func (e *Engine) runThresholds(ctx context.Context) {
	ticker := time.NewTicker(ThresholdTickRate)
	for {
		select {
		case <-ticker.C:
			for m, sink := range e.Metrics {
				scripts, ok := e.thresholds[m.Name]
				if !ok {
					continue
				}

				sample := sink.Format()
				for key, value := range sample {
					if m.Contains == stats.Time {
						value = value / float64(time.Millisecond)
					}
					// log.WithFields(log.Fields{"k": key, "v": value}).Debug("setting threshold data")
					e.thresholdVM.Set(key, value)
				}

				taint := false
				for _, script := range scripts {
					v, err := e.thresholdVM.Run(script.Script)
					if err != nil {
						log.WithError(err).WithField("metric", m.Name).Error("Threshold Error")
						taint = true
						continue
					}
					// log.WithFields(log.Fields{"metric": m.Name, "v": v, "s": sample}).Debug("threshold tick")
					bV, err := v.ToBoolean()
					if err != nil {
						log.WithError(err).WithField("metric", m.Name).Error("Threshold result is invalid")
						taint = true
						continue
					}
					if !bV {
						taint = true
					}
				}

				for key, _ := range sample {
					e.thresholdVM.Set(key, otto.UndefinedValue())
				}

				if taint {
					m.Tainted = true
					e.Status.Tainted.Bool = true
				}
			}
		case <-ctx.Done():
			return
		}
	}
}

// Returns a value sink for a metric, created from the type if unavailable.
func (e *Engine) getSink(m *stats.Metric) stats.Sink {
	s, ok := e.Metrics[m]
	if !ok {
		switch m.Type {
		case stats.Counter:
			s = &stats.CounterSink{}
		case stats.Gauge:
			s = &stats.GaugeSink{}
		case stats.Trend:
			s = &stats.TrendSink{}
		case stats.Rate:
			s = &stats.RateSink{}
		}
		e.Metrics[m] = s
	}
	return s
}

func (e *Engine) consumeBuffer(buffer []stats.Sample) {
	for _, sample := range buffer {
		e.getSink(sample.Metric).Add(sample)
	}
	if e.Collector != nil {
		e.Collector.Collect(buffer)
	}
}

func (e *Engine) consumeEngineStats() {
	t := time.Now()
	e.consumeBuffer([]stats.Sample{
		stats.Sample{Metric: MetricVUs, Time: t, Value: float64(e.Status.VUs.Int64)},
		stats.Sample{Metric: MetricVUsMax, Time: t, Value: float64(e.Status.VUsMax.Int64)},
	})
}
<commit_msg>[fix] Unset ctx when shutting down, fixes IsRunning()<commit_after>package lib

import (
	"context"
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/loadimpact/speedboat/stats"
	"github.com/robertkrimen/otto"
	"gopkg.in/guregu/null.v3"
	"strconv"
	"sync"
	"time"
)

const (
	TickRate          = 1 * time.Millisecond
	ThresholdTickRate = 2 * time.Second
	ShutdownTimeout   = 10 * time.Second
)

var (
	MetricVUs    = &stats.Metric{Name: "vus", Type: stats.Gauge}
	MetricVUsMax = &stats.Metric{Name: "vus_max", Type: stats.Gauge}
	MetricErrors = &stats.Metric{Name: "errors", Type: stats.Counter}

	ErrTooManyVUs = errors.New("More VUs than the maximum requested")
	ErrMaxTooLow  = errors.New("Can't lower max below current VU count")

	// Special error used to taint a test, without printing an error.
	ErrVUWantsTaint = errors.New("[ErrVUWantsTaint is never logged]")
)

type vuEntry struct {
	VU     VU
	Buffer []stats.Sample
	Cancel context.CancelFunc
}

type Engine struct {
	Runner    Runner
	Status    Status
	Stages    []Stage
	Collector stats.Collector
	Pause     sync.WaitGroup
	Metrics   map[*stats.Metric]stats.Sink

	thresholds  map[string][]Threshold
	thresholdVM *otto.Otto

	ctx    context.Context
	vus    []*vuEntry
	nextID int64

	vuMutex   sync.Mutex
	waitGroup sync.WaitGroup
}

func NewEngine(r Runner) (*Engine, error) {
	e := &Engine{
		Runner: r,
		Status: Status{
			Running: null.BoolFrom(false),
			Tainted: null.BoolFrom(false),
			VUs:     null.IntFrom(0),
			VUsMax:  null.IntFrom(0),
			AtTime:  null.IntFrom(0),
		},
		Metrics:     make(map[*stats.Metric]stats.Sink),
		thresholds:  make(map[string][]Threshold),
		thresholdVM: otto.New(),
	}
	e.Pause.Add(1)

	return e, nil
}

func (e *Engine) Apply(opts Options) error {
	if opts.Run.Valid {
		e.SetRunning(opts.Run.Bool)
	}
	if opts.VUsMax.Valid {
		if err := e.SetMaxVUs(opts.VUs.Int64); err != nil {
			return err
		}
	}
	if opts.VUs.Valid {
		if !opts.VUsMax.Valid {
			if err := e.SetMaxVUs(opts.VUs.Int64); err != nil {
				return err
			}
		}
		if err := e.SetVUs(opts.VUs.Int64); err != nil {
			return err
		}
	}
	if opts.Duration.Valid {
		duration, err := time.ParseDuration(opts.Duration.String)
		if err != nil {
			return err
		}
		e.Stages = []Stage{Stage{Duration: null.IntFrom(int64(duration))}}
	}

	if opts.Quit.Valid {
		e.Status.Quit = opts.Quit
	}
	if opts.QuitOnTaint.Valid {
		e.Status.QuitOnTaint = opts.QuitOnTaint
	}

	if opts.Thresholds != nil {
		e.thresholds = opts.Thresholds

		// Make sure all scripts are compiled!
		for m, scripts := range e.thresholds {
			for i, script := range scripts {
				if script.Script != nil {
					continue
				}

				s, err := e.thresholdVM.Compile(fmt.Sprintf("threshold$%s:%i", m, i), script.Source)
				if err != nil {
					return err
				}
				script.Script = s
				scripts[i] = script
			}
		}
	}

	return nil
}

func (e *Engine) Run(ctx context.Context, opts Options) error {
	subctx, cancel := context.WithCancel(context.Background())
	e.ctx = subctx
	e.nextID = 1

	e.Apply(opts)

	if e.Collector != nil {
		e.waitGroup.Add(1)
		go func() {
			e.Collector.Run(subctx)
			log.Debug("Engine: Collector shut down")
			e.waitGroup.Done()
		}()
	} else {
		log.Debug("Engine: No Collector")
	}

	e.waitGroup.Add(1)
	go func() {
		e.runThresholds(subctx)
		log.Debug("Engine: Thresholds shut down")
		e.waitGroup.Done()
	}()

	e.consumeEngineStats()

	ticker := time.NewTicker(TickRate)
	lastTick := time.Now()

loop:
	for {
		select {
		case now := <-ticker.C:
			timeDelta := now.Sub(lastTick)
			e.Status.AtTime.Int64 += int64(timeDelta)
			lastTick = now

			stage, left, ok := StageAt(e.Stages, time.Duration(e.Status.AtTime.Int64))
			if stage.StartVUs.Valid && stage.EndVUs.Valid {
				progress := (float64(stage.Duration.Int64-int64(left)) / float64(stage.Duration.Int64))
				vus := Lerp(stage.StartVUs.Int64, stage.EndVUs.Int64, progress)
				e.SetVUs(vus)
			}

			for _, vu := range e.vus {
				buffer := vu.Buffer
				if buffer == nil {
					continue
				}
				vu.Buffer = nil
				e.consumeBuffer(buffer)
			}

			if !ok {
				e.SetRunning(false)

				if e.Status.Quit.Bool {
					break loop
				} else {
					log.Info("Test finished, press Ctrl+C to exit")
					<-ctx.Done()
					break loop
				}
			}

			if e.Status.QuitOnTaint.Bool && e.Status.Tainted.Bool {
				log.Warn("Test tainted, ending early...")
				break loop
			}

			e.consumeEngineStats()
		case <-ctx.Done():
			break loop
		}
	}

	e.SetRunning(false)
	e.SetVUs(0)
	e.SetMaxVUs(0)
	e.consumeEngineStats()

	e.ctx = nil
	cancel()

	log.Debug("Engine: Waiting for subsystem shutdown...")

	done := make(chan interface{})
	go func() {
		e.waitGroup.Wait()
		close(done)
	}()
	timeout := time.After(ShutdownTimeout)
	select {
	case <-done:
	case <-timeout:
		log.Warn("VUs took too long to finish, shutting down anyways")
	}

	return nil
}

func (e *Engine) IsRunning() bool {
	return e.ctx != nil
}

func (e *Engine) TotalTime() (total time.Duration, finite bool) {
	for _, stage := range e.Stages {
		if stage.Duration.Valid {
			total += time.Duration(stage.Duration.Int64)
			finite = true
		}
	}
	return total, finite
}

func (e *Engine) SetRunning(running bool) {
	if running && !e.Status.Running.Bool {
		e.Pause.Done()
		log.Debug("Engine Unpaused")
	} else if !running && e.Status.Running.Bool {
		e.Pause.Add(1)
		log.Debug("Engine Paused")
	}
	e.Status.Running.Bool = running
}

func (e *Engine) SetVUs(v int64) error {
	if e.Status.VUs.Int64 == v {
		return nil
	}

	log.WithFields(log.Fields{"from": e.Status.VUs.Int64, "to": v}).Debug("Setting VUs")

	e.vuMutex.Lock()
	defer e.vuMutex.Unlock()

	if v > e.Status.VUsMax.Int64 {
		return ErrTooManyVUs
	}

	current := e.Status.VUs.Int64
	for i := current; i < v; i++ {
		entry := e.vus[i]
		if entry.Cancel != nil {
			panic(errors.New("ATTEMPTED TO RESCHEDULE RUNNING VU"))
		}

		ctx, cancel := context.WithCancel(e.ctx)
		entry.Cancel = cancel

		if err := entry.VU.Reconfigure(e.nextID); err != nil {
			return err
		}
		e.waitGroup.Add(1)
		go func() {
			id := e.nextID
			e.runVU(ctx, id, entry)
			log.WithField("id", id).Debug("Engine: VU terminated")
			e.waitGroup.Done()
		}()
		e.nextID++
	}
	for i := current - 1; i >= v; i-- {
		entry := e.vus[i]
		entry.Cancel()
		entry.Cancel = nil
	}

	e.Status.VUs.Int64 = v
	return nil
}

func (e *Engine) SetMaxVUs(v int64) error {
	if e.Status.VUsMax.Int64 == v {
		return nil
	}

	log.WithFields(log.Fields{"from": e.Status.VUsMax.Int64, "to": v}).Debug("Setting Max VUs")

	e.vuMutex.Lock()
	defer e.vuMutex.Unlock()

	if v < e.Status.VUs.Int64 {
		return ErrMaxTooLow
	}

	current := e.Status.VUsMax.Int64
	if v > current {
		vus := e.vus
		for i := current; i < v; i++ {
			vu, err := e.Runner.NewVU()
			if err != nil {
				return err
			}
			entry := &vuEntry{VU: vu}
			vus = append(vus, entry)
		}
		e.vus = vus
	} else if v < current {
		e.vus = e.vus[:v]
	}

	e.Status.VUsMax.Int64 = v
	return nil
}

func (e *Engine) runVU(ctx context.Context, id int64, vu *vuEntry) {
	idString := strconv.FormatInt(id, 10)

waitForPause:
	e.Pause.Wait()

	for {
		samples, err := vu.VU.RunOnce(ctx)

		// If the context is cancelled, the iteration is likely to produce erroneous output
		// due to cancelled HTTP requests and whatnot. Discard output from such runs.
		select {
		case <-ctx.Done():
			return
		default:
		}

		if err != nil {
			e.Status.Tainted.Bool = true

			if err != ErrVUWantsTaint {
				if s, ok := err.(fmt.Stringer); ok {
					log.Error(s.String())
				} else {
					log.WithError(err).Error("Runtime Error")
				}

				samples = append(samples, stats.Sample{
					Metric: MetricErrors,
					Time:   time.Now(),
					Tags:   map[string]string{"vu": idString, "error": err.Error()},
					Value:  float64(1),
				})
			}
		}

		if samples != nil {
			buffer := vu.Buffer
			if buffer == nil {
				buffer = samples
			} else {
				buffer = append(buffer, samples...)
			}
			vu.Buffer = buffer
		}

		if !e.Status.Running.Bool {
			goto waitForPause
		}
	}
}

func (e *Engine) runThresholds(ctx context.Context) {
	ticker := time.NewTicker(ThresholdTickRate)
	for {
		select {
		case <-ticker.C:
			for m, sink := range e.Metrics {
				scripts, ok := e.thresholds[m.Name]
				if !ok {
					continue
				}

				sample := sink.Format()
				for key, value := range sample {
					if m.Contains == stats.Time {
						value = value / float64(time.Millisecond)
					}
					// log.WithFields(log.Fields{"k": key, "v": value}).Debug("setting threshold data")
					e.thresholdVM.Set(key, value)
				}

				taint := false
				for _, script := range scripts {
					v, err := e.thresholdVM.Run(script.Script)
					if err != nil {
						log.WithError(err).WithField("metric", m.Name).Error("Threshold Error")
						taint = true
						continue
					}
					// log.WithFields(log.Fields{"metric": m.Name, "v": v, "s": sample}).Debug("threshold tick")
					bV, err := v.ToBoolean()
					if err != nil {
						log.WithError(err).WithField("metric", m.Name).Error("Threshold result is invalid")
						taint = true
						continue
					}
					if !bV {
						taint = true
					}
				}

				for key, _ := range sample {
					e.thresholdVM.Set(key, otto.UndefinedValue())
				}

				if taint {
					m.Tainted = true
					e.Status.Tainted.Bool = true
				}
			}
		case <-ctx.Done():
			return
		}
	}
}

// Returns a value sink for a metric, created from the type if unavailable.
func (e *Engine) getSink(m *stats.Metric) stats.Sink {
	s, ok := e.Metrics[m]
	if !ok {
		switch m.Type {
		case stats.Counter:
			s = &stats.CounterSink{}
		case stats.Gauge:
			s = &stats.GaugeSink{}
		case stats.Trend:
			s = &stats.TrendSink{}
		case stats.Rate:
			s = &stats.RateSink{}
		}
		e.Metrics[m] = s
	}
	return s
}

func (e *Engine) consumeBuffer(buffer []stats.Sample) {
	for _, sample := range buffer {
		e.getSink(sample.Metric).Add(sample)
	}
	if e.Collector != nil {
		e.Collector.Collect(buffer)
	}
}

func (e *Engine) consumeEngineStats() {
	t := time.Now()
	e.consumeBuffer([]stats.Sample{
		stats.Sample{Metric: MetricVUs, Time: t, Value: float64(e.Status.VUs.Int64)},
		stats.Sample{Metric: MetricVUsMax, Time: t, Value: float64(e.Status.VUsMax.Int64)},
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"sync"
	"time"
)

func main() {
	var blockObj sync.WaitGroup
	blockObj.Add(1)
	go func() {
		time.Sleep(time.Second * 10)
		blockObj.Done()
	}()
	blockObj.Wait()
	fmt.Println(123)
}
<commit_msg>线程锁测试<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"sync"
	"syscall"
	"time"
)

func main() {
	var blockObj sync.WaitGroup
	blockObj.Add(1)
	go func() {
		time.Sleep(time.Second * 10)
		blockObj.Done()
	}()
	blockObj.Wait()
	fmt.Println(123)
	blockObj.Add(1)
	go threadDownasdf(&blockObj)
	go threadDownasdf(&blockObj)
	time.Sleep(10 * time.Second)
	blockObj.Done()
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGTERM)
	//信号等待
	<-c
}

func threadDownasdf(wg *sync.WaitGroup) {
	jsTmr := time.NewTimer(1 * time.Minute)
	go func() {
		wg.Wait()
		fmt.Println(5)
		jsTmr.Stop()
	}()
	for {
		fmt.Println(4)
		jsTmr.Reset(3 * time.Second)
		<-jsTmr.C
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"flag"
	"github.com/KIT-MAMID/mamid/master"
	"github.com/KIT-MAMID/mamid/master/masterapi"
	"github.com/KIT-MAMID/mamid/model"
	"github.com/KIT-MAMID/mamid/msp"
	"github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"io/ioutil"
	"net/http"
	"time"
)

var masterLog = logrus.WithField("module", "master")

type LogLevelFlag struct {
	// flag.Value
	lvl logrus.Level
}

func (f LogLevelFlag) String() string {
	return f.lvl.String()
}
func (f LogLevelFlag) Set(val string) error {
	l, err := logrus.ParseLevel(val)
	if err != nil {
		f.lvl = l
	}
	return err
}

func main() {

	// Command Line Flags
	var (
		logLevel                                                                 LogLevelFlag = LogLevelFlag{logrus.DebugLevel}
		listenString                                                             string
		slaveVerifyCA, slaveAuthCert, slaveAuthKey, apiCert, apiKey, apiVerifyCA string
		dbDriver, dbDSN                                                          string
		monitorInterval                                                          time.Duration
	)

	flag.Var(&logLevel, "log.level", "possible values: debug, info, warning, error, fatal, panic")
	flag.StringVar(&dbDriver, "db.driver", "postgres", "the database driver to use. See https://golang.org/pkg/database/sql/#Open")
	flag.StringVar(&dbDSN, "db.dsn", "", "the data source name to use. for PostgreSQL, checkout https://godoc.org/github.com/lib/pq")
	flag.StringVar(&listenString, "listen", ":8080", "net.Listen() string, e.g. addr:port")
	flag.StringVar(&slaveVerifyCA, "slave.verifyCA", "", "The CA certificate to verify slaves against")
	flag.StringVar(&slaveAuthCert, "slave.auth.cert", "", "The client certificate for authentication against the slave")
	flag.StringVar(&slaveAuthKey, "slave.auth.key", "", "The key for the client certificate for authentication against the slave")
	flag.StringVar(&apiCert, "api.cert", "", "Optional: a certificate for the api/webinterface")
	flag.StringVar(&apiKey, "api.key", "", "Optional: the key for the certificate for the api/webinterface")
	flag.StringVar(&apiVerifyCA, "api.verifyCA", "", "Optional: a ca to check client certs of webinterface/api users. Implies authentication")

	flag.DurationVar(&monitorInterval, "monitor.interval", time.Duration(10*time.Second),
		"Interval in which the monitoring component should poll slaves for status updates. Specify with suffix [ms,s,min,...]")
	flag.Parse()

	if dbDriver != "postgres" {
		masterLog.Fatal("-db.driver: only 'postgres' is supported")
	}
	if dbDSN == "" {
		masterLog.Fatal("-db.dsn cannot be empty")
	}
	if slaveVerifyCA == "" {
		masterLog.Fatal("No root certificate for the slave server communication passed. Specify with -slave.verifyCA")
	}
	if slaveAuthCert == "" {
		masterLog.Fatal("No client certificate for the slave server communication passed. Specify with -slave.auth.cert")
	}
	if slaveAuthKey == "" {
		masterLog.Fatal("No key for the client certificate for the slave server communication passed. Specify with -slave.auth.key")
	}
	if check := apiKey + apiCert; check != "" && (check == apiKey || check == apiCert) {
		masterLog.Fatal("Either -apiCert specified without -apiKey or vice versa.")
	}
	// Start application
	logrus.SetLevel(logLevel.lvl)
	masterLog.Info("Startup")

	// Setup controllers

	bus := master.NewBus()
	go bus.Run()

	db, err := model.InitializeDB(dbDriver, dbDSN)
	dieOnError(err)

	clusterAllocatorBusWriteChannel := bus.GetNewWriteChannel()
	clusterAllocator := &master.ClusterAllocator{
		BusWriteChannel: &clusterAllocatorBusWriteChannel,
	}

	tx := db.Begin()
	if err := clusterAllocator.InitializeGlobalSecrets(tx); err != nil {
		tx.Rollback()
		masterLog.Fatalf("Error initializing global secrets: %s", err)
	}
	tx.Commit()

	go clusterAllocator.Run(db)

	mainRouter := mux.NewRouter().StrictSlash(true)

	httpStatic := http.FileServer(http.Dir("./gui/"))
	mainRouter.Handle("/", httpStatic)
	mainRouter.PathPrefix("/static/").Handler(httpStatic)
	mainRouter.PathPrefix("/pages/").Handler(httpStatic)

	masterAPI := &masterapi.MasterAPI{
		DB:               db,
		ClusterAllocator: clusterAllocator,
		Router:           mainRouter.PathPrefix("/api/").Subrouter(),
	}
	masterAPI.Setup()

	certPool := x509.NewCertPool()
	cert, err := loadCertificateFromFile(slaveVerifyCA)
	dieOnError(err)
	certPool.AddCert(cert)
	clientAuthCert, err := tls.LoadX509KeyPair(slaveAuthCert, slaveAuthKey)
	dieOnError(err)
	httpTransport := &http.Transport{
		TLSClientConfig: &tls.Config{
			RootCAs:      certPool,
			Certificates: []tls.Certificate{clientAuthCert},
		},
	}
	mspClient := msp.MSPClientImpl{HttpClient: http.Client{Transport: httpTransport}}

	monitor := master.Monitor{
		DB:              db,
		BusWriteChannel: bus.GetNewWriteChannel(),
		MSPClient:       mspClient,
		Interval:        monitorInterval,
	}
	go monitor.Run()

	deployer := master.Deployer{
		DB:             db,
		BusReadChannel: bus.GetNewReadChannel(),
		MSPClient:      mspClient,
	}
	go deployer.Run()

	problemManager := master.ProblemManager{
		DB:             db,
		BusReadChannel: bus.GetNewReadChannel(),
	}
	go problemManager.Run()

	listenAndServe(listenString, mainRouter, apiCert, apiKey, apiVerifyCA)
}

func dieOnError(err error) {
	if err != nil {
		panic(err)
	}
}

func loadCertificateFromFile(file string) (cert *x509.Certificate, err error) {
	certFile, err := ioutil.ReadFile(file)
	if err != nil {
		return
	}
	block, _ := pem.Decode(certFile)
	cert, err = x509.ParseCertificate(block.Bytes)
	return
}

func listenAndServe(listenString string, mainRouter *mux.Router, apiCert string, apiKey string, apiVerifyCA string) {
	// Listen...
	if apiCert != "" {
		// ...with TLS but WITHOUT client cert auth
		if apiVerifyCA == "" {
			err := http.ListenAndServeTLS(listenString, apiCert, apiKey, mainRouter)
			dieOnError(err)
		} else { // ...with TLS AND client cert auth
			certPool := x509.NewCertPool()
			caCertContent, err := ioutil.ReadFile(apiVerifyCA)
			dieOnError(err)
			certPool.AppendCertsFromPEM(caCertContent)
			tlsConfig := &tls.Config{
				ClientCAs:  certPool,
				ClientAuth: tls.RequireAndVerifyClientCert,
			}
			server := &http.Server{
				TLSConfig: tlsConfig,
				Addr:      listenString,
				Handler:   mainRouter,
			}
			err = server.ListenAndServeTLS(apiCert, apiKey)
			dieOnError(err)
		}
	} else {
		// ...insecure and unauthenticated
		err := http.ListenAndServe(listenString, mainRouter)
		dieOnError(err)
	}
}
<commit_msg>ADD: master: configurable timeout for MSPClient<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/pem"
	"flag"
	"github.com/KIT-MAMID/mamid/master"
	"github.com/KIT-MAMID/mamid/master/masterapi"
	"github.com/KIT-MAMID/mamid/model"
	"github.com/KIT-MAMID/mamid/msp"
	"github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"
	"io/ioutil"
	"net/http"
	"time"
)

var masterLog = logrus.WithField("module", "master")

type LogLevelFlag struct {
	// flag.Value
	lvl logrus.Level
}

func (f LogLevelFlag) String() string {
	return f.lvl.String()
}
func (f LogLevelFlag) Set(val string) error {
	l, err := logrus.ParseLevel(val)
	if err != nil {
		f.lvl = l
	}
	return err
}

func main() {

	// Command Line Flags
	var (
		logLevel                                                                 LogLevelFlag = LogLevelFlag{logrus.DebugLevel}
		listenString                                                             string
		slaveVerifyCA, slaveAuthCert, slaveAuthKey, apiCert, apiKey, apiVerifyCA string
		dbDriver, dbDSN                                                          string
		monitorInterval                                                          = 10 * time.Second
		slaveTimeout                                                             = 5 * time.Second
	)

	flag.Var(&logLevel, "log.level", "possible values: debug, info, warning, error, fatal, panic")
	flag.StringVar(&dbDriver, "db.driver", "postgres", "the database driver to use. See https://golang.org/pkg/database/sql/#Open")
	flag.StringVar(&dbDSN, "db.dsn", "", "the data source name to use. for PostgreSQL, checkout https://godoc.org/github.com/lib/pq")
	flag.StringVar(&listenString, "listen", ":8080", "net.Listen() string, e.g. addr:port")
	flag.StringVar(&slaveVerifyCA, "slave.verifyCA", "", "The CA certificate to verify slaves against")
	flag.StringVar(&slaveAuthCert, "slave.auth.cert", "", "The client certificate for authentication against the slave")
	flag.StringVar(&slaveAuthKey, "slave.auth.key", "", "The key for the client certificate for authentication against the slave")
	flag.DurationVar(&slaveTimeout, "slave.timeout", slaveTimeout,
		"Timeout for requests from master to slave. Specify with suffix [ms,s,min,...]")
	flag.StringVar(&apiCert, "api.cert", "", "Optional: a certificate for the api/webinterface")
	flag.StringVar(&apiKey, "api.key", "", "Optional: the key for the certificate for the api/webinterface")
	flag.StringVar(&apiVerifyCA, "api.verifyCA", "", "Optional: a ca to check client certs of webinterface/api users. Implies authentication")

	flag.DurationVar(&monitorInterval, "monitor.interval", monitorInterval,
		"Interval in which the monitoring component should poll slaves for status updates."+
			"Note that the monitor waits for the slowest slave before a new monitor run starts (-slave.timeout). Specify with suffix [ms,s,min,...]")
	flag.Parse()

	if dbDriver != "postgres" {
		masterLog.Fatal("-db.driver: only 'postgres' is supported")
	}
	if dbDSN == "" {
		masterLog.Fatal("-db.dsn cannot be empty")
	}
	if slaveVerifyCA == "" {
		masterLog.Fatal("No root certificate for the slave server communication passed. Specify with -slave.verifyCA")
	}
	if slaveAuthCert == "" {
		masterLog.Fatal("No client certificate for the slave server communication passed. Specify with -slave.auth.cert")
	}
	if slaveAuthKey == "" {
		masterLog.Fatal("No key for the client certificate for the slave server communication passed. Specify with -slave.auth.key")
	}
	if check := apiKey + apiCert; check != "" && (check == apiKey || check == apiCert) {
		masterLog.Fatal("Either -apiCert specified without -apiKey or vice versa.")
	}
	// Start application
	logrus.SetLevel(logLevel.lvl)
	masterLog.Info("Startup")

	// Setup controllers

	bus := master.NewBus()
	go bus.Run()

	db, err := model.InitializeDB(dbDriver, dbDSN)
	dieOnError(err)

	clusterAllocatorBusWriteChannel := bus.GetNewWriteChannel()
	clusterAllocator := &master.ClusterAllocator{
		BusWriteChannel: &clusterAllocatorBusWriteChannel,
	}

	tx := db.Begin()
	if err := clusterAllocator.InitializeGlobalSecrets(tx); err != nil {
		tx.Rollback()
		masterLog.Fatalf("Error initializing global secrets: %s", err)
	}
	tx.Commit()

	go clusterAllocator.Run(db)

	mainRouter := mux.NewRouter().StrictSlash(true)

	httpStatic := http.FileServer(http.Dir("./gui/"))
	mainRouter.Handle("/", httpStatic)
	mainRouter.PathPrefix("/static/").Handler(httpStatic)
	mainRouter.PathPrefix("/pages/").Handler(httpStatic)

	masterAPI := &masterapi.MasterAPI{
		DB:               db,
		ClusterAllocator: clusterAllocator,
		Router:           mainRouter.PathPrefix("/api/").Subrouter(),
	}
	masterAPI.Setup()

	certPool := x509.NewCertPool()
	cert, err := loadCertificateFromFile(slaveVerifyCA)
	dieOnError(err)
	certPool.AddCert(cert)
	clientAuthCert, err := tls.LoadX509KeyPair(slaveAuthCert, slaveAuthKey)
	dieOnError(err)
	httpTransport := &http.Transport{
		TLSClientConfig: &tls.Config{
			RootCAs:      certPool,
			Certificates: []tls.Certificate{clientAuthCert},
		},
	}
	mspClient := msp.MSPClientImpl{
		HttpClient: http.Client{
			Transport: httpTransport,
			Timeout:   slaveTimeout,
		},
	}

	monitor := master.Monitor{
		DB:              db,
		BusWriteChannel: bus.GetNewWriteChannel(),
		MSPClient:       mspClient,
		Interval:        monitorInterval,
	}
	go monitor.Run()

	deployer := master.Deployer{
		DB:             db,
		BusReadChannel: bus.GetNewReadChannel(),
		MSPClient:      mspClient,
	}
	go deployer.Run()

	problemManager := master.ProblemManager{
		DB:             db,
		BusReadChannel: bus.GetNewReadChannel(),
	}
	go problemManager.Run()

	listenAndServe(listenString, mainRouter, apiCert, apiKey, apiVerifyCA)
}

func dieOnError(err error) {
	if err != nil {
		panic(err)
	}
}

func loadCertificateFromFile(file string) (cert *x509.Certificate, err error) {
	certFile, err := ioutil.ReadFile(file)
	if err != nil {
		return
	}
	block, _ := pem.Decode(certFile)
	cert, err = x509.ParseCertificate(block.Bytes)
	return
}

func listenAndServe(listenString string, mainRouter *mux.Router, apiCert string, apiKey string, apiVerifyCA string) {
	// Listen...
	if apiCert != "" {
		// ...with TLS but WITHOUT client cert auth
		if apiVerifyCA == "" {
			err := http.ListenAndServeTLS(listenString, apiCert, apiKey, mainRouter)
			dieOnError(err)
		} else { // ...with TLS AND client cert auth
			certPool := x509.NewCertPool()
			caCertContent, err := ioutil.ReadFile(apiVerifyCA)
			dieOnError(err)
			certPool.AppendCertsFromPEM(caCertContent)
			tlsConfig := &tls.Config{
				ClientCAs:  certPool,
				ClientAuth: tls.RequireAndVerifyClientCert,
			}
			server := &http.Server{
				TLSConfig: tlsConfig,
				Addr:      listenString,
				Handler:   mainRouter,
			}
			err = server.ListenAndServeTLS(apiCert, apiKey)
			dieOnError(err)
		}
	} else {
		// ...insecure and unauthenticated
		err := http.ListenAndServe(listenString, mainRouter)
		dieOnError(err)
	}
}
<|endoftext|>


<commit_before>// +build !arm

package bcd

import (
	sys "golang.org/x/sys/unix"
)

func gettid() (int, error) {
	return sys.Gettid(), nil
}

// Call this function to allow other (non-parent) processes to trace this one.
// Alternatively, set kernel.yama.ptrace_scope = 0 in
// /etc/sysctl.d/10-ptrace.conf.
//
// This is a Linux-specific utility function.
func EnableTracing() error {
	// PR_SET_PTRACER_ANY may be a negative integer constant on some
	// systems, so we need to store it in a separate variable to bypass
	// Go's const conversion restrictions.
	flag := sys.PR_SET_PTRACER_ANY

	return sys.Prctl(sys.PR_SET_PTRACER, uintptr(flag), 0, 0, 0)
}
<commit_msg>Use sys.PR_SET_PTRACER_ANY directly<commit_after>// +build !arm

package bcd

import (
	sys "golang.org/x/sys/unix"
)

func gettid() (int, error) {
	return sys.Gettid(), nil
}

// Call this function to allow other (non-parent) processes to trace this one.
// Alternatively, set kernel.yama.ptrace_scope = 0 in
// /etc/sysctl.d/10-ptrace.conf.
//
// This is a Linux-specific utility function.
func EnableTracing() error {
	return sys.Prctl(sys.PR_SET_PTRACER, sys.PR_SET_PTRACER_ANY, 0, 0, 0)
}
<|endoftext|>


<commit_before>//
// markup.go
// memeposting markup parser
//
package srnd

import (
  "github.com/mvdan/xurls"
  "html"
  "regexp"
  "strings"
)

// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
// var re_external_link = regexp.MustCompile(`((?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?)`);
var re_backlink = regexp.MustCompile(`>> ?([0-9a-f]+)`)

// parse backlink
func backlink(word string) (markup string) {
  link := re_backlink.FindString(word)
  if len(link) > 2 {
    link = strings.Trim(link[2:], " ")
    if len(link) > 2 {
      url := template.findLink(link)
      if len(url) == 0 {
        return ">>" + link[2:] + ""
      }
      // backlink exists
      return`<a href="`+url+`">>>` + link + ""
    } else {
      return html.EscapeString(word)
    }
  }
  return html.EscapeString(word)
}

func formatline(line string) (markup string) {
  line = strings.Trim(line, "\t\r\n ")
  if len(line) > 0 {
    if strings.HasPrefix(line, ">") && ! ( strings.HasPrefix(line, ">>") && re_backlink.MatchString(strings.Split(line, " ")[0])) {
      // le ebin meme arrows
      markup += "
"
      markup += html.EscapeString(line)
      markup += "
"
    } else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
      // redtext
      markup += "
"
      markup += html.EscapeString(line[2:len(line)-2])
      markup += "
"
    } else {
      // regular line
      markup += "
"
      // for each word
      for _, word := range strings.Split(line, " ") {
        // check for backlink
        if re_backlink.MatchString(word) {
          markup += backlink(word)
        } else {
          // linkify as needed
          word = html.EscapeString(word)
          markup += xurls.Strict.ReplaceAllString(word, `<a href="$1">$1</a>`)
        }
        markup += " "
      }
      markup += "
"
    }
  }
  return
}

// format lines inside a code tag
func formatcodeline(line string) (markup string) {
  markup += html.EscapeString(line)
  markup += "\n"
  return
}

func memeposting(src string) (markup string) {
  found_tag := false
  tag_content := ""
  tag := ""
  // for each line...
  for _, line := range strings.Split(src, "\n") {
    // beginning of code tag ?
    if strings.Count(line, "[code]") > 0 {
      // yes there's a code tag
      found_tag = true
      tag = "code"
    } else if strings.Count(line, "[spoiler]") > 0 {
      // spoiler tag
      found_tag = true
      tag = "spoiler"
    } else if strings.Count(line, "[psy]") > 0 {
      // psy tag
      found_tag = true
      tag = "psy"
    }
    if found_tag {
      // collect content of tag
      tag_content += line + "\n"
      // end of our tag ?
      if strings.Count(line, "[/"+tag+"]") == 1 {
        // yah
        found_tag = false
        var tag_open, tag_close string
        if tag == "code" {
          tag_open = "
"
          tag_close = "
"
        } else if tag == "spoiler" {
          tag_open = ""
          tag_close = ""
        } else if tag == "psy" {
          tag_open = "
"
          tag_close = "
"
        }
        markup += tag_open
        // remove open tag, only once so we can have a code tag verbatum inside
        tag_content = strings.Replace(tag_content, "["+tag+"]", "", 1)
        // remove all close tags, should only have 1
        tag_content = strings.Replace(tag_content, "[/"+tag+"]", "", -1)
        // make into lines
        for _, tag_line := range strings.Split(tag_content, "\n") {
          if tag == "code" {
            markup += formatcodeline(tag_line)
          } else {
            markup += formatline(tag_line)
          }
        }
        // close pre tag
        markup += tag_close
        // reset content buffer
        tag_content = ""
      }
      // next line
      continue
    }
    // format line regularlly
    markup += formatline(line)
  }
  // flush the rest of an incomplete code tag
  for _, line := range strings.Split(tag_content, "\n") {
    markup += formatline(line)
  }
  return
}
<commit_msg>try fixing markup for backlinks<commit_after>//
// markup.go
// memeposting markup parser
//
package srnd

import (
  "github.com/mvdan/xurls"
  "html"
  "regexp"
  "strings"
)

// copypasted from https://stackoverflow.com/questions/161738/what-is-the-best-regular-expression-to-check-if-a-string-is-a-valid-url
// var re_external_link = regexp.MustCompile(`((?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,}))\.?)(?::\d{2,5})?(?:[/?#]\S*)?)`);
var re_backlink = regexp.MustCompile(`>> ?([0-9a-f]+)`)

// parse backlink
func backlink(word string) (markup string) {
  link := re_backlink.FindString(word)
  if len(link) > 2 {
    link = strings.Trim(link, " ")
    if len(link) > 2 {
      url := template.findLink(link)
      if len(url) == 0 {
        return ">>" + link + ""
      }
      // backlink exists
      return`<a href="`+url+`">>>` + link + ""
    } else {
      return html.EscapeString(word)
    }
  }
  return html.EscapeString(word)
}

func formatline(line string) (markup string) {
  line = strings.Trim(line, "\t\r\n ")
  if len(line) > 0 {
    if strings.HasPrefix(line, ">") && ! ( strings.HasPrefix(line, ">>") && re_backlink.MatchString(strings.Split(line, " ")[0])) {
      // le ebin meme arrows
      markup += "
"
      markup += html.EscapeString(line)
      markup += "
"
    } else if strings.HasPrefix(line, "==") && strings.HasSuffix(line, "==") {
      // redtext
      markup += "
"
      markup += html.EscapeString(line[2:len(line)-2])
      markup += "
"
    } else {
      // regular line
      markup += "
"
      // for each word
      for _, word := range strings.Split(line, " ") {
        // check for backlink
        if re_backlink.MatchString(word) {
          markup += backlink(word)
        } else {
          // linkify as needed
          word = html.EscapeString(word)
          markup += xurls.Strict.ReplaceAllString(word, `<a href="$1">$1</a>`)
        }
        markup += " "
      }
      markup += "
"
    }
  }
  return
}

// format lines inside a code tag
func formatcodeline(line string) (markup string) {
  markup += html.EscapeString(line)
  markup += "\n"
  return
}

func memeposting(src string) (markup string) {
  found_tag := false
  tag_content := ""
  tag := ""
  // for each line...
  for _, line := range strings.Split(src, "\n") {
    // beginning of code tag ?
    if strings.Count(line, "[code]") > 0 {
      // yes there's a code tag
      found_tag = true
      tag = "code"
    } else if strings.Count(line, "[spoiler]") > 0 {
      // spoiler tag
      found_tag = true
      tag = "spoiler"
    } else if strings.Count(line, "[psy]") > 0 {
      // psy tag
      found_tag = true
      tag = "psy"
    }
    if found_tag {
      // collect content of tag
      tag_content += line + "\n"
      // end of our tag ?
      if strings.Count(line, "[/"+tag+"]") == 1 {
        // yah
        found_tag = false
        var tag_open, tag_close string
        if tag == "code" {
          tag_open = "
"
          tag_close = "
"
        } else if tag == "spoiler" {
          tag_open = ""
          tag_close = ""
        } else if tag == "psy" {
          tag_open = "
"
          tag_close = "
"
        }
        markup += tag_open
        // remove open tag, only once so we can have a code tag verbatum inside
        tag_content = strings.Replace(tag_content, "["+tag+"]", "", 1)
        // remove all close tags, should only have 1
        tag_content = strings.Replace(tag_content, "[/"+tag+"]", "", -1)
        // make into lines
        for _, tag_line := range strings.Split(tag_content, "\n") {
          if tag == "code" {
            markup += formatcodeline(tag_line)
          } else {
            markup += formatline(tag_line)
          }
        }
        // close pre tag
        markup += tag_close
        // reset content buffer
        tag_content = ""
      }
      // next line
      continue
    }
    // format line regularlly
    markup += formatline(line)
  }
  // flush the rest of an incomplete code tag
  for _, line := range strings.Split(tag_content, "\n") {
    markup += formatline(line)
  }
  return
}
<|endoftext|>


<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"fmt"
	"log"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"

	"github.com/aws/aws-sdk-go/aws/awserr"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

var S3AllowEmptyValues = []string{"tags."}

var S3AdditionalFields = map[string]string{}

type S3Generator struct {
	AWSService
}

// createResources iterate on all buckets
// for each bucket we check region and choose only bucket from set region
// for each bucket try get bucket policy, if policy exist create additional NewTerraformResource for policy
func (S3Generator) createResources(buckets *s3.ListBucketsOutput, region string) []terraform_utils.Resource {
	resources := []terraform_utils.Resource{}
	sess, _ := session.NewSession(&aws.Config{Region: aws.String(region)})
	svc := s3.New(sess)
	for _, bucket := range buckets.Buckets {
		resourceName := aws.StringValue(bucket.Name)
		location, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: bucket.Name})
		if err != nil {
			log.Println(err)
			continue
		}
		// check if bucket in region
		if s3.NormalizeBucketLocation(aws.StringValue(location.LocationConstraint)) == region {
			resources = append(resources, terraform_utils.NewResource(
				resourceName,
				resourceName,
				"aws_s3_bucket",
				"aws",
				map[string]string{
					"force_destroy": "false",
					"acl":           "private",
				},
				S3AllowEmptyValues,
				S3AdditionalFields))
			// try get policy
			_, err := svc.GetBucketPolicy(&s3.GetBucketPolicyInput{
				Bucket: bucket.Name,
			})

			if err != nil {
				if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoSuchBucketPolicy" {
					// Bucket without policy
					continue
				}
				log.Println(err)
				continue
			}
			// if bucket policy exist create TerraformResource with bucket name as ID
			resources = append(resources, terraform_utils.NewResource(
				resourceName,
				resourceName,
				"aws_s3_bucket_policy",
				"aws",
				map[string]string{},
				S3AllowEmptyValues,
				S3AdditionalFields))
		}
	}
	return resources
}

// Generate TerraformResources from AWS API,
// from each s3 bucket create 2 TerraformResource(bucket and bucket policy)
// Need bucket name as ID for terraform resource
func (g *S3Generator) InitResources() error {
	sess, _ := session.NewSession(&aws.Config{Region: aws.String(g.GetArgs()["region"])})
	svc := s3.New(sess)
	buckets, err := svc.ListBuckets(&s3.ListBucketsInput{})
	if err != nil {
		return err
	}
	g.Resources = g.createResources(buckets, g.GetArgs()["region"])
	g.PopulateIgnoreKeys()
	return nil
}

// PostGenerateHook for add bucket policy json as heredoc
// support only bucket with policy
func (g *S3Generator) PostConvertHook() error {
	for i, resource := range g.Resources {
		if resource.InstanceInfo.Type != "aws_s3_bucket_policy" {
			continue
		}
		policy := resource.Item["policy"].(string)
		g.Resources[i].Item["policy"] = fmt.Sprintf(`<<POLICY
%s
POLICY`, policy)
	}
	return nil
}
<commit_msg>Improved AWS provider<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"fmt"
	"log"
	"strings"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"

	"github.com/aws/aws-sdk-go/aws/awserr"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
)

var S3AllowEmptyValues = []string{"tags."}

var S3AdditionalFields = map[string]string{}

type S3Generator struct {
	AWSService
}

// createResources iterate on all buckets
// for each bucket we check region and choose only bucket from set region
// for each bucket try get bucket policy, if policy exist create additional NewTerraformResource for policy
func (S3Generator) createResources(buckets *s3.ListBucketsOutput, region string) []terraform_utils.Resource {
	resources := []terraform_utils.Resource{}
	sess, _ := session.NewSession(&aws.Config{Region: aws.String(region)})
	svc := s3.New(sess)
	for _, bucket := range buckets.Buckets {
		resourceName := aws.StringValue(bucket.Name)
		location, err := svc.GetBucketLocation(&s3.GetBucketLocationInput{Bucket: bucket.Name})
		if err != nil {
			log.Println(err)
			continue
		}
		// check if bucket in region
		if s3.NormalizeBucketLocation(aws.StringValue(location.LocationConstraint)) == region {
			resources = append(resources, terraform_utils.NewResource(
				resourceName,
				resourceName,
				"aws_s3_bucket",
				"aws",
				map[string]string{
					"force_destroy": "false",
					"acl":           "private",
				},
				S3AllowEmptyValues,
				S3AdditionalFields))
			// try get policy
			_, err := svc.GetBucketPolicy(&s3.GetBucketPolicyInput{
				Bucket: bucket.Name,
			})

			if err != nil {
				if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoSuchBucketPolicy" {
					// Bucket without policy
					continue
				}
				log.Println(err)
				continue
			}
			// if bucket policy exist create TerraformResource with bucket name as ID
			resources = append(resources, terraform_utils.NewResource(
				resourceName,
				resourceName,
				"aws_s3_bucket_policy",
				"aws",
				map[string]string{},
				S3AllowEmptyValues,
				S3AdditionalFields))
		}
	}
	return resources
}

// Generate TerraformResources from AWS API,
// from each s3 bucket create 2 TerraformResource(bucket and bucket policy)
// Need bucket name as ID for terraform resource
func (g *S3Generator) InitResources() error {
	sess, _ := session.NewSession(&aws.Config{Region: aws.String(g.GetArgs()["region"])})
	svc := s3.New(sess)
	buckets, err := svc.ListBuckets(&s3.ListBucketsInput{})
	if err != nil {
		return err
	}
	g.Resources = g.createResources(buckets, g.GetArgs()["region"])
	g.PopulateIgnoreKeys()
	return nil
}

// PostGenerateHook for add bucket policy json as heredoc
// support only bucket with policy
func (g *S3Generator) PostConvertHook() error {
	for i, resource := range g.Resources {
		if resource.InstanceInfo.Type != "aws_s3_bucket_policy" {
			continue
		}
		policy := resource.Item["policy"].(string)
		g.Resources[i].Item["policy"] = fmt.Sprintf(`<<POLICY
%s
POLICY`, policy)
	}
	return nil
}

func (g *S3Generator) ParseFilter(rawFilter []string) {
	g.Filter = map[string][]string{}
	for _, resource := range rawFilter {
		t := strings.Split(resource, "=")
		if len(t) != 2 {
			log.Println("Pattern for filter must be resource_type=id1:id2:id4")
			continue
		}
		resourceName, resourcesID := t[0], t[1]
		g.Filter[resourceName] = strings.Split(resourcesID, ":")
		if resourceName == "aws_s3_bucket" {
			g.Filter["aws_s3_bucket_policy"] = strings.Split(resourcesID, ":")
		}
	}
}<|endoftext|>


<commit_before>//  Copyright Project Harbor Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package member

import (
	"context"
	"fmt"
	"github.com/goharbor/harbor/src/common"
	"github.com/goharbor/harbor/src/core/auth"
	"github.com/goharbor/harbor/src/lib/errors"
	"github.com/goharbor/harbor/src/lib/q"
	"github.com/goharbor/harbor/src/pkg/member"
	"github.com/goharbor/harbor/src/pkg/member/models"
	"github.com/goharbor/harbor/src/pkg/project"
	"github.com/goharbor/harbor/src/pkg/user"
	"github.com/goharbor/harbor/src/pkg/usergroup"
	ugModel "github.com/goharbor/harbor/src/pkg/usergroup/model"
)

// Controller defines the operation related to project member
type Controller interface {
	// Get get the project member with ID
	Get(ctx context.Context, projectNameOrID interface{}, memberID int) (*models.Member, error)
	// Create add project member to project
	Create(ctx context.Context, projectNameOrID interface{}, req Request) (int, error)
	// Delete member from project
	Delete(ctx context.Context, projectNameOrID interface{}, memberID int) error
	// List list all project members with condition
	List(ctx context.Context, projectNameOrID interface{}, entityName string, query *q.Query) ([]*models.Member, error)
	// UpdateRole update the project member role
	UpdateRole(ctx context.Context, projectNameOrID interface{}, memberID int, role int) error
	// Count get the total amount of project members
	Count(ctx context.Context, projectNameOrID interface{}, query *q.Query) (int, error)
}

// Request - Project Member Request
type Request struct {
	ProjectID   int64     `json:"project_id"`
	Role        int       `json:"role_id,omitempty"`
	MemberUser  User      `json:"member_user,omitempty"`
	MemberGroup UserGroup `json:"member_group,omitempty"`
}

// User ...
type User struct {
	UserID   int    `json:"user_id"`
	Username string `json:"username"`
}

// UserGroup ...
type UserGroup struct {
	ID          int    `json:"id,omitempty"`
	GroupName   string `json:"group_name,omitempty"`
	GroupType   int    `json:"group_type,omitempty"`
	LdapGroupDN string `json:"ldap_group_dn,omitempty"`
}

// ErrDuplicateProjectMember ...
var ErrDuplicateProjectMember = errors.New("The project member specified already exist")

// ErrInvalidRole ...
var ErrInvalidRole = errors.New("Failed to update project member, role is not in 1,2,3")

type controller struct {
	userManager user.Manager
	mgr         member.Manager
	projectMgr  project.Manager
}

// NewController ...
func NewController() Controller {
	return &controller{mgr: member.Mgr, projectMgr: project.Mgr, userManager: user.New()}
}

func (c *controller) Count(ctx context.Context, projectNameOrID interface{}, query *q.Query) (int, error) {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return 0, err
	}
	return c.mgr.GetTotalOfProjectMembers(ctx, p.ProjectID, query)
}

func (c *controller) UpdateRole(ctx context.Context, projectNameOrID interface{}, memberID int, role int) error {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return err
	}
	if p == nil {
		return errors.BadRequestError(nil).WithMessage("project is not found")
	}
	return c.mgr.UpdateRole(ctx, p.ProjectID, memberID, role)
}

func (c *controller) Get(ctx context.Context, projectNameOrID interface{}, memberID int) (*models.Member, error) {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return nil, err
	}
	if p == nil {
		return nil, errors.BadRequestError(nil).WithMessage("project is not found")
	}
	return c.mgr.Get(ctx, p.ProjectID, memberID)
}

func (c *controller) Create(ctx context.Context, projectNameOrID interface{}, req Request) (int, error) {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return 0, err
	}
	if p == nil {
		return 0, errors.BadRequestError(nil).WithMessage("project is not found")
	}
	var member models.Member
	member.ProjectID = p.ProjectID
	member.Role = req.Role
	member.EntityType = common.GroupMember

	if req.MemberUser.UserID > 0 {
		member.EntityID = req.MemberUser.UserID
		member.EntityType = common.UserMember
	} else if req.MemberGroup.ID > 0 {
		member.EntityID = req.MemberGroup.ID
	} else if len(req.MemberUser.Username) > 0 {
		// If username is provided, search userid by username
		var userID int
		member.EntityType = common.UserMember
		u, err := c.userManager.GetByName(ctx, req.MemberUser.Username)
		if err != nil {
			return 0, err
		}
		if u != nil {
			userID = u.UserID
		} else {
			userID, err = auth.SearchAndOnBoardUser(req.MemberUser.Username)
			if err != nil {
				return 0, err
			}
		}
		member.EntityID = userID
	} else if len(req.MemberGroup.LdapGroupDN) > 0 {
		req.MemberGroup.GroupType = common.LDAPGroupType
		// If groupname provided, use the provided groupname to name this group
		groupID, err := auth.SearchAndOnBoardGroup(req.MemberGroup.LdapGroupDN, req.MemberGroup.GroupName)
		if err != nil {
			return 0, err
		}
		member.EntityID = groupID
	} else if len(req.MemberGroup.GroupName) > 0 && req.MemberGroup.GroupType == common.HTTPGroupType || req.MemberGroup.GroupType == common.OIDCGroupType {
		ugs, err := usergroup.Mgr.List(ctx, ugModel.UserGroup{GroupName: req.MemberGroup.GroupName, GroupType: req.MemberGroup.GroupType})
		if err != nil {
			return 0, err
		}
		if len(ugs) == 0 {
			groupID, err := auth.SearchAndOnBoardGroup(req.MemberGroup.GroupName, "")
			if err != nil {
				return 0, err
			}
			member.EntityID = groupID
		} else {
			member.EntityID = ugs[0].ID
		}

	}
	if member.EntityID <= 0 {
		return 0, fmt.Errorf("Can not get valid member entity, request: %+v", req)
	}

	// Check if member already exist in current project
	memberList, err := c.mgr.List(ctx, models.Member{
		ProjectID:  member.ProjectID,
		EntityID:   member.EntityID,
		EntityType: member.EntityType,
	}, nil)
	if err != nil {
		return 0, err
	}
	if len(memberList) > 0 {
		return 0, ErrDuplicateProjectMember
	}

	if !isValidRole(member.Role) {
		// Return invalid role error
		return 0, ErrInvalidRole
	}
	return c.mgr.AddProjectMember(ctx, member)
}

func isValidRole(role int) bool {
	switch role {
	case common.RoleProjectAdmin,
		common.RoleMaintainer,
		common.RoleDeveloper,
		common.RoleGuest,
		common.RoleLimitedGuest:
		return true
	default:
		return false
	}
}

func (c *controller) List(ctx context.Context, projectNameOrID interface{}, entityName string, query *q.Query) ([]*models.Member, error) {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return nil, err
	}
	if p == nil {
		return nil, errors.BadRequestError(nil).WithMessage("project is not found")
	}
	pm := models.Member{
		ProjectID:  p.ProjectID,
		Entityname: entityName,
	}
	return c.mgr.List(ctx, pm, query)
}

func (c *controller) Delete(ctx context.Context, projectNameOrID interface{}, memberID int) error {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return err
	}
	return c.mgr.Delete(ctx, p.ProjectID, memberID)
}
<commit_msg>Fix nightly issue: can not add user that never logged in Handle the NotFoundError in userManager.GetUserByName()<commit_after>//  Copyright Project Harbor Authors
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package member

import (
	"context"
	"fmt"
	"github.com/goharbor/harbor/src/common"
	"github.com/goharbor/harbor/src/core/auth"
	"github.com/goharbor/harbor/src/lib/errors"
	"github.com/goharbor/harbor/src/lib/q"
	"github.com/goharbor/harbor/src/pkg/member"
	"github.com/goharbor/harbor/src/pkg/member/models"
	"github.com/goharbor/harbor/src/pkg/project"
	"github.com/goharbor/harbor/src/pkg/user"
	"github.com/goharbor/harbor/src/pkg/usergroup"
	ugModel "github.com/goharbor/harbor/src/pkg/usergroup/model"
)

// Controller defines the operation related to project member
type Controller interface {
	// Get get the project member with ID
	Get(ctx context.Context, projectNameOrID interface{}, memberID int) (*models.Member, error)
	// Create add project member to project
	Create(ctx context.Context, projectNameOrID interface{}, req Request) (int, error)
	// Delete member from project
	Delete(ctx context.Context, projectNameOrID interface{}, memberID int) error
	// List list all project members with condition
	List(ctx context.Context, projectNameOrID interface{}, entityName string, query *q.Query) ([]*models.Member, error)
	// UpdateRole update the project member role
	UpdateRole(ctx context.Context, projectNameOrID interface{}, memberID int, role int) error
	// Count get the total amount of project members
	Count(ctx context.Context, projectNameOrID interface{}, query *q.Query) (int, error)
}

// Request - Project Member Request
type Request struct {
	ProjectID   int64     `json:"project_id"`
	Role        int       `json:"role_id,omitempty"`
	MemberUser  User      `json:"member_user,omitempty"`
	MemberGroup UserGroup `json:"member_group,omitempty"`
}

// User ...
type User struct {
	UserID   int    `json:"user_id"`
	Username string `json:"username"`
}

// UserGroup ...
type UserGroup struct {
	ID          int    `json:"id,omitempty"`
	GroupName   string `json:"group_name,omitempty"`
	GroupType   int    `json:"group_type,omitempty"`
	LdapGroupDN string `json:"ldap_group_dn,omitempty"`
}

// ErrDuplicateProjectMember ...
var ErrDuplicateProjectMember = errors.New("The project member specified already exist")

// ErrInvalidRole ...
var ErrInvalidRole = errors.New("Failed to update project member, role is not in 1,2,3")

type controller struct {
	userManager user.Manager
	mgr         member.Manager
	projectMgr  project.Manager
}

// NewController ...
func NewController() Controller {
	return &controller{mgr: member.Mgr, projectMgr: project.Mgr, userManager: user.New()}
}

func (c *controller) Count(ctx context.Context, projectNameOrID interface{}, query *q.Query) (int, error) {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return 0, err
	}
	return c.mgr.GetTotalOfProjectMembers(ctx, p.ProjectID, query)
}

func (c *controller) UpdateRole(ctx context.Context, projectNameOrID interface{}, memberID int, role int) error {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return err
	}
	if p == nil {
		return errors.BadRequestError(nil).WithMessage("project is not found")
	}
	return c.mgr.UpdateRole(ctx, p.ProjectID, memberID, role)
}

func (c *controller) Get(ctx context.Context, projectNameOrID interface{}, memberID int) (*models.Member, error) {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return nil, err
	}
	if p == nil {
		return nil, errors.BadRequestError(nil).WithMessage("project is not found")
	}
	return c.mgr.Get(ctx, p.ProjectID, memberID)
}

func (c *controller) Create(ctx context.Context, projectNameOrID interface{}, req Request) (int, error) {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return 0, err
	}
	if p == nil {
		return 0, errors.BadRequestError(nil).WithMessage("project is not found")
	}
	var member models.Member
	member.ProjectID = p.ProjectID
	member.Role = req.Role
	member.EntityType = common.GroupMember

	if req.MemberUser.UserID > 0 {
		member.EntityID = req.MemberUser.UserID
		member.EntityType = common.UserMember
	} else if req.MemberGroup.ID > 0 {
		member.EntityID = req.MemberGroup.ID
	} else if len(req.MemberUser.Username) > 0 {
		// If username is provided, search userid by username
		var userID int
		member.EntityType = common.UserMember
		u, err := c.userManager.GetByName(ctx, req.MemberUser.Username)
		if err != nil && !errors.IsNotFoundErr(err) {
			return 0, err
		}
		if u != nil {
			userID = u.UserID
		} else {
			userID, err = auth.SearchAndOnBoardUser(req.MemberUser.Username)
			if err != nil {
				return 0, err
			}
		}
		member.EntityID = userID
	} else if len(req.MemberGroup.LdapGroupDN) > 0 {
		req.MemberGroup.GroupType = common.LDAPGroupType
		// If groupname provided, use the provided groupname to name this group
		groupID, err := auth.SearchAndOnBoardGroup(req.MemberGroup.LdapGroupDN, req.MemberGroup.GroupName)
		if err != nil {
			return 0, err
		}
		member.EntityID = groupID
	} else if len(req.MemberGroup.GroupName) > 0 && req.MemberGroup.GroupType == common.HTTPGroupType || req.MemberGroup.GroupType == common.OIDCGroupType {
		ugs, err := usergroup.Mgr.List(ctx, ugModel.UserGroup{GroupName: req.MemberGroup.GroupName, GroupType: req.MemberGroup.GroupType})
		if err != nil {
			return 0, err
		}
		if len(ugs) == 0 {
			groupID, err := auth.SearchAndOnBoardGroup(req.MemberGroup.GroupName, "")
			if err != nil {
				return 0, err
			}
			member.EntityID = groupID
		} else {
			member.EntityID = ugs[0].ID
		}

	}
	if member.EntityID <= 0 {
		return 0, fmt.Errorf("Can not get valid member entity, request: %+v", req)
	}

	// Check if member already exist in current project
	memberList, err := c.mgr.List(ctx, models.Member{
		ProjectID:  member.ProjectID,
		EntityID:   member.EntityID,
		EntityType: member.EntityType,
	}, nil)
	if err != nil {
		return 0, err
	}
	if len(memberList) > 0 {
		return 0, ErrDuplicateProjectMember
	}

	if !isValidRole(member.Role) {
		// Return invalid role error
		return 0, ErrInvalidRole
	}
	return c.mgr.AddProjectMember(ctx, member)
}

func isValidRole(role int) bool {
	switch role {
	case common.RoleProjectAdmin,
		common.RoleMaintainer,
		common.RoleDeveloper,
		common.RoleGuest,
		common.RoleLimitedGuest:
		return true
	default:
		return false
	}
}

func (c *controller) List(ctx context.Context, projectNameOrID interface{}, entityName string, query *q.Query) ([]*models.Member, error) {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return nil, err
	}
	if p == nil {
		return nil, errors.BadRequestError(nil).WithMessage("project is not found")
	}
	pm := models.Member{
		ProjectID:  p.ProjectID,
		Entityname: entityName,
	}
	return c.mgr.List(ctx, pm, query)
}

func (c *controller) Delete(ctx context.Context, projectNameOrID interface{}, memberID int) error {
	p, err := c.projectMgr.Get(ctx, projectNameOrID)
	if err != nil {
		return err
	}
	return c.mgr.Delete(ctx, p.ProjectID, memberID)
}
<|endoftext|>


<commit_before>package bcsgo

import (
	"strconv"
	"testing"
	"time"
)

func randomGlobalBucketName(index int) string {
	return ak[:2] + "-sdk-" + strconv.FormatInt(time.Now().Unix(), 10)[5:] + "-" + strconv.Itoa(index)
}

func createBucketTempForTest(t *testing.T) *Bucket {
	bucketName := randomGlobalBucketName(1)
	bucket := bcs.Bucket(bucketName)
	err := bucket.Create()
	if err != nil {
		t.Error(err)
	}
	return bucket
}

func deleteBucketForTest(t *testing.T, bucket *Bucket) {
	deleteErr := bucket.Delete()
	if deleteErr != nil {
		t.Error(deleteErr)
	}
}

func TestBucketCreateWithACLAndDelete(t *testing.T) {
	bucketName := randomGlobalBucketName(1)
	newBucket := bcs.Bucket(bucketName)
	bucketErr := newBucket.CreateWithACL(ACL_PUBLIC_READ)
	if bucketErr != nil {
		t.Error(bucketErr)
	}

	// bucketACL, bucketACLErr := newBucket.GetACL()
	// expectedBucketACL := fmt.Sprintf(`{"statements":[{"action":["*"],"effect":"allow","resource":["testsml2\/"],"user":["psp:egg90"]},{"action":["get_object"],"effect":"allow","resource":["%s\/"],"user":["*"]}]}`, bucketName)
	// if bucketACLErr != nil {
	// 	fmt.Println(bucketACLErr)
	// 	t.Fail()
	// }
	// if bucketACL != expectedBucketACL {
	// 	fmt.Println(bucketACL)
	// 	fmt.Println(expectedBucketACL)
	// 	t.Fail()
	// }

	bucketErr = newBucket.Delete()
	if bucketErr != nil {
		t.Error(bucketErr)
	}
}

func TestBucketCreateWithInvalidName(t *testing.T) {
	newBucket := bcs.Bucket("testErrorBucket")
	bucketErr := newBucket.Create()
	// It shall be failed.
	if bucketErr == nil {
		t.Error("create bucket with invaid name should failed")
	}
}

func TestBucketListObjects(t *testing.T) {
	bucket := createBucketTempForTest(t)

	// todo prefix
	objects, e := bucket.ListObjects("", 0, 5)
	if e != nil {
		t.Error("object list shouldn't be nil")
	}
	for _, pObject := range objects.Objects {
		if pObject == nil {
			t.Error("object should not be nil")
		}
	}

	deleteBucketForTest(t, bucket)
}

func TestBucketACL(t *testing.T) {
	bucket := createBucketTempForTest(t)

	acl, aclErr := bucket.GetACL()
	if aclErr != nil {
		t.Error(aclErr)
	}
	if acl == "" {
		t.Error("acl string shouldn't be nil")
	}
	putErr := bucket.SetACL(ACL_PUBLIC_READ)
	if putErr != nil {
		t.Error(putErr)
	}

	deleteBucketForTest(t, bucket)
	t.Fail()
}
<commit_msg>fix fail test<commit_after>package bcsgo

import (
	"strconv"
	"testing"
	"time"
)

func randomGlobalBucketName(index int) string {
	return ak[:2] + "-sdk-" + strconv.FormatInt(time.Now().Unix(), 10)[5:] + "-" + strconv.Itoa(index)
}

func createBucketTempForTest(t *testing.T) *Bucket {
	bucketName := randomGlobalBucketName(1)
	bucket := bcs.Bucket(bucketName)
	err := bucket.Create()
	if err != nil {
		t.Error(err)
	}
	return bucket
}

func deleteBucketForTest(t *testing.T, bucket *Bucket) {
	deleteErr := bucket.Delete()
	if deleteErr != nil {
		t.Error(deleteErr)
	}
}

func TestBucketCreateWithACLAndDelete(t *testing.T) {
	bucketName := randomGlobalBucketName(1)
	newBucket := bcs.Bucket(bucketName)
	bucketErr := newBucket.CreateWithACL(ACL_PUBLIC_READ)
	if bucketErr != nil {
		t.Error(bucketErr)
	}

	// bucketACL, bucketACLErr := newBucket.GetACL()
	// expectedBucketACL := fmt.Sprintf(`{"statements":[{"action":["*"],"effect":"allow","resource":["testsml2\/"],"user":["psp:egg90"]},{"action":["get_object"],"effect":"allow","resource":["%s\/"],"user":["*"]}]}`, bucketName)
	// if bucketACLErr != nil {
	// 	fmt.Println(bucketACLErr)
	// 	t.Fail()
	// }
	// if bucketACL != expectedBucketACL {
	// 	fmt.Println(bucketACL)
	// 	fmt.Println(expectedBucketACL)
	// 	t.Fail()
	// }

	bucketErr = newBucket.Delete()
	if bucketErr != nil {
		t.Error(bucketErr)
	}
}

func TestBucketCreateWithInvalidName(t *testing.T) {
	newBucket := bcs.Bucket("testErrorBucket")
	bucketErr := newBucket.Create()
	// It shall be failed.
	if bucketErr == nil {
		t.Error("create bucket with invaid name should failed")
	}
}

func TestBucketListObjects(t *testing.T) {
	bucket := createBucketTempForTest(t)

	// todo prefix
	objects, e := bucket.ListObjects("", 0, 5)
	if e != nil {
		t.Error("object list shouldn't be nil")
	}
	for _, pObject := range objects.Objects {
		if pObject == nil {
			t.Error("object should not be nil")
		}
	}

	deleteBucketForTest(t, bucket)
}

func TestBucketACL(t *testing.T) {
	bucket := createBucketTempForTest(t)

	acl, aclErr := bucket.GetACL()
	if aclErr != nil {
		t.Error(aclErr)
	}
	if acl == "" {
		t.Error("acl string shouldn't be nil")
	}
	putErr := bucket.SetACL(ACL_PUBLIC_READ)
	if putErr != nil {
		t.Error(putErr)
	}

	deleteBucketForTest(t, bucket)
}
<|endoftext|>


<commit_before>// Package azureFS implements fs.FileProvider
// and handles Azure blob storage
package azureFS

//"github.com/Azure/azure-sdk-for-go/storage"
import (
	"fmt"
	"strings"
	"time"

	"github.com/Azure/azure-sdk-for-go/storage"
	log "github.com/Sirupsen/logrus"
	"github.com/mindflavor/ftpserver2/ftp/fs"
	"github.com/mindflavor/ftpserver2/ftp/fs/azure/azureBlob"
	"github.com/mindflavor/ftpserver2/ftp/fs/azure/azureContainer"
	"github.com/mindflavor/ftpserver2/identity"
)

type azureFS struct {
	id                   identity.Identity
	client               storage.BlobStorageClient
	currentRealDirectory string
}

func (pfs *azureFS) String() string {
	return fmt.Sprintf("id:%s, currentRealDirectory: %s", pfs.id, pfs.currentRealDirectory)
}

// New initializes a new fs.FileProvider with a specific Azure account and key
func New(account, secret string) (fs.FileProvider, error) {
	cli, err := storage.NewClient(account, secret, storage.DefaultBaseURL, storage.DefaultAPIVersion, true)
	if err != nil {
		return nil, err
	}

	return &azureFS{
		id:                   nil,
		client:               cli.GetBlobService(),
		currentRealDirectory: "",
	}, nil
}

func (pfs *azureFS) Identity() identity.Identity {
	return pfs.id
}
func (pfs *azureFS) SetIdentity(identity identity.Identity) {
	pfs.id = identity
}

func (pfs *azureFS) CurrentDirectory() string {
	return "/" + pfs.currentRealDirectory
}

func (pfs *azureFS) List() ([]fs.File, error) {
	if pfs.CurrentDirectory() == "/" {
		// list containers

		// TODO
		// we should check for more than 1000 entries
		lcParams := storage.ListContainersParameters{MaxResults: 1000}
		lbr, err := pfs.client.ListContainers(lcParams)
		if err != nil {
			return nil, err
		}

		cnts := make([]fs.File, len(lbr.Containers))

		for i, item := range lbr.Containers {
			cnts[i] = azureContainer.New(item.Name, parseAzureTime(item.Properties.LastModified), pfs.client)
		}

		return cnts, nil
	}

	// files!
	lbParams := storage.ListBlobsParameters{MaxResults: 1000}
	lbr, err := pfs.client.ListBlobs(pfs.currentRealDirectory, lbParams)
	if err != nil {
		return nil, err
	}

	blobs := make([]fs.File, len(lbr.Blobs))

	for i, item := range lbr.Blobs {
		blobs[i] = azureBlob.New(item.Name, pfs.currentRealDirectory, item.Properties.ContentLength, parseAzureTime(item.Properties.LastModified), 0666, pfs.client)
	}

	return blobs, nil
}

func (pfs *azureFS) Get(filename string) (fs.File, error) {
	fullpath := filename
	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + filename
	}

	toks := splitAndCleanPath(fullpath)
	log.WithFields(log.Fields{"pfs": pfs, "filename": filename, "fullpath": fullpath, "toks": toks}).Debug("azureFS::azureFS::Get called")

	if len(toks) == 0 { // root
		return azureContainer.New("", time.Now(), pfs.client), nil
	}
	if len(toks) == 1 { // containter
		return azureContainer.New(filename, time.Now(), pfs.client), nil
	}

	// else blob
	props, err := pfs.client.GetBlobProperties(toks[0], strings.Join(toks[1:], "/"))
	if err != nil {
		return nil, err
	}
	return azureBlob.New(strings.Join(toks[1:], "/"), toks[0], props.ContentLength, parseAzureTime(props.LastModified), 0666, pfs.client), nil
}

func (pfs *azureFS) New(filename string, isDirectory bool) (fs.File, error) {
	fullpath := filename
	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + filename
	}

	log.WithFields(log.Fields{"pfs": pfs, "filename": filename, "fullpath": fullpath, "isDirectory": isDirectory}).Debug("azureFS::azureFS::New called")

	toks := splitAndCleanPath(fullpath)

	if len(toks) == 1 { // container
		return azureContainer.New(filename, time.Now(), pfs.client), nil
	}

	return azureBlob.New(toks[1], toks[0], 0, time.Now(), 0666, pfs.client), nil
}

func (pfs *azureFS) Clone() fs.FileProvider {
	return &azureFS{
		id:                   pfs.id,
		client:               pfs.client,
		currentRealDirectory: pfs.currentRealDirectory,
	}
}

func (pfs *azureFS) ChangeDirectory(path string) error {
	fullpath := path

	if len(path) == 0 {
		pfs.currentRealDirectory = ""
		log.WithFields(log.Fields{"pfs": pfs, "path": path}).Debug("azureFS::azureFS::ChangeDirectory changed to root /")
		return nil
	}

	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + path
	}

	log.WithFields(log.Fields{"pfs": pfs, "path": path, "fullpath": fullpath}).Debug("azureFS::azureFS::ChangeDirectory called")

	toks := splitAndCleanPath(fullpath)

	if len(toks) == 0 {
		pfs.currentRealDirectory = ""
		log.WithFields(log.Fields{"pfs": pfs, "path": path, "len(toks)": len(toks), "toks": toks}).Debug("azureFS::azureFS::ChangeDirectory changed to root /")
		return nil
	}

	if toks[len(toks)-1] == ".." { // strip .. folder
		if len(toks) == 1 { // root
			pfs.currentRealDirectory = ""
			log.WithFields(log.Fields{"pfs": pfs, "path": path, "len(toks)": len(toks), "toks": toks}).Debug("azureFS::azureFS::ChangeDirectory changed to root /")
			return nil
		}
		toks = toks[:len(toks)-2]
	}

	if len(toks) == 1 {
		pfs.currentRealDirectory = toks[0]
		log.WithFields(log.Fields{"pfs": pfs, "path": path, "len(toks)": len(toks), "toks": toks}).Debug("azureFS::azureFS::ChangeDirectory changed to container")
		return nil
	}

	log.WithFields(log.Fields{"pfs": pfs, "path": path, "len(toks)": len(toks), "toks": toks}).Warn("azureFS::azureFS::ChangeDirectory azure storage does not support nested containers")
	return fmt.Errorf("cannot change directory: azure storage supports only root level container")
}

func (pfs *azureFS) CreateDirectory(path string) error {
	fullpath := path
	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + path
	}

	log.WithFields(log.Fields{"pfs": pfs, "path": path, "fullpath": fullpath}).Debug("azureFS::azureFS::CreateDirectory called")

	toks := splitAndCleanPath(fullpath)

	if len(toks) > 1 {
		return fmt.Errorf("cannot nest subdirectories in azure storage")
	}

	return pfs.client.CreateContainer(path, storage.ContainerAccessTypePrivate)

}

func (pfs *azureFS) RemoveDirectory(path string) error {
	fullpath := path
	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + path
	}

	log.WithFields(log.Fields{"pfs": pfs, "path": path, "fullpath": fullpath}).Debug("azureFS::azureFS::RemoveDirectory called")

	toks := splitAndCleanPath(fullpath)

	if len(toks) > 1 {
		return fmt.Errorf("there are no subdirectories in Azure storage")
	}

	return pfs.client.DeleteContainer(path)
}

func parseAzureTime(tToParse string) time.Time {
	log.WithFields(log.Fields{"tToParse": tToParse}).Debug("azureFS::parseAzureTime called")
	t, _ := time.Parse(time.RFC1123, tToParse)
	return t
}

func splitAndCleanPath(s string) []string {
	var toks []string
	for _, item := range strings.Split(s, "/") {
		if item != "" {
			toks = append(toks, item)
		}
	}

	return toks
}
<commit_msg>Add nested CWD&MKD support<commit_after>// Package azureFS implements fs.FileProvider
// and handles Azure blob storage
package azureFS

//"github.com/Azure/azure-sdk-for-go/storage"
import (
	"fmt"
	"strings"
	"time"

	"github.com/Azure/azure-sdk-for-go/storage"
	log "github.com/Sirupsen/logrus"
	"github.com/mindflavor/ftpserver2/ftp/fs"
	"github.com/mindflavor/ftpserver2/ftp/fs/azure/azureBlob"
	"github.com/mindflavor/ftpserver2/ftp/fs/azure/azureContainer"
	"github.com/mindflavor/ftpserver2/identity"
)

type azureFS struct {
	id                   identity.Identity
	client               storage.BlobStorageClient
	currentRealDirectory string
}

func (pfs *azureFS) String() string {
	return fmt.Sprintf("id:%s, currentRealDirectory: %s", pfs.id, pfs.currentRealDirectory)
}

// New initializes a new fs.FileProvider with a specific Azure account and key
func New(account, secret string) (fs.FileProvider, error) {
	cli, err := storage.NewClient(account, secret, storage.DefaultBaseURL, storage.DefaultAPIVersion, true)
	if err != nil {
		return nil, err
	}

	return &azureFS{
		id:                   nil,
		client:               cli.GetBlobService(),
		currentRealDirectory: "",
	}, nil
}

func (pfs *azureFS) Identity() identity.Identity {
	return pfs.id
}
func (pfs *azureFS) SetIdentity(identity identity.Identity) {
	pfs.id = identity
}

func (pfs *azureFS) CurrentDirectory() string {
	return "/" + pfs.currentRealDirectory
}

func (pfs *azureFS) List() ([]fs.File, error) {
	if pfs.CurrentDirectory() == "/" {
		// list containers

		// TODO
		// we should check for more than 1000 entries
		lcParams := storage.ListContainersParameters{MaxResults: 1000}
		lbr, err := pfs.client.ListContainers(lcParams)
		if err != nil {
			return nil, err
		}

		cnts := make([]fs.File, len(lbr.Containers))

		for i, item := range lbr.Containers {
			cnts[i] = azureContainer.New(item.Name, parseAzureTime(item.Properties.LastModified), pfs.client)
		}

		return cnts, nil
	}

	// files!
	lbParams := storage.ListBlobsParameters{MaxResults: 1000}
	lbr, err := pfs.client.ListBlobs(pfs.currentRealDirectory, lbParams)
	if err != nil {
		return nil, err
	}

	blobs := make([]fs.File, len(lbr.Blobs))

	for i, item := range lbr.Blobs {
		blobs[i] = azureBlob.New(item.Name, pfs.currentRealDirectory, item.Properties.ContentLength, parseAzureTime(item.Properties.LastModified), 0666, pfs.client)
	}

	return blobs, nil
}

func (pfs *azureFS) Get(filename string) (fs.File, error) {
	fullpath := filename
	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + filename
	}

	toks := splitAndCleanPath(fullpath)
	log.WithFields(log.Fields{"pfs": pfs, "filename": filename, "fullpath": fullpath, "toks": toks}).Debug("azureFS::azureFS::Get called")

	if len(toks) == 0 { // root
		return azureContainer.New("", time.Now(), pfs.client), nil
	}
	if len(toks) == 1 { // containter
		return azureContainer.New(filename, time.Now(), pfs.client), nil
	}

	// else blob
	props, err := pfs.client.GetBlobProperties(toks[0], strings.Join(toks[1:], "/"))
	if err != nil {
		return nil, err
	}
	return azureBlob.New(strings.Join(toks[1:], "/"), toks[0], props.ContentLength, parseAzureTime(props.LastModified), 0666, pfs.client), nil
}

func (pfs *azureFS) New(filename string, isDirectory bool) (fs.File, error) {
	fullpath := filename
	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + filename
	}

	log.WithFields(log.Fields{"pfs": pfs, "filename": filename, "fullpath": fullpath, "isDirectory": isDirectory}).Debug("azureFS::azureFS::New called")

	toks := splitAndCleanPath(fullpath)

	if len(toks) == 1 { // container
		return azureContainer.New(filename, time.Now(), pfs.client), nil
	}

	return azureBlob.New(toks[1], toks[0], 0, time.Now(), 0666, pfs.client), nil
}

func (pfs *azureFS) Clone() fs.FileProvider {
	return &azureFS{
		id:                   pfs.id,
		client:               pfs.client,
		currentRealDirectory: pfs.currentRealDirectory,
	}
}

func (pfs *azureFS) ChangeDirectory(path string) error {
	fullpath := path

	if len(path) == 0 {
		pfs.currentRealDirectory = ""
		log.WithFields(log.Fields{"pfs": pfs, "path": path}).Debug("azureFS::azureFS::ChangeDirectory changed to root /")
		return nil
	}

	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + path
	}

	log.WithFields(log.Fields{"pfs": pfs, "path": path, "fullpath": fullpath}).Debug("azureFS::azureFS::ChangeDirectory called")

	toks := splitAndCleanPath(fullpath)

	if len(toks) == 0 {
		pfs.currentRealDirectory = ""
		log.WithFields(log.Fields{"pfs": pfs, "path": path, "len(toks)": len(toks), "toks": toks}).Debug("azureFS::azureFS::ChangeDirectory changed to root /")
		return nil
	}

	if toks[len(toks)-1] == ".." { // strip .. folder
		if len(toks) == 1 { // root
			pfs.currentRealDirectory = ""
			log.WithFields(log.Fields{"pfs": pfs, "path": path, "len(toks)": len(toks), "toks": toks}).Debug("azureFS::azureFS::ChangeDirectory changed to root /")
			return nil
		}
		toks = toks[:len(toks)-2]
	}

	// Container
	if len(toks) == 1 {
		exists, err := pfs.client.ContainerExists(toks[0])
		if err != nil {
			return err
		}
		if !exists {
			return fmt.Errorf("cannot change directory: container not found")
		}
		pfs.currentRealDirectory = toks[0]
		log.WithFields(log.Fields{"pfs": pfs, "path": path, "len(toks)": len(toks), "toks": toks}).Debug("azureFS::azureFS::ChangeDirectory changed to container")
		return nil
	}

	// Then, Blob
	exists, err := pfs.client.BlobExists(toks[0], strings.Join(toks[1:], "/"))
	if err != nil {
		return err
	}
	if !exists {
		return fmt.Errorf("cannot change directory: blob not found")
	}
	pfs.currentRealDirectory = strings.Join(toks, "/")

	log.WithFields(log.Fields{"pfs": pfs, "path": path, "len(toks)": len(toks), "toks": toks}).Debug("azureFS::azureFS::ChangeDirectory changed to blob")

	return nil
}

func (pfs *azureFS) CreateDirectory(path string) error {
	fullpath := path
	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + path
	}

	log.WithFields(log.Fields{"pfs": pfs, "path": path, "fullpath": fullpath}).Debug("azureFS::azureFS::CreateDirectory called")

	toks := splitAndCleanPath(fullpath)

	if _, err := pfs.client.CreateContainerIfNotExists(toks[0], storage.ContainerAccessTypePrivate); err != nil {
		return err
	}

	// Container only
	if len(toks) == 1 {
		return nil
	}

	// Blob
	return pfs.client.CreateBlockBlob(toks[0], strings.Join(toks[1:], "/"))
}

func (pfs *azureFS) RemoveDirectory(path string) error {
	fullpath := path
	if fullpath[0] != '/' {
		fullpath = "/" + pfs.currentRealDirectory + "/" + path
	}

	log.WithFields(log.Fields{"pfs": pfs, "path": path, "fullpath": fullpath}).Debug("azureFS::azureFS::RemoveDirectory called")

	toks := splitAndCleanPath(fullpath)

	if len(toks) > 1 {
		return fmt.Errorf("there are no subdirectories in Azure storage")
	}

	return pfs.client.DeleteContainer(path)
}

func parseAzureTime(tToParse string) time.Time {
	log.WithFields(log.Fields{"tToParse": tToParse}).Debug("azureFS::parseAzureTime called")
	t, _ := time.Parse(time.RFC1123, tToParse)
	return t
}

func splitAndCleanPath(s string) []string {
	var toks []string
	for _, item := range strings.Split(s, "/") {
		if item != "" {
			toks = append(toks, item)
		}
	}

	return toks
}
<|endoftext|>