<commit_before>package main

import (
	"encoding/json"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/datawire/ambassador/pkg/consulwatch"
	"github.com/datawire/ambassador/pkg/limiter"
	"github.com/datawire/ambassador/pkg/watt"

	"github.com/datawire/ambassador/pkg/k8s"
	"github.com/datawire/ambassador/pkg/supervisor"
)

type WatchHook func(p *supervisor.Process, snapshot string) WatchSet

type aggregator struct {
	// Input channel used to tell us about kubernetes state.
	KubernetesEvents chan k8sEvent
	// Input channel used to tell us about consul endpoints.
	ConsulEvents chan consulEvent
	// Output channel used to communicate with the k8s watch manager.
	k8sWatches chan<- []KubernetesWatchSpec
	// Output channel used to communicate with the consul watch manager.
	consulWatches chan<- []ConsulWatchSpec
	// Output channel used to communicate with the invoker.
	snapshots chan<- string
	// We won't consider ourselves "bootstrapped" until we hear
	// about all these kinds.
	requiredKinds       []string
	watchHook           WatchHook
	limiter             limiter.Limiter
	ids                 map[string]bool
	kubernetesResources map[string]map[string][]k8s.Resource
	consulEndpoints     map[string]consulwatch.Endpoints
	bootstrapped        bool
	notifyMux           sync.Mutex
	errors              map[string][]watt.Error
}

func NewAggregator(snapshots chan<- string, k8sWatches chan<- []KubernetesWatchSpec, consulWatches chan<- []ConsulWatchSpec,
	requiredKinds []string, watchHook WatchHook, limiter limiter.Limiter) *aggregator {
	return &aggregator{
		KubernetesEvents:    make(chan k8sEvent),
		ConsulEvents:        make(chan consulEvent),
		k8sWatches:          k8sWatches,
		consulWatches:       consulWatches,
		snapshots:           snapshots,
		requiredKinds:       requiredKinds,
		watchHook:           watchHook,
		limiter:             limiter,
		ids:                 make(map[string]bool),
		kubernetesResources: make(map[string]map[string][]k8s.Resource),
		consulEndpoints:     make(map[string]consulwatch.Endpoints),
		errors:              make(map[string][]watt.Error),
	}
}

func (a *aggregator) Work(p *supervisor.Process) error {
	// In order to invoke `maybeNotify`, which is a very time consuming
	// operation, we coalesce events:
	//
	// 1. Be continuously reading all available events from
	//    a.KubernetesEvents and a.ConsulEvents and store k8sEvents
	//    in the potentialKubernetesEventSignal variable. This means
	//    at any given point (modulo caveats below), the
	//    potentialKubernetesEventSignal variable will have the
	//    latest Kubernetes event available.
	//
	// 2. At the same time, whenever there is capacity to write
	//    down the kubernetesEventProcessor channel, we send
	//    potentialKubernetesEventSignal to be processed.
	//
	//    The anonymous goroutine below will be constantly reading
	//    from the kubernetesEventProcessor channel and performing
	//    a blocking a.maybeNotify(). This means that we can only
	//    *write* to the kubernetesEventProcessor channel when we are
	//    not currently processing an event, but when that happens, we
	//    will still read from a.KubernetesEvents and a.ConsulEvents
	//    and update potentialKubernetesEventSignal.
	//
	// There are three caveats to the above:
	//
	// 1. At startup, we don't yet have a event to write, but
	//    we're not processing anything, so we will try to write
	//    something down the kubernetesEventProcessor channel.
	//    To cope with this, the invoking goroutine will ignore events
	//    signals that have a event.skip flag.
	//
	// 2. If we process an event quickly, or if there aren't new
	//    events available, then we end up busy looping and
	//    sending the same potentialKubernetesEventSignal value down
	//    the kubernetesEventProcessor channel multiple times. To cope
	//    with this, whenever we have successfully written to the
	//    kubernetesEventProcessor channel, we do a *blocking* read of
	//    the next event from a.KubernetesEvents and a.ConsulEvents.
	//
	// 3. Always be calling a.setKubernetesResources as soon as we
	//    receive an event. This is a fast non-blocking call that
	//    update watches, we can't coalesce this call.

	p.Ready()

	type eventSignal struct {
		kubernetesEvent k8sEvent
		skip            bool
	}

	kubernetesEventProcessor := make(chan eventSignal)
	go func() {
		for event := range kubernetesEventProcessor {
			if event.skip {
				// ignore the initial eventSignal to deal with the
				// corner case where we haven't yet received an event yet.
				continue
			}
			a.maybeNotify(p)
		}
	}()

	potentialKubernetesEventSignal := eventSignal{kubernetesEvent: k8sEvent{}, skip: true}
	for {
		select {
		case potentialKubernetesEvent := <-a.KubernetesEvents:
			// if a new KubernetesEvents is available to be read,
			// and we can't write to the kubernetesEventProcessor channel,
			// then we will overwrite potentialKubernetesEvent
			// with a newer event while still processing a.setKubernetesResources
			a.setKubernetesResources(potentialKubernetesEvent)
			potentialKubernetesEventSignal = eventSignal{kubernetesEvent: potentialKubernetesEvent, skip: false}
		case kubernetesEventProcessor <- potentialKubernetesEventSignal:
			// if we aren't currently blocked in
			// a.maybeNotify() then the above goroutine will be
			// reading from the kubernetesEventProcessor channel and we
			// will send the current potentialKubernetesEventSignal
			// value over the kubernetesEventProcessor channel to be
			// processed
			select {
			case potentialKubernetesEvent := <-a.KubernetesEvents:
				a.setKubernetesResources(potentialKubernetesEvent)
				potentialKubernetesEventSignal = eventSignal{kubernetesEvent: potentialKubernetesEvent, skip: false}
			case event := <-a.ConsulEvents:
				a.updateConsulResources(event)
				a.maybeNotify(p)
			case <-p.Shutdown():
				return nil
			}
		case event := <-a.ConsulEvents:
			// we are always reading and processing ConsulEvents directly,
			// not coalescing them.
			a.updateConsulResources(event)
			a.maybeNotify(p)
		case <-p.Shutdown():
			return nil
		}
	}
}

func (a *aggregator) updateConsulResources(event consulEvent) {
	a.ids[event.WatchId] = true
	a.consulEndpoints[event.Endpoints.Service] = event.Endpoints
}

func (a *aggregator) setKubernetesResources(event k8sEvent) {
	if len(event.errors) > 0 {
		for _, kError := range event.errors {
			a.errors[kError.Source] = append(a.errors[kError.Source], kError)
		}
		return
	}
	a.ids[event.watchId] = true
	submap, ok := a.kubernetesResources[event.watchId]
	if !ok {
		submap = make(map[string][]k8s.Resource)
		a.kubernetesResources[event.watchId] = submap
	}
	submap[event.kind] = event.resources
}

func (a *aggregator) generateSnapshot() (string, error) {
	k8sResources := make(map[string][]k8s.Resource)
	for _, submap := range a.kubernetesResources {
		for k, v := range submap {
			k8sResources[k] = append(k8sResources[k], v...)
		}
	}
	s := watt.Snapshot{
		Consul:     watt.ConsulSnapshot{Endpoints: a.consulEndpoints},
		Kubernetes: k8sResources,
		Errors:     a.errors,
	}

	jsonBytes, err := json.MarshalIndent(s, "", "    ")
	if err != nil {
		return "{}", err
	}

	return string(jsonBytes), nil
}

func (a *aggregator) isKubernetesBootstrapped(p *supervisor.Process) bool {
	submap, sok := a.kubernetesResources[""]
	if !sok {
		return false
	}
	for _, k := range a.requiredKinds {
		_, ok := submap[k]
		if !ok {
			return false
		}
	}
	return true
}

// Returns true if the current state of the world is complete. The
// kubernetes state of the world is always complete by definition
// because the kubernetes client provides that guarantee. The
// aggregate state of the world is complete when any consul services
// referenced by kubernetes have populated endpoint information (even
// if the value of the populated info is an empty set of endpoints).
func (a *aggregator) isComplete(p *supervisor.Process, watchset WatchSet) bool {
	complete := true

	for _, w := range watchset.KubernetesWatches {
		if _, ok := a.ids[w.WatchId()]; ok {
			p.Logf("initialized k8s watch: %s", w.WatchId())
		} else {
			complete = false
			p.Logf("waiting for k8s watch: %s", w.WatchId())
		}
	}

	for _, w := range watchset.ConsulWatches {
		if _, ok := a.ids[w.WatchId()]; ok {
			p.Logf("initialized consul watch: %s", w.WatchId())
		} else {
			complete = false
			p.Logf("waiting for consul watch: %s", w.WatchId())
		}
	}

	return complete
}

func (a *aggregator) maybeNotify(p *supervisor.Process) {
	now := time.Now()
	delay := a.limiter.Limit(now)
	if delay == 0 {
		a.notify(p)
	} else if delay > 0 {
		time.AfterFunc(delay, func() {
			a.notify(p)
		})
	}
}

func (a *aggregator) notify(p *supervisor.Process) {
	a.notifyMux.Lock()
	defer a.notifyMux.Unlock()

	if !a.isKubernetesBootstrapped(p) {
		return
	}

	watchset := a.getWatches(p)

	p.Logf("found %d kubernetes watches", len(watchset.KubernetesWatches))
	p.Logf("found %d consul watches", len(watchset.ConsulWatches))
	a.k8sWatches <- watchset.KubernetesWatches
	a.consulWatches <- watchset.ConsulWatches

	if !a.bootstrapped && a.isComplete(p, watchset) {
		p.Logf("bootstrapped!")
		a.bootstrapped = true
	}

	if a.bootstrapped {
		snapshot, err := a.generateSnapshot()
		if err != nil {
			p.Logf("generate snapshot failed %v", err)
			return
		}

		a.snapshots <- snapshot
	}
}

func (a *aggregator) getWatches(p *supervisor.Process) WatchSet {
	snapshot, err := a.generateSnapshot()
	if err != nil {
		p.Logf("generate snapshot failed %v", err)
		return WatchSet{}
	}
	result := a.watchHook(p, snapshot)
	return result.interpolate()
}

func ExecWatchHook(watchHooks []string) WatchHook {
	return func(p *supervisor.Process, snapshot string) WatchSet {
		result := WatchSet{}

		for _, hook := range watchHooks {
			ws := invokeHook(p, hook, snapshot)
			result.KubernetesWatches = append(result.KubernetesWatches, ws.KubernetesWatches...)
			result.ConsulWatches = append(result.ConsulWatches, ws.ConsulWatches...)
		}

		return result
	}
}

func lines(st string) []string {
	return strings.Split(st, "\n")
}

func invokeHook(p *supervisor.Process, hook, snapshot string) WatchSet {
	cmd := exec.Command("sh", "-c", hook)
	cmd.Stdin = strings.NewReader(snapshot)
	var watches, errors strings.Builder
	cmd.Stdout = &watches
	cmd.Stderr = &errors
	err := cmd.Run()
	stderr := errors.String()
	if stderr != "" {
		for _, line := range lines(stderr) {
			p.Logf("watch hook stderr: %s", line)
		}
	}
	if err != nil {
		p.Logf("watch hook failed: %v", err)
		return WatchSet{}
	}

	encoded := watches.String()

	decoder := json.NewDecoder(strings.NewReader(encoded))
	decoder.DisallowUnknownFields()
	result := WatchSet{}
	err = decoder.Decode(&result)
	if err != nil {
		for _, line := range lines(encoded) {
			p.Logf("watch hook: %s", line)
		}
		p.Logf("watchset decode failed: %v", err)
		return WatchSet{}
	}

	return result
}
<commit_msg>Added additional comment about blocking read<commit_after>package main

import (
	"encoding/json"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/datawire/ambassador/pkg/consulwatch"
	"github.com/datawire/ambassador/pkg/limiter"
	"github.com/datawire/ambassador/pkg/watt"

	"github.com/datawire/ambassador/pkg/k8s"
	"github.com/datawire/ambassador/pkg/supervisor"
)

type WatchHook func(p *supervisor.Process, snapshot string) WatchSet

type aggregator struct {
	// Input channel used to tell us about kubernetes state.
	KubernetesEvents chan k8sEvent
	// Input channel used to tell us about consul endpoints.
	ConsulEvents chan consulEvent
	// Output channel used to communicate with the k8s watch manager.
	k8sWatches chan<- []KubernetesWatchSpec
	// Output channel used to communicate with the consul watch manager.
	consulWatches chan<- []ConsulWatchSpec
	// Output channel used to communicate with the invoker.
	snapshots chan<- string
	// We won't consider ourselves "bootstrapped" until we hear
	// about all these kinds.
	requiredKinds       []string
	watchHook           WatchHook
	limiter             limiter.Limiter
	ids                 map[string]bool
	kubernetesResources map[string]map[string][]k8s.Resource
	consulEndpoints     map[string]consulwatch.Endpoints
	bootstrapped        bool
	notifyMux           sync.Mutex
	errors              map[string][]watt.Error
}

func NewAggregator(snapshots chan<- string, k8sWatches chan<- []KubernetesWatchSpec, consulWatches chan<- []ConsulWatchSpec,
	requiredKinds []string, watchHook WatchHook, limiter limiter.Limiter) *aggregator {
	return &aggregator{
		KubernetesEvents:    make(chan k8sEvent),
		ConsulEvents:        make(chan consulEvent),
		k8sWatches:          k8sWatches,
		consulWatches:       consulWatches,
		snapshots:           snapshots,
		requiredKinds:       requiredKinds,
		watchHook:           watchHook,
		limiter:             limiter,
		ids:                 make(map[string]bool),
		kubernetesResources: make(map[string]map[string][]k8s.Resource),
		consulEndpoints:     make(map[string]consulwatch.Endpoints),
		errors:              make(map[string][]watt.Error),
	}
}

func (a *aggregator) Work(p *supervisor.Process) error {
	// In order to invoke `maybeNotify`, which is a very time consuming
	// operation, we coalesce events:
	//
	// 1. Be continuously reading all available events from
	//    a.KubernetesEvents and a.ConsulEvents and store k8sEvents
	//    in the potentialKubernetesEventSignal variable. This means
	//    at any given point (modulo caveats below), the
	//    potentialKubernetesEventSignal variable will have the
	//    latest Kubernetes event available.
	//
	// 2. At the same time, whenever there is capacity to write
	//    down the kubernetesEventProcessor channel, we send
	//    potentialKubernetesEventSignal to be processed.
	//
	//    The anonymous goroutine below will be constantly reading
	//    from the kubernetesEventProcessor channel and performing
	//    a blocking a.maybeNotify(). This means that we can only
	//    *write* to the kubernetesEventProcessor channel when we are
	//    not currently processing an event, but when that happens, we
	//    will still read from a.KubernetesEvents and a.ConsulEvents
	//    and update potentialKubernetesEventSignal.
	//
	// There are three caveats to the above:
	//
	// 1. At startup, we don't yet have a event to write, but
	//    we're not processing anything, so we will try to write
	//    something down the kubernetesEventProcessor channel.
	//    To cope with this, the invoking goroutine will ignore events
	//    signals that have a event.skip flag.
	//
	// 2. If we process an event quickly, or if there aren't new
	//    events available, then we end up busy looping and
	//    sending the same potentialKubernetesEventSignal value down
	//    the kubernetesEventProcessor channel multiple times. To cope
	//    with this, whenever we have successfully written to the
	//    kubernetesEventProcessor channel, we do a *blocking* read of
	//    the next event from a.KubernetesEvents and a.ConsulEvents.
	//
	// 3. Always be calling a.setKubernetesResources as soon as we
	//    receive an event. This is a fast non-blocking call that
	//    update watches, we can't coalesce this call.

	p.Ready()

	type eventSignal struct {
		kubernetesEvent k8sEvent
		skip            bool
	}

	kubernetesEventProcessor := make(chan eventSignal)
	go func() {
		for event := range kubernetesEventProcessor {
			if event.skip {
				// ignore the initial eventSignal to deal with the
				// corner case where we haven't yet received an event yet.
				continue
			}
			a.maybeNotify(p)
		}
	}()

	potentialKubernetesEventSignal := eventSignal{kubernetesEvent: k8sEvent{}, skip: true}
	for {
		select {
		case potentialKubernetesEvent := <-a.KubernetesEvents:
			// if a new KubernetesEvents is available to be read,
			// and we can't write to the kubernetesEventProcessor channel,
			// then we will overwrite potentialKubernetesEvent
			// with a newer event while still processing a.setKubernetesResources
			a.setKubernetesResources(potentialKubernetesEvent)
			potentialKubernetesEventSignal = eventSignal{kubernetesEvent: potentialKubernetesEvent, skip: false}
		case kubernetesEventProcessor <- potentialKubernetesEventSignal:
			// if we aren't currently blocked in
			// a.maybeNotify() then the above goroutine will be
			// reading from the kubernetesEventProcessor channel and we
			// will send the current potentialKubernetesEventSignal
			// value over the kubernetesEventProcessor channel to be
			// processed
			select {
			case potentialKubernetesEvent := <-a.KubernetesEvents:
				// here we do blocking read of the next event for caveat #2.
				a.setKubernetesResources(potentialKubernetesEvent)
				potentialKubernetesEventSignal = eventSignal{kubernetesEvent: potentialKubernetesEvent, skip: false}
			case event := <-a.ConsulEvents:
				a.updateConsulResources(event)
				a.maybeNotify(p)
			case <-p.Shutdown():
				return nil
			}
		case event := <-a.ConsulEvents:
			// we are always reading and processing ConsulEvents directly,
			// not coalescing them.
			a.updateConsulResources(event)
			a.maybeNotify(p)
		case <-p.Shutdown():
			return nil
		}
	}
}

func (a *aggregator) updateConsulResources(event consulEvent) {
	a.ids[event.WatchId] = true
	a.consulEndpoints[event.Endpoints.Service] = event.Endpoints
}

func (a *aggregator) setKubernetesResources(event k8sEvent) {
	if len(event.errors) > 0 {
		for _, kError := range event.errors {
			a.errors[kError.Source] = append(a.errors[kError.Source], kError)
		}
		return
	}
	a.ids[event.watchId] = true
	submap, ok := a.kubernetesResources[event.watchId]
	if !ok {
		submap = make(map[string][]k8s.Resource)
		a.kubernetesResources[event.watchId] = submap
	}
	submap[event.kind] = event.resources
}

func (a *aggregator) generateSnapshot() (string, error) {
	k8sResources := make(map[string][]k8s.Resource)
	for _, submap := range a.kubernetesResources {
		for k, v := range submap {
			k8sResources[k] = append(k8sResources[k], v...)
		}
	}
	s := watt.Snapshot{
		Consul:     watt.ConsulSnapshot{Endpoints: a.consulEndpoints},
		Kubernetes: k8sResources,
		Errors:     a.errors,
	}

	jsonBytes, err := json.MarshalIndent(s, "", "    ")
	if err != nil {
		return "{}", err
	}

	return string(jsonBytes), nil
}

func (a *aggregator) isKubernetesBootstrapped(p *supervisor.Process) bool {
	submap, sok := a.kubernetesResources[""]
	if !sok {
		return false
	}
	for _, k := range a.requiredKinds {
		_, ok := submap[k]
		if !ok {
			return false
		}
	}
	return true
}

// Returns true if the current state of the world is complete. The
// kubernetes state of the world is always complete by definition
// because the kubernetes client provides that guarantee. The
// aggregate state of the world is complete when any consul services
// referenced by kubernetes have populated endpoint information (even
// if the value of the populated info is an empty set of endpoints).
func (a *aggregator) isComplete(p *supervisor.Process, watchset WatchSet) bool {
	complete := true

	for _, w := range watchset.KubernetesWatches {
		if _, ok := a.ids[w.WatchId()]; ok {
			p.Logf("initialized k8s watch: %s", w.WatchId())
		} else {
			complete = false
			p.Logf("waiting for k8s watch: %s", w.WatchId())
		}
	}

	for _, w := range watchset.ConsulWatches {
		if _, ok := a.ids[w.WatchId()]; ok {
			p.Logf("initialized consul watch: %s", w.WatchId())
		} else {
			complete = false
			p.Logf("waiting for consul watch: %s", w.WatchId())
		}
	}

	return complete
}

func (a *aggregator) maybeNotify(p *supervisor.Process) {
	now := time.Now()
	delay := a.limiter.Limit(now)
	if delay == 0 {
		a.notify(p)
	} else if delay > 0 {
		time.AfterFunc(delay, func() {
			a.notify(p)
		})
	}
}

func (a *aggregator) notify(p *supervisor.Process) {
	a.notifyMux.Lock()
	defer a.notifyMux.Unlock()

	if !a.isKubernetesBootstrapped(p) {
		return
	}

	watchset := a.getWatches(p)

	p.Logf("found %d kubernetes watches", len(watchset.KubernetesWatches))
	p.Logf("found %d consul watches", len(watchset.ConsulWatches))
	a.k8sWatches <- watchset.KubernetesWatches
	a.consulWatches <- watchset.ConsulWatches

	if !a.bootstrapped && a.isComplete(p, watchset) {
		p.Logf("bootstrapped!")
		a.bootstrapped = true
	}

	if a.bootstrapped {
		snapshot, err := a.generateSnapshot()
		if err != nil {
			p.Logf("generate snapshot failed %v", err)
			return
		}

		a.snapshots <- snapshot
	}
}

func (a *aggregator) getWatches(p *supervisor.Process) WatchSet {
	snapshot, err := a.generateSnapshot()
	if err != nil {
		p.Logf("generate snapshot failed %v", err)
		return WatchSet{}
	}
	result := a.watchHook(p, snapshot)
	return result.interpolate()
}

func ExecWatchHook(watchHooks []string) WatchHook {
	return func(p *supervisor.Process, snapshot string) WatchSet {
		result := WatchSet{}

		for _, hook := range watchHooks {
			ws := invokeHook(p, hook, snapshot)
			result.KubernetesWatches = append(result.KubernetesWatches, ws.KubernetesWatches...)
			result.ConsulWatches = append(result.ConsulWatches, ws.ConsulWatches...)
		}

		return result
	}
}

func lines(st string) []string {
	return strings.Split(st, "\n")
}

func invokeHook(p *supervisor.Process, hook, snapshot string) WatchSet {
	cmd := exec.Command("sh", "-c", hook)
	cmd.Stdin = strings.NewReader(snapshot)
	var watches, errors strings.Builder
	cmd.Stdout = &watches
	cmd.Stderr = &errors
	err := cmd.Run()
	stderr := errors.String()
	if stderr != "" {
		for _, line := range lines(stderr) {
			p.Logf("watch hook stderr: %s", line)
		}
	}
	if err != nil {
		p.Logf("watch hook failed: %v", err)
		return WatchSet{}
	}

	encoded := watches.String()

	decoder := json.NewDecoder(strings.NewReader(encoded))
	decoder.DisallowUnknownFields()
	result := WatchSet{}
	err = decoder.Decode(&result)
	if err != nil {
		for _, line := range lines(encoded) {
			p.Logf("watch hook: %s", line)
		}
		p.Logf("watchset decode failed: %v", err)
		return WatchSet{}
	}

	return result
}
<|endoftext|>


<commit_before>package ifviva

import (
	"encoding/json"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"path"
)

const (
	contentType    = "Content-Type"
	contentText    = "text/plain"
	contentJSON    = "application/json"
	contentHTML    = "text/html"
	defaultCharset = "UTF-8"
)

var (
	cacheTemplate *template.Template
)

type Controller struct {
	Req        *http.Request
	Res        http.ResponseWriter
	Params     map[string]string
	statusCode int
	Err        error
	Charset    string
}

func SetViewPath(dir string) {
	viewPaths := []string{}
	scanDir(dir, func(viewPath string) {
		viewPaths = append(viewPaths, viewPath)
	})
	var err error
	cacheTemplate, err = template.ParseFiles(viewPaths...)
	if err != nil {
		log.Println("[ifviva]Set view path error: ", err)
	}
}

func scanDir(dir string, fn func(string)) {
	fileInfos, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Println("[ifviva]Set view path error: ", err)
		return
	}
	for _, fileInfo := range fileInfos {
		if fileInfo.IsDir() {
			scanDir(path.Join(dir, fileInfo.Name()), fn)
		} else {
			fn(path.Join(dir, fileInfo.Name()))
		}
	}
}

func (ctrl *Controller) Init(ctx Context) {
	ctrl.Req = ctx.Req
	ctrl.Res = ctx.Res
	ctrl.Params = ctx.Params
	ctrl.statusCode = 200
	ctrl.Charset = defaultCharset
}

func (ctrl *Controller) Status(status int) {
	ctrl.statusCode = status
}

func (ctrl *Controller) Text(text string) {
	ctrl.Res.Header().Set(contentType, appendCharset(contentText, ctrl.Charset))
	ctrl.Res.WriteHeader(ctrl.statusCode)
	ctrl.Res.Write([]byte(text))
}

func (ctrl *Controller) Json(v interface{}) {
	result, err := json.MarshalIndent(v, "", "  ")
	if err != nil {
		ctrl.InternalError(err)
		return
	}
	ctrl.Res.Header().Set(contentType, appendCharset(contentJSON, ctrl.Charset))
	ctrl.Res.WriteHeader(ctrl.statusCode)
	ctrl.Res.Write(result)
}

func (ctrl *Controller) View(name string, data interface{}) {
	ctrl.Res.Header().Set(contentType, appendCharset(contentHTML, ctrl.Charset))
	ctrl.Res.WriteHeader(ctrl.statusCode)
	err := cacheTemplate.ExecuteTemplate(ctrl.Res, name, data)
	if err != nil {
		ctrl.InternalError(err)
		return
	}
}

func (ctrl *Controller) InternalError(err error) {
	ctrl.Err = err
	ctrl.Res.WriteHeader(500)
	ctrl.Res.Write([]byte("Internal Server Error"))
}

func appendCharset(content string, charset string) string {
	return content + "; charset=" + charset
}
<commit_msg>添加开启和关闭视图缓存的功能<commit_after>package ifviva

import (
	"encoding/json"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"path"
)

const (
	contentType    = "Content-Type"
	contentText    = "text/plain"
	contentJSON    = "application/json"
	contentHTML    = "text/html"
	defaultCharset = "UTF-8"
)

var (
	isCache       bool = true
	cacheTemplate *template.Template
	viewDir       string
)

type Controller struct {
	Req        *http.Request
	Res        http.ResponseWriter
	Params     map[string]string
	statusCode int
	Err        error
	Charset    string
}

func SetViewPath(dir string) {
	viewDir = dir
	setCacheTemplate()
}

func OpenViewCache() {
	isCache = true
}

func CloseViewCache() {
	isCache = false
}

func setCacheTemplate() {
	viewPaths := []string{}
	scanDir(viewDir, func(viewPath string) {
		viewPaths = append(viewPaths, viewPath)
	})
	var err error
	cacheTemplate, err = template.ParseFiles(viewPaths...)
	if err != nil {
		log.Println("[ifviva]Set view path error: ", err)
	}
}

func scanDir(dir string, fn func(string)) {
	fileInfos, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Println("[ifviva]Set view path error: ", err)
		return
	}
	for _, fileInfo := range fileInfos {
		if fileInfo.IsDir() {
			scanDir(path.Join(dir, fileInfo.Name()), fn)
		} else {
			fn(path.Join(dir, fileInfo.Name()))
		}
	}
}

func (ctrl *Controller) Init(ctx Context) {
	ctrl.Req = ctx.Req
	ctrl.Res = ctx.Res
	ctrl.Params = ctx.Params
	ctrl.statusCode = 200
	ctrl.Charset = defaultCharset
}

func (ctrl *Controller) Status(status int) {
	ctrl.statusCode = status
}

func (ctrl *Controller) Text(text string) {
	ctrl.Res.Header().Set(contentType, appendCharset(contentText, ctrl.Charset))
	ctrl.Res.WriteHeader(ctrl.statusCode)
	ctrl.Res.Write([]byte(text))
}

func (ctrl *Controller) Json(v interface{}) {
	result, err := json.MarshalIndent(v, "", "  ")
	if err != nil {
		ctrl.InternalError(err)
		return
	}
	ctrl.Res.Header().Set(contentType, appendCharset(contentJSON, ctrl.Charset))
	ctrl.Res.WriteHeader(ctrl.statusCode)
	ctrl.Res.Write(result)
}

func (ctrl *Controller) View(name string, data interface{}) {
	if isCache != true {
		setCacheTemplate()
	}

	ctrl.Res.Header().Set(contentType, appendCharset(contentHTML, ctrl.Charset))
	ctrl.Res.WriteHeader(ctrl.statusCode)
	err := cacheTemplate.ExecuteTemplate(ctrl.Res, name, data)
	if err != nil {
		ctrl.InternalError(err)
		return
	}
}

func (ctrl *Controller) InternalError(err error) {
	ctrl.Err = err
	ctrl.Res.WriteHeader(500)
	ctrl.Res.Write([]byte("Internal Server Error"))
}

func appendCharset(content string, charset string) string {
	return content + "; charset=" + charset
}
<|endoftext|>


<commit_before>package dogo

import (
	"net/http"
)

type dogoController interface {
	handler(http.ResponseWriter, *http.Request)
}

type BaseController struct {
	Get  map[string][]string
	Post map[string][]string
	// Get and Post merge
	Request map[string][]string
}

func (c *BaseController) handler(response http.ResponseWriter, request *http.Request) {

	response.Write([]byte("wuciyou hello word"))
}
<commit_msg>删除控制器文件<commit_after><|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/terraform/config"
)

// NodeApplyableResource represents a resource that is "applyable":
// it is ready to be applied and is represented by a diff.
type NodeApplyableResource struct {
	Addr          *ResourceAddress // Addr is the address for this resource
	Config        *config.Resource // Config is the resource in the config
	ResourceState *ResourceState   // ResourceState is the ResourceState for this
}

func (n *NodeApplyableResource) Name() string {
	return n.Addr.String()
}

// GraphNodeSubPath
func (n *NodeApplyableResource) Path() []string {
	return n.Addr.Path
}

// GraphNodeProviderConsumer
func (n *NodeApplyableResource) ProvidedBy() []string {
	// If we have a config we prefer that above all else
	if n.Config != nil {
		return []string{resourceProvider(n.Config.Type, n.Config.Provider)}
	}

	// If we have state, then we will use the provider from there
	if n.ResourceState != nil {
		return []string{n.ResourceState.Provider}
	}

	// Use our type
	return []string{resourceProvider(n.Addr.Type, "")}
}

// GraphNodeProvisionerConsumer
func (n *NodeApplyableResource) ProvisionedBy() []string {
	// If we have no configuration, then we have no provisioners
	if n.Config == nil {
		return nil
	}

	// Build the list of provisioners we need based on the configuration.
	// It is okay to have duplicates here.
	result := make([]string, len(n.Config.Provisioners))
	for i, p := range n.Config.Provisioners {
		result[i] = p.Type
	}

	return result
}

// GraphNodeEvalable
func (n *NodeApplyableResource) EvalTree() EvalNode {
	// stateId is the ID to put into the state
	stateId := n.Addr.stateId()
	if n.Addr.Index > -1 {
		stateId = fmt.Sprintf("%s.%d", stateId, n.Addr.Index)
	}

	// Build the instance info. More of this will be populated during eval
	info := &InstanceInfo{
		Id:   stateId,
		Type: n.Addr.Type,
	}

	// Build the resource for eval
	resource := &Resource{
		Name:       n.Addr.Name,
		Type:       n.Addr.Type,
		CountIndex: n.Addr.Index,
	}
	if resource.CountIndex < 0 {
		resource.CountIndex = 0
	}

	// Determine the dependencies for the state. We use some older
	// code for this that we've used for a long time.
	var stateDeps []string
	{
		oldN := &graphNodeExpandedResource{Resource: n.Config}
		stateDeps = oldN.StateDependencies()
	}

	// Declare a bunch of variables that are used for state during
	// evaluation. Most of this are written to by-address below.
	var provider ResourceProvider
	var diff *InstanceDiff
	var state *InstanceState
	var resourceConfig *ResourceConfig
	var err error
	var createNew bool
	var createBeforeDestroyEnabled bool

	return &EvalSequence{
		Nodes: []EvalNode{
			// Build the instance info
			&EvalInstanceInfo{
				Info: info,
			},

			// Get the saved diff for apply
			&EvalReadDiff{
				Name: stateId,
				Diff: &diff,
			},

			// We don't want to do any destroys
			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					if diff == nil {
						return true, EvalEarlyExitError{}
					}

					if diff.GetDestroy() && diff.GetAttributesLen() == 0 {
						return true, EvalEarlyExitError{}
					}

					diff.SetDestroy(false)
					return true, nil
				},
				Then: EvalNoop{},
			},

			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					destroy := false
					if diff != nil {
						destroy = diff.GetDestroy() || diff.RequiresNew()
					}

					createBeforeDestroyEnabled =
						n.Config.Lifecycle.CreateBeforeDestroy &&
							destroy

					return createBeforeDestroyEnabled, nil
				},
				Then: &EvalDeposeState{
					Name: stateId,
				},
			},

			&EvalInterpolate{
				Config:   n.Config.RawConfig.Copy(),
				Resource: resource,
				Output:   &resourceConfig,
			},
			&EvalGetProvider{
				Name:   n.ProvidedBy()[0],
				Output: &provider,
			},
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},
			// Re-run validation to catch any errors we missed, e.g. type
			// mismatches on computed values.
			&EvalValidateResource{
				Provider:       &provider,
				Config:         &resourceConfig,
				ResourceName:   n.Config.Name,
				ResourceType:   n.Config.Type,
				ResourceMode:   n.Config.Mode,
				IgnoreWarnings: true,
			},
			&EvalDiff{
				Info:       info,
				Config:     &resourceConfig,
				Resource:   n.Config,
				Provider:   &provider,
				Diff:       &diff,
				State:      &state,
				OutputDiff: &diff,
			},

			// Get the saved diff
			&EvalReadDiff{
				Name: stateId,
				Diff: &diff,
			},

			// Compare the diffs
			&EvalCompareDiff{
				Info: info,
				One:  &diff,
				Two:  &diff,
			},

			&EvalGetProvider{
				Name:   n.ProvidedBy()[0],
				Output: &provider,
			},
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},
			&EvalApply{
				Info:      info,
				State:     &state,
				Diff:      &diff,
				Provider:  &provider,
				Output:    &state,
				Error:     &err,
				CreateNew: &createNew,
			},
			&EvalWriteState{
				Name:         stateId,
				ResourceType: n.Config.Type,
				Provider:     n.Config.Provider,
				Dependencies: stateDeps,
				State:        &state,
			},
			/*
				TODO: this has to work
				&EvalApplyProvisioners{
					Info:           info,
					State:          &state,
					Resource:       n.Config,
					InterpResource: resource,
					CreateNew:      &createNew,
					Error:          &err,
				},
			*/
			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					return createBeforeDestroyEnabled && err != nil, nil
				},
				Then: &EvalUndeposeState{
					Name:  stateId,
					State: &state,
				},
				Else: &EvalWriteState{
					Name:         stateId,
					ResourceType: n.Config.Type,
					Provider:     n.Config.Provider,
					Dependencies: stateDeps,
					State:        &state,
				},
			},

			// We clear the diff out here so that future nodes
			// don't see a diff that is already complete. There
			// is no longer a diff!
			&EvalWriteDiff{
				Name: stateId,
				Diff: nil,
			},

			&EvalApplyPost{
				Info:  info,
				State: &state,
				Error: &err,
			},
			&EvalUpdateStateHook{},
		},
	}
}
<commit_msg>terraform: enable provisioners to execute<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/terraform/config"
)

// NodeApplyableResource represents a resource that is "applyable":
// it is ready to be applied and is represented by a diff.
type NodeApplyableResource struct {
	Addr          *ResourceAddress // Addr is the address for this resource
	Config        *config.Resource // Config is the resource in the config
	ResourceState *ResourceState   // ResourceState is the ResourceState for this
}

func (n *NodeApplyableResource) Name() string {
	return n.Addr.String()
}

// GraphNodeSubPath
func (n *NodeApplyableResource) Path() []string {
	return n.Addr.Path
}

// GraphNodeProviderConsumer
func (n *NodeApplyableResource) ProvidedBy() []string {
	// If we have a config we prefer that above all else
	if n.Config != nil {
		return []string{resourceProvider(n.Config.Type, n.Config.Provider)}
	}

	// If we have state, then we will use the provider from there
	if n.ResourceState != nil {
		return []string{n.ResourceState.Provider}
	}

	// Use our type
	return []string{resourceProvider(n.Addr.Type, "")}
}

// GraphNodeProvisionerConsumer
func (n *NodeApplyableResource) ProvisionedBy() []string {
	// If we have no configuration, then we have no provisioners
	if n.Config == nil {
		return nil
	}

	// Build the list of provisioners we need based on the configuration.
	// It is okay to have duplicates here.
	result := make([]string, len(n.Config.Provisioners))
	for i, p := range n.Config.Provisioners {
		result[i] = p.Type
	}

	return result
}

// GraphNodeEvalable
func (n *NodeApplyableResource) EvalTree() EvalNode {
	// stateId is the ID to put into the state
	stateId := n.Addr.stateId()
	if n.Addr.Index > -1 {
		stateId = fmt.Sprintf("%s.%d", stateId, n.Addr.Index)
	}

	// Build the instance info. More of this will be populated during eval
	info := &InstanceInfo{
		Id:   stateId,
		Type: n.Addr.Type,
	}

	// Build the resource for eval
	resource := &Resource{
		Name:       n.Addr.Name,
		Type:       n.Addr.Type,
		CountIndex: n.Addr.Index,
	}
	if resource.CountIndex < 0 {
		resource.CountIndex = 0
	}

	// Determine the dependencies for the state. We use some older
	// code for this that we've used for a long time.
	var stateDeps []string
	{
		oldN := &graphNodeExpandedResource{Resource: n.Config}
		stateDeps = oldN.StateDependencies()
	}

	// Declare a bunch of variables that are used for state during
	// evaluation. Most of this are written to by-address below.
	var provider ResourceProvider
	var diff *InstanceDiff
	var state *InstanceState
	var resourceConfig *ResourceConfig
	var err error
	var createNew bool
	var createBeforeDestroyEnabled bool

	return &EvalSequence{
		Nodes: []EvalNode{
			// Build the instance info
			&EvalInstanceInfo{
				Info: info,
			},

			// Get the saved diff for apply
			&EvalReadDiff{
				Name: stateId,
				Diff: &diff,
			},

			// We don't want to do any destroys
			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					if diff == nil {
						return true, EvalEarlyExitError{}
					}

					if diff.GetDestroy() && diff.GetAttributesLen() == 0 {
						return true, EvalEarlyExitError{}
					}

					diff.SetDestroy(false)
					return true, nil
				},
				Then: EvalNoop{},
			},

			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					destroy := false
					if diff != nil {
						destroy = diff.GetDestroy() || diff.RequiresNew()
					}

					createBeforeDestroyEnabled =
						n.Config.Lifecycle.CreateBeforeDestroy &&
							destroy

					return createBeforeDestroyEnabled, nil
				},
				Then: &EvalDeposeState{
					Name: stateId,
				},
			},

			&EvalInterpolate{
				Config:   n.Config.RawConfig.Copy(),
				Resource: resource,
				Output:   &resourceConfig,
			},
			&EvalGetProvider{
				Name:   n.ProvidedBy()[0],
				Output: &provider,
			},
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},
			// Re-run validation to catch any errors we missed, e.g. type
			// mismatches on computed values.
			&EvalValidateResource{
				Provider:       &provider,
				Config:         &resourceConfig,
				ResourceName:   n.Config.Name,
				ResourceType:   n.Config.Type,
				ResourceMode:   n.Config.Mode,
				IgnoreWarnings: true,
			},
			&EvalDiff{
				Info:       info,
				Config:     &resourceConfig,
				Resource:   n.Config,
				Provider:   &provider,
				Diff:       &diff,
				State:      &state,
				OutputDiff: &diff,
			},

			// Get the saved diff
			&EvalReadDiff{
				Name: stateId,
				Diff: &diff,
			},

			// Compare the diffs
			&EvalCompareDiff{
				Info: info,
				One:  &diff,
				Two:  &diff,
			},

			&EvalGetProvider{
				Name:   n.ProvidedBy()[0],
				Output: &provider,
			},
			&EvalReadState{
				Name:   stateId,
				Output: &state,
			},
			&EvalApply{
				Info:      info,
				State:     &state,
				Diff:      &diff,
				Provider:  &provider,
				Output:    &state,
				Error:     &err,
				CreateNew: &createNew,
			},
			&EvalWriteState{
				Name:         stateId,
				ResourceType: n.Config.Type,
				Provider:     n.Config.Provider,
				Dependencies: stateDeps,
				State:        &state,
			},
			&EvalApplyProvisioners{
				Info:           info,
				State:          &state,
				Resource:       n.Config,
				InterpResource: resource,
				CreateNew:      &createNew,
				Error:          &err,
			},
			&EvalIf{
				If: func(ctx EvalContext) (bool, error) {
					return createBeforeDestroyEnabled && err != nil, nil
				},
				Then: &EvalUndeposeState{
					Name:  stateId,
					State: &state,
				},
				Else: &EvalWriteState{
					Name:         stateId,
					ResourceType: n.Config.Type,
					Provider:     n.Config.Provider,
					Dependencies: stateDeps,
					State:        &state,
				},
			},

			// We clear the diff out here so that future nodes
			// don't see a diff that is already complete. There
			// is no longer a diff!
			&EvalWriteDiff{
				Name: stateId,
				Diff: nil,
			},

			&EvalApplyPost{
				Info:  info,
				State: &state,
				Error: &err,
			},
			&EvalUpdateStateHook{},
		},
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 - The Event Horizon authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scheduler

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"time"

	eh "github.com/looplab/eventhorizon"
	"github.com/looplab/eventhorizon/uuid"
)

// ErrCanceled is when a scheduled command has been canceled.
var ErrCanceled = errors.New("canceled")

// Command is a scheduled command with an execution time.
type Command interface {
	eh.Command

	// ExecuteAt returns the time when the command will execute.
	ExecuteAt() time.Time
}

// CommandWithExecuteTime returns a wrapped command with a execution time set.
func CommandWithExecuteTime(cmd eh.Command, t time.Time) Command {
	return &command{Command: cmd, t: t}
}

// private implementation to wrap ordinary commands and add a execution time.
type command struct {
	eh.Command
	t time.Time
}

// ExecuteAt implements the ExecuteAt method of the Command interface.
func (c *command) ExecuteAt() time.Time {
	return c.t
}

// NewMiddleware returns a new command handler middleware and a scheduler helper.
func NewMiddleware(repo eh.ReadWriteRepo, codec eh.CommandCodec) (eh.CommandHandlerMiddleware, *Scheduler) {
	s := &Scheduler{
		repo:             repo,
		cmdCh:            make(chan *scheduledCommand, 100),
		cancelScheduling: map[uuid.UUID]chan struct{}{},
		errCh:            make(chan error, 100),
		codec:            codec,
	}

	return eh.CommandHandlerMiddleware(func(h eh.CommandHandler) eh.CommandHandler {
		s.setHandler(h)

		return eh.CommandHandlerFunc(func(ctx context.Context, cmd eh.Command) error {
			// Delayed command execution if there is time set.
			if c, ok := cmd.(Command); ok && !c.ExecuteAt().IsZero() {
				// Use the wrapped command when created by the helper func.
				innerCmd, ok := c.(*command)
				if ok {
					cmd = innerCmd.Command
				}

				// Ignore the persisted command ID in this case.
				_, err := s.ScheduleCommand(ctx, cmd, c.ExecuteAt())

				return err
			}

			// Immediate command execution.
			return h.HandleCommand(ctx, cmd)
		})
	}), s
}

// PersistedCommand is a persisted command.
type PersistedCommand struct {
	ID         uuid.UUID       `json:"_"                 bson:"_id"`
	IDStr      string          `json:"id"                bson:"_"`
	RawCommand []byte          `json:"command"           bson:"command"`
	ExecuteAt  time.Time       `json:"timestamp"         bson:"timestamp"`
	Command    eh.Command      `json:"-"                 bson:"-"`
	Context    context.Context `json:"-"                 bson:"-"`
}

// EntityID implements the EntityID method of the eventhorizon.Entity interface.
func (c *PersistedCommand) EntityID() uuid.UUID {
	return c.ID
}

// Scheduler is a scheduled of commands.
type Scheduler struct {
	h                  eh.CommandHandler
	hMu                sync.Mutex
	repo               eh.ReadWriteRepo
	cmdCh              chan *scheduledCommand
	cancelScheduling   map[uuid.UUID]chan struct{}
	cancelSchedulingMu sync.Mutex
	errCh              chan error
	cctx               context.Context
	cancel             context.CancelFunc
	done               chan struct{}
	codec              eh.CommandCodec
}

func (s *Scheduler) setHandler(h eh.CommandHandler) {
	s.hMu.Lock()
	defer s.hMu.Unlock()

	if s.h != nil {
		panic("eventhorizon: handler already set for outbox")
	}

	s.h = h
}

// Start starts the scheduler by first loading all persisted commands.
func (s *Scheduler) Start() error {
	if s.h == nil {
		return fmt.Errorf("command handler not set")
	}

	if err := s.loadCommands(); err != nil {
		return fmt.Errorf("could not load commands: %w", err)
	}

	s.cctx, s.cancel = context.WithCancel(context.Background())
	s.done = make(chan struct{})

	go s.run()

	return nil
}

// Stop stops all scheduled commands.
func (s *Scheduler) Stop() error {
	s.cancel()

	<-s.done

	return nil
}

// Errors returns an error channel that will receive errors from handling of
// scheduled commands.
func (s *Scheduler) Errors() <-chan error {
	return s.errCh
}

type scheduledCommand struct {
	id        uuid.UUID
	ctx       context.Context
	cmd       eh.Command
	executeAt time.Time
}

// ScheduleCommand schedules a command to be executed at `executeAt`. It is persisted
// to the repo.
func (s *Scheduler) ScheduleCommand(ctx context.Context, cmd eh.Command, executeAt time.Time) (uuid.UUID, error) {
	b, err := s.codec.MarshalCommand(ctx, cmd)
	if err != nil {
		return uuid.Nil, &Error{
			Err:     fmt.Errorf("could not marshal command: %w", err),
			Ctx:     ctx,
			Command: cmd,
		}
	}

	// Use the command ID as persisted ID if available.
	var id uuid.UUID
	if cmd, ok := cmd.(eh.CommandIDer); ok {
		id = cmd.CommandID()
	} else {
		id = uuid.New()
	}

	pc := &PersistedCommand{
		ID:         id,
		IDStr:      id.String(),
		RawCommand: b,
		ExecuteAt:  executeAt,
	}

	if err := s.repo.Save(context.Background(), pc); err != nil {
		return uuid.Nil, &Error{
			Err:     fmt.Errorf("could not persist command: %w", err),
			Ctx:     ctx,
			Command: cmd,
		}
	}

	select {
	case s.cmdCh <- &scheduledCommand{id, ctx, cmd, executeAt}:
	default:
		return uuid.Nil, &Error{
			Err:     fmt.Errorf("command queue full"),
			Ctx:     ctx,
			Command: cmd,
		}
	}

	return pc.ID, nil
}

// Commands returns all scheduled commands.
func (s *Scheduler) Commands(ctx context.Context) ([]*PersistedCommand, error) {
	entities, err := s.repo.FindAll(ctx)
	if err != nil {
		return nil, fmt.Errorf("could not load scheduled commands: %w", err)
	}

	commands := make([]*PersistedCommand, len(entities))

	for i, entity := range entities {
		c, ok := entity.(*PersistedCommand)
		if !ok {
			return nil, fmt.Errorf("command is not schedulable: %T", entity)
		}

		if c.Command, c.Context, err = s.codec.UnmarshalCommand(ctx, c.RawCommand); err != nil {
			return nil, fmt.Errorf("could not unmarshal command: %w", err)
		}

		c.RawCommand = nil

		if c.IDStr != "" {
			id, err := uuid.Parse(c.IDStr)
			if err != nil {
				return nil, fmt.Errorf("could not parse command ID: %w", err)
			}

			c.ID = id
		}

		commands[i] = c
	}

	return commands, nil
}

// CancelCommand cancels a scheduled command.
func (s *Scheduler) CancelCommand(ctx context.Context, id uuid.UUID) error {
	s.cancelSchedulingMu.Lock()
	defer s.cancelSchedulingMu.Unlock()

	cancel, ok := s.cancelScheduling[id]
	if !ok {
		return fmt.Errorf("command %s not scheduled", id)
	}

	close(cancel)

	return nil
}

func (s *Scheduler) loadCommands() error {
	commands, err := s.Commands(context.Background())
	if err != nil {
		return fmt.Errorf("could not load scheduled commands: %w", err)
	}

	for _, pc := range commands {
		sc := &scheduledCommand{
			id:        pc.ID,
			ctx:       pc.Context,
			cmd:       pc.Command,
			executeAt: pc.ExecuteAt,
		}

		select {
		case s.cmdCh <- sc:
		default:
			return fmt.Errorf("could not schedule command: %w", err)
		}
	}

	return nil
}

func (s *Scheduler) run() {
	var wg sync.WaitGroup

loop:
	for {
		select {
		case <-s.cctx.Done():
			break loop
		case sc := <-s.cmdCh:
			wg.Add(1)

			s.cancelSchedulingMu.Lock()
			cancel := make(chan struct{})
			s.cancelScheduling[sc.id] = cancel
			s.cancelSchedulingMu.Unlock()

			go func(cancel chan struct{}) {
				defer wg.Done()

				t := time.NewTimer(time.Until(sc.executeAt))
				defer t.Stop()

				select {
				case <-s.cctx.Done():
					// Stop without removing persisted cmd.
				case <-t.C:
					if err := s.h.HandleCommand(sc.ctx, sc.cmd); err != nil {
						// Always try to deliver errors.
						s.errCh <- &Error{
							Err:     err,
							Ctx:     sc.ctx,
							Command: sc.cmd,
						}
					}

					if err := s.repo.Remove(context.Background(), sc.id); err != nil {
						s.errCh <- &Error{
							Err:     fmt.Errorf("could not remove persisted command: %w", err),
							Ctx:     sc.ctx,
							Command: sc.cmd,
						}
					}
				case <-cancel:
					if err := s.repo.Remove(context.Background(), sc.id); err != nil {
						s.errCh <- &Error{
							Err:     fmt.Errorf("could not remove persisted command: %w", err),
							Ctx:     sc.ctx,
							Command: sc.cmd,
						}
					}

					s.errCh <- &Error{
						Err:     ErrCanceled,
						Ctx:     sc.ctx,
						Command: sc.cmd,
					}
				}
			}(cancel)
		}
	}

	wg.Wait()

	close(s.done)
}

// Error is an async error containing the error and the command.
type Error struct {
	// Err is the error that happened when handling the command.
	Err error
	// Ctx is the context used when the error happened.
	Ctx context.Context
	// Command is the command handeled when the error happened.
	Command eh.Command
}

// Error implements the Error method of the error interface.
func (e *Error) Error() string {
	return fmt.Sprintf("%s (%s): %s", e.Command.CommandType(), e.Command.AggregateID(), e.Err.Error())
}

// Unwrap implements the errors.Unwrap method.
func (e *Error) Unwrap() error {
	return e.Err
}

// Cause implements the github.com/pkg/errors Unwrap method.
func (e *Error) Cause() error {
	return e.Unwrap()
}
<commit_msg>fix: use correct context when removing scheduled items<commit_after>// Copyright (c) 2017 - The Event Horizon authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scheduler

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"time"

	eh "github.com/looplab/eventhorizon"
	"github.com/looplab/eventhorizon/uuid"
)

// ErrCanceled is when a scheduled command has been canceled.
var ErrCanceled = errors.New("canceled")

// Command is a scheduled command with an execution time.
type Command interface {
	eh.Command

	// ExecuteAt returns the time when the command will execute.
	ExecuteAt() time.Time
}

// CommandWithExecuteTime returns a wrapped command with a execution time set.
func CommandWithExecuteTime(cmd eh.Command, t time.Time) Command {
	return &command{Command: cmd, t: t}
}

// private implementation to wrap ordinary commands and add a execution time.
type command struct {
	eh.Command
	t time.Time
}

// ExecuteAt implements the ExecuteAt method of the Command interface.
func (c *command) ExecuteAt() time.Time {
	return c.t
}

// NewMiddleware returns a new command handler middleware and a scheduler helper.
func NewMiddleware(repo eh.ReadWriteRepo, codec eh.CommandCodec) (eh.CommandHandlerMiddleware, *Scheduler) {
	s := &Scheduler{
		repo:             repo,
		cmdCh:            make(chan *scheduledCommand, 100),
		cancelScheduling: map[uuid.UUID]chan struct{}{},
		errCh:            make(chan error, 100),
		codec:            codec,
	}

	return eh.CommandHandlerMiddleware(func(h eh.CommandHandler) eh.CommandHandler {
		s.setHandler(h)

		return eh.CommandHandlerFunc(func(ctx context.Context, cmd eh.Command) error {
			// Delayed command execution if there is time set.
			if c, ok := cmd.(Command); ok && !c.ExecuteAt().IsZero() {
				// Use the wrapped command when created by the helper func.
				innerCmd, ok := c.(*command)
				if ok {
					cmd = innerCmd.Command
				}

				// Ignore the persisted command ID in this case.
				_, err := s.ScheduleCommand(ctx, cmd, c.ExecuteAt())

				return err
			}

			// Immediate command execution.
			return h.HandleCommand(ctx, cmd)
		})
	}), s
}

// PersistedCommand is a persisted command.
type PersistedCommand struct {
	ID         uuid.UUID       `json:"_"                 bson:"_id"`
	IDStr      string          `json:"id"                bson:"_"`
	RawCommand []byte          `json:"command"           bson:"command"`
	ExecuteAt  time.Time       `json:"timestamp"         bson:"timestamp"`
	Command    eh.Command      `json:"-"                 bson:"-"`
	Context    context.Context `json:"-"                 bson:"-"`
}

// EntityID implements the EntityID method of the eventhorizon.Entity interface.
func (c *PersistedCommand) EntityID() uuid.UUID {
	return c.ID
}

// Scheduler is a scheduled of commands.
type Scheduler struct {
	h                  eh.CommandHandler
	hMu                sync.Mutex
	repo               eh.ReadWriteRepo
	cmdCh              chan *scheduledCommand
	cancelScheduling   map[uuid.UUID]chan struct{}
	cancelSchedulingMu sync.Mutex
	errCh              chan error
	cctx               context.Context
	cancel             context.CancelFunc
	done               chan struct{}
	codec              eh.CommandCodec
}

func (s *Scheduler) setHandler(h eh.CommandHandler) {
	s.hMu.Lock()
	defer s.hMu.Unlock()

	if s.h != nil {
		panic("eventhorizon: handler already set for outbox")
	}

	s.h = h
}

// Start starts the scheduler by first loading all persisted commands.
func (s *Scheduler) Start() error {
	if s.h == nil {
		return fmt.Errorf("command handler not set")
	}

	if err := s.loadCommands(); err != nil {
		return fmt.Errorf("could not load commands: %w", err)
	}

	s.cctx, s.cancel = context.WithCancel(context.Background())
	s.done = make(chan struct{})

	go s.run()

	return nil
}

// Stop stops all scheduled commands.
func (s *Scheduler) Stop() error {
	s.cancel()

	<-s.done

	return nil
}

// Errors returns an error channel that will receive errors from handling of
// scheduled commands.
func (s *Scheduler) Errors() <-chan error {
	return s.errCh
}

type scheduledCommand struct {
	id        uuid.UUID
	ctx       context.Context
	cmd       eh.Command
	executeAt time.Time
}

// ScheduleCommand schedules a command to be executed at `executeAt`. It is persisted
// to the repo.
func (s *Scheduler) ScheduleCommand(ctx context.Context, cmd eh.Command, executeAt time.Time) (uuid.UUID, error) {
	b, err := s.codec.MarshalCommand(ctx, cmd)
	if err != nil {
		return uuid.Nil, &Error{
			Err:     fmt.Errorf("could not marshal command: %w", err),
			Ctx:     ctx,
			Command: cmd,
		}
	}

	// Use the command ID as persisted ID if available.
	var id uuid.UUID
	if cmd, ok := cmd.(eh.CommandIDer); ok {
		id = cmd.CommandID()
	} else {
		id = uuid.New()
	}

	pc := &PersistedCommand{
		ID:         id,
		IDStr:      id.String(),
		RawCommand: b,
		ExecuteAt:  executeAt,
	}

	if err := s.repo.Save(ctx, pc); err != nil {
		return uuid.Nil, &Error{
			Err:     fmt.Errorf("could not persist command: %w", err),
			Ctx:     ctx,
			Command: cmd,
		}
	}

	select {
	case s.cmdCh <- &scheduledCommand{id, ctx, cmd, executeAt}:
	default:
		return uuid.Nil, &Error{
			Err:     fmt.Errorf("command queue full"),
			Ctx:     ctx,
			Command: cmd,
		}
	}

	return pc.ID, nil
}

// Commands returns all scheduled commands.
func (s *Scheduler) Commands(ctx context.Context) ([]*PersistedCommand, error) {
	entities, err := s.repo.FindAll(ctx)
	if err != nil {
		return nil, fmt.Errorf("could not load scheduled commands: %w", err)
	}

	commands := make([]*PersistedCommand, len(entities))

	for i, entity := range entities {
		c, ok := entity.(*PersistedCommand)
		if !ok {
			return nil, fmt.Errorf("command is not schedulable: %T", entity)
		}

		if c.Command, c.Context, err = s.codec.UnmarshalCommand(ctx, c.RawCommand); err != nil {
			return nil, fmt.Errorf("could not unmarshal command: %w", err)
		}

		c.RawCommand = nil

		if c.IDStr != "" {
			id, err := uuid.Parse(c.IDStr)
			if err != nil {
				return nil, fmt.Errorf("could not parse command ID: %w", err)
			}

			c.ID = id
		}

		commands[i] = c
	}

	return commands, nil
}

// CancelCommand cancels a scheduled command.
func (s *Scheduler) CancelCommand(ctx context.Context, id uuid.UUID) error {
	s.cancelSchedulingMu.Lock()
	defer s.cancelSchedulingMu.Unlock()

	cancel, ok := s.cancelScheduling[id]
	if !ok {
		return fmt.Errorf("command %s not scheduled", id)
	}

	close(cancel)

	return nil
}

func (s *Scheduler) loadCommands() error {
	commands, err := s.Commands(context.Background())
	if err != nil {
		return fmt.Errorf("could not load scheduled commands: %w", err)
	}

	for _, pc := range commands {
		sc := &scheduledCommand{
			id:        pc.ID,
			ctx:       pc.Context,
			cmd:       pc.Command,
			executeAt: pc.ExecuteAt,
		}

		select {
		case s.cmdCh <- sc:
		default:
			return fmt.Errorf("could not schedule command: %w", err)
		}
	}

	return nil
}

func (s *Scheduler) run() {
	var wg sync.WaitGroup

loop:
	for {
		select {
		case <-s.cctx.Done():
			break loop
		case sc := <-s.cmdCh:
			wg.Add(1)

			s.cancelSchedulingMu.Lock()
			cancel := make(chan struct{})
			s.cancelScheduling[sc.id] = cancel
			s.cancelSchedulingMu.Unlock()

			go func(cancel chan struct{}) {
				defer wg.Done()

				t := time.NewTimer(time.Until(sc.executeAt))
				defer t.Stop()

				select {
				case <-s.cctx.Done():
					// Stop without removing persisted cmd.
				case <-t.C:
					if err := s.h.HandleCommand(sc.ctx, sc.cmd); err != nil {
						// Always try to deliver errors.
						s.errCh <- &Error{
							Err:     err,
							Ctx:     sc.ctx,
							Command: sc.cmd,
						}
					}

					if err := s.repo.Remove(sc.ctx, sc.id); err != nil {
						s.errCh <- &Error{
							Err:     fmt.Errorf("could not remove persisted command: %w", err),
							Ctx:     sc.ctx,
							Command: sc.cmd,
						}
					}
				case <-cancel:
					if err := s.repo.Remove(sc.ctx, sc.id); err != nil {
						s.errCh <- &Error{
							Err:     fmt.Errorf("could not remove persisted command: %w", err),
							Ctx:     sc.ctx,
							Command: sc.cmd,
						}
					}

					s.errCh <- &Error{
						Err:     ErrCanceled,
						Ctx:     sc.ctx,
						Command: sc.cmd,
					}
				}
			}(cancel)
		}
	}

	wg.Wait()

	close(s.done)
}

// Error is an async error containing the error and the command.
type Error struct {
	// Err is the error that happened when handling the command.
	Err error
	// Ctx is the context used when the error happened.
	Ctx context.Context
	// Command is the command handeled when the error happened.
	Command eh.Command
}

// Error implements the Error method of the error interface.
func (e *Error) Error() string {
	return fmt.Sprintf("%s (%s): %s", e.Command.CommandType(), e.Command.AggregateID(), e.Err.Error())
}

// Unwrap implements the errors.Unwrap method.
func (e *Error) Unwrap() error {
	return e.Err
}

// Cause implements the github.com/pkg/errors Unwrap method.
func (e *Error) Cause() error {
	return e.Unwrap()
}
<|endoftext|>


<commit_before>// Http provider
package http

import (
	"fmt"
	"github.com/pierrre/imageserver"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
)

var contentTypeRegexp, _ = regexp.Compile("^image/(.+)$")

// Returns image from an http source
//
// If the source is not an url, the string representation of the source will be used to create one.
//
// Returns an error if the http status code is not 200 (OK).
//
// The image type is determined by the "Content-Type" header.
type HttpProvider struct {
}

func (provider *HttpProvider) Get(source interface{}, parameters imageserver.Parameters) (*imageserver.Image, error) {
	sourceUrl, err := provider.getSourceUrl(source)
	if err != nil {
		return nil, err
	}
	response, err := provider.request(sourceUrl)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	if err = provider.checkResponse(response); err != nil {
		return nil, err
	}
	image, err := provider.createImage(response)
	if err != nil {
		return nil, err
	}
	return image, nil
}

func (provider *HttpProvider) getSourceUrl(source interface{}) (*url.URL, error) {
	sourceUrl, ok := source.(*url.URL)
	if !ok {
		var err error
		sourceUrl, err = url.ParseRequestURI(fmt.Sprint(source))
		if err != nil {
			return nil, imageserver.NewError("Invalid source url")
		}
	}
	if sourceUrl.Scheme != "http" && sourceUrl.Scheme != "https" {
		return nil, imageserver.NewError("Invalid source scheme")
	}
	return sourceUrl, nil
}

func (provider *HttpProvider) request(sourceUrl *url.URL) (*http.Response, error) {
	//TODO optional http client
	return http.Get(sourceUrl.String())
}

func (provider *HttpProvider) checkResponse(response *http.Response) error {
	if response.StatusCode != http.StatusOK {
		return imageserver.NewError(fmt.Sprintf("Error %d while downloading source", response.StatusCode))
	}
	return nil
}

func (provider *HttpProvider) createImage(response *http.Response) (*imageserver.Image, error) {
	image := &imageserver.Image{}
	provider.parseType(response, image)
	if err := provider.parseData(response, image); err != nil {
		return nil, err
	}
	return image, nil
}

func (provider *HttpProvider) parseType(response *http.Response, image *imageserver.Image) {
	contentType := response.Header.Get("Content-Type")
	if len(contentType) > 0 {
		matches := contentTypeRegexp.FindStringSubmatch(contentType)
		if matches != nil && len(matches) == 2 {
			image.Type = matches[1]
		}
	}
}

func (provider *HttpProvider) parseData(response *http.Response, image *imageserver.Image) error {
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}
	image.Data = data
	return nil
}
<commit_msg>Refactor header parsing in http provider<commit_after>// Http provider
package http

import (
	"fmt"
	"github.com/pierrre/imageserver"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
)

var contentTypeRegexp, _ = regexp.Compile("^image/(.+)$")

// Returns image from an http source
//
// If the source is not an url, the string representation of the source will be used to create one.
//
// Returns an error if the http status code is not 200 (OK).
//
// The image type is determined by the "Content-Type" header.
type HttpProvider struct {
}

func (provider *HttpProvider) Get(source interface{}, parameters imageserver.Parameters) (*imageserver.Image, error) {
	sourceUrl, err := provider.getSourceUrl(source)
	if err != nil {
		return nil, err
	}
	response, err := provider.request(sourceUrl)
	if err != nil {
		return nil, err
	}
	defer response.Body.Close()
	if err = provider.checkResponse(response); err != nil {
		return nil, err
	}
	image, err := provider.createImage(response)
	if err != nil {
		return nil, err
	}
	return image, nil
}

func (provider *HttpProvider) getSourceUrl(source interface{}) (*url.URL, error) {
	sourceUrl, ok := source.(*url.URL)
	if !ok {
		var err error
		sourceUrl, err = url.ParseRequestURI(fmt.Sprint(source))
		if err != nil {
			return nil, imageserver.NewError("Invalid source url")
		}
	}
	if sourceUrl.Scheme != "http" && sourceUrl.Scheme != "https" {
		return nil, imageserver.NewError("Invalid source scheme")
	}
	return sourceUrl, nil
}

func (provider *HttpProvider) request(sourceUrl *url.URL) (*http.Response, error) {
	//TODO optional http client
	return http.Get(sourceUrl.String())
}

func (provider *HttpProvider) checkResponse(response *http.Response) error {
	if response.StatusCode != http.StatusOK {
		return imageserver.NewError(fmt.Sprintf("Error %d while downloading source", response.StatusCode))
	}
	return nil
}

func (provider *HttpProvider) createImage(response *http.Response) (*imageserver.Image, error) {
	image := &imageserver.Image{}
	provider.parseType(response, image)
	if err := provider.parseData(response, image); err != nil {
		return nil, err
	}
	return image, nil
}

func (provider *HttpProvider) parseType(response *http.Response, image *imageserver.Image) {
	contentType := response.Header.Get("Content-Type")
	if len(contentType) == 0 {
		return
	}
	matches := contentTypeRegexp.FindStringSubmatch(contentType)
	if matches == nil || len(matches) != 2 {
		return
	}
	image.Type = matches[1]
}

func (provider *HttpProvider) parseData(response *http.Response, image *imageserver.Image) error {
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}
	image.Data = data
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package report contains helpers for writing comments and updating
// statuses in Github.
package report

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	"k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/plugins"
)

const (
	commentTag = ""
)

// GithubClient provides a client interface to report job status updates
// through Github comments.
type GithubClient interface {
	BotName() (string, error)
	CreateStatus(org, repo, ref string, s github.Status) error
	ListIssueComments(org, repo string, number int) ([]github.IssueComment, error)
	CreateComment(org, repo string, number int, comment string) error
	DeleteComment(org, repo string, ID int) error
	EditComment(org, repo string, ID int, comment string) error
}

// prowjobStateToGithubStatus maps prowjob status to github states.
// Github states can be one of error, failure, pending, or success.
// https://developer.github.com/v3/repos/statuses/#create-a-status
func prowjobStateToGithubStatus(pjState v1.ProwJobState) (string, error) {
	switch pjState {
	case v1.TriggeredState:
		return github.StatusPending, nil
	case v1.PendingState:
		return github.StatusPending, nil
	case v1.SuccessState:
		return github.StatusSuccess, nil
	case v1.ErrorState:
		return github.StatusError, nil
	case v1.FailureState:
		return github.StatusFailure, nil
	case v1.AbortedState:
		return github.StatusFailure, nil
	}
	return "", fmt.Errorf("Unknown prowjob state: %v", pjState)
}

const (
	maxLen = 140 // https://developer.github.com/v3/repos/deployments/#parameters-2
	elide  = " ... "
)

// truncate converts "really long messages" into "really ... messages".
func truncate(in string) string {
	const (
		half = (maxLen - len(elide)) / 2
	)
	if len(in) <= maxLen {
		return in
	}
	return in[:half] + elide + in[len(in)-half:]
}

// reportStatus should be called on status different from Success.
// Once a parent ProwJob is pending, all children should be marked as Pending
// Same goes for failed status.
func reportStatus(ghc GithubClient, pj v1.ProwJob, childDescription string) error {
	refs := pj.Spec.Refs
	if pj.Spec.Report {
		contextState, err := prowjobStateToGithubStatus(pj.Status.State)
		if err != nil {
			return err
		}
		if err := ghc.CreateStatus(refs.Org, refs.Repo, refs.Pulls[0].SHA, github.Status{
			State:       contextState,
			Description: truncate(pj.Status.Description),
			Context:     pj.Spec.Context, // consider truncating this too
			TargetURL:   pj.Status.URL,
		}); err != nil {
			return err
		}
	}
	return nil
}

// Report is creating/updating/removing reports in Github based on the state of
// the provided ProwJob.
func Report(ghc GithubClient, reportTemplate *template.Template, pj v1.ProwJob) error {
	if ghc == nil {
		return fmt.Errorf("trying to report pj %s, but found empty github client", pj.ObjectMeta.Name)
	}
	if !pj.Spec.Report {
		return nil
	}
	refs := pj.Spec.Refs
	if len(refs.Pulls) != 1 {
		return fmt.Errorf("prowjob %s has %d pulls, not 1", pj.ObjectMeta.Name, len(refs.Pulls))
	}
	childDescription := fmt.Sprintf("Waiting on: %s", pj.Spec.Context)
	if err := reportStatus(ghc, pj, childDescription); err != nil {
		return fmt.Errorf("error setting status: %v", err)
	}
	// Report manually aborted Jenkins jobs and jobs with invalid pod specs alongside
	// test successes/failures.
	if !pj.Complete() {
		return nil
	}
	ics, err := ghc.ListIssueComments(refs.Org, refs.Repo, refs.Pulls[0].Number)
	if err != nil {
		return fmt.Errorf("error listing comments: %v", err)
	}
	botName, err := ghc.BotName()
	if err != nil {
		return fmt.Errorf("error getting bot name: %v", err)
	}
	deletes, entries, updateID := parseIssueComments(pj, botName, ics)
	for _, delete := range deletes {
		if err := ghc.DeleteComment(refs.Org, refs.Repo, delete); err != nil {
			return fmt.Errorf("error deleting comment: %v", err)
		}
	}
	if len(entries) > 0 {
		comment, err := createComment(reportTemplate, pj, entries)
		if err != nil {
			return fmt.Errorf("generating comment: %v", err)
		}
		if updateID == 0 {
			if err := ghc.CreateComment(refs.Org, refs.Repo, refs.Pulls[0].Number, comment); err != nil {
				return fmt.Errorf("error creating comment: %v", err)
			}
		} else {
			if err := ghc.EditComment(refs.Org, refs.Repo, updateID, comment); err != nil {
				return fmt.Errorf("error updating comment: %v", err)
			}
		}
	}
	return nil
}

// parseIssueComments returns a list of comments to delete, a list of table
// entries, and the ID of the comment to update. If there are no table entries
// then don't make a new comment. Otherwise, if the comment to update is 0,
// create a new comment.
func parseIssueComments(pj v1.ProwJob, botName string, ics []github.IssueComment) ([]int, []string, int) {
	var delete []int
	var previousComments []int
	var latestComment int
	var entries []string
	// First accumulate result entries and comment IDs
	for _, ic := range ics {
		if ic.User.Login != botName {
			continue
		}
		// Old report comments started with the context. Delete them.
		// TODO(spxtr): Delete this check a few weeks after this merges.
		if strings.HasPrefix(ic.Body, pj.Spec.Context) {
			delete = append(delete, ic.ID)
		}
		if !strings.Contains(ic.Body, commentTag) {
			continue
		}
		if latestComment != 0 {
			previousComments = append(previousComments, latestComment)
		}
		latestComment = ic.ID
		var tracking bool
		for _, line := range strings.Split(ic.Body, "\n") {
			line = strings.TrimSpace(line)
			if strings.HasPrefix(line, "---") {
				tracking = true
			} else if len(line) == 0 {
				tracking = false
			} else if tracking {
				entries = append(entries, line)
			}
		}
	}
	var newEntries []string
	// Next decide which entries to keep.
	for i := range entries {
		keep := true
		f1 := strings.Split(entries[i], " | ")
		for j := range entries {
			if i == j {
				continue
			}
			f2 := strings.Split(entries[j], " | ")
			// Use the newer results if there are multiple.
			if j > i && f2[0] == f1[0] {
				keep = false
			}
		}
		// Use the current result if there is an old one.
		if pj.Spec.Context == f1[0] {
			keep = false
		}
		if keep {
			newEntries = append(newEntries, entries[i])
		}
	}
	var createNewComment bool
	if string(pj.Status.State) == github.StatusFailure {
		newEntries = append(newEntries, createEntry(pj))
		createNewComment = true
	}
	delete = append(delete, previousComments...)
	if (createNewComment || len(newEntries) == 0) && latestComment != 0 {
		delete = append(delete, latestComment)
		latestComment = 0
	}
	return delete, newEntries, latestComment
}

func createEntry(pj v1.ProwJob) string {
	return strings.Join([]string{
		pj.Spec.Context,
		pj.Spec.Refs.Pulls[0].SHA,
		fmt.Sprintf("[link](%s)", pj.Status.URL),
		fmt.Sprintf("`%s`", pj.Spec.RerunCommand),
	}, " | ")
}

// createComment take a ProwJob and a list of entries generated with
// createEntry and returns a nicely formatted comment. It may fail if template
// execution fails.
func createComment(reportTemplate *template.Template, pj v1.ProwJob, entries []string) (string, error) {
	plural := ""
	if len(entries) > 1 {
		plural = "s"
	}
	var b bytes.Buffer
	if reportTemplate != nil {
		if err := reportTemplate.Execute(&b, &pj); err != nil {
			return "", err
		}
	}
	lines := []string{
		fmt.Sprintf("@%s: The following test%s **failed**, say `/retest` to rerun them all:", pj.Spec.Refs.Pulls[0].Author, plural),
		"",
		"Test name | Commit | Details | Rerun command",
		"--- | --- | --- | ---",
	}
	lines = append(lines, entries...)
	if reportTemplate != nil {
		lines = append(lines, "", b.String())
	}
	lines = append(lines, []string{
		"",
		"",
		"",
		plugins.AboutThisBot,
		"",
		commentTag,
	}...)
	return strings.Join(lines, "\n"), nil
}
<commit_msg>Use the prowapi import alias in the report lib<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package report contains helpers for writing comments and updating
// statuses in Github.
package report

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	prowapi "k8s.io/test-infra/prow/apis/prowjobs/v1"
	"k8s.io/test-infra/prow/github"
	"k8s.io/test-infra/prow/plugins"
)

const (
	commentTag = ""
)

// GithubClient provides a client interface to report job status updates
// through Github comments.
type GithubClient interface {
	BotName() (string, error)
	CreateStatus(org, repo, ref string, s github.Status) error
	ListIssueComments(org, repo string, number int) ([]github.IssueComment, error)
	CreateComment(org, repo string, number int, comment string) error
	DeleteComment(org, repo string, ID int) error
	EditComment(org, repo string, ID int, comment string) error
}

// prowjobStateToGithubStatus maps prowjob status to github states.
// Github states can be one of error, failure, pending, or success.
// https://developer.github.com/v3/repos/statuses/#create-a-status
func prowjobStateToGithubStatus(pjState prowapi.ProwJobState) (string, error) {
	switch pjState {
	case prowapi.TriggeredState:
		return github.StatusPending, nil
	case prowapi.PendingState:
		return github.StatusPending, nil
	case prowapi.SuccessState:
		return github.StatusSuccess, nil
	case prowapi.ErrorState:
		return github.StatusError, nil
	case prowapi.FailureState:
		return github.StatusFailure, nil
	case prowapi.AbortedState:
		return github.StatusFailure, nil
	}
	return "", fmt.Errorf("Unknown prowjob state: %v", pjState)
}

const (
	maxLen = 140 // https://developer.github.com/v3/repos/deployments/#parameters-2
	elide  = " ... "
)

// truncate converts "really long messages" into "really ... messages".
func truncate(in string) string {
	const (
		half = (maxLen - len(elide)) / 2
	)
	if len(in) <= maxLen {
		return in
	}
	return in[:half] + elide + in[len(in)-half:]
}

// reportStatus should be called on status different from Success.
// Once a parent ProwJob is pending, all children should be marked as Pending
// Same goes for failed status.
func reportStatus(ghc GithubClient, pj prowapi.ProwJob, childDescription string) error {
	refs := pj.Spec.Refs
	if pj.Spec.Report {
		contextState, err := prowjobStateToGithubStatus(pj.Status.State)
		if err != nil {
			return err
		}
		if err := ghc.CreateStatus(refs.Org, refs.Repo, refs.Pulls[0].SHA, github.Status{
			State:       contextState,
			Description: truncate(pj.Status.Description),
			Context:     pj.Spec.Context, // consider truncating this too
			TargetURL:   pj.Status.URL,
		}); err != nil {
			return err
		}
	}
	return nil
}

// Report is creating/updating/removing reports in Github based on the state of
// the provided ProwJob.
func Report(ghc GithubClient, reportTemplate *template.Template, pj prowapi.ProwJob) error {
	if ghc == nil {
		return fmt.Errorf("trying to report pj %s, but found empty github client", pj.ObjectMeta.Name)
	}
	if !pj.Spec.Report {
		return nil
	}
	refs := pj.Spec.Refs
	if len(refs.Pulls) != 1 {
		return fmt.Errorf("prowjob %s has %d pulls, not 1", pj.ObjectMeta.Name, len(refs.Pulls))
	}
	childDescription := fmt.Sprintf("Waiting on: %s", pj.Spec.Context)
	if err := reportStatus(ghc, pj, childDescription); err != nil {
		return fmt.Errorf("error setting status: %v", err)
	}
	// Report manually aborted Jenkins jobs and jobs with invalid pod specs alongside
	// test successes/failures.
	if !pj.Complete() {
		return nil
	}
	ics, err := ghc.ListIssueComments(refs.Org, refs.Repo, refs.Pulls[0].Number)
	if err != nil {
		return fmt.Errorf("error listing comments: %v", err)
	}
	botName, err := ghc.BotName()
	if err != nil {
		return fmt.Errorf("error getting bot name: %v", err)
	}
	deletes, entries, updateID := parseIssueComments(pj, botName, ics)
	for _, delete := range deletes {
		if err := ghc.DeleteComment(refs.Org, refs.Repo, delete); err != nil {
			return fmt.Errorf("error deleting comment: %v", err)
		}
	}
	if len(entries) > 0 {
		comment, err := createComment(reportTemplate, pj, entries)
		if err != nil {
			return fmt.Errorf("generating comment: %v", err)
		}
		if updateID == 0 {
			if err := ghc.CreateComment(refs.Org, refs.Repo, refs.Pulls[0].Number, comment); err != nil {
				return fmt.Errorf("error creating comment: %v", err)
			}
		} else {
			if err := ghc.EditComment(refs.Org, refs.Repo, updateID, comment); err != nil {
				return fmt.Errorf("error updating comment: %v", err)
			}
		}
	}
	return nil
}

// parseIssueComments returns a list of comments to delete, a list of table
// entries, and the ID of the comment to update. If there are no table entries
// then don't make a new comment. Otherwise, if the comment to update is 0,
// create a new comment.
func parseIssueComments(pj prowapi.ProwJob, botName string, ics []github.IssueComment) ([]int, []string, int) {
	var delete []int
	var previousComments []int
	var latestComment int
	var entries []string
	// First accumulate result entries and comment IDs
	for _, ic := range ics {
		if ic.User.Login != botName {
			continue
		}
		// Old report comments started with the context. Delete them.
		// TODO(spxtr): Delete this check a few weeks after this merges.
		if strings.HasPrefix(ic.Body, pj.Spec.Context) {
			delete = append(delete, ic.ID)
		}
		if !strings.Contains(ic.Body, commentTag) {
			continue
		}
		if latestComment != 0 {
			previousComments = append(previousComments, latestComment)
		}
		latestComment = ic.ID
		var tracking bool
		for _, line := range strings.Split(ic.Body, "\n") {
			line = strings.TrimSpace(line)
			if strings.HasPrefix(line, "---") {
				tracking = true
			} else if len(line) == 0 {
				tracking = false
			} else if tracking {
				entries = append(entries, line)
			}
		}
	}
	var newEntries []string
	// Next decide which entries to keep.
	for i := range entries {
		keep := true
		f1 := strings.Split(entries[i], " | ")
		for j := range entries {
			if i == j {
				continue
			}
			f2 := strings.Split(entries[j], " | ")
			// Use the newer results if there are multiple.
			if j > i && f2[0] == f1[0] {
				keep = false
			}
		}
		// Use the current result if there is an old one.
		if pj.Spec.Context == f1[0] {
			keep = false
		}
		if keep {
			newEntries = append(newEntries, entries[i])
		}
	}
	var createNewComment bool
	if string(pj.Status.State) == github.StatusFailure {
		newEntries = append(newEntries, createEntry(pj))
		createNewComment = true
	}
	delete = append(delete, previousComments...)
	if (createNewComment || len(newEntries) == 0) && latestComment != 0 {
		delete = append(delete, latestComment)
		latestComment = 0
	}
	return delete, newEntries, latestComment
}

func createEntry(pj prowapi.ProwJob) string {
	return strings.Join([]string{
		pj.Spec.Context,
		pj.Spec.Refs.Pulls[0].SHA,
		fmt.Sprintf("[link](%s)", pj.Status.URL),
		fmt.Sprintf("`%s`", pj.Spec.RerunCommand),
	}, " | ")
}

// createComment take a ProwJob and a list of entries generated with
// createEntry and returns a nicely formatted comment. It may fail if template
// execution fails.
func createComment(reportTemplate *template.Template, pj prowapi.ProwJob, entries []string) (string, error) {
	plural := ""
	if len(entries) > 1 {
		plural = "s"
	}
	var b bytes.Buffer
	if reportTemplate != nil {
		if err := reportTemplate.Execute(&b, &pj); err != nil {
			return "", err
		}
	}
	lines := []string{
		fmt.Sprintf("@%s: The following test%s **failed**, say `/retest` to rerun them all:", pj.Spec.Refs.Pulls[0].Author, plural),
		"",
		"Test name | Commit | Details | Rerun command",
		"--- | --- | --- | ---",
	}
	lines = append(lines, entries...)
	if reportTemplate != nil {
		lines = append(lines, "", b.String())
	}
	lines = append(lines, []string{
		"",
		"",
		"",
		plugins.AboutThisBot,
		"",
		commentTag,
	}...)
	return strings.Join(lines, "\n"), nil
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"errors"

	clientv2 "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/clientv3"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/samuel/go-zookeeper/zk"
	"golang.org/x/net/context"
)

type ReqHandler func(ctx context.Context, req *request) error

func newPutEtcd2(conn clientv2.KeysAPI) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.etcdv2Op
		_, err := conn.Set(context.Background(), op.key, op.value, nil)
		return err
	}
}

func newPutEtcd3(conn clientv3.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Do(ctx, req.etcdv3Op)
		return err
	}
}

func newPutOverwriteZK(conn *zk.Conn) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.zkOp
		_, err := conn.Set(op.key, op.value, int32(-1))
		return err
	}
}

func newPutCreateZK(conn *zk.Conn) ReqHandler {
	// samekey
	return func(ctx context.Context, req *request) error {
		op := req.zkOp
		_, err := conn.Create(op.key, op.value, zkCreateFlags, zkCreateAcl)
		return err
	}
}

func newPutConsul(conn *consulapi.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.consulOp
		_, err := conn.Put(&consulapi.KVPair{Key: op.key, Value: op.value}, nil)
		return err
	}
}

func newGetEtcd2(conn clientv2.KeysAPI) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Get(ctx, req.etcdv2Op.key, nil)
		return err
	}
}

func newGetEtcd3(conn clientv3.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Do(ctx, req.etcdv3Op)
		return err
	}
}

func newGetZK(conn *zk.Conn) ReqHandler {
	return func(ctx context.Context, req *request) error {
		errt := ""
		if !req.zkOp.staleRead {
			_, err := conn.Sync(req.zkOp.key)
			if err != nil {
				errt += err.Error()
			}
		}
		_, _, err := conn.Get(req.zkOp.key)
		if err != nil {
			errt += ";" + err.Error()
		}
		if errt != "" {
			return errors.New(errt)
		}
		return nil
	}
}

func newGetConsul(conn *consulapi.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, _, err := conn.Get(req.consulOp.key, &consulapi.QueryOptions{AllowStale: req.consulOp.staleRead})
		return err
	}
}
<commit_msg>control: add more logging, Zk get with prefix "/"<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package control

import (
	"errors"

	"fmt"

	clientv2 "github.com/coreos/etcd/client"
	"github.com/coreos/etcd/clientv3"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/samuel/go-zookeeper/zk"
	"golang.org/x/net/context"
)

type ReqHandler func(ctx context.Context, req *request) error

func newPutEtcd2(conn clientv2.KeysAPI) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.etcdv2Op
		_, err := conn.Set(context.Background(), op.key, op.value, nil)
		return err
	}
}

func newPutEtcd3(conn clientv3.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Do(ctx, req.etcdv3Op)
		return err
	}
}

func newPutOverwriteZK(conn *zk.Conn) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.zkOp
		_, err := conn.Set(op.key, op.value, int32(-1))
		return err
	}
}

func newPutCreateZK(conn *zk.Conn) ReqHandler {
	// samekey
	return func(ctx context.Context, req *request) error {
		op := req.zkOp
		_, err := conn.Create(op.key, op.value, zkCreateFlags, zkCreateAcl)
		return err
	}
}

func newPutConsul(conn *consulapi.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		op := req.consulOp
		_, err := conn.Put(&consulapi.KVPair{Key: op.key, Value: op.value}, nil)
		return err
	}
}

func newGetEtcd2(conn clientv2.KeysAPI) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Get(ctx, req.etcdv2Op.key, nil)
		return err
	}
}

func newGetEtcd3(conn clientv3.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, err := conn.Do(ctx, req.etcdv3Op)
		return err
	}
}

func newGetZK(conn *zk.Conn) ReqHandler {
	return func(ctx context.Context, req *request) error {
		errt := ""
		if !req.zkOp.staleRead {
			_, err := conn.Sync("/" + req.zkOp.key)
			if err != nil {
				errt += err.Error()
			}
		}
		_, _, err := conn.Get("/" + req.zkOp.key)
		if err != nil {
			if errt != "" {
				errt += "; "
			}
			errt += fmt.Sprintf("%q while getting %q", err.Error(), "/"+req.zkOp.key)
		}
		if errt != "" {
			return errors.New(errt)
		}
		return nil
	}
}

func newGetConsul(conn *consulapi.KV) ReqHandler {
	return func(ctx context.Context, req *request) error {
		_, _, err := conn.Get(req.consulOp.key, &consulapi.QueryOptions{AllowStale: req.consulOp.staleRead})
		return err
	}
}
<|endoftext|>


<commit_before>package apptail

import (
	"fmt"
	"github.com/ActiveState/log"
	"github.com/ActiveState/tail"
	"github.com/ActiveState/zmqpubsub"
	"logyard"
	"logyard/clients/messagecommon"
	"os"
	"time"
)

// Instance is the NATS message sent by dea_ng to notify of new instances.
type Instance struct {
	AppGUID  string
	AppName  string
	AppSpace string `json:"space"`
	Type     string
	Index    int
	DockerId string `json:"docker_id"`
	LogFiles map[string]string
}

func (instance *Instance) Identifier() string {
	return fmt.Sprintf("%v[%v:%v]", instance.AppName, instance.Index, instance.DockerId[:ID_LENGTH])
}

// Tail begins tailing the files for this instance.
func (instance *Instance) Tail() {
	log.Infof("Tailing %v logs for %v -- %+v",
		instance.Type, instance.Identifier(), instance)

	stopCh := make(chan bool)

	for name, filename := range instance.LogFiles {
		go instance.tailFile(name, filename, stopCh)
	}

	go func() {
		DockerListener.WaitForContainer(instance.DockerId)
		log.Infof("Container for %v exited", instance.Identifier())
		close(stopCh)
	}()
}

func (instance *Instance) tailFile(name, filename string, stopCh chan bool) {
	var err error

	pub := logyard.Broker.NewPublisherMust()
	defer pub.Stop()

	limit, err := instance.getReadLimit(pub, name, filename)
	if err != nil {
		log.Warn(err)
		return
	}

	tail, err := tail.TailFile(filename, tail.Config{
		MaxLineSize: GetConfig().MaxRecordSize,
		MustExist:   true,
		Follow:      true,
		Location:    &tail.SeekInfo{-limit, os.SEEK_END},
		ReOpen:      false,
		Poll:        false,
		LimitRate:   GetConfig().RateLimit})
	if err != nil {
		log.Warnf("Cannot tail file (%s); %s", filename, err)
		return
	}

FORLOOP:
	for {
		select {
		case line, ok := <-tail.Lines:
			if !ok {
				err = tail.Wait()
				break FORLOOP
			}
			instance.publishLine(pub, name, line)
		case <-stopCh:
			err = tail.Stop()
			break FORLOOP
		}
	}

	if err != nil {
		log.Warn(err)
	}

	log.Infof("Completed tailing %v log for %v", name, instance.Identifier())
}

func (instance *Instance) getReadLimit(
	pub *zmqpubsub.Publisher,
	logname string,
	filename string) (int64, error) {
	// convert MB to limit in bytes.
	filesizeLimit := GetConfig().FileSizeLimit * 1024 * 1024
	if !(filesizeLimit > 0) {
		panic("invalid value for `read_limit' in apptail config")
	}

	fi, err := os.Stat(filename)
	if err != nil {
		return -1, fmt.Errorf("Cannot stat file (%s); %s", filename, err)
	}
	size := fi.Size()
	limit := filesizeLimit
	if size > filesizeLimit {
		err := fmt.Errorf("Skipping much of a large log file (%s); size (%v bytes) > read_limit (%v bytes)",
			logname, size, filesizeLimit)
		// Publish special error message.
		instance.publishLine(pub, logname, &tail.Line{
			Text: err.Error(),
			Time: time.Now(),
			Err:  err})
	} else {
		limit = size
	}
	return limit, nil
}

// publishLine zmq-publishes a log line corresponding to this instance
func (instance *Instance) publishLine(
	pub *zmqpubsub.Publisher,
	logname string,
	line *tail.Line) {

	if line == nil {
		panic("line is nil")
	}

	msg := &Message{
		LogFilename:   logname,
		Source:        instance.Type,
		InstanceIndex: instance.Index,
		AppGUID:       instance.AppGUID,
		AppName:       instance.AppName,
		AppSpace:      instance.AppSpace,
		MessageCommon: messagecommon.New(line.Text, line.Time, LocalNodeId()),
	}

	if line.Err != nil {
		// Mark this as a special error record, as it is
		// coming from tail, not the app.
		msg.Source = "stackato.apptail"
		msg.LogFilename = ""
		log.Warnf("[%s] %s", instance.AppName, line.Text)
	}

	err := msg.Publish(pub, false)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>expecting AppSpace (case insensitive) as the field in nats message<commit_after>package apptail

import (
	"fmt"
	"github.com/ActiveState/log"
	"github.com/ActiveState/tail"
	"github.com/ActiveState/zmqpubsub"
	"logyard"
	"logyard/clients/messagecommon"
	"os"
	"time"
)

// Instance is the NATS message sent by dea_ng to notify of new instances.
type Instance struct {
	AppGUID  string
	AppName  string
	AppSpace string
	Type     string
	Index    int
	DockerId string `json:"docker_id"`
	LogFiles map[string]string
}

func (instance *Instance) Identifier() string {
	return fmt.Sprintf("%v[%v:%v]", instance.AppName, instance.Index, instance.DockerId[:ID_LENGTH])
}

// Tail begins tailing the files for this instance.
func (instance *Instance) Tail() {
	log.Infof("Tailing %v logs for %v -- %+v",
		instance.Type, instance.Identifier(), instance)

	stopCh := make(chan bool)

	for name, filename := range instance.LogFiles {
		go instance.tailFile(name, filename, stopCh)
	}

	go func() {
		DockerListener.WaitForContainer(instance.DockerId)
		log.Infof("Container for %v exited", instance.Identifier())
		close(stopCh)
	}()
}

func (instance *Instance) tailFile(name, filename string, stopCh chan bool) {
	var err error

	pub := logyard.Broker.NewPublisherMust()
	defer pub.Stop()

	limit, err := instance.getReadLimit(pub, name, filename)
	if err != nil {
		log.Warn(err)
		return
	}

	tail, err := tail.TailFile(filename, tail.Config{
		MaxLineSize: GetConfig().MaxRecordSize,
		MustExist:   true,
		Follow:      true,
		Location:    &tail.SeekInfo{-limit, os.SEEK_END},
		ReOpen:      false,
		Poll:        false,
		LimitRate:   GetConfig().RateLimit})
	if err != nil {
		log.Warnf("Cannot tail file (%s); %s", filename, err)
		return
	}

FORLOOP:
	for {
		select {
		case line, ok := <-tail.Lines:
			if !ok {
				err = tail.Wait()
				break FORLOOP
			}
			instance.publishLine(pub, name, line)
		case <-stopCh:
			err = tail.Stop()
			break FORLOOP
		}
	}

	if err != nil {
		log.Warn(err)
	}

	log.Infof("Completed tailing %v log for %v", name, instance.Identifier())
}

func (instance *Instance) getReadLimit(
	pub *zmqpubsub.Publisher,
	logname string,
	filename string) (int64, error) {
	// convert MB to limit in bytes.
	filesizeLimit := GetConfig().FileSizeLimit * 1024 * 1024
	if !(filesizeLimit > 0) {
		panic("invalid value for `read_limit' in apptail config")
	}

	fi, err := os.Stat(filename)
	if err != nil {
		return -1, fmt.Errorf("Cannot stat file (%s); %s", filename, err)
	}
	size := fi.Size()
	limit := filesizeLimit
	if size > filesizeLimit {
		err := fmt.Errorf("Skipping much of a large log file (%s); size (%v bytes) > read_limit (%v bytes)",
			logname, size, filesizeLimit)
		// Publish special error message.
		instance.publishLine(pub, logname, &tail.Line{
			Text: err.Error(),
			Time: time.Now(),
			Err:  err})
	} else {
		limit = size
	}
	return limit, nil
}

// publishLine zmq-publishes a log line corresponding to this instance
func (instance *Instance) publishLine(
	pub *zmqpubsub.Publisher,
	logname string,
	line *tail.Line) {

	if line == nil {
		panic("line is nil")
	}

	msg := &Message{
		LogFilename:   logname,
		Source:        instance.Type,
		InstanceIndex: instance.Index,
		AppGUID:       instance.AppGUID,
		AppName:       instance.AppName,
		AppSpace:      instance.AppSpace,
		MessageCommon: messagecommon.New(line.Text, line.Time, LocalNodeId()),
	}

	if line.Err != nil {
		// Mark this as a special error record, as it is
		// coming from tail, not the app.
		msg.Source = "stackato.apptail"
		msg.LogFilename = ""
		log.Warnf("[%s] %s", instance.AppName, line.Text)
	}

	err := msg.Publish(pub, false)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>