<commit_before>package buildkit

import (
	"context"
	"io"
	"strings"
	"sync"
	"time"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/platforms"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/backend"
	"github.com/docker/docker/builder"
	"github.com/docker/docker/daemon/images"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/system"
	controlapi "github.com/moby/buildkit/api/services/control"
	"github.com/moby/buildkit/control"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/util/tracing"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
	grpcmetadata "google.golang.org/grpc/metadata"
)

// Opt is option struct required for creating the builder
type Opt struct {
	SessionManager *session.Manager
	Root           string
	Dist           images.DistributionServices
}

// Builder can build using BuildKit backend
type Builder struct {
	controller     *control.Controller
	reqBodyHandler *reqBodyHandler

	mu   sync.Mutex
	jobs map[string]*buildJob
}

// New creates a new builder
func New(opt Opt) (*Builder, error) {
	reqHandler := newReqBodyHandler(tracing.DefaultTransport)

	c, err := newController(reqHandler, opt)
	if err != nil {
		return nil, err
	}
	b := &Builder{
		controller:     c,
		reqBodyHandler: reqHandler,
		jobs:           map[string]*buildJob{},
	}
	return b, nil
}

// Cancel cancels a build using ID
func (b *Builder) Cancel(ctx context.Context, id string) error {
	b.mu.Lock()
	if j, ok := b.jobs[id]; ok && j.cancel != nil {
		j.cancel()
	}
	b.mu.Unlock()
	return nil
}

// DiskUsage returns a report about space used by build cache
func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
	duResp, err := b.controller.DiskUsage(ctx, &controlapi.DiskUsageRequest{})
	if err != nil {
		return nil, err
	}

	var items []*types.BuildCache
	for _, r := range duResp.Record {
		items = append(items, &types.BuildCache{
			ID:      r.ID,
			Mutable: r.Mutable,
			InUse:   r.InUse,
			Size:    r.Size_,

			CreatedAt:   r.CreatedAt,
			LastUsedAt:  r.LastUsedAt,
			UsageCount:  int(r.UsageCount),
			Parent:      r.Parent,
			Description: r.Description,
		})
	}
	return items, nil
}

// Prune clears all reclaimable build cache
func (b *Builder) Prune(ctx context.Context) (int64, error) {
	ch := make(chan *controlapi.UsageRecord)

	eg, ctx := errgroup.WithContext(ctx)

	eg.Go(func() error {
		defer close(ch)
		return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
			streamProxy: streamProxy{ctx: ctx},
			ch:          ch,
		})
	})

	var size int64
	eg.Go(func() error {
		for r := range ch {
			size += r.Size_
		}
		return nil
	})

	if err := eg.Wait(); err != nil {
		return 0, err
	}

	return size, nil
}

// Build executes a build request
func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.Result, error) {
	var rc = opt.Source

	if buildID := opt.Options.BuildID; buildID != "" {
		b.mu.Lock()

		upload := false
		if strings.HasPrefix(buildID, "upload-request:") {
			upload = true
			buildID = strings.TrimPrefix(buildID, "upload-request:")
		}

		if _, ok := b.jobs[buildID]; !ok {
			b.jobs[buildID] = newBuildJob()
		}
		j := b.jobs[buildID]
		var cancel func()
		ctx, cancel = context.WithCancel(ctx)
		j.cancel = cancel
		b.mu.Unlock()

		if upload {
			ctx2, cancel := context.WithTimeout(ctx, 5*time.Second)
			defer cancel()
			err := j.SetUpload(ctx2, rc)
			return nil, err
		}

		if remoteContext := opt.Options.RemoteContext; remoteContext == "upload-request" {
			ctx2, cancel := context.WithTimeout(ctx, 5*time.Second)
			defer cancel()
			var err error
			rc, err = j.WaitUpload(ctx2)
			if err != nil {
				return nil, err
			}
			opt.Options.RemoteContext = ""
		}

		defer func() {
			delete(b.jobs, buildID)
		}()
	}

	var out builder.Result

	id := identity.NewID()

	frontendAttrs := map[string]string{}

	if opt.Options.Target != "" {
		frontendAttrs["target"] = opt.Options.Target
	}

	if opt.Options.Dockerfile != "" && opt.Options.Dockerfile != "." {
		frontendAttrs["filename"] = opt.Options.Dockerfile
	}

	if opt.Options.RemoteContext != "" {
		if opt.Options.RemoteContext != "client-session" {
			frontendAttrs["context"] = opt.Options.RemoteContext
		}
	} else {
		url, cancel := b.reqBodyHandler.newRequest(rc)
		defer cancel()
		frontendAttrs["context"] = url
	}

	cacheFrom := append([]string{}, opt.Options.CacheFrom...)

	frontendAttrs["cache-from"] = strings.Join(cacheFrom, ",")

	for k, v := range opt.Options.BuildArgs {
		if v == nil {
			continue
		}
		frontendAttrs["build-arg:"+k] = *v
	}

	for k, v := range opt.Options.Labels {
		frontendAttrs["label:"+k] = v
	}

	if opt.Options.NoCache {
		frontendAttrs["no-cache"] = ""
	}

	if opt.Options.Platform != "" {
		// same as in newBuilder in builder/dockerfile.builder.go
		// TODO: remove once opt.Options.Platform is of type specs.Platform
		sp, err := platforms.Parse(opt.Options.Platform)
		if err != nil {
			return nil, err
		}
		if err := system.ValidatePlatform(sp); err != nil {
			return nil, err
		}
		frontendAttrs["platform"] = opt.Options.Platform
	}

	exporterAttrs := map[string]string{}

	if len(opt.Options.Tags) > 0 {
		exporterAttrs["name"] = strings.Join(opt.Options.Tags, ",")
	}

	req := &controlapi.SolveRequest{
		Ref:           id,
		Exporter:      "moby",
		ExporterAttrs: exporterAttrs,
		Frontend:      "dockerfile.v0",
		FrontendAttrs: frontendAttrs,
		Session:       opt.Options.SessionID,
	}

	aux := streamformatter.AuxFormatter{Writer: opt.ProgressWriter.Output}

	eg, ctx := errgroup.WithContext(ctx)

	eg.Go(func() error {
		resp, err := b.controller.Solve(ctx, req)
		if err != nil {
			return err
		}
		id, ok := resp.ExporterResponse["containerimage.digest"]
		if !ok {
			return errors.Errorf("missing image id")
		}
		out.ImageID = id
		return aux.Emit("moby.image.id", types.BuildResult{ID: id})
	})

	ch := make(chan *controlapi.StatusResponse)

	eg.Go(func() error {
		defer close(ch)
		return b.controller.Status(&controlapi.StatusRequest{
			Ref: id,
		}, &statusProxy{streamProxy: streamProxy{ctx: ctx}, ch: ch})
	})

	eg.Go(func() error {
		for sr := range ch {
			dt, err := sr.Marshal()
			if err != nil {
				return err
			}
			if err := aux.Emit("moby.buildkit.trace", dt); err != nil {
				return err
			}
		}
		return nil
	})

	if err := eg.Wait(); err != nil {
		return nil, err
	}

	return &out, nil
}

type streamProxy struct {
	ctx context.Context
}

func (sp *streamProxy) SetHeader(_ grpcmetadata.MD) error {
	return nil
}

func (sp *streamProxy) SendHeader(_ grpcmetadata.MD) error {
	return nil
}

func (sp *streamProxy) SetTrailer(_ grpcmetadata.MD) {
}

func (sp *streamProxy) Context() context.Context {
	return sp.ctx
}
func (sp *streamProxy) RecvMsg(m interface{}) error {
	return io.EOF
}

type statusProxy struct {
	streamProxy
	ch chan *controlapi.StatusResponse
}

func (sp *statusProxy) Send(resp *controlapi.StatusResponse) error {
	return sp.SendMsg(resp)
}
func (sp *statusProxy) SendMsg(m interface{}) error {
	if sr, ok := m.(*controlapi.StatusResponse); ok {
		sp.ch <- sr
	}
	return nil
}

type pruneProxy struct {
	streamProxy
	ch chan *controlapi.UsageRecord
}

func (sp *pruneProxy) Send(resp *controlapi.UsageRecord) error {
	return sp.SendMsg(resp)
}
func (sp *pruneProxy) SendMsg(m interface{}) error {
	if sr, ok := m.(*controlapi.UsageRecord); ok {
		sp.ch <- sr
	}
	return nil
}

type contentStoreNoLabels struct {
	content.Store
}

func (c *contentStoreNoLabels) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
	return content.Info{}, nil
}

type wrapRC struct {
	io.ReadCloser
	once   sync.Once
	err    error
	waitCh chan struct{}
}

func (w *wrapRC) Read(b []byte) (int, error) {
	n, err := w.ReadCloser.Read(b)
	if err != nil {
		e := err
		if e == io.EOF {
			e = nil
		}
		w.close(e)
	}
	return n, err
}

func (w *wrapRC) Close() error {
	err := w.ReadCloser.Close()
	w.close(err)
	return err
}

func (w *wrapRC) close(err error) {
	w.once.Do(func() {
		w.err = err
		close(w.waitCh)
	})
}

func (w *wrapRC) wait() error {
	<-w.waitCh
	return w.err
}

type buildJob struct {
	cancel func()
	waitCh chan func(io.ReadCloser) error
}

func newBuildJob() *buildJob {
	return &buildJob{waitCh: make(chan func(io.ReadCloser) error)}
}

func (j *buildJob) WaitUpload(ctx context.Context) (io.ReadCloser, error) {
	done := make(chan struct{})

	var upload io.ReadCloser
	fn := func(rc io.ReadCloser) error {
		w := &wrapRC{ReadCloser: rc, waitCh: make(chan struct{})}
		upload = w
		close(done)
		return w.wait()
	}

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case j.waitCh <- fn:
		<-done
		return upload, nil
	}
}

func (j *buildJob) SetUpload(ctx context.Context, rc io.ReadCloser) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	case fn := <-j.waitCh:
		return fn(rc)
	}
}
<commit_msg>builder: do not cancel buildkit status request<commit_after>package buildkit

import (
	"context"
	"io"
	"strings"
	"sync"
	"time"

	"github.com/containerd/containerd/content"
	"github.com/containerd/containerd/platforms"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/backend"
	"github.com/docker/docker/builder"
	"github.com/docker/docker/daemon/images"
	"github.com/docker/docker/pkg/streamformatter"
	"github.com/docker/docker/pkg/system"
	controlapi "github.com/moby/buildkit/api/services/control"
	"github.com/moby/buildkit/control"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/session"
	"github.com/moby/buildkit/util/tracing"
	"github.com/pkg/errors"
	"golang.org/x/sync/errgroup"
	grpcmetadata "google.golang.org/grpc/metadata"
)

// Opt is option struct required for creating the builder
type Opt struct {
	SessionManager *session.Manager
	Root           string
	Dist           images.DistributionServices
}

// Builder can build using BuildKit backend
type Builder struct {
	controller     *control.Controller
	reqBodyHandler *reqBodyHandler

	mu   sync.Mutex
	jobs map[string]*buildJob
}

// New creates a new builder
func New(opt Opt) (*Builder, error) {
	reqHandler := newReqBodyHandler(tracing.DefaultTransport)

	c, err := newController(reqHandler, opt)
	if err != nil {
		return nil, err
	}
	b := &Builder{
		controller:     c,
		reqBodyHandler: reqHandler,
		jobs:           map[string]*buildJob{},
	}
	return b, nil
}

// Cancel cancels a build using ID
func (b *Builder) Cancel(ctx context.Context, id string) error {
	b.mu.Lock()
	if j, ok := b.jobs[id]; ok && j.cancel != nil {
		j.cancel()
	}
	b.mu.Unlock()
	return nil
}

// DiskUsage returns a report about space used by build cache
func (b *Builder) DiskUsage(ctx context.Context) ([]*types.BuildCache, error) {
	duResp, err := b.controller.DiskUsage(ctx, &controlapi.DiskUsageRequest{})
	if err != nil {
		return nil, err
	}

	var items []*types.BuildCache
	for _, r := range duResp.Record {
		items = append(items, &types.BuildCache{
			ID:      r.ID,
			Mutable: r.Mutable,
			InUse:   r.InUse,
			Size:    r.Size_,

			CreatedAt:   r.CreatedAt,
			LastUsedAt:  r.LastUsedAt,
			UsageCount:  int(r.UsageCount),
			Parent:      r.Parent,
			Description: r.Description,
		})
	}
	return items, nil
}

// Prune clears all reclaimable build cache
func (b *Builder) Prune(ctx context.Context) (int64, error) {
	ch := make(chan *controlapi.UsageRecord)

	eg, ctx := errgroup.WithContext(ctx)

	eg.Go(func() error {
		defer close(ch)
		return b.controller.Prune(&controlapi.PruneRequest{}, &pruneProxy{
			streamProxy: streamProxy{ctx: ctx},
			ch:          ch,
		})
	})

	var size int64
	eg.Go(func() error {
		for r := range ch {
			size += r.Size_
		}
		return nil
	})

	if err := eg.Wait(); err != nil {
		return 0, err
	}

	return size, nil
}

// Build executes a build request
func (b *Builder) Build(ctx context.Context, opt backend.BuildConfig) (*builder.Result, error) {
	var rc = opt.Source

	if buildID := opt.Options.BuildID; buildID != "" {
		b.mu.Lock()

		upload := false
		if strings.HasPrefix(buildID, "upload-request:") {
			upload = true
			buildID = strings.TrimPrefix(buildID, "upload-request:")
		}

		if _, ok := b.jobs[buildID]; !ok {
			b.jobs[buildID] = newBuildJob()
		}
		j := b.jobs[buildID]
		var cancel func()
		ctx, cancel = context.WithCancel(ctx)
		j.cancel = cancel
		b.mu.Unlock()

		if upload {
			ctx2, cancel := context.WithTimeout(ctx, 5*time.Second)
			defer cancel()
			err := j.SetUpload(ctx2, rc)
			return nil, err
		}

		if remoteContext := opt.Options.RemoteContext; remoteContext == "upload-request" {
			ctx2, cancel := context.WithTimeout(ctx, 5*time.Second)
			defer cancel()
			var err error
			rc, err = j.WaitUpload(ctx2)
			if err != nil {
				return nil, err
			}
			opt.Options.RemoteContext = ""
		}

		defer func() {
			delete(b.jobs, buildID)
		}()
	}

	var out builder.Result

	id := identity.NewID()

	frontendAttrs := map[string]string{}

	if opt.Options.Target != "" {
		frontendAttrs["target"] = opt.Options.Target
	}

	if opt.Options.Dockerfile != "" && opt.Options.Dockerfile != "." {
		frontendAttrs["filename"] = opt.Options.Dockerfile
	}

	if opt.Options.RemoteContext != "" {
		if opt.Options.RemoteContext != "client-session" {
			frontendAttrs["context"] = opt.Options.RemoteContext
		}
	} else {
		url, cancel := b.reqBodyHandler.newRequest(rc)
		defer cancel()
		frontendAttrs["context"] = url
	}

	cacheFrom := append([]string{}, opt.Options.CacheFrom...)

	frontendAttrs["cache-from"] = strings.Join(cacheFrom, ",")

	for k, v := range opt.Options.BuildArgs {
		if v == nil {
			continue
		}
		frontendAttrs["build-arg:"+k] = *v
	}

	for k, v := range opt.Options.Labels {
		frontendAttrs["label:"+k] = v
	}

	if opt.Options.NoCache {
		frontendAttrs["no-cache"] = ""
	}

	if opt.Options.Platform != "" {
		// same as in newBuilder in builder/dockerfile.builder.go
		// TODO: remove once opt.Options.Platform is of type specs.Platform
		sp, err := platforms.Parse(opt.Options.Platform)
		if err != nil {
			return nil, err
		}
		if err := system.ValidatePlatform(sp); err != nil {
			return nil, err
		}
		frontendAttrs["platform"] = opt.Options.Platform
	}

	exporterAttrs := map[string]string{}

	if len(opt.Options.Tags) > 0 {
		exporterAttrs["name"] = strings.Join(opt.Options.Tags, ",")
	}

	req := &controlapi.SolveRequest{
		Ref:           id,
		Exporter:      "moby",
		ExporterAttrs: exporterAttrs,
		Frontend:      "dockerfile.v0",
		FrontendAttrs: frontendAttrs,
		Session:       opt.Options.SessionID,
	}

	aux := streamformatter.AuxFormatter{Writer: opt.ProgressWriter.Output}

	eg, ctx := errgroup.WithContext(ctx)

	eg.Go(func() error {
		resp, err := b.controller.Solve(ctx, req)
		if err != nil {
			return err
		}
		id, ok := resp.ExporterResponse["containerimage.digest"]
		if !ok {
			return errors.Errorf("missing image id")
		}
		out.ImageID = id
		return aux.Emit("moby.image.id", types.BuildResult{ID: id})
	})

	ch := make(chan *controlapi.StatusResponse)

	eg.Go(func() error {
		defer close(ch)
		// streamProxy.ctx is not set to ctx because when request is cancelled,
		// only the build request has to be cancelled, not the status request.
		stream := &statusProxy{streamProxy: streamProxy{ctx: context.TODO()}, ch: ch}
		return b.controller.Status(&controlapi.StatusRequest{Ref: id}, stream)
	})

	eg.Go(func() error {
		for sr := range ch {
			dt, err := sr.Marshal()
			if err != nil {
				return err
			}
			if err := aux.Emit("moby.buildkit.trace", dt); err != nil {
				return err
			}
		}
		return nil
	})

	if err := eg.Wait(); err != nil {
		return nil, err
	}

	return &out, nil
}

type streamProxy struct {
	ctx context.Context
}

func (sp *streamProxy) SetHeader(_ grpcmetadata.MD) error {
	return nil
}

func (sp *streamProxy) SendHeader(_ grpcmetadata.MD) error {
	return nil
}

func (sp *streamProxy) SetTrailer(_ grpcmetadata.MD) {
}

func (sp *streamProxy) Context() context.Context {
	return sp.ctx
}
func (sp *streamProxy) RecvMsg(m interface{}) error {
	return io.EOF
}

type statusProxy struct {
	streamProxy
	ch chan *controlapi.StatusResponse
}

func (sp *statusProxy) Send(resp *controlapi.StatusResponse) error {
	return sp.SendMsg(resp)
}
func (sp *statusProxy) SendMsg(m interface{}) error {
	if sr, ok := m.(*controlapi.StatusResponse); ok {
		sp.ch <- sr
	}
	return nil
}

type pruneProxy struct {
	streamProxy
	ch chan *controlapi.UsageRecord
}

func (sp *pruneProxy) Send(resp *controlapi.UsageRecord) error {
	return sp.SendMsg(resp)
}
func (sp *pruneProxy) SendMsg(m interface{}) error {
	if sr, ok := m.(*controlapi.UsageRecord); ok {
		sp.ch <- sr
	}
	return nil
}

type contentStoreNoLabels struct {
	content.Store
}

func (c *contentStoreNoLabels) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
	return content.Info{}, nil
}

type wrapRC struct {
	io.ReadCloser
	once   sync.Once
	err    error
	waitCh chan struct{}
}

func (w *wrapRC) Read(b []byte) (int, error) {
	n, err := w.ReadCloser.Read(b)
	if err != nil {
		e := err
		if e == io.EOF {
			e = nil
		}
		w.close(e)
	}
	return n, err
}

func (w *wrapRC) Close() error {
	err := w.ReadCloser.Close()
	w.close(err)
	return err
}

func (w *wrapRC) close(err error) {
	w.once.Do(func() {
		w.err = err
		close(w.waitCh)
	})
}

func (w *wrapRC) wait() error {
	<-w.waitCh
	return w.err
}

type buildJob struct {
	cancel func()
	waitCh chan func(io.ReadCloser) error
}

func newBuildJob() *buildJob {
	return &buildJob{waitCh: make(chan func(io.ReadCloser) error)}
}

func (j *buildJob) WaitUpload(ctx context.Context) (io.ReadCloser, error) {
	done := make(chan struct{})

	var upload io.ReadCloser
	fn := func(rc io.ReadCloser) error {
		w := &wrapRC{ReadCloser: rc, waitCh: make(chan struct{})}
		upload = w
		close(done)
		return w.wait()
	}

	select {
	case <-ctx.Done():
		return nil, ctx.Err()
	case j.waitCh <- fn:
		<-done
		return upload, nil
	}
}

func (j *buildJob) SetUpload(ctx context.Context, rc io.ReadCloser) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	case fn := <-j.waitCh:
		return fn(rc)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"syscall"
	"path/filepath"
	"strconv"
	"time"
	"bytes"
	"io"
	"strings"

	"github.com/joho/godotenv"
)

type Backend struct {
	appPath      string
	port         int
	process      *os.Process
	startedAt    time.Time
	exited       bool
	activityChan chan interface{}
}

func (b *Backend) Close() {
	log.Println("Terminating", b.appPath, "pid", b.process.Pid)

	err := b.process.Signal(syscall.SIGTERM)
	if err != nil {
		log.Println("failed to kill process: ", err)
		return
	}
	_, err = b.process.Wait()
	if err != nil {
		log.Println("failed to wait for process: ", err)
		return
	}

	log.Println("Terminated", b.appPath)
}

func (b *Backend) IsRestartRequested() bool {
	if b.exited {
		return true
	}
	fi, err := os.Stat(b.appPath + "/tmp/restart.txt")
	if err != nil {
		return false
	}
	return fi.ModTime().After(b.startedAt)
}

type BootCrash struct {
	Log bytes.Buffer
	Env []string
	Cmd string
	Path string
}
func (b BootCrash) Error() string {
	return "app crashed during boot"
}

func SpawnBackend(appName string) (*Backend, error) {
	pathToApp, err := appDir(appName)
	if err != nil {
		return nil, err
	}
	port, err := getFreeTCPPort()
	if err != nil {
		return nil, err
	}
	log.Println("Spawning", pathToApp, "on port", port)

	env := os.Environ()

	pathbytes, err := ioutil.ReadFile(os.Getenv("HOME") + "/.pow/.path")
	path := os.Getenv("PATH")
	if err == nil {
		path = string(pathbytes)
	} else {
		log.Println("while reading path file:", err)
	}
	// remove the old PATH
	for i, v := range env {
		if strings.Index(v, "PATH=") == 0 {
			env = append(env[:i], env[i+1:]...)
		}
	}
	env = append(env, "PATH="+path, "PORT="+strconv.Itoa(port))

	// add .env
	entries, err := godotenv.Read(pathToApp+"/.env")
	if err != nil {
		return nil, err
	}
	for k, v := range entries {
		env = append(env, k+"="+v)
	}

	procfile, err := ReadProcfile(pathToApp+"/Procfile")
	if err != nil {
		return nil, err
	}
	var CmdName string
	for _, v := range procfile.Entries {
		if v.Name == "web" {
			CmdName = v.Command
		}
	}

	if CmdName == "" {
		return nil, errors.New("No 'web' entry found in Procfile")
	}

	cmd := exec.Command("bash", "-c", "exec "+CmdName)

	var bootlog bytes.Buffer

	toStderrWithCapture := io.MultiWriter(os.Stderr, &bootlog)

	cmd.Stdout = toStderrWithCapture // never write to gowd's stdout
	cmd.Stderr = toStderrWithCapture
	cmd.Dir = pathToApp
	cmd.Env = env

	err = cmd.Start()
	if err != nil {
		return nil, err
	}

	b := &Backend{appPath: pathToApp, port: port, process: cmd.Process, startedAt: time.Now(), activityChan: make(chan interface{})}
	booting := true
	crashChan := make(chan error, 1)
	go func() {
		cmd.Wait()
		b.exited = true

		if booting {
			crashChan <- BootCrash{Log: bootlog, Env: env, Cmd: CmdName, Path: pathToApp}
		}
	}()

	log.Println("waiting for spawn result for", pathToApp)

	select {
	case <-awaitTCP(b.Address()):
		log.Println(pathToApp, "came up successfully")
		booting = false
		go b.watchForActivity()

		return b, nil
	case <-time.After(30 * time.Second):
		log.Println(pathToApp, "failed to bind")
		cmd.Process.Kill()
		return nil, errors.New("app failed to bind")
	case err := <-crashChan:
		log.Println(pathToApp, "crashed while starting")
		return nil, err
	}
}

func (b *Backend) Touch() {
	if b.activityChan != nil {
		b.activityChan <- new(interface{})
	}
}

func (b *Backend) Address() string {
	return "127.0.0.1:" + strconv.Itoa(b.port)
}

// Close the backend after inactivity
func (b *Backend) watchForActivity() {
outer:
	for {
		select {
		case _, ok := <-b.activityChan:
			if ok {
				continue
			} else {
				b.Close()
				b.activityChan = nil
				break outer
			}

		case <-time.After(30 * time.Minute):
			log.Println(b.appPath, "backend idling.")
			b.Close()
			b.activityChan = nil
			break outer
		}
	}
}

func awaitTCP(address string) chan bool {
	c := make(chan bool)
	go func() {
		for {
			_, err := net.Dial("tcp", address)
			if err == nil {
				c <- true
				break
			}
			time.Sleep(200 * time.Millisecond)
		}
	}()
	return c
}

func getFreeTCPPort() (port int, err error) {
	// We still have a small race condition here, but meh.
	l, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		return 0, err
	}
	port = l.Addr().(*net.TCPAddr).Port
	l.Close()
	return port, nil
}

func appDir(name string) (path string, err error) {
	path, err = filepath.EvalSymlinks(os.Getenv("HOME") + "/.pow/" + name)
	return
}
<commit_msg>don't ever wait(2) twice on the child<commit_after>package main

import (
	"errors"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"syscall"
	"path/filepath"
	"strconv"
	"time"
	"bytes"
	"io"
	"strings"

	"github.com/joho/godotenv"
)

type Backend struct {
	appPath      string
	port         int
	process      *os.Process
	startedAt    time.Time
	exited       bool
	exitChan 		 chan interface{}
	activityChan chan interface{}
}

func (b *Backend) Close() {
	log.Println("Terminating", b.appPath, "pid", b.process.Pid)

	err := b.process.Signal(syscall.SIGTERM)
	if err != nil {
		log.Println("failed to kill process: ", err)
		return
	}

	<-b.exitChan

	log.Println("Terminated", b.appPath)
}

func (b *Backend) IsRestartRequested() bool {
	if b.exited {
		return true
	}
	fi, err := os.Stat(b.appPath + "/tmp/restart.txt")
	if err != nil {
		return false
	}
	return fi.ModTime().After(b.startedAt)
}

type BootCrash struct {
	Log bytes.Buffer
	Env []string
	Cmd string
	Path string
}
func (b BootCrash) Error() string {
	return "app crashed during boot"
}

func SpawnBackend(appName string) (*Backend, error) {
	pathToApp, err := appDir(appName)
	if err != nil {
		return nil, err
	}
	port, err := getFreeTCPPort()
	if err != nil {
		return nil, err
	}
	log.Println("Spawning", pathToApp, "on port", port)

	env := os.Environ()

	pathbytes, err := ioutil.ReadFile(os.Getenv("HOME") + "/.pow/.path")
	path := os.Getenv("PATH")
	if err == nil {
		path = string(pathbytes)
	} else {
		log.Println("while reading path file:", err)
	}
	// remove the old PATH
	for i, v := range env {
		if strings.Index(v, "PATH=") == 0 {
			env = append(env[:i], env[i+1:]...)
		}
	}
	env = append(env, "PATH="+path, "PORT="+strconv.Itoa(port))

	// add .env
	entries, err := godotenv.Read(pathToApp+"/.env")
	if err != nil {
		return nil, err
	}
	for k, v := range entries {
		env = append(env, k+"="+v)
	}

	procfile, err := ReadProcfile(pathToApp+"/Procfile")
	if err != nil {
		return nil, err
	}
	var CmdName string
	for _, v := range procfile.Entries {
		if v.Name == "web" {
			CmdName = v.Command
		}
	}

	if CmdName == "" {
		return nil, errors.New("No 'web' entry found in Procfile")
	}

	cmd := exec.Command("bash", "-c", "exec "+CmdName)

	var bootlog bytes.Buffer

	toStderrWithCapture := io.MultiWriter(os.Stderr, &bootlog)

	cmd.Stdout = toStderrWithCapture // never write to gowd's stdout
	cmd.Stderr = toStderrWithCapture
	cmd.Dir = pathToApp
	cmd.Env = env

	err = cmd.Start()
	if err != nil {
		return nil, err
	}

	exitChan := make(chan interface{}, 1)
	b := &Backend{appPath: pathToApp, port: port, process: cmd.Process, startedAt: time.Now(), activityChan: make(chan interface{}), exitChan: exitChan}
	booting := true
	crashChan := make(chan error, 1)
	go func() {
		cmd.Wait()
		b.exited = true
		b.exitChan <- new(interface{})

		if booting {
			crashChan <- BootCrash{Log: bootlog, Env: env, Cmd: CmdName, Path: pathToApp}
		}
	}()

	log.Println("waiting for spawn result for", pathToApp)

	select {
	case <-awaitTCP(b.Address()):
		log.Println(pathToApp, "came up successfully")
		booting = false
		go b.watchForActivity()

		return b, nil
	case <-time.After(30 * time.Second):
		log.Println(pathToApp, "failed to bind")
		cmd.Process.Kill()
		return nil, errors.New("app failed to bind")
	case err := <-crashChan:
		log.Println(pathToApp, "crashed while starting")
		return nil, err
	}
}

func (b *Backend) Touch() {
	if b.activityChan != nil {
		b.activityChan <- new(interface{})
	}
}

func (b *Backend) Address() string {
	return "127.0.0.1:" + strconv.Itoa(b.port)
}

// Close the backend after inactivity
func (b *Backend) watchForActivity() {
outer:
	for {
		select {
		case _, ok := <-b.activityChan:
			if ok {
				continue
			} else {
				b.Close()
				b.activityChan = nil
				break outer
			}

		case <-time.After(30 * time.Minute):
			log.Println(b.appPath, "backend idling.")
			b.Close()
			b.activityChan = nil
			break outer
		}
	}
}

func awaitTCP(address string) chan bool {
	c := make(chan bool)
	go func() {
		for {
			_, err := net.Dial("tcp", address)
			if err == nil {
				c <- true
				break
			}
			time.Sleep(200 * time.Millisecond)
		}
	}()
	return c
}

func getFreeTCPPort() (port int, err error) {
	// We still have a small race condition here, but meh.
	l, err := net.Listen("tcp", "localhost:0")
	if err != nil {
		return 0, err
	}
	port = l.Addr().(*net.TCPAddr).Port
	l.Close()
	return port, nil
}

func appDir(name string) (path string, err error) {
	path, err = filepath.EvalSymlinks(os.Getenv("HOME") + "/.pow/" + name)
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

// +genclient
// +k8s:openapi-gen=true
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// +kubebuilder:resource:path=certificates
// Certificate is a type to represent a Certificate from ACME
type Certificate struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   CertificateSpec   `json:"spec,omitempty"`
	Status CertificateStatus `json:"status,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// CertificateList is a list of Certificates
type CertificateList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`

	Items []Certificate `json:"items"`
}

type KeyAlgorithm string

const (
	RSAKeyAlgorithm   KeyAlgorithm = "rsa"
	ECDSAKeyAlgorithm KeyAlgorithm = "ecdsa"
)

// CertificateSpec defines the desired state of Certificate
type CertificateSpec struct {
	// CommonName is a common name to be used on the Certificate
	CommonName string `json:"commonName,omitempty"`

	// Organization is the organization to be used on the Certificate
	Organization []string `json:"organization,omitempty"`

	// Certificate default Duration
	Duration *metav1.Duration `json:"duration,omitempty"`

	// Certificate renew before expiration duration
	RenewBefore *metav1.Duration `json:"renewBefore,omitempty"`

	// DNSNames is a list of subject alt names to be used on the Certificate
	DNSNames []string `json:"dnsNames,omitempty"`

	// SecretName is the name of the secret resource to store this secret in
	SecretName string `json:"secretName"`

	// IssuerRef is a reference to the issuer for this certificate.
	// If the 'kind' field is not set, or set to 'Issuer', an Issuer resource
	// with the given name in the same namespace as the Certificate will be used.
	// If the 'kind' field is set to 'ClusterIssuer', a ClusterIssuer with the
	// provided name will be used.
	// The 'name' field in this stanza is required at all times.
	IssuerRef ObjectReference `json:"issuerRef"`

	// IsCA will mark this Certificate as valid for signing.
	// This implies that the 'signing' usage is set
	IsCA bool `json:"isCA,omitempty"`

	// ACME contains configuration specific to ACME Certificates.
	// Notably, this contains details on how the domain names listed on this
	// Certificate resource should be 'solved', i.e. mapping HTTP01 and DNS01
	// providers to DNS names.
	ACME *ACMECertificateConfig `json:"acme,omitempty"`

	// KeySize is the key bit size of the corresponding private key for this certificate.
	// If provided, value must be between 2048 and 8192 inclusive when KeyAlgorithm is
	// empty or is set to "rsa", and value must be one of (256, 384, 521) when
	// KeyAlgorithm is set to "ecdsa".
	KeySize int `json:"keySize,omitempty"`
	// KeyAlgorithm is the private key algorithm of the corresponding private key
	// for this certificate. If provided, allowed values are either "rsa" or "ecdsa"
	// If KeyAlgorithm is specified and KeySize is not provided,
	// key size of 256 will be used for "ecdsa" key algorithm and
	// key size of 2048 will be used for "rsa" key algorithm.
	KeyAlgorithm KeyAlgorithm `json:"keyAlgorithm,omitempty"`
}

// ACMECertificateConfig contains the configuration for the ACME certificate provider
type ACMECertificateConfig struct {
	Config []DomainSolverConfig `json:"config"`
}

// CertificateStatus defines the observed state of Certificate
type CertificateStatus struct {
	Conditions      []CertificateCondition `json:"conditions,omitempty"`
	LastFailureTime *metav1.Time           `json:"lastFailureTime,omitempty"`

	// The expiration time of the certificate stored in the secret named
	// by this resource in spec.secretName.
	NotAfter *metav1.Time `json:"notAfter,omitempty"`
}

// CertificateCondition contains condition information for an Certificate.
type CertificateCondition struct {
	// Type of the condition, currently ('Ready').
	Type CertificateConditionType `json:"type"`

	// Status of the condition, one of ('True', 'False', 'Unknown').
	Status ConditionStatus `json:"status"`

	// LastTransitionTime is the timestamp corresponding to the last status
	// change of this condition.
	LastTransitionTime metav1.Time `json:"lastTransitionTime"`

	// Reason is a brief machine readable explanation for the condition's last
	// transition.
	Reason string `json:"reason"`

	// Message is a human readable description of the details of the last
	// transition, complementing reason.
	Message string `json:"message"`
}

// CertificateConditionType represents an Certificate condition value.
type CertificateConditionType string

const (
	// CertificateConditionReady represents the fact that a given Certificate condition
	// is in ready state.
	CertificateConditionReady CertificateConditionType = "Ready"

	// CertificateConditionValidationFailed is used to indicate whether a
	// validation for a Certificate has failed.
	// This is currently used by the ACME issuer to track when the last
	// validation was attempted.
	CertificateConditionValidationFailed CertificateConditionType = "ValidateFailed"
)
<commit_msg>Update comment on Ready condition API type<commit_after>/*
Copyright 2018 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

// +genclient
// +k8s:openapi-gen=true
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// +kubebuilder:resource:path=certificates
// Certificate is a type to represent a Certificate from ACME
type Certificate struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Spec   CertificateSpec   `json:"spec,omitempty"`
	Status CertificateStatus `json:"status,omitempty"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// CertificateList is a list of Certificates
type CertificateList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`

	Items []Certificate `json:"items"`
}

type KeyAlgorithm string

const (
	RSAKeyAlgorithm   KeyAlgorithm = "rsa"
	ECDSAKeyAlgorithm KeyAlgorithm = "ecdsa"
)

// CertificateSpec defines the desired state of Certificate
type CertificateSpec struct {
	// CommonName is a common name to be used on the Certificate
	CommonName string `json:"commonName,omitempty"`

	// Organization is the organization to be used on the Certificate
	Organization []string `json:"organization,omitempty"`

	// Certificate default Duration
	Duration *metav1.Duration `json:"duration,omitempty"`

	// Certificate renew before expiration duration
	RenewBefore *metav1.Duration `json:"renewBefore,omitempty"`

	// DNSNames is a list of subject alt names to be used on the Certificate
	DNSNames []string `json:"dnsNames,omitempty"`

	// SecretName is the name of the secret resource to store this secret in
	SecretName string `json:"secretName"`

	// IssuerRef is a reference to the issuer for this certificate.
	// If the 'kind' field is not set, or set to 'Issuer', an Issuer resource
	// with the given name in the same namespace as the Certificate will be used.
	// If the 'kind' field is set to 'ClusterIssuer', a ClusterIssuer with the
	// provided name will be used.
	// The 'name' field in this stanza is required at all times.
	IssuerRef ObjectReference `json:"issuerRef"`

	// IsCA will mark this Certificate as valid for signing.
	// This implies that the 'signing' usage is set
	IsCA bool `json:"isCA,omitempty"`

	// ACME contains configuration specific to ACME Certificates.
	// Notably, this contains details on how the domain names listed on this
	// Certificate resource should be 'solved', i.e. mapping HTTP01 and DNS01
	// providers to DNS names.
	ACME *ACMECertificateConfig `json:"acme,omitempty"`

	// KeySize is the key bit size of the corresponding private key for this certificate.
	// If provided, value must be between 2048 and 8192 inclusive when KeyAlgorithm is
	// empty or is set to "rsa", and value must be one of (256, 384, 521) when
	// KeyAlgorithm is set to "ecdsa".
	KeySize int `json:"keySize,omitempty"`
	// KeyAlgorithm is the private key algorithm of the corresponding private key
	// for this certificate. If provided, allowed values are either "rsa" or "ecdsa"
	// If KeyAlgorithm is specified and KeySize is not provided,
	// key size of 256 will be used for "ecdsa" key algorithm and
	// key size of 2048 will be used for "rsa" key algorithm.
	KeyAlgorithm KeyAlgorithm `json:"keyAlgorithm,omitempty"`
}

// ACMECertificateConfig contains the configuration for the ACME certificate provider
type ACMECertificateConfig struct {
	Config []DomainSolverConfig `json:"config"`
}

// CertificateStatus defines the observed state of Certificate
type CertificateStatus struct {
	Conditions      []CertificateCondition `json:"conditions,omitempty"`
	LastFailureTime *metav1.Time           `json:"lastFailureTime,omitempty"`

	// The expiration time of the certificate stored in the secret named
	// by this resource in spec.secretName.
	NotAfter *metav1.Time `json:"notAfter,omitempty"`
}

// CertificateCondition contains condition information for an Certificate.
type CertificateCondition struct {
	// Type of the condition, currently ('Ready').
	Type CertificateConditionType `json:"type"`

	// Status of the condition, one of ('True', 'False', 'Unknown').
	Status ConditionStatus `json:"status"`

	// LastTransitionTime is the timestamp corresponding to the last status
	// change of this condition.
	LastTransitionTime metav1.Time `json:"lastTransitionTime"`

	// Reason is a brief machine readable explanation for the condition's last
	// transition.
	Reason string `json:"reason"`

	// Message is a human readable description of the details of the last
	// transition, complementing reason.
	Message string `json:"message"`
}

// CertificateConditionType represents an Certificate condition value.
type CertificateConditionType string

const (
	// CertificateConditionReady indicates that a certificate is ready for use.
	// This is defined as:
	// - The target secret exists
	// - The target secret contains a certificate that has not expired
	// - The target secret contains a private key valid for the certificate
	// - The commonName and dnsNames attributes match those specified on the Certificate
	CertificateConditionReady CertificateConditionType = "Ready"
)
<|endoftext|>


<commit_before>package anaconda

import (
	"time"

	"github.com/azr/backoff"
)

/*
Reconnecting(from https://dev.twitter.com/streaming/overview/connecting) :

Once an established connection drops, attempt to reconnect immediately.
If the reconnect fails, slow down your reconnect attempts according to the type of error experienced:
*/

//Back off linearly for TCP/IP level network errors.
//	These problems are generally temporary and tend to clear quickly.
//	Increase the delay in reconnects by 250ms each attempt, up to 16 seconds.
func NewTCPIPErrBackoff() backoff.BackOffer {
	return backoff.NewLinear(0, time.Second*16, time.Millisecond*250, 1)
}

//Back off exponentially for HTTP errors for which reconnecting would be appropriate.
//	Start with a 5 second wait, doubling each attempt, up to 320 seconds.
func NewHTTPErrBackoff() backoff.BackOffer {
	eb := backoff.NewExponential()
	eb.InitialInterval = time.Second * 5
	eb.MaxInterval = time.Second * 320
	eb.Multiplier = 2
	eb.Reset()
	return eb
}

// Back off exponentially for HTTP 420 errors.
// 	Start with a 1 minute wait and double each attempt.
// 	Note that every HTTP 420 received increases the time you must
// 	wait until rate limiting will no longer will be in effect for your account.
func NewHTTP420ErrBackoff() backoff.BackOffer {
	eb := backoff.NewExponential()
	eb.InitialInterval = time.Minute * 1
	eb.Multiplier = 2
	eb.MaxInterval = time.Minute * 20
	eb.Reset()
	return eb
}
<commit_msg>Update streaming funcs for latest version of azr/backoff<commit_after>package anaconda

import (
	"time"

	"github.com/azr/backoff"
)

/*
Reconnecting(from https://dev.twitter.com/streaming/overview/connecting) :

Once an established connection drops, attempt to reconnect immediately.
If the reconnect fails, slow down your reconnect attempts according to the type of error experienced:
*/

//Back off linearly for TCP/IP level network errors.
//	These problems are generally temporary and tend to clear quickly.
//	Increase the delay in reconnects by 250ms each attempt, up to 16 seconds.
func NewTCPIPErrBackoff() backoff.Interface {
	return backoff.NewLinear(0, time.Second*16, time.Millisecond*250, 1)
}

//Back off exponentially for HTTP errors for which reconnecting would be appropriate.
//	Start with a 5 second wait, doubling each attempt, up to 320 seconds.
func NewHTTPErrBackoff() backoff.Interface {
	eb := backoff.NewExponential()
	eb.InitialInterval = time.Second * 5
	eb.MaxInterval = time.Second * 320
	eb.Multiplier = 2
	eb.Reset()
	return eb
}

// Back off exponentially for HTTP 420 errors.
// 	Start with a 1 minute wait and double each attempt.
// 	Note that every HTTP 420 received increases the time you must
// 	wait until rate limiting will no longer will be in effect for your account.
func NewHTTP420ErrBackoff() backoff.Interface {
	eb := backoff.NewExponential()
	eb.InitialInterval = time.Minute * 1
	eb.Multiplier = 2
	eb.MaxInterval = time.Minute * 20
	eb.Reset()
	return eb
}
<|endoftext|>


<commit_before>package types

import (
	"encoding/json"
	"strconv"
	"testing"
	"time"
)

// TestJourneyString tests the String method
func TestJourneyString(t *testing.T) {
	from := Address{
		ID:    "2.399803859568057;48.88150165806373",
		Label: "54 Boulevard d'Algérie (Paris)",
		Name:  "Boulevard d'Algérie",
	}

	to := Address{
		ID:    "2.344404;48.835114",
		Label: "54 Boulevard Arago (Paris)",
		Name:  "Boulevard Arago",
	}

	departure, err := time.Parse("2006-01-02T15:04:05", "2017-04-11T21:33:55")
	if err != nil {
		t.Fatal(err)
	}
	arrival, err := time.Parse("2006-01-02T15:04:05", "2017-04-11T22:24:13")
	if err != nil {
		t.Fatal(err)
	}

	section := Section{
		From:      from,
		To:        to,
		Departure: departure,
		Arrival:   arrival,
		Duration:  time.Duration(3018) * time.Second,
		Display: DisplayInformations{
			Label:        "11",
			PhysicalMode: "Métro",
		},
	}

	journey := Journey{
		From:      from,
		To:        to,
		Departure: departure,
		Arrival:   arrival,
		Duration:  time.Duration(3018) * time.Second,
		Sections:  []Section{section},
	}

	t.Logf("For journey we have: %s", journey.String())
}

// TestJourneyUnmarshal_NoCompare tries to unmarshal all json test data for this type, but doesn't compare its response to a known correct output.
func TestJourneyUnmarshal_NoCompare(t *testing.T) {
	// Get the input
	input := testData["journey"]
	if len(input) == 0 {
		t.Skip("No data to test")
	}

	// For each of them, let's run a subtest
	for i, reader := range input {
		// Create a name for this run
		name := strconv.Itoa(i)

		// Create the run function
		rfunc := func(t *testing.T) {
			var j = &Journey{}
			dec := json.NewDecoder(reader)
			err := dec.Decode(j)
			if err != nil {
				t.Errorf("Error while unmarshalling: %v", err)
			}
		}

		// Run !
		t.Run(name, rfunc)
	}
}
<commit_msg>types: Add validity check for TestJourneyString<commit_after>package types

import (
	"encoding/json"
	"strconv"
	"testing"
	"time"
)

// TestJourneyString tests the String method
func TestJourneyString(t *testing.T) {
	from := Address{
		ID:    "2.399803859568057;48.88150165806373",
		Label: "54 Boulevard d'Algérie (Paris)",
		Name:  "Boulevard d'Algérie",
	}

	to := Address{
		ID:    "2.344404;48.835114",
		Label: "54 Boulevard Arago (Paris)",
		Name:  "Boulevard Arago",
	}

	departure, err := time.Parse("2006-01-02T15:04:05", "2017-04-11T21:33:55")
	if err != nil {
		t.Fatal(err)
	}
	arrival, err := time.Parse("2006-01-02T15:04:05", "2017-04-11T22:24:13")
	if err != nil {
		t.Fatal(err)
	}

	section := Section{
		From:      from,
		To:        to,
		Departure: departure,
		Arrival:   arrival,
		Duration:  time.Duration(3018) * time.Second,
		Display: DisplayInformations{
			Label:        "11",
			PhysicalMode: "Métro",
		},
	}

	journey := Journey{
		From:      from,
		To:        to,
		Departure: departure,
		Arrival:   arrival,
		Duration:  time.Duration(3018) * time.Second,
		Sections:  []Section{section},
	}

	want := "Boulevard d'Algérie (11/04 @ 21:33) --(50m18s)--> Boulevard Arago (11/04 @ 22:24)\n\tBoulevard d'Algérie (11/04 @ 21:33) --(Métro 11 | 50m18s)--> Boulevard Arago (11/04 @ 22:24)"

	if journey.String() != want {
		t.Error("Output of String isn't what was expected")
	}

	t.Logf("For journey we have: %s", journey.String())
}

// TestJourneyUnmarshal_NoCompare tries to unmarshal all json test data for this type, but doesn't compare its response to a known correct output.
func TestJourneyUnmarshal_NoCompare(t *testing.T) {
	// Get the input
	input := testData["journey"]
	if len(input) == 0 {
		t.Skip("No data to test")
	}

	// For each of them, let's run a subtest
	for i, reader := range input {
		// Create a name for this run
		name := strconv.Itoa(i)

		// Create the run function
		rfunc := func(t *testing.T) {
			var j = &Journey{}
			dec := json.NewDecoder(reader)
			err := dec.Decode(j)
			if err != nil {
				t.Errorf("Error while unmarshalling: %v", err)
			}
		}

		// Run !
		t.Run(name, rfunc)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/janekolszak/idp/core"
	"github.com/janekolszak/idp/helpers"
	"github.com/janekolszak/idp/providers"
	"github.com/janekolszak/idp/providers/cookie"

	"flag"
	"fmt"
	"github.com/gorilla/sessions"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"text/template"
	"time"
)

const (
	consent = `
	Hi {{.User}}!
	Do you agree to grant {{.Client}} access to those scopes?
	{{range .Scopes}}
	{{.}}
	{{end}}

	<form method="post">
		<input type="submit" name="answer" value="y">
		<input type="submit" name="answer" value="n">
	</form>

	`
)

var (
	// Configuration file
	config         *helpers.HydraConfig
	idp            *core.IDP
	provider       *providers.BasicAuth
	cookieProvider *cookie.CookieAuth

	// Command line options
	// clientID     = flag.String("id", "someid", "OAuth2 client ID of the IdP")
	// clientSecret = flag.String("secret", "somesecret", "OAuth2 client secret")
	hydraURL     = flag.String("hydra", "https://hydra:4444", "Hydra's URL")
	configPath   = flag.String("conf", ".hydra.yml", "Path to Hydra's configuration")
	htpasswdPath = flag.String("htpasswd", "/etc/idp/htpasswd", "Path to credentials in htpasswd format")
	cookieDBPath = flag.String("cookie-db", "/etc/idp/remember.db3", "Path to a database with remember me cookies")
)

func HandleChallengeGET() httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
		fmt.Println("Challenge!")

		user, err := cookieProvider.Check(r)
		if err == nil {
			fmt.Println("Authenticated with Cookie")
		} else {
			// Can't authenticate with "Remember Me" cookie,
			// so try with another provider:

			user, err = provider.Check(r)
			if err != nil {
				// Authentication failed, or any other error
				fmt.Println(err.Error())
				provider.Respond(w, r)
				return
			}
			fmt.Println("Authenticated with Basic Auth")

		}

		// Authentication success, save the "Remember Me" cookie
		// TODO: Implement Update method. Add should be used only for creating new cookies.
		err = cookieProvider.Add(w, r, user)
		if err != nil {
			fmt.Println(err.Error())
		}

		challenge, err := idp.NewChallenge(r, user)
		if err != nil {
			fmt.Println(err.Error())
			provider.Respond(w, r)
			return
		}

		err = challenge.Save(w, r)
		if err != nil {
			fmt.Println(err.Error())
			provider.Respond(w, r)
		}

		http.Redirect(w, r, "/consent", http.StatusFound)
	}
}

func HandleConsentGET() httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
		challenge, err := idp.GetChallenge(r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		fmt.Println("Data ", challenge.User)

		t := template.Must(template.New("tmpl").Parse(consent))

		t.Execute(w, challenge)
	}
}

func HandleConsentPOST() httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {

		fmt.Println("Consent POST!")
		challenge, err := idp.GetChallenge(r)
		if err != nil {
			fmt.Println(err.Error())
			provider.Respond(w, r)
		}

		answer := r.FormValue("answer")
		fmt.Println("Answer: ", answer)

		if answer != "y" {
			// No challenge token
			// TODO: Handle negative answer
			return
		}

		err = challenge.GrantAccessToAll(w, r)
		if err != nil {
			// Server error
			fmt.Println(err.Error())
			provider.Respond(w, r)
			return
		}
	}
}

func main() {
	fmt.Println("Identity Provider started!")

	flag.Parse()
	// Read the configuration file
	hydraConfig := helpers.NewHydraConfig(*configPath)

	// Setup the providers
	var err error
	provider, err = providers.NewBasicAuth(*htpasswdPath, "localhost")
	if err != nil {
		panic(err)
	}

	cookieProvider, err = cookie.NewCookieAuth(*cookieDBPath)
	if err != nil {
		panic(err)
	}

	config := core.IDPConfig{
		HydraAddress:            *hydraURL,
		ClientID:                hydraConfig.ClientID,
		ClientSecret:            hydraConfig.ClientSecret,
		KeyCacheExpiration:      10 * time.Minute,
		KeyCacheCleanupInterval: 30 * time.Second,

		// TODO: [IMPORTANT] Don't use CookieStore here
		ChallengeStore: sessions.NewCookieStore([]byte("something-very-secret")),
	}

	idp = core.NewIDP(&config)

	// Connect with Hydra
	err = idp.Connect()
	if err != nil {
		panic(err)
	}

	router := httprouter.New()
	router.GET("/", HandleChallengeGET())
	router.POST("/", HandleChallengeGET())
	router.GET("/consent", HandleConsentGET())
	router.POST("/consent", HandleConsentPOST())
	http.ListenAndServe(":3000", router)

	idp.Close()
}
<commit_msg>remove unused global var "config"<commit_after>package main

import (
	"github.com/janekolszak/idp/core"
	"github.com/janekolszak/idp/helpers"
	"github.com/janekolszak/idp/providers"
	"github.com/janekolszak/idp/providers/cookie"

	"flag"
	"fmt"
	"github.com/gorilla/sessions"
	"github.com/julienschmidt/httprouter"
	"net/http"
	"text/template"
	"time"
)

const (
	consent = `
	Hi {{.User}}!
	Do you agree to grant {{.Client}} access to those scopes?
	{{range .Scopes}}
	{{.}}
	{{end}}

	<form method="post">
		<input type="submit" name="answer" value="y">
		<input type="submit" name="answer" value="n">
	</form>

	`
)

var (
	idp            *core.IDP
	provider       *providers.BasicAuth
	cookieProvider *cookie.CookieAuth

	// Command line options
	// clientID     = flag.String("id", "someid", "OAuth2 client ID of the IdP")
	// clientSecret = flag.String("secret", "somesecret", "OAuth2 client secret")
	hydraURL     = flag.String("hydra", "https://hydra:4444", "Hydra's URL")
	configPath   = flag.String("conf", ".hydra.yml", "Path to Hydra's configuration")
	htpasswdPath = flag.String("htpasswd", "/etc/idp/htpasswd", "Path to credentials in htpasswd format")
	cookieDBPath = flag.String("cookie-db", "/etc/idp/remember.db3", "Path to a database with remember me cookies")
)

func HandleChallengeGET() httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
		fmt.Println("Challenge!")

		user, err := cookieProvider.Check(r)
		if err == nil {
			fmt.Println("Authenticated with Cookie")
		} else {
			// Can't authenticate with "Remember Me" cookie,
			// so try with another provider:

			user, err = provider.Check(r)
			if err != nil {
				// Authentication failed, or any other error
				fmt.Println(err.Error())
				provider.Respond(w, r)
				return
			}
			fmt.Println("Authenticated with Basic Auth")

		}

		// Authentication success, save the "Remember Me" cookie
		// TODO: Implement Update method. Add should be used only for creating new cookies.
		err = cookieProvider.Add(w, r, user)
		if err != nil {
			fmt.Println(err.Error())
		}

		challenge, err := idp.NewChallenge(r, user)
		if err != nil {
			fmt.Println(err.Error())
			provider.Respond(w, r)
			return
		}

		err = challenge.Save(w, r)
		if err != nil {
			fmt.Println(err.Error())
			provider.Respond(w, r)
		}

		http.Redirect(w, r, "/consent", http.StatusFound)
	}
}

func HandleConsentGET() httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
		challenge, err := idp.GetChallenge(r)
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}

		fmt.Println("Data ", challenge.User)

		t := template.Must(template.New("tmpl").Parse(consent))

		t.Execute(w, challenge)
	}
}

func HandleConsentPOST() httprouter.Handle {
	return func(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {

		fmt.Println("Consent POST!")
		challenge, err := idp.GetChallenge(r)
		if err != nil {
			fmt.Println(err.Error())
			provider.Respond(w, r)
		}

		answer := r.FormValue("answer")
		fmt.Println("Answer: ", answer)

		if answer != "y" {
			// No challenge token
			// TODO: Handle negative answer
			return
		}

		err = challenge.GrantAccessToAll(w, r)
		if err != nil {
			// Server error
			fmt.Println(err.Error())
			provider.Respond(w, r)
			return
		}
	}
}

func main() {
	fmt.Println("Identity Provider started!")

	flag.Parse()
	// Read the configuration file
	hydraConfig := helpers.NewHydraConfig(*configPath)

	// Setup the providers
	var err error
	provider, err = providers.NewBasicAuth(*htpasswdPath, "localhost")
	if err != nil {
		panic(err)
	}

	cookieProvider, err = cookie.NewCookieAuth(*cookieDBPath)
	if err != nil {
		panic(err)
	}

	config := core.IDPConfig{
		HydraAddress:            *hydraURL,
		ClientID:                hydraConfig.ClientID,
		ClientSecret:            hydraConfig.ClientSecret,
		KeyCacheExpiration:      10 * time.Minute,
		KeyCacheCleanupInterval: 30 * time.Second,

		// TODO: [IMPORTANT] Don't use CookieStore here
		ChallengeStore: sessions.NewCookieStore([]byte("something-very-secret")),
	}

	idp = core.NewIDP(&config)

	// Connect with Hydra
	err = idp.Connect()
	if err != nil {
		panic(err)
	}

	router := httprouter.New()
	router.GET("/", HandleChallengeGET())
	router.POST("/", HandleChallengeGET())
	router.GET("/consent", HandleConsentGET())
	router.POST("/consent", HandleConsentPOST())
	http.ListenAndServe(":3000", router)

	idp.Close()
}
<|endoftext|>


<commit_before>package main

import "bytes"
import "flag"
import "encoding/asn1"
import "fmt"
import "gss/proxy"
import "gss/misc"
import "net"
import "os"
import "strings"

func connectOnce(pconn *net.Conn, pcc *proxy.CallCtx, host string, port int, service string, mcount int, quiet bool, plain []byte, v1 bool, nmech *asn1.ObjectIdentifier, mech asn1.ObjectIdentifier, delegate, seq, noreplay, nomutual, noauth, nowrap, noenc, nomic bool) {
	var ctx proxy.SecCtx
	var status proxy.Status
	var tag byte
	var ptoken *[]byte
	var major, minor uint64
	var sname proxy.Name
	var localstate, openstate string
	var flags proxy.Flags

	/* Open the connection. */
	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", host, port))
	if err != nil {
		fmt.Printf("Error connecting: %s\n", err)
		os.Exit(2)
	}
	defer conn.Close()

	/* Import the remote service's name. */
	if strings.Contains(service, "@") {
		sname.DisplayName = service
	} else {
		sname.DisplayName = service + "@" + host
	}
	sname.NameType = proxy.NT_HOSTBASED_SERVICE
	if nmech != nil {
		icnr, err := proxy.ImportAndCanonName(pconn, pcc, sname, *nmech, nil, nil)
		if err != nil {
			fmt.Printf("Error importing remote service name: %s\n", err)
			return
		}
		if icnr.Status.MajorStatus != 0 {
			DisplayProxyStatus("importing remote service name", icnr.Status)
			return
		}
		sname = *icnr.Name
	}

	if noauth {
		misc.SendToken(conn, misc.TOKEN_NOOP, nil)
	} else {
		if !v1 {
			misc.SendToken(conn, misc.TOKEN_NOOP|misc.TOKEN_CONTEXT_NEXT, nil)
		}
		flags = proxy.Flags{Deleg: delegate, Sequence: seq, Replay: !noreplay, Conf: !noenc, Integ: !nomic, Mutual: !nomutual}
		for true {
			/* Start/continue. */
			iscr, err := proxy.InitSecContext(pconn, pcc, &ctx, nil, &sname, mech, flags, proxy.C_INDEFINITE, nil, ptoken, nil)
			if err != nil {
				fmt.Printf("Error initializing security context: %s\n", err)
				return
			}
			status = iscr.Status
			major = status.MajorStatus
			if major != proxy.S_COMPLETE && major != proxy.S_CONTINUE_NEEDED {
				DisplayProxyStatus("initializing security context", iscr.Status)
				return
			}
			/* If we have an output token, we need to send it. */
			if iscr.OutputToken != nil {
				if !quiet {
					fmt.Printf("Sending init_sec_context token (size=%d)...", len(*iscr.OutputToken))
				}
				if v1 {
					tag = 0
				} else {
					tag = misc.TOKEN_CONTEXT
				}
				misc.SendToken(conn, tag, *iscr.OutputToken)
			}
			if major == proxy.S_CONTINUE_NEEDED {
				/* CONTINUE_NEEDED means we expect a token from the far end to be fed back in to InitSecContext(). */
				var token []byte
				if !quiet {
					fmt.Printf("continue needed...")
				}
				tag, token = misc.RecvToken(conn)
				if !quiet {
					fmt.Printf("\n")
				}
				if len(token) == 0 {
					if !quiet {
						fmt.Printf("server closed connection.\n")
					}
					break
				}
				ptoken = &token
			} else {
				/* COMPLETE means we're done, everything succeeded. */
				if !quiet {
					fmt.Printf("\n")
				}
				break
			}
		}
		if major != proxy.S_COMPLETE {
			fmt.Printf("Error authenticating to server: %08x/%08x.\n", major, minor)
			return
		}
		if !quiet {
			DisplayProxyFlags(flags, false, os.Stdout)
		}

		/* Describe the context. */
		if ctx.LocallyInitiated {
			localstate = "locally initiated"
		} else {
			localstate = "remotely initiated"
		}
		if ctx.Open {
			openstate = "open"
		} else {
			openstate = "closed"
		}
		if !quiet {
			fmt.Printf("\"%s\" to \"%s\", lifetime %d, flags %x, %s, %s\n", ctx.SrcName.DisplayName, ctx.TargName.DisplayName, ctx.Lifetime, proxy.FlagsToRaw(ctx.Flags), localstate, openstate)
		}
		if !quiet {
			fmt.Printf("Name type of source name is %s.\n", ctx.SrcName.NameType.String())
		}

		imr, err := proxy.IndicateMechs(pconn, pcc)
		if err != nil {
			fmt.Printf("Error indicating mechanisms: %s\n", err)
			return
		}
		status = imr.Status
		major = status.MajorStatus
		if major != proxy.S_COMPLETE && major != proxy.S_CONTINUE_NEEDED {
			DisplayProxyStatus("indicating mechanisms", imr.Status)
			return
		}

		for _, mech := range imr.Mechs {
			if !mech.Mech.Equal(ctx.Mech) {
				continue
			}
			if !quiet {
				fmt.Printf("Mechanism %s supports %d names\n", mech.Mech, len(mech.NameTypes))
			}
			for i, nametype := range mech.NameTypes {
				if !quiet {
					fmt.Printf("%3d: %s\n", i, nametype.String())
				}
			}
		}
	}

	for i := 0; i < mcount; i++ {
		var wrapped []byte
		var major uint64

		if nowrap {
			wrapped = plain
		} else {
			plains := make([][]byte, 1)
			plains[0] = plain
			wr, err := proxy.Wrap(pconn, pcc, &ctx, !noenc, plains, proxy.C_QOP_DEFAULT)
			if err != nil {
				fmt.Printf("Error wrapping message: %s\n", err)
				return
			}
			status = wr.Status
			major = status.MajorStatus
			if major != proxy.S_COMPLETE {
				DisplayProxyStatus("wrapping data", status)
				return
			}
			if !noenc && !wr.ConfState && !quiet {
				fmt.Printf("Warning!  Message not encrypted.\n")
			}
			wrapped = wr.TokenBuffer[0]
		}

		tag = misc.TOKEN_DATA
		if !nowrap {
			tag |= misc.TOKEN_WRAPPED
		}
		if !noenc {
			tag |= misc.TOKEN_ENCRYPTED
		}
		if !nomic {
			tag |= misc.TOKEN_SEND_MIC
		}
		if v1 {
			tag = 0
		}

		misc.SendToken(conn, tag, wrapped)
		tag, mictoken := misc.RecvToken(conn)
		if tag == 0 && len(mictoken) == 0 {
			if !quiet {
				fmt.Printf("Server closed connection unexpectedly.\n")
			}
			return
		}
		if nomic {
			if bytes.Equal(plain, mictoken) {
				if !quiet {
					fmt.Printf("Response differed.\n")
				}
				return
			}
			if !quiet {
				fmt.Printf("Response received.\n")
			}
		} else {
			vr, err := proxy.VerifyMic(pconn, pcc, &ctx, plain, mictoken)
			if err != nil {
				fmt.Printf("Error verifying mic: %s\n", err)
				return
			}
			status = vr.Status
			major = status.MajorStatus
			if major != proxy.S_COMPLETE {
				DisplayProxyStatus("verifying signature", status)
				return
			}
			if !quiet {
				fmt.Printf("Signature verified.\n")
			}
		}
	}
	if !v1 {
		misc.SendToken(conn, misc.TOKEN_NOOP, nil)
	}
}

func main() {
	port := flag.Int("port", 4444, "port")
	mechstr := flag.String("mech", "", "mechanism")
	spnego := flag.Bool("spnego", false, "use SPNEGO")
	iakerb := flag.Bool("iakerb", false, "use IAKERB")
	krb5 := flag.Bool("krb5", false, "use Kerberos 5")
	delegate := flag.Bool("d", false, "delegate")
	seq := flag.Bool("seq", false, "use sequence number checking")
	noreplay := flag.Bool("noreplay", false, "disable replay checking")
	nomutual := flag.Bool("nomutual", false, "perform one-way authentication")
	file := flag.Bool("f", false, "read message from file")
	v1 := flag.Bool("v1", false, "use version 1 protocol")
	quiet := flag.Bool("q", false, "quiet")
	ccount := flag.Int("ccount", 1, "connection count")
	mcount := flag.Int("mcount", 1, "message count")
	noauth := flag.Bool("na", false, "no authentication")
	nowrap := flag.Bool("nw", false, "no wrapping")
	noenc := flag.Bool("nx", false, "no encryption")
	nomic := flag.Bool("nm", false, "no MICs")
	var plain []byte
	var nmech *asn1.ObjectIdentifier
	var mech asn1.ObjectIdentifier
	var call proxy.CallCtx

	flag.Parse()
	sockaddr := flag.Arg(0)
	host := flag.Arg(1)
	service := flag.Arg(2)
	msg := flag.Arg(3)
	if flag.NArg() < 4 {
		fmt.Printf("Usage: proxy-client [options] socket host gss-service-name message-or-file\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if *file {
		msgfile, err := os.Open(msg)
		if err != nil {
			fmt.Printf("Error opening \"%s\": %s", msg, err)
			return
		}
		fi, err := msgfile.Stat()
		if err != nil {
			fmt.Printf("Error statting \"%s\": %s", msg, err)
			return
		}
		plain = make([]byte, fi.Size())
		n, err := msgfile.Read(plain)
		if int64(n) != fi.Size() {
			fmt.Printf("Error reading \"%s\": %s", msg, err)
			return
		}
	} else {
		buffer := bytes.NewBufferString(msg)
		plain = buffer.Bytes()
	}
	if *spnego {
		/* If we're doing SPNEGO, then a passed-in mechanism OID is the one we want to negotiate, but we can't. */
		fmt.Printf("Warning: set_neg_mechs is not available.\n")
		tmpmech := misc.ParseOid("1.3.6.1.5.5.2")
		mech = tmpmech
	} else if *krb5 {
		/* This is the OID from the RFC.  The native tests would use the pre-RFC OID. */
		tmpmech := misc.ParseOid("1.2.840.113554.1.2.2")
		nmech = &tmpmech
		mech = tmpmech
	} else if *iakerb {
		tmpmech := misc.ParseOid("1.3.6.1.5.2.5")
		nmech = &tmpmech
		mech = tmpmech
	} else if len(*mechstr) > 0 {
		tmpmech := misc.ParseOid(*mechstr)
		nmech = &tmpmech
		mech = tmpmech
	} else {
		tmpmech := misc.ParseOid("1.2.840.113554.1.2.2")
		nmech = &tmpmech
		mech = tmpmech
	}
	if *noauth {
		*nowrap = true
		*noenc = true
		*nomic = true
	}

	pconn, err := net.Dial("unix", sockaddr)
	if err != nil {
		fmt.Printf("Error connecting to gss-proxy at \"%s\": %s", sockaddr, err)
		return
	}

	gccr, err := proxy.GetCallContext(&pconn, &call, nil)
	if err != nil {
		fmt.Printf("Error getting a calling context: %s", err)
		return
	}
	if gccr.Status.MajorStatus != proxy.S_COMPLETE {
		DisplayProxyStatus("getting calling context", gccr.Status)
		return
	}

	for c := 0; c < *ccount; c++ {
		connectOnce(&pconn, &call, host, *port, service, *mcount, *quiet, plain, *v1, nmech, mech, *delegate, *seq, *noreplay, *nomutual, *noauth, *nowrap, *noenc, *nomic)
	}
}
<commit_msg>Revert "Default to the krb5 mechanism"<commit_after>package main

import "bytes"
import "flag"
import "encoding/asn1"
import "fmt"
import "gss/proxy"
import "gss/misc"
import "net"
import "os"
import "strings"

func connectOnce(pconn *net.Conn, pcc *proxy.CallCtx, host string, port int, service string, mcount int, quiet bool, plain []byte, v1 bool, nmech *asn1.ObjectIdentifier, mech asn1.ObjectIdentifier, delegate, seq, noreplay, nomutual, noauth, nowrap, noenc, nomic bool) {
	var ctx proxy.SecCtx
	var status proxy.Status
	var tag byte
	var ptoken *[]byte
	var major, minor uint64
	var sname proxy.Name
	var localstate, openstate string
	var flags proxy.Flags

	/* Open the connection. */
	conn, err := net.Dial("tcp", fmt.Sprintf("%s:%d", host, port))
	if err != nil {
		fmt.Printf("Error connecting: %s\n", err)
		os.Exit(2)
	}
	defer conn.Close()

	/* Import the remote service's name. */
	if strings.Contains(service, "@") {
		sname.DisplayName = service
	} else {
		sname.DisplayName = service + "@" + host
	}
	sname.NameType = proxy.NT_HOSTBASED_SERVICE
	if nmech != nil {
		icnr, err := proxy.ImportAndCanonName(pconn, pcc, sname, *nmech, nil, nil)
		if err != nil {
			fmt.Printf("Error importing remote service name: %s\n", err)
			return
		}
		if icnr.Status.MajorStatus != 0 {
			DisplayProxyStatus("importing remote service name", icnr.Status)
			return
		}
		sname = *icnr.Name
	}

	if noauth {
		misc.SendToken(conn, misc.TOKEN_NOOP, nil)
	} else {
		if !v1 {
			misc.SendToken(conn, misc.TOKEN_NOOP|misc.TOKEN_CONTEXT_NEXT, nil)
		}
		flags = proxy.Flags{Deleg: delegate, Sequence: seq, Replay: !noreplay, Conf: !noenc, Integ: !nomic, Mutual: !nomutual}
		for true {
			/* Start/continue. */
			iscr, err := proxy.InitSecContext(pconn, pcc, &ctx, nil, &sname, mech, flags, proxy.C_INDEFINITE, nil, ptoken, nil)
			if err != nil {
				fmt.Printf("Error initializing security context: %s\n", err)
				return
			}
			status = iscr.Status
			major = status.MajorStatus
			if major != proxy.S_COMPLETE && major != proxy.S_CONTINUE_NEEDED {
				DisplayProxyStatus("initializing security context", iscr.Status)
				return
			}
			/* If we have an output token, we need to send it. */
			if iscr.OutputToken != nil {
				if !quiet {
					fmt.Printf("Sending init_sec_context token (size=%d)...", len(*iscr.OutputToken))
				}
				if v1 {
					tag = 0
				} else {
					tag = misc.TOKEN_CONTEXT
				}
				misc.SendToken(conn, tag, *iscr.OutputToken)
			}
			if major == proxy.S_CONTINUE_NEEDED {
				/* CONTINUE_NEEDED means we expect a token from the far end to be fed back in to InitSecContext(). */
				var token []byte
				if !quiet {
					fmt.Printf("continue needed...")
				}
				tag, token = misc.RecvToken(conn)
				if !quiet {
					fmt.Printf("\n")
				}
				if len(token) == 0 {
					if !quiet {
						fmt.Printf("server closed connection.\n")
					}
					break
				}
				ptoken = &token
			} else {
				/* COMPLETE means we're done, everything succeeded. */
				if !quiet {
					fmt.Printf("\n")
				}
				break
			}
		}
		if major != proxy.S_COMPLETE {
			fmt.Printf("Error authenticating to server: %08x/%08x.\n", major, minor)
			return
		}
		if !quiet {
			DisplayProxyFlags(flags, false, os.Stdout)
		}

		/* Describe the context. */
		if ctx.LocallyInitiated {
			localstate = "locally initiated"
		} else {
			localstate = "remotely initiated"
		}
		if ctx.Open {
			openstate = "open"
		} else {
			openstate = "closed"
		}
		if !quiet {
			fmt.Printf("\"%s\" to \"%s\", lifetime %d, flags %x, %s, %s\n", ctx.SrcName.DisplayName, ctx.TargName.DisplayName, ctx.Lifetime, proxy.FlagsToRaw(ctx.Flags), localstate, openstate)
		}
		if !quiet {
			fmt.Printf("Name type of source name is %s.\n", ctx.SrcName.NameType.String())
		}

		imr, err := proxy.IndicateMechs(pconn, pcc)
		if err != nil {
			fmt.Printf("Error indicating mechanisms: %s\n", err)
			return
		}
		status = imr.Status
		major = status.MajorStatus
		if major != proxy.S_COMPLETE && major != proxy.S_CONTINUE_NEEDED {
			DisplayProxyStatus("indicating mechanisms", imr.Status)
			return
		}

		for _, mech := range imr.Mechs {
			if !mech.Mech.Equal(ctx.Mech) {
				continue
			}
			if !quiet {
				fmt.Printf("Mechanism %s supports %d names\n", mech.Mech, len(mech.NameTypes))
			}
			for i, nametype := range mech.NameTypes {
				if !quiet {
					fmt.Printf("%3d: %s\n", i, nametype.String())
				}
			}
		}
	}

	for i := 0; i < mcount; i++ {
		var wrapped []byte
		var major uint64

		if nowrap {
			wrapped = plain
		} else {
			plains := make([][]byte, 1)
			plains[0] = plain
			wr, err := proxy.Wrap(pconn, pcc, &ctx, !noenc, plains, proxy.C_QOP_DEFAULT)
			if err != nil {
				fmt.Printf("Error wrapping message: %s\n", err)
				return
			}
			status = wr.Status
			major = status.MajorStatus
			if major != proxy.S_COMPLETE {
				DisplayProxyStatus("wrapping data", status)
				return
			}
			if !noenc && !wr.ConfState && !quiet {
				fmt.Printf("Warning!  Message not encrypted.\n")
			}
			wrapped = wr.TokenBuffer[0]
		}

		tag = misc.TOKEN_DATA
		if !nowrap {
			tag |= misc.TOKEN_WRAPPED
		}
		if !noenc {
			tag |= misc.TOKEN_ENCRYPTED
		}
		if !nomic {
			tag |= misc.TOKEN_SEND_MIC
		}
		if v1 {
			tag = 0
		}

		misc.SendToken(conn, tag, wrapped)
		tag, mictoken := misc.RecvToken(conn)
		if tag == 0 && len(mictoken) == 0 {
			if !quiet {
				fmt.Printf("Server closed connection unexpectedly.\n")
			}
			return
		}
		if nomic {
			if bytes.Equal(plain, mictoken) {
				if !quiet {
					fmt.Printf("Response differed.\n")
				}
				return
			}
			if !quiet {
				fmt.Printf("Response received.\n")
			}
		} else {
			vr, err := proxy.VerifyMic(pconn, pcc, &ctx, plain, mictoken)
			if err != nil {
				fmt.Printf("Error verifying mic: %s\n", err)
				return
			}
			status = vr.Status
			major = status.MajorStatus
			if major != proxy.S_COMPLETE {
				DisplayProxyStatus("verifying signature", status)
				return
			}
			if !quiet {
				fmt.Printf("Signature verified.\n")
			}
		}
	}
	if !v1 {
		misc.SendToken(conn, misc.TOKEN_NOOP, nil)
	}
}

func main() {
	port := flag.Int("port", 4444, "port")
	mechstr := flag.String("mech", "", "mechanism")
	spnego := flag.Bool("spnego", false, "use SPNEGO")
	iakerb := flag.Bool("iakerb", false, "use IAKERB")
	krb5 := flag.Bool("krb5", false, "use Kerberos 5")
	delegate := flag.Bool("d", false, "delegate")
	seq := flag.Bool("seq", false, "use sequence number checking")
	noreplay := flag.Bool("noreplay", false, "disable replay checking")
	nomutual := flag.Bool("nomutual", false, "perform one-way authentication")
	file := flag.Bool("f", false, "read message from file")
	v1 := flag.Bool("v1", false, "use version 1 protocol")
	quiet := flag.Bool("q", false, "quiet")
	ccount := flag.Int("ccount", 1, "connection count")
	mcount := flag.Int("mcount", 1, "message count")
	noauth := flag.Bool("na", false, "no authentication")
	nowrap := flag.Bool("nw", false, "no wrapping")
	noenc := flag.Bool("nx", false, "no encryption")
	nomic := flag.Bool("nm", false, "no MICs")
	var plain []byte
	var nmech *asn1.ObjectIdentifier
	var mech asn1.ObjectIdentifier
	var call proxy.CallCtx

	flag.Parse()
	sockaddr := flag.Arg(0)
	host := flag.Arg(1)
	service := flag.Arg(2)
	msg := flag.Arg(3)
	if flag.NArg() < 4 {
		fmt.Printf("Usage: proxy-client [options] socket host gss-service-name message-or-file\n")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if *file {
		msgfile, err := os.Open(msg)
		if err != nil {
			fmt.Printf("Error opening \"%s\": %s", msg, err)
			return
		}
		fi, err := msgfile.Stat()
		if err != nil {
			fmt.Printf("Error statting \"%s\": %s", msg, err)
			return
		}
		plain = make([]byte, fi.Size())
		n, err := msgfile.Read(plain)
		if int64(n) != fi.Size() {
			fmt.Printf("Error reading \"%s\": %s", msg, err)
			return
		}
	} else {
		buffer := bytes.NewBufferString(msg)
		plain = buffer.Bytes()
	}
	if *spnego {
		/* If we're doing SPNEGO, then a passed-in mechanism OID is the one we want to negotiate, but we can't. */
		fmt.Printf("Warning: set_neg_mechs is not available.\n")
		tmpmech := misc.ParseOid("1.3.6.1.5.5.2")
		mech = tmpmech
	} else if *krb5 {
		/* This is the OID from the RFC.  The native tests would use the pre-RFC OID. */
		tmpmech := misc.ParseOid("1.2.840.113554.1.2.2")
		nmech = &tmpmech
		mech = tmpmech
	} else if *iakerb {
		tmpmech := misc.ParseOid("1.3.6.1.5.2.5")
		nmech = &tmpmech
		mech = tmpmech
	} else if len(*mechstr) > 0 {
		tmpmech := misc.ParseOid(*mechstr)
		nmech = &tmpmech
		mech = tmpmech
	}
	if *noauth {
		*nowrap = true
		*noenc = true
		*nomic = true
	}

	pconn, err := net.Dial("unix", sockaddr)
	if err != nil {
		fmt.Printf("Error connecting to gss-proxy at \"%s\": %s", sockaddr, err)
		return
	}

	gccr, err := proxy.GetCallContext(&pconn, &call, nil)
	if err != nil {
		fmt.Printf("Error getting a calling context: %s", err)
		return
	}
	if gccr.Status.MajorStatus != proxy.S_COMPLETE {
		DisplayProxyStatus("getting calling context", gccr.Status)
		return
	}

	for c := 0; c < *ccount; c++ {
		connectOnce(&pconn, &call, host, *port, service, *mcount, *quiet, plain, *v1, nmech, mech, *delegate, *seq, *noreplay, *nomutual, *noauth, *nowrap, *noenc, *nomic)
	}
}
<|endoftext|>


<commit_before>package nomad

import (
	"fmt"
	"time"

	"github.com/armon/go-metrics"
	"github.com/hashicorp/nomad/nomad/state"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/raft"
)

// planApply is a long lived goroutine that reads plan allocations from
// the plan queue, determines if they can be applied safely and applies
// them via Raft.
//
// Naively, we could simply dequeue a plan, verify, apply and then respond.
// However, the plan application is bounded by the Raft apply time and
// subject to some latency. This creates a stall condition, where we are
// not evaluating, but simply waiting for a transaction to complete.
//
// To avoid this, we overlap verification with apply. This means once
// we've verified plan N we attempt to apply it. However, while waiting
// for apply, we begin to verify plan N+1 under the assumption that plan
// N has succeeded.
//
// In this sense, we track two parallel versions of the world. One is
// the pessimistic one driven by the Raft log which is replicated. The
// other is optimistic and assumes our transactions will succeed. In the
// happy path, this lets us do productive work during the latency of
// apply.
//
// In the unhappy path (Raft transaction fails), effectively we only
// wasted work during a time we would have been waiting anyways. However,
// in anticipation of this case we cannot respond to the plan until
// the Raft log is updated. This means our schedulers will stall,
// but there are many of those and only a single plan verifier.
//
func (s *Server) planApply() {
	// waitCh is used to track an outstanding application
	// while snap holds an optimistic state which includes
	// that plan application.
	var waitCh chan struct{}
	var snap *state.StateSnapshot

	for {
		// Pull the next pending plan, exit if we are no longer leader
		pending, err := s.planQueue.Dequeue(0)
		if err != nil {
			return
		}

		// Verify the evaluation is outstanding, and that the tokens match.
		token, ok := s.evalBroker.Outstanding(pending.plan.EvalID)
		if !ok {
			s.logger.Printf("[ERR] nomad: plan received for non-outstanding evaluation %s",
				pending.plan.EvalID)
			pending.respond(nil, fmt.Errorf("evaluation is not outstanding"))
			continue
		}
		if pending.plan.EvalToken != token {
			s.logger.Printf("[ERR] nomad: plan received for evaluation %s with wrong token",
				pending.plan.EvalID)
			pending.respond(nil, fmt.Errorf("evaluation token does not match"))
			continue
		}

		// Check if out last plan has completed
		select {
		case <-waitCh:
			waitCh = nil
			snap = nil
		default:
		}

		// Snapshot the state so that we have a consistent view of the world
		// if no snapshot is available
		if snap == nil {
			snap, err = s.fsm.State().Snapshot()
			if err != nil {
				s.logger.Printf("[ERR] nomad: failed to snapshot state: %v", err)
				pending.respond(nil, err)
				continue
			}
		}

		// Evaluate the plan
		result, err := evaluatePlan(snap, pending.plan)
		if err != nil {
			s.logger.Printf("[ERR] nomad: failed to evaluate plan: %v", err)
			pending.respond(nil, err)
			continue
		}

		// Fast-path the response if there is nothing to do
		if result.IsNoOp() {
			pending.respond(result, nil)
			continue
		}

		// Ensure any parallel apply is complete before
		// starting the next one. This also limits how out
		// of date our snapshot can be.
		if waitCh != nil {
			<-waitCh
			snap, err = s.fsm.State().Snapshot()
			if err != nil {
				s.logger.Printf("[ERR] nomad: failed to snapshot state: %v", err)
				pending.respond(nil, err)
				continue
			}
		}

		// Dispatch the Raft transaction for the plan
		future, err := s.applyPlan(result, snap)
		if err != nil {
			s.logger.Printf("[ERR] nomad: failed to submit plan: %v", err)
			pending.respond(nil, err)
			continue
		}

		// Respond to the plan in async
		waitCh = make(chan struct{})
		go s.asyncPlanWait(waitCh, future, result, pending)
	}
}

// applyPlan is used to apply the plan result and to return the alloc index
func (s *Server) applyPlan(result *structs.PlanResult, snap *state.StateSnapshot) (raft.ApplyFuture, error) {
	req := structs.AllocUpdateRequest{}
	for _, updateList := range result.NodeUpdate {
		req.Alloc = append(req.Alloc, updateList...)
	}
	for _, allocList := range result.NodeAllocation {
		req.Alloc = append(req.Alloc, allocList...)
	}
	req.Alloc = append(req.Alloc, result.FailedAllocs...)

	// Dispatch the Raft transaction
	future, err := s.raftApplyFuture(structs.AllocUpdateRequestType, &req)
	if err != nil {
		return nil, err
	}

	// Optimistically apply to our state view
	if snap != nil {
		nextIdx := s.raft.AppliedIndex() + 1
		if err := snap.UpsertAllocs(nextIdx, req.Alloc); err != nil {
			return future, err
		}
	}
	return future, nil
}

// asyncPlanWait is used to apply and respond to a plan async
func (s *Server) asyncPlanWait(waitCh chan struct{}, future raft.ApplyFuture,
	result *structs.PlanResult, pending *pendingPlan) {
	defer metrics.MeasureSince([]string{"nomad", "plan", "apply"}, time.Now())
	defer close(waitCh)

	// Wait for the plan to apply
	if err := future.Error(); err != nil {
		s.logger.Printf("[ERR] nomad: failed to apply plan: %v", err)
		pending.respond(nil, err)
		return
	}

	// Respond to the plan
	result.AllocIndex = future.Index()
	pending.respond(result, nil)
}

// evaluatePlan is used to determine what portions of a plan
// can be applied if any. Returns if there should be a plan application
// which may be partial or if there was an error
func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanResult, error) {
	defer metrics.MeasureSince([]string{"nomad", "plan", "evaluate"}, time.Now())

	// Create a result holder for the plan
	result := &structs.PlanResult{
		NodeUpdate:     make(map[string][]*structs.Allocation),
		NodeAllocation: make(map[string][]*structs.Allocation),
		FailedAllocs:   plan.FailedAllocs,
	}

	// Collect all the nodeIDs
	nodeIDs := make(map[string]struct{})
	for nodeID := range plan.NodeUpdate {
		nodeIDs[nodeID] = struct{}{}
	}
	for nodeID := range plan.NodeAllocation {
		nodeIDs[nodeID] = struct{}{}
	}

	// Check each allocation to see if it should be allowed
	for nodeID := range nodeIDs {
		// Evaluate the plan for this node
		fit, err := evaluateNodePlan(snap, plan, nodeID)
		if err != nil {
			return nil, err
		}
		if !fit {
			// Scheduler must have stale data, RefreshIndex should force
			// the latest view of allocations and nodes
			allocIndex, err := snap.Index("allocs")
			if err != nil {
				return nil, err
			}
			nodeIndex, err := snap.Index("nodes")
			if err != nil {
				return nil, err
			}
			result.RefreshIndex = maxUint64(nodeIndex, allocIndex)

			// If we require all-at-once scheduling, there is no point
			// to continue the evaluation, as we've already failed.
			if plan.AllAtOnce {
				result.NodeUpdate = nil
				result.NodeAllocation = nil
				return result, nil
			}

			// Skip this node, since it cannot be used.
			continue
		}

		// Add this to the plan result
		if nodeUpdate := plan.NodeUpdate[nodeID]; len(nodeUpdate) > 0 {
			result.NodeUpdate[nodeID] = nodeUpdate
		}
		if nodeAlloc := plan.NodeAllocation[nodeID]; len(nodeAlloc) > 0 {
			result.NodeAllocation[nodeID] = nodeAlloc
		}
	}
	return result, nil
}

// evaluateNodePlan is used to evalute the plan for a single node,
// returning if the plan is valid or if an error is encountered
func evaluateNodePlan(snap *state.StateSnapshot, plan *structs.Plan, nodeID string) (bool, error) {
	// If this is an evict-only plan, it always 'fits' since we are removing things.
	if len(plan.NodeAllocation[nodeID]) == 0 {
		return true, nil
	}

	// Get the node itself
	node, err := snap.NodeByID(nodeID)
	if err != nil {
		return false, fmt.Errorf("failed to get node '%s': %v", nodeID, err)
	}

	// If the node does not exist or is not ready for schduling it is not fit
	// XXX: There is a potential race between when we do this check and when
	// the Raft commit happens.
	if node == nil || node.Status != structs.NodeStatusReady || node.Drain {
		return false, nil
	}

	// Get the existing allocations
	existingAlloc, err := snap.AllocsByNode(nodeID)
	if err != nil {
		return false, fmt.Errorf("failed to get existing allocations for '%s': %v", nodeID, err)
	}

	// Filter on alloc state
	existingAlloc = structs.FilterTerminalAllocs(existingAlloc)

	// Determine the proposed allocation by first removing allocations
	// that are planned evictions and adding the new allocations.
	proposed := existingAlloc
	var remove []*structs.Allocation
	if update := plan.NodeUpdate[nodeID]; len(update) > 0 {
		remove = append(remove, update...)
	}
	if updated := plan.NodeAllocation[nodeID]; len(updated) > 0 {
		for _, alloc := range updated {
			remove = append(remove, alloc)
		}
	}
	proposed = structs.RemoveAllocs(existingAlloc, remove)
	proposed = append(proposed, plan.NodeAllocation[nodeID]...)

	// Check if these allocations fit
	fit, _, _, err := structs.AllocsFit(node, proposed, nil)
	return fit, err
}
<commit_msg>nomad: refresh snapshot under error return<commit_after>package nomad

import (
	"fmt"
	"time"

	"github.com/armon/go-metrics"
	"github.com/hashicorp/nomad/nomad/state"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/raft"
)

// planApply is a long lived goroutine that reads plan allocations from
// the plan queue, determines if they can be applied safely and applies
// them via Raft.
//
// Naively, we could simply dequeue a plan, verify, apply and then respond.
// However, the plan application is bounded by the Raft apply time and
// subject to some latency. This creates a stall condition, where we are
// not evaluating, but simply waiting for a transaction to complete.
//
// To avoid this, we overlap verification with apply. This means once
// we've verified plan N we attempt to apply it. However, while waiting
// for apply, we begin to verify plan N+1 under the assumption that plan
// N has succeeded.
//
// In this sense, we track two parallel versions of the world. One is
// the pessimistic one driven by the Raft log which is replicated. The
// other is optimistic and assumes our transactions will succeed. In the
// happy path, this lets us do productive work during the latency of
// apply.
//
// In the unhappy path (Raft transaction fails), effectively we only
// wasted work during a time we would have been waiting anyways. However,
// in anticipation of this case we cannot respond to the plan until
// the Raft log is updated. This means our schedulers will stall,
// but there are many of those and only a single plan verifier.
//
func (s *Server) planApply() {
	// waitCh is used to track an outstanding application
	// while snap holds an optimistic state which includes
	// that plan application.
	var waitCh chan struct{}
	var snap *state.StateSnapshot

	for {
		// Pull the next pending plan, exit if we are no longer leader
		pending, err := s.planQueue.Dequeue(0)
		if err != nil {
			return
		}

		// Verify the evaluation is outstanding, and that the tokens match.
		token, ok := s.evalBroker.Outstanding(pending.plan.EvalID)
		if !ok {
			s.logger.Printf("[ERR] nomad: plan received for non-outstanding evaluation %s",
				pending.plan.EvalID)
			pending.respond(nil, fmt.Errorf("evaluation is not outstanding"))
			continue
		}
		if pending.plan.EvalToken != token {
			s.logger.Printf("[ERR] nomad: plan received for evaluation %s with wrong token",
				pending.plan.EvalID)
			pending.respond(nil, fmt.Errorf("evaluation token does not match"))
			continue
		}

		// Check if out last plan has completed
		select {
		case <-waitCh:
			waitCh = nil
			snap = nil
		default:
		}

		// Snapshot the state so that we have a consistent view of the world
		// if no snapshot is available
		if waitCh == nil || snap == nil {
			snap, err = s.fsm.State().Snapshot()
			if err != nil {
				s.logger.Printf("[ERR] nomad: failed to snapshot state: %v", err)
				pending.respond(nil, err)
				continue
			}
		}

		// Evaluate the plan
		result, err := evaluatePlan(snap, pending.plan)
		if err != nil {
			s.logger.Printf("[ERR] nomad: failed to evaluate plan: %v", err)
			pending.respond(nil, err)
			continue
		}

		// Fast-path the response if there is nothing to do
		if result.IsNoOp() {
			pending.respond(result, nil)
			continue
		}

		// Ensure any parallel apply is complete before
		// starting the next one. This also limits how out
		// of date our snapshot can be.
		if waitCh != nil {
			<-waitCh
			snap, err = s.fsm.State().Snapshot()
			if err != nil {
				s.logger.Printf("[ERR] nomad: failed to snapshot state: %v", err)
				pending.respond(nil, err)
				continue
			}
		}

		// Dispatch the Raft transaction for the plan
		future, err := s.applyPlan(result, snap)
		if err != nil {
			s.logger.Printf("[ERR] nomad: failed to submit plan: %v", err)
			pending.respond(nil, err)
			continue
		}

		// Respond to the plan in async
		waitCh = make(chan struct{})
		go s.asyncPlanWait(waitCh, future, result, pending)
	}
}

// applyPlan is used to apply the plan result and to return the alloc index
func (s *Server) applyPlan(result *structs.PlanResult, snap *state.StateSnapshot) (raft.ApplyFuture, error) {
	req := structs.AllocUpdateRequest{}
	for _, updateList := range result.NodeUpdate {
		req.Alloc = append(req.Alloc, updateList...)
	}
	for _, allocList := range result.NodeAllocation {
		req.Alloc = append(req.Alloc, allocList...)
	}
	req.Alloc = append(req.Alloc, result.FailedAllocs...)

	// Dispatch the Raft transaction
	future, err := s.raftApplyFuture(structs.AllocUpdateRequestType, &req)
	if err != nil {
		return nil, err
	}

	// Optimistically apply to our state view
	if snap != nil {
		nextIdx := s.raft.AppliedIndex() + 1
		if err := snap.UpsertAllocs(nextIdx, req.Alloc); err != nil {
			return future, err
		}
	}
	return future, nil
}

// asyncPlanWait is used to apply and respond to a plan async
func (s *Server) asyncPlanWait(waitCh chan struct{}, future raft.ApplyFuture,
	result *structs.PlanResult, pending *pendingPlan) {
	defer metrics.MeasureSince([]string{"nomad", "plan", "apply"}, time.Now())
	defer close(waitCh)

	// Wait for the plan to apply
	if err := future.Error(); err != nil {
		s.logger.Printf("[ERR] nomad: failed to apply plan: %v", err)
		pending.respond(nil, err)
		return
	}

	// Respond to the plan
	result.AllocIndex = future.Index()
	pending.respond(result, nil)
}

// evaluatePlan is used to determine what portions of a plan
// can be applied if any. Returns if there should be a plan application
// which may be partial or if there was an error
func evaluatePlan(snap *state.StateSnapshot, plan *structs.Plan) (*structs.PlanResult, error) {
	defer metrics.MeasureSince([]string{"nomad", "plan", "evaluate"}, time.Now())

	// Create a result holder for the plan
	result := &structs.PlanResult{
		NodeUpdate:     make(map[string][]*structs.Allocation),
		NodeAllocation: make(map[string][]*structs.Allocation),
		FailedAllocs:   plan.FailedAllocs,
	}

	// Collect all the nodeIDs
	nodeIDs := make(map[string]struct{})
	for nodeID := range plan.NodeUpdate {
		nodeIDs[nodeID] = struct{}{}
	}
	for nodeID := range plan.NodeAllocation {
		nodeIDs[nodeID] = struct{}{}
	}

	// Check each allocation to see if it should be allowed
	for nodeID := range nodeIDs {
		// Evaluate the plan for this node
		fit, err := evaluateNodePlan(snap, plan, nodeID)
		if err != nil {
			return nil, err
		}
		if !fit {
			// Scheduler must have stale data, RefreshIndex should force
			// the latest view of allocations and nodes
			allocIndex, err := snap.Index("allocs")
			if err != nil {
				return nil, err
			}
			nodeIndex, err := snap.Index("nodes")
			if err != nil {
				return nil, err
			}
			result.RefreshIndex = maxUint64(nodeIndex, allocIndex)

			// If we require all-at-once scheduling, there is no point
			// to continue the evaluation, as we've already failed.
			if plan.AllAtOnce {
				result.NodeUpdate = nil
				result.NodeAllocation = nil
				return result, nil
			}

			// Skip this node, since it cannot be used.
			continue
		}

		// Add this to the plan result
		if nodeUpdate := plan.NodeUpdate[nodeID]; len(nodeUpdate) > 0 {
			result.NodeUpdate[nodeID] = nodeUpdate
		}
		if nodeAlloc := plan.NodeAllocation[nodeID]; len(nodeAlloc) > 0 {
			result.NodeAllocation[nodeID] = nodeAlloc
		}
	}
	return result, nil
}

// evaluateNodePlan is used to evalute the plan for a single node,
// returning if the plan is valid or if an error is encountered
func evaluateNodePlan(snap *state.StateSnapshot, plan *structs.Plan, nodeID string) (bool, error) {
	// If this is an evict-only plan, it always 'fits' since we are removing things.
	if len(plan.NodeAllocation[nodeID]) == 0 {
		return true, nil
	}

	// Get the node itself
	node, err := snap.NodeByID(nodeID)
	if err != nil {
		return false, fmt.Errorf("failed to get node '%s': %v", nodeID, err)
	}

	// If the node does not exist or is not ready for schduling it is not fit
	// XXX: There is a potential race between when we do this check and when
	// the Raft commit happens.
	if node == nil || node.Status != structs.NodeStatusReady || node.Drain {
		return false, nil
	}

	// Get the existing allocations
	existingAlloc, err := snap.AllocsByNode(nodeID)
	if err != nil {
		return false, fmt.Errorf("failed to get existing allocations for '%s': %v", nodeID, err)
	}

	// Filter on alloc state
	existingAlloc = structs.FilterTerminalAllocs(existingAlloc)

	// Determine the proposed allocation by first removing allocations
	// that are planned evictions and adding the new allocations.
	proposed := existingAlloc
	var remove []*structs.Allocation
	if update := plan.NodeUpdate[nodeID]; len(update) > 0 {
		remove = append(remove, update...)
	}
	if updated := plan.NodeAllocation[nodeID]; len(updated) > 0 {
		for _, alloc := range updated {
			remove = append(remove, alloc)
		}
	}
	proposed = structs.RemoveAllocs(existingAlloc, remove)
	proposed = append(proposed, plan.NodeAllocation[nodeID]...)

	// Check if these allocations fit
	fit, _, _, err := structs.AllocsFit(node, proposed, nil)
	return fit, err
}
<|endoftext|>


<commit_before>package data

import "math"

// Resample resamples the raw data. It either streches the data to fit the witdth
// or it uses the Largest Triangle Three Bucket algorithm to fit the data to the new width.
func (d *Data) Resample(width int) {
	if len(d.raw) < width {
		d.raw = d.stretch(width)
	} else if len(d.raw) > width {
		d.raw = d.lttb(width)
	}
}

// stretch stretches the raw array into a new width.
// It just stretches using the same values without any interpolation.
func (d *Data) stretch(width int) []float64 {
	newdata := make([]float64, width)
	max := len(d.raw)
	for i := 0; i < width; i++ {
		idx := float64(max) / float64(width) * float64(i)
		v := d.raw[int(idx)]
		f := idx - float64(int(idx))
		if f < d.gap || f > 1-d.gap {
			v = 0
		}
		newdata[i] = v
	}
	return newdata
}

// lttb implements Largest Triangle Three Bucket downsampling algorithm.
// Converted to Go from several implementations found online.
func (d *Data) lttb(width int) []float64 {
	L := len(d.raw)
	res := make([]float64, width)

	every := float64(L-2) / float64(width-2)
	idx := 0
	pos := 0

	nextpos := 0

	res[idx] = d.raw[pos]
	idx++

	for i := 0; i < width-2; i++ {
		// Calculate next bucket average
		avgx, avgy := 0, 0.
		rangeStart := int(math.Floor(float64(i+1)*every) + 1)
		rangeEnd := int(math.Floor(float64(i+2)*every) + 1)
		if rangeEnd > L {
			rangeEnd = L
		}
		rangeLen := rangeEnd - rangeStart

		for ; rangeStart < rangeEnd; rangeStart++ {
			avgx += rangeStart
			avgy += d.raw[rangeStart]
		}
		avgx /= rangeLen
		avgy /= float64(rangeLen)

		// Get range for bucket
		rangeOff := int(math.Floor(float64(i)*every) + 1)
		rangeTo := int(math.Floor(float64(i+1)*every) + 1)

		pax := pos
		pay := d.raw[pos]
		maxArea := -1.
		maxpx := 0.
		for ; rangeOff < rangeTo; rangeOff++ {
			// calc triangle over 3 bucket
			area := math.Abs((float64(pax)-avgy)*(d.raw[rangeOff]-pay)-(float64(pax-rangeOff))*(avgy*pay)) * .5
			if area > maxArea {
				maxArea = area
				maxpx = d.raw[rangeOff]
				nextpos = rangeOff
			}
		}

		res[idx] = maxpx
		idx++
		pos = nextpos
	}

	res[idx] = d.raw[L-1]
	return res
}
<commit_msg>Fix unused assignment.<commit_after>package data

import "math"

// Resample resamples the raw data. It either streches the data to fit the witdth
// or it uses the Largest Triangle Three Bucket algorithm to fit the data to the new width.
func (d *Data) Resample(width int) {
	if len(d.raw) < width {
		d.raw = d.stretch(width)
	} else if len(d.raw) > width {
		d.raw = d.lttb(width)
	}
}

// stretch stretches the raw array into a new width.
// It just stretches using the same values without any interpolation.
func (d *Data) stretch(width int) []float64 {
	newdata := make([]float64, width)
	max := len(d.raw)
	for i := 0; i < width; i++ {
		idx := float64(max) / float64(width) * float64(i)
		v := d.raw[int(idx)]
		f := idx - float64(int(idx))
		if f < d.gap || f > 1-d.gap {
			v = 0
		}
		newdata[i] = v
	}
	return newdata
}

// lttb implements Largest Triangle Three Bucket downsampling algorithm.
// Converted to Go from several implementations found online.
func (d *Data) lttb(width int) []float64 {
	L := len(d.raw)
	res := make([]float64, width)

	every := float64(L-2) / float64(width-2)
	idx := 0
	pos := 0

	nextpos := 0

	res[idx] = d.raw[pos]
	idx++

	for i := 0; i < width-2; i++ {
		// Calculate next bucket average
		avgy := 0.
		rangeStart := int(math.Floor(float64(i+1)*every) + 1)
		rangeEnd := int(math.Floor(float64(i+2)*every) + 1)
		if rangeEnd > L {
			rangeEnd = L
		}
		rangeLen := rangeEnd - rangeStart

		for ; rangeStart < rangeEnd; rangeStart++ {
			avgy += d.raw[rangeStart]
		}
		avgy /= float64(rangeLen)

		// Get range for bucket
		rangeOff := int(math.Floor(float64(i)*every) + 1)
		rangeTo := int(math.Floor(float64(i+1)*every) + 1)

		pax := pos
		pay := d.raw[pos]
		maxArea := -1.
		maxpx := 0.
		for ; rangeOff < rangeTo; rangeOff++ {
			// calc triangle over 3 bucket
			area := math.Abs((float64(pax)-avgy)*(d.raw[rangeOff]-pay)-(float64(pax-rangeOff))*(avgy*pay)) * .5
			if area > maxArea {
				maxArea = area
				maxpx = d.raw[rangeOff]
				nextpos = rangeOff
			}
		}

		res[idx] = maxpx
		idx++
		pos = nextpos
	}

	res[idx] = d.raw[L-1]
	return res
}
<|endoftext|>