<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/dispatcher"
	"go.chromium.org/luci/common/sync/dispatcher/buffer"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/hardcoded/chromeinfra"
	"golang.org/x/time/rate"

	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"
)

var (
	errInvalidConfig = errors.New("invalid configuration")
	errNoWorkToDo    = errors.New("no work to do")

	leaseRetryParams = func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   time.Second,
				Retries: -1,
			},
			Multiplier: 1.25,
			MaxDelay:   time.Minute * 10,
		}
	}

	ackChannelOptions = &dispatcher.Options{
		QPSLimit: rate.NewLimiter(1, 1), // 1 QPS max rate
		Buffer: buffer.Options{
			MaxLeases:     10,
			BatchSize:     500,
			BatchDuration: 10 * time.Minute,
			FullBehavior: &buffer.BlockNewItems{
				MaxItems: 10 * 500,
			},
			Retry: func() retry.Iterator {
				return &retry.ExponentialBackoff{
					Limited: retry.Limited{
						Delay:   time.Second,
						Retries: 10,
					},
					Multiplier: 1.25,
					MaxDelay:   time.Minute * 10,
				}
			},
		},
	}

	mkJobChannelOptions = func(maxWorkers int) *dispatcher.Options {
		return &dispatcher.Options{
			Buffer: buffer.Options{
				MaxLeases: maxWorkers,
				BatchSize: 1,
				FullBehavior: &buffer.BlockNewItems{
					MaxItems: 2 * maxWorkers,
				},
			},
		}
	}

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service

	maxConcurrentTasks int
}

// runForever runs the archivist loop forever.
func runForever(ctx context.Context, taskConcurrency int, ar archivist.Archivist) {
	type archiveJob struct {
		deadline time.Time
		task     *logdog.ArchiveTask
	}

	ackChan, err := dispatcher.NewChannel(ctx, ackChannelOptions, func(batch *buffer.Batch) error {
		var req *logdog.DeleteRequest
		if batch.Meta != nil {
			req = batch.Meta.(*logdog.DeleteRequest)
		} else {
			tasks := make([]*logdog.ArchiveTask, len(batch.Data))
			for i, datum := range batch.Data {
				tasks[i] = datum.(*logdog.ArchiveTask)
				batch.Data[i] = nil
			}
			req = &logdog.DeleteRequest{Tasks: tasks}
			batch.Meta = req
		}
		_, err := ar.Service.DeleteArchiveTasks(ctx, req)
		if err == nil {
			tsAckCount.Add(ctx, int64(len(req.Tasks)))
		}
		return transient.Tag.Apply(err)
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "draining ACK channel")
		ackChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "ACK channel drained")
	}()

	jobChanOpts := mkJobChannelOptions(taskConcurrency)
	jobChan, err := dispatcher.NewChannel(ctx, jobChanOpts, func(data *buffer.Batch) error {
		job := data.Data[0].(*archiveJob)

		nc, cancel := context.WithDeadline(ctx, job.deadline)
		defer cancel()

		startTime := clock.Now(ctx)
		err := ar.ArchiveTask(nc, job.task)
		duration := clock.Now(ctx).Sub(startTime)
		tsTaskProcessingTime.Add(ctx, float64(duration.Nanoseconds())/1000000, err == nil)

		if err == nil {
			select {
			case ackChan.C <- job.task:
			case <-ctx.Done():
				logging.Errorf(ctx, "Failed to ACK task %v due to context: %s", job.task, ctx.Err())
			}
		} else {
			tsNackCount.Add(ctx, 1)
			logging.Errorf(ctx, "Failed to archive task %v: %s", job.task, err)
		}

		return nil
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "Job channel draining")
		jobChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "Job channel drained")
	}()

	// now we spin forever, pushing items into jobChan.
	sleepTime := 1
	var previousCycle time.Time
	for ctx.Err() == nil {
		var tasks *logdog.LeaseResponse
		var deadline time.Time

		err := retry.Retry(ctx, leaseRetryParams, func() (err error) {
			// we grab here to avoid getting stuck with some bad loopParams.
			loopParams := grabLoopParams()
			req := loopParams.mkRequest(ctx)
			logging.Infof(ctx, "Leasing max %d tasks for %s", loopParams.batchSize, loopParams.deadline)
			deadline = clock.Now(ctx).Add(loop.deadline)
			tasks, err = ar.Service.LeaseArchiveTasks(ctx, req)
			return
		}, retry.LogCallback(ctx, "LeaseArchiveTasks"))
		if ctx.Err() == nil && err != nil {
			panic("impossible: infinite retry stopped: " + err.Error())
		}

		now := clock.Now(ctx)
		if !previousCycle.IsZero() {
			tsLoopCycleTime.Add(ctx, float64(now.Sub(previousCycle).Nanoseconds()/1000000))
		}
		previousCycle = now

		if len(tasks.Tasks) == 0 {
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			logging.Infof(ctx, "no work to do, sleeping for %d seconds", sleepTime)
			clock.Sleep(ctx, time.Duration(sleepTime)*time.Second)
			previousCycle = time.Time{}
			continue
		} else {
			tsLeaseCount.Add(ctx, int64(len(tasks.Tasks)))
			sleepTime = 1
		}

		for _, task := range tasks.Tasks {
			select {
			case jobChan.C <- &archiveJob{deadline, task}:
			case <-ctx.Done():
				logging.Infof(ctx, "lease thread got context err: %s", ctx.Err())
				break
			}
		}
	}

	logging.Infof(ctx, "runForever no longer running forever: ctx.Err() == %s", ctx.Err())
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, project string) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, project)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Load our settings and update them periodically.
	fetchLoopParams(c)
	go loopParamsUpdater(c)

	runForever(c, a.maxConcurrentTasks, ar)

	return nil
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Flags.IntVar(&a.maxConcurrentTasks, "max-concurrent-tasks", 1,
		"Maximum number of archive tasks to process concurrently. "+
			"Pass 0 to set infinite limit.")
	a.Run(context.Background(), a.runArchivist)
}
<commit_msg>[logdog] Get loop cycle data even when sleeping.<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/dispatcher"
	"go.chromium.org/luci/common/sync/dispatcher/buffer"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/hardcoded/chromeinfra"
	"golang.org/x/time/rate"

	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"
)

var (
	errInvalidConfig = errors.New("invalid configuration")
	errNoWorkToDo    = errors.New("no work to do")

	leaseRetryParams = func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   time.Second,
				Retries: -1,
			},
			Multiplier: 1.25,
			MaxDelay:   time.Minute * 10,
		}
	}

	ackChannelOptions = &dispatcher.Options{
		QPSLimit: rate.NewLimiter(1, 1), // 1 QPS max rate
		Buffer: buffer.Options{
			MaxLeases:     10,
			BatchSize:     500,
			BatchDuration: 10 * time.Minute,
			FullBehavior: &buffer.BlockNewItems{
				MaxItems: 10 * 500,
			},
			Retry: func() retry.Iterator {
				return &retry.ExponentialBackoff{
					Limited: retry.Limited{
						Delay:   time.Second,
						Retries: 10,
					},
					Multiplier: 1.25,
					MaxDelay:   time.Minute * 10,
				}
			},
		},
	}

	mkJobChannelOptions = func(maxWorkers int) *dispatcher.Options {
		return &dispatcher.Options{
			Buffer: buffer.Options{
				MaxLeases: maxWorkers,
				BatchSize: 1,
				FullBehavior: &buffer.BlockNewItems{
					MaxItems: 2 * maxWorkers,
				},
			},
		}
	}

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service

	maxConcurrentTasks int
}

// runForever runs the archivist loop forever.
func runForever(ctx context.Context, taskConcurrency int, ar archivist.Archivist) {
	type archiveJob struct {
		deadline time.Time
		task     *logdog.ArchiveTask
	}

	ackChan, err := dispatcher.NewChannel(ctx, ackChannelOptions, func(batch *buffer.Batch) error {
		var req *logdog.DeleteRequest
		if batch.Meta != nil {
			req = batch.Meta.(*logdog.DeleteRequest)
		} else {
			tasks := make([]*logdog.ArchiveTask, len(batch.Data))
			for i, datum := range batch.Data {
				tasks[i] = datum.(*logdog.ArchiveTask)
				batch.Data[i] = nil
			}
			req = &logdog.DeleteRequest{Tasks: tasks}
			batch.Meta = req
		}
		_, err := ar.Service.DeleteArchiveTasks(ctx, req)
		if err == nil {
			tsAckCount.Add(ctx, int64(len(req.Tasks)))
		}
		return transient.Tag.Apply(err)
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "draining ACK channel")
		ackChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "ACK channel drained")
	}()

	jobChanOpts := mkJobChannelOptions(taskConcurrency)
	jobChan, err := dispatcher.NewChannel(ctx, jobChanOpts, func(data *buffer.Batch) error {
		job := data.Data[0].(*archiveJob)

		nc, cancel := context.WithDeadline(ctx, job.deadline)
		defer cancel()

		startTime := clock.Now(ctx)
		err := ar.ArchiveTask(nc, job.task)
		duration := clock.Now(ctx).Sub(startTime)
		tsTaskProcessingTime.Add(ctx, float64(duration.Nanoseconds())/1000000, err == nil)

		if err == nil {
			select {
			case ackChan.C <- job.task:
			case <-ctx.Done():
				logging.Errorf(ctx, "Failed to ACK task %v due to context: %s", job.task, ctx.Err())
			}
		} else {
			tsNackCount.Add(ctx, 1)
			logging.Errorf(ctx, "Failed to archive task %v: %s", job.task, err)
		}

		return nil
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "Job channel draining")
		jobChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "Job channel drained")
	}()

	// now we spin forever, pushing items into jobChan.
	sleepTime := 1
	var previousCycle time.Time
	for ctx.Err() == nil {
		var tasks *logdog.LeaseResponse
		var deadline time.Time

		err := retry.Retry(ctx, leaseRetryParams, func() (err error) {
			// we grab here to avoid getting stuck with some bad loopParams.
			loopParams := grabLoopParams()
			req := loopParams.mkRequest(ctx)
			logging.Infof(ctx, "Leasing max %d tasks for %s", loopParams.batchSize, loopParams.deadline)
			deadline = clock.Now(ctx).Add(loop.deadline)
			tasks, err = ar.Service.LeaseArchiveTasks(ctx, req)
			return
		}, retry.LogCallback(ctx, "LeaseArchiveTasks"))
		if ctx.Err() == nil && err != nil {
			panic("impossible: infinite retry stopped: " + err.Error())
		}

		now := clock.Now(ctx)
		if !previousCycle.IsZero() {
			tsLoopCycleTime.Add(ctx, float64(now.Sub(previousCycle).Nanoseconds()/1000000))
		}
		previousCycle = now

		if len(tasks.Tasks) == 0 {
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			logging.Infof(ctx, "no work to do, sleeping for %d seconds", sleepTime)
			clock.Sleep(ctx, time.Duration(sleepTime)*time.Second)
			continue
		} else {
			tsLeaseCount.Add(ctx, int64(len(tasks.Tasks)))
			sleepTime = 1
		}

		for _, task := range tasks.Tasks {
			select {
			case jobChan.C <- &archiveJob{deadline, task}:
			case <-ctx.Done():
				logging.Infof(ctx, "lease thread got context err: %s", ctx.Err())
				break
			}
		}
	}

	logging.Infof(ctx, "runForever no longer running forever: ctx.Err() == %s", ctx.Err())
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, project string) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, project)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Load our settings and update them periodically.
	fetchLoopParams(c)
	go loopParamsUpdater(c)

	runForever(c, a.maxConcurrentTasks, ar)

	return nil
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Flags.IntVar(&a.maxConcurrentTasks, "max-concurrent-tasks", 1,
		"Maximum number of archive tasks to process concurrently. "+
			"Pass 0 to set infinite limit.")
	a.Run(context.Background(), a.runArchivist)
}
<|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/dispatcher"
	"go.chromium.org/luci/common/sync/dispatcher/buffer"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/hardcoded/chromeinfra"
	"golang.org/x/time/rate"

	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"
)

var (
	errInvalidConfig = errors.New("invalid configuration")
	errNoWorkToDo    = errors.New("no work to do")

	leaseRetryParams = func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   time.Second,
				Retries: -1,
			},
			Multiplier: 1.25,
			MaxDelay:   time.Minute * 10,
		}
	}

	ackChannelOptions = &dispatcher.Options{
		QPSLimit: rate.NewLimiter(1, 1), // 1 QPS max rate
		Buffer: buffer.Options{
			MaxLeases:     10,
			BatchSize:     500,
			BatchDuration: 10 * time.Minute,
			FullBehavior: &buffer.BlockNewItems{
				MaxItems: 10 * 500,
			},
			Retry: func() retry.Iterator {
				return &retry.ExponentialBackoff{
					Limited: retry.Limited{
						Delay:   time.Second,
						Retries: 10,
					},
					Multiplier: 1.25,
					MaxDelay:   time.Minute * 10,
				}
			},
		},
	}

	mkJobChannelOptions = func(maxWorkers int) *dispatcher.Options {
		return &dispatcher.Options{
			Buffer: buffer.Options{
				MaxLeases: maxWorkers,
				BatchSize: 1,
				FullBehavior: &buffer.BlockNewItems{
					MaxItems: 2 * maxWorkers,
				},
			},
		}
	}

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service

	maxConcurrentTasks int
}

// runForever runs the archivist loop forever.
func runForever(ctx context.Context, taskConcurrency int, ar archivist.Archivist) {
	type archiveJob struct {
		deadline time.Time
		task     *logdog.ArchiveTask
	}

	ackChan, err := dispatcher.NewChannel(ctx, ackChannelOptions, func(batch *buffer.Batch) error {
		var req *logdog.DeleteRequest
		if batch.Meta != nil {
			req = batch.Meta.(*logdog.DeleteRequest)
		} else {
			tasks := make([]*logdog.ArchiveTask, len(batch.Data))
			for i, datum := range batch.Data {
				tasks[i] = datum.(*logdog.ArchiveTask)
				batch.Data[i] = nil
			}
			req = &logdog.DeleteRequest{Tasks: tasks}
			batch.Meta = req
		}
		_, err := ar.Service.DeleteArchiveTasks(ctx, req)
		if err == nil {
			tsAckCount.Add(ctx, int64(len(req.Tasks)))
		}
		return transient.Tag.Apply(err)
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "draining ACK channel")
		ackChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "ACK channel drained")
	}()

	jobChanOpts := mkJobChannelOptions(taskConcurrency)
	jobChan, err := dispatcher.NewChannel(ctx, jobChanOpts, func(data *buffer.Batch) error {
		job := data.Data[0].(*archiveJob)

		nc, cancel := context.WithDeadline(ctx, job.deadline)
		defer cancel()

		startTime := clock.Now(ctx)
		err := ar.ArchiveTask(nc, job.task)
		duration := clock.Now(ctx).Sub(startTime)
		tsTaskProcessingTime.Add(ctx, float64(duration.Nanoseconds())/1000000, err == nil)

		if err == nil {
			select {
			case ackChan.C <- job.task:
			case <-ctx.Done():
				logging.Errorf(ctx, "Failed to ACK task %v due to context: %s", job.task, ctx.Err())
			}
		} else {
			tsNackCount.Add(ctx, 1)
			logging.Errorf(ctx, "Failed to archive task %v: %s", job.task, err)
		}

		return nil
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "Job channel draining")
		jobChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "Job channel drained")
	}()

	// now we spin forever, pushing items into jobChan.
	sleepTime := 1
	var previousCycle time.Time
	for ctx.Err() == nil {
		var tasks *logdog.LeaseResponse
		var deadline time.Time

		err := retry.Retry(ctx, leaseRetryParams, func() (err error) {
			// we grab here to avoid getting stuck with some bad loopParams.
			loopParams := grabLoopParams()
			req := loopParams.mkRequest(ctx)
			logging.Infof(ctx, "Leasing max %d tasks for %s", loopParams.batchSize, loopParams.deadline)
			deadline = clock.Now(ctx).Add(loop.deadline)
			tasks, err = ar.Service.LeaseArchiveTasks(ctx, req)
			return
		}, retry.LogCallback(ctx, "LeaseArchiveTasks"))
		if ctx.Err() == nil && err != nil {
			panic("impossible: infinite retry stopped: " + err.Error())
		}

		now := clock.Now(ctx)
		if !previousCycle.IsZero() {
			tsLoopCycleTime.Add(ctx, float64(now.Sub(previousCycle).Nanoseconds()/1000000))
		}
		previousCycle = now

		if len(tasks.Tasks) == 0 {
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			logging.Infof(ctx, "no work to do, sleeping for %d seconds", sleepTime)
			clock.Sleep(ctx, time.Duration(sleepTime)*time.Second)
			continue
		} else {
			tsLeaseCount.Add(ctx, int64(len(tasks.Tasks)))
			sleepTime = 1
		}

		for _, task := range tasks.Tasks {
			select {
			case jobChan.C <- &archiveJob{deadline, task}:
			case <-ctx.Done():
				logging.Infof(ctx, "lease thread got context err: %s", ctx.Err())
				break
			}
		}
	}

	logging.Infof(ctx, "runForever no longer running forever: ctx.Err() == %s", ctx.Err())
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, project string) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, project)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Load our settings and update them periodically.
	fetchLoopParams(c)
	go loopParamsUpdater(c)

	runForever(c, a.maxConcurrentTasks, ar)

	return nil
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Flags.IntVar(&a.maxConcurrentTasks, "max-concurrent-tasks", 1,
		"Maximum number of archive tasks to process concurrently. "+
			"Pass 0 to set infinite limit.")
	a.Run(context.Background(), a.runArchivist)
}
<commit_msg>[logdog] Actually allow dispatcher channels to run at full parallelism.<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/dispatcher"
	"go.chromium.org/luci/common/sync/dispatcher/buffer"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/hardcoded/chromeinfra"
	"golang.org/x/time/rate"

	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"
)

var (
	errInvalidConfig = errors.New("invalid configuration")
	errNoWorkToDo    = errors.New("no work to do")

	leaseRetryParams = func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   time.Second,
				Retries: -1,
			},
			Multiplier: 1.25,
			MaxDelay:   time.Minute * 10,
		}
	}

	ackChannelOptions = &dispatcher.Options{
		QPSLimit: rate.NewLimiter(rate.Inf, 1), // no limit
		Buffer: buffer.Options{
			MaxLeases:     10,
			BatchSize:     500,
			BatchDuration: 10 * time.Minute,
			FullBehavior: &buffer.BlockNewItems{
				MaxItems: 10 * 500,
			},
			Retry: func() retry.Iterator {
				return &retry.ExponentialBackoff{
					Limited: retry.Limited{
						Delay:   time.Second,
						Retries: 10,
					},
					Multiplier: 1.25,
					MaxDelay:   time.Minute * 10,
				}
			},
		},
	}

	mkJobChannelOptions = func(maxWorkers int) *dispatcher.Options {
		return &dispatcher.Options{
			QPSLimit: rate.NewLimiter(rate.Inf, 1), // no limit
			Buffer: buffer.Options{
				MaxLeases: maxWorkers,
				BatchSize: 1,
				FullBehavior: &buffer.BlockNewItems{
					MaxItems: 2 * maxWorkers,
				},
			},
		}
	}

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service

	maxConcurrentTasks int
}

// runForever runs the archivist loop forever.
func runForever(ctx context.Context, taskConcurrency int, ar archivist.Archivist) {
	type archiveJob struct {
		deadline time.Time
		task     *logdog.ArchiveTask
	}

	ackChan, err := dispatcher.NewChannel(ctx, ackChannelOptions, func(batch *buffer.Batch) error {
		var req *logdog.DeleteRequest
		if batch.Meta != nil {
			req = batch.Meta.(*logdog.DeleteRequest)
		} else {
			tasks := make([]*logdog.ArchiveTask, len(batch.Data))
			for i, datum := range batch.Data {
				tasks[i] = datum.(*logdog.ArchiveTask)
				batch.Data[i] = nil
			}
			req = &logdog.DeleteRequest{Tasks: tasks}
			batch.Meta = req
		}
		_, err := ar.Service.DeleteArchiveTasks(ctx, req)
		if err == nil {
			tsAckCount.Add(ctx, int64(len(req.Tasks)))
		}
		return transient.Tag.Apply(err)
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "draining ACK channel")
		ackChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "ACK channel drained")
	}()

	jobChanOpts := mkJobChannelOptions(taskConcurrency)
	jobChan, err := dispatcher.NewChannel(ctx, jobChanOpts, func(data *buffer.Batch) error {
		job := data.Data[0].(*archiveJob)

		nc, cancel := context.WithDeadline(ctx, job.deadline)
		defer cancel()

		startTime := clock.Now(ctx)
		err := ar.ArchiveTask(nc, job.task)
		duration := clock.Now(ctx).Sub(startTime)
		tsTaskProcessingTime.Add(ctx, float64(duration.Nanoseconds())/1000000, err == nil)

		if err == nil {
			select {
			case ackChan.C <- job.task:
			case <-ctx.Done():
				logging.Errorf(ctx, "Failed to ACK task %v due to context: %s", job.task, ctx.Err())
			}
		} else {
			tsNackCount.Add(ctx, 1)
			logging.Errorf(ctx, "Failed to archive task %v: %s", job.task, err)
		}

		return nil
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "Job channel draining")
		jobChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "Job channel drained")
	}()

	// now we spin forever, pushing items into jobChan.
	sleepTime := 1
	var previousCycle time.Time
	for ctx.Err() == nil {
		var tasks *logdog.LeaseResponse
		var deadline time.Time

		err := retry.Retry(ctx, leaseRetryParams, func() (err error) {
			// we grab here to avoid getting stuck with some bad loopParams.
			loopParams := grabLoopParams()
			req := loopParams.mkRequest(ctx)
			logging.Infof(ctx, "Leasing max %d tasks for %s", loopParams.batchSize, loopParams.deadline)
			deadline = clock.Now(ctx).Add(loop.deadline)
			tasks, err = ar.Service.LeaseArchiveTasks(ctx, req)
			return
		}, retry.LogCallback(ctx, "LeaseArchiveTasks"))
		if ctx.Err() == nil && err != nil {
			panic("impossible: infinite retry stopped: " + err.Error())
		}

		now := clock.Now(ctx)
		if !previousCycle.IsZero() {
			tsLoopCycleTime.Add(ctx, float64(now.Sub(previousCycle).Nanoseconds()/1000000))
		}
		previousCycle = now

		if len(tasks.Tasks) == 0 {
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			logging.Infof(ctx, "no work to do, sleeping for %d seconds", sleepTime)
			clock.Sleep(ctx, time.Duration(sleepTime)*time.Second)
			continue
		} else {
			tsLeaseCount.Add(ctx, int64(len(tasks.Tasks)))
			sleepTime = 1
		}

		for _, task := range tasks.Tasks {
			select {
			case jobChan.C <- &archiveJob{deadline, task}:
			case <-ctx.Done():
				logging.Infof(ctx, "lease thread got context err: %s", ctx.Err())
				break
			}
		}
	}

	logging.Infof(ctx, "runForever no longer running forever: ctx.Err() == %s", ctx.Err())
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, project string) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, project)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Load our settings and update them periodically.
	fetchLoopParams(c)
	go loopParamsUpdater(c)

	runForever(c, a.maxConcurrentTasks, ar)

	return nil
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Flags.IntVar(&a.maxConcurrentTasks, "max-concurrent-tasks", 1,
		"Maximum number of archive tasks to process concurrently. "+
			"Pass 0 to set infinite limit.")
	a.Run(context.Background(), a.runArchivist)
}
<|endoftext|>


<commit_before>package io

import (
	"github.com/valep27/chip8/emu"
	"github.com/veandco/go-sdl2/sdl"
)

// SdlFrontend implements basic drawing using SDL2.
type SdlFrontend struct {
	window *sdl.Window
}

func NewSdlFrontend() SdlFrontend {
	return SdlFrontend{nil}
}

func (sf *SdlFrontend) Initialize() {
	sdl.Init(sdl.INIT_EVERYTHING)

	window, err := sdl.CreateWindow("Chip8",
		sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED,
		800, 600,
		sdl.WINDOW_SHOWN|sdl.WINDOW_OPENGL)

	if err != nil {
		panic(err)
	}

	sf.window = window
}

func (sf *SdlFrontend) Draw(emulator emu.Chip8) {
	surface, err := sf.window.GetSurface()
	if err != nil {
		panic(err)
	}

	rect := sdl.Rect{0, 0, 800, 600}
	surface.FillRect(&rect, 0xffff0000)
	sf.window.UpdateSurface()
}

func (sf *SdlFrontend) Close() {
	defer sf.window.Destroy()
	sdl.Delay(3000)
	sdl.Quit()
}
<commit_msg>Add comments to SDL2 frontend methods<commit_after>package io

import (
	"github.com/valep27/chip8/emu"
	"github.com/veandco/go-sdl2/sdl"
)

// SdlFrontend implements basic drawing using SDL2.
type SdlFrontend struct {
	window *sdl.Window
}

// NewSdlFrontend creates a new uninitialized frontend that uses SDL2.
func NewSdlFrontend() SdlFrontend {
	return SdlFrontend{nil}
}

// Initialize creates the window and sets up any internal state for the frontend.
func (sf *SdlFrontend) Initialize() {
	sdl.Init(sdl.INIT_EVERYTHING)

	window, err := sdl.CreateWindow("Chip8",
		sdl.WINDOWPOS_UNDEFINED, sdl.WINDOWPOS_UNDEFINED,
		800, 600,
		sdl.WINDOW_SHOWN|sdl.WINDOW_OPENGL)

	if err != nil {
		panic(err)
	}

	sf.window = window
}

// Draw will draw on the window the contents of the emulator framebuffer.
func (sf *SdlFrontend) Draw(emulator emu.Chip8) {
	surface, err := sf.window.GetSurface()
	if err != nil {
		panic(err)
	}

	rect := sdl.Rect{0, 0, 800, 600}
	surface.FillRect(&rect, 0xffff0000)
	sf.window.UpdateSurface()
}

// Close will free any resources, the window and quit the application.
// Best used with defer.
func (sf *SdlFrontend) Close() {
	defer sf.window.Destroy()
	sdl.Delay(3000)
	sdl.Quit()
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package envoy

import (
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	"github.com/hashicorp/go-multierror"

	"istio.io/api/proxy/v1/config"
	"istio.io/manager/model"
	"istio.io/manager/proxy"
)

type ingressWatcher struct {
	agent   proxy.Agent
	ctl     model.Controller
	context *IngressConfig
}

// NewIngressWatcher creates a new ingress watcher instance with an agent
func NewIngressWatcher(ctl model.Controller, context *IngressConfig) (Watcher, error) {
	agent := proxy.NewAgent(runEnvoy(context.Mesh, "ingress"), cleanupEnvoy(context.Mesh), 10, 100*time.Millisecond)

	out := &ingressWatcher{
		agent:   agent,
		ctl:     ctl,
		context: context,
	}

	if err := ctl.AppendConfigHandler(model.IngressRule, func(model.Key, proto.Message, model.Event) {
		out.reload()
	}); err != nil {
		return nil, err
	}

	// ingress rule listing depends on the service declaration being up to date
	if err := ctl.AppendServiceHandler(func(*model.Service, model.Event) {
		out.reload()
	}); err != nil {
		return nil, err
	}

	return out, nil
}

func (w *ingressWatcher) reload() {
	w.agent.ScheduleConfigUpdate(generateIngress(w.context))
}

func (w *ingressWatcher) Run(stop <-chan struct{}) {
	go w.agent.Run(stop)

	// Initialize envoy according to the current model state,
	// instead of waiting for the first event to arrive.
	// Note that this is currently done synchronously (blocking),
	// to avoid racing with controller events lurking around the corner.
	// This can be improved once we switch to a mechanism where reloads
	// are linearized (e.g., by a single goroutine reloader).
	w.reload()
	w.ctl.Run(stop)
}

// IngressConfig defines information for ingress
type IngressConfig struct {
	// TODO: cert/key filenames will need to be dynamic for multiple key/cert pairs
	CertFile  string
	KeyFile   string
	Namespace string
	Secrets   model.SecretRegistry
	Registry  *model.IstioRegistry
	Mesh      *config.ProxyMeshConfig
	Port      int
	SSLPort   int
}

func generateIngress(conf *IngressConfig) *Config {
	listeners, clusters := buildIngressListeners(conf)

	return &Config{
		Listeners: listeners,
		Admin: Admin{
			AccessLogPath: DefaultAccessLog,
			Address:       fmt.Sprintf("tcp://%s:%d", WildcardAddress, conf.Mesh.ProxyAdminPort),
		},
		ClusterManager: ClusterManager{
			Clusters: clusters,
			SDS: &SDS{
				Cluster:        buildDiscoveryCluster(conf.Mesh.DiscoveryAddress, "sds", conf.Mesh.ConnectTimeout),
				RefreshDelayMs: (int)(convertDuration(conf.Mesh.DiscoveryRefreshDelay) / time.Millisecond),
			},
		},
	}
}

func buildIngressVhosts(conf *IngressConfig) ([]*VirtualHost, []*VirtualHost, *model.TLSSecret) {
	rules := conf.Registry.IngressRules(conf.Namespace)

	rulesByHost := make(map[string][]*config.RouteRule, len(rules))
	for _, rule := range rules {
		host := "*"
		if rule.Match != nil {
			if authority, ok := rule.Match.HttpHeaders["authority"]; ok {
				switch match := authority.GetMatchType().(type) {
				case *config.StringMatch_Exact:
					host = match.Exact
				default:
					glog.Warningf("Unsupported match type for authority condition: %T", match)
				}
			}
		}

		rulesByHost[host] = append(rulesByHost[host], rule)
	}

	// figure out which hosts are configured to terminate TLS
	// ensure that only one TLS config is used
	// TODO: extensive tests for the TLS logic
	var tls *model.TLSSecret
	tlsValid := true
	tlsHosts := make(map[string]bool)
	for host := range rulesByHost {
		if t, err := conf.Secrets.GetTLSSecret(host); err != nil {
			tlsValid = false
			tlsHosts[host] = true // count this as a TLS host so that it is omitted from the config

			glog.Warningf("Error retrieving TLS context for %q: %v", host, err)
		} else if t != nil {
			if tls == nil {
				tls = t
			} else if string(tls.PrivateKey) != string(t.PrivateKey) || string(tls.Certificate) != string(t.Certificate) {
				glog.Warningf("Unsupported ingress configuration for host %q: multiple TLS configs", host)
				tlsValid = false
			}
			tlsHosts[host] = true
		}
	}

	// build vhosts
	vhosts := make([]*VirtualHost, 0, len(rulesByHost))
	vhostsTLS := make([]*VirtualHost, 0, len(rulesByHost))
	for host, hostRules := range rulesByHost {
		routes := make([]*HTTPRoute, 0, len(hostRules))
		for _, rule := range hostRules {
			route, err := buildIngressRoute(rule)
			if err != nil {
				glog.Warningf("Error constructing Envoy route from ingress rule: %v", err)
				continue
			}
			routes = append(routes, route)
		}
		sort.Sort(RoutesByPath(routes))
		vhost := &VirtualHost{
			Name:    host,
			Domains: []string{host},
			Routes:  routes,
		}

		// sort into TLS and non-TLS
		if tlsHosts[host] {
			if tlsValid { // only add TLS vhosts when the TLS config is valid
				vhostsTLS = append(vhostsTLS, vhost)
			}
		} else {
			vhosts = append(vhosts, vhost)
		}
	}

	return vhosts, vhostsTLS, tls
}

func buildIngressListeners(conf *IngressConfig) (Listeners, Clusters) {
	vhosts, vhostsTLS, tls := buildIngressVhosts(conf)

	clusters := make(Clusters, 0)
	listeners := make([]*Listener, 0)

	if len(vhosts) > 0 {
		sort.Slice(vhosts, func(i, j int) bool { return vhosts[i].Name < vhosts[j].Name })

		rConfig := &HTTPRouteConfig{VirtualHosts: vhosts}
		listener := &Listener{
			Address:    fmt.Sprintf("tcp://%s:%d", WildcardAddress, conf.Port),
			BindToPort: true,
			Filters: []*NetworkFilter{
				{
					Type: "read",
					Name: HTTPConnectionManager,
					Config: HTTPFilterConfig{
						CodecType:   "auto",
						StatPrefix:  "http",
						AccessLog:   []AccessLog{{Path: DefaultAccessLog}},
						RouteConfig: rConfig,
						Filters: []HTTPFilter{
							{
								Type:   "decoder",
								Name:   "router",
								Config: FilterRouterConfig{},
							},
						},
					},
				},
			},
		}

		listeners = append(listeners, listener)
		clusters = append(clusters, rConfig.clusters()...)
	}

	if len(vhostsTLS) > 0 {
		if err := writeTLS(conf.CertFile, conf.KeyFile, tls); err != nil {
			glog.Warning("Failed to get and save secrets. Envoy will crash and trigger a retry...")
		}

		sort.Slice(vhostsTLS, func(i, j int) bool { return vhostsTLS[i].Name < vhostsTLS[j].Name })

		rConfig := &HTTPRouteConfig{VirtualHosts: vhostsTLS}
		listener := &Listener{
			Address: fmt.Sprintf("tcp://%s:%d", WildcardAddress, conf.SSLPort),
			SSLContext: &SSLContext{
				CertChainFile:  conf.CertFile,
				PrivateKeyFile: conf.KeyFile,
			},
			BindToPort: true,
			Filters: []*NetworkFilter{
				{
					Type: "read",
					Name: HTTPConnectionManager,
					Config: HTTPFilterConfig{
						CodecType:   "auto",
						StatPrefix:  "https",
						AccessLog:   []AccessLog{{Path: DefaultAccessLog}},
						RouteConfig: rConfig,
						Filters: []HTTPFilter{
							{
								Type:   "decoder",
								Name:   "router",
								Config: FilterRouterConfig{},
							},
						},
					},
				},
			},
		}

		listeners = append(listeners, listener)
		clusters = append(clusters, rConfig.clusters()...)
	}

	clusters = clusters.normalize()
	clusters.setTimeout(conf.Mesh.ConnectTimeout)

	return listeners, clusters
}

func writeTLS(certFile, keyFile string, tls *model.TLSSecret) error {
	if err := ioutil.WriteFile(certFile, tls.Certificate, 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(keyFile, tls.PrivateKey, 0755); err != nil {
		return err
	}

	return nil
}

// buildIngressRoute translates an ingress rule to an Envoy route
func buildIngressRoute(rule *config.RouteRule) (*HTTPRoute, error) {
	route := &HTTPRoute{
		Path:   "",
		Prefix: "/",
	}

	if rule.Match != nil && rule.Match.HttpHeaders != nil {
		if uri, ok := rule.Match.HttpHeaders[HeaderURI]; ok {
			switch m := uri.MatchType.(type) {
			case *config.StringMatch_Exact:
				route.Path = m.Exact
				route.Prefix = ""
			case *config.StringMatch_Prefix:
				route.Path = ""
				route.Prefix = m.Prefix
			case *config.StringMatch_Regex:
				return nil, fmt.Errorf("unsupported route match condition: regex")
			}
		}
	}

	clusters := make([]*WeightedClusterEntry, 0)
	for _, dst := range rule.Route {
		// fetch route destination, or fallback to rule destination
		destination := dst.Destination
		if destination == "" {
			destination = rule.Destination
		}

		port, tags, err := extractPortAndTags(dst)
		if err != nil {
			return nil, multierror.Append(fmt.Errorf("failed to extract routing rule destination port"), err)
		}

		cluster := buildOutboundCluster(destination, port, tags)
		clusters = append(clusters, &WeightedClusterEntry{
			Name:   cluster.Name,
			Weight: int(dst.Weight),
		})
		route.clusters = append(route.clusters, cluster)
	}
	route.WeightedClusters = &WeightedCluster{Clusters: clusters}

	// rewrite to a single cluster if it's one weighted cluster
	if len(rule.Route) == 1 {
		route.Cluster = route.WeightedClusters.Clusters[0].Name
		route.WeightedClusters = nil
	}

	// Ensure all destination clusters have the same port number.
	//
	// This is currently required for doing host header rewrite (host:port),
	// which is scoped to the entire route.
	// This restriction can be relaxed by constructing multiple envoy.Route objects
	// per config.RouteRule, and doing weighted load balancing using Runtime.
	portSet := make(map[int]struct{}, 1)
	for _, cluster := range route.clusters {
		portSet[cluster.port.Port] = struct{}{}
	}
	if len(portSet) > 1 {
		return nil, fmt.Errorf("unsupported multiple destination ports per ingress route rule")
	}

	// Rewrite the host header so that inbound proxies can match incoming traffic
	route.HostRewrite = fmt.Sprintf("%s:%d", rule.Destination, route.clusters[0].port.Port)

	return route, nil
}

// extractPortAndTags extracts the destination service port from the given destination,
// as well as its tags (after clearing meta-tags describing the port).
// Note that this is a temporary measure to communicate the destination service's port
// to the proxy configuration generator. This can be improved by using
// a dedicated model object for IngressRule (instead of reusing RouteRule),
// which exposes the necessary target port field within the "Route" field.
func extractPortAndTags(dst *config.DestinationWeight) (*model.Port, model.Tags, error) {
	portNum, err := strconv.Atoi(dst.Tags["servicePort.port"])
	if err != nil {
		return nil, nil, err
	}
	portName, ok := dst.Tags["servicePort.name"]
	if !ok {
		return nil, nil, fmt.Errorf("no name specified for service port %d", portNum)
	}
	portProto, ok := dst.Tags["servicePort.protocol"]
	if !ok {
		return nil, nil, fmt.Errorf("no protocol specified for service port %d", portNum)
	}

	port := &model.Port{
		Port:     portNum,
		Name:     portName,
		Protocol: model.Protocol(portProto),
	}

	var tags model.Tags
	if len(dst.Tags) > 3 {
		tags = make(model.Tags, len(dst.Tags)-3)
		for k, v := range dst.Tags {
			tags[k] = v
		}
		delete(tags, "servicePort.port")
		delete(tags, "servicePort.name")
		delete(tags, "servicePort.protocol")
	}

	return port, tags, nil
}
<commit_msg>Improve ingress failure handling (#540)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package envoy

import (
	"fmt"
	"io/ioutil"
	"sort"
	"strconv"
	"time"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	"github.com/hashicorp/go-multierror"

	"istio.io/api/proxy/v1/config"
	"istio.io/manager/model"
	"istio.io/manager/proxy"
)

type ingressWatcher struct {
	agent   proxy.Agent
	ctl     model.Controller
	context *IngressConfig
}

// NewIngressWatcher creates a new ingress watcher instance with an agent
func NewIngressWatcher(ctl model.Controller, context *IngressConfig) (Watcher, error) {
	agent := proxy.NewAgent(runEnvoy(context.Mesh, "ingress"), cleanupEnvoy(context.Mesh), 10, 100*time.Millisecond)

	out := &ingressWatcher{
		agent:   agent,
		ctl:     ctl,
		context: context,
	}

	if err := ctl.AppendConfigHandler(model.IngressRule, func(model.Key, proto.Message, model.Event) {
		out.reload()
	}); err != nil {
		return nil, err
	}

	// ingress rule listing depends on the service declaration being up to date
	if err := ctl.AppendServiceHandler(func(*model.Service, model.Event) {
		out.reload()
	}); err != nil {
		return nil, err
	}

	return out, nil
}

func (w *ingressWatcher) reload() {
	w.agent.ScheduleConfigUpdate(generateIngress(w.context))
}

func (w *ingressWatcher) Run(stop <-chan struct{}) {
	go w.agent.Run(stop)

	// Initialize envoy according to the current model state,
	// instead of waiting for the first event to arrive.
	// Note that this is currently done synchronously (blocking),
	// to avoid racing with controller events lurking around the corner.
	// This can be improved once we switch to a mechanism where reloads
	// are linearized (e.g., by a single goroutine reloader).
	w.reload()
	w.ctl.Run(stop)
}

// IngressConfig defines information for ingress
type IngressConfig struct {
	// TODO: cert/key filenames will need to be dynamic for multiple key/cert pairs
	CertFile  string
	KeyFile   string
	Namespace string
	Secrets   model.SecretRegistry
	Registry  *model.IstioRegistry
	Mesh      *config.ProxyMeshConfig
	Port      int
	SSLPort   int
}

func generateIngress(conf *IngressConfig) *Config {
	listeners, clusters := buildIngressListeners(conf)

	return &Config{
		Listeners: listeners,
		Admin: Admin{
			AccessLogPath: DefaultAccessLog,
			Address:       fmt.Sprintf("tcp://%s:%d", WildcardAddress, conf.Mesh.ProxyAdminPort),
		},
		ClusterManager: ClusterManager{
			Clusters: clusters,
			SDS: &SDS{
				Cluster:        buildDiscoveryCluster(conf.Mesh.DiscoveryAddress, "sds", conf.Mesh.ConnectTimeout),
				RefreshDelayMs: (int)(convertDuration(conf.Mesh.DiscoveryRefreshDelay) / time.Millisecond),
			},
		},
	}
}

func buildIngressVhosts(conf *IngressConfig) ([]*VirtualHost, []*VirtualHost, *model.TLSSecret) {
	rules := conf.Registry.IngressRules(conf.Namespace)

	rulesByHost := make(map[string][]*config.RouteRule, len(rules))
	for _, rule := range rules {
		host := "*"
		if rule.Match != nil {
			if authority, ok := rule.Match.HttpHeaders["authority"]; ok {
				switch match := authority.GetMatchType().(type) {
				case *config.StringMatch_Exact:
					host = match.Exact
				default:
					glog.Warningf("Unsupported match type for authority condition: %T", match)
				}
			}
		}

		rulesByHost[host] = append(rulesByHost[host], rule)
	}

	// figure out which hosts are configured to terminate TLS
	// ensure that only one TLS config is used
	// TODO: extensive tests for the TLS logic
	var tls *model.TLSSecret
	tlsValid := true
	tlsHosts := make(map[string]bool)
	for host := range rulesByHost {
		if t, err := conf.Secrets.GetTLSSecret(host); err != nil {
			tlsValid = false
			tlsHosts[host] = true // count this as a TLS host so that it is omitted from the config

			glog.Warningf("Error retrieving TLS context for %q: %v", host, err)
		} else if t != nil {
			if tls == nil {
				tls = t
			} else if string(tls.PrivateKey) != string(t.PrivateKey) || string(tls.Certificate) != string(t.Certificate) {
				glog.Warningf("Unsupported ingress configuration for host %q: multiple TLS configs", host)
				tlsValid = false
			}
			tlsHosts[host] = true
		}
	}

	// build vhosts
	vhosts := make([]*VirtualHost, 0, len(rulesByHost))
	vhostsTLS := make([]*VirtualHost, 0, len(rulesByHost))
	for host, hostRules := range rulesByHost {
		routes := make([]*HTTPRoute, 0, len(hostRules))
		for _, rule := range hostRules {
			route, err := buildIngressRoute(rule)
			if err != nil {
				glog.Warningf("Error constructing Envoy route from ingress rule: %v", err)
				continue
			}
			routes = append(routes, route)
		}
		sort.Sort(RoutesByPath(routes))
		vhost := &VirtualHost{
			Name:    host,
			Domains: []string{host},
			Routes:  routes,
		}

		// sort into TLS and non-TLS
		if tlsHosts[host] {
			if tlsValid { // only add TLS vhosts when the TLS config is valid
				vhostsTLS = append(vhostsTLS, vhost)
			}
		} else {
			vhosts = append(vhosts, vhost)
		}
	}

	return vhosts, vhostsTLS, tls
}

func buildIngressListener(address, statPrefix string, routeConfig *HTTPRouteConfig, ssl *SSLContext) *Listener {
	return &Listener{
		Address:    address,
		SSLContext: ssl,
		BindToPort: true,
		Filters: []*NetworkFilter{
			{
				Type: "read",
				Name: HTTPConnectionManager,
				Config: HTTPFilterConfig{
					CodecType:   "auto",
					StatPrefix:  statPrefix,
					AccessLog:   []AccessLog{{Path: DefaultAccessLog}},
					RouteConfig: routeConfig,
					Filters: []HTTPFilter{
						{
							Type:   "decoder",
							Name:   "router",
							Config: FilterRouterConfig{},
						},
					},
				},
			},
		},
	}
}

func buildIngressListeners(conf *IngressConfig) (Listeners, Clusters) {
	vhosts, vhostsTLS, tls := buildIngressVhosts(conf)

	clusters := make(Clusters, 0)
	listeners := make([]*Listener, 0)

	if len(vhosts) > 0 {
		sort.Slice(vhosts, func(i, j int) bool { return vhosts[i].Name < vhosts[j].Name })

		rConfig := &HTTPRouteConfig{VirtualHosts: vhosts}
		address := fmt.Sprintf("tcp://%s:%d", WildcardAddress, conf.Port)
		listeners = append(listeners, buildIngressListener(address, "http", rConfig, nil))
		clusters = append(clusters, rConfig.clusters()...)
	}

	if len(vhostsTLS) > 0 && tls != nil {
		if err := writeTLS(conf.CertFile, conf.KeyFile, tls); err != nil {
			glog.Warning("Failed to write cert/key")
		} else {
			sort.Slice(vhostsTLS, func(i, j int) bool { return vhostsTLS[i].Name < vhostsTLS[j].Name })

			rConfig := &HTTPRouteConfig{VirtualHosts: vhostsTLS}
			ssl := &SSLContext{
				CertChainFile:  conf.CertFile,
				PrivateKeyFile: conf.KeyFile,
			}
			address := fmt.Sprintf("tcp://%s:%d", WildcardAddress, conf.SSLPort)
			listeners = append(listeners, buildIngressListener(address, "https", rConfig, ssl))
			clusters = append(clusters, rConfig.clusters()...)
		}
	}

	clusters = clusters.normalize()
	clusters.setTimeout(conf.Mesh.ConnectTimeout)

	return listeners, clusters
}

func writeTLS(certFile, keyFile string, tls *model.TLSSecret) error {
	if err := ioutil.WriteFile(certFile, tls.Certificate, 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(keyFile, tls.PrivateKey, 0755); err != nil {
		return err
	}

	return nil
}

// buildIngressRoute translates an ingress rule to an Envoy route
func buildIngressRoute(rule *config.RouteRule) (*HTTPRoute, error) {
	route := &HTTPRoute{
		Path:   "",
		Prefix: "/",
	}

	if rule.Match != nil && rule.Match.HttpHeaders != nil {
		if uri, ok := rule.Match.HttpHeaders[HeaderURI]; ok {
			switch m := uri.MatchType.(type) {
			case *config.StringMatch_Exact:
				route.Path = m.Exact
				route.Prefix = ""
			case *config.StringMatch_Prefix:
				route.Path = ""
				route.Prefix = m.Prefix
			case *config.StringMatch_Regex:
				return nil, fmt.Errorf("unsupported route match condition: regex")
			}
		}
	}

	clusters := make([]*WeightedClusterEntry, 0)
	for _, dst := range rule.Route {
		// fetch route destination, or fallback to rule destination
		destination := dst.Destination
		if destination == "" {
			destination = rule.Destination
		}

		port, tags, err := extractPortAndTags(dst)
		if err != nil {
			return nil, multierror.Append(fmt.Errorf("failed to extract routing rule destination port"), err)
		}

		cluster := buildOutboundCluster(destination, port, tags)
		clusters = append(clusters, &WeightedClusterEntry{
			Name:   cluster.Name,
			Weight: int(dst.Weight),
		})
		route.clusters = append(route.clusters, cluster)
	}
	route.WeightedClusters = &WeightedCluster{Clusters: clusters}

	// rewrite to a single cluster if it's one weighted cluster
	if len(rule.Route) == 1 {
		route.Cluster = route.WeightedClusters.Clusters[0].Name
		route.WeightedClusters = nil
	}

	// Ensure all destination clusters have the same port number.
	//
	// This is currently required for doing host header rewrite (host:port),
	// which is scoped to the entire route.
	// This restriction can be relaxed by constructing multiple envoy.Route objects
	// per config.RouteRule, and doing weighted load balancing using Runtime.
	portSet := make(map[int]struct{}, 1)
	for _, cluster := range route.clusters {
		portSet[cluster.port.Port] = struct{}{}
	}
	if len(portSet) > 1 {
		return nil, fmt.Errorf("unsupported multiple destination ports per ingress route rule")
	}

	// Rewrite the host header so that inbound proxies can match incoming traffic
	route.HostRewrite = fmt.Sprintf("%s:%d", rule.Destination, route.clusters[0].port.Port)

	return route, nil
}

// extractPortAndTags extracts the destination service port from the given destination,
// as well as its tags (after clearing meta-tags describing the port).
// Note that this is a temporary measure to communicate the destination service's port
// to the proxy configuration generator. This can be improved by using
// a dedicated model object for IngressRule (instead of reusing RouteRule),
// which exposes the necessary target port field within the "Route" field.
func extractPortAndTags(dst *config.DestinationWeight) (*model.Port, model.Tags, error) {
	portNum, err := strconv.Atoi(dst.Tags["servicePort.port"])
	if err != nil {
		return nil, nil, err
	}
	portName, ok := dst.Tags["servicePort.name"]
	if !ok {
		return nil, nil, fmt.Errorf("no name specified for service port %d", portNum)
	}
	portProto, ok := dst.Tags["servicePort.protocol"]
	if !ok {
		return nil, nil, fmt.Errorf("no protocol specified for service port %d", portNum)
	}

	port := &model.Port{
		Port:     portNum,
		Name:     portName,
		Protocol: model.Protocol(portProto),
	}

	var tags model.Tags
	if len(dst.Tags) > 3 {
		tags = make(model.Tags, len(dst.Tags)-3)
		for k, v := range dst.Tags {
			tags[k] = v
		}
		delete(tags, "servicePort.port")
		delete(tags, "servicePort.name")
		delete(tags, "servicePort.protocol")
	}

	return port, tags, nil
}
<|endoftext|>


<commit_before>// +build darwin dragonfly freebsd linux netbsd openbsd solaris

package proxy

import (
	"encoding/binary"
	"errors"
	"net"
	"strconv"
	"syscall"
	"unsafe"

	"github.com/coyove/goflyway/pkg/fd"
)

func vpnDial(address string) (net.Conn, error) {
	sock, err := fd.Socket(syscall.AF_INET)
	if err != nil {
		return nil, err
	}

	// Send our conn fd to shadowsocks vpn thread for protection
	if err := protectFD(sock); err != nil {
		return nil, err
	}

	// If succeeded, this fd will be closed while we still need it.
	// So we dial a new conn, replace its fd with this one
	return fd.DialWithFD(sock, address)
}

func protectFD(fd int) error {
	sock, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return err
	}

	var addr syscall.SockaddrUnix
	addr.Name = "protect_path"

	if err := (syscall.Connect(sock, &addr)); err != nil {
		return err
	}

	if err := sendFD(sock, fd); err != nil {
		return err
	}

	ret := []byte{9}
	if n, err := syscall.Read(sock, ret); err != nil {
		return err
	} else if n != 1 {
		return errors.New("protecting failed")
	}

	syscall.Close(sock)

	if ret[0] != 0 {
		return errors.New("protecting failed")
	}

	return nil
}

func sendTrafficStats(stat *trafficSurvey) error {
	sock, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return err
	}

	var addr syscall.SockaddrUnix
	addr.Name = "stat_path"

	if err := (syscall.Connect(sock, &addr)); err != nil {
		return err
	}

	payload := make([]byte, 16)
	binary.LittleEndian.PutUint64(payload, stat.totalSent)
	binary.LittleEndian.PutUint64(payload[8:], stat.totalRecved)

	ret := []byte{9}
	if n, err := syscall.Read(sock, ret); err != nil {
		return err
	} else if n != 1 {
		return errors.New("sending traffic stats failed")
	}

	syscall.Close(sock)

	if ret[0] != 0 {
		return errors.New("sending traffic stats failed")
	}

	return nil
}

var _int_value_one int = 1
var _little_endian = *(*byte)(unsafe.Pointer(&_int_value_one)) == 1

func sendFD(sock int, fd int) error {
	cmsg := &syscall.Cmsghdr{
		Level: syscall.SOL_SOCKET,
		Type:  syscall.SCM_RIGHTS,
	}

	const hdrsize = syscall.SizeofCmsghdr
	ln := byte(hdrsize + strconv.IntSize/8)
	h := (*[8]byte)(unsafe.Pointer(&cmsg.Len))

	if _little_endian {
		h[0] = ln
	} else {
		h[3+4*(^cmsg.Len<<32>>63)] = ln
	}

	buffer := make([]byte, cmsg.Len)

	copy(buffer, (*[hdrsize]byte)(unsafe.Pointer(cmsg))[:])
	*(*int)(unsafe.Pointer(&buffer[hdrsize])) = fd

	return syscall.Sendmsg(sock, []byte{'!'}, buffer, nil, 0)
}
<commit_msg>Fix a silly bug<commit_after>// +build darwin dragonfly freebsd linux netbsd openbsd solaris

package proxy

import (
	"encoding/binary"
	"errors"
	"net"
	"strconv"
	"syscall"
	"unsafe"

	"github.com/coyove/goflyway/pkg/fd"
)

func vpnDial(address string) (net.Conn, error) {
	sock, err := fd.Socket(syscall.AF_INET)
	if err != nil {
		return nil, err
	}

	// Send our conn fd to shadowsocks vpn thread for protection
	if err := protectFD(sock); err != nil {
		return nil, err
	}

	// If succeeded, this fd will be closed while we still need it.
	// So we dial a new conn, replace its fd with this one
	return fd.DialWithFD(sock, address)
}

func protectFD(fd int) error {
	sock, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return err
	}

	var addr syscall.SockaddrUnix
	addr.Name = "protect_path"

	if err := (syscall.Connect(sock, &addr)); err != nil {
		return err
	}

	if err := sendFD(sock, fd); err != nil {
		return err
	}

	ret := []byte{9}
	if n, err := syscall.Read(sock, ret); err != nil {
		return err
	} else if n != 1 {
		return errors.New("protecting failed")
	}

	syscall.Close(sock)

	if ret[0] != 0 {
		return errors.New("protecting failed")
	}

	return nil
}

func sendTrafficStats(stat *trafficSurvey) error {
	const errm = "sending traffic stats failed"

	sock, err := syscall.Socket(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return err
	}

	var addr syscall.SockaddrUnix
	addr.Name = "stat_path"

	if err := (syscall.Connect(sock, &addr)); err != nil {
		return err
	}

	payload := make([]byte, 16)
	binary.LittleEndian.PutUint64(payload, stat.totalSent)
	binary.LittleEndian.PutUint64(payload[8:], stat.totalRecved)

	if n, err := syscall.Write(sock, payload); err != nil {
		return err
	} else if n != 16 {
		return errors.New(errm)
	}

	ret := []byte{9}
	if n, err := syscall.Read(sock, ret); err != nil {
		return err
	} else if n != 1 {
		return errors.New(errm)
	}

	syscall.Close(sock)

	if ret[0] != 0 {
		return errors.New(errm)
	}

	return nil
}

var _int_value_one int = 1
var _little_endian = *(*byte)(unsafe.Pointer(&_int_value_one)) == 1

func sendFD(sock int, fd int) error {
	cmsg := &syscall.Cmsghdr{
		Level: syscall.SOL_SOCKET,
		Type:  syscall.SCM_RIGHTS,
	}

	const hdrsize = syscall.SizeofCmsghdr
	ln := byte(hdrsize + strconv.IntSize/8)
	h := (*[8]byte)(unsafe.Pointer(&cmsg.Len))

	if _little_endian {
		h[0] = ln
	} else {
		h[3+4*(^cmsg.Len<<32>>63)] = ln
	}

	buffer := make([]byte, cmsg.Len)

	copy(buffer, (*[hdrsize]byte)(unsafe.Pointer(cmsg))[:])
	*(*int)(unsafe.Pointer(&buffer[hdrsize])) = fd

	return syscall.Sendmsg(sock, []byte{'!'}, buffer, nil, 0)
}
<|endoftext|>


<commit_before>package clang_test

import (
	"strings"
	"testing"

	"github.com/sbinet/go-clang"
)

func TestCompleteAt(t *testing.T) {
	idx := clang.NewIndex(0, 0)
	defer idx.Dispose()
	tu := idx.Parse("visitorwrap.c", nil, nil, 0)
	if !tu.IsValid() {
		t.Fatal("TranslationUnit is not valid")
	}
	defer tu.Dispose()

	res := tu.CompleteAt("visitorwrap.c", 11, 16, nil, 0)
	if !res.IsValid() {
		t.Fatal("CompleteResults are not valid")
	}
	defer res.Dispose()
	if n := len(res.Results()); n < 10 {
		t.Errorf("Expected more results than %d", n)
	}
	t.Logf("%+v", res)
	for _, r := range res.Results() {
		t.Logf("%+v", r)
		for _, c := range r.CompletionString.Chunks() {
			t.Logf("\t%+v", c)
		}
	}

	diags := res.Diagnostics()
	defer diags.Dispose()
	ok := false
	for _, d := range diags {
		if strings.Contains(d.Spelling(), "_cgo_export.h") {
			ok = true
		}
		t.Log(d.Severity(), d.Spelling())
	}
	if !ok {
		t.Errorf("Expected to find a diagnostic regarding _cgo_export.h")
	}
}
<commit_msg>test: document what we complete-at<commit_after>package clang_test

import (
	"strings"
	"testing"

	"github.com/sbinet/go-clang"
)

func TestCompleteAt(t *testing.T) {
	idx := clang.NewIndex(0, 0)
	defer idx.Dispose()
	tu := idx.Parse("visitorwrap.c", nil, nil, 0)
	if !tu.IsValid() {
		t.Fatal("TranslationUnit is not valid")
	}
	defer tu.Dispose()

	const lineno = 10 // ie: call to clang_visitChildren
	res := tu.CompleteAt("visitorwrap.c", lineno, 16, nil, 0)
	if !res.IsValid() {
		t.Fatal("CompleteResults are not valid")
	}
	defer res.Dispose()
	if n := len(res.Results()); n < 10 {
		t.Errorf("Expected more results than %d", n)
	}
	t.Logf("%+v", res)
	for _, r := range res.Results() {
		t.Logf("%+v", r)
		for _, c := range r.CompletionString.Chunks() {
			t.Logf("\t%+v", c)
		}
	}

	diags := res.Diagnostics()
	defer diags.Dispose()
	ok := false
	for _, d := range diags {
		if strings.Contains(d.Spelling(), "_cgo_export.h") {
			ok = true
		}
		t.Log(d.Severity(), d.Spelling())
	}
	if !ok {
		t.Errorf("Expected to find a diagnostic regarding _cgo_export.h")
	}
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package output

import (
	"encoding/json"
	"strings"

	"github.com/r3labs/graph"
	"github.com/r3labs/workflow"
)

// FSMMessage is the JSON payload that will be sent to the FSM to create a
// service.
type FSMMessage struct {
	ID            string   `json:"id"`
	Body          string   `json:"body"`
	Endpoint      string   `json:"endpoint"`
	Service       string   `json:"service"`
	Bootstrapping string   `json:"bootstrapping"`
	ErnestIP      []string `json:"ernest_ip"`
	ServiceIP     string   `json:"service_ip"`
	Parent        string   `json:"existing_service"`
	Workflow      struct {
		Arcs []graph.Edge `json:"arcs"`
	} `json:"workflow"`
	ServiceName string `json:"name"`
	Client      string `json:"client"` // TODO: Use client or client_id not both!
	ClientID    string `json:"client_id"`
	ClientName  string `json:"client_name"`
	Started     string `json:"started"`
	Finished    string `json:"finished"`
	Status      string `json:"status"`
	Type        string `json:"type"`
	Datacenters struct {
		Started  string       `json:"started"`
		Finished string       `json:"finished"`
		Status   string       `json:"status"`
		Items    []Datacenter `json:"items"`
	} `json:"datacenters"`
	Networks struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks"`
	NetworksToCreate struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks_to_create"`
	NetworksToDelete struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks_to_delete"`
	Instances struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances"`
	InstancesToCreate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_create"`
	InstancesToUpdate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_update"`
	InstancesToDelete struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_delete"`
	Firewalls struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls"`
	FirewallsToCreate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_create"`
	FirewallsToUpdate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_update"`
	FirewallsToDelete struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_delete"`
	Nats struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats"`
	NatsToCreate struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_create"`
	NatsToDelete struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_delete"`
}

// Diff compares against an existing FSMMessage from a previous fsm message
func (m *FSMMessage) Diff(om FSMMessage) {
	// build new networks
	for i, network := range m.Networks.Items {
		if o := om.FindNetwork(network.Name); o == nil {
			m.NetworksToCreate.Items = append(m.NetworksToCreate.Items, network)
		} else {
			m.Networks.Items[i] = *o
		}
	}

	// build new and existing instances
	for i, instance := range m.Instances.Items {
		if oi := om.FindInstance(instance.Name); oi == nil {
			m.InstancesToCreate.Items = append(m.InstancesToCreate.Items, instance)
		} else if instance.HasChanged(oi) {
			m.InstancesToUpdate.Items = append(m.InstancesToUpdate.Items, instance)
		} else {
			m.Instances.Items[i] = *oi
		}
	}

	// build old instances to delete
	for _, instance := range om.Instances.Items {
		if m.FindInstance(instance.Name) == nil {
			m.InstancesToDelete.Items = append(m.InstancesToDelete.Items, instance)
		}
	}

	// build new and existing firewalls
	for i, firewall := range m.Firewalls.Items {
		if of := om.FindFirewall(firewall.Name); of == nil {
			m.FirewallsToCreate.Items = append(m.FirewallsToCreate.Items, firewall)
		} else if firewall.HasChanged(of) {
			m.FirewallsToUpdate.Items = append(m.FirewallsToUpdate.Items, firewall)
		} else {
			m.Firewalls.Items[i] = *of
		}
	}

	// build old firewalls to delete
	for _, firewall := range om.Firewalls.Items {
		if m.FindFirewall(firewall.Name) == nil {
			m.FirewallsToDelete.Items = append(m.FirewallsToDelete.Items, firewall)
		}
	}

	// build new and existing nats
	for i, nat := range m.Nats.Items {
		if on := om.FindNat(nat.Name); on == nil {
			m.NatsToCreate.Items = append(m.NatsToCreate.Items, nat)
		} else {
			m.Nats.Items[i] = *on
		}
	}

	// build old nats to delete
	for _, nat := range om.Nats.Items {
		if m.FindNat(nat.Name) == nil {
			m.NatsToDelete.Items = append(m.NatsToDelete.Items, nat)
		}
	}
}

// GenerateWorkflow creates a fsm workflow based upon actionable tasks, such as creation or deletion of an entity.
func (m *FSMMessage) GenerateWorkflow(path string) error {
	w := workflow.New()
	err := w.LoadFile("./output/arcs/" + path)
	if err != nil {
		return err
	}

	// Set network items
	w.SetCount("creating_networks", len(m.NetworksToCreate.Items))
	w.SetCount("networks_created", len(m.NetworksToCreate.Items))
	w.SetCount("deleting_networks", len(m.NetworksToDelete.Items))
	w.SetCount("networks_deleted", len(m.NetworksToDelete.Items))

	// Set instance items
	w.SetCount("creating_instances", len(m.InstancesToCreate.Items))
	w.SetCount("instances_created", len(m.InstancesToCreate.Items))
	w.SetCount("updating_instances", len(m.InstancesToUpdate.Items))
	w.SetCount("instances_updated", len(m.InstancesToUpdate.Items))
	w.SetCount("deleting_instances", len(m.InstancesToDelete.Items))
	w.SetCount("instances_deleted", len(m.InstancesToDelete.Items))

	// Set firewall items
	w.SetCount("creating_firewalls", len(m.FirewallsToCreate.Items))
	w.SetCount("firewalls_created", len(m.FirewallsToCreate.Items))
	w.SetCount("updating_firewalls", len(m.FirewallsToUpdate.Items))
	w.SetCount("firewalls_updated", len(m.FirewallsToUpdate.Items))
	w.SetCount("deleting_firewalls", len(m.FirewallsToDelete.Items))
	w.SetCount("firewalls_deleted", len(m.FirewallsToDelete.Items))

	// Set nat items
	w.SetCount("creating_nats", len(m.NatsToCreate.Items))
	w.SetCount("nats_created", len(m.NatsToCreate.Items))
	w.SetCount("deleting_nats", len(m.NatsToDelete.Items))
	w.SetCount("nats_deleted", len(m.NatsToDelete.Items))

	// Optimize the graph, removing unused arcs/verticies
	w.Optimize()

	m.Workflow.Arcs = w.Arcs()

	return nil
}

// FindNetwork returns true if a router with a given name exists
func (m *FSMMessage) FindNetwork(name string) *Network {
	for i, network := range m.Networks.Items {
		if network.Name == name {
			return &m.Networks.Items[i]
		}
	}
	return nil
}

// FindInstance returns true if a router with a given name exists
func (m *FSMMessage) FindInstance(name string) *Instance {
	for i, instance := range m.Instances.Items {
		if instance.Name == name {
			return &m.Instances.Items[i]
		}
	}
	return nil
}

// FindFirewall returns true if a router with a given name exists
func (m *FSMMessage) FindFirewall(name string) *Firewall {
	for i, firewall := range m.Firewalls.Items {
		if firewall.Name == name {
			return &m.Firewalls.Items[i]
		}
	}
	return nil
}

// FindNat returns true if a router with a given name exists
func (m *FSMMessage) FindNat(name string) *Nat {
	for i, nat := range m.Nats.Items {
		if nat.Name == name {
			return &m.Nats.Items[i]
		}
	}
	return nil
}

// FilterNewInstances will return any new instances that match a certain pattern
func (m *FSMMessage) FilterNewInstances(name string) []Instance {
	var instances []Instance
	for _, instance := range m.InstancesToCreate.Items {
		if strings.Contains(instance.Name, name) {
			instances = append(instances, instance)
		}
	}
	return instances
}

// ToJSON : Get this service as a json
func (m *FSMMessage) ToJSON() []byte {
	json, _ := json.Marshal(m)

	return json
}
<commit_msg>Map networks to delete<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package output

import (
	"encoding/json"
	"strings"

	"github.com/r3labs/graph"
	"github.com/r3labs/workflow"
)

// FSMMessage is the JSON payload that will be sent to the FSM to create a
// service.
type FSMMessage struct {
	ID            string   `json:"id"`
	Body          string   `json:"body"`
	Endpoint      string   `json:"endpoint"`
	Service       string   `json:"service"`
	Bootstrapping string   `json:"bootstrapping"`
	ErnestIP      []string `json:"ernest_ip"`
	ServiceIP     string   `json:"service_ip"`
	Parent        string   `json:"existing_service"`
	Workflow      struct {
		Arcs []graph.Edge `json:"arcs"`
	} `json:"workflow"`
	ServiceName string `json:"name"`
	Client      string `json:"client"` // TODO: Use client or client_id not both!
	ClientID    string `json:"client_id"`
	ClientName  string `json:"client_name"`
	Started     string `json:"started"`
	Finished    string `json:"finished"`
	Status      string `json:"status"`
	Type        string `json:"type"`
	Datacenters struct {
		Started  string       `json:"started"`
		Finished string       `json:"finished"`
		Status   string       `json:"status"`
		Items    []Datacenter `json:"items"`
	} `json:"datacenters"`
	Networks struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks"`
	NetworksToCreate struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks_to_create"`
	NetworksToDelete struct {
		Started  string    `json:"started"`
		Finished string    `json:"finished"`
		Status   string    `json:"status"`
		Items    []Network `json:"items"`
	} `json:"networks_to_delete"`
	Instances struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances"`
	InstancesToCreate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_create"`
	InstancesToUpdate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_update"`
	InstancesToDelete struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Instance `json:"items"`
	} `json:"instances_to_delete"`
	Firewalls struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls"`
	FirewallsToCreate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_create"`
	FirewallsToUpdate struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_update"`
	FirewallsToDelete struct {
		Started  string     `json:"started"`
		Finished string     `json:"finished"`
		Status   string     `json:"status"`
		Items    []Firewall `json:"items"`
	} `json:"firewalls_to_delete"`
	Nats struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats"`
	NatsToCreate struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_create"`
	NatsToDelete struct {
		Started  string `json:"started"`
		Finished string `json:"finished"`
		Status   string `json:"status"`
		Items    []Nat  `json:"items"`
	} `json:"nats_to_delete"`
}

// Diff compares against an existing FSMMessage from a previous fsm message
func (m *FSMMessage) Diff(om FSMMessage) {
	// build new networks
	for i, network := range m.Networks.Items {
		if o := om.FindNetwork(network.Name); o == nil {
			m.NetworksToCreate.Items = append(m.NetworksToCreate.Items, network)
		} else {
			m.Networks.Items[i] = *o
		}
	}

	// build old networks to delete
	for _, network := range om.Networks.Items {
		if m.FindNetwork(network.Name) == nil {
			m.NetworksToDelete.Items = append(m.NetworksToDelete.Items, network)
		}
	}

	// build new and existing instances
	for i, instance := range m.Instances.Items {
		if oi := om.FindInstance(instance.Name); oi == nil {
			m.InstancesToCreate.Items = append(m.InstancesToCreate.Items, instance)
		} else if instance.HasChanged(oi) {
			m.InstancesToUpdate.Items = append(m.InstancesToUpdate.Items, instance)
		} else {
			m.Instances.Items[i] = *oi
		}
	}

	// build old instances to delete
	for _, instance := range om.Instances.Items {
		if m.FindInstance(instance.Name) == nil {
			m.InstancesToDelete.Items = append(m.InstancesToDelete.Items, instance)
		}
	}

	// build new and existing firewalls
	for i, firewall := range m.Firewalls.Items {
		if of := om.FindFirewall(firewall.Name); of == nil {
			m.FirewallsToCreate.Items = append(m.FirewallsToCreate.Items, firewall)
		} else if firewall.HasChanged(of) {
			m.FirewallsToUpdate.Items = append(m.FirewallsToUpdate.Items, firewall)
		} else {
			m.Firewalls.Items[i] = *of
		}
	}

	// build old firewalls to delete
	for _, firewall := range om.Firewalls.Items {
		if m.FindFirewall(firewall.Name) == nil {
			m.FirewallsToDelete.Items = append(m.FirewallsToDelete.Items, firewall)
		}
	}

	// build new and existing nats
	for i, nat := range m.Nats.Items {
		if on := om.FindNat(nat.Name); on == nil {
			m.NatsToCreate.Items = append(m.NatsToCreate.Items, nat)
		} else {
			m.Nats.Items[i] = *on
		}
	}

	// build old nats to delete
	for _, nat := range om.Nats.Items {
		if m.FindNat(nat.Name) == nil {
			m.NatsToDelete.Items = append(m.NatsToDelete.Items, nat)
		}
	}
}

// GenerateWorkflow creates a fsm workflow based upon actionable tasks, such as creation or deletion of an entity.
func (m *FSMMessage) GenerateWorkflow(path string) error {
	w := workflow.New()
	err := w.LoadFile("./output/arcs/" + path)
	if err != nil {
		return err
	}

	// Set network items
	w.SetCount("creating_networks", len(m.NetworksToCreate.Items))
	w.SetCount("networks_created", len(m.NetworksToCreate.Items))
	w.SetCount("deleting_networks", len(m.NetworksToDelete.Items))
	w.SetCount("networks_deleted", len(m.NetworksToDelete.Items))

	// Set instance items
	w.SetCount("creating_instances", len(m.InstancesToCreate.Items))
	w.SetCount("instances_created", len(m.InstancesToCreate.Items))
	w.SetCount("updating_instances", len(m.InstancesToUpdate.Items))
	w.SetCount("instances_updated", len(m.InstancesToUpdate.Items))
	w.SetCount("deleting_instances", len(m.InstancesToDelete.Items))
	w.SetCount("instances_deleted", len(m.InstancesToDelete.Items))

	// Set firewall items
	w.SetCount("creating_firewalls", len(m.FirewallsToCreate.Items))
	w.SetCount("firewalls_created", len(m.FirewallsToCreate.Items))
	w.SetCount("updating_firewalls", len(m.FirewallsToUpdate.Items))
	w.SetCount("firewalls_updated", len(m.FirewallsToUpdate.Items))
	w.SetCount("deleting_firewalls", len(m.FirewallsToDelete.Items))
	w.SetCount("firewalls_deleted", len(m.FirewallsToDelete.Items))

	// Set nat items
	w.SetCount("creating_nats", len(m.NatsToCreate.Items))
	w.SetCount("nats_created", len(m.NatsToCreate.Items))
	w.SetCount("deleting_nats", len(m.NatsToDelete.Items))
	w.SetCount("nats_deleted", len(m.NatsToDelete.Items))

	// Optimize the graph, removing unused arcs/verticies
	w.Optimize()

	m.Workflow.Arcs = w.Arcs()

	return nil
}

// FindNetwork returns true if a router with a given name exists
func (m *FSMMessage) FindNetwork(name string) *Network {
	for i, network := range m.Networks.Items {
		if network.Name == name {
			return &m.Networks.Items[i]
		}
	}
	return nil
}

// FindInstance returns true if a router with a given name exists
func (m *FSMMessage) FindInstance(name string) *Instance {
	for i, instance := range m.Instances.Items {
		if instance.Name == name {
			return &m.Instances.Items[i]
		}
	}
	return nil
}

// FindFirewall returns true if a router with a given name exists
func (m *FSMMessage) FindFirewall(name string) *Firewall {
	for i, firewall := range m.Firewalls.Items {
		if firewall.Name == name {
			return &m.Firewalls.Items[i]
		}
	}
	return nil
}

// FindNat returns true if a router with a given name exists
func (m *FSMMessage) FindNat(name string) *Nat {
	for i, nat := range m.Nats.Items {
		if nat.Name == name {
			return &m.Nats.Items[i]
		}
	}
	return nil
}

// FilterNewInstances will return any new instances that match a certain pattern
func (m *FSMMessage) FilterNewInstances(name string) []Instance {
	var instances []Instance
	for _, instance := range m.InstancesToCreate.Items {
		if strings.Contains(instance.Name, name) {
			instances = append(instances, instance)
		}
	}
	return instances
}

// ToJSON : Get this service as a json
func (m *FSMMessage) ToJSON() []byte {
	json, _ := json.Marshal(m)

	return json
}
<|endoftext|>