<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/dispatcher"
	"go.chromium.org/luci/common/sync/dispatcher/buffer"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/hardcoded/chromeinfra"
	"golang.org/x/time/rate"

	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"
)

var (
	errInvalidConfig = errors.New("invalid configuration")
	errNoWorkToDo    = errors.New("no work to do")

	leaseRetryParams = func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   time.Second,
				Retries: -1,
			},
			Multiplier: 1.25,
			MaxDelay:   time.Minute * 10,
		}
	}

	ackChannelOptions = &dispatcher.Options{
		QPSLimit: rate.NewLimiter(1, 1), // 1 QPS max rate
		Buffer: buffer.Options{
			MaxLeases:     10,
			BatchSize:     500,
			BatchDuration: 10 * time.Minute,
			FullBehavior: &buffer.BlockNewItems{
				MaxItems: 10 * 500,
			},
			Retry: func() retry.Iterator {
				return &retry.ExponentialBackoff{
					Limited: retry.Limited{
						Delay:   time.Second,
						Retries: 10,
					},
					Multiplier: 1.25,
					MaxDelay:   time.Minute * 10,
				}
			},
		},
	}

	mkJobChannelOptions = func(maxWorkers int) *dispatcher.Options {
		return &dispatcher.Options{
			Buffer: buffer.Options{
				MaxLeases: maxWorkers,
				BatchSize: 1,
				FullBehavior: &buffer.BlockNewItems{
					MaxItems: 2 * maxWorkers,
				},
			},
		}
	}

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service

	maxConcurrentTasks int
}

// runForever runs the archivist loop forever.
func runForever(ctx context.Context, taskConcurrency int, ar archivist.Archivist) {
	type archiveJob struct {
		deadline time.Time
		task     *logdog.ArchiveTask
	}

	ackChan, err := dispatcher.NewChannel(ctx, ackChannelOptions, func(batch *buffer.Batch) error {
		var req *logdog.DeleteRequest
		if batch.Meta != nil {
			req = batch.Meta.(*logdog.DeleteRequest)
		} else {
			tasks := make([]*logdog.ArchiveTask, len(batch.Data))
			for i, datum := range batch.Data {
				tasks[i] = datum.(*logdog.ArchiveTask)
				batch.Data[i] = nil
			}
			req = &logdog.DeleteRequest{Tasks: tasks}
			batch.Meta = req
		}
		_, err := ar.Service.DeleteArchiveTasks(ctx, req)
		return transient.Tag.Apply(err)
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "draining ACK channel")
		ackChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "ACK channel drained")
	}()

	jobChanOpts := mkJobChannelOptions(taskConcurrency)
	jobChan, err := dispatcher.NewChannel(ctx, jobChanOpts, func(data *buffer.Batch) error {
		job := data.Data[0].(*archiveJob)

		nc, cancel := context.WithDeadline(ctx, job.deadline)
		defer cancel()

		startTime := clock.Now(ctx)
		err := ar.ArchiveTask(nc, job.task)
		duration := clock.Now(ctx).Sub(startTime)
		tsTaskProcessingTime.Add(ctx, float64(duration.Nanoseconds())/1000000, err == nil)

		if err == nil {
			select {
			case ackChan.C <- job:
			case <-ctx.Done():
				logging.Errorf(ctx, "Failed to ACK task %v due to context: %s", job.task, ctx.Err())
			}
		} else {
			logging.Errorf(ctx, "Failed to archive task %v: %s", job.task, err)
		}

		return nil
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "Job channel draining")
		jobChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "Job channel drained")
	}()

	// now we spin forever, pushing items into jobChan.
	sleepTime := 1
	var previousCycle time.Time
	for ctx.Err() == nil {
		loopParams := grabLoopParams()
		req := loopParams.mkRequest(ctx)

		var tasks *logdog.LeaseResponse
		var deadline time.Time

		logging.Infof(ctx, "Leasing max %d tasks for %s", loopParams.batchSize, loopParams.deadline)
		err := retry.Retry(ctx, leaseRetryParams, func() (err error) {
			deadline = clock.Now(ctx).Add(loop.deadline)
			tasks, err = ar.Service.LeaseArchiveTasks(ctx, req)
			return
		}, retry.LogCallback(ctx, "LeaseArchiveTasks"))
		if ctx.Err() != nil && err != nil {
			panic("impossible: infinite retry stopped: " + err.Error())
		}

		if !previousCycle.IsZero() {
			now := clock.Now(ctx)
			tsLoopCycleTime.Add(ctx, float64(now.Sub(previousCycle).Nanoseconds()/1000000))
			previousCycle = now
		}

		if len(tasks.Tasks) == 0 {
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			logging.Infof(ctx, "no work to do, sleeping for %d seconds", sleepTime)
			clock.Sleep(ctx, time.Duration(sleepTime)*time.Second)
			previousCycle = time.Time{}
			continue
		} else {
			sleepTime = 1
		}

		for _, task := range tasks.Tasks {
			select {
			case jobChan.C <- &archiveJob{deadline, task}:
			case <-ctx.Done():
				logging.Infof(ctx, "lease thread got context err: %s", ctx.Err())
				break
			}
		}
	}

	logging.Infof(ctx, "runForever no longer running forever: ctx.Err() == %s", ctx.Err())
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, project string) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, project)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Load our settings and update them periodically.
	fetchLoopParams(c)
	go loopParamsUpdater(c)

	runForever(c, a.maxConcurrentTasks, ar)

	return nil
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Flags.IntVar(&a.maxConcurrentTasks, "max-concurrent-tasks", 1,
		"Maximum number of archive tasks to process concurrently. "+
			"Pass 0 to set infinite limit.")
	a.Run(context.Background(), a.runArchivist)
}
<commit_msg>[logdog] Fix typesafety issue.<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/dispatcher"
	"go.chromium.org/luci/common/sync/dispatcher/buffer"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/hardcoded/chromeinfra"
	"golang.org/x/time/rate"

	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"
)

var (
	errInvalidConfig = errors.New("invalid configuration")
	errNoWorkToDo    = errors.New("no work to do")

	leaseRetryParams = func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   time.Second,
				Retries: -1,
			},
			Multiplier: 1.25,
			MaxDelay:   time.Minute * 10,
		}
	}

	ackChannelOptions = &dispatcher.Options{
		QPSLimit: rate.NewLimiter(1, 1), // 1 QPS max rate
		Buffer: buffer.Options{
			MaxLeases:     10,
			BatchSize:     500,
			BatchDuration: 10 * time.Minute,
			FullBehavior: &buffer.BlockNewItems{
				MaxItems: 10 * 500,
			},
			Retry: func() retry.Iterator {
				return &retry.ExponentialBackoff{
					Limited: retry.Limited{
						Delay:   time.Second,
						Retries: 10,
					},
					Multiplier: 1.25,
					MaxDelay:   time.Minute * 10,
				}
			},
		},
	}

	mkJobChannelOptions = func(maxWorkers int) *dispatcher.Options {
		return &dispatcher.Options{
			Buffer: buffer.Options{
				MaxLeases: maxWorkers,
				BatchSize: 1,
				FullBehavior: &buffer.BlockNewItems{
					MaxItems: 2 * maxWorkers,
				},
			},
		}
	}

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service

	maxConcurrentTasks int
}

// runForever runs the archivist loop forever.
func runForever(ctx context.Context, taskConcurrency int, ar archivist.Archivist) {
	type archiveJob struct {
		deadline time.Time
		task     *logdog.ArchiveTask
	}

	ackChan, err := dispatcher.NewChannel(ctx, ackChannelOptions, func(batch *buffer.Batch) error {
		var req *logdog.DeleteRequest
		if batch.Meta != nil {
			req = batch.Meta.(*logdog.DeleteRequest)
		} else {
			tasks := make([]*logdog.ArchiveTask, len(batch.Data))
			for i, datum := range batch.Data {
				tasks[i] = datum.(*logdog.ArchiveTask)
				batch.Data[i] = nil
			}
			req = &logdog.DeleteRequest{Tasks: tasks}
			batch.Meta = req
		}
		_, err := ar.Service.DeleteArchiveTasks(ctx, req)
		return transient.Tag.Apply(err)
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "draining ACK channel")
		ackChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "ACK channel drained")
	}()

	jobChanOpts := mkJobChannelOptions(taskConcurrency)
	jobChan, err := dispatcher.NewChannel(ctx, jobChanOpts, func(data *buffer.Batch) error {
		job := data.Data[0].(*archiveJob)

		nc, cancel := context.WithDeadline(ctx, job.deadline)
		defer cancel()

		startTime := clock.Now(ctx)
		err := ar.ArchiveTask(nc, job.task)
		duration := clock.Now(ctx).Sub(startTime)
		tsTaskProcessingTime.Add(ctx, float64(duration.Nanoseconds())/1000000, err == nil)

		if err == nil {
			select {
			case ackChan.C <- job.task:
			case <-ctx.Done():
				logging.Errorf(ctx, "Failed to ACK task %v due to context: %s", job.task, ctx.Err())
			}
		} else {
			logging.Errorf(ctx, "Failed to archive task %v: %s", job.task, err)
		}

		return nil
	})
	if err != nil {
		panic(err) // only occurs if Options is invalid
	}
	defer func() {
		logging.Infof(ctx, "Job channel draining")
		jobChan.CloseAndDrain(ctx)
		logging.Infof(ctx, "Job channel drained")
	}()

	// now we spin forever, pushing items into jobChan.
	sleepTime := 1
	var previousCycle time.Time
	for ctx.Err() == nil {
		loopParams := grabLoopParams()
		req := loopParams.mkRequest(ctx)

		var tasks *logdog.LeaseResponse
		var deadline time.Time

		logging.Infof(ctx, "Leasing max %d tasks for %s", loopParams.batchSize, loopParams.deadline)
		err := retry.Retry(ctx, leaseRetryParams, func() (err error) {
			deadline = clock.Now(ctx).Add(loop.deadline)
			tasks, err = ar.Service.LeaseArchiveTasks(ctx, req)
			return
		}, retry.LogCallback(ctx, "LeaseArchiveTasks"))
		if ctx.Err() != nil && err != nil {
			panic("impossible: infinite retry stopped: " + err.Error())
		}

		if !previousCycle.IsZero() {
			now := clock.Now(ctx)
			tsLoopCycleTime.Add(ctx, float64(now.Sub(previousCycle).Nanoseconds()/1000000))
			previousCycle = now
		}

		if len(tasks.Tasks) == 0 {
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			logging.Infof(ctx, "no work to do, sleeping for %d seconds", sleepTime)
			clock.Sleep(ctx, time.Duration(sleepTime)*time.Second)
			previousCycle = time.Time{}
			continue
		} else {
			sleepTime = 1
		}

		for _, task := range tasks.Tasks {
			select {
			case jobChan.C <- &archiveJob{deadline, task}:
			case <-ctx.Done():
				logging.Infof(ctx, "lease thread got context err: %s", ctx.Err())
				break
			}
		}
	}

	logging.Infof(ctx, "runForever no longer running forever: ctx.Err() == %s", ctx.Err())
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, project string) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, project)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Load our settings and update them periodically.
	fetchLoopParams(c)
	go loopParamsUpdater(c)

	runForever(c, a.maxConcurrentTasks, ar)

	return nil
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Flags.IntVar(&a.maxConcurrentTasks, "max-concurrent-tasks", 1,
		"Maximum number of archive tasks to process concurrently. "+
			"Pass 0 to set infinite limit.")
	a.Run(context.Background(), a.runArchivist)
}
<|endoftext|>


<commit_before>// +build noos

package syscall

import (
	"bits"
	"internal"
	"unsafe"
)

const (
	NEWTASK    = internal.NEWTASK
	KILLTASK   = internal.KILLTASK
	TASKUNLOCK = internal.TASKUNLOCK
	MAXTASKS   = iota
	SCHEDNEXT
	EVENTWAIT
	SETSYSTIM
	NANOSEC
	SETALARM
	SETAT
	SETIRQENA
	SETIRQPRIO
	SETIRQHANDLER
	IRQSTATUS
	TRIGGERIRQ
	SETPRIVLEVEL
	DEBUGOUT
)

// NewTask creates new task that starts execute f. If lock is true tasker stops
// scheduling current task and waits until new task will call TaskUnlock. When
// success it returns TID of new task.
func NewTask(f func(), lock bool) (int, Errno) {
	tid, e := internal.Syscall2(NEWTASK, ftou(f), uintptr(bits.One(lock)))
	return int(tid), Errno(e)
}

// KillTask kills task with specified tid. tid == 0 means current task.
func KillTask(tid int) Errno {
	_, e := internal.Syscall1(KILLTASK, uintptr(tid))
	return Errno(e)
}

// TaskUnlock can be used when task was created with lock option. It informs
// tasker that now it can safely run parent task.
func TaskUnlock() {
	internal.Syscall0(TASKUNLOCK)
}

// MaxTasks: see rtos package.
func MaxTasks() int {
	n, _ := internal.Syscall0(MAXTASKS)
	return int(n)
}

// SchedYield causes the calling task to relinquish the CPU.
func SchedYield() {
	internal.Syscall0(SCHEDNEXT)
}

// SchedNext informs tasker that it need to schedule next ready to run task.
// It is safe to call SchedNext from interrupt handler.
func SchedNext() {
	schedNext()
}

// SetSysTimer registers two functions that the runtime uses to communicate with
// the system timer.
//
// Nanosec is used to implement Nanosec system call. It should return the
// monotonic time in nanoseconds (typically the time of system timer run).
//
// SetWakeUp is called by scheduler to ask system timer to wake it up at time t
// (using SchedNext function). T can be a monotonic time in nanoseconds or -1
// if the scheduler does not want to be woken. System timer must guarantee that
// it will wake up the scheduler at t or after t. Additional awakenings before
// and after t are acceptable but not recommended. Ticking timer simply wakes up
// the scheduler with a constant period and its setWakeUp function does nothing.
// Tickless timer schould wake up the scheduler only once, at t or just after t.
func SetSysTimer(nanosec func() int64, setWakeUp func(t int64)) {
	internal.Syscall2(SETSYSTIM, fr64tou(nanosec), f64tou(setWakeUp))
}

// Nanosec: see rtos package.
func Nanosec() int64 {
	return internal.Syscall0r64(NANOSEC)
}

// SetAlarm asks the runtime to send Alarm event at t. T is only a hint for the
// runtime, because it can send alarm at any time: before t, at t and after t.
// Typically, task use SetAlarm in conjunction with Alarm.Wait and Nanosec.
func SetAlarm(t int64) {
	internal.Syscall1i64(SETALARM, t)
}

// SetAt works like SetAlarm but additionaly sets task local variable, used to
// implement internal.TimeChan.
func SetAt(t int64) {
	internal.Syscall1i64(SETAT, t)
}

// TimeChan returns channel that can be used to wait for time set using
// SetAt. TimeChan is used mainly for deadline/timeout in select statements.
func TimeChan() <-chan int64 {
	ch := &internal.TimeChan
	return *(*<-chan int64)(unsafe.Pointer(&ch))
}

// SetIRQEna enables or disables irq.
func SetIRQEna(irq int, ena bool) Errno {
	_, e := internal.Syscall2(SETIRQENA, uintptr(irq), uintptr(bits.One(ena)))
	return Errno(e)
}

// SetIRQPrio sets priority for irq.
func SetIRQPrio(irq, prio int) Errno {
	_, err := internal.Syscall2(SETIRQPRIO, uintptr(irq), uintptr(prio))
	return Errno(err)
}

// SetIRQHandler: see rtos package.
func SetIRQHandler(irq int, f func()) Errno {
	_, e := internal.Syscall2(SETIRQHANDLER, uintptr(irq), ftou(f))
	return Errno(e)
}

// IRQStatus: ee rtos package.
func IRQStatus(irq int) (int, Errno) {
	s, e := internal.Syscall1(IRQSTATUS, uintptr(irq))
	return int(s), Errno(e)
}

// TriggerIRQ: see rtos package.
func TriggerIRQ(irq int) Errno {
	_, e := internal.Syscall1(IRQSTATUS, uintptr(irq))
	return Errno(e)
}

// SetPrivLevel: see rtos package.
func SetPrivLevel(n int) (int, Errno) {
	old, e := internal.Syscall1(SETPRIVLEVEL, uintptr(n))
	return int(old), Errno(e)
}

// DebugOutString allows write debug message.
func DebugOutString(port int, s string) (int, Errno) {
	p := (*internal.String)(unsafe.Pointer(&s))
	n, e := internal.Syscall3(DEBUGOUT, uintptr(port), p.Addr, p.Len)
	return int(n), Errno(e)
}

// DebugOut allows write debug message.
func DebugOut(port int, b []byte) (int, Errno) {
	return DebugOutString(port, *(*string)(unsafe.Pointer(&b)))
}
<commit_msg>syscall: Fix bug in TriggerIRQ.<commit_after>// +build noos

package syscall

import (
	"bits"
	"internal"
	"unsafe"
)

const (
	NEWTASK    = internal.NEWTASK
	KILLTASK   = internal.KILLTASK
	TASKUNLOCK = internal.TASKUNLOCK
	MAXTASKS   = iota
	SCHEDNEXT
	EVENTWAIT
	SETSYSTIM
	NANOSEC
	SETALARM
	SETAT
	SETIRQENA
	SETIRQPRIO
	SETIRQHANDLER
	IRQSTATUS
	TRIGGERIRQ
	SETPRIVLEVEL
	DEBUGOUT
)

// NewTask creates new task that starts execute f. If lock is true tasker stops
// scheduling current task and waits until new task will call TaskUnlock. When
// success it returns TID of new task.
func NewTask(f func(), lock bool) (int, Errno) {
	tid, e := internal.Syscall2(NEWTASK, ftou(f), uintptr(bits.One(lock)))
	return int(tid), Errno(e)
}

// KillTask kills task with specified tid. tid == 0 means current task.
func KillTask(tid int) Errno {
	_, e := internal.Syscall1(KILLTASK, uintptr(tid))
	return Errno(e)
}

// TaskUnlock can be used when task was created with lock option. It informs
// tasker that now it can safely run parent task.
func TaskUnlock() {
	internal.Syscall0(TASKUNLOCK)
}

// MaxTasks: see rtos package.
func MaxTasks() int {
	n, _ := internal.Syscall0(MAXTASKS)
	return int(n)
}

// SchedYield causes the calling task to relinquish the CPU.
func SchedYield() {
	internal.Syscall0(SCHEDNEXT)
}

// SchedNext informs tasker that it need to schedule next ready to run task.
// It is safe to call SchedNext from interrupt handler.
func SchedNext() {
	schedNext()
}

// SetSysTimer registers two functions that the runtime uses to communicate with
// the system timer.
//
// Nanosec is used to implement Nanosec system call. It should return the
// monotonic time in nanoseconds (typically the time of system timer run).
//
// SetWakeUp is called by scheduler to ask system timer to wake it up at time t
// (using SchedNext function). T can be a monotonic time in nanoseconds or -1
// if the scheduler does not want to be woken. System timer must guarantee that
// it will wake up the scheduler at t or after t. Additional awakenings before
// and after t are acceptable but not recommended. Ticking timer simply wakes up
// the scheduler with a constant period and its setWakeUp function does nothing.
// Tickless timer schould wake up the scheduler only once, at t or just after t.
func SetSysTimer(nanosec func() int64, setWakeUp func(t int64)) {
	internal.Syscall2(SETSYSTIM, fr64tou(nanosec), f64tou(setWakeUp))
}

// Nanosec: see rtos package.
func Nanosec() int64 {
	return internal.Syscall0r64(NANOSEC)
}

// SetAlarm asks the runtime to send Alarm event at t. T is only a hint for the
// runtime, because it can send alarm at any time: before t, at t and after t.
// Typically, task use SetAlarm in conjunction with Alarm.Wait and Nanosec.
func SetAlarm(t int64) {
	internal.Syscall1i64(SETALARM, t)
}

// SetAt works like SetAlarm but additionaly sets task local variable, used to
// implement internal.TimeChan.
func SetAt(t int64) {
	internal.Syscall1i64(SETAT, t)
}

// TimeChan returns channel that can be used to wait for time set using
// SetAt. TimeChan is used mainly for deadline/timeout in select statements.
func TimeChan() <-chan int64 {
	ch := &internal.TimeChan
	return *(*<-chan int64)(unsafe.Pointer(&ch))
}

// SetIRQEna enables or disables irq.
func SetIRQEna(irq int, ena bool) Errno {
	_, e := internal.Syscall2(SETIRQENA, uintptr(irq), uintptr(bits.One(ena)))
	return Errno(e)
}

// SetIRQPrio sets priority for irq.
func SetIRQPrio(irq, prio int) Errno {
	_, err := internal.Syscall2(SETIRQPRIO, uintptr(irq), uintptr(prio))
	return Errno(err)
}

// SetIRQHandler: see rtos package.
func SetIRQHandler(irq int, f func()) Errno {
	_, e := internal.Syscall2(SETIRQHANDLER, uintptr(irq), ftou(f))
	return Errno(e)
}

// IRQStatus: ee rtos package.
func IRQStatus(irq int) (int, Errno) {
	s, e := internal.Syscall1(IRQSTATUS, uintptr(irq))
	return int(s), Errno(e)
}

// TriggerIRQ: see rtos package.
func TriggerIRQ(irq int) Errno {
	_, e := internal.Syscall1(TRIGGERIRQ, uintptr(irq))
	return Errno(e)
}

// SetPrivLevel: see rtos package.
func SetPrivLevel(n int) (int, Errno) {
	old, e := internal.Syscall1(SETPRIVLEVEL, uintptr(n))
	return int(old), Errno(e)
}

// DebugOutString allows write debug message.
func DebugOutString(port int, s string) (int, Errno) {
	p := (*internal.String)(unsafe.Pointer(&s))
	n, e := internal.Syscall3(DEBUGOUT, uintptr(port), p.Addr, p.Len)
	return int(n), Errno(e)
}

// DebugOut allows write debug message.
func DebugOut(port int, b []byte) (int, Errno) {
	return DebugOutString(port, *(*string)(unsafe.Pointer(&b)))
}
<|endoftext|>


<commit_before>package kafka

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/json-iterator/go"

	"github.com/qiniu/log"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/sender"
	. "github.com/qiniu/logkit/utils/models"
)

var _ sender.SkipDeepCopySender = &Sender{}

type Sender struct {
	name  string
	hosts []string
	topic []string
	cfg   *sarama.Config

	lastError error //用于防止所有的错误都被 kafka熔断的错误提示刷掉
	producer  sarama.SyncProducer
}

var (
	compressionModes = map[string]sarama.CompressionCodec{
		sender.KeyKafkaCompressionNone:   sarama.CompressionNone,
		sender.KeyKafkaCompressionGzip:   sarama.CompressionGZIP,
		sender.KeyKafkaCompressionSnappy: sarama.CompressionSnappy,
	}
)

func init() {
	sender.RegisterConstructor(sender.TypeKafka, NewSender)
}

// kafka sender
func NewSender(conf conf.MapConf) (kafkaSender sender.Sender, err error) {
	hosts, err := conf.GetStringList(sender.KeyKafkaHost)
	if err != nil {
		return
	}
	topic, err := conf.GetStringList(sender.KeyKafkaTopic)
	if err != nil {
		return
	}
	topic, err = ExtractField(topic)
	if err != nil {
		return
	}
	hostName, err := os.Hostname()
	if err != nil {
		hostName = "getHostnameErr:" + err.Error()
		err = nil
	}
	clientID, _ := conf.GetStringOr(sender.KeyKafkaClientId, hostName)
	//num, _ := conf.GetIntOr(KeyKafkaFlushNum, 200)
	//frequency, _ := conf.GetIntOr(KeyKafkaFlushFrequency, 5)
	retryMax, _ := conf.GetIntOr(sender.KeyKafkaRetryMax, 3)
	compression, _ := conf.GetStringOr(sender.KeyKafkaCompression, sender.KeyKafkaCompressionNone)
	timeout, _ := conf.GetStringOr(sender.KeyKafkaTimeout, "30s")
	keepAlive, _ := conf.GetStringOr(sender.KeyKafkaKeepAlive, "0")
	maxMessageBytes, _ := conf.GetIntOr(sender.KeyMaxMessageBytes, 4*1024*1024)

	name, _ := conf.GetStringOr(sender.KeyName, fmt.Sprintf("kafkaSender:(kafkaUrl:%s,topic:%s)", hosts, topic))
	cfg := sarama.NewConfig()
	cfg.Producer.Return.Successes = true
	cfg.Producer.Return.Errors = true
	//cfg.Producer.Return.Successes = false
	//cfg.Producer.Return.Errors = false
	//客户端ID
	cfg.ClientID = clientID
	//批量发送条数
	//cfg.Producer.Flush.Messages = num
	//批量发送间隔
	//cfg.Producer.Flush.Frequency =  time.Duration(frequency) * time.Second
	cfg.Producer.Retry.Max = retryMax
	compressionMode, ok := compressionModes[strings.ToLower(compression)]
	if !ok {
		return nil, fmt.Errorf("unknown compression mode: '%v'", compression)
	}
	cfg.Producer.Compression = compressionMode
	cfg.Net.DialTimeout, err = time.ParseDuration(timeout)
	if err != nil {
		return
	}
	cfg.Net.KeepAlive, err = time.ParseDuration(keepAlive)
	if err != nil {
		return
	}
	cfg.Producer.MaxMessageBytes = maxMessageBytes

	producer, err := sarama.NewSyncProducer(hosts, cfg)
	if err != nil {
		return
	}

	kafkaSender = newSender(name, hosts, topic, cfg, producer)
	return
}

func newSender(name string, hosts []string, topic []string, cfg *sarama.Config, producer sarama.SyncProducer) (k *Sender) {
	k = &Sender{
		name:     name,
		hosts:    hosts,
		topic:    topic,
		cfg:      cfg,
		producer: producer,
	}
	return
}

func (this *Sender) Name() string {
	return this.name
}

func (this *Sender) Send(data []Data) error {
	producer := this.producer
	var msgs []*sarama.ProducerMessage
	ss := &StatsError{}
	var lastErr error
	for _, doc := range data {
		message, err := this.getEventMessage(doc)
		if err != nil {
			log.Debugf("Dropping event: %v", err)
			ss.AddErrors()
			lastErr = err
			continue
		}
		msgs = append(msgs, message)
	}
	err := producer.SendMessages(msgs)
	if err != nil {
		ss.AddErrorsNum(len(msgs))
		if pde, ok := err.(sarama.ProducerErrors); ok {
			var allcir = true
			for _, v := range pde {
				//对于熔断的错误提示，没有任何帮助，过滤掉
				if strings.Contains(v.Error(), "circuit breaker is open") {
					continue
				}
				allcir = false
				ss.ErrorDetail = fmt.Errorf("%v detail: %v", ss.ErrorDetail, v.Error())
				this.lastError = v
				break
			}
			if allcir {
				ss.ErrorDetail = fmt.Errorf("%v, all error is circuit breaker is open , last error %v", err, this.lastError)
			}
		} else {
			ss.ErrorDetail = err
		}
		return ss
	}
	ss.AddSuccessNum(len(msgs))
	if lastErr != nil {
		ss.LastError = lastErr.Error()
		return ss
	}
	//本次发送成功, lastError 置为 nil
	this.lastError = nil
	return ss
}

func (kf *Sender) getEventMessage(event map[string]interface{}) (pm *sarama.ProducerMessage, err error) {
	var topic string
	if len(kf.topic) == 2 {
		if event[kf.topic[0]] == nil || event[kf.topic[0]] == "" {
			topic = kf.topic[1]
		} else {
			if mytopic, ok := event[kf.topic[0]].(string); ok {
				topic = mytopic
			} else {
				topic = kf.topic[1]
			}
		}
	} else {
		topic = kf.topic[0]
	}
	value, err := jsoniter.Marshal(event)
	if err != nil {
		return
	}
	pm = &sarama.ProducerMessage{
		Topic: topic,
		Value: sarama.StringEncoder(string(value)),
	}
	return
}

func (this *Sender) Close() (err error) {
	log.Infof("kafka sender was closed")
	this.producer.Close()
	this.producer = nil
	return nil
}

func (_ *Sender) SkipDeepCopy() bool { return true }
<commit_msg>kafka sender unpack message with binary when sender error is message too large<commit_after>package kafka

import (
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/Shopify/sarama"
	"github.com/json-iterator/go"

	"github.com/qiniu/log"
	"github.com/qiniu/pandora-go-sdk/base/reqerr"

	"github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/sender"
	. "github.com/qiniu/logkit/utils/models"
)

var _ sender.SkipDeepCopySender = &Sender{}

type Sender struct {
	name  string
	hosts []string
	topic []string
	cfg   *sarama.Config

	lastError error //用于防止所有的错误都被 kafka熔断的错误提示刷掉
	producer  sarama.SyncProducer
}

var (
	compressionModes = map[string]sarama.CompressionCodec{
		sender.KeyKafkaCompressionNone:   sarama.CompressionNone,
		sender.KeyKafkaCompressionGzip:   sarama.CompressionGZIP,
		sender.KeyKafkaCompressionSnappy: sarama.CompressionSnappy,
	}
)

func init() {
	sender.RegisterConstructor(sender.TypeKafka, NewSender)
}

// kafka sender
func NewSender(conf conf.MapConf) (kafkaSender sender.Sender, err error) {
	hosts, err := conf.GetStringList(sender.KeyKafkaHost)
	if err != nil {
		return
	}
	topic, err := conf.GetStringList(sender.KeyKafkaTopic)
	if err != nil {
		return
	}
	topic, err = ExtractField(topic)
	if err != nil {
		return
	}
	hostName, err := os.Hostname()
	if err != nil {
		hostName = "getHostnameErr:" + err.Error()
		err = nil
	}
	clientID, _ := conf.GetStringOr(sender.KeyKafkaClientId, hostName)
	//num, _ := conf.GetIntOr(KeyKafkaFlushNum, 200)
	//frequency, _ := conf.GetIntOr(KeyKafkaFlushFrequency, 5)
	retryMax, _ := conf.GetIntOr(sender.KeyKafkaRetryMax, 3)
	compression, _ := conf.GetStringOr(sender.KeyKafkaCompression, sender.KeyKafkaCompressionNone)
	timeout, _ := conf.GetStringOr(sender.KeyKafkaTimeout, "30s")
	keepAlive, _ := conf.GetStringOr(sender.KeyKafkaKeepAlive, "0")
	maxMessageBytes, _ := conf.GetIntOr(sender.KeyMaxMessageBytes, 4*1024*1024)

	name, _ := conf.GetStringOr(sender.KeyName, fmt.Sprintf("kafkaSender:(kafkaUrl:%s,topic:%s)", hosts, topic))
	cfg := sarama.NewConfig()
	cfg.Producer.Return.Successes = true
	cfg.Producer.Return.Errors = true
	//cfg.Producer.Return.Successes = false
	//cfg.Producer.Return.Errors = false
	//客户端ID
	cfg.ClientID = clientID
	//批量发送条数
	//cfg.Producer.Flush.Messages = num
	//批量发送间隔
	//cfg.Producer.Flush.Frequency =  time.Duration(frequency) * time.Second
	cfg.Producer.Retry.Max = retryMax
	compressionMode, ok := compressionModes[strings.ToLower(compression)]
	if !ok {
		return nil, fmt.Errorf("unknown compression mode: '%v'", compression)
	}
	cfg.Producer.Compression = compressionMode
	cfg.Net.DialTimeout, err = time.ParseDuration(timeout)
	if err != nil {
		return
	}
	cfg.Net.KeepAlive, err = time.ParseDuration(keepAlive)
	if err != nil {
		return
	}
	cfg.Producer.MaxMessageBytes = maxMessageBytes

	producer, err := sarama.NewSyncProducer(hosts, cfg)
	if err != nil {
		return
	}

	kafkaSender = newSender(name, hosts, topic, cfg, producer)
	return
}

func newSender(name string, hosts []string, topic []string, cfg *sarama.Config, producer sarama.SyncProducer) (k *Sender) {
	k = &Sender{
		name:     name,
		hosts:    hosts,
		topic:    topic,
		cfg:      cfg,
		producer: producer,
	}
	return
}

func (this *Sender) Name() string {
	return this.name
}

func (this *Sender) Send(data []Data) error {
	producer := this.producer
	var msgs []*sarama.ProducerMessage
	ss := &StatsError{}
	var lastErr error
	for _, doc := range data {
		message, err := this.getEventMessage(doc)
		if err != nil {
			log.Debugf("Dropping event: %v", err)
			ss.AddErrors()
			lastErr = err
			continue
		}
		msgs = append(msgs, message)
	}
	err := producer.SendMessages(msgs)
	if err != nil {
		ss.AddErrorsNum(len(msgs))
		if pde, ok := err.(sarama.ProducerErrors); ok {
			var allcir = true
			for _, v := range pde {
				//对于熔断的错误提示，没有任何帮助，过滤掉
				if strings.Contains(v.Error(), "circuit breaker is open") {
					continue
				}
				allcir = false
				ss.ErrorDetail = fmt.Errorf("%v detail: %v", ss.ErrorDetail, v.Error())
				this.lastError = v
				//发送错误为message too large时，启用二分策略重新发送
				if v.Err == sarama.ErrMessageSizeTooLarge {
					ss.ErrorDetail = reqerr.NewSendError("Sender[Kafka]:Message was too large, server rejected it to avoid allocation error", sender.ConvertDatasBack(data), reqerr.TypeBinaryUnpack)
				}
				break
			}
			if allcir {
				ss.ErrorDetail = fmt.Errorf("%v, all error is circuit breaker is open , last error %v", err, this.lastError)
			}
		} else {
			ss.ErrorDetail = err
		}
		return ss
	}
	ss.AddSuccessNum(len(msgs))
	if lastErr != nil {
		ss.LastError = lastErr.Error()
		return ss
	}
	//本次发送成功, lastError 置为 nil
	this.lastError = nil
	return ss
}

func (kf *Sender) getEventMessage(event map[string]interface{}) (pm *sarama.ProducerMessage, err error) {
	var topic string
	if len(kf.topic) == 2 {
		if event[kf.topic[0]] == nil || event[kf.topic[0]] == "" {
			topic = kf.topic[1]
		} else {
			if mytopic, ok := event[kf.topic[0]].(string); ok {
				topic = mytopic
			} else {
				topic = kf.topic[1]
			}
		}
	} else {
		topic = kf.topic[0]
	}
	value, err := jsoniter.Marshal(event)
	if err != nil {
		return
	}
	pm = &sarama.ProducerMessage{
		Topic: topic,
		Value: sarama.StringEncoder(string(value)),
	}
	return
}

func (this *Sender) Close() (err error) {
	log.Infof("kafka sender was closed")
	this.producer.Close()
	this.producer = nil
	return nil
}

func (_ *Sender) SkipDeepCopy() bool { return true }
<|endoftext|>


<commit_before>package agent

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strings"
	"sync"
	"time"

	"github.com/jrallison/go-workers"
	"github.com/pborman/uuid"
	"github.com/spf13/viper"
)

// Agent runs codeflow and collects data based on the given config
type Agent struct {
	Queueing   bool
	Events     chan Event
	TestEvents chan Event
	Shutdown   chan struct{}
	Plugins    []*RunningPlugin
}

// NewAgent returns an Agent struct based off the given Config
func NewAgent() (*Agent, error) {
	if len(viper.GetStringMap("plugins")) == 0 {
		log.Fatalf("Error: no plugins found, did you provide a valid config file?")
	}

	agent := &Agent{}

	// channel shared between all plugin threads for accumulating events
	agent.Events = make(chan Event, 10000)

	// channel shared between all plugin threads to trigger shutdown
	agent.Shutdown = make(chan struct{})

	if err := agent.LoadPlugins(); err != nil {
		log.Fatal(err)
	}

	return agent, nil
}

// NewTestAgent returns an Agent struct based off the given Config
func NewTestAgent(config []byte) (*Agent, error) {
	var err error
	var agent *Agent

	viper.SetConfigType("yaml")
	viper.ReadConfig(bytes.NewBuffer(config))

	if agent, err = NewAgent(); err != nil {
		log.Fatalf("Error while initializing agent: %v", err)
	}

	agent.TestEvents = make(chan Event, 10000)
	agent.Queueing = false

	return agent, nil
}

func (a *Agent) LoadPlugins() error {
	var err error

	ep := viper.GetString("run")
	runPlugins := strings.Split(strings.Trim(ep, "[]"), ",")
	for name := range viper.GetStringMap("plugins") {
		if err = a.addPlugin(name); err != nil {
			return fmt.Errorf("Error parsing %s, %s", name, err)
		}
		if ep == "" || SliceContains(name, runPlugins) {
			if err = a.enablePlugin(name); err != nil {
				return fmt.Errorf("Error parsing %s, %s", name, err)
			}
		}
	}

	return nil
}

// Returns a list of strings of the configured plugins.
func (a *Agent) PluginNames() []string {
	var name []string
	for key, _ := range viper.GetStringMap("plugins") {
		name = append(name, key)
	}
	return name
}

func (a *Agent) addPlugin(name string) error {
	if len(a.PluginNames()) > 0 && !SliceContains(name, a.PluginNames()) {
		return nil
	}

	creator, ok := PluginRegistry[name]
	if !ok {
		return fmt.Errorf("Undefined but requested Plugin: %s", name)
	}
	plugin := creator()

	viper.UnmarshalKey(fmt.Sprint("plugins.", name), plugin)

	work := func(message *workers.Msg) {
		e, _ := json.Marshal(message.Args())
		event := Event{}
		json.Unmarshal([]byte(e), &event)
		if err := MapPayload(event.PayloadModel, &event); err != nil {
			event.Error = fmt.Errorf("PayloadModel not found: %s. Did you add it to ApiRegistry?", event.PayloadModel)
		}

		// For debugging purposes
		event.Dump()

		plugin.Process(event)
	}

	rp := &RunningPlugin{
		Name:    name,
		Plugin:  plugin,
		Work:    work,
		Enabled: false,
		Workers: viper.GetInt("plugins." + name + ".workers"),
	}

	a.Plugins = append(a.Plugins, rp)

	return nil
}

func (a *Agent) enablePlugin(name string) error {
	if len(a.PluginNames()) > 0 && !SliceContains(name, a.PluginNames()) {
		return nil
	}

	for _, rp := range a.Plugins {
		if rp.Name == name {
			rp.Enabled = true
		}
	}

	return nil
}

// flusher monitors the events plugin channel and schedules them to correct queues
func (a *Agent) flusher() {
	for {
		select {
		case <-a.Shutdown:
			log.Println("Hang on, flushing any cached metrics before shutdown")
			return
		case e := <-a.Events:
			ev_handled := false

			for _, plugin := range a.Plugins {
				if plugin.Workers > 0 {
					subscribedTo := plugin.Plugin.Subscribe()
					if SliceContains(e.PayloadModel, subscribedTo) || SliceContains(e.Name, subscribedTo) {
						ev_handled = true
						if a.Queueing {
							log.Printf("Enqueue event %v for %v\n", e.Name, plugin.Name)
							workers.Enqueue(plugin.Name, "Event", e)
						} else {
							plugin.Plugin.Process(e)
						}
					}
				}
			}

			if a.TestEvents != nil {
				a.TestEvents <- e
			} else if !ev_handled {
				log.Printf("Event not handled by any plugin: %s\n", e.Name)
				e.Dump()
			}
		}
	}
}

// Run runs the agent daemon
func (a *Agent) Run() error {
	var wg sync.WaitGroup

	if a.Queueing {
		workers.Middleware = workers.NewMiddleware(
			&workers.MiddlewareRetry{},
			&workers.MiddlewareStats{},
		)

		workers.Configure(map[string]string{
			"server":   viper.GetString("redis.server"),
			"database": viper.GetString("redis.database"),
			"pool":     viper.GetString("redis.pool"),
			"process":  uuid.New(),
		})
	}

	for _, plugin := range a.Plugins {
		if !plugin.Enabled {
			continue
		}

		// Start service of any Plugins
		switch p := plugin.Plugin.(type) {
		case Plugin:
			if err := p.Start(a.Events); err != nil {
				log.Printf("Service for plugin %s failed to start, exiting\n%s\n",
					plugin.Name, err.Error())
				return err
			}
			defer p.Stop()

			if a.Queueing {
				workers.Process(plugin.Name, plugin.Work, viper.GetInt("plugins."+plugin.Name+".workers"))
			}
		}
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		a.flusher()
	}()

	if a.Queueing {
		wg.Add(1)
		go func() {
			defer wg.Done()
			workers.Run()
			a.Stop()
		}()

		//wg.Add(1)
		//go func() {
		//	defer wg.Done()
		//	workers.StatsServer(8080)
		//}()
	}

	wg.Wait()
	return nil
}

// Shutdown the agent daemon
func (a *Agent) Stop() {
	close(a.Shutdown)
}

// GetTestEvent listens and returns requested event
func (a *Agent) GetTestEvent(name string, timeout time.Duration) Event {
	// timeout in the case that we don't get requested event
	timer := time.NewTimer(time.Second * timeout)
	go func() {
		<-timer.C
		a.Stop()
		log.Fatalf("Timer expired waiting for event: %v", name)
	}()

	for e := range a.TestEvents {
		if e.Name == name {
			timer.Stop()
			return e
		}
	}

	return Event{}
}
<commit_msg>Fix agent run plugins<commit_after>package agent

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/jrallison/go-workers"
	"github.com/pborman/uuid"
	"github.com/spf13/viper"
)

// Agent runs codeflow and collects data based on the given config
type Agent struct {
	Queueing   bool
	Events     chan Event
	TestEvents chan Event
	Shutdown   chan struct{}
	Plugins    []*RunningPlugin
}

// NewAgent returns an Agent struct based off the given Config
func NewAgent() (*Agent, error) {
	if len(viper.GetStringMap("plugins")) == 0 {
		log.Fatalf("Error: no plugins found, did you provide a valid config file?")
	}

	agent := &Agent{}

	// channel shared between all plugin threads for accumulating events
	agent.Events = make(chan Event, 10000)

	// channel shared between all plugin threads to trigger shutdown
	agent.Shutdown = make(chan struct{})

	if err := agent.LoadPlugins(); err != nil {
		log.Fatal(err)
	}

	return agent, nil
}

// NewTestAgent returns an Agent struct based off the given Config
func NewTestAgent(config []byte) (*Agent, error) {
	var err error
	var agent *Agent

	viper.SetConfigType("yaml")
	viper.ReadConfig(bytes.NewBuffer(config))

	if agent, err = NewAgent(); err != nil {
		log.Fatalf("Error while initializing agent: %v", err)
	}

	agent.TestEvents = make(chan Event, 10000)
	agent.Queueing = false

	return agent, nil
}

func (a *Agent) LoadPlugins() error {
	var err error

	runPlugins := viper.GetStringSlice("run")
	for name := range viper.GetStringMap("plugins") {
		if err = a.addPlugin(name); err != nil {
			return fmt.Errorf("Error parsing %s, %s", name, err)
		}
		if len(runPlugins) == 0 || SliceContains(name, runPlugins) {
			if err = a.enablePlugin(name); err != nil {
				return fmt.Errorf("Error parsing %s, %s", name, err)
			}
		}
	}

	return nil
}

// Returns a list of strings of the configured plugins.
func (a *Agent) PluginNames() []string {
	var name []string
	for key, _ := range viper.GetStringMap("plugins") {
		name = append(name, key)
	}
	return name
}

func (a *Agent) addPlugin(name string) error {
	if len(a.PluginNames()) > 0 && !SliceContains(name, a.PluginNames()) {
		return nil
	}

	creator, ok := PluginRegistry[name]
	if !ok {
		return fmt.Errorf("Undefined but requested Plugin: %s", name)
	}
	plugin := creator()

	viper.UnmarshalKey(fmt.Sprint("plugins.", name), plugin)

	work := func(message *workers.Msg) {
		e, _ := json.Marshal(message.Args())
		event := Event{}
		json.Unmarshal([]byte(e), &event)
		if err := MapPayload(event.PayloadModel, &event); err != nil {
			event.Error = fmt.Errorf("PayloadModel not found: %s. Did you add it to ApiRegistry?", event.PayloadModel)
		}

		// For debugging purposes
		event.Dump()

		plugin.Process(event)
	}

	rp := &RunningPlugin{
		Name:    name,
		Plugin:  plugin,
		Work:    work,
		Enabled: false,
		Workers: viper.GetInt("plugins." + name + ".workers"),
	}

	a.Plugins = append(a.Plugins, rp)

	return nil
}

func (a *Agent) enablePlugin(name string) error {
	if len(a.PluginNames()) > 0 && !SliceContains(name, a.PluginNames()) {
		return nil
	}

	for _, rp := range a.Plugins {
		if rp.Name == name {
			rp.Enabled = true
		}
	}

	return nil
}

// flusher monitors the events plugin channel and schedules them to correct queues
func (a *Agent) flusher() {
	for {
		select {
		case <-a.Shutdown:
			log.Println("Hang on, flushing any cached metrics before shutdown")
			return
		case e := <-a.Events:
			ev_handled := false

			for _, plugin := range a.Plugins {
				if plugin.Workers > 0 {
					subscribedTo := plugin.Plugin.Subscribe()
					if SliceContains(e.PayloadModel, subscribedTo) || SliceContains(e.Name, subscribedTo) {
						ev_handled = true
						if a.Queueing {
							log.Printf("Enqueue event %v for %v\n", e.Name, plugin.Name)
							workers.Enqueue(plugin.Name, "Event", e)
						} else {
							plugin.Plugin.Process(e)
						}
					}
				}
			}

			if a.TestEvents != nil {
				a.TestEvents <- e
			} else if !ev_handled {
				log.Printf("Event not handled by any plugin: %s\n", e.Name)
				e.Dump()
			}
		}
	}
}

// Run runs the agent daemon
func (a *Agent) Run() error {
	var wg sync.WaitGroup

	if a.Queueing {
		workers.Middleware = workers.NewMiddleware(
			&workers.MiddlewareRetry{},
			&workers.MiddlewareStats{},
		)

		workers.Configure(map[string]string{
			"server":   viper.GetString("redis.server"),
			"database": viper.GetString("redis.database"),
			"pool":     viper.GetString("redis.pool"),
			"process":  uuid.New(),
		})
	}

	for _, plugin := range a.Plugins {
		if !plugin.Enabled {
			continue
		}

		// Start service of any Plugins
		switch p := plugin.Plugin.(type) {
		case Plugin:
			if err := p.Start(a.Events); err != nil {
				log.Printf("Service for plugin %s failed to start, exiting\n%s\n",
					plugin.Name, err.Error())
				return err
			}
			defer p.Stop()

			if a.Queueing {
				workers.Process(plugin.Name, plugin.Work, viper.GetInt("plugins."+plugin.Name+".workers"))
			}
		}
	}

	wg.Add(1)
	go func() {
		defer wg.Done()
		a.flusher()
	}()

	if a.Queueing {
		wg.Add(1)
		go func() {
			defer wg.Done()
			workers.Run()
			a.Stop()
		}()

		//wg.Add(1)
		//go func() {
		//	defer wg.Done()
		//	workers.StatsServer(8080)
		//}()
	}

	wg.Wait()
	return nil
}

// Shutdown the agent daemon
func (a *Agent) Stop() {
	close(a.Shutdown)
}

// GetTestEvent listens and returns requested event
func (a *Agent) GetTestEvent(name string, timeout time.Duration) Event {
	// timeout in the case that we don't get requested event
	timer := time.NewTimer(time.Second * timeout)
	go func() {
		<-timer.C
		a.Stop()
		log.Fatalf("Timer expired waiting for event: %v", name)
	}()

	for e := range a.TestEvents {
		if e.Name == name {
			timer.Stop()
			return e
		}
	}

	return Event{}
}
<|endoftext|>


<commit_before>package server

import (
	"errors"
	"github.com/itsankoff/gotcha/common"
	"log"
)

// Store for all authenticated outputs
// Main facility to send a message to the remote user
type OutputStore struct {
	outputs map[string]chan<- *common.Message
}

func NewOutputStore() *OutputStore {
	out := &OutputStore{
		outputs: make(map[string]chan<- *common.Message),
	}

	return out
}

// AddOutput adds user output in the store by id
func (store *OutputStore) AddOutput(id string,
	output chan<- *common.Message) error {
	_, ok := store.outputs[id]
	if ok {
		return errors.New("Output for already exists for id " + id)
	}

	store.outputs[id] = output
	log.Println("Add output in store", id)
	return nil
}

// RemoveOutput removes user output from the store by id
func (store *OutputStore) RemoveOutput(id string) error {
	_, ok := store.outputs[id]
	if ok {
		delete(store.outputs, id)
		log.Println("Remove output from store", id)
		return nil
	}

	return errors.New("Failed to remove output from store " + id)
}

// GetOutput returns a user output from the store by id
func (store OutputStore) GetOutput(id string) chan<- *common.Message {
	output, ok := store.outputs[id]
	if ok {
		return output
	}

	log.Println("Failed to get output for id", id)
	return nil
}

// Send sends a message to user's output
func (store OutputStore) Send(msg *common.Message) {
	output := store.GetOutput(msg.To())
	if output != nil {
		output <- msg
	} else {
		log.Println("Failed to send message", msg)
	}
}
<commit_msg>* Add more comments<commit_after>package server

import (
	"errors"
	"github.com/itsankoff/gotcha/common"
	"log"
)

// Store for all authenticated outputs. Main facility to send messages to remote
// users
type OutputStore struct {
	outputs map[string]chan<- *common.Message
}

// NewOutputStore returns a valid store.
func NewOutputStore() *OutputStore {
	out := &OutputStore{
		outputs: make(map[string]chan<- *common.Message),
	}

	return out
}

// AddOutput adds user output in the store by id
func (store *OutputStore) AddOutput(id string,
	output chan<- *common.Message) error {
	_, ok := store.outputs[id]
	if ok {
		return errors.New("Output for already exists for id " + id)
	}

	store.outputs[id] = output
	log.Println("Add output in store", id)
	return nil
}

// RemoveOutput removes user output from the store by id
func (store *OutputStore) RemoveOutput(id string) error {
	_, ok := store.outputs[id]
	if ok {
		delete(store.outputs, id)
		log.Println("Remove output from store", id)
		return nil
	}

	return errors.New("Failed to remove output from store " + id)
}

// GetOutput returns a user output from the store by id
func (store OutputStore) GetOutput(id string) chan<- *common.Message {
	output, ok := store.outputs[id]
	if ok {
		return output
	}

	log.Println("Failed to get output for id", id)
	return nil
}

// Send sends a message to user's output
func (store OutputStore) Send(msg *common.Message) {
	output := store.GetOutput(msg.To())
	if output != nil {
		output <- msg
	} else {
		log.Println("Failed to send message", msg)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"

	"github.com/fatih/color"
)

var warn = color.New(color.FgYellow).Add(color.Bold).Println

func procTestResponse(res *http.Response, t *testing.T, expectedCode int) {
	body, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		t.Error(err)
	}
	fmt.Println(string(body) + "\n")
	if res.StatusCode != expectedCode {
		t.Error(res.Request.Method + ": " + res.Request.URL.Path + " returned " + strconv.Itoa(res.StatusCode) + ". Expected " + strconv.Itoa(expectedCode) + ".")
		return
	}
	if res.StatusCode == 204 {
		return
	}
	if res.StatusCode == 404 {
		return
	}
	var marsh interface{}
	err = json.Unmarshal(body, &marsh)
	if err != nil {
		t.Error(errors.New(res.Request.Method + ": failed to unmarshal response from " + res.Request.URL.Path + ". Status code was: " + strconv.Itoa(res.StatusCode)))
	}
	_, err = json.MarshalIndent(marsh, "", "  ")
	if err != nil {
		t.Error("failed to Marshal")
	}
}

func TestEndpoints(t *testing.T) {

	s := NewServer()
	r := s.NewRouter()
	server := httptest.NewServer(r)
	defer server.Close()

	// a few closures to save time below
	get := func(endpoint string, expectedCode int) {
		res, err := http.Get(server.URL + endpoint)
		if err != nil {
			t.Error(err)
			return
		}
		procTestResponse(res, t, expectedCode)
	}

	post := func(endpoint, msg string, expectedCode int) {
		res, err := http.Post(server.URL+endpoint, "application/json", bytes.NewBuffer([]byte(msg)))
		if err != nil {
			t.Error(err)
			return
		}
		procTestResponse(res, t, expectedCode)
	}

	put := func(endpoint, msg string, expectedCode int) {
		req, err := http.NewRequest("PUT", server.URL+endpoint, bytes.NewBuffer([]byte(msg)))
		if err != nil {
			t.Error(err)
			return
		}
		c := &http.Client{}
		res, err := c.Do(req)
		if err != nil {
			t.Error(err)
			return
		}
		procTestResponse(res, t, expectedCode)
	}

	del := func(endpoint string, expectedCode int) {
		req, err := http.NewRequest("DELETE", server.URL+endpoint, nil)
		if err != nil {
			t.Error(err)
			return
		}
		c := &http.Client{}
		res, err := c.Do(req)
		if err != nil {
			t.Error(err)
			return
		}
		procTestResponse(res, t, expectedCode)
	}

	// set up a group (1)
	post("/groups", `{"parent":0}`, 200)

	// set up a + block (2)
	post("/blocks", `{"type":"+","parent":1}`, 200)

	// label group 1
	put("/groups/1/label", `"The Best Group Ever"`, 204)

	// label the plus block
	put("/blocks/2/label", `"my bestest adder"`, 204)

	// move the plus block
	put("/blocks/2/position", `{"x":10,"y":10}`, 204)

	// get all the groups
	get("/groups", 200)

	// get group 1
	get("/groups/1", 200)

	// move group 1
	put("/groups/1/position", `{"x":20,"y":20}`, 204)

	// get all the blocks
	get("/blocks", 200)

	// get the + block
	get("/blocks/2", 200)

	// make a delay block (3)
	post("/blocks", `{"type":"delay", "parent":1}`, 200)

	// set the delay's value
	put("/blocks/3/routes/1", `{"data":"1s"}`, 204)

	// make a log block (4)
	post("/blocks", `{"type":"log", "parent":1}`, 200)

	// connect the + block to the delay block (5)
	post("/connections", `{"from":{"id":2, "route":0}, "to":{"id":3, "route":0}}`, 200)

	// connect the delay block to the log block (6)
	post("/connections", `{"from":{"id":3, "route":0}, "to":{"id":4, "route":0}}`, 200)

	// set the value of the plus inputs
	put("/blocks/2/routes/0", `{"data":1}`, 204)
	put("/blocks/2/routes/1", `{"data":1}`, 204)

	// make a set block (7)
	post("/blocks", `{"type":"set", "parent":1}`, 200)

	// disconnect the log block from the delay block
	del("/connections/6", 204)

	// connect the set block to the log block and delay block (8) (9)
	post("/connections", `{"from":{"id":7, "route":0}, "to":{"id":4, "route":0}}`, 200)
	post("/connections", `{"from":{"id":3, "route":0}, "to":{"id":7, "route":1}}`, 200)

	// list connections
	get("/connections", 200)
	// describe connection 8
	get("/connections/8", 200)

	// set the value of the set key
	put("/blocks/7/routes/0", `{"data":"myResult"}`, 204)

	// move log block to root group
	put("/groups/0/children/4", "", 204)
	// move + block to root group (we will generate some errors with this later)
	put("/groups/0/children/2", "", 204)

	// create a keyvalue source (10)
	post("/sources", `{"type":"key_value"}`, 200)

	// get the keyvalue source
	get("/sources/10", 200)

	// make a stream source (11)
	post("/sources", `{"type":"stream"}`, 200)

	// change a parameter in the stream
	put("/sources/11/params", `{"topic":"test"}`, 204)

	// get all the sources
	get("/sources", 200)

	// make a key value get block (12)
	post("/blocks", `{"type":"kvGet"}`, 200)

	// link the key value get block to the key value source (13)
	post("/links", `{"source":{"id":10},"block":{"id":12}}`, 200)

	// list the links
	get("/links", 200)

	// this doesn't exist yet - TODO use case?
	// get the link
	// get("/links/13", 200)

	// delete the link
	del("/links/13", 204)

	// delete the keyvalue store
	del("/sources/10", 204)

	// export the pattern
	get("/groups/0/export", 200)

	// import a pattern
	pattern := `{"blocks":[{"label":"","type":"delay","id":3,"inputs":[{"name":"passthrough","value":null},{"name":"duration","value":{"data":"1s"}}],"outputs":[{"name":"passthrough"}],"position":{"x":0,"y":0}},{"label":"","type":"set","id":7,"inputs":[{"name":"key","value":{"data":"myResult"}},{"name":"value","value":null}],"outputs":[{"name":"object"}],"position":{"x":0,"y":0}},{"label":"","type":"log","id":4,"inputs":[{"name":"log","value":null}],"outputs":[],"position":{"x":0,"y":0}},{"label":"my bestest adder","type":"+","id":2,"inputs":[{"name":"addend","value":{"data":1}},{"name":"addend","value":{"data":1}}],"outputs":[{"name":"sum"}],"position":{"x":10,"y":10}},{"label":"","type":"kvGet","id":12,"inputs":[{"name":"key","value":null}],"outputs":[{"name":"value"}],"position":{"x":0,"y":0}}],"connections":[{"from":{"id":2,"route":0},"to":{"id":3,"route":0},"id":5},{"from":{"id":7,"route":0},"to":{"id":4,"route":0},"id":8},{"from":{"id":3,"route":0},"to":{"id":7,"route":1},"id":9}],"groups":[{"id":0,"label":"root","children":[1,4,2,11,12],"position":{"x":0,"y":0}},{"id":1,"label":"The Best Group Ever","children":[3,7],"position":{"x":20,"y":20}}],"sources":[{"label":"","type":"stream","id":11,"position":{"x":0,"y":0},"params":{"channel":"","lookupdAddr":"","maxInFlight":"10","topic":""}}],"links":null}`
	post("/groups/1/import", pattern, 204)

	// delete the log block
	del("/blocks/4", 204)

	// delete group 1
	del("/groups/1", 204)

	// get the blocks library
	get("/blocks/library", 200)

	// get the blocks library
	get("/sources/library", 200)

	// generate some errors
	del("/groups/1", 400)                                                                 // delete a group we've already deleted
	del("/groups/", 404)                                                                  // delete unspecified group
	del("/blocks/246", 400)                                                               // delete an unknown block
	post("/groups/1/import", "{}", 400)                                                   // import empty
	post("/groups/1/import", "{bla}", 400)                                                // import malformed
	get("/groups/6/export", 400)                                                          // export an unknown group
	post("/sources", `{"type":"GodHead"}`, 400)                                           // create an unknown source
	get("/sources/45", 400)                                                               // get an unknown source
	post("/links", `{"from":100,"block":12}`, 400)                                        // link to an unknown source
	post("/links", `{"from":10,"block":120}`, 400)                                        // link to an unknown block
	get("/links/450", 404)                                                                // get an unknown link
	put("/groups/8/children/4", "", 400)                                                  // modify an unknown group
	put("/groups/0/children/34", "", 400)                                                 // move an unknown block to group 0
	put("/blocks/2/routes/0", `{bobo}`, 400)                                              // set the + block's route using malformed json
	post("/groups", `{"parent":10}`, 400)                                                 // create a group with an unknown parent
	post("/groups", `{"parent"10}`, 400)                                                  // create a group with malformed JSON
	post("/blocks", `{"type":"invalid", "parent":0}`, 400)                                // create a block of invalid type
	post("/blocks", `{"type"lid", "parent":1}`, 400)                                      // create a block with malformed json
	post("/blocks", `{"type":"latch", "parent":10}`, 400)                                 // create a block witha group that doesn't exist
	post("/connections", `{"from":{"id":700, "route":0}, "to":{"id":2, "route":0}}`, 400) //connect unknown source
	post("/connections", `{"from":{"id":2, "route":0}, "to":{"id":200, "route":0}}`, 400) //connect unknown target
	post("/connections", `{"from":{"i:0}, "ta200, "route":0}}`, 400)                      //connect with malformed json
	post("/connections", `{}`, 400)                                                       //connect with empty json
	post("/connections", "", 400)                                                         //connect with empty string
	del("/connections/289", 400)                                                          //delete unknown connection
	del("/connections/", 404)                                                             //delete unspecified connection
	del("/connections/invalid", 400)                                                      //delete malformed connection
}
<commit_msg>fixing test<commit_after>package server

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"

	"github.com/fatih/color"
)

var warn = color.New(color.FgYellow).Add(color.Bold).Println

func procTestResponse(res *http.Response, t *testing.T, expectedCode int) {
	body, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		t.Error(err)
	}
	fmt.Println(string(body) + "\n")
	if res.StatusCode != expectedCode {
		t.Error(res.Request.Method + ": " + res.Request.URL.Path + " returned " + strconv.Itoa(res.StatusCode) + ". Expected " + strconv.Itoa(expectedCode) + ".")
		return
	}
	if res.StatusCode == 204 {
		return
	}
	if res.StatusCode == 404 {
		return
	}
	var marsh interface{}
	err = json.Unmarshal(body, &marsh)
	if err != nil {
		t.Error(errors.New(res.Request.Method + ": failed to unmarshal response from " + res.Request.URL.Path + ". Status code was: " + strconv.Itoa(res.StatusCode)))
	}
	_, err = json.MarshalIndent(marsh, "", "  ")
	if err != nil {
		t.Error("failed to Marshal")
	}
}

func TestEndpoints(t *testing.T) {

	s := NewServer()
	r := s.NewRouter()
	server := httptest.NewServer(r)
	defer server.Close()

	// a few closures to save time below
	get := func(endpoint string, expectedCode int) {
		res, err := http.Get(server.URL + endpoint)
		if err != nil {
			t.Error(err)
			return
		}
		procTestResponse(res, t, expectedCode)
	}

	post := func(endpoint, msg string, expectedCode int) {
		res, err := http.Post(server.URL+endpoint, "application/json", bytes.NewBuffer([]byte(msg)))
		if err != nil {
			t.Error(err)
			return
		}
		procTestResponse(res, t, expectedCode)
	}

	put := func(endpoint, msg string, expectedCode int) {
		req, err := http.NewRequest("PUT", server.URL+endpoint, bytes.NewBuffer([]byte(msg)))
		if err != nil {
			t.Error(err)
			return
		}
		c := &http.Client{}
		res, err := c.Do(req)
		if err != nil {
			t.Error(err)
			return
		}
		procTestResponse(res, t, expectedCode)
	}

	del := func(endpoint string, expectedCode int) {
		req, err := http.NewRequest("DELETE", server.URL+endpoint, nil)
		if err != nil {
			t.Error(err)
			return
		}
		c := &http.Client{}
		res, err := c.Do(req)
		if err != nil {
			t.Error(err)
			return
		}
		procTestResponse(res, t, expectedCode)
	}

	// set up a group (1)
	post("/groups", `{"parent":0}`, 200)

	// set up a + block (2)
	post("/blocks", `{"type":"+","parent":1}`, 200)

	// label group 1
	put("/groups/1/label", `"The Best Group Ever"`, 204)

	// label the plus block
	put("/blocks/2/label", `"my bestest adder"`, 204)

	// move the plus block
	put("/blocks/2/position", `{"x":10,"y":10}`, 204)

	// get all the groups
	get("/groups", 200)

	// get group 1
	get("/groups/1", 200)

	// move group 1
	put("/groups/1/position", `{"x":20,"y":20}`, 204)

	// get all the blocks
	get("/blocks", 200)

	// get the + block
	get("/blocks/2", 200)

	// make a delay block (3)
	post("/blocks", `{"type":"delay", "parent":1}`, 200)

	// set the delay's value
	put("/blocks/3/routes/1", `{"data":"1s"}`, 204)

	// make a log block (4)
	post("/blocks", `{"type":"log", "parent":1}`, 200)

	// connect the + block to the delay block (5)
	post("/connections", `{"from":{"id":2, "route":0}, "to":{"id":3, "route":0}}`, 200)

	// connect the delay block to the log block (6)
	post("/connections", `{"from":{"id":3, "route":0}, "to":{"id":4, "route":0}}`, 200)

	// set the value of the plus inputs
	put("/blocks/2/routes/0", `{"data":1}`, 204)
	put("/blocks/2/routes/1", `{"data":1}`, 204)

	// make a set block (7)
	post("/blocks", `{"type":"set", "parent":1}`, 200)

	// disconnect the log block from the delay block
	del("/connections/6", 204)

	// connect the set block to the log block and delay block (8) (9)
	post("/connections", `{"from":{"id":7, "route":0}, "to":{"id":4, "route":0}}`, 200)
	post("/connections", `{"from":{"id":3, "route":0}, "to":{"id":7, "route":1}}`, 200)

	// list connections
	get("/connections", 200)
	// describe connection 8
	get("/connections/8", 200)

	// set the value of the set key
	put("/blocks/7/routes/0", `{"data":"myResult"}`, 204)

	// move log block to root group
	put("/groups/0/children/4", "", 204)
	// move + block to root group (we will generate some errors with this later)
	put("/groups/0/children/2", "", 204)

	// create a keyvalue source (10)
	post("/sources", `{"type":"key_value"}`, 200)

	// get the keyvalue source
	get("/sources/10", 200)

	// make a stream source (11)
	post("/sources", `{"type":"stream"}`, 200)

	// change a parameter in the stream
	put("/sources/11/params", `{"topic":"test"}`, 204)

	// get all the sources
	get("/sources", 200)

	// make a key value get block (12)
	post("/blocks", `{"type":"kvGet"}`, 200)

	// link the key value get block to the key value source (13)
	post("/links", `{"source":{"id":10},"block":{"id":12}}`, 200)

	// list the links
	get("/links", 200)

	// this doesn't exist yet - TODO use case?
	// get the link
	// get("/links/13", 200)

	// delete the link
	del("/links/13", 204)

	// delete the keyvalue store
	del("/sources/10", 204)

	// export the pattern
	get("/groups/0/export", 200)

	// import a pattern
	pattern := `{"blocks":[{"label":"","type":"delay","id":3,"inputs":[{"name":"passthrough","value":null},{"name":"duration","value":{"data":"1s"}}],"outputs":[{"name":"passthrough"}],"position":{"x":0,"y":0}},{"label":"","type":"set","id":7,"inputs":[{"name":"key","value":{"data":"myResult"}},{"name":"value","value":null}],"outputs":[{"name":"object"}],"position":{"x":0,"y":0}},{"label":"","type":"log","id":4,"inputs":[{"name":"log","value":null}],"outputs":[],"position":{"x":0,"y":0}},{"label":"my bestest adder","type":"+","id":2,"inputs":[{"name":"addend","value":{"data":1}},{"name":"addend","value":{"data":1}}],"outputs":[{"name":"sum"}],"position":{"x":10,"y":10}},{"label":"","type":"kvGet","id":12,"inputs":[{"name":"key","value":null}],"outputs":[{"name":"value"}],"position":{"x":0,"y":0}}],"connections":[{"from":{"id":2,"route":0},"to":{"id":3,"route":0},"id":5},{"from":{"id":7,"route":0},"to":{"id":4,"route":0},"id":8},{"from":{"id":3,"route":0},"to":{"id":7,"route":1},"id":9}],"groups":[{"id":0,"label":"root","children":[1,4,2,11,12],"position":{"x":0,"y":0}},{"id":1,"label":"The Best Group Ever","children":[3,7],"position":{"x":20,"y":20}}],"sources":[{"label":"","type":"stream","id":11,"position":{"x":0,"y":0},"params":{"channel":"","lookupdAddr":"","maxInFlight":"10","topic":""}}],"links":null}`
	post("/groups/1/import", pattern, 200)

	// delete the log block
	del("/blocks/4", 204)

	// delete group 1
	del("/groups/1", 204)

	// get the blocks library
	get("/blocks/library", 200)

	// get the blocks library
	get("/sources/library", 200)

	// generate some errors
	del("/groups/1", 400)                                                                 // delete a group we've already deleted
	del("/groups/", 404)                                                                  // delete unspecified group
	del("/blocks/246", 400)                                                               // delete an unknown block
	post("/groups/1/import", "{}", 400)                                                   // import empty
	post("/groups/1/import", "{bla}", 400)                                                // import malformed
	get("/groups/6/export", 400)                                                          // export an unknown group
	post("/sources", `{"type":"GodHead"}`, 400)                                           // create an unknown source
	get("/sources/45", 400)                                                               // get an unknown source
	post("/links", `{"from":100,"block":12}`, 400)                                        // link to an unknown source
	post("/links", `{"from":10,"block":120}`, 400)                                        // link to an unknown block
	get("/links/450", 404)                                                                // get an unknown link
	put("/groups/8/children/4", "", 400)                                                  // modify an unknown group
	put("/groups/0/children/34", "", 400)                                                 // move an unknown block to group 0
	put("/blocks/2/routes/0", `{bobo}`, 400)                                              // set the + block's route using malformed json
	post("/groups", `{"parent":10}`, 400)                                                 // create a group with an unknown parent
	post("/groups", `{"parent"10}`, 400)                                                  // create a group with malformed JSON
	post("/blocks", `{"type":"invalid", "parent":0}`, 400)                                // create a block of invalid type
	post("/blocks", `{"type"lid", "parent":1}`, 400)                                      // create a block with malformed json
	post("/blocks", `{"type":"latch", "parent":10}`, 400)                                 // create a block witha group that doesn't exist
	post("/connections", `{"from":{"id":700, "route":0}, "to":{"id":2, "route":0}}`, 400) //connect unknown source
	post("/connections", `{"from":{"id":2, "route":0}, "to":{"id":200, "route":0}}`, 400) //connect unknown target
	post("/connections", `{"from":{"i:0}, "ta200, "route":0}}`, 400)                      //connect with malformed json
	post("/connections", `{}`, 400)                                                       //connect with empty json
	post("/connections", "", 400)                                                         //connect with empty string
	del("/connections/289", 400)                                                          //delete unknown connection
	del("/connections/", 404)                                                             //delete unspecified connection
	del("/connections/invalid", 400)                                                      //delete malformed connection
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package subnets_test

import (
	stdtesting "testing"

	"github.com/juju/juju/testing"
)

// TestAll is the main test function for this package
func TestAll(t *stdtesting.T) {
	testing.MgoTestPackage(t)
}
<commit_msg>Not using Mongo, so shouldn't use MgoTestPackage.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package subnets_test

import (
	"testing"

	gc "gopkg.in/check.v1"
)

// TestAll is the main test function for this package
func TestAll(t *testing.T) {
	gc.TestingT(t)
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssa

import (
	"cmd/internal/src"
)

// findlive returns the reachable blocks and live values in f.
// The caller should call f.retDeadcodeLive(live) when it is done with it.
func findlive(f *Func) (reachable []bool, live []bool) {
	reachable = ReachableBlocks(f)
	var order []*Value
	live, order = liveValues(f, reachable)
	f.retDeadcodeLiveOrderStmts(order)
	return
}

// ReachableBlocks returns the reachable blocks in f.
func ReachableBlocks(f *Func) []bool {
	reachable := make([]bool, f.NumBlocks())
	reachable[f.Entry.ID] = true
	p := make([]*Block, 0, 64) // stack-like worklist
	p = append(p, f.Entry)
	for len(p) > 0 {
		// Pop a reachable block
		b := p[len(p)-1]
		p = p[:len(p)-1]
		// Mark successors as reachable
		s := b.Succs
		if b.Kind == BlockFirst {
			s = s[:1]
		}
		for _, e := range s {
			c := e.b
			if int(c.ID) >= len(reachable) {
				f.Fatalf("block %s >= f.NumBlocks()=%d?", c, len(reachable))
			}
			if !reachable[c.ID] {
				reachable[c.ID] = true
				p = append(p, c) // push
			}
		}
	}
	return reachable
}

// liveValues returns the live values in f and a list of values that are eligible
// to be statements in reversed data flow order.
// The second result is used to help conserve statement boundaries for debugging.
// reachable is a map from block ID to whether the block is reachable.
// The caller should call f.retDeadcodeLive(live) and f.retDeadcodeLiveOrderStmts(liveOrderStmts)
// when they are done with the return values.
func liveValues(f *Func, reachable []bool) (live []bool, liveOrderStmts []*Value) {
	live = f.newDeadcodeLive()
	if cap(live) < f.NumValues() {
		live = make([]bool, f.NumValues())
	} else {
		live = live[:f.NumValues()]
		for i := range live {
			live[i] = false
		}
	}

	liveOrderStmts = f.newDeadcodeLiveOrderStmts()
	liveOrderStmts = liveOrderStmts[:0]

	// After regalloc, consider all values to be live.
	// See the comment at the top of regalloc.go and in deadcode for details.
	if f.RegAlloc != nil {
		for i := range live {
			live[i] = true
		}
		return
	}

	// Record all the inline indexes we need
	var liveInlIdx map[int]bool
	pt := f.Config.ctxt.PosTable
	for _, b := range f.Blocks {
		for _, v := range b.Values {
			i := pt.Pos(v.Pos).Base().InliningIndex()
			if i < 0 {
				continue
			}
			if liveInlIdx == nil {
				liveInlIdx = map[int]bool{}
			}
			liveInlIdx[i] = true
		}
		i := pt.Pos(b.Pos).Base().InliningIndex()
		if i < 0 {
			continue
		}
		if liveInlIdx == nil {
			liveInlIdx = map[int]bool{}
		}
		liveInlIdx[i] = true
	}

	// Find all live values
	q := f.Cache.deadcode.q[:0]
	defer func() { f.Cache.deadcode.q = q }()

	// Starting set: all control values of reachable blocks are live.
	// Calls are live (because callee can observe the memory state).
	for _, b := range f.Blocks {
		if !reachable[b.ID] {
			continue
		}
		for _, v := range b.ControlValues() {
			if !live[v.ID] {
				live[v.ID] = true
				q = append(q, v)
				if v.Pos.IsStmt() != src.PosNotStmt {
					liveOrderStmts = append(liveOrderStmts, v)
				}
			}
		}
		for _, v := range b.Values {
			if (opcodeTable[v.Op].call || opcodeTable[v.Op].hasSideEffects) && !live[v.ID] {
				live[v.ID] = true
				q = append(q, v)
				if v.Pos.IsStmt() != src.PosNotStmt {
					liveOrderStmts = append(liveOrderStmts, v)
				}
			}
			if v.Type.IsVoid() && !live[v.ID] {
				// The only Void ops are nil checks and inline marks.  We must keep these.
				if v.Op == OpInlMark && !liveInlIdx[int(v.AuxInt)] {
					// We don't need marks for bodies that
					// have been completely optimized away.
					// TODO: save marks only for bodies which
					// have a faulting instruction or a call?
					continue
				}
				live[v.ID] = true
				q = append(q, v)
				if v.Pos.IsStmt() != src.PosNotStmt {
					liveOrderStmts = append(liveOrderStmts, v)
				}
			}
		}
	}

	// Compute transitive closure of live values.
	for len(q) > 0 {
		// pop a reachable value
		v := q[len(q)-1]
		q = q[:len(q)-1]
		for i, x := range v.Args {
			if v.Op == OpPhi && !reachable[v.Block.Preds[i].b.ID] {
				continue
			}
			if !live[x.ID] {
				live[x.ID] = true
				q = append(q, x) // push
				if x.Pos.IsStmt() != src.PosNotStmt {
					liveOrderStmts = append(liveOrderStmts, x)
				}
			}
		}
	}

	return
}

// deadcode removes dead code from f.
func deadcode(f *Func) {
	// deadcode after regalloc is forbidden for now. Regalloc
	// doesn't quite generate legal SSA which will lead to some
	// required moves being eliminated. See the comment at the
	// top of regalloc.go for details.
	if f.RegAlloc != nil {
		f.Fatalf("deadcode after regalloc")
	}

	// Find reachable blocks.
	reachable := ReachableBlocks(f)

	// Get rid of edges from dead to live code.
	for _, b := range f.Blocks {
		if reachable[b.ID] {
			continue
		}
		for i := 0; i < len(b.Succs); {
			e := b.Succs[i]
			if reachable[e.b.ID] {
				b.removeEdge(i)
			} else {
				i++
			}
		}
	}

	// Get rid of dead edges from live code.
	for _, b := range f.Blocks {
		if !reachable[b.ID] {
			continue
		}
		if b.Kind != BlockFirst {
			continue
		}
		b.removeEdge(1)
		b.Kind = BlockPlain
		b.Likely = BranchUnknown
	}

	// Splice out any copies introduced during dead block removal.
	copyelim(f)

	// Find live values.
	live, order := liveValues(f, reachable)
	defer f.retDeadcodeLive(live)
	defer f.retDeadcodeLiveOrderStmts(order)

	// Remove dead & duplicate entries from namedValues map.
	s := f.newSparseSet(f.NumValues())
	defer f.retSparseSet(s)
	i := 0
	for _, name := range f.Names {
		j := 0
		s.clear()
		values := f.NamedValues[name]
		for _, v := range values {
			if live[v.ID] && !s.contains(v.ID) {
				values[j] = v
				j++
				s.add(v.ID)
			}
		}
		if j == 0 {
			delete(f.NamedValues, name)
		} else {
			f.Names[i] = name
			i++
			for k := len(values) - 1; k >= j; k-- {
				values[k] = nil
			}
			f.NamedValues[name] = values[:j]
		}
	}
	for k := len(f.Names) - 1; k >= i; k-- {
		f.Names[k] = LocalSlot{}
	}
	f.Names = f.Names[:i]

	pendingLines := f.cachedLineStarts // Holds statement boundaries that need to be moved to a new value/block
	pendingLines.clear()

	// Unlink values and conserve statement boundaries
	for i, b := range f.Blocks {
		if !reachable[b.ID] {
			// TODO what if control is statement boundary? Too late here.
			b.ResetControls()
		}
		for _, v := range b.Values {
			if !live[v.ID] {
				v.resetArgs()
				if v.Pos.IsStmt() == src.PosIsStmt && reachable[b.ID] {
					pendingLines.set(v.Pos, int32(i)) // TODO could be more than one pos for a line
				}
			}
		}
	}

	// Find new homes for lost lines -- require earliest in data flow with same line that is also in same block
	for i := len(order) - 1; i >= 0; i-- {
		w := order[i]
		if j := pendingLines.get(w.Pos); j > -1 && f.Blocks[j] == w.Block {
			w.Pos = w.Pos.WithIsStmt()
			pendingLines.remove(w.Pos)
		}
	}

	// Any boundary that failed to match a live value can move to a block end
	pendingLines.foreachEntry(func(j int32, l uint, bi int32) {
		b := f.Blocks[bi]
		if b.Pos.Line() == l && b.Pos.FileIndex() == j {
			b.Pos = b.Pos.WithIsStmt()
		}
	})

	// Remove dead values from blocks' value list. Return dead
	// values to the allocator.
	for _, b := range f.Blocks {
		i := 0
		for _, v := range b.Values {
			if live[v.ID] {
				b.Values[i] = v
				i++
			} else {
				f.freeValue(v)
			}
		}
		// aid GC
		tail := b.Values[i:]
		for j := range tail {
			tail[j] = nil
		}
		b.Values = b.Values[:i]
	}

	// Remove dead blocks from WBLoads list.
	i = 0
	for _, b := range f.WBLoads {
		if reachable[b.ID] {
			f.WBLoads[i] = b
			i++
		}
	}
	for j := i; j < len(f.WBLoads); j++ {
		f.WBLoads[j] = nil
	}
	f.WBLoads = f.WBLoads[:i]

	// Remove unreachable blocks. Return dead blocks to allocator.
	i = 0
	for _, b := range f.Blocks {
		if reachable[b.ID] {
			f.Blocks[i] = b
			i++
		} else {
			if len(b.Values) > 0 {
				b.Fatalf("live values in unreachable block %v: %v", b, b.Values)
			}
			f.freeBlock(b)
		}
	}
	// zero remainder to help GC
	tail := f.Blocks[i:]
	for j := range tail {
		tail[j] = nil
	}
	f.Blocks = f.Blocks[:i]
}

// removeEdge removes the i'th outgoing edge from b (and
// the corresponding incoming edge from b.Succs[i].b).
func (b *Block) removeEdge(i int) {
	e := b.Succs[i]
	c := e.b
	j := e.i

	// Adjust b.Succs
	b.removeSucc(i)

	// Adjust c.Preds
	c.removePred(j)

	// Remove phi args from c's phis.
	n := len(c.Preds)
	for _, v := range c.Values {
		if v.Op != OpPhi {
			continue
		}
		v.Args[j].Uses--
		v.Args[j] = v.Args[n]
		v.Args[n] = nil
		v.Args = v.Args[:n]
		phielimValue(v)
		// Note: this is trickier than it looks. Replacing
		// a Phi with a Copy can in general cause problems because
		// Phi and Copy don't have exactly the same semantics.
		// Phi arguments always come from a predecessor block,
		// whereas copies don't. This matters in loops like:
		// 1: x = (Phi y)
		//    y = (Add x 1)
		//    goto 1
		// If we replace Phi->Copy, we get
		// 1: x = (Copy y)
		//    y = (Add x 1)
		//    goto 1
		// (Phi y) refers to the *previous* value of y, whereas
		// (Copy y) refers to the *current* value of y.
		// The modified code has a cycle and the scheduler
		// will barf on it.
		//
		// Fortunately, this situation can only happen for dead
		// code loops. We know the code we're working with is
		// not dead, so we're ok.
		// Proof: If we have a potential bad cycle, we have a
		// situation like this:
		//   x = (Phi z)
		//   y = (op1 x ...)
		//   z = (op2 y ...)
		// Where opX are not Phi ops. But such a situation
		// implies a cycle in the dominator graph. In the
		// example, x.Block dominates y.Block, y.Block dominates
		// z.Block, and z.Block dominates x.Block (treating
		// "dominates" as reflexive).  Cycles in the dominator
		// graph can only happen in an unreachable cycle.
	}
}
<commit_msg>cmd/compile: use optimized slice zeroing in deadcode<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ssa

import (
	"cmd/internal/src"
)

// findlive returns the reachable blocks and live values in f.
// The caller should call f.retDeadcodeLive(live) when it is done with it.
func findlive(f *Func) (reachable []bool, live []bool) {
	reachable = ReachableBlocks(f)
	var order []*Value
	live, order = liveValues(f, reachable)
	f.retDeadcodeLiveOrderStmts(order)
	return
}

// ReachableBlocks returns the reachable blocks in f.
func ReachableBlocks(f *Func) []bool {
	reachable := make([]bool, f.NumBlocks())
	reachable[f.Entry.ID] = true
	p := make([]*Block, 0, 64) // stack-like worklist
	p = append(p, f.Entry)
	for len(p) > 0 {
		// Pop a reachable block
		b := p[len(p)-1]
		p = p[:len(p)-1]
		// Mark successors as reachable
		s := b.Succs
		if b.Kind == BlockFirst {
			s = s[:1]
		}
		for _, e := range s {
			c := e.b
			if int(c.ID) >= len(reachable) {
				f.Fatalf("block %s >= f.NumBlocks()=%d?", c, len(reachable))
			}
			if !reachable[c.ID] {
				reachable[c.ID] = true
				p = append(p, c) // push
			}
		}
	}
	return reachable
}

// liveValues returns the live values in f and a list of values that are eligible
// to be statements in reversed data flow order.
// The second result is used to help conserve statement boundaries for debugging.
// reachable is a map from block ID to whether the block is reachable.
// The caller should call f.retDeadcodeLive(live) and f.retDeadcodeLiveOrderStmts(liveOrderStmts)
// when they are done with the return values.
func liveValues(f *Func, reachable []bool) (live []bool, liveOrderStmts []*Value) {
	live = f.newDeadcodeLive()
	if cap(live) < f.NumValues() {
		live = make([]bool, f.NumValues())
	} else {
		live = live[:f.NumValues()]
		for i := range live {
			live[i] = false
		}
	}

	liveOrderStmts = f.newDeadcodeLiveOrderStmts()
	liveOrderStmts = liveOrderStmts[:0]

	// After regalloc, consider all values to be live.
	// See the comment at the top of regalloc.go and in deadcode for details.
	if f.RegAlloc != nil {
		for i := range live {
			live[i] = true
		}
		return
	}

	// Record all the inline indexes we need
	var liveInlIdx map[int]bool
	pt := f.Config.ctxt.PosTable
	for _, b := range f.Blocks {
		for _, v := range b.Values {
			i := pt.Pos(v.Pos).Base().InliningIndex()
			if i < 0 {
				continue
			}
			if liveInlIdx == nil {
				liveInlIdx = map[int]bool{}
			}
			liveInlIdx[i] = true
		}
		i := pt.Pos(b.Pos).Base().InliningIndex()
		if i < 0 {
			continue
		}
		if liveInlIdx == nil {
			liveInlIdx = map[int]bool{}
		}
		liveInlIdx[i] = true
	}

	// Find all live values
	q := f.Cache.deadcode.q[:0]
	defer func() { f.Cache.deadcode.q = q }()

	// Starting set: all control values of reachable blocks are live.
	// Calls are live (because callee can observe the memory state).
	for _, b := range f.Blocks {
		if !reachable[b.ID] {
			continue
		}
		for _, v := range b.ControlValues() {
			if !live[v.ID] {
				live[v.ID] = true
				q = append(q, v)
				if v.Pos.IsStmt() != src.PosNotStmt {
					liveOrderStmts = append(liveOrderStmts, v)
				}
			}
		}
		for _, v := range b.Values {
			if (opcodeTable[v.Op].call || opcodeTable[v.Op].hasSideEffects) && !live[v.ID] {
				live[v.ID] = true
				q = append(q, v)
				if v.Pos.IsStmt() != src.PosNotStmt {
					liveOrderStmts = append(liveOrderStmts, v)
				}
			}
			if v.Type.IsVoid() && !live[v.ID] {
				// The only Void ops are nil checks and inline marks.  We must keep these.
				if v.Op == OpInlMark && !liveInlIdx[int(v.AuxInt)] {
					// We don't need marks for bodies that
					// have been completely optimized away.
					// TODO: save marks only for bodies which
					// have a faulting instruction or a call?
					continue
				}
				live[v.ID] = true
				q = append(q, v)
				if v.Pos.IsStmt() != src.PosNotStmt {
					liveOrderStmts = append(liveOrderStmts, v)
				}
			}
		}
	}

	// Compute transitive closure of live values.
	for len(q) > 0 {
		// pop a reachable value
		v := q[len(q)-1]
		q = q[:len(q)-1]
		for i, x := range v.Args {
			if v.Op == OpPhi && !reachable[v.Block.Preds[i].b.ID] {
				continue
			}
			if !live[x.ID] {
				live[x.ID] = true
				q = append(q, x) // push
				if x.Pos.IsStmt() != src.PosNotStmt {
					liveOrderStmts = append(liveOrderStmts, x)
				}
			}
		}
	}

	return
}

// deadcode removes dead code from f.
func deadcode(f *Func) {
	// deadcode after regalloc is forbidden for now. Regalloc
	// doesn't quite generate legal SSA which will lead to some
	// required moves being eliminated. See the comment at the
	// top of regalloc.go for details.
	if f.RegAlloc != nil {
		f.Fatalf("deadcode after regalloc")
	}

	// Find reachable blocks.
	reachable := ReachableBlocks(f)

	// Get rid of edges from dead to live code.
	for _, b := range f.Blocks {
		if reachable[b.ID] {
			continue
		}
		for i := 0; i < len(b.Succs); {
			e := b.Succs[i]
			if reachable[e.b.ID] {
				b.removeEdge(i)
			} else {
				i++
			}
		}
	}

	// Get rid of dead edges from live code.
	for _, b := range f.Blocks {
		if !reachable[b.ID] {
			continue
		}
		if b.Kind != BlockFirst {
			continue
		}
		b.removeEdge(1)
		b.Kind = BlockPlain
		b.Likely = BranchUnknown
	}

	// Splice out any copies introduced during dead block removal.
	copyelim(f)

	// Find live values.
	live, order := liveValues(f, reachable)
	defer f.retDeadcodeLive(live)
	defer f.retDeadcodeLiveOrderStmts(order)

	// Remove dead & duplicate entries from namedValues map.
	s := f.newSparseSet(f.NumValues())
	defer f.retSparseSet(s)
	i := 0
	for _, name := range f.Names {
		j := 0
		s.clear()
		values := f.NamedValues[name]
		for _, v := range values {
			if live[v.ID] && !s.contains(v.ID) {
				values[j] = v
				j++
				s.add(v.ID)
			}
		}
		if j == 0 {
			delete(f.NamedValues, name)
		} else {
			f.Names[i] = name
			i++
			for k := len(values) - 1; k >= j; k-- {
				values[k] = nil
			}
			f.NamedValues[name] = values[:j]
		}
	}
	clearNames := f.Names[i:]
	for j := range clearNames {
		clearNames[j] = LocalSlot{}
	}
	f.Names = f.Names[:i]

	pendingLines := f.cachedLineStarts // Holds statement boundaries that need to be moved to a new value/block
	pendingLines.clear()

	// Unlink values and conserve statement boundaries
	for i, b := range f.Blocks {
		if !reachable[b.ID] {
			// TODO what if control is statement boundary? Too late here.
			b.ResetControls()
		}
		for _, v := range b.Values {
			if !live[v.ID] {
				v.resetArgs()
				if v.Pos.IsStmt() == src.PosIsStmt && reachable[b.ID] {
					pendingLines.set(v.Pos, int32(i)) // TODO could be more than one pos for a line
				}
			}
		}
	}

	// Find new homes for lost lines -- require earliest in data flow with same line that is also in same block
	for i := len(order) - 1; i >= 0; i-- {
		w := order[i]
		if j := pendingLines.get(w.Pos); j > -1 && f.Blocks[j] == w.Block {
			w.Pos = w.Pos.WithIsStmt()
			pendingLines.remove(w.Pos)
		}
	}

	// Any boundary that failed to match a live value can move to a block end
	pendingLines.foreachEntry(func(j int32, l uint, bi int32) {
		b := f.Blocks[bi]
		if b.Pos.Line() == l && b.Pos.FileIndex() == j {
			b.Pos = b.Pos.WithIsStmt()
		}
	})

	// Remove dead values from blocks' value list. Return dead
	// values to the allocator.
	for _, b := range f.Blocks {
		i := 0
		for _, v := range b.Values {
			if live[v.ID] {
				b.Values[i] = v
				i++
			} else {
				f.freeValue(v)
			}
		}
		// aid GC
		tail := b.Values[i:]
		for j := range tail {
			tail[j] = nil
		}
		b.Values = b.Values[:i]
	}

	// Remove dead blocks from WBLoads list.
	i = 0
	for _, b := range f.WBLoads {
		if reachable[b.ID] {
			f.WBLoads[i] = b
			i++
		}
	}
	clearWBLoads := f.WBLoads[i:]
	for j := range clearWBLoads {
		clearWBLoads[j] = nil
	}
	f.WBLoads = f.WBLoads[:i]

	// Remove unreachable blocks. Return dead blocks to allocator.
	i = 0
	for _, b := range f.Blocks {
		if reachable[b.ID] {
			f.Blocks[i] = b
			i++
		} else {
			if len(b.Values) > 0 {
				b.Fatalf("live values in unreachable block %v: %v", b, b.Values)
			}
			f.freeBlock(b)
		}
	}
	// zero remainder to help GC
	tail := f.Blocks[i:]
	for j := range tail {
		tail[j] = nil
	}
	f.Blocks = f.Blocks[:i]
}

// removeEdge removes the i'th outgoing edge from b (and
// the corresponding incoming edge from b.Succs[i].b).
func (b *Block) removeEdge(i int) {
	e := b.Succs[i]
	c := e.b
	j := e.i

	// Adjust b.Succs
	b.removeSucc(i)

	// Adjust c.Preds
	c.removePred(j)

	// Remove phi args from c's phis.
	n := len(c.Preds)
	for _, v := range c.Values {
		if v.Op != OpPhi {
			continue
		}
		v.Args[j].Uses--
		v.Args[j] = v.Args[n]
		v.Args[n] = nil
		v.Args = v.Args[:n]
		phielimValue(v)
		// Note: this is trickier than it looks. Replacing
		// a Phi with a Copy can in general cause problems because
		// Phi and Copy don't have exactly the same semantics.
		// Phi arguments always come from a predecessor block,
		// whereas copies don't. This matters in loops like:
		// 1: x = (Phi y)
		//    y = (Add x 1)
		//    goto 1
		// If we replace Phi->Copy, we get
		// 1: x = (Copy y)
		//    y = (Add x 1)
		//    goto 1
		// (Phi y) refers to the *previous* value of y, whereas
		// (Copy y) refers to the *current* value of y.
		// The modified code has a cycle and the scheduler
		// will barf on it.
		//
		// Fortunately, this situation can only happen for dead
		// code loops. We know the code we're working with is
		// not dead, so we're ok.
		// Proof: If we have a potential bad cycle, we have a
		// situation like this:
		//   x = (Phi z)
		//   y = (op1 x ...)
		//   z = (op2 y ...)
		// Where opX are not Phi ops. But such a situation
		// implies a cycle in the dominator graph. In the
		// example, x.Block dominates y.Block, y.Block dominates
		// z.Block, and z.Block dominates x.Block (treating
		// "dominates" as reflexive).  Cycles in the dominator
		// graph can only happen in an unreachable cycle.
	}
}
<|endoftext|>