<commit_before>package main

import (
	"io/ioutil"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"

	log "github.com/lxc/lxd/shared/log15"
)

var instancesCmd = APIEndpoint{
	Name:    "instances",
	Path:    "instances",
	Aliases: []APIEndpointAlias{{Name: "containers", Path: "containers"}},

	Get:  APIEndpointAction{Handler: containersGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post: APIEndpointAction{Handler: containersPost, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
}

var instanceCmd = APIEndpoint{
	Name:    "instance",
	Path:    "instances/{name}",
	Aliases: []APIEndpointAlias{{Name: "container", Path: "containers/{name}"}},

	Get:    APIEndpointAction{Handler: containerGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Put:    APIEndpointAction{Handler: containerPut, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Delete: APIEndpointAction{Handler: containerDelete, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Post:   APIEndpointAction{Handler: containerPost, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Patch:  APIEndpointAction{Handler: containerPatch, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
}

var instanceStateCmd = APIEndpoint{
	Name:    "instanceState",
	Path:    "instances/{name}/state",
	Aliases: []APIEndpointAlias{{Name: "containerState", Path: "containers/{name}/state"}},

	Get: APIEndpointAction{Handler: containerState, AccessHandler: AllowProjectPermission("containers", "view")},
	Put: APIEndpointAction{Handler: containerStatePut, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceFileCmd = APIEndpoint{
	Name:    "instanceFile",
	Path:    "instances/{name}/files",
	Aliases: []APIEndpointAlias{{Name: "containerFile", Path: "containers/{name}/files"}},

	Get:    APIEndpointAction{Handler: containerFileHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Post:   APIEndpointAction{Handler: containerFileHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Delete: APIEndpointAction{Handler: containerFileHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceSnapshotsCmd = APIEndpoint{
	Name:    "instanceSnapshots",
	Path:    "instances/{name}/snapshots",
	Aliases: []APIEndpointAlias{{Name: "containerSnapshots", Path: "containers/{name}/snapshots"}},

	Get:  APIEndpointAction{Handler: containerSnapshotsGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post: APIEndpointAction{Handler: containerSnapshotsPost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceSnapshotCmd = APIEndpoint{
	Name:    "instanceSnapshot",
	Path:    "instances/{name}/snapshots/{snapshotName}",
	Aliases: []APIEndpointAlias{{Name: "containerSnapshot", Path: "containers/{name}/snapshots/{snapshotName}"}},

	Get:    APIEndpointAction{Handler: containerSnapshotHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Post:   APIEndpointAction{Handler: containerSnapshotHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Delete: APIEndpointAction{Handler: containerSnapshotHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Put:    APIEndpointAction{Handler: containerSnapshotHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceConsoleCmd = APIEndpoint{
	Name:    "instanceConsole",
	Path:    "instances/{name}/console",
	Aliases: []APIEndpointAlias{{Name: "containerConsole", Path: "containers/{name}/console"}},

	Get:    APIEndpointAction{Handler: containerConsoleLogGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post:   APIEndpointAction{Handler: containerConsolePost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Delete: APIEndpointAction{Handler: containerConsoleLogDelete, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceExecCmd = APIEndpoint{
	Name:    "instanceExec",
	Path:    "instances/{name}/exec",
	Aliases: []APIEndpointAlias{{Name: "containerExec", Path: "containers/{name}/exec"}},

	Post: APIEndpointAction{Handler: containerExecPost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceMetadataCmd = APIEndpoint{
	Name:    "instanceMetadata",
	Path:    "instances/{name}/metadata",
	Aliases: []APIEndpointAlias{{Name: "containerMetadata", Path: "containers/{name}/metadata"}},

	Get: APIEndpointAction{Handler: containerMetadataGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Put: APIEndpointAction{Handler: containerMetadataPut, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
}

var instanceMetadataTemplatesCmd = APIEndpoint{
	Name:    "instanceMetadataTemplates",
	Path:    "instances/{name}/metadata/templates",
	Aliases: []APIEndpointAlias{{Name: "containerMetadataTemplates", Path: "containers/{name}/metadata/templates"}},

	Get:    APIEndpointAction{Handler: containerMetadataTemplatesGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post:   APIEndpointAction{Handler: containerMetadataTemplatesPostPut, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Put:    APIEndpointAction{Handler: containerMetadataTemplatesPostPut, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Delete: APIEndpointAction{Handler: containerMetadataTemplatesDelete, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
}

var instanceBackupsCmd = APIEndpoint{
	Name:    "instanceBackups",
	Path:    "instances/{name}/backups",
	Aliases: []APIEndpointAlias{{Name: "containerBackups", Path: "containers/{name}/backups"}},

	Get:  APIEndpointAction{Handler: containerBackupsGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post: APIEndpointAction{Handler: containerBackupsPost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceBackupCmd = APIEndpoint{
	Name:    "instanceBackup",
	Path:    "instances/{name}/backups/{backupName}",
	Aliases: []APIEndpointAlias{{Name: "containerBackup", Path: "containers/{name}/backups/{backupName}"}},

	Get:    APIEndpointAction{Handler: containerBackupGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post:   APIEndpointAction{Handler: containerBackupPost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Delete: APIEndpointAction{Handler: containerBackupDelete, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceBackupExportCmd = APIEndpoint{
	Name:    "instanceBackupExport",
	Path:    "instances/{name}/backups/{backupName}/export",
	Aliases: []APIEndpointAlias{{Name: "containerBackupExport", Path: "containers/{name}/backups/{backupName}/export"}},

	Get: APIEndpointAction{Handler: containerBackupExportGet, AccessHandler: AllowProjectPermission("containers", "view")},
}

type containerAutostartList []Instance

func (slice containerAutostartList) Len() int {
	return len(slice)
}

func (slice containerAutostartList) Less(i, j int) bool {
	iOrder := slice[i].ExpandedConfig()["boot.autostart.priority"]
	jOrder := slice[j].ExpandedConfig()["boot.autostart.priority"]

	if iOrder != jOrder {
		iOrderInt, _ := strconv.Atoi(iOrder)
		jOrderInt, _ := strconv.Atoi(jOrder)
		return iOrderInt > jOrderInt
	}

	return slice[i].Name() < slice[j].Name()
}

func (slice containerAutostartList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func containersRestart(s *state.State) error {
	// Get all the instances
	result, err := instanceLoadNodeAll(s)
	if err != nil {
		return err
	}

	instances := []Instance{}

	for _, c := range result {
		instances = append(instances, c)
	}

	sort.Sort(containerAutostartList(instances))

	// Restart the instances
	for _, c := range instances {
		config := c.ExpandedConfig()
		lastState := config["volatile.last_state.power"]

		autoStart := config["boot.autostart"]
		autoStartDelay := config["boot.autostart.delay"]

		if shared.IsTrue(autoStart) || (autoStart == "" && lastState == "RUNNING") {
			if c.IsRunning() {
				continue
			}

			err = c.Start(false)
			if err != nil {
				logger.Errorf("Failed to start container '%s': %v", c.Name(), err)
			}

			autoStartDelayInt, err := strconv.Atoi(autoStartDelay)
			if err == nil {
				time.Sleep(time.Duration(autoStartDelayInt) * time.Second)
			}
		}
	}

	return nil
}

type containerStopList []Instance

func (slice containerStopList) Len() int {
	return len(slice)
}

func (slice containerStopList) Less(i, j int) bool {
	iOrder := slice[i].ExpandedConfig()["boot.stop.priority"]
	jOrder := slice[j].ExpandedConfig()["boot.stop.priority"]

	if iOrder != jOrder {
		iOrderInt, _ := strconv.Atoi(iOrder)
		jOrderInt, _ := strconv.Atoi(jOrder)
		return iOrderInt > jOrderInt // check this line (prob <)
	}

	return slice[i].Name() < slice[j].Name()
}

func (slice containerStopList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

// Return the names of all local containers, grouped by project. The
// information is obtained by reading the data directory.
func containersOnDisk() (map[string][]string, error) {
	containers := map[string][]string{}

	files, err := ioutil.ReadDir(shared.VarPath("containers"))
	if err != nil {
		return nil, err
	}

	for _, file := range files {
		name := file.Name()
		project := "default"
		if strings.Contains(name, "_") {
			fields := strings.Split(file.Name(), "_")
			project = fields[0]
			name = fields[1]
		}
		names, ok := containers[project]
		if !ok {
			names = []string{}
		}
		containers[project] = append(names, name)
	}

	return containers, nil
}

func containersShutdown(s *state.State) error {
	var wg sync.WaitGroup

	dbAvailable := true

	// Get all the instances
	instances, err := instanceLoadNodeAll(s)
	if err != nil {
		// Mark database as offline
		dbAvailable = false
		instances = []Instance{}

		// List all containers on disk
		cnames, err := containersOnDisk()
		if err != nil {
			return err
		}

		for project, names := range cnames {
			for _, name := range names {
				inst, err := instanceLoad(s, db.InstanceArgs{
					Project: project,
					Name:    name,
					Config:  make(map[string]string),
				}, nil)
				if err != nil {
					return err
				}

				instances = append(instances, inst)
			}
		}
	}

	sort.Sort(containerStopList(instances))

	if dbAvailable {
		// Reset all container states
		err = s.Cluster.ContainersResetState()
		if err != nil {
			return err
		}
	}

	var lastPriority int

	if len(instances) != 0 {
		lastPriority, _ = strconv.Atoi(instances[0].ExpandedConfig()["boot.stop.priority"])
	}

	for _, c := range instances {
		priority, _ := strconv.Atoi(c.ExpandedConfig()["boot.stop.priority"])

		// Enforce shutdown priority
		if priority != lastPriority {
			lastPriority = priority

			// Wait for instances with higher priority to finish
			wg.Wait()
		}

		// Record the current state
		lastState := c.State()

		// Stop the container
		if lastState != "BROKEN" && lastState != "STOPPED" {
			// Determinate how long to wait for the instance to shutdown cleanly
			var timeoutSeconds int
			value, ok := c.ExpandedConfig()["boot.host_shutdown_timeout"]
			if ok {
				timeoutSeconds, _ = strconv.Atoi(value)
			} else {
				timeoutSeconds = 30
			}

			// Stop the instance
			wg.Add(1)
			go func(c Instance, lastState string) {
				c.Shutdown(time.Second * time.Duration(timeoutSeconds))
				c.Stop(false)
				c.VolatileSet(map[string]string{"volatile.last_state.power": lastState})

				wg.Done()
			}(c, lastState)
		} else {
			c.VolatileSet(map[string]string{"volatile.last_state.power": lastState})
		}
	}
	wg.Wait()

	return nil
}

func containerDeleteSnapshots(s *state.State, project, cname string) error {
	results, err := s.Cluster.ContainerGetSnapshots(project, cname)
	if err != nil {
		return err
	}

	for _, sname := range results {
		sc, err := instanceLoadByProjectAndName(s, project, sname)
		if err != nil {
			logger.Error(
				"containerDeleteSnapshots: Failed to load the snapshot container",
				log.Ctx{"container": cname, "snapshot": sname, "err": err})

			continue
		}

		if err := sc.Delete(); err != nil {
			logger.Error(
				"containerDeleteSnapshots: Failed to delete a snapshot container",
				log.Ctx{"container": cname, "snapshot": sname, "err": err})
		}
	}

	return nil
}
<commit_msg>lxd/containers: Renames containerDeleteSnapshots to instanceDeleteSnapshots<commit_after>package main

import (
	"io/ioutil"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/logger"

	log "github.com/lxc/lxd/shared/log15"
)

var instancesCmd = APIEndpoint{
	Name:    "instances",
	Path:    "instances",
	Aliases: []APIEndpointAlias{{Name: "containers", Path: "containers"}},

	Get:  APIEndpointAction{Handler: containersGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post: APIEndpointAction{Handler: containersPost, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
}

var instanceCmd = APIEndpoint{
	Name:    "instance",
	Path:    "instances/{name}",
	Aliases: []APIEndpointAlias{{Name: "container", Path: "containers/{name}"}},

	Get:    APIEndpointAction{Handler: containerGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Put:    APIEndpointAction{Handler: containerPut, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Delete: APIEndpointAction{Handler: containerDelete, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Post:   APIEndpointAction{Handler: containerPost, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Patch:  APIEndpointAction{Handler: containerPatch, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
}

var instanceStateCmd = APIEndpoint{
	Name:    "instanceState",
	Path:    "instances/{name}/state",
	Aliases: []APIEndpointAlias{{Name: "containerState", Path: "containers/{name}/state"}},

	Get: APIEndpointAction{Handler: containerState, AccessHandler: AllowProjectPermission("containers", "view")},
	Put: APIEndpointAction{Handler: containerStatePut, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceFileCmd = APIEndpoint{
	Name:    "instanceFile",
	Path:    "instances/{name}/files",
	Aliases: []APIEndpointAlias{{Name: "containerFile", Path: "containers/{name}/files"}},

	Get:    APIEndpointAction{Handler: containerFileHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Post:   APIEndpointAction{Handler: containerFileHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Delete: APIEndpointAction{Handler: containerFileHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceSnapshotsCmd = APIEndpoint{
	Name:    "instanceSnapshots",
	Path:    "instances/{name}/snapshots",
	Aliases: []APIEndpointAlias{{Name: "containerSnapshots", Path: "containers/{name}/snapshots"}},

	Get:  APIEndpointAction{Handler: containerSnapshotsGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post: APIEndpointAction{Handler: containerSnapshotsPost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceSnapshotCmd = APIEndpoint{
	Name:    "instanceSnapshot",
	Path:    "instances/{name}/snapshots/{snapshotName}",
	Aliases: []APIEndpointAlias{{Name: "containerSnapshot", Path: "containers/{name}/snapshots/{snapshotName}"}},

	Get:    APIEndpointAction{Handler: containerSnapshotHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Post:   APIEndpointAction{Handler: containerSnapshotHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Delete: APIEndpointAction{Handler: containerSnapshotHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Put:    APIEndpointAction{Handler: containerSnapshotHandler, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceConsoleCmd = APIEndpoint{
	Name:    "instanceConsole",
	Path:    "instances/{name}/console",
	Aliases: []APIEndpointAlias{{Name: "containerConsole", Path: "containers/{name}/console"}},

	Get:    APIEndpointAction{Handler: containerConsoleLogGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post:   APIEndpointAction{Handler: containerConsolePost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Delete: APIEndpointAction{Handler: containerConsoleLogDelete, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceExecCmd = APIEndpoint{
	Name:    "instanceExec",
	Path:    "instances/{name}/exec",
	Aliases: []APIEndpointAlias{{Name: "containerExec", Path: "containers/{name}/exec"}},

	Post: APIEndpointAction{Handler: containerExecPost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceMetadataCmd = APIEndpoint{
	Name:    "instanceMetadata",
	Path:    "instances/{name}/metadata",
	Aliases: []APIEndpointAlias{{Name: "containerMetadata", Path: "containers/{name}/metadata"}},

	Get: APIEndpointAction{Handler: containerMetadataGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Put: APIEndpointAction{Handler: containerMetadataPut, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
}

var instanceMetadataTemplatesCmd = APIEndpoint{
	Name:    "instanceMetadataTemplates",
	Path:    "instances/{name}/metadata/templates",
	Aliases: []APIEndpointAlias{{Name: "containerMetadataTemplates", Path: "containers/{name}/metadata/templates"}},

	Get:    APIEndpointAction{Handler: containerMetadataTemplatesGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post:   APIEndpointAction{Handler: containerMetadataTemplatesPostPut, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Put:    APIEndpointAction{Handler: containerMetadataTemplatesPostPut, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
	Delete: APIEndpointAction{Handler: containerMetadataTemplatesDelete, AccessHandler: AllowProjectPermission("containers", "manage-containers")},
}

var instanceBackupsCmd = APIEndpoint{
	Name:    "instanceBackups",
	Path:    "instances/{name}/backups",
	Aliases: []APIEndpointAlias{{Name: "containerBackups", Path: "containers/{name}/backups"}},

	Get:  APIEndpointAction{Handler: containerBackupsGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post: APIEndpointAction{Handler: containerBackupsPost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceBackupCmd = APIEndpoint{
	Name:    "instanceBackup",
	Path:    "instances/{name}/backups/{backupName}",
	Aliases: []APIEndpointAlias{{Name: "containerBackup", Path: "containers/{name}/backups/{backupName}"}},

	Get:    APIEndpointAction{Handler: containerBackupGet, AccessHandler: AllowProjectPermission("containers", "view")},
	Post:   APIEndpointAction{Handler: containerBackupPost, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
	Delete: APIEndpointAction{Handler: containerBackupDelete, AccessHandler: AllowProjectPermission("containers", "operate-containers")},
}

var instanceBackupExportCmd = APIEndpoint{
	Name:    "instanceBackupExport",
	Path:    "instances/{name}/backups/{backupName}/export",
	Aliases: []APIEndpointAlias{{Name: "containerBackupExport", Path: "containers/{name}/backups/{backupName}/export"}},

	Get: APIEndpointAction{Handler: containerBackupExportGet, AccessHandler: AllowProjectPermission("containers", "view")},
}

type containerAutostartList []Instance

func (slice containerAutostartList) Len() int {
	return len(slice)
}

func (slice containerAutostartList) Less(i, j int) bool {
	iOrder := slice[i].ExpandedConfig()["boot.autostart.priority"]
	jOrder := slice[j].ExpandedConfig()["boot.autostart.priority"]

	if iOrder != jOrder {
		iOrderInt, _ := strconv.Atoi(iOrder)
		jOrderInt, _ := strconv.Atoi(jOrder)
		return iOrderInt > jOrderInt
	}

	return slice[i].Name() < slice[j].Name()
}

func (slice containerAutostartList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

func containersRestart(s *state.State) error {
	// Get all the instances
	result, err := instanceLoadNodeAll(s)
	if err != nil {
		return err
	}

	instances := []Instance{}

	for _, c := range result {
		instances = append(instances, c)
	}

	sort.Sort(containerAutostartList(instances))

	// Restart the instances
	for _, c := range instances {
		config := c.ExpandedConfig()
		lastState := config["volatile.last_state.power"]

		autoStart := config["boot.autostart"]
		autoStartDelay := config["boot.autostart.delay"]

		if shared.IsTrue(autoStart) || (autoStart == "" && lastState == "RUNNING") {
			if c.IsRunning() {
				continue
			}

			err = c.Start(false)
			if err != nil {
				logger.Errorf("Failed to start container '%s': %v", c.Name(), err)
			}

			autoStartDelayInt, err := strconv.Atoi(autoStartDelay)
			if err == nil {
				time.Sleep(time.Duration(autoStartDelayInt) * time.Second)
			}
		}
	}

	return nil
}

type containerStopList []Instance

func (slice containerStopList) Len() int {
	return len(slice)
}

func (slice containerStopList) Less(i, j int) bool {
	iOrder := slice[i].ExpandedConfig()["boot.stop.priority"]
	jOrder := slice[j].ExpandedConfig()["boot.stop.priority"]

	if iOrder != jOrder {
		iOrderInt, _ := strconv.Atoi(iOrder)
		jOrderInt, _ := strconv.Atoi(jOrder)
		return iOrderInt > jOrderInt // check this line (prob <)
	}

	return slice[i].Name() < slice[j].Name()
}

func (slice containerStopList) Swap(i, j int) {
	slice[i], slice[j] = slice[j], slice[i]
}

// Return the names of all local containers, grouped by project. The
// information is obtained by reading the data directory.
func containersOnDisk() (map[string][]string, error) {
	containers := map[string][]string{}

	files, err := ioutil.ReadDir(shared.VarPath("containers"))
	if err != nil {
		return nil, err
	}

	for _, file := range files {
		name := file.Name()
		project := "default"
		if strings.Contains(name, "_") {
			fields := strings.Split(file.Name(), "_")
			project = fields[0]
			name = fields[1]
		}
		names, ok := containers[project]
		if !ok {
			names = []string{}
		}
		containers[project] = append(names, name)
	}

	return containers, nil
}

func containersShutdown(s *state.State) error {
	var wg sync.WaitGroup

	dbAvailable := true

	// Get all the instances
	instances, err := instanceLoadNodeAll(s)
	if err != nil {
		// Mark database as offline
		dbAvailable = false
		instances = []Instance{}

		// List all containers on disk
		cnames, err := containersOnDisk()
		if err != nil {
			return err
		}

		for project, names := range cnames {
			for _, name := range names {
				inst, err := instanceLoad(s, db.InstanceArgs{
					Project: project,
					Name:    name,
					Config:  make(map[string]string),
				}, nil)
				if err != nil {
					return err
				}

				instances = append(instances, inst)
			}
		}
	}

	sort.Sort(containerStopList(instances))

	if dbAvailable {
		// Reset all container states
		err = s.Cluster.ContainersResetState()
		if err != nil {
			return err
		}
	}

	var lastPriority int

	if len(instances) != 0 {
		lastPriority, _ = strconv.Atoi(instances[0].ExpandedConfig()["boot.stop.priority"])
	}

	for _, c := range instances {
		priority, _ := strconv.Atoi(c.ExpandedConfig()["boot.stop.priority"])

		// Enforce shutdown priority
		if priority != lastPriority {
			lastPriority = priority

			// Wait for instances with higher priority to finish
			wg.Wait()
		}

		// Record the current state
		lastState := c.State()

		// Stop the container
		if lastState != "BROKEN" && lastState != "STOPPED" {
			// Determinate how long to wait for the instance to shutdown cleanly
			var timeoutSeconds int
			value, ok := c.ExpandedConfig()["boot.host_shutdown_timeout"]
			if ok {
				timeoutSeconds, _ = strconv.Atoi(value)
			} else {
				timeoutSeconds = 30
			}

			// Stop the instance
			wg.Add(1)
			go func(c Instance, lastState string) {
				c.Shutdown(time.Second * time.Duration(timeoutSeconds))
				c.Stop(false)
				c.VolatileSet(map[string]string{"volatile.last_state.power": lastState})

				wg.Done()
			}(c, lastState)
		} else {
			c.VolatileSet(map[string]string{"volatile.last_state.power": lastState})
		}
	}
	wg.Wait()

	return nil
}

// instanceDeleteSnapshots calls the Delete() function on each of the supplied instance's snapshots.
func instanceDeleteSnapshots(s *state.State, projectName, instanceName string) error {
	results, err := s.Cluster.ContainerGetSnapshots(projectName, instanceName)
	if err != nil {
		return err
	}

	for _, snapName := range results {
		snapInst, err := instanceLoadByProjectAndName(s, projectName, snapName)
		if err != nil {
			logger.Error("instanceDeleteSnapshots: Failed to load the snapshot", log.Ctx{"project": projectName, "instance": instanceName, "snapshot": snapName, "err": err})
			continue
		}

		if err := snapInst.Delete(); err != nil {
			logger.Error("instanceDeleteSnapshots: Failed to delete the snapshot", log.Ctx{"project": projectName, "instance": instanceName, "snapshot": snapName, "err": err})
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package lzb

type readBuffer struct {
	*buffer
	dict *dict
	head int64
}

func (r *readBuffer) Seek(offset int64, whence int) (off int64, err error) {
	switch whence {
	case 0:
		off = offset
	case 1:
		off = r.head + offset
	case 2:
		off = r.top + offset
	default:
		return r.head, errWhence
	}
	if !(r.bottom <= off && off <= r.top) {
		return r.head, errOffset
	}
	r.head = off
	r.writeLimit = off + int64(r.capacity())
	return
}

func newReadBuffer(capacity int64, dictsize int64) (r *readBuffer, err error) {
	b, err := newBuffer(capacity)
	if err != nil {
		return nil, err
	}
	d, err := newDict(b, 0, dictsize)
	if err != nil {
		return nil, err
	}
	r = &readBuffer{buffer: b, dict: d, head: 0}
	return r, nil
}

func (r *readBuffer) Read(p []byte) (n int, err error) {
	n, err = r.ReadAt(p, r.head)
	if err != nil {
		return 0, err
	}
	_, err = r.Seek(int64(n), 1)
	if err != nil {
		return 0, err
	}
	return
}

func (r *readBuffer) Write(p []byte) (n int, err error) {
	n, err = r.Write(p)
	_, serr := r.dict.Seek(int64(n), 1)
	if err == nil {
		err = serr
	}
	return
}

func (r *readBuffer) WriteByte(c byte) error {
	err := r.WriteByte(c)
	if err != nil {
		return err
	}
	_, err = r.dict.Seek(1, 1)
	return err
}
<commit_msg>lzb: fixed bug in readBuffer.Write and readBuffer.WriteByte<commit_after>package lzb

type readBuffer struct {
	*buffer
	dict *dict
	head int64
}

func (r *readBuffer) Seek(offset int64, whence int) (off int64, err error) {
	switch whence {
	case 0:
		off = offset
	case 1:
		off = r.head + offset
	case 2:
		off = r.top + offset
	default:
		return r.head, errWhence
	}
	if !(r.bottom <= off && off <= r.top) {
		return r.head, errOffset
	}
	r.head = off
	r.writeLimit = off + int64(r.capacity())
	return
}

func newReadBuffer(capacity int64, dictsize int64) (r *readBuffer, err error) {
	b, err := newBuffer(capacity)
	if err != nil {
		return nil, err
	}
	d, err := newDict(b, 0, dictsize)
	if err != nil {
		return nil, err
	}
	r = &readBuffer{buffer: b, dict: d, head: 0}
	return r, nil
}

func (r *readBuffer) Read(p []byte) (n int, err error) {
	n, err = r.ReadAt(p, r.head)
	if err != nil {
		return 0, err
	}
	_, err = r.Seek(int64(n), 1)
	if err != nil {
		return 0, err
	}
	return
}

func (r *readBuffer) Write(p []byte) (n int, err error) {
	n, err = r.buffer.Write(p)
	_, serr := r.dict.Seek(int64(n), 1)
	if err == nil {
		err = serr
	}
	return
}

func (r *readBuffer) WriteByte(c byte) error {
	err := r.buffer.WriteByte(c)
	if err != nil {
		return err
	}
	_, err = r.dict.Seek(1, 1)
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ava-labs/avalanchego/chains"
	"github.com/ava-labs/avalanchego/database/manager"
	"github.com/ava-labs/avalanchego/node"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/logging"
	"os"
	"path/filepath"
	"runtime"
	"syscall"
)

type migrationManager struct {
	nodeManager *nodeManager
	rootConfig  node.Config
	log         logging.Logger
}

func newMigrationManager(nodeManager *nodeManager, rootConfig node.Config, log logging.Logger) *migrationManager {
	return &migrationManager{
		nodeManager: nodeManager,
		rootConfig:  rootConfig,
		log:         log,
	}
}

// Runs migration if required. See runMigration().
func (m *migrationManager) migrate() error {
	shouldMigrate, err := m.shouldMigrate()
	if err != nil {
		return err
	}
	if !shouldMigrate {
		return nil
	}
	vdErr := m.verifyDiskStorage()
	if vdErr != nil {
		return vdErr
	}

	return m.runMigration()
}

func dirSize(path string) (uint64, error) {
	var size int64
	err := filepath.Walk(path,
		func(_ string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() {
				size += info.Size()
			}
			return err
		})
	return uint64(size), err
}

func windowsVerifyDiskStorage(path string) (uint64, uint64, error) {
	return 0, 0, fmt.Errorf("storage space verification not yet implemented for windows")
}

func unixVerifyDiskStorage(storagePath string) (uint64, uint64, error) {
	var stat syscall.Statfs_t
	err := syscall.Statfs(storagePath, &stat)
	if err != nil {
		return 0, 0, err
	}
	size, dsErr := dirSize(storagePath)
	if dsErr != nil {
		return 0, 0, dsErr
	}
	avail := stat.Bavail * uint64(stat.Bsize)
	twox := size + size
	saftyBuf := (twox * 15) / 100
	return avail, size + saftyBuf, nil
}

func (m *migrationManager) verifyDiskStorage() error {
	storagePath := m.rootConfig.DBPath
	avail, required, err := uint64(0), uint64(0), error(nil)
	if runtime.GOOS == "windows" {
		avail, required, err = windowsVerifyDiskStorage(storagePath)
	} else {
		avail, required, err = unixVerifyDiskStorage(storagePath)
	}
	if err != nil {
		return err
	}
	if avail < required {
		return fmt.Errorf("available space %d is less then required space %d for migration", avail, required)
	}
	if avail < 214748364800 {
		print("WARNING: 200G available is recommended")
	}
	return nil
}

// Returns true if the database should be migrated from the previous database version.
// Should migrate if the previous database version exists and
// if the latest database version has not finished bootstrapping.
func (m *migrationManager) shouldMigrate() (bool, error) {
	if !m.rootConfig.DBEnabled {
		return false, nil
	}
	dbManager, err := manager.New(m.rootConfig.DBPath, logging.NoLog{}, node.DatabaseVersion, true)
	if err != nil {
		return false, fmt.Errorf("couldn't create db manager at %s: %w", m.rootConfig.DBPath, err)
	}
	defer func() {
		if err := dbManager.Close(); err != nil {
			m.log.Error("error closing db manager: %s", err)
		}
	}()

	currentDBBootstrapped, err := dbManager.Current().Database.Has(chains.BootstrappedKey)
	if err != nil {
		return false, fmt.Errorf("couldn't get if database version %s is bootstrapped: %w", node.DatabaseVersion, err)
	}
	if currentDBBootstrapped {
		return false, nil
	}
	_, exists := dbManager.Previous()
	return exists, nil
}

// Run two nodes simultaneously: one is a version before the database upgrade and the other after.
// The latter will bootstrap from the former.
// When the new node version is done bootstrapping, both nodes are stopped.
// Returns nil if the new node version successfully bootstrapped.
// Some configuration flags are modified before being passed into the 2 nodes.
func (m *migrationManager) runMigration() error {
	m.log.Info("starting database migration")
	m.nodeManager.lock.Lock()
	if m.nodeManager.hasShutdown {
		m.nodeManager.lock.Unlock()
		return nil
	}

	preDBUpgradeNode, err := m.nodeManager.preDBUpgradeNode()
	if err != nil {
		m.nodeManager.lock.Unlock()
		return fmt.Errorf("couldn't create pre-upgrade node during migration: %w", err)
	}
	m.log.Info("starting pre-database upgrade node")
	preDBUpgradeNodeExitCodeChan := preDBUpgradeNode.start()
	defer func() {
		if err := m.nodeManager.Stop(preDBUpgradeNode.path); err != nil {
			m.log.Error("%s", fmt.Errorf("error while stopping node at %s: %s", preDBUpgradeNode.path, err))
		}
	}()

	m.log.Info("starting latest node version")
	latestVersion, err := m.nodeManager.latestVersionNodeFetchOnly(m.rootConfig)
	if err != nil {
		m.nodeManager.lock.Unlock()
		return fmt.Errorf("couldn't create latest version during migration: %w", err)
	}
	latestVersionExitCodeChan := latestVersion.start()
	defer func() {
		if err := m.nodeManager.Stop(latestVersion.path); err != nil {
			m.log.Error("error while stopping latest version node: %s", err)
		}
	}()
	m.nodeManager.lock.Unlock()

	// Wait until one of the nodes finishes.
	// If the bootstrapping node finishes with an exit code other than
	// the one indicating it is done bootstrapping, error.
	select {
	case exitCode := <-preDBUpgradeNodeExitCodeChan:
		// If this node ended because the node manager shut down,
		// don't return an error
		m.nodeManager.lock.Lock()
		hasShutdown := m.nodeManager.hasShutdown
		m.nodeManager.lock.Unlock()
		if hasShutdown {
			return nil
		}
		return fmt.Errorf("previous version node stopped with exit code %d", exitCode)
	case exitCode := <-latestVersionExitCodeChan:
		if exitCode != constants.ExitCodeDoneMigrating {
			return fmt.Errorf("latest version died with exit code %d", exitCode)
		}

		return nil
	}
}
<commit_msg>address linting issues<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"syscall"

	"github.com/ava-labs/avalanchego/chains"
	"github.com/ava-labs/avalanchego/database/manager"
	"github.com/ava-labs/avalanchego/node"
	"github.com/ava-labs/avalanchego/utils/constants"
	"github.com/ava-labs/avalanchego/utils/logging"
)

type migrationManager struct {
	nodeManager *nodeManager
	rootConfig  node.Config
	log         logging.Logger
}

func newMigrationManager(nodeManager *nodeManager, rootConfig node.Config, log logging.Logger) *migrationManager {
	return &migrationManager{
		nodeManager: nodeManager,
		rootConfig:  rootConfig,
		log:         log,
	}
}

// Runs migration if required. See runMigration().
func (m *migrationManager) migrate() error {
	shouldMigrate, err := m.shouldMigrate()
	if err != nil {
		return err
	}
	if !shouldMigrate {
		return nil
	}
	vdErr := m.verifyDiskStorage()
	if vdErr != nil {
		return vdErr
	}

	return m.runMigration()
}

func dirSize(path string) (uint64, error) {
	var size int64
	err := filepath.Walk(path,
		func(_ string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() {
				size += info.Size()
			}
			return err
		})
	return uint64(size), err
}

func windowsVerifyDiskStorage(path string) (uint64, uint64, error) {
	return 0, 0, fmt.Errorf("storage space verification not yet implemented for windows")
}

func unixVerifyDiskStorage(storagePath string) (uint64, uint64, error) {
	var stat syscall.Statfs_t
	err := syscall.Statfs(storagePath, &stat)
	if err != nil {
		return 0, 0, err
	}
	size, dsErr := dirSize(storagePath)
	if dsErr != nil {
		return 0, 0, dsErr
	}
	avail := stat.Bavail * uint64(stat.Bsize)
	twox := size + size
	saftyBuf := (twox * 15) / 100
	return avail, size + saftyBuf, nil
}

func (m *migrationManager) verifyDiskStorage() error {
	storagePath := m.rootConfig.DBPath
	var avail uint64
	var required uint64
	var err error
	if runtime.GOOS == "windows" {
		avail, required, err = windowsVerifyDiskStorage(storagePath)
	} else {
		avail, required, err = unixVerifyDiskStorage(storagePath)
	}
	if err != nil {
		return err
	}
	if avail < required {
		return fmt.Errorf("available space %d is less then required space %d for migration", avail, required)
	}
	if avail < 214748364800 {
		print("WARNING: 200G available is recommended")
	}
	return nil
}

// Returns true if the database should be migrated from the previous database version.
// Should migrate if the previous database version exists and
// if the latest database version has not finished bootstrapping.
func (m *migrationManager) shouldMigrate() (bool, error) {
	if !m.rootConfig.DBEnabled {
		return false, nil
	}
	dbManager, err := manager.New(m.rootConfig.DBPath, logging.NoLog{}, node.DatabaseVersion, true)
	if err != nil {
		return false, fmt.Errorf("couldn't create db manager at %s: %w", m.rootConfig.DBPath, err)
	}
	defer func() {
		if err := dbManager.Close(); err != nil {
			m.log.Error("error closing db manager: %s", err)
		}
	}()

	currentDBBootstrapped, err := dbManager.Current().Database.Has(chains.BootstrappedKey)
	if err != nil {
		return false, fmt.Errorf("couldn't get if database version %s is bootstrapped: %w", node.DatabaseVersion, err)
	}
	if currentDBBootstrapped {
		return false, nil
	}
	_, exists := dbManager.Previous()
	return exists, nil
}

// Run two nodes simultaneously: one is a version before the database upgrade and the other after.
// The latter will bootstrap from the former.
// When the new node version is done bootstrapping, both nodes are stopped.
// Returns nil if the new node version successfully bootstrapped.
// Some configuration flags are modified before being passed into the 2 nodes.
func (m *migrationManager) runMigration() error {
	m.log.Info("starting database migration")
	m.nodeManager.lock.Lock()
	if m.nodeManager.hasShutdown {
		m.nodeManager.lock.Unlock()
		return nil
	}

	preDBUpgradeNode, err := m.nodeManager.preDBUpgradeNode()
	if err != nil {
		m.nodeManager.lock.Unlock()
		return fmt.Errorf("couldn't create pre-upgrade node during migration: %w", err)
	}
	m.log.Info("starting pre-database upgrade node")
	preDBUpgradeNodeExitCodeChan := preDBUpgradeNode.start()
	defer func() {
		if err := m.nodeManager.Stop(preDBUpgradeNode.path); err != nil {
			m.log.Error("%s", fmt.Errorf("error while stopping node at %s: %s", preDBUpgradeNode.path, err))
		}
	}()

	m.log.Info("starting latest node version")
	latestVersion, err := m.nodeManager.latestVersionNodeFetchOnly(m.rootConfig)
	if err != nil {
		m.nodeManager.lock.Unlock()
		return fmt.Errorf("couldn't create latest version during migration: %w", err)
	}
	latestVersionExitCodeChan := latestVersion.start()
	defer func() {
		if err := m.nodeManager.Stop(latestVersion.path); err != nil {
			m.log.Error("error while stopping latest version node: %s", err)
		}
	}()
	m.nodeManager.lock.Unlock()

	// Wait until one of the nodes finishes.
	// If the bootstrapping node finishes with an exit code other than
	// the one indicating it is done bootstrapping, error.
	select {
	case exitCode := <-preDBUpgradeNodeExitCodeChan:
		// If this node ended because the node manager shut down,
		// don't return an error
		m.nodeManager.lock.Lock()
		hasShutdown := m.nodeManager.hasShutdown
		m.nodeManager.lock.Unlock()
		if hasShutdown {
			return nil
		}
		return fmt.Errorf("previous version node stopped with exit code %d", exitCode)
	case exitCode := <-latestVersionExitCodeChan:
		if exitCode != constants.ExitCodeDoneMigrating {
			return fmt.Errorf("latest version died with exit code %d", exitCode)
		}

		return nil
	}
}
<|endoftext|>


<commit_before>package manager

import (
	"fmt"
	"github.com/sellweek/TOGY/updater"
	"os"
	"strconv"
	"time"
)

func updateManager(mgr *Manager, t <-chan time.Time) {
	for {
		select {
		case _ = <-t:
			mgr.config.Log.Println("Getting update info")
			i, err := updater.GetInfo(mgr.config)
			if err != nil {
				mgr.config.Log.Println("Error while downloading info: ", err)
				continue
			}
			if i.Broadcast {
				mgr.config.Log.Println("Updating broadcast")
				err = updateBroadcast(mgr, i.FileType)
				if err != nil {
					mgr.config.Log.Println(err)
				}
				mgr.config.Log.Println("Broadcast updated")
			}

			if i.Config {
				mgr.config.Log.Println("Updating config")
				err = updateConfig(mgr)
				if err != nil {
					mgr.config.Log.Println(err)
				}
				mgr.reloadSignal <- true
				mgr.config.Log.Println("Config updated, restarting manager.")
				return
			}
		}
	}
}

func updateConfig(mgr *Manager) error {
	err := updater.DownloadConfig(mgr.config, mgr.config.CentralPath)
	if err != nil {
		return fmt.Errorf("Error while updating config: %v", err)
	}
	mgr.config.Log.Println("Config succesfully downloaded, announcing.")
	err = updater.AnnounceConfig(mgr.config)
	if err != nil {
		return fmt.Errorf("Error while announcing the download of config: %v", err)
	}
	return nil
}

func updateBroadcast(mgr *Manager, ft string) error {
	path, err := makeTempDir()
	if err != nil {
		return fmt.Errorf("Error while creating temporary directory: %v", err)
	}
	err = updater.DownloadBroadcast(mgr.config, ft, path)
	if err != nil {
		return fmt.Errorf("Error while downloading new broadcast: %v", err)
	}
	mgr.config.Log.Println("Broadcast successfully downloaded, announcing.")

	err = updater.AnnounceBroadcast(mgr.config)
	if err != nil {
		return fmt.Errorf("Error while announcing the download of broadcast: %v", err)
	}

	err = mgr.stopBroadcast()
	if err != nil {
		return fmt.Errorf("Error while stopping broadcast for update: %v", err)
	}

	mgr.block()
	defer mgr.unblock()

	mgr.config.Log.Println("Moving new broadcast into place")
	err = deleteAll(mgr.config.BroadcastDir)
	if err != nil {
		return fmt.Errorf("Error while deleting old broadcast: %v", err)
	}

	err = moveFiles(path, mgr.config.BroadcastDir)
	if err != nil {
		return fmt.Errorf("Error while moving new broadcast into place: %v", err)
	}

	err = mgr.startBroadcast()
	if err != nil {
		return fmt.Errorf("Error while starting new broadcast: %v", err)
	}

	_ = deleteAll(path)
	return nil
}

func makeTempDir() (path string, err error) {
	path = os.TempDir() + string(os.PathSeparator) + "broadcast-download-" + strconv.Itoa(int(time.Now().Unix()))
	err = os.Mkdir(path, os.ModePerm)
	if err != nil {
		return
	}
	return
}

func moveFiles(src, dest string) (err error) {
	sf, err := os.Open(src)
	if err != nil {
		return
	}

	fns, err := sf.Readdirnames(0)
	if err != nil {
		return
	}

	for _, name := range fns {
		path := src + string(os.PathSeparator) + name
		err = os.Rename(path, dest+string(os.PathSeparator)+name)
		if err != nil {
			return
		}
	}
	return
}

func deleteAll(dir string) (err error) {
	d, err := os.Open(dir)
	if err != nil {
		return
	}

	fns, err := d.Readdirnames(0)
	if err != nil {
		return
	}

	for _, f := range fns {
		path := dir + string(os.PathSeparator) + f
		err = os.Remove(path)
		if err != nil {
			return
		}
	}
	return nil
}
<commit_msg>Added documentation in update.go<commit_after>package manager

import (
	"fmt"
	"github.com/sellweek/TOGY/updater"
	"os"
	"strconv"
	"time"
)

//updateManager receives messages from a Ticker,
//checking server for update info and downloading
//updated config or broadcast.
func updateManager(mgr *Manager, t <-chan time.Time) {
	for {
		select {
		case _ = <-t:
			mgr.config.Log.Println("Getting update info")
			i, err := updater.GetInfo(mgr.config)
			if err != nil {
				mgr.config.Log.Println("Error while downloading info: ", err)
				continue
			}
			if i.Broadcast {
				mgr.config.Log.Println("Updating broadcast")
				err = updateBroadcast(mgr, i.FileType)
				if err != nil {
					mgr.config.Log.Println(err)
				}
				mgr.config.Log.Println("Broadcast updated")
			}

			//If a new config is downloaded, a reload signal is sent
			//and updateManager terminates.
			if i.Config {
				mgr.config.Log.Println("Updating config")
				err = updateConfig(mgr)
				if err != nil {
					mgr.config.Log.Println(err)
				}
				mgr.reloadSignal <- true
				mgr.config.Log.Println("Config updated, restarting manager.")
				return
			}
		}
	}
}

//updateConfig downloads a new config from server and announces its
//succesful download back.
func updateConfig(mgr *Manager) error {
	err := updater.DownloadConfig(mgr.config, mgr.config.CentralPath)
	if err != nil {
		return fmt.Errorf("Error while updating config: %v", err)
	}
	mgr.config.Log.Println("Config succesfully downloaded, announcing.")
	err = updater.AnnounceConfig(mgr.config)
	if err != nil {
		return fmt.Errorf("Error while announcing the download of config: %v", err)
	}
	return nil
}

//updateBroadcast downloads a new broadcast from server,
//announces its succesful download and moves it into place.
func updateBroadcast(mgr *Manager, ft string) error {
	path, err := makeTempDir()
	if err != nil {
		return fmt.Errorf("Error while creating temporary directory: %v", err)
	}
	err = updater.DownloadBroadcast(mgr.config, ft, path)
	if err != nil {
		return fmt.Errorf("Error while downloading new broadcast: %v", err)
	}
	mgr.config.Log.Println("Broadcast successfully downloaded, announcing.")

	err = updater.AnnounceBroadcast(mgr.config)
	if err != nil {
		return fmt.Errorf("Error while announcing the download of broadcast: %v", err)
	}

	err = mgr.stopBroadcast()
	if err != nil {
		return fmt.Errorf("Error while stopping broadcast for update: %v", err)
	}

	//scheduleManager has to be blocked from interfering
	//with us moving the broadcast into place.
	mgr.block()
	defer mgr.unblock()

	mgr.config.Log.Println("Moving new broadcast into place")
	err = deleteAll(mgr.config.BroadcastDir)
	if err != nil {
		return fmt.Errorf("Error while deleting old broadcast: %v", err)
	}

	err = moveFiles(path, mgr.config.BroadcastDir)
	if err != nil {
		return fmt.Errorf("Error while moving new broadcast into place: %v", err)
	}

	err = mgr.startBroadcast()
	if err != nil {
		return fmt.Errorf("Error while starting new broadcast: %v", err)
	}

	_ = deleteAll(path)
	return nil
}

//makeTempDir creates a directory in the temporary directory
//returned by os.TempDir() and returns its path.
func makeTempDir() (path string, err error) {
	path = os.TempDir() + string(os.PathSeparator) + "broadcast-download-" + strconv.Itoa(int(time.Now().Unix()))
	err = os.Mkdir(path, os.ModePerm)
	if err != nil {
		return
	}
	return
}

//moveFiles moves all the files from one directory into another.
func moveFiles(src, dest string) (err error) {
	sf, err := os.Open(src)
	if err != nil {
		return
	}

	fns, err := sf.Readdirnames(0)
	if err != nil {
		return
	}

	for _, name := range fns {
		path := src + string(os.PathSeparator) + name
		err = os.Rename(path, dest+string(os.PathSeparator)+name)
		if err != nil {
			return
		}
	}
	return
}

//deleteAll deletes all the files in given directory.
func deleteAll(dir string) (err error) {
	d, err := os.Open(dir)
	if err != nil {
		return
	}

	fns, err := d.Readdirnames(0)
	if err != nil {
		return
	}

	for _, f := range fns {
		path := dir + string(os.PathSeparator) + f
		err = os.Remove(path)
		if err != nil {
			return
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tq

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"sync/atomic"

	taskspb "google.golang.org/genproto/googleapis/cloud/tasks/v2"

	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/server/tq/tqtesting"
)

// TestingContext creates a scheduler that executes tasks through the given
// dispatcher (or Default one if nil) and puts it into the context as Submitter,
// so AddTask calls eventually submit tasks into this scheduler.
//
// The end result is that tasks submitted using such context end up in the
// returned Scheduler (allowing them to be examined), and when the Scheduler
// delivers them, they result in calls to corresponding handlers registered in
// the Dispatcher.
func TestingContext(ctx context.Context, d *Dispatcher) (context.Context, *tqtesting.Scheduler) {
	if d == nil {
		d = &Default
	}
	sched := &tqtesting.Scheduler{Executor: &directExecutor{d, 0}}
	return UseSubmitter(ctx, sched), sched
}

// directExecutor implements tqtesting.Executor via handlePush.
type directExecutor struct {
	d   *Dispatcher
	cnt int64
}

func (e *directExecutor) Execute(ctx context.Context, t *tqtesting.Task, done func(retry bool)) {
	retry := false
	defer done(retry)

	if t.Message != nil {
		panic("Executing PubSub tasks is not supported yet") // break tests loudly
	}

	var body []byte
	var headers map[string]string
	switch mt := t.Task.MessageType.(type) {
	case *taskspb.Task_HttpRequest:
		body = mt.HttpRequest.Body
		headers = mt.HttpRequest.Headers
	case *taskspb.Task_AppEngineHttpRequest:
		body = mt.AppEngineHttpRequest.Body
		headers = mt.AppEngineHttpRequest.Headers
	default:
		panic(fmt.Sprintf("Bad task, no payload: %q", t.Task))
	}

	hdr := make(http.Header, len(headers))
	for k, v := range headers {
		hdr.Set(k, v)
	}
	info := parseHeaders(hdr)

	// The direct executor doesn't emulate X-CloudTasks-* headers.
	info.ExecutionCount = t.Attempts - 1

	ctx = logging.SetField(ctx, "TQ#", strconv.FormatInt(atomic.AddInt64(&e.cnt, 1), 10))
	err := e.d.handlePush(ctx, body, info)
	if err != nil {
		logging.Errorf(ctx, "server/tq task error: %s", err)
		retry = !Fatal.In(err)
	}
}
<commit_msg>[server][tq] fix bug in directExecutor.<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tq

import (
	"context"
	"fmt"
	"net/http"
	"strconv"
	"sync/atomic"

	taskspb "google.golang.org/genproto/googleapis/cloud/tasks/v2"

	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/server/tq/tqtesting"
)

// TestingContext creates a scheduler that executes tasks through the given
// dispatcher (or Default one if nil) and puts it into the context as Submitter,
// so AddTask calls eventually submit tasks into this scheduler.
//
// The end result is that tasks submitted using such context end up in the
// returned Scheduler (allowing them to be examined), and when the Scheduler
// delivers them, they result in calls to corresponding handlers registered in
// the Dispatcher.
func TestingContext(ctx context.Context, d *Dispatcher) (context.Context, *tqtesting.Scheduler) {
	if d == nil {
		d = &Default
	}
	sched := &tqtesting.Scheduler{Executor: &directExecutor{d, 0}}
	return UseSubmitter(ctx, sched), sched
}

// directExecutor implements tqtesting.Executor via handlePush.
type directExecutor struct {
	d   *Dispatcher
	cnt int64
}

func (e *directExecutor) Execute(ctx context.Context, t *tqtesting.Task, done func(retry bool)) {
	retry := false
	defer func() { done(retry) }()

	if t.Message != nil {
		panic("Executing PubSub tasks is not supported yet") // break tests loudly
	}

	var body []byte
	var headers map[string]string
	switch mt := t.Task.MessageType.(type) {
	case *taskspb.Task_HttpRequest:
		body = mt.HttpRequest.Body
		headers = mt.HttpRequest.Headers
	case *taskspb.Task_AppEngineHttpRequest:
		body = mt.AppEngineHttpRequest.Body
		headers = mt.AppEngineHttpRequest.Headers
	default:
		panic(fmt.Sprintf("Bad task, no payload: %q", t.Task))
	}

	hdr := make(http.Header, len(headers))
	for k, v := range headers {
		hdr.Set(k, v)
	}
	info := parseHeaders(hdr)

	// The direct executor doesn't emulate X-CloudTasks-* headers.
	info.ExecutionCount = t.Attempts - 1

	ctx = logging.SetField(ctx, "TQ#", strconv.FormatInt(atomic.AddInt64(&e.cnt, 1), 10))
	err := e.d.handlePush(ctx, body, info)
	if err != nil {
		logging.Errorf(ctx, "server/tq task error: %s", err)
		retry = !Fatal.In(err)
	}
}
<|endoftext|>


<commit_before>package sarama

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"time"

	"github.com/eapache/go-xerial-snappy"
	"github.com/pierrec/lz4"
)

const recordBatchOverhead = 49

type recordsArray []*Record

func (e recordsArray) encode(pe packetEncoder) error {
	for _, r := range e {
		if err := r.encode(pe); err != nil {
			return err
		}
	}
	return nil
}

func (e recordsArray) decode(pd packetDecoder) error {
	for i := range e {
		rec := &Record{}
		if err := rec.decode(pd); err != nil {
			return err
		}
		e[i] = rec
	}
	return nil
}

type RecordBatch struct {
	FirstOffset           int64
	PartitionLeaderEpoch  int32
	Version               int8
	Codec                 CompressionCodec
	Control               bool
	LastOffsetDelta       int32
	FirstTimestamp        time.Time
	MaxTimestamp          time.Time
	ProducerID            int64
	ProducerEpoch         int16
	FirstSequence         int32
	Records               []*Record
	PartialTrailingRecord bool

	compressedRecords []byte
	recordsLen        int // uncompressed records size
}

func (b *RecordBatch) encode(pe packetEncoder) error {
	if b.Version != 2 {
		return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
	}
	pe.putInt64(b.FirstOffset)
	pe.push(&lengthField{})
	pe.putInt32(b.PartitionLeaderEpoch)
	pe.putInt8(b.Version)
	pe.push(newCRC32Field(crcCastagnoli))
	pe.putInt16(b.computeAttributes())
	pe.putInt32(int32(len(b.Records)))

	if err := (Timestamp{&b.FirstTimestamp}).encode(pe); err != nil {
		return err
	}

	if err := (Timestamp{&b.MaxTimestamp}).encode(pe); err != nil {
		return err
	}

	pe.putInt64(b.ProducerID)
	pe.putInt16(b.ProducerEpoch)
	pe.putInt32(b.FirstSequence)

	if err := pe.putArrayLength(len(b.Records)); err != nil {
		return err
	}

	if b.compressedRecords == nil {
		if err := b.encodeRecords(pe); err != nil {
			return err
		}
	}
	if err := pe.putRawBytes(b.compressedRecords); err != nil {
		return err
	}

	if err := pe.pop(); err != nil {
		return err
	}
	return pe.pop()
}

func (b *RecordBatch) decode(pd packetDecoder) (err error) {
	if b.FirstOffset, err = pd.getInt64(); err != nil {
		return err
	}

	batchLen, err := pd.getInt32()
	if err != nil {
		return err
	}

	if b.PartitionLeaderEpoch, err = pd.getInt32(); err != nil {
		return err
	}

	if b.Version, err = pd.getInt8(); err != nil {
		return err
	}

	if err = pd.push(&crc32Field{polynomial: crcCastagnoli}); err != nil {
		return err
	}

	attributes, err := pd.getInt16()
	if err != nil {
		return err
	}
	b.Codec = CompressionCodec(int8(attributes) & compressionCodecMask)
	b.Control = attributes&controlMask == controlMask

	if b.LastOffsetDelta, err = pd.getInt32(); err != nil {
		return err
	}

	if err = (Timestamp{&b.FirstTimestamp}).decode(pd); err != nil {
		return err
	}

	if err = (Timestamp{&b.MaxTimestamp}).decode(pd); err != nil {
		return err
	}

	if b.ProducerID, err = pd.getInt64(); err != nil {
		return err
	}

	if b.ProducerEpoch, err = pd.getInt16(); err != nil {
		return err
	}

	if b.FirstSequence, err = pd.getInt32(); err != nil {
		return err
	}

	numRecs, err := pd.getArrayLength()
	if err != nil {
		return err
	}
	if numRecs >= 0 {
		b.Records = make([]*Record, numRecs)
	}

	bufSize := int(batchLen) - recordBatchOverhead
	recBuffer, err := pd.getRawBytes(bufSize)
	if err != nil {
		if err == ErrInsufficientData {
			b.PartialTrailingRecord = true
			b.Records = nil
			return nil
		}
		return err
	}

	if err = pd.pop(); err != nil {
		return err
	}

	switch b.Codec {
	case CompressionNone:
	case CompressionGZIP:
		reader, err := gzip.NewReader(bytes.NewReader(recBuffer))
		if err != nil {
			return err
		}
		if recBuffer, err = ioutil.ReadAll(reader); err != nil {
			return err
		}
	case CompressionSnappy:
		if recBuffer, err = snappy.Decode(recBuffer); err != nil {
			return err
		}
	case CompressionLZ4:
		reader := lz4.NewReader(bytes.NewReader(recBuffer))
		if recBuffer, err = ioutil.ReadAll(reader); err != nil {
			return err
		}
	default:
		return PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", b.Codec)}
	}

	b.recordsLen = len(recBuffer)
	err = decode(recBuffer, recordsArray(b.Records))
	if err == ErrInsufficientData {
		b.PartialTrailingRecord = true
		b.Records = nil
		return nil
	}
	return err
}

func (b *RecordBatch) encodeRecords(pe packetEncoder) error {
	var raw []byte
	if b.Codec != CompressionNone {
		var err error
		if raw, err = encode(recordsArray(b.Records), nil); err != nil {
			return err
		}
		b.recordsLen = len(raw)
	}

	switch b.Codec {
	case CompressionNone:
		offset := pe.offset()
		if err := recordsArray(b.Records).encode(pe); err != nil {
			return err
		}
		b.recordsLen = pe.offset() - offset
	case CompressionGZIP:
		var buf bytes.Buffer
		writer := gzip.NewWriter(&buf)
		if _, err := writer.Write(raw); err != nil {
			return err
		}
		if err := writer.Close(); err != nil {
			return err
		}
		b.compressedRecords = buf.Bytes()
	case CompressionSnappy:
		b.compressedRecords = snappy.Encode(raw)
	case CompressionLZ4:
		var buf bytes.Buffer
		writer := lz4.NewWriter(&buf)
		if _, err := writer.Write(raw); err != nil {
			return err
		}
		if err := writer.Close(); err != nil {
			return err
		}
		b.compressedRecords = buf.Bytes()
	default:
		return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
	}

	return nil
}

func (b *RecordBatch) computeAttributes() int16 {
	attr := int16(b.Codec) & int16(compressionCodecMask)
	if b.Control {
		attr |= controlMask
	}
	return attr
}

func (b *RecordBatch) addRecord(r *Record) {
	b.Records = append(b.Records, r)
}
<commit_msg>Simplify record batch encoding a bit<commit_after>package sarama

import (
	"bytes"
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"time"

	"github.com/eapache/go-xerial-snappy"
	"github.com/pierrec/lz4"
)

const recordBatchOverhead = 49

type recordsArray []*Record

func (e recordsArray) encode(pe packetEncoder) error {
	for _, r := range e {
		if err := r.encode(pe); err != nil {
			return err
		}
	}
	return nil
}

func (e recordsArray) decode(pd packetDecoder) error {
	for i := range e {
		rec := &Record{}
		if err := rec.decode(pd); err != nil {
			return err
		}
		e[i] = rec
	}
	return nil
}

type RecordBatch struct {
	FirstOffset           int64
	PartitionLeaderEpoch  int32
	Version               int8
	Codec                 CompressionCodec
	Control               bool
	LastOffsetDelta       int32
	FirstTimestamp        time.Time
	MaxTimestamp          time.Time
	ProducerID            int64
	ProducerEpoch         int16
	FirstSequence         int32
	Records               []*Record
	PartialTrailingRecord bool

	compressedRecords []byte
	recordsLen        int // uncompressed records size
}

func (b *RecordBatch) encode(pe packetEncoder) error {
	if b.Version != 2 {
		return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
	}
	pe.putInt64(b.FirstOffset)
	pe.push(&lengthField{})
	pe.putInt32(b.PartitionLeaderEpoch)
	pe.putInt8(b.Version)
	pe.push(newCRC32Field(crcCastagnoli))
	pe.putInt16(b.computeAttributes())
	pe.putInt32(int32(len(b.Records)))

	if err := (Timestamp{&b.FirstTimestamp}).encode(pe); err != nil {
		return err
	}

	if err := (Timestamp{&b.MaxTimestamp}).encode(pe); err != nil {
		return err
	}

	pe.putInt64(b.ProducerID)
	pe.putInt16(b.ProducerEpoch)
	pe.putInt32(b.FirstSequence)

	if err := pe.putArrayLength(len(b.Records)); err != nil {
		return err
	}

	if b.compressedRecords == nil {
		if err := b.encodeRecords(pe); err != nil {
			return err
		}
	}
	if err := pe.putRawBytes(b.compressedRecords); err != nil {
		return err
	}

	if err := pe.pop(); err != nil {
		return err
	}
	return pe.pop()
}

func (b *RecordBatch) decode(pd packetDecoder) (err error) {
	if b.FirstOffset, err = pd.getInt64(); err != nil {
		return err
	}

	batchLen, err := pd.getInt32()
	if err != nil {
		return err
	}

	if b.PartitionLeaderEpoch, err = pd.getInt32(); err != nil {
		return err
	}

	if b.Version, err = pd.getInt8(); err != nil {
		return err
	}

	if err = pd.push(&crc32Field{polynomial: crcCastagnoli}); err != nil {
		return err
	}

	attributes, err := pd.getInt16()
	if err != nil {
		return err
	}
	b.Codec = CompressionCodec(int8(attributes) & compressionCodecMask)
	b.Control = attributes&controlMask == controlMask

	if b.LastOffsetDelta, err = pd.getInt32(); err != nil {
		return err
	}

	if err = (Timestamp{&b.FirstTimestamp}).decode(pd); err != nil {
		return err
	}

	if err = (Timestamp{&b.MaxTimestamp}).decode(pd); err != nil {
		return err
	}

	if b.ProducerID, err = pd.getInt64(); err != nil {
		return err
	}

	if b.ProducerEpoch, err = pd.getInt16(); err != nil {
		return err
	}

	if b.FirstSequence, err = pd.getInt32(); err != nil {
		return err
	}

	numRecs, err := pd.getArrayLength()
	if err != nil {
		return err
	}
	if numRecs >= 0 {
		b.Records = make([]*Record, numRecs)
	}

	bufSize := int(batchLen) - recordBatchOverhead
	recBuffer, err := pd.getRawBytes(bufSize)
	if err != nil {
		if err == ErrInsufficientData {
			b.PartialTrailingRecord = true
			b.Records = nil
			return nil
		}
		return err
	}

	if err = pd.pop(); err != nil {
		return err
	}

	switch b.Codec {
	case CompressionNone:
	case CompressionGZIP:
		reader, err := gzip.NewReader(bytes.NewReader(recBuffer))
		if err != nil {
			return err
		}
		if recBuffer, err = ioutil.ReadAll(reader); err != nil {
			return err
		}
	case CompressionSnappy:
		if recBuffer, err = snappy.Decode(recBuffer); err != nil {
			return err
		}
	case CompressionLZ4:
		reader := lz4.NewReader(bytes.NewReader(recBuffer))
		if recBuffer, err = ioutil.ReadAll(reader); err != nil {
			return err
		}
	default:
		return PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", b.Codec)}
	}

	b.recordsLen = len(recBuffer)
	err = decode(recBuffer, recordsArray(b.Records))
	if err == ErrInsufficientData {
		b.PartialTrailingRecord = true
		b.Records = nil
		return nil
	}
	return err
}

func (b *RecordBatch) encodeRecords(pe packetEncoder) error {
	var raw []byte
	var err error
	if raw, err = encode(recordsArray(b.Records), pe.metricRegistry()); err != nil {
		return err
	}
	b.recordsLen = len(raw)

	switch b.Codec {
	case CompressionNone:
		b.compressedRecords = raw
	case CompressionGZIP:
		var buf bytes.Buffer
		writer := gzip.NewWriter(&buf)
		if _, err := writer.Write(raw); err != nil {
			return err
		}
		if err := writer.Close(); err != nil {
			return err
		}
		b.compressedRecords = buf.Bytes()
	case CompressionSnappy:
		b.compressedRecords = snappy.Encode(raw)
	case CompressionLZ4:
		var buf bytes.Buffer
		writer := lz4.NewWriter(&buf)
		if _, err := writer.Write(raw); err != nil {
			return err
		}
		if err := writer.Close(); err != nil {
			return err
		}
		b.compressedRecords = buf.Bytes()
	default:
		return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)}
	}

	return nil
}

func (b *RecordBatch) computeAttributes() int16 {
	attr := int16(b.Codec) & int16(compressionCodecMask)
	if b.Control {
		attr |= controlMask
	}
	return attr
}

func (b *RecordBatch) addRecord(r *Record) {
	b.Records = append(b.Records, r)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/gateway"
	"github.com/google/gops/agent"
	log "github.com/sirupsen/logrus"
	prefixed "github.com/x-cray/logrus-prefixed-formatter"
	"os"
	"strings"
)

var (
	version = "1.9.1"
	githash string
)

func main() {
	log.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: true})
	flog := log.WithFields(log.Fields{"prefix": "main"})
	flagConfig := flag.String("conf", "matterbridge.toml", "config file")
	flagDebug := flag.Bool("debug", false, "enable debug")
	flagVersion := flag.Bool("version", false, "show version")
	flagGops := flag.Bool("gops", false, "enable gops agent")
	flag.Parse()
	if *flagGops {
		agent.Listen(&agent.Options{})
		defer agent.Close()
	}
	if *flagVersion {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}
	if *flagDebug || os.Getenv("DEBUG") == "1" {
		log.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: false, ForceFormatting: true})
		flog.Info("Enabling debug")
		log.SetLevel(log.DebugLevel)
	}
	flog.Printf("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		flog.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}
	cfg := config.NewConfig(*flagConfig)
	cfg.General.Debug = *flagDebug
	r, err := gateway.NewRouter(cfg)
	if err != nil {
		flog.Fatalf("Starting gateway failed: %s", err)
	}
	err = r.Start()
	if err != nil {
		flog.Fatalf("Starting gateway failed: %s", err)
	}
	flog.Printf("Gateway(s) started succesfully. Now relaying messages")
	select {}
}
<commit_msg>Bump version<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/gateway"
	"github.com/google/gops/agent"
	log "github.com/sirupsen/logrus"
	prefixed "github.com/x-cray/logrus-prefixed-formatter"
	"os"
	"strings"
)

var (
	version = "1.9.2-dev"
	githash string
)

func main() {
	log.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: true})
	flog := log.WithFields(log.Fields{"prefix": "main"})
	flagConfig := flag.String("conf", "matterbridge.toml", "config file")
	flagDebug := flag.Bool("debug", false, "enable debug")
	flagVersion := flag.Bool("version", false, "show version")
	flagGops := flag.Bool("gops", false, "enable gops agent")
	flag.Parse()
	if *flagGops {
		agent.Listen(&agent.Options{})
		defer agent.Close()
	}
	if *flagVersion {
		fmt.Printf("version: %s %s\n", version, githash)
		return
	}
	if *flagDebug || os.Getenv("DEBUG") == "1" {
		log.SetFormatter(&prefixed.TextFormatter{PrefixPadding: 13, DisableColors: true, FullTimestamp: false, ForceFormatting: true})
		flog.Info("Enabling debug")
		log.SetLevel(log.DebugLevel)
	}
	flog.Printf("Running version %s %s", version, githash)
	if strings.Contains(version, "-dev") {
		flog.Println("WARNING: THIS IS A DEVELOPMENT VERSION. Things may break.")
	}
	cfg := config.NewConfig(*flagConfig)
	cfg.General.Debug = *flagDebug
	r, err := gateway.NewRouter(cfg)
	if err != nil {
		flog.Fatalf("Starting gateway failed: %s", err)
	}
	err = r.Start()
	if err != nil {
		flog.Fatalf("Starting gateway failed: %s", err)
	}
	flog.Printf("Gateway(s) started succesfully. Now relaying messages")
	select {}
}
<|endoftext|>


<commit_before>package md3

import (
	"bytes"
	"fmt"
	"io"
	"log"
)

const (
	maxQPath       = 64
	maxFrameLength = 16

	md3HeaderIdent  = "IDP3"
	md3SurfaceIdent = md3HeaderIdent
	md3MaxVersion   = 15
	md3VertexSize   = 8
)

type surfaceHeader struct {
	name string

	flags int32 // Unused.

	num_frames    int32
	num_shaders   int32
	num_verts     int32
	num_triangles int32

	ofs_triangles int32
	ofs_shaders   int32
	ofs_st        int32
	ofs_xyznormal int32
	ofs_end       int32
}

type fileHeader struct {
	name string

	version int32
	flags   int32 // Unused.

	num_frames   int32
	num_tags     int32
	num_surfaces int32
	num_skins    int32 // Unused.

	ofs_frames   int32
	ofs_tags     int32
	ofs_surfaces int32
	ofs_eof      int32
}

func Read(data []byte) (*Model, error) {
	var (
		header *fileHeader
		err    error
	)

	r := bytes.NewReader(data)
	header, err = readMD3Header(r)
	if err != nil {
		log.Println("Error reading header:", err)
		return nil, err
	}

	model := new(Model)
	surfaces := make([]*Surface, 0, 1)

	model.name = header.name

	surfaceOutput := readSurfaceList(data[header.ofs_surfaces:], int(header.num_surfaces))
	tagOutput := readTagList(data[header.ofs_tags:], int(header.num_tags), int(header.num_frames))
	frameOutput := readFrameList(data[header.ofs_frames:], int(header.num_frames))

	for completions := header.num_surfaces + 1; completions > 0; completions-- {
		select {
		case surface := <-surfaceOutput:
			if surface != nil {
				surfaces = append(surfaces, surface)
			}
		case tags := <-tagOutput:
			model.tags = tags
		case frames := <-frameOutput:
			model.frames = frames
		}
	}

	model.surfaces = surfaces

	return model, nil
}

func readTagList(data []byte, count int, numFrames int) <-chan []*Tag {
	output := make(chan []*Tag)

	go func(output chan<- []*Tag) {
		// defer close(output)

		r := bytes.NewReader(data)
		tagMap := make(map[string]*Tag)
		tags := make([]*Tag, 0, count)
		var ok bool
		var tag *Tag
		var frame TagFrame
		var name string
		var err error
		var numTagsToRead = count * numFrames

		for i := 0; i < numTagsToRead; i++ {
			name, frame, err = readTag(r)

			if err != nil {
				log.Println("Error reading tag list:", err)
				break
			}

			if tag, ok = tagMap[name]; !ok {
				tag = new(Tag)
				tag.name = name
				tags = append(tags, tag)
				tagMap[name] = tag
			}

			tag.frames = append(tag.frames, frame)
		}

		output <- tags
	}(output)

	return output
}

func readTag(r io.Reader) (string, TagFrame, error) {
	var err error
	var name string
	var frame TagFrame

	name, err = readNulString(r, maxQPath)
	if err != nil {
		return name, frame, err
	}

	vecPointers := []*Vec3{
		&frame.Origin,
		&frame.XOrientation,
		&frame.YOrientation,
		&frame.ZOrientation,
	}

	for _, ptr := range vecPointers {
		*ptr, err = readF32Vec3(r)
		if err != nil {
			return name, frame, err
		}
	}

	return name, frame, nil
}

func readMD3Header(r io.Reader) (*fileHeader, error) {
	header := new(fileHeader)
	var ident string
	var err error

	ident, err = readFixedString(r, 4)
	switch {
	case err != nil:
		log.Println("Error reading header identifier", err)
		return nil, err
	case ident != md3HeaderIdent:
		return nil, fmt.Errorf("MD3 header identifier is %q, should be %q", ident, md3HeaderIdent)
	}

	header.version, err = readS32(r)
	switch {
	case err != nil:
		log.Println("Error reading header version", err)
		return nil, err
	case header.version > md3MaxVersion:
		return nil, fmt.Errorf("MD3 header version (%d) exceeds max version (%d)", header.version, md3MaxVersion)
	}

	header.name, err = readNulString(r, maxQPath)
	if err != nil {
		log.Println("Error reading header model name", err)
		return nil, err
	}

	var s32Fields = [...]*int32{
		&header.flags,
		&header.num_frames,
		&header.num_tags,
		&header.num_surfaces,
		&header.num_skins,
		&header.ofs_frames,
		&header.ofs_tags,
		&header.ofs_surfaces,
		&header.ofs_eof,
	}

	for _, x := range s32Fields {
		*x, err = readS32(r)
		if err != nil {
			return nil, err
		}
	}

	return header, nil
}

func readSurfaceHeader(r io.Reader) (*surfaceHeader, error) {
	var err error
	var ident string

	header := new(surfaceHeader)

	ident, err = readFixedString(r, 4)
	if err != nil {
		return nil, err
	} else if ident != md3SurfaceIdent {
		return nil, fmt.Errorf("Surface header identifier is %q, should be %q", ident, md3SurfaceIdent)
	}

	header.name, err = readNulString(r, maxQPath)
	if err != nil {
		return nil, err
	}

	var s32Fields = [...]*int32{
		&header.flags,
		&header.num_frames,
		&header.num_shaders,
		&header.num_verts,
		&header.num_triangles,
		&header.ofs_triangles,
		&header.ofs_shaders,
		&header.ofs_st,
		&header.ofs_xyznormal,
		&header.ofs_end,
	}

	for _, x := range s32Fields {
		*x, err = readS32(r)
		if err != nil {
			return nil, err
		}
	}

	return header, nil
}

func readVertex(r io.Reader) (Vertex, error) {
	var result Vertex
	var err error

	result.Origin, err = readF16Vec3(r)
	if err != nil {
		return result, err
	}

	result.Normal, err = readSphereNormal(r)

	return result, err
}

func readXYZNormals(r io.Reader, count int) ([]Vertex, error) {
	vertices := make([]Vertex, count)
	for index := range vertices {
		var err error
		vertices[index], err = readVertex(r)
		if err != nil {
			return nil, err
		}
	}
	return vertices, nil
}

func readTriangleList(data []byte, count int) <-chan []Triangle {
	output := make(chan []Triangle)
	go func() {
		var err error
		tris := make([]Triangle, count)
		r := bytes.NewReader(data)
		for index := range tris {
			tri := Triangle{}

			tri.A, err = readS32(r)
			if err != nil {
				break
			}

			tri.B, err = readS32(r)
			if err != nil {
				break
			}

			tri.C, err = readS32(r)
			if err != nil {
				break
			}

			tris[index] = tri
		}

		if err != nil {
			log.Println("Error reading triangles:", err)
		}

		output <- tris
	}()
	return output
}

func readTexCoordList(data []byte, count int) <-chan []TexCoord {
	output := make(chan []TexCoord)
	go func() {
		var err error
		tcs := make([]TexCoord, count)
		r := bytes.NewReader(data)
		for index := range tcs {
			tc := TexCoord{}

			tc.S, err = readF32(r)
			if err != nil {
				break
			}

			tc.T, err = readF32(r)
			if err != nil {
				break
			}

			tcs[index] = tc
		}

		if err != nil {
			log.Println("Error reading texcoords:", err)
		}

		output <- tcs
	}()
	return output
}

func readShaderList(data []byte, count int) <-chan []Shader {
	output := make(chan []Shader)
	go func() {
		var err error
		shaders := make([]Shader, count)
		r := bytes.NewReader(data)
		for index := range shaders {
			shader := Shader{}

			shader.Name, err = readNulString(r, maxQPath)
			if err != nil {
				break
			}

			shader.Index, err = readS32(r)
			if err != nil {
				break
			}

			shaders[index] = shader
		}

		if err != nil {
			log.Println("Error reading shaders:", err)
		}

		output <- shaders
	}()
	return output
}

func readSurfaceList(data []byte, count int) <-chan *Surface {
	output := make(chan *Surface)
	go func(data []byte, output chan<- *Surface) {
		for index := 0; index < count; index++ {
			reader := bytes.NewReader(data[:])
			header, err := readSurfaceHeader(reader)
			if err != nil {
				log.Println("Error reading surface header:", err)
				break
			}

			go func(data []byte) {
				surf, err := readSurface(header, data)
				if err != nil {
					log.Printf("Error reading surface %q: %s\n", header.name, err)
				}

				surf.name = header.name
				surf.numFrames = int(header.num_frames)

				output <- surf
			}(data)

			data = data[header.ofs_end:]
		}
	}(data, output)
	return output
}

func readSurface(h *surfaceHeader, data []byte) (*Surface, error) {
	surface := new(Surface)

	triangleOutput := readTriangleList(data[h.ofs_triangles:], int(h.num_triangles))
	shaderOutput := readShaderList(data[h.ofs_shaders:], int(h.num_shaders))
	texcoordOutput := readTexCoordList(data[h.ofs_st:], int(h.num_verts))
	verticesOutput := readVertexFrames(data[h.ofs_xyznormal:], int(h.num_frames), int(h.num_verts))

	surface.vertices = <-verticesOutput
	surface.triangles = <-triangleOutput
	surface.texcoords = <-texcoordOutput
	surface.shaders = <-shaderOutput

	return surface, nil
}

type frameAndVertices struct {
	index    int
	vertices []Vertex
}

func readVertexFrames(data []byte, numVertices, numFrames int) <-chan [][]Vertex {
	output := make(chan [][]Vertex)

	go func(data []byte, output chan<- [][]Vertex) {
		var (
			frameVertices = make([][]Vertex, numFrames)
			frameReceiver = make(chan frameAndVertices)
			frameSize     = numVertices * md3VertexSize
		)

		for frame := range frameVertices {
			go func(frame int, data []byte) {
				reader := bytes.NewReader(data)
				vertices, err := readXYZNormals(reader, numVertices)
				if err != nil {
					log.Println("Error reading vertices:", err)
				}
				frameReceiver <- frameAndVertices{frame, vertices}
			}(frame, data[:frameSize])
			data = data[frameSize:]
		}

		for _ = range frameVertices {
			pack := <-frameReceiver
			frameVertices[pack.index] = pack.vertices
		}

		output <- frameVertices
	}(data, output)

	return output
}

func readFrame(r io.Reader) (*Frame, error) {
	var err error
	frame := new(Frame)

	frame.name, err = readNulString(r, maxFrameLength)
	if err != nil {
		return nil, err
	}

	for _, ptr := range []*Vec3{&frame.min, &frame.max, &frame.origin} {
		*ptr, err = readF32Vec3(r)
		if err != nil {
			return nil, err
		}
	}

	frame.radius, err = readF32(r)
	if err != nil {
		return nil, err
	}

	return frame, nil
}

func readFrameList(data []byte, count int) <-chan []*Frame {
	output := make(chan []*Frame)

	go func(output chan<- []*Frame) {
		// defer close(output)

		var err error
		reader := bytes.NewReader(data)
		frames := make([]*Frame, count)

		for index := range frames {
			frames[index], err = readFrame(reader)

			if err != nil {
				log.Println("Error reading frame list:", err)
				frames = nil
				break
			}
		}

		output <- frames
	}(output)

	return output
}
<commit_msg>Simplify s32Fields decls.<commit_after>package md3

import (
	"bytes"
	"fmt"
	"io"
	"log"
)

const (
	maxQPath       = 64
	maxFrameLength = 16

	md3HeaderIdent  = "IDP3"
	md3SurfaceIdent = md3HeaderIdent
	md3MaxVersion   = 15
	md3VertexSize   = 8
)

type surfaceHeader struct {
	name string

	flags int32 // Unused.

	num_frames    int32
	num_shaders   int32
	num_verts     int32
	num_triangles int32

	ofs_triangles int32
	ofs_shaders   int32
	ofs_st        int32
	ofs_xyznormal int32
	ofs_end       int32
}

type fileHeader struct {
	name string

	version int32
	flags   int32 // Unused.

	num_frames   int32
	num_tags     int32
	num_surfaces int32
	num_skins    int32 // Unused.

	ofs_frames   int32
	ofs_tags     int32
	ofs_surfaces int32
	ofs_eof      int32
}

func Read(data []byte) (*Model, error) {
	var (
		header *fileHeader
		err    error
	)

	r := bytes.NewReader(data)
	header, err = readMD3Header(r)
	if err != nil {
		log.Println("Error reading header:", err)
		return nil, err
	}

	model := new(Model)
	surfaces := make([]*Surface, 0, 1)

	model.name = header.name

	surfaceOutput := readSurfaceList(data[header.ofs_surfaces:], int(header.num_surfaces))
	tagOutput := readTagList(data[header.ofs_tags:], int(header.num_tags), int(header.num_frames))
	frameOutput := readFrameList(data[header.ofs_frames:], int(header.num_frames))

	for completions := header.num_surfaces + 1; completions > 0; completions-- {
		select {
		case surface := <-surfaceOutput:
			if surface != nil {
				surfaces = append(surfaces, surface)
			}
		case tags := <-tagOutput:
			model.tags = tags
		case frames := <-frameOutput:
			model.frames = frames
		}
	}

	model.surfaces = surfaces

	return model, nil
}

func readTagList(data []byte, count int, numFrames int) <-chan []*Tag {
	output := make(chan []*Tag)

	go func(output chan<- []*Tag) {
		// defer close(output)

		r := bytes.NewReader(data)
		tagMap := make(map[string]*Tag)
		tags := make([]*Tag, 0, count)
		var ok bool
		var tag *Tag
		var frame TagFrame
		var name string
		var err error
		var numTagsToRead = count * numFrames

		for i := 0; i < numTagsToRead; i++ {
			name, frame, err = readTag(r)

			if err != nil {
				log.Println("Error reading tag list:", err)
				break
			}

			if tag, ok = tagMap[name]; !ok {
				tag = new(Tag)
				tag.name = name
				tags = append(tags, tag)
				tagMap[name] = tag
			}

			tag.frames = append(tag.frames, frame)
		}

		output <- tags
	}(output)

	return output
}

func readTag(r io.Reader) (string, TagFrame, error) {
	var err error
	var name string
	var frame TagFrame

	name, err = readNulString(r, maxQPath)
	if err != nil {
		return name, frame, err
	}

	vecPointers := [...]*Vec3{
		&frame.Origin,
		&frame.XOrientation,
		&frame.YOrientation,
		&frame.ZOrientation,
	}

	for _, ptr := range vecPointers {
		*ptr, err = readF32Vec3(r)
		if err != nil {
			return name, frame, err
		}
	}

	return name, frame, nil
}

func readMD3Header(r io.Reader) (*fileHeader, error) {
	header := new(fileHeader)
	var ident string
	var err error

	ident, err = readFixedString(r, 4)
	switch {
	case err != nil:
		log.Println("Error reading header identifier", err)
		return nil, err
	case ident != md3HeaderIdent:
		return nil, fmt.Errorf("MD3 header identifier is %q, should be %q", ident, md3HeaderIdent)
	}

	header.version, err = readS32(r)
	switch {
	case err != nil:
		log.Println("Error reading header version", err)
		return nil, err
	case header.version > md3MaxVersion:
		return nil, fmt.Errorf("MD3 header version (%d) exceeds max version (%d)", header.version, md3MaxVersion)
	}

	header.name, err = readNulString(r, maxQPath)
	if err != nil {
		log.Println("Error reading header model name", err)
		return nil, err
	}

	s32Fields := [...]*int32{
		&header.flags,
		&header.num_frames,
		&header.num_tags,
		&header.num_surfaces,
		&header.num_skins,
		&header.ofs_frames,
		&header.ofs_tags,
		&header.ofs_surfaces,
		&header.ofs_eof,
	}

	for _, x := range s32Fields {
		*x, err = readS32(r)
		if err != nil {
			return nil, err
		}
	}

	return header, nil
}

func readSurfaceHeader(r io.Reader) (*surfaceHeader, error) {
	var err error
	var ident string

	header := new(surfaceHeader)

	ident, err = readFixedString(r, 4)
	if err != nil {
		return nil, err
	} else if ident != md3SurfaceIdent {
		return nil, fmt.Errorf("Surface header identifier is %q, should be %q", ident, md3SurfaceIdent)
	}

	header.name, err = readNulString(r, maxQPath)
	if err != nil {
		return nil, err
	}

	s32Fields := [...]*int32{
		&header.flags,
		&header.num_frames,
		&header.num_shaders,
		&header.num_verts,
		&header.num_triangles,
		&header.ofs_triangles,
		&header.ofs_shaders,
		&header.ofs_st,
		&header.ofs_xyznormal,
		&header.ofs_end,
	}

	for _, x := range s32Fields {
		*x, err = readS32(r)
		if err != nil {
			return nil, err
		}
	}

	return header, nil
}

func readVertex(r io.Reader) (Vertex, error) {
	var result Vertex
	var err error

	result.Origin, err = readF16Vec3(r)
	if err != nil {
		return result, err
	}

	result.Normal, err = readSphereNormal(r)

	return result, err
}

func readXYZNormals(r io.Reader, count int) ([]Vertex, error) {
	vertices := make([]Vertex, count)
	for index := range vertices {
		var err error
		vertices[index], err = readVertex(r)
		if err != nil {
			return nil, err
		}
	}
	return vertices, nil
}

func readTriangleList(data []byte, count int) <-chan []Triangle {
	output := make(chan []Triangle)
	go func() {
		var err error
		tris := make([]Triangle, count)
		r := bytes.NewReader(data)
		for index := range tris {
			tri := Triangle{}

			tri.A, err = readS32(r)
			if err != nil {
				break
			}

			tri.B, err = readS32(r)
			if err != nil {
				break
			}

			tri.C, err = readS32(r)
			if err != nil {
				break
			}

			tris[index] = tri
		}

		if err != nil {
			log.Println("Error reading triangles:", err)
		}

		output <- tris
	}()
	return output
}

func readTexCoordList(data []byte, count int) <-chan []TexCoord {
	output := make(chan []TexCoord)
	go func() {
		var err error
		tcs := make([]TexCoord, count)
		r := bytes.NewReader(data)
		for index := range tcs {
			tc := TexCoord{}

			tc.S, err = readF32(r)
			if err != nil {
				break
			}

			tc.T, err = readF32(r)
			if err != nil {
				break
			}

			tcs[index] = tc
		}

		if err != nil {
			log.Println("Error reading texcoords:", err)
		}

		output <- tcs
	}()
	return output
}

func readShaderList(data []byte, count int) <-chan []Shader {
	output := make(chan []Shader)
	go func() {
		var err error
		shaders := make([]Shader, count)
		r := bytes.NewReader(data)
		for index := range shaders {
			shader := Shader{}

			shader.Name, err = readNulString(r, maxQPath)
			if err != nil {
				break
			}

			shader.Index, err = readS32(r)
			if err != nil {
				break
			}

			shaders[index] = shader
		}

		if err != nil {
			log.Println("Error reading shaders:", err)
		}

		output <- shaders
	}()
	return output
}

func readSurfaceList(data []byte, count int) <-chan *Surface {
	output := make(chan *Surface)
	go func(data []byte, output chan<- *Surface) {
		for index := 0; index < count; index++ {
			reader := bytes.NewReader(data[:])
			header, err := readSurfaceHeader(reader)
			if err != nil {
				log.Println("Error reading surface header:", err)
				break
			}

			go func(data []byte) {
				surf, err := readSurface(header, data)
				if err != nil {
					log.Printf("Error reading surface %q: %s\n", header.name, err)
				}

				surf.name = header.name
				surf.numFrames = int(header.num_frames)

				output <- surf
			}(data)

			data = data[header.ofs_end:]
		}
	}(data, output)
	return output
}

func readSurface(h *surfaceHeader, data []byte) (*Surface, error) {
	surface := new(Surface)

	triangleOutput := readTriangleList(data[h.ofs_triangles:], int(h.num_triangles))
	shaderOutput := readShaderList(data[h.ofs_shaders:], int(h.num_shaders))
	texcoordOutput := readTexCoordList(data[h.ofs_st:], int(h.num_verts))
	verticesOutput := readVertexFrames(data[h.ofs_xyznormal:], int(h.num_frames), int(h.num_verts))

	surface.vertices = <-verticesOutput
	surface.triangles = <-triangleOutput
	surface.texcoords = <-texcoordOutput
	surface.shaders = <-shaderOutput

	return surface, nil
}

type frameAndVertices struct {
	index    int
	vertices []Vertex
}

func readVertexFrames(data []byte, numVertices, numFrames int) <-chan [][]Vertex {
	output := make(chan [][]Vertex)

	go func(data []byte, output chan<- [][]Vertex) {
		var (
			frameVertices = make([][]Vertex, numFrames)
			frameReceiver = make(chan frameAndVertices)
			frameSize     = numVertices * md3VertexSize
		)

		for frame := range frameVertices {
			go func(frame int, data []byte) {
				reader := bytes.NewReader(data)
				vertices, err := readXYZNormals(reader, numVertices)
				if err != nil {
					log.Println("Error reading vertices:", err)
				}
				frameReceiver <- frameAndVertices{frame, vertices}
			}(frame, data[:frameSize])
			data = data[frameSize:]
		}

		for _ = range frameVertices {
			pack := <-frameReceiver
			frameVertices[pack.index] = pack.vertices
		}

		output <- frameVertices
	}(data, output)

	return output
}

func readFrame(r io.Reader) (*Frame, error) {
	var err error
	frame := new(Frame)

	frame.name, err = readNulString(r, maxFrameLength)
	if err != nil {
		return nil, err
	}

	for _, ptr := range []*Vec3{&frame.min, &frame.max, &frame.origin} {
		*ptr, err = readF32Vec3(r)
		if err != nil {
			return nil, err
		}
	}

	frame.radius, err = readF32(r)
	if err != nil {
		return nil, err
	}

	return frame, nil
}

func readFrameList(data []byte, count int) <-chan []*Frame {
	output := make(chan []*Frame)

	go func(output chan<- []*Frame) {
		// defer close(output)

		var err error
		reader := bytes.NewReader(data)
		frames := make([]*Frame, count)

		for index := range frames {
			frames[index], err = readFrame(reader)

			if err != nil {
				log.Println("Error reading frame list:", err)
				frames = nil
				break
			}
		}

		output <- frames
	}(output)

	return output
}
<|endoftext|>


<commit_before>package gonx

import (
	. "github.com/smartystreets/goconvey/convey"
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestReducer(t *testing.T) {
	Convey("Test process input channel with reducers", t, func() {
		input := make(chan *Entry, 10)

		Convey("ReadAll reducer", func() {
			reducer := new(ReadAll)

			// Prepare import channel
			entry := NewEmptyEntry()
			input <- entry
			close(input)

			output := make(chan *Entry, 1) // Make it buffered to avoid deadlock
			reducer.Reduce(input, output)

			// ReadAll reducer writes input channel to the output
			result, ok := <-output
			So(ok, ShouldBeTrue)
			So(result, ShouldEqual, entry)
		})

		Convey("With filled input channel", func() {
			// Prepare import channel
			input <- NewEntry(Fields{
				"uri": "/asd/fgh",
				"foo": "123",
				"bar": "234",
				"baz": "345",
			})
			input <- NewEntry(Fields{
				"uri": "/zxc/vbn",
				"foo": "456",
				"bar": "567",
				"baz": "678",
			})
			close(input)

			output := make(chan *Entry, 1) // Make it buffered to avoid deadlock

			Convey("Count reducer", func() {
				reducer := new(Count)
				reducer.Reduce(input, output)

				result, ok := <-output
				So(ok, ShouldBeTrue)
				count, err := result.Field("count")
				So(err, ShouldBeNil)
				So(count, ShouldEqual, "2")
			})

			Convey("Sum reducer", func() {
				reducer := &Sum{[]string{"foo", "bar"}}
				reducer.Reduce(input, output)

				result, ok := <-output
				So(ok, ShouldBeTrue)

				value, err := result.FloatField("foo")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, 123.0+456)

				value, err = result.FloatField("bar")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, 234.0+567)

				_, err = result.Field("buz")
				So(err, ShouldNotBeNil)
			})

			Convey("Avg reducer", func() {
				reducer := &Avg{[]string{"foo", "bar"}}
				reducer.Reduce(input, output)

				result, ok := <-output
				So(ok, ShouldBeTrue)

				value, err := result.FloatField("foo")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, (123.0+456.0)/2.0)

				value, err = result.FloatField("bar")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, (234.0+567.0)/2.0)

				_, err = result.Field("buz")
				So(err, ShouldNotBeNil)
			})

			Convey("Chain reducer", func() {
				reducer := NewChain(&Avg{[]string{"foo", "bar"}}, &Count{})
				reducer.Reduce(input, output)

				result, ok := <-output
				So(ok, ShouldBeTrue)

				value, err := result.FloatField("foo")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, (123.0+456.0)/2.0)

				value, err = result.FloatField("bar")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, (234.0+567.0)/2.0)

				count, err := result.Field("count")
				So(err, ShouldBeNil)
				So(count, ShouldEqual, "2")

				_, err = result.Field("buz")
				So(err, ShouldNotBeNil)
			})
		})
	})
}

func TestGroupByReducer(t *testing.T) {
	reducer := NewGroupBy(
		// Fields to group by
		[]string{"host"},
		// Result reducers
		&Sum{[]string{"foo", "bar"}},
		new(Count),
	)
	assert.Implements(t, (*Reducer)(nil), reducer)

	// Prepare import channel
	input := make(chan *Entry, 10)
	input <- NewEntry(Fields{
		"uri":  "/asd/fgh",
		"host": "alpha.example.com",
		"foo":  "1",
		"bar":  "2",
		"baz":  "3",
	})
	input <- NewEntry(Fields{
		"uri":  "/zxc/vbn",
		"host": "beta.example.com",
		"foo":  "4",
		"bar":  "5",
		"baz":  "6",
	})
	input <- NewEntry(Fields{
		"uri":  "/ijk/lmn",
		"host": "beta.example.com",
		"foo":  "7",
		"bar":  "8",
		"baz":  "9",
	})
	close(input)
	output := make(chan *Entry, 2) // Make it buffered to avoid deadlock
	reducer.Reduce(input, output)

	// Collect result entries from output channel to the map, because reading
	// from channel can be in any order, it depends on each reducer processing
	resultMap := make(map[string]*Entry)
	for result := range output {
		value, err := result.Field("host")
		assert.NoError(t, err)
		resultMap[value] = result
	}
	assert.Equal(t, len(resultMap), 2)

	// Read and assert first group result
	result := resultMap["alpha.example.com"]

	floatVal, err := result.FloatField("foo")
	assert.NoError(t, err)
	assert.Equal(t, floatVal, 1.0)

	floatVal, err = result.FloatField("bar")
	assert.NoError(t, err)
	assert.Equal(t, floatVal, 2.0)

	value, err := result.Field("count")
	assert.NoError(t, err)
	assert.Equal(t, value, "1")

	// Read and assert second group result
	result = resultMap["beta.example.com"]

	floatVal, err = result.FloatField("foo")
	assert.NoError(t, err)
	assert.Equal(t, floatVal, 4.0+7.0)

	floatVal, err = result.FloatField("bar")
	assert.NoError(t, err)
	assert.Equal(t, floatVal, 5.0+8.0)

	value, err = result.Field("count")
	assert.NoError(t, err)
	assert.Equal(t, value, "2")
}
<commit_msg>better input channel fixture<commit_after>package gonx

import (
	. "github.com/smartystreets/goconvey/convey"
	"github.com/stretchr/testify/assert"
	"testing"
)

func TestReducer(t *testing.T) {
	Convey("Test process input channel with reducers", t, func() {
		input := make(chan *Entry, 10)

		Convey("ReadAll reducer", func() {
			reducer := new(ReadAll)

			// Prepare import channel
			entry := NewEmptyEntry()
			input <- entry
			close(input)

			output := make(chan *Entry, 1) // Make it buffered to avoid deadlock
			reducer.Reduce(input, output)

			// ReadAll reducer writes input channel to the output
			result, ok := <-output
			So(ok, ShouldBeTrue)
			So(result, ShouldEqual, entry)
		})

		Convey("With filled input channel", func() {
			// Prepare import channel
			input <- NewEntry(Fields{
				"uri":  "/asd/fgh",
				"host": "alpha.example.com",
				"foo":  "1",
				"bar":  "2",
				"baz":  "3",
			})
			input <- NewEntry(Fields{
				"uri":  "/zxc/vbn",
				"host": "beta.example.com",
				"foo":  "4",
				"bar":  "5",
				"baz":  "6",
			})
			input <- NewEntry(Fields{
				"uri":  "/ijk/lmn",
				"host": "beta.example.com",
				"foo":  "7",
				"bar":  "8",
				"baz":  "9",
			})
			close(input)
			total := float64(len(input))

			output := make(chan *Entry, 1) // Make it buffered to avoid deadlock

			Convey("Count reducer", func() {
				reducer := new(Count)
				reducer.Reduce(input, output)

				result, ok := <-output
				So(ok, ShouldBeTrue)
				count, err := result.FloatField("count")
				So(err, ShouldBeNil)
				So(count, ShouldEqual, total)
			})

			Convey("Sum reducer", func() {
				reducer := &Sum{[]string{"foo", "bar"}}
				reducer.Reduce(input, output)

				result, ok := <-output
				So(ok, ShouldBeTrue)

				value, err := result.FloatField("foo")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, 1+4+7)

				value, err = result.FloatField("bar")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, 2+5+8)

				_, err = result.Field("buz")
				So(err, ShouldNotBeNil)
			})

			Convey("Avg reducer", func() {
				reducer := &Avg{[]string{"foo", "bar"}}
				reducer.Reduce(input, output)

				result, ok := <-output
				So(ok, ShouldBeTrue)

				value, err := result.FloatField("foo")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, (1.0+4+7)/total)

				value, err = result.FloatField("bar")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, (2.0+5+8)/total)

				_, err = result.Field("buz")
				So(err, ShouldNotBeNil)
			})

			Convey("Chain reducer", func() {
				reducer := NewChain(&Avg{[]string{"foo", "bar"}}, &Count{})
				reducer.Reduce(input, output)

				result, ok := <-output
				So(ok, ShouldBeTrue)

				value, err := result.FloatField("foo")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, (1.0+4+7)/total)

				value, err = result.FloatField("bar")
				So(err, ShouldBeNil)
				So(value, ShouldEqual, (2.0+5+8)/total)

				count, err := result.FloatField("count")
				So(err, ShouldBeNil)
				So(count, ShouldEqual, total)

				_, err = result.Field("buz")
				So(err, ShouldNotBeNil)
			})
		})
	})
}

func TestGroupByReducer(t *testing.T) {
	reducer := NewGroupBy(
		// Fields to group by
		[]string{"host"},
		// Result reducers
		&Sum{[]string{"foo", "bar"}},
		new(Count),
	)
	assert.Implements(t, (*Reducer)(nil), reducer)

	// Prepare import channel
	input := make(chan *Entry, 10)
	input <- NewEntry(Fields{
		"uri":  "/asd/fgh",
		"host": "alpha.example.com",
		"foo":  "1",
		"bar":  "2",
		"baz":  "3",
	})
	input <- NewEntry(Fields{
		"uri":  "/zxc/vbn",
		"host": "beta.example.com",
		"foo":  "4",
		"bar":  "5",
		"baz":  "6",
	})
	input <- NewEntry(Fields{
		"uri":  "/ijk/lmn",
		"host": "beta.example.com",
		"foo":  "7",
		"bar":  "8",
		"baz":  "9",
	})
	close(input)
	output := make(chan *Entry, 2) // Make it buffered to avoid deadlock
	reducer.Reduce(input, output)

	// Collect result entries from output channel to the map, because reading
	// from channel can be in any order, it depends on each reducer processing
	resultMap := make(map[string]*Entry)
	for result := range output {
		value, err := result.Field("host")
		assert.NoError(t, err)
		resultMap[value] = result
	}
	assert.Equal(t, len(resultMap), 2)

	// Read and assert first group result
	result := resultMap["alpha.example.com"]

	floatVal, err := result.FloatField("foo")
	assert.NoError(t, err)
	assert.Equal(t, floatVal, 1.0)

	floatVal, err = result.FloatField("bar")
	assert.NoError(t, err)
	assert.Equal(t, floatVal, 2.0)

	value, err := result.Field("count")
	assert.NoError(t, err)
	assert.Equal(t, value, "1")

	// Read and assert second group result
	result = resultMap["beta.example.com"]

	floatVal, err = result.FloatField("foo")
	assert.NoError(t, err)
	assert.Equal(t, floatVal, 4.0+7.0)

	floatVal, err = result.FloatField("bar")
	assert.NoError(t, err)
	assert.Equal(t, floatVal, 5.0+8.0)

	value, err = result.Field("count")
	assert.NoError(t, err)
	assert.Equal(t, value, "2")
}
<|endoftext|>