<commit_before>package profitbricks

import (
	"fmt"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/profitbricks/profitbricks-sdk-go"
	"log"
	"strings"
)

func resourceProfitBricksIPBlock() *schema.Resource {
	return &schema.Resource{
		Create: resourceProfitBricksIPBlockCreate,
		Read:   resourceProfitBricksIPBlockRead,
		//Update: resourceProfitBricksIPBlockUpdate,
		Delete: resourceProfitBricksIPBlockDelete,
		Schema: map[string]*schema.Schema{
			"location": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"size": {
				Type:     schema.TypeInt,
				Required: true,
				ForceNew: true,
			},
			"ips": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceProfitBricksIPBlockCreate(d *schema.ResourceData, meta interface{}) error {
	ipblock := profitbricks.IpBlock{
		Properties: profitbricks.IpBlockProperties{
			Size:     d.Get("size").(int),
			Location: d.Get("location").(string),
		},
	}

	ipblock = profitbricks.ReserveIpBlock(ipblock)

	if ipblock.StatusCode > 299 {
		return fmt.Errorf("An error occured while reserving an ip block: %s", ipblock.Response)
	}
	err := waitTillProvisioned(meta, ipblock.Headers.Get("Location"))
	if err != nil {
		return err
	}
	d.SetId(ipblock.Id)

	return resourceProfitBricksIPBlockRead(d, meta)
}

func resourceProfitBricksIPBlockRead(d *schema.ResourceData, meta interface{}) error {
	ipblock := profitbricks.GetIpBlock(d.Id())

	if ipblock.StatusCode > 299 {
		if ipblock.StatusCode == 404 {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("An error occured while fetching an ip block ID %s %s", d.Id(), ipblock.Response)
	}

	log.Printf("[INFO] IPS: %s", strings.Join(ipblock.Properties.Ips, ","))

	d.Set("ips", strings.Join(ipblock.Properties.Ips, ","))
	d.Set("location", ipblock.Properties.Location)
	d.Set("size", ipblock.Properties.Size)

	return nil
}

func resourceProfitBricksIPBlockDelete(d *schema.ResourceData, meta interface{}) error {
	resp := profitbricks.ReleaseIpBlock(d.Id())
	if resp.StatusCode > 299 {
		return fmt.Errorf("An error occured while releasing an ipblock ID: %s %s", d.Id(), string(resp.Body))
	}

	err := waitTillProvisioned(meta, resp.Headers.Get("Location"))
	if err != nil {
		return err
	}
	d.SetId("")
	return nil
}
<commit_msg>Changed output type of ips variable of ip_block ProfitBricks resource (#13290)<commit_after>package profitbricks

import (
	"fmt"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/profitbricks/profitbricks-sdk-go"
	"log"
	"strings"
)

func resourceProfitBricksIPBlock() *schema.Resource {
	return &schema.Resource{
		Create: resourceProfitBricksIPBlockCreate,
		Read:   resourceProfitBricksIPBlockRead,
		//Update: resourceProfitBricksIPBlockUpdate,
		Delete: resourceProfitBricksIPBlockDelete,
		Schema: map[string]*schema.Schema{
			"location": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"size": {
				Type:     schema.TypeInt,
				Required: true,
				ForceNew: true,
			},
			"ips": {
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
			},
		},
	}
}

func resourceProfitBricksIPBlockCreate(d *schema.ResourceData, meta interface{}) error {
	ipblock := profitbricks.IpBlock{
		Properties: profitbricks.IpBlockProperties{
			Size:     d.Get("size").(int),
			Location: d.Get("location").(string),
		},
	}

	ipblock = profitbricks.ReserveIpBlock(ipblock)

	if ipblock.StatusCode > 299 {
		return fmt.Errorf("An error occured while reserving an ip block: %s", ipblock.Response)
	}
	err := waitTillProvisioned(meta, ipblock.Headers.Get("Location"))
	if err != nil {
		return err
	}
	d.SetId(ipblock.Id)

	return resourceProfitBricksIPBlockRead(d, meta)
}

func resourceProfitBricksIPBlockRead(d *schema.ResourceData, meta interface{}) error {
	ipblock := profitbricks.GetIpBlock(d.Id())

	if ipblock.StatusCode > 299 {
		if ipblock.StatusCode == 404 {
			d.SetId("")
			return nil
		}
		return fmt.Errorf("An error occured while fetching an ip block ID %s %s", d.Id(), ipblock.Response)
	}

	log.Printf("[INFO] IPS: %s", strings.Join(ipblock.Properties.Ips, ","))

	d.Set("ips", ipblock.Properties.Ips)
	d.Set("location", ipblock.Properties.Location)
	d.Set("size", ipblock.Properties.Size)

	return nil
}

func resourceProfitBricksIPBlockDelete(d *schema.ResourceData, meta interface{}) error {
	resp := profitbricks.ReleaseIpBlock(d.Id())
	if resp.StatusCode > 299 {
		return fmt.Errorf("An error occured while releasing an ipblock ID: %s %s", d.Id(), string(resp.Body))
	}

	err := waitTillProvisioned(meta, resp.Headers.Get("Location"))
	if err != nil {
		return err
	}
	d.SetId("")
	return nil
}
<|endoftext|>


<commit_before>// +build local

/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package local

import (
	"context"
	"fmt"
	"sync"

	"github.com/compose-spec/compose-go/types"
	"golang.org/x/sync/errgroup"
)

type ServiceStatus int

const (
	ServiceStopped ServiceStatus = iota
	ServiceStarted
)

func inDependencyOrder(ctx context.Context, project *types.Project, fn func(context.Context, types.ServiceConfig) error) error {
	g := NewGraph(project.Services)
	leaves := g.Leaves()

	eg, _ := errgroup.WithContext(ctx)
	eg.Go(func() error {
		return run(ctx, g, eg, leaves, fn)
	})

	return eg.Wait()
}

// Note: this could be `graph.walk` or whatever
func run(ctx context.Context, graph *Graph, eg *errgroup.Group, nodes []*Vertex, fn func(context.Context, types.ServiceConfig) error) error {
	for _, node := range nodes {
		n := node
		// Don't start this service yet if all of their children have
		// not been started yet.
		if len(graph.FilterChildren(n.Service.Name, ServiceStopped)) != 0 {
			continue
		}

		eg.Go(func() error {
			err := fn(ctx, n.Service)
			if err != nil {
				return err
			}

			graph.UpdateStatus(n.Service.Name, ServiceStarted)

			return run(ctx, graph, eg, n.GetParents(), fn)
		})
	}

	return nil
}

type Graph struct {
	Vertices map[string]*Vertex
	lock     sync.RWMutex
}

type Vertex struct {
	Key      string
	Service  types.ServiceConfig
	Status   ServiceStatus
	Children map[string]*Vertex
	Parents  map[string]*Vertex
}

func (v *Vertex) GetParents() []*Vertex {
	var res []*Vertex
	for _, p := range v.Parents {
		res = append(res, p)
	}
	return res
}

func NewGraph(services types.Services) *Graph {
	graph := &Graph{
		lock:     sync.RWMutex{},
		Vertices: map[string]*Vertex{},
	}

	for _, s := range services {
		graph.AddVertex(s.Name, s)
	}

	for _, s := range services {
		for _, name := range s.GetDependencies() {
			graph.AddEdge(s.Name, name)
		}
	}

	return graph
}

// We then create a constructor function for the Vertex
func NewVertex(key string, service types.ServiceConfig) *Vertex {
	return &Vertex{
		Key:      key,
		Service:  service,
		Status:   ServiceStopped,
		Parents:  map[string]*Vertex{},
		Children: map[string]*Vertex{},
	}
}

func (g *Graph) AddVertex(key string, service types.ServiceConfig) {
	g.lock.Lock()
	defer g.lock.Unlock()

	v := NewVertex(key, service)
	g.Vertices[key] = v
}

func (g *Graph) AddEdge(source string, destination string) error {
	g.lock.Lock()
	defer g.lock.Unlock()

	sourceVertex := g.Vertices[source]
	destinationVertex := g.Vertices[destination]

	if sourceVertex == nil {
		return fmt.Errorf("could not find %s", source)
	}
	if destinationVertex == nil {
		return fmt.Errorf("could not find %s", destination)
	}

	// If they are already connected
	if _, ok := sourceVertex.Children[destination]; ok {
		return nil
	}

	sourceVertex.Children[destination] = destinationVertex
	destinationVertex.Parents[source] = sourceVertex

	g.Vertices[source] = sourceVertex
	g.Vertices[destination] = destinationVertex
	return nil
}

func (g *Graph) Leaves() []*Vertex {
	g.lock.Lock()
	defer g.lock.Unlock()

	var res []*Vertex
	for _, v := range g.Vertices {
		if len(v.Children) == 0 {
			res = append(res, v)
		}
	}

	return res
}

func (g *Graph) UpdateStatus(key string, status ServiceStatus) {
	g.lock.Lock()
	defer g.lock.Unlock()
	g.Vertices[key].Status = status
}

func (g *Graph) FilterChildren(key string, status ServiceStatus) []*Vertex {
	g.lock.Lock()
	defer g.lock.Unlock()

	var res []*Vertex
	vertex := g.Vertices[key]

	for _, child := range vertex.Children {
		if child.Status == status {
			res = append(res, child)
		}
	}

	return res
}
<commit_msg>Detect cycles<commit_after>// +build local

/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package local

import (
	"context"
	"fmt"
	"strings"
	"sync"

	"github.com/compose-spec/compose-go/types"
	"golang.org/x/sync/errgroup"
)

type ServiceStatus int

const (
	ServiceStopped ServiceStatus = iota
	ServiceStarted
)

func inDependencyOrder(ctx context.Context, project *types.Project, fn func(context.Context, types.ServiceConfig) error) error {
	g := NewGraph(project.Services)
	if b, err := g.HasCycles(); b {
		return err
	}

	leaves := g.Leaves()

	eg, _ := errgroup.WithContext(ctx)
	eg.Go(func() error {
		return run(ctx, g, eg, leaves, fn)
	})

	return eg.Wait()
}

// Note: this could be `graph.walk` or whatever
func run(ctx context.Context, graph *Graph, eg *errgroup.Group, nodes []*Vertex, fn func(context.Context, types.ServiceConfig) error) error {
	for _, node := range nodes {
		n := node
		// Don't start this service yet if all of its children have
		// not been started yet.
		if len(graph.FilterChildren(n.Service.Name, ServiceStopped)) != 0 {
			continue
		}

		eg.Go(func() error {
			err := fn(ctx, n.Service)
			if err != nil {
				return err
			}

			graph.UpdateStatus(n.Service.Name, ServiceStarted)

			return run(ctx, graph, eg, n.GetParents(), fn)
		})
	}

	return nil
}

type Graph struct {
	Vertices map[string]*Vertex
	lock     sync.RWMutex
}

type Vertex struct {
	Key      string
	Service  types.ServiceConfig
	Status   ServiceStatus
	Children map[string]*Vertex
	Parents  map[string]*Vertex
}

func (v *Vertex) GetParents() []*Vertex {
	var res []*Vertex
	for _, p := range v.Parents {
		res = append(res, p)
	}
	return res
}

func NewGraph(services types.Services) *Graph {
	graph := &Graph{
		lock:     sync.RWMutex{},
		Vertices: map[string]*Vertex{},
	}

	for _, s := range services {
		graph.AddVertex(s.Name, s)
	}

	for _, s := range services {
		for _, name := range s.GetDependencies() {
			graph.AddEdge(s.Name, name)
		}
	}

	return graph
}

// We then create a constructor function for the Vertex
func NewVertex(key string, service types.ServiceConfig) *Vertex {
	return &Vertex{
		Key:      key,
		Service:  service,
		Status:   ServiceStopped,
		Parents:  map[string]*Vertex{},
		Children: map[string]*Vertex{},
	}
}

func (g *Graph) AddVertex(key string, service types.ServiceConfig) {
	g.lock.Lock()
	defer g.lock.Unlock()

	v := NewVertex(key, service)
	g.Vertices[key] = v
}

func (g *Graph) AddEdge(source string, destination string) error {
	g.lock.Lock()
	defer g.lock.Unlock()

	sourceVertex := g.Vertices[source]
	destinationVertex := g.Vertices[destination]

	if sourceVertex == nil {
		return fmt.Errorf("could not find %s", source)
	}
	if destinationVertex == nil {
		return fmt.Errorf("could not find %s", destination)
	}

	// If they are already connected
	if _, ok := sourceVertex.Children[destination]; ok {
		return nil
	}

	sourceVertex.Children[destination] = destinationVertex
	destinationVertex.Parents[source] = sourceVertex

	return nil
}

func (g *Graph) Leaves() []*Vertex {
	g.lock.Lock()
	defer g.lock.Unlock()

	var res []*Vertex
	for _, v := range g.Vertices {
		if len(v.Children) == 0 {
			res = append(res, v)
		}
	}

	return res
}

func (g *Graph) UpdateStatus(key string, status ServiceStatus) {
	g.lock.Lock()
	defer g.lock.Unlock()
	g.Vertices[key].Status = status
}

func (g *Graph) FilterChildren(key string, status ServiceStatus) []*Vertex {
	g.lock.Lock()
	defer g.lock.Unlock()

	var res []*Vertex
	vertex := g.Vertices[key]

	for _, child := range vertex.Children {
		if child.Status == status {
			res = append(res, child)
		}
	}

	return res
}

func (g *Graph) HasCycles() (bool, error) {
	discovered := []string{}
	finished := []string{}

	for _, vertex := range g.Vertices {
		path := []string{
			vertex.Key,
		}
		if !contains(discovered, vertex.Key) && !contains(finished, vertex.Key) {
			var err error
			discovered, finished, err = g.visit(vertex.Key, path, discovered, finished)

			if err != nil {
				return true, err
			}
		}
	}

	return false, nil
}

func (g *Graph) visit(key string, path []string, discovered []string, finished []string) ([]string, []string, error) {
	discovered = append(discovered, key)

	for _, v := range g.Vertices[key].Children {
		path := append(path, v.Key)
		if contains(discovered, v.Key) {
			return nil, nil, fmt.Errorf("cycle found: %s", strings.Join(path, " -> "))
		}

		if !contains(finished, v.Key) {
			if _, _, err := g.visit(v.Key, path, discovered, finished); err != nil {
				return nil, nil, err
			}
		}
	}

	discovered = remove(discovered, key)
	finished = append(finished, key)
	return discovered, finished, nil
}

func remove(slice []string, item string) []string {
	var s []string
	for _, i := range slice {
		if i != item {
			s = append(s, i)
		}
	}
	return s
}
<|endoftext|>


<commit_before><commit_msg>Generated from 03fe2bf0385ffbd9b4681743ab2e6f33279c418a<commit_after><|endoftext|>


<commit_before>package swarm

import (
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"time"

	types "github.com/docker/docker/api/types"
	context "golang.org/x/net/context"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/hatchery"
	"github.com/ovh/cds/sdk/log"
)

const (
	bridge  = "bridge"
	docker0 = "docker0"
)

func (h *HatcherySwarm) killAndRemove(dockerClient *dockerClient, ID string) error {
	ctxList, cancelList := context.WithTimeout(context.Background(), 3*time.Second)
	defer cancelList()
	container, err := dockerClient.ContainerInspect(ctxList, ID)
	if err != nil {
		//If there is an error, we try to remove the container
		if strings.Contains(err.Error(), "No such container") {
			log.Debug("hatchery> swarm> killAndRemove> cannot InspectContainer: %v on %s", err, dockerClient.name)
			return nil
		}
		log.Info("hatchery> swarm> killAndRemove> cannot InspectContainer: %v on %s", err, dockerClient.name)
	} else {
		// If its a worker "register", check registration before deleting it
		if strings.Contains(container.Name, "register-") {
			modelID, err := strconv.ParseInt(container.Config.Labels["worker_model"], 10, 64)
			if err != nil {
				log.Error("hatchery> swarm> killAndRemove> unable to get model from registering container %s", container.Name)
			} else {
				if err := hatchery.CheckWorkerModelRegister(h, modelID); err != nil {
					ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)
					defer cancel()
					logsOpts := types.ContainerLogsOptions{
						Details:    true,
						ShowStderr: true,
						ShowStdout: true,
						Timestamps: true,
						Since:      "10s",
					}
					var spawnErr = sdk.SpawnErrorForm{
						Error: err.Error(),
					}

					logsReader, errL := dockerClient.ContainerLogs(ctx, container.ID, logsOpts)
					if errL != nil {
						log.Error("hatchery> swarm> killAndRemove> cannot get logs from docker for containers service %s %v : %v", container.ID, container.Name, errL)
						spawnErr.Logs = []byte(fmt.Sprintf("unable to get container logs: %v", errL))

					} else if logsReader != nil {
						defer logsReader.Close()
						logs, errR := ioutil.ReadAll(logsReader)
						if errR != nil {
							log.Error("hatchery> swarm> killAndRemove> cannot read logs for containers service %s %v : %v", container.ID, container.Name, errR)
						} else if logs != nil {
							spawnErr.Logs = logs
						}
					}

					if err := h.CDSClient().WorkerModelSpawnError(modelID, spawnErr); err != nil {
						log.Error("hatchery> swarm> killAndRemove> error on call client.WorkerModelSpawnError on worker model %d for register: %s", modelID, err)
					}
				}
			}
		}
	}

	if err := h.killAndRemoveContainer(dockerClient, ID); err != nil {
		return sdk.WrapError(err, "%s on %s", ID[:7], dockerClient.name)
	}

	//If there is no network settings, stop here
	if container.NetworkSettings == nil {
		return nil
	}

	for _, cnetwork := range container.NetworkSettings.Networks {
		//Get the network
		ctxList, cancelList := context.WithTimeout(context.Background(), 3*time.Second)
		defer cancelList()
		network, err := dockerClient.NetworkInspect(ctxList, cnetwork.NetworkID, types.NetworkInspectOptions{})
		if err != nil {
			if !strings.Contains(err.Error(), "No such network") {
				return sdk.WrapError(err, "unable to get network for %s on %s", ID[:7], dockerClient.name)
			}
			continue
		}

		//If it's the default docker bridge... skip
		if network.Driver != bridge || network.Name == docker0 || network.Name == bridge {
			continue
		}

		// If we succeed to get the network, kill and remove all the container on the network
		if netname, ok := network.Labels["worker_net"]; ok {
			log.Debug("hatchery> swarm> killAndRemove> Remove network %s", netname)
			for id := range network.Containers {
				if err := h.killAndRemoveContainer(dockerClient, id); err != nil {
					log.Error("hatchery> swarm> killAndRemove> unable to kill and remove container %s on %s err:%s", id[:12], dockerClient.name, err)
				}
			}
		}

		//Finally remove the network
		log.Info("hatchery> swarm> remove network %s (%s)", network.Name, network.ID)
		ctxDocker, cancelList := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancelList()
		if err := dockerClient.NetworkRemove(ctxDocker, network.ID); err != nil {
			log.Error("hatchery> swarm> killAndRemove> unable to kill and remove network %s from %s err:%s", network.ID[:12], dockerClient.name, err)
		}
	}
	return nil
}

func (h *HatcherySwarm) killAndRemoveContainer(dockerClient *dockerClient, ID string) error {
	log.Debug("hatchery> swarm> killAndRemove> remove container %s on %s", ID, dockerClient.name)
	ctxDocker, cancelList := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancelList()
	if err := dockerClient.ContainerKill(ctxDocker, ID, "SIGKILL"); err != nil {
		if !strings.Contains(err.Error(), "is not running") && !strings.Contains(err.Error(), "No such container") {
			return sdk.WrapError(err, "err on kill container %v from %s", err, dockerClient.name)
		}
	}

	ctxDockerRemove, cancelList := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancelList()
	if err := dockerClient.ContainerRemove(ctxDockerRemove, ID, types.ContainerRemoveOptions{RemoveVolumes: true, RemoveLinks: true, Force: true}); err != nil {
		// container could be already removed by a previous call to docker
		if !strings.Contains(err.Error(), "No such container") && !strings.Contains(err.Error(), "is already in progress") {
			return sdk.WrapError(err, "Unable to remove container %s from %s", ID, dockerClient.name)
		}
	}

	return nil
}

func (h *HatcherySwarm) killAwolNetworks() error {
	for _, dockerClient := range h.dockerClients {
		//Checking networks
		ctxDocker, cancelList := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancelList()
		nets, errLN := dockerClient.NetworkList(ctxDocker, types.NetworkListOptions{})
		if errLN != nil {
			log.Warning("hatchery> swarm> killAwolNetworks> Cannot get networks on %s: %s", dockerClient.name, errLN)
			return errLN
		}

		for i := range nets {
			ctxDocker, cancelList := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancelList()
			n, err := dockerClient.NetworkInspect(ctxDocker, nets[i].ID, types.NetworkInspectOptions{})
			if err != nil {
				log.Warning("hatchery> swarm> killAwolNetworks> Unable to get network info: %v", err)
				continue
			}

			if n.Driver != bridge || n.Name == docker0 || n.Name == bridge {
				continue
			}

			if _, ok := n.Labels["worker_net"]; !ok {
				continue
			}

			if len(n.Containers) > 0 {
				continue
			}

			// if network created less than 10 min, keep it alive for now
			if time.Since(n.Created) < 10*time.Minute {
				continue
			}

			log.Info("hatchery> swarm> killAwolNetworks> remove network[%s] %s on %s (created on %v)", n.ID, n.Name, dockerClient.name, n.Created)
			ctxDocker2, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			if err := dockerClient.NetworkRemove(ctxDocker2, n.ID); err != nil {
				log.Warning("hatchery> swarm> killAwolNetworks> Unable to delete network %s err:%s", n.Name, err)
			}
		}
	}
	return nil
}
<commit_msg>fix(hatchery/swarm): remove volume only (#4707)<commit_after>package swarm

import (
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"time"

	types "github.com/docker/docker/api/types"
	context "golang.org/x/net/context"

	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/hatchery"
	"github.com/ovh/cds/sdk/log"
)

const (
	bridge  = "bridge"
	docker0 = "docker0"
)

func (h *HatcherySwarm) killAndRemove(dockerClient *dockerClient, ID string) error {
	ctxList, cancelList := context.WithTimeout(context.Background(), 3*time.Second)
	defer cancelList()
	container, err := dockerClient.ContainerInspect(ctxList, ID)
	if err != nil {
		//If there is an error, we try to remove the container
		if strings.Contains(err.Error(), "No such container") {
			log.Debug("hatchery> swarm> killAndRemove> cannot InspectContainer: %v on %s", err, dockerClient.name)
			return nil
		}
		log.Info("hatchery> swarm> killAndRemove> cannot InspectContainer: %v on %s", err, dockerClient.name)
	} else {
		// If its a worker "register", check registration before deleting it
		if strings.Contains(container.Name, "register-") {
			modelID, err := strconv.ParseInt(container.Config.Labels["worker_model"], 10, 64)
			if err != nil {
				log.Error("hatchery> swarm> killAndRemove> unable to get model from registering container %s", container.Name)
			} else {
				if err := hatchery.CheckWorkerModelRegister(h, modelID); err != nil {
					ctx, cancel := context.WithTimeout(context.Background(), time.Minute*2)
					defer cancel()
					logsOpts := types.ContainerLogsOptions{
						Details:    true,
						ShowStderr: true,
						ShowStdout: true,
						Timestamps: true,
						Since:      "10s",
					}
					var spawnErr = sdk.SpawnErrorForm{
						Error: err.Error(),
					}

					logsReader, errL := dockerClient.ContainerLogs(ctx, container.ID, logsOpts)
					if errL != nil {
						log.Error("hatchery> swarm> killAndRemove> cannot get logs from docker for containers service %s %v : %v", container.ID, container.Name, errL)
						spawnErr.Logs = []byte(fmt.Sprintf("unable to get container logs: %v", errL))

					} else if logsReader != nil {
						defer logsReader.Close()
						logs, errR := ioutil.ReadAll(logsReader)
						if errR != nil {
							log.Error("hatchery> swarm> killAndRemove> cannot read logs for containers service %s %v : %v", container.ID, container.Name, errR)
						} else if logs != nil {
							spawnErr.Logs = logs
						}
					}

					if err := h.CDSClient().WorkerModelSpawnError(modelID, spawnErr); err != nil {
						log.Error("hatchery> swarm> killAndRemove> error on call client.WorkerModelSpawnError on worker model %d for register: %s", modelID, err)
					}
				}
			}
		}
	}

	if err := h.killAndRemoveContainer(dockerClient, ID); err != nil {
		return sdk.WrapError(err, "%s on %s", ID[:7], dockerClient.name)
	}

	//If there is no network settings, stop here
	if container.NetworkSettings == nil {
		return nil
	}

	for _, cnetwork := range container.NetworkSettings.Networks {
		//Get the network
		ctxList, cancelList := context.WithTimeout(context.Background(), 3*time.Second)
		defer cancelList()
		network, err := dockerClient.NetworkInspect(ctxList, cnetwork.NetworkID, types.NetworkInspectOptions{})
		if err != nil {
			if !strings.Contains(err.Error(), "No such network") {
				return sdk.WrapError(err, "unable to get network for %s on %s", ID[:7], dockerClient.name)
			}
			continue
		}

		//If it's the default docker bridge... skip
		if network.Driver != bridge || network.Name == docker0 || network.Name == bridge {
			continue
		}

		// If we succeed to get the network, kill and remove all the container on the network
		if netname, ok := network.Labels["worker_net"]; ok {
			log.Debug("hatchery> swarm> killAndRemove> Remove network %s", netname)
			for id := range network.Containers {
				if err := h.killAndRemoveContainer(dockerClient, id); err != nil {
					log.Error("hatchery> swarm> killAndRemove> unable to kill and remove container %s on %s err:%s", id[:12], dockerClient.name, err)
				}
			}
		}

		//Finally remove the network
		log.Info("hatchery> swarm> remove network %s (%s)", network.Name, network.ID)
		ctxDocker, cancelList := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancelList()
		if err := dockerClient.NetworkRemove(ctxDocker, network.ID); err != nil {
			log.Error("hatchery> swarm> killAndRemove> unable to kill and remove network %s from %s err:%s", network.ID[:12], dockerClient.name, err)
		}
	}
	return nil
}

func (h *HatcherySwarm) killAndRemoveContainer(dockerClient *dockerClient, ID string) error {
	log.Debug("hatchery> swarm> killAndRemove> remove container %s on %s", ID, dockerClient.name)
	ctxDocker, cancelList := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancelList()
	if err := dockerClient.ContainerKill(ctxDocker, ID, "SIGKILL"); err != nil {
		if !strings.Contains(err.Error(), "is not running") && !strings.Contains(err.Error(), "No such container") {
			return sdk.WrapError(err, "err on kill container %v from %s", err, dockerClient.name)
		}
	}

	ctxDockerRemove, cancelList := context.WithTimeout(context.Background(), 20*time.Second)
	defer cancelList()
	if err := dockerClient.ContainerRemove(ctxDockerRemove, ID, types.ContainerRemoveOptions{RemoveVolumes: true, Force: true}); err != nil {
		// container could be already removed by a previous call to docker
		if !strings.Contains(err.Error(), "No such container") && !strings.Contains(err.Error(), "is already in progress") {
      log.Error("Unable to remove container %s from %s: %v", ID, dockerClient.name, err)
		}
	}

	return nil
}

func (h *HatcherySwarm) killAwolNetworks() error {
	for _, dockerClient := range h.dockerClients {
		//Checking networks
		ctxDocker, cancelList := context.WithTimeout(context.Background(), 5*time.Second)
		defer cancelList()
		nets, errLN := dockerClient.NetworkList(ctxDocker, types.NetworkListOptions{})
		if errLN != nil {
			log.Warning("hatchery> swarm> killAwolNetworks> Cannot get networks on %s: %s", dockerClient.name, errLN)
			return errLN
		}

		for i := range nets {
			ctxDocker, cancelList := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancelList()
			n, err := dockerClient.NetworkInspect(ctxDocker, nets[i].ID, types.NetworkInspectOptions{})
			if err != nil {
				log.Warning("hatchery> swarm> killAwolNetworks> Unable to get network info: %v", err)
				continue
			}

			if n.Driver != bridge || n.Name == docker0 || n.Name == bridge {
				continue
			}

			if _, ok := n.Labels["worker_net"]; !ok {
				continue
			}

			if len(n.Containers) > 0 {
				continue
			}

			// if network created less than 10 min, keep it alive for now
			if time.Since(n.Created) < 10*time.Minute {
				continue
			}

			log.Info("hatchery> swarm> killAwolNetworks> remove network[%s] %s on %s (created on %v)", n.ID, n.Name, dockerClient.name, n.Created)
			ctxDocker2, cancel := context.WithTimeout(context.Background(), 5*time.Second)
			defer cancel()
			if err := dockerClient.NetworkRemove(ctxDocker2, n.ID); err != nil {
				log.Warning("hatchery> swarm> killAwolNetworks> Unable to delete network %s err:%s", n.Name, err)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package model

import (
	"database/sql"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/postgres"
	"github.com/lib/pq"
	"math/rand"
	"os"
	"time"
)

var modelLog = logrus.WithField("module", "model")

/*
	The structs defined in this file are stored in a database using the `gorm` package.

	Remember to
		Set primary key for a struct.
		Set constraints on specific fields where appropriate.
		Define UniqueIndexes either through a tag or through gorm.DB.AddUniqueIndex()
			for a Unique constraint over multiple fields

	Unless you have a good reason, declare attributes of a struct not null.

	Example:

		type MyType struct {
			Name string `gorm:"not null"`
		}

	Special Cases:

	Enums: 	EnumType.EnumItem => const EnumTypeEnumItem

		Structs using such 'enums' should declare appropriate constraints in the corresponding FieldTag,
		using go-sqlite3 syntax

		Example:

			type MyType struct {
				Name string `sql:"unique"`
			}

*/

type Slave struct {
	ID                   int64  `gorm:"primary_key"`
	Hostname             string `gorm:"unique_index"`
	Port                 PortNumber
	MongodPortRangeBegin PortNumber
	MongodPortRangeEnd   PortNumber
	PersistentStorage    bool
	Mongods              []*Mongod `gorm:"ForeignKey:ParentSlaveID"`
	ConfiguredState      SlaveState

	Problems []*Problem

	// Foreign keys
	RiskGroupID sql.NullInt64 `sql:"type:integer NULL REFERENCES risk_groups(id) DEFERRABLE INITIALLY DEFERRED"`

	ObservationError   *MSPError     // error in observation that is not tied to a specific Mongod
	ObservationErrorID sql.NullInt64 `sql:"type:integer NULL REFERENCES msp_errors(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"` // TODO not cleaned up on slave deletion right now
}

type PortNumber uint16

const (
	PortNumberMin PortNumber = 1
	PortNumberMax            = 65535
)

type SlaveState uint

const (
	_                           = 0
	SlaveStateActive SlaveState = iota
	SlaveStateMaintenance
	SlaveStateDisabled
)

type ReplicaSet struct {
	ID                              int64  `gorm:"primary_key"` //TODO needs to start incrementing at 1
	Name                            string `gorm:"unique_index"`
	PersistentMemberCount           uint
	VolatileMemberCount             uint
	ConfigureAsShardingConfigServer bool
	Mongods                         []*Mongod

	Problems []*Problem
}

type RiskGroup struct {
	ID     int64  `gorm:"primary_key"` //TODO needs to start incrementing at 1, 0 is special value for slaves "out of risk" => define a constant?
	Name   string `gorm:"unique_index"`
	Slaves []*Slave
}

type Mongod struct {
	// TODO missing UNIQUE constraint
	ID          int64 `gorm:"primary_key"`
	Port        PortNumber
	ReplSetName string

	ObservationError   MSPError
	ObservationErrorID sql.NullInt64 `sql:"type:integer NULL REFERENCES msp_errors(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"` // TODO not cleaned up on Mongod deletion right now

	LastEstablishStateError   MSPError
	LastEstablishStateErrorID sql.NullInt64 `sql:"type:integer NULL REFERENCES msp_errors(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"` // TODO not cleaned up on Mongod deletion right now

	ParentSlave   *Slave
	ParentSlaveID int64 `sql:"type:integer REFERENCES slaves(id) DEFERRABLE INITIALLY DEFERRED"`

	ReplicaSet   *ReplicaSet
	ReplicaSetID int64 `sql:"type:integer NULL REFERENCES replica_sets(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"`

	DesiredState   MongodState
	DesiredStateID int64 `sql:"type:integer NOT NULL REFERENCES mongod_states(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"` // NOTE: we cascade on delete because Monogd cannot be without DesiredState

	ObservedState   MongodState
	ObservedStateID sql.NullInt64 `sql:"type:integer NULL REFERENCES mongod_states(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"`
}

type MongodState struct {
	ID                     int64 `gorm:"primary_key"`
	ParentMongod           *Mongod
	ParentMongodID         int64 `sql:"type:integer NOT NULL REFERENCES mongods(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"`
	IsShardingConfigServer bool
	ExecutionState         MongodExecutionState
	ReplicaSetMembers      []ReplicaSetMember
}

type MongodExecutionState uint

const (
	_                                                  = 0
	MongodExecutionStateDestroyed MongodExecutionState = iota
	MongodExecutionStateNotRunning
	MongodExecutionStateRecovering // invalid for a desired MongodState
	MongodExecutionStateRunning
)

type ReplicaSetMember struct { // was ReplicaSetMember in UML
	// TODO missing primary key.
	ID       int64 `gorm:"primary_key"`
	Hostname string
	Port     PortNumber

	// Foreign key to parent MongodState
	MongodStateID int64 `sql:"type:integer REFERENCES mongod_states(id) DEFERRABLE INITIALLY DEFERRED"`
}

// msp.Error
// duplicated for decoupling protocol & internal representation
type MSPError struct {
	ID              int64 `gorm:"primary_key"`
	Identifier      string
	Description     string
	LongDescription string
}

type ProblemType uint

const (
	_                                 = 0
	ProblemTypeConnection ProblemType = iota
	ProblemTypeMismatch
	ProblemTypeDesiredReplicaSetConstraint
	ProblemTypeObservedReplicaSetConstraint
)

type Problem struct {
	ID              int64 `gorm:"primary_key"`
	Description     string
	LongDescription string
	ProblemType     ProblemType
	FirstOccurred   time.Time
	LastUpdated     time.Time

	Slave   *Slave
	SlaveID sql.NullInt64 `sql:"type:integer NULL REFERENCES slaves(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"`

	ReplicaSet   *ReplicaSet
	ReplicaSetID sql.NullInt64 `sql:"type:integer NULL REFERENCES replica_sets(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"`

	Mongod   *Mongod
	MongodID sql.NullInt64 `sql:"type:integer NULL REFERENCES mongods(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"`
}

type DB struct {
	gormDB  *gorm.DB
	dbName  sql.NullString
	connDSN sql.NullString
}

func (db *DB) Begin() *gorm.DB {
	tx := db.gormDB.Begin()
	return tx
}

func (db *DB) CloseAndDrop() {

	if !(db.dbName.Valid && db.connDSN.Valid) {
		modelLog.Fatalf("model.DB object not initialized for dropping database")
	}

	if err := db.gormDB.Close(); err != nil {
		modelLog.Fatalf("could not close connection with database open: %s", err)
	}

	const driver = "postgres"
	c, err := sql.Open(driver, db.connDSN.String)
	if err != nil {
		modelLog.Fatalf("cannot connect to test database: %s", err)
	}
	defer c.Close()

	res, err := c.Exec("DROP DATABASE ?", db.dbName.String)
	if err != nil {
		modelLog.Fatalf("could not drop database `%s`: %s", db.dbName.String, err)
	} else {
		modelLog.Infof("dropped database `%s`: %s", db.dbName.String, res)
	}

}

// Idempotently migrate the database schema.
// Currenlty, only creation of the schema is supported.
func (dbWrapper *DB) migrate() (err error) {

	db := dbWrapper.gormDB

	// if mamid_metadata table exists, the database must have been populated
	// we don't support migrations yet, hence throw an error and exit
	res := db.Raw(`
	SELECT EXISTS (
		SELECT 1
		FROM   information_schema.tables
		WHERE  table_schema = 'public'
		AND    table_name = 'mamid_metadata'
	)
	`)

	if res.Error != nil {
		return fmt.Errorf("the database has already been populated, migrations are not supported: %s", res.Error)
	}

	// run the populating query

	ddlStatements, err := Asset("model/sql/mamid_postgresql.sql")
	if err != nil {
		return fmt.Errorf("sql DDL data not found: %s", err)
	}

	err = db.Exec(string(ddlStatements), []interface{}{}).Error
	if err != nil {
		return fmt.Errorf("error running DDL statements: %s", err)
	}

	return nil

}

func InitializeDB(driver, dsn string) (*DB, error) {

	gormDB, err := gorm.Open(driver, dsn)
	if err != nil {
		return nil, err
	}

	gormDB.SetLogger(modelLog)

	db := &DB{gormDB: gormDB}

	if err := db.migrate(); err != nil {
		return nil, fmt.Errorf("could not migrate database: %s", err)
	}

	return db, err

}

func InitializeTestDB() (db *DB, dsn string, err error) {

	const driver = "postgres"
	connDSN := os.Getenv("MAMID_TESTDB_DSN")
	if connDSN == "" {
		modelLog.Panic("MAMID_TESTDB_DSN environment variable is not set")
	}

	c, err := sql.Open(driver, connDSN)
	if err != nil {
		modelLog.Fatalf("cannot connect to test database: %s", err)
	}
	defer c.Close()

	dbName := randomDBName("mamid_testing_", 20)
	_, err = c.Exec("CREATE DATABASE " + dbName)
	if err != nil {
		modelLog.Fatalf("cannot create test database `%s`: %s", dbName, err)
	}
	c.Close()

	// NOTE: in the current implementation of pq (postgres driver), the last key-value pair wins over previous ones with the same key
	dsn = fmt.Sprintf("%s dbname=%s", connDSN, dbName)
	gormDB, err := gorm.Open(driver, dsn)
	if err != nil {
		modelLog.Fatalf("cannot open just created test database `%s`: %s", dsn, err)
	}

	gormDB.SetLogger(modelLog)

	db = &DB{
		gormDB:  gormDB,
		dbName:  sql.NullString{String: dbName, Valid: true},
		connDSN: sql.NullString{String: connDSN, Valid: true},
	}

	if err := db.migrate(); err != nil {
		return nil, dsn, fmt.Errorf("could not migrate database: %s", err)
	}

	return db, dsn, nil

}

func randomDBName(prefix string, strlen int) string {
	rand.Seed(time.Now().UTC().UnixNano())
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := make([]byte, strlen)
	for i := 0; i < strlen; i++ {
		result[i] = chars[rand.Intn(len(chars))]
	}
	return prefix + string(result)
}

func NullIntValue(value int64) sql.NullInt64 {
	return sql.NullInt64{Int64: value, Valid: true}
}

func NullInt() sql.NullInt64 {
	return sql.NullInt64{}
}

func NullIntToPtr(nullint sql.NullInt64) *int64 {
	if nullint.Valid {
		value := nullint.Int64
		return &value
	} else {
		return nil
	}
}

func PtrToNullInt(value *int64) sql.NullInt64 {
	if value != nil {
		return NullIntValue(*value)
	} else {
		return NullInt()
	}
}

func IsIntegrityConstraintViolation(err error) bool {
	if driverErr, ok := err.(*pq.Error); ok && driverErr.Code.Class() == "23" { // Integrity Constraint Violation
		return true
	} else {
		return false
	}
}
<commit_msg>FIX: remove outdated comment on sqlite3<commit_after>package model

import (
	"database/sql"
	"fmt"
	"github.com/Sirupsen/logrus"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/postgres"
	"github.com/lib/pq"
	"math/rand"
	"os"
	"time"
)

var modelLog = logrus.WithField("module", "model")

/*
	The structs defined in this file are stored in a database using the `gorm` package.

	Remember to
		Set primary key for a struct.
		Set constraints on specific fields where appropriate.
		Define UniqueIndexes either through a tag or through gorm.DB.AddUniqueIndex()
			for a Unique constraint over multiple fields

	Unless you have a good reason, declare attributes of a struct not null.

	Example:

		type MyType struct {
			Name string `gorm:"not null"`
		}

	Special Cases:

	Enums: 	EnumType.EnumItem => const EnumTypeEnumItem

		Structs using such 'enums' should declare appropriate constraints in the corresponding FieldTag

		Example:

			type MyType struct {
				Name string `sql:"unique"`
			}

*/

type Slave struct {
	ID                   int64  `gorm:"primary_key"`
	Hostname             string `gorm:"unique_index"`
	Port                 PortNumber
	MongodPortRangeBegin PortNumber
	MongodPortRangeEnd   PortNumber
	PersistentStorage    bool
	Mongods              []*Mongod `gorm:"ForeignKey:ParentSlaveID"`
	ConfiguredState      SlaveState

	Problems []*Problem

	// Foreign keys
	RiskGroupID sql.NullInt64 `sql:"type:integer NULL REFERENCES risk_groups(id) DEFERRABLE INITIALLY DEFERRED"`

	ObservationError   *MSPError     // error in observation that is not tied to a specific Mongod
	ObservationErrorID sql.NullInt64 `sql:"type:integer NULL REFERENCES msp_errors(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"` // TODO not cleaned up on slave deletion right now
}

type PortNumber uint16

const (
	PortNumberMin PortNumber = 1
	PortNumberMax            = 65535
)

type SlaveState uint

const (
	_                           = 0
	SlaveStateActive SlaveState = iota
	SlaveStateMaintenance
	SlaveStateDisabled
)

type ReplicaSet struct {
	ID                              int64  `gorm:"primary_key"` //TODO needs to start incrementing at 1
	Name                            string `gorm:"unique_index"`
	PersistentMemberCount           uint
	VolatileMemberCount             uint
	ConfigureAsShardingConfigServer bool
	Mongods                         []*Mongod

	Problems []*Problem
}

type RiskGroup struct {
	ID     int64  `gorm:"primary_key"` //TODO needs to start incrementing at 1, 0 is special value for slaves "out of risk" => define a constant?
	Name   string `gorm:"unique_index"`
	Slaves []*Slave
}

type Mongod struct {
	// TODO missing UNIQUE constraint
	ID          int64 `gorm:"primary_key"`
	Port        PortNumber
	ReplSetName string

	ObservationError   MSPError
	ObservationErrorID sql.NullInt64 `sql:"type:integer NULL REFERENCES msp_errors(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"` // TODO not cleaned up on Mongod deletion right now

	LastEstablishStateError   MSPError
	LastEstablishStateErrorID sql.NullInt64 `sql:"type:integer NULL REFERENCES msp_errors(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"` // TODO not cleaned up on Mongod deletion right now

	ParentSlave   *Slave
	ParentSlaveID int64 `sql:"type:integer REFERENCES slaves(id) DEFERRABLE INITIALLY DEFERRED"`

	ReplicaSet   *ReplicaSet
	ReplicaSetID int64 `sql:"type:integer NULL REFERENCES replica_sets(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"`

	DesiredState   MongodState
	DesiredStateID int64 `sql:"type:integer NOT NULL REFERENCES mongod_states(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"` // NOTE: we cascade on delete because Monogd cannot be without DesiredState

	ObservedState   MongodState
	ObservedStateID sql.NullInt64 `sql:"type:integer NULL REFERENCES mongod_states(id) ON DELETE SET NULL DEFERRABLE INITIALLY DEFERRED"`
}

type MongodState struct {
	ID                     int64 `gorm:"primary_key"`
	ParentMongod           *Mongod
	ParentMongodID         int64 `sql:"type:integer NOT NULL REFERENCES mongods(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"`
	IsShardingConfigServer bool
	ExecutionState         MongodExecutionState
	ReplicaSetMembers      []ReplicaSetMember
}

type MongodExecutionState uint

const (
	_                                                  = 0
	MongodExecutionStateDestroyed MongodExecutionState = iota
	MongodExecutionStateNotRunning
	MongodExecutionStateRecovering // invalid for a desired MongodState
	MongodExecutionStateRunning
)

type ReplicaSetMember struct { // was ReplicaSetMember in UML
	// TODO missing primary key.
	ID       int64 `gorm:"primary_key"`
	Hostname string
	Port     PortNumber

	// Foreign key to parent MongodState
	MongodStateID int64 `sql:"type:integer REFERENCES mongod_states(id) DEFERRABLE INITIALLY DEFERRED"`
}

// msp.Error
// duplicated for decoupling protocol & internal representation
type MSPError struct {
	ID              int64 `gorm:"primary_key"`
	Identifier      string
	Description     string
	LongDescription string
}

type ProblemType uint

const (
	_                                 = 0
	ProblemTypeConnection ProblemType = iota
	ProblemTypeMismatch
	ProblemTypeDesiredReplicaSetConstraint
	ProblemTypeObservedReplicaSetConstraint
)

type Problem struct {
	ID              int64 `gorm:"primary_key"`
	Description     string
	LongDescription string
	ProblemType     ProblemType
	FirstOccurred   time.Time
	LastUpdated     time.Time

	Slave   *Slave
	SlaveID sql.NullInt64 `sql:"type:integer NULL REFERENCES slaves(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"`

	ReplicaSet   *ReplicaSet
	ReplicaSetID sql.NullInt64 `sql:"type:integer NULL REFERENCES replica_sets(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"`

	Mongod   *Mongod
	MongodID sql.NullInt64 `sql:"type:integer NULL REFERENCES mongods(id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED"`
}

type DB struct {
	gormDB  *gorm.DB
	dbName  sql.NullString
	connDSN sql.NullString
}

func (db *DB) Begin() *gorm.DB {
	tx := db.gormDB.Begin()
	return tx
}

func (db *DB) CloseAndDrop() {

	if !(db.dbName.Valid && db.connDSN.Valid) {
		modelLog.Fatalf("model.DB object not initialized for dropping database")
	}

	if err := db.gormDB.Close(); err != nil {
		modelLog.Fatalf("could not close connection with database open: %s", err)
	}

	const driver = "postgres"
	c, err := sql.Open(driver, db.connDSN.String)
	if err != nil {
		modelLog.Fatalf("cannot connect to test database: %s", err)
	}
	defer c.Close()

	res, err := c.Exec("DROP DATABASE ?", db.dbName.String)
	if err != nil {
		modelLog.Fatalf("could not drop database `%s`: %s", db.dbName.String, err)
	} else {
		modelLog.Infof("dropped database `%s`: %s", db.dbName.String, res)
	}

}

// Idempotently migrate the database schema.
// Currenlty, only creation of the schema is supported.
func (dbWrapper *DB) migrate() (err error) {

	db := dbWrapper.gormDB

	// if mamid_metadata table exists, the database must have been populated
	// we don't support migrations yet, hence throw an error and exit
	res := db.Raw(`
	SELECT EXISTS (
		SELECT 1
		FROM   information_schema.tables
		WHERE  table_schema = 'public'
		AND    table_name = 'mamid_metadata'
	)
	`)

	if res.Error != nil {
		return fmt.Errorf("the database has already been populated, migrations are not supported: %s", res.Error)
	}

	// run the populating query

	ddlStatements, err := Asset("model/sql/mamid_postgresql.sql")
	if err != nil {
		return fmt.Errorf("sql DDL data not found: %s", err)
	}

	err = db.Exec(string(ddlStatements), []interface{}{}).Error
	if err != nil {
		return fmt.Errorf("error running DDL statements: %s", err)
	}

	return nil

}

func InitializeDB(driver, dsn string) (*DB, error) {

	gormDB, err := gorm.Open(driver, dsn)
	if err != nil {
		return nil, err
	}

	gormDB.SetLogger(modelLog)

	db := &DB{gormDB: gormDB}

	if err := db.migrate(); err != nil {
		return nil, fmt.Errorf("could not migrate database: %s", err)
	}

	return db, err

}

func InitializeTestDB() (db *DB, dsn string, err error) {

	const driver = "postgres"
	connDSN := os.Getenv("MAMID_TESTDB_DSN")
	if connDSN == "" {
		modelLog.Panic("MAMID_TESTDB_DSN environment variable is not set")
	}

	c, err := sql.Open(driver, connDSN)
	if err != nil {
		modelLog.Fatalf("cannot connect to test database: %s", err)
	}
	defer c.Close()

	dbName := randomDBName("mamid_testing_", 20)
	_, err = c.Exec("CREATE DATABASE " + dbName)
	if err != nil {
		modelLog.Fatalf("cannot create test database `%s`: %s", dbName, err)
	}
	c.Close()

	// NOTE: in the current implementation of pq (postgres driver), the last key-value pair wins over previous ones with the same key
	dsn = fmt.Sprintf("%s dbname=%s", connDSN, dbName)
	gormDB, err := gorm.Open(driver, dsn)
	if err != nil {
		modelLog.Fatalf("cannot open just created test database `%s`: %s", dsn, err)
	}

	gormDB.SetLogger(modelLog)

	db = &DB{
		gormDB:  gormDB,
		dbName:  sql.NullString{String: dbName, Valid: true},
		connDSN: sql.NullString{String: connDSN, Valid: true},
	}

	if err := db.migrate(); err != nil {
		return nil, dsn, fmt.Errorf("could not migrate database: %s", err)
	}

	return db, dsn, nil

}

func randomDBName(prefix string, strlen int) string {
	rand.Seed(time.Now().UTC().UnixNano())
	const chars = "abcdefghijklmnopqrstuvwxyz0123456789"
	result := make([]byte, strlen)
	for i := 0; i < strlen; i++ {
		result[i] = chars[rand.Intn(len(chars))]
	}
	return prefix + string(result)
}

func NullIntValue(value int64) sql.NullInt64 {
	return sql.NullInt64{Int64: value, Valid: true}
}

func NullInt() sql.NullInt64 {
	return sql.NullInt64{}
}

func NullIntToPtr(nullint sql.NullInt64) *int64 {
	if nullint.Valid {
		value := nullint.Int64
		return &value
	} else {
		return nil
	}
}

func PtrToNullInt(value *int64) sql.NullInt64 {
	if value != nil {
		return NullIntValue(*value)
	} else {
		return NullInt()
	}
}

func IsIntegrityConstraintViolation(err error) bool {
	if driverErr, ok := err.(*pq.Error); ok && driverErr.Code.Class() == "23" { // Integrity Constraint Violation
		return true
	} else {
		return false
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"log"
	"io/ioutil"
	"os"
	"path"
	"strconv"
	"strings"

	"github.com/phil-mansfield/gotetra/cosmo"
	"github.com/phil-mansfield/gotetra/render/io"
	"github.com/phil-mansfield/gotetra/render/halo"
	rgeom "github.com/phil-mansfield/gotetra/render/geom"
	"github.com/phil-mansfield/gotetra/los/geom"
	"github.com/phil-mansfield/gotetra/los/analyze"
)

type Params struct {
	MaxMult float64
}

func main() {
	p := parseCmd()
	ids, snaps, coeffs, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }
	snapBins, coeffBins, idxBins := binBySnap(snaps, ids, coeffs)

	masses := make([]float64, len(ids))

	for snap, snapIDs := range snapBins {
		idxs := idxBins[snap]
		snapCoeffs := coeffBins[snap]
		if snap == -1 { continue }

		hds, files, err := readHeaders(snap)
		if err != nil { log.Fatal(err.Error()) }
		hBounds, err := boundingSpheres(snap, &hds[0], snapIDs, p)
		if err != nil { log.Fatal(err.Error()) }
		intrBins := binIntersections(hds, hBounds)

		xs := []rgeom.Vec{}
		for i := range hds {
			if len(intrBins[i]) == 0 { continue }
			hd := &hds[i]

			n := hd.GridWidth*hd.GridWidth*hd.GridWidth
			if len(xs) == 0 { xs = make([]rgeom.Vec, n) }
			err := io.ReadSheetPositionsAt(files[i], xs)
			if err != nil { log.Fatal(err.Error()) }

			for j := range idxs {
				masses[idxs[j]] += massContained(
					&hds[i], xs, snapCoeffs[j], hBounds[j],
				)
			}
		}
	}

	printMasses(ids, snaps, masses)
}

func parseStdin() (ids, snaps []int, coeffs [][]float64, err error) {
	ids, snaps, coeffs = []int{}, []int{}, [][]float64{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, nil, err }
	for i, line := range lines {
		rawTokens := strings.Split(line, " ")
		tokens := make([]string, 0, len(rawTokens))
		for _, tok := range rawTokens {
			if len(tok) != 0 { tokens = append(tokens, tok) }
		}

		var (
			id, snap int
			hCoeffs []float64
			err error
		)
		switch {
		case len(tokens) == 0:
			continue
		case len(tokens) <= 2:
			if tokens[0] == "" { continue }
			return nil, nil, nil, fmt.Errorf(
				"Line %d of stdin has 1 token, but >2 are required.", i + 1,
			)
		case len(tokens) > 2:
			id, err = strconv.Atoi(tokens[0])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[0],
				)
			}
			snap, err = strconv.Atoi(tokens[1])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[1],
				)
			}

			hCoeffs = make([]float64, len(tokens) - 2)
			for i := range hCoeffs {
				hCoeffs[i], err = strconv.ParseFloat(tokens[i + 2], 64)
			}
		}

		ids = append(ids, id)
		snaps = append(snaps, snap)
		coeffs = append(coeffs, hCoeffs)
	}

	return ids, snaps, coeffs, nil
}

func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func parseCmd() *Params {
	p := &Params{}
	flag.Float64Var(&p.MaxMult, "MaxMult", 3,
		"Ending radius of LoSs as a multiple of R_200m. " +
			"Should be the same value as used in gtet_shell.")
	flag.Parse()
	return p
}

func binBySnap(
	snaps, ids []int, coeffs [][]float64,
) (snapBins map[int][]int,coeffBins map[int][][]float64,idxBins map[int][]int) {
	snapBins = make(map[int][]int)
	coeffBins = make(map[int][][]float64)
	idxBins = make(map[int][]int)
	for i, snap := range snaps {
		snapBins[snap] = append(snapBins[snap], ids[i])
		coeffBins[snap] = append(coeffBins[snap], coeffs[i])
		idxBins[snap] = append(idxBins[snap], i)
	}
	return snapBins, coeffBins, idxBins
}

func readHeaders(snap int) ([]io.SheetHeader, []string, error) {
	memoDir := os.Getenv("GTET_MEMO_DIR")
	if memoDir == "" {
		// You don't want to memoize? Fine. Deal with the consequences.
		return readHeadersFromSheet(snap)
	}
	if _, err := os.Stat(memoDir); err != nil {
		return nil, nil, err
	}

	memoFile := path.Join(memoDir, fmt.Sprintf("hd_snap%d.dat", snap))

	if _, err := os.Stat(memoFile); err != nil {
		// File not written yet.
		hds, files, err := readHeadersFromSheet(snap)
		if err != nil { return nil, nil, err }

        f, err := os.Create(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()
        binary.Write(f, binary.LittleEndian, hds)

		return hds, files, nil
	} else {
		// File exists: read from it instead.

		f, err := os.Open(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()

		n, err := sheetNum(snap)
		if err != nil { return nil, nil, err }
		hds := make([]io.SheetHeader, n)
        binary.Read(f, binary.LittleEndian, hds)

		gtetFmt := os.Getenv("GTET_FMT")
		dir := fmt.Sprintf(gtetFmt, snap)
		files, err := dirContents(dir)
		if err != nil { return nil, nil, err }

		return hds, files, nil
	}
}

func sheetNum(snap int) (int, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return 0, err }
	return len(files), nil
}

func readHeadersFromSheet(snap int) ([]io.SheetHeader, []string, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return nil, nil, err }

	hds := make([]io.SheetHeader, len(files))
	for i := range files {
		err = io.ReadSheetHeaderAt(files[i], &hds[i])
		if err != nil { return nil, nil, err }
	}
	return hds, files, nil
}

func dirContents(dir string) ([]string, error) {
	infos, err := ioutil.ReadDir(dir)
	if err != nil { return nil, err }

	files := make([]string, len(infos))
	for i := range infos {
		files[i] = path.Join(dir, infos[i].Name())
	}

	return files, nil
}

func wrapDist(x1, x2, width float32) float32 {
	dist := x1 - x2
	if dist > width / 2 {
		return dist - width
	} else if dist < width / -2 {
		return dist + width
	} else {
		return dist
	}
}

func inRange(x, r, low, width, tw float32) bool {
	return wrapDist(x, low, tw) > -r && wrapDist(x, low + width, tw) < r
}

// SheetIntersect returns true if the given halo and sheet intersect one another
// and false otherwise.
func sheetIntersect(s geom.Sphere, hd *io.SheetHeader) bool {
	tw := float32(hd.TotalWidth)
	return inRange(s.C[0], s.R, hd.Origin[0], hd.Width[0], tw) &&
		inRange(s.C[1], s.R, hd.Origin[1], hd.Width[1], tw) &&
		inRange(s.C[2], s.R, hd.Origin[2], hd.Width[2], tw)
}

func binIntersections(
	hds []io.SheetHeader, spheres []geom.Sphere,
) [][]geom.Sphere {
	bins := make([][]geom.Sphere, len(hds))
	for i := range hds {
		for si := range spheres {
			if sheetIntersect(spheres[si], &hds[i]) {
				bins[i] = append(bins[i], spheres[si])
			}
		}
	}
	return bins
}

func boundingSpheres(
	snap int, hd *io.SheetHeader, ids []int, p *Params,
) ([]geom.Sphere, error) {
	rockstarDir := os.Getenv("GTET_ROCKSTAR_DIR")
	if rockstarDir == "" {
		return nil, fmt.Errorf("$GTET_ROCKSTAR_DIR not set.")
	}

	hlists, err := dirContents(rockstarDir)
	if err != nil { return nil, err }
	rids, vals, err := halo.ReadRockstarVals(
		hlists[snap - 1], &hd.Cosmo, halo.X, halo.Y, halo.Z, halo.Rad200b,
	)
	xs, ys, zs, rs := vals[0], vals[1], vals[2], vals[3]

	spheres := make([]geom.Sphere, len(ids))
	for i := range spheres {
		j := -1
		for idx := range xs {
			if rids[idx] == ids[i] {
				j = idx
				break
			}
		}

		if j == -1 {
			return nil, fmt.Errorf("Halo %d not found in snap %d.",
				ids[i], snap)
		}
		spheres[i].C = geom.Vec{float32(xs[j]), float32(ys[j]), float32(zs[j])}
		spheres[i].R = float32(rs[j])
	}

	return spheres, nil
}

func findOrder(coeffs []float64) int {
	i := 1
	for {
		if 2*i*i == len(coeffs) {
			return i
		} else if 2*i*i > len(coeffs) {
			panic("Impossible")
		}
		i++
	}
}

func wrap(x, tw2 float32) float32 {
	if x > tw2 {
		return x - tw2
	} else if x < -tw2 {
		return x + tw2
	}
	return x
}

func coords(idx, cells int64) (x, y, z int64) {
    x = idx % cells
    y = (idx % (cells * cells)) / cells
    z = idx / (cells * cells)
    return x, y, z
}

func massContained(
	hd *io.SheetHeader, xs []rgeom.Vec, coeffs []float64, sphere geom.Sphere,
) float64 {
	c := &hd.Cosmo
	rhoM := cosmo.RhoAverage(c.H100 * 100, c.OmegaM, c.OmegaL, c.Z )
	dx := hd.TotalWidth / float64(hd.CountWidth) / (1 + c.Z)
	ptMass := rhoM * (dx*dx*dx)
	tw2 := float32(hd.TotalWidth) / 2

	order := findOrder(coeffs)
	shell := analyze.PennaFunc(coeffs, order, order, 2)

	// This prevents excess calls to the shell function:
	low, high := shell.RadialRange(10 * 1000)
	low2, high2 := float32(low*low), float32(high*high)

	sum := 0.0
	sw := hd.SegmentWidth
	for si := int64(0); si < sw*sw*sw; si++ {
		xi, yi, zi := coords(si, hd.SegmentWidth)
		i := xi + yi*sw + zi*sw*sw
		x, y, z := xs[i][0], xs[i][1], xs[i][2]
		x, y, z = x - sphere.C[0], y - sphere.C[1], z - sphere.C[2]
		x = wrap(x, tw2)
		y = wrap(y, tw2)
		z = wrap(z, tw2)

		r2 := x*x + y*y +z*z

		if r2 < low2 || ( r2 < high2 &&
			shell.Contains(float64(x), float64(y), float64(z))) {
			sum += ptMass
		}
	}
	return sum
}

func printMasses(ids, snaps []int, masses []float64) {
	idWidth, snapWidth, massWidth := 0, 0, 0
	for i := range ids {
		iWidth := len(fmt.Sprintf("%d", ids[i]))
		sWidth := len(fmt.Sprintf("%d", snaps[i]))
		mWidth := len(fmt.Sprintf("%.5g", masses[i]))
		if iWidth > idWidth { idWidth = iWidth }
		if sWidth > snapWidth { snapWidth = sWidth }
		if mWidth > massWidth { massWidth = mWidth }
	}

	rowFmt := fmt.Sprintf("%%%dd %%%dd %%%d.5g\n",
		idWidth, snapWidth, massWidth)
	for i := range ids { fmt.Printf(rowFmt, ids[i], snaps[i], masses[i]) }
}
<commit_msg>Changed gtet_mass to also report radius.<commit_after>package main

import (
	"encoding/binary"
	"flag"
	"fmt"
	"log"
	"io/ioutil"
	"math"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"

	"github.com/phil-mansfield/gotetra/cosmo"
	"github.com/phil-mansfield/gotetra/render/io"
	"github.com/phil-mansfield/gotetra/render/halo"
	rgeom "github.com/phil-mansfield/gotetra/render/geom"
	"github.com/phil-mansfield/gotetra/los/geom"
	"github.com/phil-mansfield/gotetra/los/analyze"
)

const (
	minSnap = 30
)

type Params struct {
	MaxMult float64
}

func main() {
	p := parseCmd()
	ids, snaps, coeffs, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }
	snapBins, coeffBins, idxBins := binBySnap(snaps, ids, coeffs)

	masses := make([]float64, len(ids))
	rads := make([]float64, len(ids))

	sortedSnaps := []int{}
	for snap := range snapBins {
		sortedSnaps = append(sortedSnaps, snap)
	}
	sort.Ints(sortedSnaps)

	log.Println("gtet_mass")
	for _, snap := range sortedSnaps {
		log.Println("Snap", snap)
		snapIDs := snapBins[snap]
		snapCoeffs := coeffBins[snap]
		idxs := idxBins[snap]

		if snap < minSnap { continue }

		hds, files, err := readHeaders(snap)
		if err != nil { log.Fatal(err.Error()) }
		hBounds, err := boundingSpheres(snap, &hds[0], snapIDs, p)
		if err != nil { log.Fatal(err.Error()) }
		intrBins := binIntersections(hds, hBounds)

		xs := []rgeom.Vec{}
		for i := range hds {
			if len(intrBins[i]) == 0 { continue }
			hd := &hds[i]

			n := hd.GridWidth*hd.GridWidth*hd.GridWidth
			if len(xs) == 0 { xs = make([]rgeom.Vec, n) }
			err := io.ReadSheetPositionsAt(files[i], xs)
			if err != nil { log.Fatal(err.Error()) }

			for j := range idxs {
				masses[idxs[j]] += massContained(
					&hds[i], xs, snapCoeffs[j], hBounds[j],
				)
			}
		}

		for j := range idxs {
			rads[idxs[j]] = rSp(&hds[0], snapCoeffs[j])
		}
	}

	printMasses(ids, snaps, masses, rads)
}

func parseStdin() (ids, snaps []int, coeffs [][]float64, err error) {
	ids, snaps, coeffs = []int{}, []int{}, [][]float64{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, nil, err }
	for i, line := range lines {
		rawTokens := strings.Split(line, " ")
		tokens := make([]string, 0, len(rawTokens))
		for _, tok := range rawTokens {
			if len(tok) != 0 { tokens = append(tokens, tok) }
		}

		var (
			id, snap int
			hCoeffs []float64
			err error
		)
		switch {
		case len(tokens) == 0:
			continue
		case len(tokens) <= 2:
			if tokens[0] == "" { continue }
			return nil, nil, nil, fmt.Errorf(
				"Line %d of stdin has 1 token, but >2 are required.", i + 1,
			)
		case len(tokens) > 2:
			id, err = strconv.Atoi(tokens[0])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[0],
				)
			}
			snap, err = strconv.Atoi(tokens[1])
			if err != nil {
				return nil, nil, nil, fmt.Errorf(
					"One line %d of stdin, %s does not parse as an int.",
					i + 1, tokens[1],
				)
			}

			hCoeffs = make([]float64, len(tokens) - 2)
			for i := range hCoeffs {
				hCoeffs[i], err = strconv.ParseFloat(tokens[i + 2], 64)
			}
		}

		ids = append(ids, id)
		snaps = append(snaps, snap)
		coeffs = append(coeffs, hCoeffs)
	}

	return ids, snaps, coeffs, nil
}

func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func parseCmd() *Params {
	p := &Params{}
	flag.Float64Var(&p.MaxMult, "MaxMult", 3,
		"Ending radius of LoSs as a multiple of R_200m. " +
			"Should be the same value as used in gtet_shell.")
	flag.Parse()
	return p
}

func binBySnap(
	snaps, ids []int, coeffs [][]float64,
) (snapBins map[int][]int,coeffBins map[int][][]float64,idxBins map[int][]int) {
	snapBins = make(map[int][]int)
	coeffBins = make(map[int][][]float64)
	idxBins = make(map[int][]int)
	for i, snap := range snaps {
		snapBins[snap] = append(snapBins[snap], ids[i])
		coeffBins[snap] = append(coeffBins[snap], coeffs[i])
		idxBins[snap] = append(idxBins[snap], i)
	}
	return snapBins, coeffBins, idxBins
}

func readHeaders(snap int) ([]io.SheetHeader, []string, error) {
	memoDir := os.Getenv("GTET_MEMO_DIR")
	if memoDir == "" {
		// You don't want to memoize? Fine. Deal with the consequences.
		return readHeadersFromSheet(snap)
	}
	if _, err := os.Stat(memoDir); err != nil {
		return nil, nil, err
	}

	memoFile := path.Join(memoDir, fmt.Sprintf("hd_snap%d.dat", snap))

	if _, err := os.Stat(memoFile); err != nil {
		// File not written yet.
		hds, files, err := readHeadersFromSheet(snap)
		if err != nil { return nil, nil, err }

        f, err := os.Create(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()
        binary.Write(f, binary.LittleEndian, hds)

		return hds, files, nil
	} else {
		// File exists: read from it instead.

		f, err := os.Open(memoFile)
        if err != nil { return nil, nil, err }
        defer f.Close()

		n, err := sheetNum(snap)
		if err != nil { return nil, nil, err }
		hds := make([]io.SheetHeader, n)
        binary.Read(f, binary.LittleEndian, hds)

		gtetFmt := os.Getenv("GTET_FMT")
		dir := fmt.Sprintf(gtetFmt, snap)
		files, err := dirContents(dir)
		if err != nil { return nil, nil, err }

		return hds, files, nil
	}
}

func sheetNum(snap int) (int, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return 0, err }
	return len(files), nil
}

func readHeadersFromSheet(snap int) ([]io.SheetHeader, []string, error) {
	gtetFmt := os.Getenv("GTET_FMT")
	dir := fmt.Sprintf(gtetFmt, snap)
	files, err := dirContents(dir)
	if err != nil { return nil, nil, err }

	hds := make([]io.SheetHeader, len(files))
	for i := range files {
		err = io.ReadSheetHeaderAt(files[i], &hds[i])
		if err != nil { return nil, nil, err }
	}
	return hds, files, nil
}

func dirContents(dir string) ([]string, error) {
	infos, err := ioutil.ReadDir(dir)
	if err != nil { return nil, err }

	files := make([]string, len(infos))
	for i := range infos {
		files[i] = path.Join(dir, infos[i].Name())
	}

	return files, nil
}

func wrapDist(x1, x2, width float32) float32 {
	dist := x1 - x2
	if dist > width / 2 {
		return dist - width
	} else if dist < width / -2 {
		return dist + width
	} else {
		return dist
	}
}

func inRange(x, r, low, width, tw float32) bool {
	return wrapDist(x, low, tw) > -r && wrapDist(x, low + width, tw) < r
}

// SheetIntersect returns true if the given halo and sheet intersect one another
// and false otherwise.
func sheetIntersect(s geom.Sphere, hd *io.SheetHeader) bool {
	tw := float32(hd.TotalWidth)
	return inRange(s.C[0], s.R, hd.Origin[0], hd.Width[0], tw) &&
		inRange(s.C[1], s.R, hd.Origin[1], hd.Width[1], tw) &&
		inRange(s.C[2], s.R, hd.Origin[2], hd.Width[2], tw)
}

func binIntersections(
	hds []io.SheetHeader, spheres []geom.Sphere,
) [][]geom.Sphere {
	bins := make([][]geom.Sphere, len(hds))
	for i := range hds {
		for si := range spheres {
			if sheetIntersect(spheres[si], &hds[i]) {
				bins[i] = append(bins[i], spheres[si])
			}
		}
	}
	return bins
}

func boundingSpheres(
	snap int, hd *io.SheetHeader, ids []int, p *Params,
) ([]geom.Sphere, error) {
	rockstarDir := os.Getenv("GTET_ROCKSTAR_DIR")
	if rockstarDir == "" {
		return nil, fmt.Errorf("$GTET_ROCKSTAR_DIR not set.")
	}

	hlists, err := dirContents(rockstarDir)
	if err != nil { return nil, err }
	rids, vals, err := halo.ReadRockstarVals(
		hlists[snap - 1], &hd.Cosmo, halo.X, halo.Y, halo.Z, halo.Rad200b,
	)
	xs, ys, zs, rs := vals[0], vals[1], vals[2], vals[3]

	spheres := make([]geom.Sphere, len(ids))
	for i := range spheres {
		j := -1
		for idx := range xs {
			if rids[idx] == ids[i] {
				j = idx
				break
			}
		}

		if j == -1 {
			return nil, fmt.Errorf("Halo %d not found in snap %d.",
				ids[i], snap)
		}
		spheres[i].C = geom.Vec{float32(xs[j]), float32(ys[j]), float32(zs[j])}
		spheres[i].R = float32(rs[j])
	}

	return spheres, nil
}

func findOrder(coeffs []float64) int {
	i := 1
	for {
		if 2*i*i == len(coeffs) {
			return i
		} else if 2*i*i > len(coeffs) {
			panic("Impossible")
		}
		i++
	}
}

func wrap(x, tw2 float32) float32 {
	if x > tw2 {
		return x - tw2
	} else if x < -tw2 {
		return x + tw2
	}
	return x
}

func coords(idx, cells int64) (x, y, z int64) {
    x = idx % cells
    y = (idx % (cells * cells)) / cells
    z = idx / (cells * cells)
    return x, y, z
}

func rSp(hd *io.SheetHeader, coeffs []float64) float64 {
	order := findOrder(coeffs)
	shell := analyze.PennaFunc(coeffs, order, order, 2)
	vol := shell.Volume(10 * 1000)

	r := math.Pow(vol / (math.Pi * 4 / 3), 0.33333)
	return r
}

func massContained(
	hd *io.SheetHeader, xs []rgeom.Vec, coeffs []float64, sphere geom.Sphere,
) float64 {
	c := &hd.Cosmo
	rhoM := cosmo.RhoAverage(c.H100 * 100, c.OmegaM, c.OmegaL, c.Z )
	dx := hd.TotalWidth / float64(hd.CountWidth) / (1 + c.Z)
	ptMass := rhoM * (dx*dx*dx)
	tw2 := float32(hd.TotalWidth) / 2

	order := findOrder(coeffs)
	shell := analyze.PennaFunc(coeffs, order, order, 2)

	// This prevents excess calls to the shell function:
	low, high := shell.RadialRange(10 * 1000)
	low2, high2 := float32(low*low), float32(high*high)

	sum := 0.0
	sw := hd.SegmentWidth
	for si := int64(0); si < sw*sw*sw; si++ {
		xi, yi, zi := coords(si, hd.SegmentWidth)
		i := xi + yi*sw + zi*sw*sw
		x, y, z := xs[i][0], xs[i][1], xs[i][2]
		x, y, z = x - sphere.C[0], y - sphere.C[1], z - sphere.C[2]
		x = wrap(x, tw2)
		y = wrap(y, tw2)
		z = wrap(z, tw2)

		r2 := x*x + y*y +z*z

		if r2 < low2 || ( r2 < high2 &&
			shell.Contains(float64(x), float64(y), float64(z))) {
			sum += ptMass
		}
	}
	return sum
}

func printMasses(ids, snaps []int, masses, rads []float64) {
	idWidth, snapWidth, massWidth, radWidth := 0, 0, 0, 0
	for i := range ids {
		iWidth := len(fmt.Sprintf("%d", ids[i]))
		sWidth := len(fmt.Sprintf("%d", snaps[i]))
		mWidth := len(fmt.Sprintf("%.5g", masses[i]))
		rWidth := len(fmt.Sprintf("%.5g", rads[i]))
		if iWidth > idWidth { idWidth = iWidth }
		if sWidth > snapWidth { snapWidth = sWidth }
		if mWidth > massWidth { massWidth = mWidth }
		if rWidth > radWidth { radWidth = rWidth }
	}

	rowFmt := fmt.Sprintf("%%%dd %%%dd %%%d.5g %%%d.5g\n",
		idWidth, snapWidth, massWidth, radWidth)
	for i := range ids { fmt.Printf(rowFmt, ids[i], snaps[i], masses[i], rads[i]) }
}
<|endoftext|>


<commit_before>package hostqueue

import (
	"bytes"
	"fmt"
	"net/http"
	"strings"

	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"

	"github.com/sendgrid/sendgrid-go"
)

func sendReminder(group Group, r *http.Request) {
	c := appengine.NewContext(r)

	creds, err := GetCreds(c)
	if err != nil {
		panic(err)
	}

	sg := sendgrid.NewSendGridClient(creds.Username, creds.Pass)
	sg.Client = urlfetch.Client(c)
	email := group.GroupEmail
	hostName := group.Hosts[group.Next].HostName
	c.Infof("Email: %v", email)
	c.Infof("Host Name: %v", hostName)

	html := hostName + hostMessage
	sendEmail(email, "This weeks hosting reminder", html, r)
}

func sendSkipMessage(group Group, r *http.Request) {
	c := appengine.NewContext(r)

	email := group.GroupEmail
	var buffer bytes.Buffer
	for _, element := range group.Hosts {
		buffer.WriteString(element.HostName)
	}

	hosts := make([]string, len(group.Hosts))
	for i, element := range group.Hosts {
		hosts[i] = element.HostName
	}
	c.Infof("hosts: %s", strings.Join(hosts[:], ","))
	c.Infof("Email: %v", email)

	html := fmt.Sprintf(skipMessage, strings.Join(hosts[:], ", "))
	sendEmail(email, "See you next week", html, r)
}

func sendHostConfirmedMessage(group Group, r *http.Request) {
	c := appengine.NewContext(r)

	email := group.GroupEmail
	var buffer bytes.Buffer
	for _, element := range group.Hosts {
		buffer.WriteString(element.HostName)
	}

	hosts := make([]string, len(group.Hosts))
	for i, element := range group.Hosts {
		hosts[i] = element.HostName
	}
	c.Infof("hosts: %s", strings.Join(hosts[:], ","))
	c.Infof("Email: %v", email)

	html := fmt.Sprintf(confirmedMessage, group.Hosts[len(group.Hosts)-1].HostName, strings.Join(hosts[:], ", "))
	sendEmail(email, "See you next week", html, r)

}

func sendEmail(email string, subject string, html string, r *http.Request) {
	c := appengine.NewContext(r)

	creds, err := GetCreds(c)
	if err != nil {
		panic(err)
	}

	sg := sendgrid.NewSendGridClient(creds.Username, creds.Pass)
	sg.Client = urlfetch.Client(c)

	message := sendgrid.NewMail()
	message.AddTo(email)
	message.SetSubject(subject)
	message.SetHTML(html)
	message.SetFrom(from)

	err = sg.Send(message)
	if err != nil {
		c.Infof("Message: %v", message)
		panic(err)
	}
}

/*App engine allows for environment variables, but they are stored in app.yaml
and I don't want my mail creds pushed to a public repo.  */
type EmailCreds struct {
	Username string `json:"username"`
	Pass     string `json:"password"`
}

func GetCreds(ctx appengine.Context) (EmailCreds, error) {
	k := datastore.NewKey(ctx, "EmailCreds", "singleton_creds", 0, nil)
	var ec EmailCreds
	err := datastore.Get(ctx, k, &ec)
	if err != nil {
		return EmailCreds{}, err
	}

	return ec, nil
}

const from = "reminder@hostqueue-1146.appspotmail.com"

const hostMessage = ` it is your turn to host!
Respond with 'yes' to host, 'no' to go to the next in line to host, or 'skip' for everyone to skip this week and host next week.
`

const skipMessage = `See you next week with the following turn order:  %s`

const confirmedMessage = `The %s has agreed to host this week.  The rotation for next week will be: %s`
<commit_msg>added todo for handing error.  Probaly a good idea where I'm checking most err.<commit_after>package hostqueue

import (
	"bytes"
	"fmt"
	"net/http"
	"strings"

	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"

	"github.com/sendgrid/sendgrid-go"
)

func sendReminder(group Group, r *http.Request) {
	c := appengine.NewContext(r)

	creds, err := GetCreds(c)
	if err != nil {
		panic(err)
	}

	sg := sendgrid.NewSendGridClient(creds.Username, creds.Pass)
	sg.Client = urlfetch.Client(c)
	email := group.GroupEmail
	hostName := group.Hosts[group.Next].HostName
	c.Infof("Email: %v", email)
	c.Infof("Host Name: %v", hostName)

	html := hostName + hostMessage
	sendEmail(email, "This weeks hosting reminder", html, r)
}

func sendSkipMessage(group Group, r *http.Request) {
	c := appengine.NewContext(r)

	email := group.GroupEmail
	var buffer bytes.Buffer
	for _, element := range group.Hosts {
		buffer.WriteString(element.HostName)
	}

	hosts := make([]string, len(group.Hosts))
	for i, element := range group.Hosts {
		hosts[i] = element.HostName
	}
	c.Infof("hosts: %s", strings.Join(hosts[:], ","))
	c.Infof("Email: %v", email)

	html := fmt.Sprintf(skipMessage, strings.Join(hosts[:], ", "))
	sendEmail(email, "See you next week", html, r)
}

func sendHostConfirmedMessage(group Group, r *http.Request) {
	c := appengine.NewContext(r)

	email := group.GroupEmail
	var buffer bytes.Buffer
	for _, element := range group.Hosts {
		buffer.WriteString(element.HostName)
	}

	hosts := make([]string, len(group.Hosts))
	for i, element := range group.Hosts {
		hosts[i] = element.HostName
	}
	c.Infof("hosts: %s", strings.Join(hosts[:], ","))
	c.Infof("Email: %v", email)

	html := fmt.Sprintf(confirmedMessage, group.Hosts[len(group.Hosts)-1].HostName, strings.Join(hosts[:], ", "))
	sendEmail(email, "See you next week", html, r)

}

func sendEmail(email string, subject string, html string, r *http.Request) {
	c := appengine.NewContext(r)

	creds, err := GetCreds(c)
	if err != nil {
		panic(err)
	}

	sg := sendgrid.NewSendGridClient(creds.Username, creds.Pass)
	sg.Client = urlfetch.Client(c)

	message := sendgrid.NewMail()
	message.AddTo(email)
	message.SetSubject(subject)
	message.SetHTML(html)
	message.SetFrom(from)

	err = sg.Send(message)
	if err != nil {
		//TODO:  autocreate an issue in github if this is unique in the last month
		c.Infof("Message: %v", message)
		panic(err)
	}
}

/*App engine allows for environment variables, but they are stored in app.yaml
and I don't want my mail creds pushed to a public repo.  */
type EmailCreds struct {
	Username string `json:"username"`
	Pass     string `json:"password"`
}

func GetCreds(ctx appengine.Context) (EmailCreds, error) {
	k := datastore.NewKey(ctx, "EmailCreds", "singleton_creds", 0, nil)
	var ec EmailCreds
	err := datastore.Get(ctx, k, &ec)
	if err != nil {
		return EmailCreds{}, err
	}

	return ec, nil
}

const from = "reminder@hostqueue-1146.appspotmail.com"

const hostMessage = ` it is your turn to host!
Respond with 'yes' to host, 'no' to go to the next in line to host, or 'skip' for everyone to skip this week and host next week.
`

const skipMessage = `See you next week with the following turn order:  %s`

const confirmedMessage = `The %s has agreed to host this week.  The rotation for next week will be: %s`
<|endoftext|>


<commit_before>package gotool

// This file contains code from the Go distribution.

// Copyright (c) 2012 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import (
	"fmt"
	"go/build"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

var (
	goroot       = filepath.Clean(runtime.GOROOT())
	gorootSrcPkg = filepath.Join(goroot, "src/pkg")
)

var buildContext = build.Default

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}

func matchPackages(pattern string) []string {
	match := func(string) bool { return true }
	if pattern != "all" && pattern != "std" {
		match = matchPattern(pattern)
	}

	have := map[string]bool{
		"builtin": true, // ignore pseudo-package that exists only for documentation
	}
	if !buildContext.CgoEnabled {
		have["runtime/cgo"] = true // ignore during walk
	}
	var pkgs []string

	for _, src := range buildContext.SrcDirs() {
		if pattern == "std" && src != gorootSrcPkg {
			continue
		}
		src = filepath.Clean(src) + string(filepath.Separator)
		filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
			if err != nil || !fi.IsDir() || path == src {
				return nil
			}

			// Avoid .foo, _foo, and testdata directory trees.
			_, elem := filepath.Split(path)
			if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
				return filepath.SkipDir
			}

			name := filepath.ToSlash(path[len(src):])
			if pattern == "std" && strings.Contains(name, ".") {
				return filepath.SkipDir
			}
			if have[name] {
				return nil
			}
			have[name] = true
			if !match(name) {
				return nil
			}
			_, err = buildContext.ImportDir(path, 0)
			if err != nil && strings.Contains(err.Error(), "no Go source files") {
				return nil
			}
			pkgs = append(pkgs, name)
			return nil
		})
	}
	return pkgs
}

// importPathsNoDotExpansion returns the import paths to use for the given
// command line, but it does no ... expansion.
func importPathsNoDotExpansion(args []string) []string {
	if len(args) == 0 {
		return []string{"."}
	}
	var out []string
	for _, a := range args {
		// Arguments are supposed to be import paths, but
		// as a courtesy to Windows developers, rewrite \ to /
		// in command-line arguments.  Handles .\... and so on.
		if filepath.Separator == '\\' {
			a = strings.Replace(a, `\`, `/`, -1)
		}

		// Put argument in canonical form, but preserve leading ./.
		if strings.HasPrefix(a, "./") {
			a = "./" + path.Clean(a)
			if a == "./." {
				a = "."
			}
		} else {
			a = path.Clean(a)
		}
		if a == "all" || a == "std" {
			out = append(out, allPackages(a)...)
			continue
		}
		out = append(out, a)
	}
	return out
}

// importPaths returns the import paths to use for the given command line.
func importPaths(args []string) []string {
	args = importPathsNoDotExpansion(args)
	var out []string
	for _, a := range args {
		if strings.Contains(a, "...") {
			if build.IsLocalImport(a) {
				out = append(out, allPackagesInFS(a)...)
			} else {
				out = append(out, allPackages(a)...)
			}
			continue
		}
		out = append(out, a)
	}
	return out
}

// allPackages returns all the packages that can be found
// under the $GOPATH directories and $GOROOT matching pattern.
// The pattern is either "all" (all packages), "std" (standard packages)
// or a path including "...".
func allPackages(pattern string) []string {
	pkgs := matchPackages(pattern)
	if len(pkgs) == 0 {
		fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
	}
	return pkgs
}

// allPackagesInFS is like allPackages but is passed a pattern
// beginning ./ or ../, meaning it should scan the tree rooted
// at the given directory.  There are ... in the pattern too.
func allPackagesInFS(pattern string) []string {
	pkgs := matchPackagesInFS(pattern)
	if len(pkgs) == 0 {
		fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
	}
	return pkgs
}

func matchPackagesInFS(pattern string) []string {
	// Find directory to begin the scan.
	// Could be smarter but this one optimization
	// is enough for now, since ... is usually at the
	// end of a path.
	i := strings.Index(pattern, "...")
	dir, _ := path.Split(pattern[:i])

	// pattern begins with ./ or ../.
	// path.Clean will discard the ./ but not the ../.
	// We need to preserve the ./ for pattern matching
	// and in the returned import paths.
	prefix := ""
	if strings.HasPrefix(pattern, "./") {
		prefix = "./"
	}
	match := matchPattern(pattern)

	var pkgs []string
	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if err != nil || !fi.IsDir() {
			return nil
		}
		if path == dir {
			// filepath.Walk starts at dir and recurses. For the recursive case,
			// the path is the result of filepath.Join, which calls filepath.Clean.
			// The initial case is not Cleaned, though, so we do this explicitly.
			//
			// This converts a path like "./io/" to "io". Without this step, running
			// "cd $GOROOT/src/pkg; go list ./io/..." would incorrectly skip the io
			// package, because prepending the prefix "./" to the unclean path would
			// result in "././io", and match("././io") returns false.
			path = filepath.Clean(path)
		}

		// Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..".
		_, elem := filepath.Split(path)
		dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".."
		if dot || strings.HasPrefix(elem, "_") || elem == "testdata" {
			return filepath.SkipDir
		}

		name := prefix + filepath.ToSlash(path)
		if !match(name) {
			return nil
		}
		if _, err = build.ImportDir(path, 0); err != nil {
			return nil
		}
		pkgs = append(pkgs, name)
		return nil
	})
	return pkgs
}
<commit_msg>use build.NoGoError instead of string comparison<commit_after>package gotool

// This file contains code from the Go distribution.

// Copyright (c) 2012 The Go Authors. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//    * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//    * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//    * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import (
	"fmt"
	"go/build"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
)

var (
	goroot       = filepath.Clean(runtime.GOROOT())
	gorootSrcPkg = filepath.Join(goroot, "src/pkg")
)

var buildContext = build.Default

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}

func matchPackages(pattern string) []string {
	match := func(string) bool { return true }
	if pattern != "all" && pattern != "std" {
		match = matchPattern(pattern)
	}

	have := map[string]bool{
		"builtin": true, // ignore pseudo-package that exists only for documentation
	}
	if !buildContext.CgoEnabled {
		have["runtime/cgo"] = true // ignore during walk
	}
	var pkgs []string

	for _, src := range buildContext.SrcDirs() {
		if pattern == "std" && src != gorootSrcPkg {
			continue
		}
		src = filepath.Clean(src) + string(filepath.Separator)
		filepath.Walk(src, func(path string, fi os.FileInfo, err error) error {
			if err != nil || !fi.IsDir() || path == src {
				return nil
			}

			// Avoid .foo, _foo, and testdata directory trees.
			_, elem := filepath.Split(path)
			if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
				return filepath.SkipDir
			}

			name := filepath.ToSlash(path[len(src):])
			if pattern == "std" && strings.Contains(name, ".") {
				return filepath.SkipDir
			}
			if have[name] {
				return nil
			}
			have[name] = true
			if !match(name) {
				return nil
			}
			_, err = buildContext.ImportDir(path, 0)
			if err != nil {
				if _, noGo := err.(*build.NoGoError); noGo {
					return nil
				}
			}
			pkgs = append(pkgs, name)
			return nil
		})
	}
	return pkgs
}

// importPathsNoDotExpansion returns the import paths to use for the given
// command line, but it does no ... expansion.
func importPathsNoDotExpansion(args []string) []string {
	if len(args) == 0 {
		return []string{"."}
	}
	var out []string
	for _, a := range args {
		// Arguments are supposed to be import paths, but
		// as a courtesy to Windows developers, rewrite \ to /
		// in command-line arguments.  Handles .\... and so on.
		if filepath.Separator == '\\' {
			a = strings.Replace(a, `\`, `/`, -1)
		}

		// Put argument in canonical form, but preserve leading ./.
		if strings.HasPrefix(a, "./") {
			a = "./" + path.Clean(a)
			if a == "./." {
				a = "."
			}
		} else {
			a = path.Clean(a)
		}
		if a == "all" || a == "std" {
			out = append(out, allPackages(a)...)
			continue
		}
		out = append(out, a)
	}
	return out
}

// importPaths returns the import paths to use for the given command line.
func importPaths(args []string) []string {
	args = importPathsNoDotExpansion(args)
	var out []string
	for _, a := range args {
		if strings.Contains(a, "...") {
			if build.IsLocalImport(a) {
				out = append(out, allPackagesInFS(a)...)
			} else {
				out = append(out, allPackages(a)...)
			}
			continue
		}
		out = append(out, a)
	}
	return out
}

// allPackages returns all the packages that can be found
// under the $GOPATH directories and $GOROOT matching pattern.
// The pattern is either "all" (all packages), "std" (standard packages)
// or a path including "...".
func allPackages(pattern string) []string {
	pkgs := matchPackages(pattern)
	if len(pkgs) == 0 {
		fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
	}
	return pkgs
}

// allPackagesInFS is like allPackages but is passed a pattern
// beginning ./ or ../, meaning it should scan the tree rooted
// at the given directory.  There are ... in the pattern too.
func allPackagesInFS(pattern string) []string {
	pkgs := matchPackagesInFS(pattern)
	if len(pkgs) == 0 {
		fmt.Fprintf(os.Stderr, "warning: %q matched no packages\n", pattern)
	}
	return pkgs
}

func matchPackagesInFS(pattern string) []string {
	// Find directory to begin the scan.
	// Could be smarter but this one optimization
	// is enough for now, since ... is usually at the
	// end of a path.
	i := strings.Index(pattern, "...")
	dir, _ := path.Split(pattern[:i])

	// pattern begins with ./ or ../.
	// path.Clean will discard the ./ but not the ../.
	// We need to preserve the ./ for pattern matching
	// and in the returned import paths.
	prefix := ""
	if strings.HasPrefix(pattern, "./") {
		prefix = "./"
	}
	match := matchPattern(pattern)

	var pkgs []string
	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
		if err != nil || !fi.IsDir() {
			return nil
		}
		if path == dir {
			// filepath.Walk starts at dir and recurses. For the recursive case,
			// the path is the result of filepath.Join, which calls filepath.Clean.
			// The initial case is not Cleaned, though, so we do this explicitly.
			//
			// This converts a path like "./io/" to "io". Without this step, running
			// "cd $GOROOT/src/pkg; go list ./io/..." would incorrectly skip the io
			// package, because prepending the prefix "./" to the unclean path would
			// result in "././io", and match("././io") returns false.
			path = filepath.Clean(path)
		}

		// Avoid .foo, _foo, and testdata directory trees, but do not avoid "." or "..".
		_, elem := filepath.Split(path)
		dot := strings.HasPrefix(elem, ".") && elem != "." && elem != ".."
		if dot || strings.HasPrefix(elem, "_") || elem == "testdata" {
			return filepath.SkipDir
		}

		name := prefix + filepath.ToSlash(path)
		if !match(name) {
			return nil
		}
		if _, err = build.ImportDir(path, 0); err != nil {
			return nil
		}
		pkgs = append(pkgs, name)
		return nil
	})
	return pkgs
}
<|endoftext|>


<commit_before>package traytor

import "math"

const (
	// Epsilon is a very small number
	Epsilon float64 = 1e-9
	Inf     float64 = 1e99
)

// Round returns the nearest int to a given float number
func Round(number float64) int {
	return int(math.Floor(number + 0.5))
}

// Round32 returns the nearest int to a given float32 number
func Round32(number float32) int {
	return int(math.Floor(float64(number + 0.5)))
}

// Pow32 is a Pow function which uses float32
func Pow32(x, a float32) float32 {
	return float32(math.Pow(float64(x), float64(a)))
}
<commit_msg>add function for solving 2D systems of equations<commit_after>package traytor

import "math"

const (
	// Epsilon is a very small number
	Epsilon float64 = 1e-9
	// Inf is a very large number
	Inf float64 = 1e99
)

// Round returns the nearest int to a given float number
func Round(number float64) int {
	return int(math.Floor(number + 0.5))
}

// Round32 returns the nearest int to a given float32 number
func Round32(number float32) int {
	return int(math.Floor(float64(number + 0.5)))
}

// Pow32 is a Pow function which uses float32
func Pow32(x, a float32) float32 {
	return float32(math.Pow(float64(x), float64(a)))
}

// SolveEquation solves the following system (returning x and y):
// | a1 * x + b1 * y + c1 = 0
// | a2 * x + b2 * y + c2 = 0
func SolveEquation(a, b, c *Vec3) (float64, float64) {
	coefficientMatrix := [2][2]float64{{a.X, b.X}, {a.Y, b.Y}}
	constantCoefficientMatrix := [2]float64{c.X, c.Y}

	det := coefficientMatrix[0][0]*coefficientMatrix[1][1] -
		coefficientMatrix[1][0]*coefficientMatrix[0][1]
	x := (constantCoefficientMatrix[0]*coefficientMatrix[1][1] -
		constantCoefficientMatrix[1]*coefficientMatrix[0][1]) / det
	y := (constantCoefficientMatrix[1]*coefficientMatrix[0][0] -
		constantCoefficientMatrix[0]*coefficientMatrix[1][0]) / det
	return x, y
}
<|endoftext|>