<commit_before>package agent

import (
	"strings"
	"testing"

	"github.com/weaveworks/flux/common/daemon"
	"github.com/weaveworks/flux/common/data"
	"github.com/weaveworks/flux/common/store"
	"github.com/weaveworks/flux/common/store/inmem"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/stretchr/testify/require"
)

type container struct {
	ID          string
	IPAddress   string
	Image       string
	Labels      map[string]string
	Env         map[string]string
	Ports       map[string]string
	NetworkMode string
}

func resetContainers(cs ...container) ContainerUpdate {
	return ContainerUpdate{
		Containers: makeContainersMap(cs),
		Reset:      true,
	}
}

func addContainer(c container) ContainerUpdate {
	return ContainerUpdate{
		Containers: makeContainersMap([]container{c}),
	}
}

func removeContainer(id string) ContainerUpdate {
	return ContainerUpdate{
		Containers: map[string]*docker.Container{id: nil},
	}
}

func makeContainersMap(cs []container) map[string]*docker.Container {
	containers := make(map[string]*docker.Container)

	for _, c := range cs {
		env := []string{}
		for k, v := range c.Env {
			env = append(env, strings.Join([]string{k, v}, "="))
		}
		ports := map[docker.Port][]docker.PortBinding{}
		for k, v := range c.Ports {
			ports[docker.Port(k)] = []docker.PortBinding{
				docker.PortBinding{
					HostIP:   "0.0.0.0",
					HostPort: v,
				},
			}
		}

		netmode := c.NetworkMode
		if netmode == "" {
			netmode = "default"
		}
		c1 := &docker.Container{
			ID: c.ID,
			HostConfig: &docker.HostConfig{
				NetworkMode: netmode,
			},
			Config: &docker.Config{
				Image:  c.Image,
				Env:    env,
				Labels: c.Labels,
			},
			NetworkSettings: &docker.NetworkSettings{
				IPAddress: c.IPAddress,
				Ports:     ports,
			},
		}
		containers[c.ID] = c1
	}

	return containers
}

const GROUP = "deliberately not default"

type harness struct {
	si *SyncInstances
	store.Store
	es             daemon.ErrorSink
	serviceUpdates chan store.ServiceUpdate
	watchServices  daemon.Component
}

func setup(st store.Store, hostIP, netmode string) (res harness) {
	if st == nil {
		st = inmem.NewInMemStore()
	}

	if netmode == "" {
		netmode = LOCAL
	}

	res.Store = st
	res.si = NewSyncInstances(Config{
		Store:   res.Store,
		Network: netmode,
		HostIP:  hostIP,
	})
	res.es = daemon.NewErrorSink()
	res.serviceUpdates = make(chan store.ServiceUpdate)
	res.watchServices = store.WatchServicesStartFunc(res.Store,
		store.QueryServiceOptions{WithContainerRules: true},
		res.serviceUpdates)(res.es)
	return
}

func (h harness) stop(t *testing.T) {
	h.watchServices.Stop()
	require.Empty(t, h.es)
}

func (h harness) addGroup(serviceName string, labels ...string) {
	if len(labels)%2 != 0 {
		panic("Expected key value ... as arguments")
	}
	sel := make(map[string]string)
	for i := 0; i < len(labels); i += 2 {
		sel[labels[i]] = labels[i+1]
	}

	h.SetContainerRule(serviceName, GROUP,
		data.ContainerRule{Selector: sel})
}

func (h harness) allInstances(t *testing.T) []data.Instance {
	var res []data.Instance
	svcs, err := h.GetAllServices(store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	for _, svc := range svcs {
		for _, inst := range svc.Instances {
			res = append(res, inst.Instance)
		}
	}
	return res
}

func TestSyncInstancesReconcile(t *testing.T) {
	h := setup(nil, "10.98.99.100", GLOBAL)
	h.AddService("foo-svc", data.Service{
		InstancePort: 80,
	})
	h.addGroup("foo-svc", "tag", "bobbins", "image", "foo-image")
	h.AddService("bar-svc", data.Service{
		InstancePort: 80,
	})
	h.addGroup("bar-svc", "flux/foo-label", "blorp")
	h.AddService("boo-svc", data.Service{
		InstancePort: 80,
	})
	h.addGroup("boo-svc", "env.SERVICE_NAME", "boo")

	selectedAddress := "192.168.45.67"

	h.si.processServiceUpdate(<-h.serviceUpdates)
	h.si.processContainerUpdate(resetContainers(container{
		ID:        "selected",
		IPAddress: selectedAddress,
		Image:     "foo-image:bobbins",
		Labels:    map[string]string{"flux/foo-label": "blorp"},
		Env:       map[string]string{"SERVICE_NAME": "boo"},
	}, container{
		ID:        "not",
		IPAddress: "111.111.111.111",
		Image:     "foo-image:not-bobbins",
		Labels:    map[string]string{"flux/foo-label": "something-else"},
		Env:       map[string]string{"SERVICE_NAME": "literally anything"},
	}))

	insts := h.allInstances(t)
	require.Len(t, insts, 3)
	for _, inst := range insts {
		require.Equal(t, selectedAddress, inst.Address)
	}

	h.stop(t)
}

func TestSyncInstancesEvents(t *testing.T) {
	h := setup(nil, "10.98.90.111", "")
	// starting condition
	require.Len(t, h.allInstances(t), 0)

	// no services defined
	h.si.processServiceUpdate(<-h.serviceUpdates)
	h.si.processContainerUpdate(resetContainers(container{
		ID:        "foo",
		Image:     "foo-image:latest",
		IPAddress: "192.168.0.67",
	}))
	require.Len(t, h.allInstances(t), 0)

	h.AddService("foo-svc", data.Service{})
	h.si.processServiceUpdate(<-h.serviceUpdates)
	h.addGroup("foo-svc", "image", "foo-image")
	h.si.processServiceUpdate(<-h.serviceUpdates)
	require.Len(t, h.allInstances(t), 1)

	h.addGroup("foo-svc", "image", "not-foo-image")
	h.si.processServiceUpdate(<-h.serviceUpdates)
	require.Len(t, h.allInstances(t), 0)

	h.si.processContainerUpdate(addContainer(container{
		ID:        "bar",
		IPAddress: "192.168.34.87",
		Image:     "not-foo-image:version",
	}))
	h.si.processContainerUpdate(addContainer(container{
		ID:        "baz",
		IPAddress: "192.168.34.99",
		Image:     "not-foo-image:version2",
	}))
	require.Len(t, h.allInstances(t), 2)

	h.si.processContainerUpdate(removeContainer("baz"))
	require.Len(t, h.allInstances(t), 1)

	h.RemoveService("foo-svc")
	h.si.processServiceUpdate(<-h.serviceUpdates)
	require.Len(t, h.allInstances(t), 0)
	h.stop(t)
}

func TestMappedPort(t *testing.T) {
	h := setup(nil, "11.98.99.98", LOCAL)

	h.AddService("blorp-svc", data.Service{
		InstancePort: 8080,
	})
	h.addGroup("blorp-svc", "image", "blorp-image")

	h.si.processContainerUpdate(resetContainers(container{
		ID:        "blorp-instance",
		IPAddress: "10.13.14.15",
		Image:     "blorp-image:tag",
		Ports: map[string]string{
			"8080/tcp": "3456",
		},
	}))
	h.si.processServiceUpdate(<-h.serviceUpdates)

	require.Len(t, h.allInstances(t), 1)
	svc, err := h.GetService("blorp-svc", store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	require.Equal(t, h.si.hostIP, svc.Instances[0].Address)
	require.Equal(t, 3456, svc.Instances[0].Port)
	require.Equal(t, data.LIVE, svc.Instances[0].State)
	h.stop(t)
}

func TestMultihostNetworking(t *testing.T) {
	instAddress := "10.13.14.15"
	instPort := 8080

	h := setup(nil, "11.98.99.98", GLOBAL)

	h.AddService("blorp-svc", data.Service{
		InstancePort: instPort,
	})
	h.addGroup("blorp-svc", "image", "blorp-image")

	h.si.processContainerUpdate(resetContainers(container{
		ID:        "blorp-instance",
		IPAddress: instAddress,
		Image:     "blorp-image:tag",
		Ports: map[string]string{
			"8080/tcp": "3456",
		},
	}))
	h.si.processServiceUpdate(<-h.serviceUpdates)

	require.Len(t, h.allInstances(t), 1)
	svc, err := h.GetService("blorp-svc", store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	require.Equal(t, instAddress, svc.Instances[0].Address)
	require.Equal(t, instPort, svc.Instances[0].Port)
	require.Equal(t, data.LIVE, svc.Instances[0].State)
	h.stop(t)
}

func TestNoAddress(t *testing.T) {
	h := setup(nil, "192.168.3.4", LOCAL)

	h.AddService("important-svc", data.Service{
		InstancePort: 80,
	})
	h.addGroup("important-svc", "image", "important-image")

	h.si.processContainerUpdate(resetContainers(container{
		ID:        "oops-instance",
		IPAddress: "10.13.14.15",
		Image:     "important-image:greatest",
		// No published port
	}))
	h.si.processServiceUpdate(<-h.serviceUpdates)

	require.Len(t, h.allInstances(t), 1)
	svc, err := h.GetService("important-svc", store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	require.Equal(t, "", svc.Instances[0].Address)
	require.Equal(t, 0, svc.Instances[0].Port)
	require.Equal(t, data.NOADDR, svc.Instances[0].State)
	h.stop(t)
}

func TestHostNetworking(t *testing.T) {
	h := setup(nil, "192.168.5.135", GLOBAL)

	h.AddService("blorp-svc", data.Service{
		InstancePort: 8080,
	})
	h.addGroup("blorp-svc", "image", "blorp-image")

	h.si.processContainerUpdate(resetContainers(container{
		NetworkMode: "host",
		ID:          "blorp-instance",
		IPAddress:   "",
		Image:       "blorp-image:tag",
	}))
	h.si.processServiceUpdate(<-h.serviceUpdates)

	require.Len(t, h.allInstances(t), 1)
	svc, err := h.GetService("blorp-svc", store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	require.Equal(t, h.si.hostIP, svc.Instances[0].Address)
	require.Equal(t, 8080, svc.Instances[0].Port)
	require.Equal(t, data.LIVE, svc.Instances[0].State)
	h.stop(t)
}

func TestOtherHostsEntries(t *testing.T) {
	st := inmem.NewInMemStore()
	h1 := setup(st, "192.168.11.34", LOCAL)
	h2 := setup(st, "192.168.11.5", LOCAL)

	h1.AddService("foo-svc", data.Service{})
	h1.addGroup("foo-svc", "image", "foo-image")
	h1.si.processContainerUpdate(resetContainers(container{
		ID:        "bar1",
		IPAddress: "192.168.34.1",
		Image:     "foo-image:version",
	}, container{
		ID:        "baz1",
		IPAddress: "192.168.34.2",
		Image:     "foo-image:version2",
	}))

	h2.si.processContainerUpdate(resetContainers(container{
		ID:        "bar2",
		IPAddress: "192.168.34.3",
		Image:     "foo-image:version",
	}, container{
		ID:        "baz2",
		IPAddress: "192.168.34.4",
		Image:     "foo-image:version2",
	}))

	// let si on the first host add its instances
	h1.si.processServiceUpdate(<-h1.serviceUpdates)
	require.Len(t, h1.allInstances(t), 2)

	// let si on the second host add its instances
	h2.si.processServiceUpdate(<-h2.serviceUpdates)
	require.Len(t, h1.allInstances(t), 4)

	// simulate an agent restart; in the meantime, a container has
	// stopped.
	h2.stop(t)
	h2 = setup(st, "192.168.11.5", LOCAL)
	h2.si.processContainerUpdate(resetContainers(container{
		ID:        "bar2",
		IPAddress: "192.168.34.3",
		Image:     "foo-image:version",
	}))
	h2.si.processServiceUpdate(<-h2.serviceUpdates)
	require.Len(t, h2.allInstances(t), 3)

	// test behaviour when the docker listener restarts:
	h2.si.processContainerUpdate(resetContainers())
	require.Len(t, h2.allInstances(t), 2)
}
<commit_msg>Oops, missed in invocation of store.WatchServicesStartFunc<commit_after>package agent

import (
	"strings"
	"testing"

	"github.com/weaveworks/flux/common/daemon"
	"github.com/weaveworks/flux/common/data"
	"github.com/weaveworks/flux/common/store"
	"github.com/weaveworks/flux/common/store/inmem"

	docker "github.com/fsouza/go-dockerclient"
	"github.com/stretchr/testify/require"
)

type container struct {
	ID          string
	IPAddress   string
	Image       string
	Labels      map[string]string
	Env         map[string]string
	Ports       map[string]string
	NetworkMode string
}

func resetContainers(cs ...container) ContainerUpdate {
	return ContainerUpdate{
		Containers: makeContainersMap(cs),
		Reset:      true,
	}
}

func addContainer(c container) ContainerUpdate {
	return ContainerUpdate{
		Containers: makeContainersMap([]container{c}),
	}
}

func removeContainer(id string) ContainerUpdate {
	return ContainerUpdate{
		Containers: map[string]*docker.Container{id: nil},
	}
}

func makeContainersMap(cs []container) map[string]*docker.Container {
	containers := make(map[string]*docker.Container)

	for _, c := range cs {
		env := []string{}
		for k, v := range c.Env {
			env = append(env, strings.Join([]string{k, v}, "="))
		}
		ports := map[docker.Port][]docker.PortBinding{}
		for k, v := range c.Ports {
			ports[docker.Port(k)] = []docker.PortBinding{
				docker.PortBinding{
					HostIP:   "0.0.0.0",
					HostPort: v,
				},
			}
		}

		netmode := c.NetworkMode
		if netmode == "" {
			netmode = "default"
		}
		c1 := &docker.Container{
			ID: c.ID,
			HostConfig: &docker.HostConfig{
				NetworkMode: netmode,
			},
			Config: &docker.Config{
				Image:  c.Image,
				Env:    env,
				Labels: c.Labels,
			},
			NetworkSettings: &docker.NetworkSettings{
				IPAddress: c.IPAddress,
				Ports:     ports,
			},
		}
		containers[c.ID] = c1
	}

	return containers
}

const GROUP = "deliberately not default"

type harness struct {
	si *SyncInstances
	store.Store
	es             daemon.ErrorSink
	serviceUpdates chan store.ServiceUpdate
	watchServices  daemon.Component
}

func setup(st store.Store, hostIP, netmode string) (res harness) {
	if st == nil {
		st = inmem.NewInMemStore()
	}

	if netmode == "" {
		netmode = LOCAL
	}

	res.Store = st
	res.si = NewSyncInstances(Config{
		Store:   res.Store,
		Network: netmode,
		HostIP:  hostIP,
	})
	res.es = daemon.NewErrorSink()
	res.serviceUpdates = make(chan store.ServiceUpdate)
	res.watchServices = store.WatchServicesStartFunc(res.Store,
		store.QueryServiceOptions{WithContainerRules: true},
		res.serviceUpdates, nil)(res.es)
	return
}

func (h harness) stop(t *testing.T) {
	h.watchServices.Stop()
	require.Empty(t, h.es)
}

func (h harness) addGroup(serviceName string, labels ...string) {
	if len(labels)%2 != 0 {
		panic("Expected key value ... as arguments")
	}
	sel := make(map[string]string)
	for i := 0; i < len(labels); i += 2 {
		sel[labels[i]] = labels[i+1]
	}

	h.SetContainerRule(serviceName, GROUP,
		data.ContainerRule{Selector: sel})
}

func (h harness) allInstances(t *testing.T) []data.Instance {
	var res []data.Instance
	svcs, err := h.GetAllServices(store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	for _, svc := range svcs {
		for _, inst := range svc.Instances {
			res = append(res, inst.Instance)
		}
	}
	return res
}

func TestSyncInstancesReconcile(t *testing.T) {
	h := setup(nil, "10.98.99.100", GLOBAL)
	h.AddService("foo-svc", data.Service{
		InstancePort: 80,
	})
	h.addGroup("foo-svc", "tag", "bobbins", "image", "foo-image")
	h.AddService("bar-svc", data.Service{
		InstancePort: 80,
	})
	h.addGroup("bar-svc", "flux/foo-label", "blorp")
	h.AddService("boo-svc", data.Service{
		InstancePort: 80,
	})
	h.addGroup("boo-svc", "env.SERVICE_NAME", "boo")

	selectedAddress := "192.168.45.67"

	h.si.processServiceUpdate(<-h.serviceUpdates)
	h.si.processContainerUpdate(resetContainers(container{
		ID:        "selected",
		IPAddress: selectedAddress,
		Image:     "foo-image:bobbins",
		Labels:    map[string]string{"flux/foo-label": "blorp"},
		Env:       map[string]string{"SERVICE_NAME": "boo"},
	}, container{
		ID:        "not",
		IPAddress: "111.111.111.111",
		Image:     "foo-image:not-bobbins",
		Labels:    map[string]string{"flux/foo-label": "something-else"},
		Env:       map[string]string{"SERVICE_NAME": "literally anything"},
	}))

	insts := h.allInstances(t)
	require.Len(t, insts, 3)
	for _, inst := range insts {
		require.Equal(t, selectedAddress, inst.Address)
	}

	h.stop(t)
}

func TestSyncInstancesEvents(t *testing.T) {
	h := setup(nil, "10.98.90.111", "")
	// starting condition
	require.Len(t, h.allInstances(t), 0)

	// no services defined
	h.si.processServiceUpdate(<-h.serviceUpdates)
	h.si.processContainerUpdate(resetContainers(container{
		ID:        "foo",
		Image:     "foo-image:latest",
		IPAddress: "192.168.0.67",
	}))
	require.Len(t, h.allInstances(t), 0)

	h.AddService("foo-svc", data.Service{})
	h.si.processServiceUpdate(<-h.serviceUpdates)
	h.addGroup("foo-svc", "image", "foo-image")
	h.si.processServiceUpdate(<-h.serviceUpdates)
	require.Len(t, h.allInstances(t), 1)

	h.addGroup("foo-svc", "image", "not-foo-image")
	h.si.processServiceUpdate(<-h.serviceUpdates)
	require.Len(t, h.allInstances(t), 0)

	h.si.processContainerUpdate(addContainer(container{
		ID:        "bar",
		IPAddress: "192.168.34.87",
		Image:     "not-foo-image:version",
	}))
	h.si.processContainerUpdate(addContainer(container{
		ID:        "baz",
		IPAddress: "192.168.34.99",
		Image:     "not-foo-image:version2",
	}))
	require.Len(t, h.allInstances(t), 2)

	h.si.processContainerUpdate(removeContainer("baz"))
	require.Len(t, h.allInstances(t), 1)

	h.RemoveService("foo-svc")
	h.si.processServiceUpdate(<-h.serviceUpdates)
	require.Len(t, h.allInstances(t), 0)
	h.stop(t)
}

func TestMappedPort(t *testing.T) {
	h := setup(nil, "11.98.99.98", LOCAL)

	h.AddService("blorp-svc", data.Service{
		InstancePort: 8080,
	})
	h.addGroup("blorp-svc", "image", "blorp-image")

	h.si.processContainerUpdate(resetContainers(container{
		ID:        "blorp-instance",
		IPAddress: "10.13.14.15",
		Image:     "blorp-image:tag",
		Ports: map[string]string{
			"8080/tcp": "3456",
		},
	}))
	h.si.processServiceUpdate(<-h.serviceUpdates)

	require.Len(t, h.allInstances(t), 1)
	svc, err := h.GetService("blorp-svc", store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	require.Equal(t, h.si.hostIP, svc.Instances[0].Address)
	require.Equal(t, 3456, svc.Instances[0].Port)
	require.Equal(t, data.LIVE, svc.Instances[0].State)
	h.stop(t)
}

func TestMultihostNetworking(t *testing.T) {
	instAddress := "10.13.14.15"
	instPort := 8080

	h := setup(nil, "11.98.99.98", GLOBAL)

	h.AddService("blorp-svc", data.Service{
		InstancePort: instPort,
	})
	h.addGroup("blorp-svc", "image", "blorp-image")

	h.si.processContainerUpdate(resetContainers(container{
		ID:        "blorp-instance",
		IPAddress: instAddress,
		Image:     "blorp-image:tag",
		Ports: map[string]string{
			"8080/tcp": "3456",
		},
	}))
	h.si.processServiceUpdate(<-h.serviceUpdates)

	require.Len(t, h.allInstances(t), 1)
	svc, err := h.GetService("blorp-svc", store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	require.Equal(t, instAddress, svc.Instances[0].Address)
	require.Equal(t, instPort, svc.Instances[0].Port)
	require.Equal(t, data.LIVE, svc.Instances[0].State)
	h.stop(t)
}

func TestNoAddress(t *testing.T) {
	h := setup(nil, "192.168.3.4", LOCAL)

	h.AddService("important-svc", data.Service{
		InstancePort: 80,
	})
	h.addGroup("important-svc", "image", "important-image")

	h.si.processContainerUpdate(resetContainers(container{
		ID:        "oops-instance",
		IPAddress: "10.13.14.15",
		Image:     "important-image:greatest",
		// No published port
	}))
	h.si.processServiceUpdate(<-h.serviceUpdates)

	require.Len(t, h.allInstances(t), 1)
	svc, err := h.GetService("important-svc", store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	require.Equal(t, "", svc.Instances[0].Address)
	require.Equal(t, 0, svc.Instances[0].Port)
	require.Equal(t, data.NOADDR, svc.Instances[0].State)
	h.stop(t)
}

func TestHostNetworking(t *testing.T) {
	h := setup(nil, "192.168.5.135", GLOBAL)

	h.AddService("blorp-svc", data.Service{
		InstancePort: 8080,
	})
	h.addGroup("blorp-svc", "image", "blorp-image")

	h.si.processContainerUpdate(resetContainers(container{
		NetworkMode: "host",
		ID:          "blorp-instance",
		IPAddress:   "",
		Image:       "blorp-image:tag",
	}))
	h.si.processServiceUpdate(<-h.serviceUpdates)

	require.Len(t, h.allInstances(t), 1)
	svc, err := h.GetService("blorp-svc", store.QueryServiceOptions{WithInstances: true})
	require.Nil(t, err)
	require.Equal(t, h.si.hostIP, svc.Instances[0].Address)
	require.Equal(t, 8080, svc.Instances[0].Port)
	require.Equal(t, data.LIVE, svc.Instances[0].State)
	h.stop(t)
}

func TestOtherHostsEntries(t *testing.T) {
	st := inmem.NewInMemStore()
	h1 := setup(st, "192.168.11.34", LOCAL)
	h2 := setup(st, "192.168.11.5", LOCAL)

	h1.AddService("foo-svc", data.Service{})
	h1.addGroup("foo-svc", "image", "foo-image")
	h1.si.processContainerUpdate(resetContainers(container{
		ID:        "bar1",
		IPAddress: "192.168.34.1",
		Image:     "foo-image:version",
	}, container{
		ID:        "baz1",
		IPAddress: "192.168.34.2",
		Image:     "foo-image:version2",
	}))

	h2.si.processContainerUpdate(resetContainers(container{
		ID:        "bar2",
		IPAddress: "192.168.34.3",
		Image:     "foo-image:version",
	}, container{
		ID:        "baz2",
		IPAddress: "192.168.34.4",
		Image:     "foo-image:version2",
	}))

	// let si on the first host add its instances
	h1.si.processServiceUpdate(<-h1.serviceUpdates)
	require.Len(t, h1.allInstances(t), 2)

	// let si on the second host add its instances
	h2.si.processServiceUpdate(<-h2.serviceUpdates)
	require.Len(t, h1.allInstances(t), 4)

	// simulate an agent restart; in the meantime, a container has
	// stopped.
	h2.stop(t)
	h2 = setup(st, "192.168.11.5", LOCAL)
	h2.si.processContainerUpdate(resetContainers(container{
		ID:        "bar2",
		IPAddress: "192.168.34.3",
		Image:     "foo-image:version",
	}))
	h2.si.processServiceUpdate(<-h2.serviceUpdates)
	require.Len(t, h2.allInstances(t), 3)

	// test behaviour when the docker listener restarts:
	h2.si.processContainerUpdate(resetContainers())
	require.Len(t, h2.allInstances(t), 2)
}
<|endoftext|>


<commit_before>package sudoku

import (
	"math"
	"math/rand"
	"testing"
)

const _NUM_RUNS_TEST_WEIGHTED_DISTRIBUTION = 10000
const _ALLOWABLE_DIFF_WEIGHTED_DISTRIBUTION = 0.01

func TestRandomWeightedIndex(t *testing.T) {

	result := randomIndexWithNormalizedWeights([]float64{1.0, 0.0})
	if result != 0 {
		t.Log("Got wrong result with random weights")
		t.Fail()
	}
	result = randomIndexWithNormalizedWeights([]float64{0.5, 0.0, 0.5})
	if result != 0 && result != 2 {
		t.Log("Didn't get one of two legal weights")
		t.Fail()
	}
	result = randomIndexWithNormalizedWeights([]float64{0.0, 0.0, 1.0})
	if result != 2 {
		t.Log("Should have gotten last item in random weights; we didn't")
		t.Fail()
	}

	if weightsNormalized([]float64{1.0, 0.000001}) {
		t.Log("thought weights were normalized when they weren't")
		t.Fail()
	}
	if !weightsNormalized([]float64{0.5, 0.25, 0.25}) {
		t.Log("Didn't think weights were normalized but they were")
		t.Fail()
	}

	if weightsNormalized([]float64{0.5, -0.25, 0.25}) {
		t.Error("A negative weight was considered normal.")
	}

	rand.Seed(1)
	result = randomIndexWithInvertedWeights([]float64{0.0, 0.0, 1.0})
	if result == 2 {
		t.Error("Got the wrong index for inverted weights")
	}

	for i := 0; i < 10; i++ {
		rand.Seed(int64(i))
		result = randomIndexWithInvertedWeights([]float64{0.5, 1.0, 0.0})
		if result == 1 {
			t.Error("RandominzedIndexWithInvertedWeights returned wrong result")
			break
		}
	}

	weightResult := normalizedWeights([]float64{2.0, 1.0, 1.0})
	if weightResult[0] != 0.5 || weightResult[1] != 0.25 || weightResult[2] != 0.25 {
		t.Log("Nomralized weights came back wrong")
		t.Fail()
	}

	weightResult = normalizedWeights([]float64{1.0, 1.0, -0.5})
	if weightResult[0] != 0.5 || weightResult[1] != 0.5 || weightResult[2] != 0 {
		t.Error("Normalized weights with a negative came back wrong: ", weightResult)
	}

	weightResult = normalizedWeights([]float64{-0.25, -0.5, 0.25})
	if weightResult[0] != 0.25 || weightResult[1] != 0 || weightResult[2] != 0.75 {
		t.Error("Normalized weights with two different negative numbers came back wrong: ", weightResult)
	}

	result = randomIndexWithWeights([]float64{1.0, 0.0})
	if result != 0 {
		t.Log("Got wrong result with random weights")
		t.Fail()
	}
	result = randomIndexWithWeights([]float64{5.0, 0.0, 5.0})
	if result != 0 && result != 2 {
		t.Log("Didn't get one of two legal weights")
		t.Fail()
	}
	result = randomIndexWithWeights([]float64{0.0, 0.0, 5.0})
	if result != 2 {
		t.Log("Should have gotten last item in random weights; we didn't")
		t.Fail()
	}
	for i := 0; i < 100; i++ {
		rand.Seed(int64(i))
		result = randomIndexWithWeights([]float64{1.0, 10.0, 0.5, -1.0, 0.0, 6.4})
		if result == 3 {
			t.Error("Random index with weights picked wrong index with seed ", i)
		}
	}
	for i := 0; i < 100; i++ {
		rand.Seed(int64(i))
		result = randomIndexWithWeights([]float64{1.0, 10.0, 0.5, 1.0, 0.0, 6.4, 0.0})
		if result == 4 || result == 6 {
			t.Error("Random index with weights that ended in zero picked wrong index with seed ", i)
		}
	}
}

func TestWeightedRandomDistribution(t *testing.T) {

	//We're just going to bother testing randomIndexWithInvertedWeights since that's the one we actually use
	//in HumanSolve.

	type distributionTestCase struct {
		input       []float64
		expected    []float64
		description string
	}

	cases := []distributionTestCase{
		{
			[]float64{
				0.0,
				1.0,
				2.0,
			},
			[]float64{
				0.3505,
				0.33333333333,
				0.3162,
			},
			"0 1 2",
		},
		{
			[]float64{
				0.0,
			},
			[]float64{
				1.0,
			},
			"0.0",
		},
		{
			[]float64{
				10.0,
			},
			[]float64{
				1.0,
			},
			"10.0",
		},
		{
			[]float64{
				0.5,
				0.5,
				1.0,
			},
			[]float64{
				0.337,
				0.337,
				0.327,
			},
			"0.5, 0.5, 1.0",
		},
		{
			[]float64{
				1.0,
				100.0,
				0.5,
				-1.0,
				0.0,
				6.4,
			},
			[]float64{
				0.2019,
				0.0012,
				0.2072,
				0.2232,
				0.2119,
				0.1546,
			},
			"1.0, 100.0, 0.5, -1.0, 0.0, 6.4",
		},
		{
			[]float64{
				3.0,
				3.0,
				4.0,
				4.0,
				4.0,
				4.0,
				100.0,
				100.0,
				400.0,
			},
			[]float64{
				0.1721,
				0.1721,
				0.1635,
				0.1624,
				0.1638,
				0.1637,
				0.0014,
				0.0011,
				0.0,
			},
			"Many at same weight; exponential incrase",
		},
		//This demonstrates the same problem as the case above, but is more pure
		{
			[]float64{
				0.0,
				1.0,
				2.0,
				4.0,
				8.0,
				16.0,
			},
			[]float64{
				0.2086,
				0.1979,
				0.1891,
				0.1714,
				0.1402,
				0.0928,
			},
			"Straight power of two increase 31",
		},
	}

	for _, testCase := range cases {
		randomIndexDistributionHelper(
			t,
			randomIndexWithInvertedWeights,
			testCase.input,
			testCase.expected,
			testCase.description)
	}

}

func randomIndexDistributionHelper(t *testing.T, theFunc func([]float64) int, input []float64, expectedDistribution []float64, testCase string) {

	if len(input) != len(expectedDistribution) {
		t.Fatal("Given differently sized input and expected distribution")
	}

	//collect the results
	results := make([]int, len(expectedDistribution))
	for i := 0; i < _NUM_RUNS_TEST_WEIGHTED_DISTRIBUTION; i++ {
		rand.Seed(int64(i))
		result := theFunc(input)
		results[result]++
	}

	//normalize the results and then calculate the diffs from expected.

	diffAccum := 0.0

	normalizedResults := make([]float64, len(results))

	for i, result := range results {
		normalizedResults[i] = float64(result) / _NUM_RUNS_TEST_WEIGHTED_DISTRIBUTION
		diffAccum += math.Abs(normalizedResults[i] - expectedDistribution[i])
	}

	if diffAccum > _ALLOWABLE_DIFF_WEIGHTED_DISTRIBUTION {
		t.Error("More than allowable difference observed in weighted random distribution:", diffAccum, testCase, "Got", normalizedResults, "Expected", expectedDistribution)
	}

}
<commit_msg>Added an extra test distribution for the inverted weights<commit_after>package sudoku

import (
	"math"
	"math/rand"
	"testing"
)

const _NUM_RUNS_TEST_WEIGHTED_DISTRIBUTION = 10000
const _ALLOWABLE_DIFF_WEIGHTED_DISTRIBUTION = 0.01

func TestRandomWeightedIndex(t *testing.T) {

	result := randomIndexWithNormalizedWeights([]float64{1.0, 0.0})
	if result != 0 {
		t.Log("Got wrong result with random weights")
		t.Fail()
	}
	result = randomIndexWithNormalizedWeights([]float64{0.5, 0.0, 0.5})
	if result != 0 && result != 2 {
		t.Log("Didn't get one of two legal weights")
		t.Fail()
	}
	result = randomIndexWithNormalizedWeights([]float64{0.0, 0.0, 1.0})
	if result != 2 {
		t.Log("Should have gotten last item in random weights; we didn't")
		t.Fail()
	}

	if weightsNormalized([]float64{1.0, 0.000001}) {
		t.Log("thought weights were normalized when they weren't")
		t.Fail()
	}
	if !weightsNormalized([]float64{0.5, 0.25, 0.25}) {
		t.Log("Didn't think weights were normalized but they were")
		t.Fail()
	}

	if weightsNormalized([]float64{0.5, -0.25, 0.25}) {
		t.Error("A negative weight was considered normal.")
	}

	rand.Seed(1)
	result = randomIndexWithInvertedWeights([]float64{0.0, 0.0, 1.0})
	if result == 2 {
		t.Error("Got the wrong index for inverted weights")
	}

	for i := 0; i < 10; i++ {
		rand.Seed(int64(i))
		result = randomIndexWithInvertedWeights([]float64{0.5, 1.0, 0.0})
		if result == 1 {
			t.Error("RandominzedIndexWithInvertedWeights returned wrong result")
			break
		}
	}

	weightResult := normalizedWeights([]float64{2.0, 1.0, 1.0})
	if weightResult[0] != 0.5 || weightResult[1] != 0.25 || weightResult[2] != 0.25 {
		t.Log("Nomralized weights came back wrong")
		t.Fail()
	}

	weightResult = normalizedWeights([]float64{1.0, 1.0, -0.5})
	if weightResult[0] != 0.5 || weightResult[1] != 0.5 || weightResult[2] != 0 {
		t.Error("Normalized weights with a negative came back wrong: ", weightResult)
	}

	weightResult = normalizedWeights([]float64{-0.25, -0.5, 0.25})
	if weightResult[0] != 0.25 || weightResult[1] != 0 || weightResult[2] != 0.75 {
		t.Error("Normalized weights with two different negative numbers came back wrong: ", weightResult)
	}

	result = randomIndexWithWeights([]float64{1.0, 0.0})
	if result != 0 {
		t.Log("Got wrong result with random weights")
		t.Fail()
	}
	result = randomIndexWithWeights([]float64{5.0, 0.0, 5.0})
	if result != 0 && result != 2 {
		t.Log("Didn't get one of two legal weights")
		t.Fail()
	}
	result = randomIndexWithWeights([]float64{0.0, 0.0, 5.0})
	if result != 2 {
		t.Log("Should have gotten last item in random weights; we didn't")
		t.Fail()
	}
	for i := 0; i < 100; i++ {
		rand.Seed(int64(i))
		result = randomIndexWithWeights([]float64{1.0, 10.0, 0.5, -1.0, 0.0, 6.4})
		if result == 3 {
			t.Error("Random index with weights picked wrong index with seed ", i)
		}
	}
	for i := 0; i < 100; i++ {
		rand.Seed(int64(i))
		result = randomIndexWithWeights([]float64{1.0, 10.0, 0.5, 1.0, 0.0, 6.4, 0.0})
		if result == 4 || result == 6 {
			t.Error("Random index with weights that ended in zero picked wrong index with seed ", i)
		}
	}
}

func TestWeightedRandomDistribution(t *testing.T) {

	//We're just going to bother testing randomIndexWithInvertedWeights since that's the one we actually use
	//in HumanSolve.

	type distributionTestCase struct {
		input       []float64
		expected    []float64
		description string
	}

	cases := []distributionTestCase{
		{
			[]float64{
				0.0,
				1.0,
				2.0,
			},
			[]float64{
				0.3505,
				0.33333333333,
				0.3162,
			},
			"0 1 2",
		},
		{
			[]float64{
				0.0,
			},
			[]float64{
				1.0,
			},
			"0.0",
		},
		{
			[]float64{
				10.0,
			},
			[]float64{
				1.0,
			},
			"10.0",
		},
		{
			[]float64{
				0.5,
				0.5,
				1.0,
			},
			[]float64{
				0.337,
				0.337,
				0.327,
			},
			"0.5, 0.5, 1.0",
		},
		{
			[]float64{
				1.0,
				100.0,
				0.5,
				-1.0,
				0.0,
				6.4,
			},
			[]float64{
				0.2019,
				0.0012,
				0.2072,
				0.2232,
				0.2119,
				0.1546,
			},
			"1.0, 100.0, 0.5, -1.0, 0.0, 6.4",
		},
		{
			[]float64{
				3.0,
				3.0,
				4.0,
				4.0,
				4.0,
				4.0,
				100.0,
				100.0,
				400.0,
			},
			[]float64{
				0.1721,
				0.1721,
				0.1635,
				0.1624,
				0.1638,
				0.1637,
				0.0014,
				0.0011,
				0.0,
			},
			"Many at same weight; exponential incrase",
		},
		//This demonstrates the same problem as the case above, but is more pure
		{
			[]float64{
				0.0,
				1.0,
				2.0,
				4.0,
				8.0,
				16.0,
			},
			[]float64{
				0.2086,
				0.1979,
				0.1891,
				0.1714,
				0.1402,
				0.0928,
			},
			"Straight power of two increase 31",
		},
		{
			[]float64{
				1.0,
				2.0,
				3.0,
				4.0,
				10.0,
				1000.0,
			},
			[]float64{
				0.2291,
				0.219,
				0.2075,
				0.1979,
				0.1465,
				0.0,
			},
			"Small numbers and very big one",
		},
	}

	for _, testCase := range cases {
		randomIndexDistributionHelper(
			t,
			randomIndexWithInvertedWeights,
			testCase.input,
			testCase.expected,
			testCase.description)
	}

}

func randomIndexDistributionHelper(t *testing.T, theFunc func([]float64) int, input []float64, expectedDistribution []float64, testCase string) {

	if len(input) != len(expectedDistribution) {
		t.Fatal("Given differently sized input and expected distribution")
	}

	//collect the results
	results := make([]int, len(expectedDistribution))
	for i := 0; i < _NUM_RUNS_TEST_WEIGHTED_DISTRIBUTION; i++ {
		rand.Seed(int64(i))
		result := theFunc(input)
		results[result]++
	}

	//normalize the results and then calculate the diffs from expected.

	diffAccum := 0.0

	normalizedResults := make([]float64, len(results))

	for i, result := range results {
		normalizedResults[i] = float64(result) / _NUM_RUNS_TEST_WEIGHTED_DISTRIBUTION
		diffAccum += math.Abs(normalizedResults[i] - expectedDistribution[i])
	}

	if diffAccum > _ALLOWABLE_DIFF_WEIGHTED_DISTRIBUTION {
		t.Error("More than allowable difference observed in weighted random distribution:", diffAccum, testCase, "Got", normalizedResults, "Expected", expectedDistribution)
	}

}
<|endoftext|>


<commit_before>package ingester

import (
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/storage/metric"
	"golang.org/x/net/context"

	"github.com/weaveworks/common/user"
	"github.com/weaveworks/cortex/util"
)

type userStates struct {
	mtx              sync.RWMutex
	states           map[string]*userState
	rateUpdatePeriod time.Duration
}

type userState struct {
	userID          string
	fpLocker        *fingerprintLocker
	fpToSeries      *seriesMap
	mapper          *fpMapper
	index           *invertedIndex
	ingestedSamples *ewmaRate
}

func newUserStates(rateUpdatePeriod time.Duration) userStates {
	return userStates{
		states:           map[string]*userState{},
		rateUpdatePeriod: rateUpdatePeriod,
	}
}

func (us *userStates) cp() map[string]*userState {
	us.mtx.Lock()
	states := make(map[string]*userState, len(us.states))
	for id, state := range us.states {
		states[id] = state
	}
	us.mtx.Unlock()
	return states
}

func (us *userStates) gc() {
	us.mtx.Lock()
	for id, state := range us.states {
		if state.fpToSeries.length() == 0 {
			delete(us.states, id)
		}
	}
	us.mtx.Unlock()
}

func (us *userStates) updateRates() {
	us.mtx.RLock()
	defer us.mtx.RUnlock()

	for _, state := range us.states {
		state.ingestedSamples.tick()
	}
}

func (us *userStates) numUsers() int {
	us.mtx.RLock()
	defer us.mtx.RUnlock()
	return len(us.states)
}

func (us *userStates) numSeries() int {
	us.mtx.RLock()
	defer us.mtx.RUnlock()
	numSeries := 0
	for _, state := range us.states {
		numSeries += state.fpToSeries.length()
	}
	return numSeries
}

func (us *userStates) get(userID string) (*userState, bool) {
	us.mtx.RLock()
	state, ok := us.states[userID]
	us.mtx.RUnlock()
	return state, ok
}

func (us *userStates) getOrCreate(ctx context.Context) (*userState, error) {
	userID, err := user.GetID(ctx)
	if err != nil {
		return nil, fmt.Errorf("no user id")
	}

	us.mtx.RLock()
	state, ok := us.states[userID]
	us.mtx.RUnlock()
	if ok {
		return state, nil
	}

	us.mtx.Lock()
	defer us.mtx.Unlock()
	return us.unlockedGetOrCreate(userID), nil
}

func (us *userStates) getOrCreateSeries(ctx context.Context, metric model.Metric) (*userState, model.Fingerprint, *memorySeries, error) {
	userID, err := user.GetID(ctx)
	if err != nil {
		return nil, 0, nil, fmt.Errorf("no user id")
	}

	var (
		state  *userState
		ok     bool
		fp     model.Fingerprint
		series *memorySeries
	)

	us.mtx.RLock()
	state, ok = us.states[userID]
	if ok {
		fp, series = state.unlockedGet(metric)
	}
	us.mtx.RUnlock()
	if ok {
		return state, fp, series, nil
	}

	us.mtx.Lock()
	defer us.mtx.Unlock()
	state = us.unlockedGetOrCreate(userID)
	fp, series = state.unlockedGet(metric)
	return state, fp, series, nil
}

func (us *userStates) unlockedGetOrCreate(userID string) *userState {
	state, ok := us.states[userID]
	if !ok {
		state = &userState{
			userID:          userID,
			fpToSeries:      newSeriesMap(),
			fpLocker:        newFingerprintLocker(16),
			index:           newInvertedIndex(),
			ingestedSamples: newEWMARate(0.2, us.rateUpdatePeriod),
		}
		state.mapper = newFPMapper(state.fpToSeries)
		us.states[userID] = state
	}
	return state
}

func (u *userState) unlockedGet(metric model.Metric) (model.Fingerprint, *memorySeries) {
	rawFP := metric.FastFingerprint()
	u.fpLocker.Lock(rawFP)
	fp := u.mapper.mapFP(rawFP, metric)
	if fp != rawFP {
		u.fpLocker.Unlock(rawFP)
		u.fpLocker.Lock(fp)
	}

	series, ok := u.fpToSeries.get(fp)
	if ok {
		return fp, series
	}

	series = newMemorySeries(metric)
	u.fpToSeries.put(fp, series)
	u.index.add(metric, fp)
	return fp, series
}

// forSeriesMatching passes all series matching the given matchers to the provided callback.
// Deals with locking and the quirks of zero-length matcher values.
func (u *userState) forSeriesMatching(allMatchers []*metric.LabelMatcher, callback func(model.Fingerprint, *memorySeries) error) error {
	filters, matchers := util.SplitFiltersAndMatchers(allMatchers)
	fps := u.index.lookup(matchers)

	// fps is sorted, lock them in order to prevent deadlocks
outer:
	for _, fp := range fps {
		u.fpLocker.Lock(fp)
		series, ok := u.fpToSeries.get(fp)
		if !ok {
			u.fpLocker.Unlock(fp)
			continue
		}

		for _, filter := range filters {
			if !filter.Match(series.metric[filter.Name]) {
				continue outer
			}
		}

		err := callback(fp, series)
		u.fpLocker.Unlock(fp)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Ensure we unlock the fingerprint if its filtered out of a query.  Fixes #241.<commit_after>package ingester

import (
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/storage/metric"
	"golang.org/x/net/context"

	"github.com/weaveworks/common/user"
	"github.com/weaveworks/cortex/util"
)

type userStates struct {
	mtx              sync.RWMutex
	states           map[string]*userState
	rateUpdatePeriod time.Duration
}

type userState struct {
	userID          string
	fpLocker        *fingerprintLocker
	fpToSeries      *seriesMap
	mapper          *fpMapper
	index           *invertedIndex
	ingestedSamples *ewmaRate
}

func newUserStates(rateUpdatePeriod time.Duration) userStates {
	return userStates{
		states:           map[string]*userState{},
		rateUpdatePeriod: rateUpdatePeriod,
	}
}

func (us *userStates) cp() map[string]*userState {
	us.mtx.Lock()
	states := make(map[string]*userState, len(us.states))
	for id, state := range us.states {
		states[id] = state
	}
	us.mtx.Unlock()
	return states
}

func (us *userStates) gc() {
	us.mtx.Lock()
	for id, state := range us.states {
		if state.fpToSeries.length() == 0 {
			delete(us.states, id)
		}
	}
	us.mtx.Unlock()
}

func (us *userStates) updateRates() {
	us.mtx.RLock()
	defer us.mtx.RUnlock()

	for _, state := range us.states {
		state.ingestedSamples.tick()
	}
}

func (us *userStates) numUsers() int {
	us.mtx.RLock()
	defer us.mtx.RUnlock()
	return len(us.states)
}

func (us *userStates) numSeries() int {
	us.mtx.RLock()
	defer us.mtx.RUnlock()
	numSeries := 0
	for _, state := range us.states {
		numSeries += state.fpToSeries.length()
	}
	return numSeries
}

func (us *userStates) get(userID string) (*userState, bool) {
	us.mtx.RLock()
	state, ok := us.states[userID]
	us.mtx.RUnlock()
	return state, ok
}

func (us *userStates) getOrCreate(ctx context.Context) (*userState, error) {
	userID, err := user.GetID(ctx)
	if err != nil {
		return nil, fmt.Errorf("no user id")
	}

	us.mtx.RLock()
	state, ok := us.states[userID]
	us.mtx.RUnlock()
	if ok {
		return state, nil
	}

	us.mtx.Lock()
	defer us.mtx.Unlock()
	return us.unlockedGetOrCreate(userID), nil
}

func (us *userStates) getOrCreateSeries(ctx context.Context, metric model.Metric) (*userState, model.Fingerprint, *memorySeries, error) {
	userID, err := user.GetID(ctx)
	if err != nil {
		return nil, 0, nil, fmt.Errorf("no user id")
	}

	var (
		state  *userState
		ok     bool
		fp     model.Fingerprint
		series *memorySeries
	)

	us.mtx.RLock()
	state, ok = us.states[userID]
	if ok {
		fp, series = state.unlockedGet(metric)
	}
	us.mtx.RUnlock()
	if ok {
		return state, fp, series, nil
	}

	us.mtx.Lock()
	defer us.mtx.Unlock()
	state = us.unlockedGetOrCreate(userID)
	fp, series = state.unlockedGet(metric)
	return state, fp, series, nil
}

func (us *userStates) unlockedGetOrCreate(userID string) *userState {
	state, ok := us.states[userID]
	if !ok {
		state = &userState{
			userID:          userID,
			fpToSeries:      newSeriesMap(),
			fpLocker:        newFingerprintLocker(16),
			index:           newInvertedIndex(),
			ingestedSamples: newEWMARate(0.2, us.rateUpdatePeriod),
		}
		state.mapper = newFPMapper(state.fpToSeries)
		us.states[userID] = state
	}
	return state
}

func (u *userState) unlockedGet(metric model.Metric) (model.Fingerprint, *memorySeries) {
	rawFP := metric.FastFingerprint()
	u.fpLocker.Lock(rawFP)
	fp := u.mapper.mapFP(rawFP, metric)
	if fp != rawFP {
		u.fpLocker.Unlock(rawFP)
		u.fpLocker.Lock(fp)
	}

	series, ok := u.fpToSeries.get(fp)
	if ok {
		return fp, series
	}

	series = newMemorySeries(metric)
	u.fpToSeries.put(fp, series)
	u.index.add(metric, fp)
	return fp, series
}

// forSeriesMatching passes all series matching the given matchers to the provided callback.
// Deals with locking and the quirks of zero-length matcher values.
func (u *userState) forSeriesMatching(allMatchers []*metric.LabelMatcher, callback func(model.Fingerprint, *memorySeries) error) error {
	filters, matchers := util.SplitFiltersAndMatchers(allMatchers)
	fps := u.index.lookup(matchers)

	// fps is sorted, lock them in order to prevent deadlocks
outer:
	for _, fp := range fps {
		u.fpLocker.Lock(fp)
		series, ok := u.fpToSeries.get(fp)
		if !ok {
			u.fpLocker.Unlock(fp)
			continue
		}

		for _, filter := range filters {
			if !filter.Match(series.metric[filter.Name]) {
				u.fpLocker.Unlock(fp)
				continue outer
			}
		}

		err := callback(fp, series)
		u.fpLocker.Unlock(fp)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuseutil

import (
	"github.com/jacobsa/fuse"
	"golang.org/x/net/context"
)

// Embed this within your file system type to inherit default implementations
// of all methods that return fuse.ENOSYS.
type NotImplementedFileSystem struct {
}

var _ fuse.FileSystem = &NotImplementedFileSystem{}

func (fs *NotImplementedFileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (*fuse.InitResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (*fuse.LookUpInodeResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	*fuse.GetInodeAttributesResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) SetInodeAttributes(
	ctx context.Context,
	req *fuse.SetInodeAttributesRequest) (
	*fuse.SetInodeAttributesResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ForgetInode(
	ctx context.Context,
	req *fuse.ForgetInodeRequest) (*fuse.ForgetInodeResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) MkDir(
	ctx context.Context,
	req *fuse.MkDirRequest) (*fuse.MkDirResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) CreateFile(
	ctx context.Context,
	req *fuse.CreateFileRequest) (*fuse.CreateFileResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) RmDir(
	ctx context.Context,
	req *fuse.RmDirRequest) (*fuse.RmDirResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (*fuse.OpenDirResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (*fuse.ReadDirResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (*fuse.ReleaseDirHandleResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) OpenFile(
	ctx context.Context,
	req *fuse.OpenFileRequest) (*fuse.OpenFileResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ReadFile(
	ctx context.Context,
	req *fuse.ReadFileRequest) (*fuse.ReadFileResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) WriteFile(
	ctx context.Context,
	req *fuse.WriteFileRequest) (*fuse.WriteFileResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ReleaseFileHandle(
	ctx context.Context,
	req *fuse.ReleaseFileHandleRequest) (*fuse.ReleaseFileHandleResponse, error) {
	return nil, fuse.ENOSYS
}
<commit_msg>Added NotImplementedFileSystem.Unlink.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuseutil

import (
	"github.com/jacobsa/fuse"
	"golang.org/x/net/context"
)

// Embed this within your file system type to inherit default implementations
// of all methods that return fuse.ENOSYS.
type NotImplementedFileSystem struct {
}

var _ fuse.FileSystem = &NotImplementedFileSystem{}

func (fs *NotImplementedFileSystem) Init(
	ctx context.Context,
	req *fuse.InitRequest) (*fuse.InitResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (*fuse.LookUpInodeResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	*fuse.GetInodeAttributesResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) SetInodeAttributes(
	ctx context.Context,
	req *fuse.SetInodeAttributesRequest) (
	*fuse.SetInodeAttributesResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ForgetInode(
	ctx context.Context,
	req *fuse.ForgetInodeRequest) (*fuse.ForgetInodeResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) MkDir(
	ctx context.Context,
	req *fuse.MkDirRequest) (*fuse.MkDirResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) CreateFile(
	ctx context.Context,
	req *fuse.CreateFileRequest) (*fuse.CreateFileResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) RmDir(
	ctx context.Context,
	req *fuse.RmDirRequest) (*fuse.RmDirResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) Unlink(
	ctx context.Context,
	req *fuse.UnlinkRequest) (*fuse.UnlinkResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (*fuse.OpenDirResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (*fuse.ReadDirResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ReleaseDirHandle(
	ctx context.Context,
	req *fuse.ReleaseDirHandleRequest) (*fuse.ReleaseDirHandleResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) OpenFile(
	ctx context.Context,
	req *fuse.OpenFileRequest) (*fuse.OpenFileResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ReadFile(
	ctx context.Context,
	req *fuse.ReadFileRequest) (*fuse.ReadFileResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) WriteFile(
	ctx context.Context,
	req *fuse.WriteFileRequest) (*fuse.WriteFileResponse, error) {
	return nil, fuse.ENOSYS
}

func (fs *NotImplementedFileSystem) ReleaseFileHandle(
	ctx context.Context,
	req *fuse.ReleaseFileHandleRequest) (*fuse.ReleaseFileHandleResponse, error) {
	return nil, fuse.ENOSYS
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build integ
// +build integ

// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilot

import (
	"context"
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/gateway-api/conformance/tests"
	"sigs.k8s.io/gateway-api/conformance/utils/suite"

	"istio.io/istio/pkg/kube"
	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/resource"
	"istio.io/istio/pkg/test/scopes"
	"istio.io/istio/pkg/test/util/retry"
)

// GatewayConformanceInputs defines inputs to the gateway conformance test.
// The upstream build requires using `testing.T` types, which we cannot pass using our framework.
// To workaround this, we set up the inputs it TestMain.
type GatewayConformanceInputs struct {
	Client  kube.ExtendedClient
	Cleanup bool
}

var gatewayConformanceInputs GatewayConformanceInputs

func TestGatewayConformance(t *testing.T) {
	framework.
		NewTest(t).
		RequiresSingleCluster().
		Features("traffic.gateway").
		Run(func(ctx framework.TestContext) {
			if !supportsCRDv1(ctx) {
				t.Skip("Not supported; requires CRDv1 support.")
			}
			if err := ctx.ConfigIstio().File("", "testdata/gateway-api-crd.yaml").Apply(resource.NoCleanup); err != nil {
				ctx.Fatal(err)
			}
			// Wait until our GatewayClass is ready
			retry.UntilSuccessOrFail(ctx, func() error {
				_, err := ctx.Clusters().Default().GatewayAPI().GatewayV1alpha2().GatewayClasses().Get(context.Background(), "istio", metav1.GetOptions{})
				return err
			})

			mapper, _ := gatewayConformanceInputs.Client.UtilFactory().ToRESTMapper()
			c, err := client.New(gatewayConformanceInputs.Client.RESTConfig(), client.Options{
				Scheme: kube.IstioScheme,
				Mapper: mapper,
			})
			if err != nil {
				t.Fatal(err)
			}

			csuite := suite.New(suite.Options{
				Client:           c,
				GatewayClassName: "istio",
				Debug:            scopes.Framework.DebugEnabled(),
				Cleanup:          gatewayConformanceInputs.Cleanup,
				RoundTripper:     nil,
			})
			csuite.Setup(t)

			for _, ct := range tests.ConformanceTests {
				t.Run(ct.ShortName, func(t *testing.T) {
					ct.Run(t, csuite)
				})
			}
		})
}
<commit_msg>Skip GW conformance for now (#38208)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:build integ
// +build integ

// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilot

import (
	"context"
	"testing"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/gateway-api/conformance/tests"
	"sigs.k8s.io/gateway-api/conformance/utils/suite"

	"istio.io/istio/pkg/kube"
	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/framework/resource"
	"istio.io/istio/pkg/test/scopes"
	"istio.io/istio/pkg/test/util/retry"
)

// GatewayConformanceInputs defines inputs to the gateway conformance test.
// The upstream build requires using `testing.T` types, which we cannot pass using our framework.
// To workaround this, we set up the inputs it TestMain.
type GatewayConformanceInputs struct {
	Client  kube.ExtendedClient
	Cleanup bool
}

var gatewayConformanceInputs GatewayConformanceInputs

func TestGatewayConformance(t *testing.T) {
	framework.
		NewTest(t).
		RequiresSingleCluster().
		Features("traffic.gateway").
		Run(func(ctx framework.TestContext) {
			// Flaky until this is merged
			t.Skip("https://github.com/kubernetes-sigs/gateway-api/pull/1080")
			if !supportsCRDv1(ctx) {
				t.Skip("Not supported; requires CRDv1 support.")
			}
			if err := ctx.ConfigIstio().File("", "testdata/gateway-api-crd.yaml").Apply(resource.NoCleanup); err != nil {
				ctx.Fatal(err)
			}
			// Wait until our GatewayClass is ready
			retry.UntilSuccessOrFail(ctx, func() error {
				_, err := ctx.Clusters().Default().GatewayAPI().GatewayV1alpha2().GatewayClasses().Get(context.Background(), "istio", metav1.GetOptions{})
				return err
			})

			mapper, _ := gatewayConformanceInputs.Client.UtilFactory().ToRESTMapper()
			c, err := client.New(gatewayConformanceInputs.Client.RESTConfig(), client.Options{
				Scheme: kube.IstioScheme,
				Mapper: mapper,
			})
			if err != nil {
				t.Fatal(err)
			}

			csuite := suite.New(suite.Options{
				Client:           c,
				GatewayClassName: "istio",
				Debug:            scopes.Framework.DebugEnabled(),
				Cleanup:          gatewayConformanceInputs.Cleanup,
				RoundTripper:     nil,
			})
			csuite.Setup(t)

			for _, ct := range tests.ConformanceTests {
				t.Run(ct.ShortName, func(t *testing.T) {
					ct.Run(t, csuite)
				})
			}
		})
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"log"
)

type ingredientData struct {
	data *sql.DB
}

func (i *ingredientData) getAllIngredients() []Ingredient {
	var (
		id     int
		name   string
		color  sql.NullString
		baseID int
		ing    Ingredient
	)
	ingredients := make([]Ingredient, 0)
	rows, err := i.data.Query("select i.id, name, color, baseid from ingredients i join baseingredients b on i.id = b.id")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&id, &name, &color, &baseID)
		if err != nil {
			log.Fatal(err)
		}
		if !color.Valid {
			ing = Ingredient{
				ID:     id,
				Name:   name,
				Color:  "",
				BaseID: baseID,
			}
		} else {
			ing = Ingredient{
				ID:     id,
				Name:   name,
				Color:  color.String,
				BaseID: baseID,
			}
		}
		ingredients = append(ingredients, ing)
	}
	return ingredients
}

func (i *ingredientData) getAllIngredientsWithIDs(ids []string) []Ingredient {
	var (
		id     int
		name   string
		color  string
		baseID int
	)
	statement := "select i.id, name, color, baseid from ingredients i join baseIngredients b on i.id = b.id  where i.id = any($1::integer[])"
	args := "{"
	for _, v := range ids {
		args += v + ","
	}
	args = args[:len(args)-1] + "}"
	ingredients := make([]Ingredient, 0)
	rows, err := i.data.Query(statement, args)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&id, &name, &color, &baseID)
		if err != nil {
			log.Fatal(err)
		}
		ingredients = append(ingredients, Ingredient{
			ID:     id,
			Name:   name,
			Color:  color,
			BaseID: baseID,
		})
	}
	return ingredients
}
<commit_msg>fixed sql errors<commit_after>package main

import (
	"database/sql"
	"log"
)

type ingredientData struct {
	data *sql.DB
}

func (i *ingredientData) getAllIngredients() []Ingredient {
	var (
		id       int
		name     string
		color    sql.NullString
		baseID   int
		category Category
		ing      Ingredient
	)
	ingredients := make([]Ingredient, 0)
	rows, err := i.data.Query("select i.id, i.name, i.color, i.baseid, b.category from ingredients i join baseingredients b on i.id = b.id")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&id, &name, &color, &baseID, &category)
		if err != nil {
			log.Fatal(err)
		}
		if !color.Valid {
			ing = Ingredient{
				ID:     id,
				Name:   name,
				Color:  "",
				BaseID: baseID,
				Cat:    category,
			}
		} else {
			ing = Ingredient{
				ID:     id,
				Name:   name,
				Color:  color.String,
				BaseID: baseID,
				Cat:    category,
			}
		}
		ingredients = append(ingredients, ing)
	}
	return ingredients
}

func (i *ingredientData) getAllIngredientsWithIDs(ids []string) []Ingredient {
	var (
		id       int
		name     string
		color    string
		baseID   int
		category Category
	)
	statement := "select i.id, i.name, i.color, i.baseid, b.category from ingredients i join baseIngredients b on i.id = b.id  where i.id = any($1::integer[])"
	args := "{"
	for _, v := range ids {
		args += v + ","
	}
	args = args[:len(args)-1] + "}"
	ingredients := make([]Ingredient, 0)
	rows, err := i.data.Query(statement, args)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	for rows.Next() {
		err := rows.Scan(&id, &name, &color, &baseID, &category)
		if err != nil {
			log.Fatal(err)
		}
		ingredients = append(ingredients, Ingredient{
			ID:     id,
			Name:   name,
			Color:  color,
			BaseID: baseID,
			Cat:    category,
		})
	}
	return ingredients
}
<|endoftext|>


<commit_before>// Copyright 2014 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package orchestrator

import (
	"fmt"
	"time"

	"allmark.io/modules/common/route"
	"allmark.io/modules/model"
	converter "allmark.io/modules/services/converter/markdowntohtml"
	"allmark.io/modules/web/view/viewmodel"
)

type ViewModelOrchestrator struct {
	*Orchestrator

	navigationOrchestrator *NavigationOrchestrator
	tagOrchestrator        *TagsOrchestrator
	fileOrchestrator       *FileOrchestrator

	latestByRoute     map[string][]*viewmodel.Model
	viewmodelsByRoute map[string]*viewmodel.Model
}

func (orchestrator *ViewModelOrchestrator) blockingCacheWarmup() {
	orchestrator.getViewModel(orchestrator.rootItem(), false)
	orchestrator.GetLatest(route.New(), 5, 1)
}

func (orchestrator *ViewModelOrchestrator) GetFullViewModel(itemRoute route.Route) (viewModel viewmodel.Model, found bool) {

	startTime := time.Now()

	// get the requested item
	item := orchestrator.getItem(itemRoute)
	if item == nil {
		return viewModel, false
	}

	// get the base view model
	viewModel = *orchestrator.getViewModel(item, false)

	// navigation
	viewModel.ToplevelNavigation = orchestrator.navigationOrchestrator.GetToplevelNavigation()
	viewModel.BreadcrumbNavigation = orchestrator.navigationOrchestrator.GetBreadcrumbNavigation(itemRoute)
	viewModel.ItemNavigation = orchestrator.navigationOrchestrator.GetItemNavigation(itemRoute)

	// childs
	viewModel.Childs = orchestrator.getChildModels(itemRoute)

	// tags
	viewModel.Tags = orchestrator.tagOrchestrator.getItemTags(itemRoute)

	// Geo Coordinates
	viewModel.GeoLocation = getGeoLocation(item)

	// Analytics Settings
	viewModel.Analytics = orchestrator.getAnalyticsSettings()

	// Hash / ETag
	viewModel.Hash = item.Hash

	// special viewmodel attributes
	isRepositoryItem := item.Type == model.TypeRepository
	if isRepositoryItem {

		// tag cloud
		repositoryIsNotEmpty := orchestrator.index().Size() >= 5 // don't bother to create a tag cloud if there aren't enough documents
		if repositoryIsNotEmpty {

			tagCloud := orchestrator.tagOrchestrator.GetTagCloud()
			viewModel.TagCloud = tagCloud

		}

	}

	endTime := time.Now()
	duration := endTime.Sub(startTime)
	orchestrator.logger.Statistics("Getting the full view model %s took %f seconds.", viewModel.Route, duration.Seconds())

	return viewModel, true
}

func (orchestrator *ViewModelOrchestrator) GetViewModel(itemRoute route.Route) (viewModel viewmodel.Model, found bool) {

	// get the requested item
	item := orchestrator.getItem(itemRoute)
	if item == nil {
		return viewModel, false
	}

	return *orchestrator.getViewModel(item, false), true
}

func (orchestrator *ViewModelOrchestrator) GetLatest(itemRoute route.Route, pageSize, page int) (latest []*viewmodel.Model, found bool) {

	cacheType := "latest"

	// load from cache
	if orchestrator.latestByRoute != nil {

		// re-prime the cache if it is stale
		if orchestrator.isCacheStale(cacheType) {
			go orchestrator.primeCache(cacheType)
		}

		// return the result
		if models, exists := orchestrator.latestByRoute[itemRoute.Value()]; exists {
			return pagedViewmodels(models, pageSize, page)
		}

		return []*viewmodel.Model{}, false

	}

	orchestrator.setCache(cacheType, func() {

		startTime := time.Now()

		latestModelsByRoute := make(map[string][]*viewmodel.Model)

		for _, childRoute := range orchestrator.repository.Routes() {

			// get the latest items
			latestItems := orchestrator.getLatestItems(childRoute)

			// store the results
			latestModelsByRoute[childRoute.Value()] = orchestrator.getLastesViewModelsFromItemList(latestItems)

		}

		endTime := time.Now()
		duration := endTime.Sub(startTime)
		orchestrator.logger.Statistics("Priming the latest cache took %f seconds.", duration.Seconds())

		// save the result
		orchestrator.latestByRoute = latestModelsByRoute

	})

	// return a result
	if models, exists := orchestrator.latestByRoute[itemRoute.Value()]; exists {
		return pagedViewmodels(models, pageSize, page)
	}

	return []*viewmodel.Model{}, false
}

// Converts a list of model.Item elements into a view models for the latest-items controller
func (orchestrator *ViewModelOrchestrator) getLastesViewModelsFromItemList(items []*model.Item) []*viewmodel.Model {

	// create viewmodels
	models := make([]*viewmodel.Model, 0, len(items))
	for _, item := range items {

		viewModel := orchestrator.getViewModel(item, false)
		if viewModel == nil {
			orchestrator.logger.Error("No view model found for item %q.", item)
			continue
		}

		// prepare lazy loading
		contentWithLazyLoadingEnabled := converter.LazyLoad(viewModel.Content)

		// create a copy (make sure we don't modify the content of the original view model)
		viewmodelCopy := *viewModel
		viewmodelCopy.Content = contentWithLazyLoadingEnabled

		models = append(models, &viewmodelCopy)
	}

	return models
}

func (orchestrator *ViewModelOrchestrator) getViewModel(item *model.Item, skipCache bool) *viewmodel.Model {

	// get the root item
	root := orchestrator.rootItem()
	if root == nil {
		panic(fmt.Sprintf("Cannot get viewmodel for route %q because no root item was found.", item))
	}

	itemRoute := item.Route()
	cacheType := "viewmodels"

	if skipCache {
		// prime the cache synchronously
		orchestrator.primeCache(cacheType)
	}

	// load from cache
	if orchestrator.viewmodelsByRoute != nil {

		// re-prime the cache if it is stale
		if orchestrator.isCacheStale(cacheType) {
			go orchestrator.primeCache(cacheType)
		}

		// return the result
		return orchestrator.viewmodelsByRoute[itemRoute.Value()]
	}

	orchestrator.setCache(cacheType, func() {

		startTime := time.Now()

		viewmodelsByRoute := make(map[string]*viewmodel.Model)

		for _, child := range orchestrator.index().GetAllItems() {

			childRoute := child.Route()

			// convert content
			convertedContent, err := orchestrator.converter.Convert(orchestrator.getItemByAlias, orchestrator.relativePather(childRoute), child)
			if err != nil {
				orchestrator.logger.Warn("Cannot convert content for item %q. Error: %s.", child.String(), err.Error())
				convertedContent = "<!-- Conversion Error -->"
			}

			// create a view model
			viewModel := &viewmodel.Model{
				Base:             getBaseModel(root, child, orchestrator.itemPather()),
				Content:          convertedContent,
				Publisher:        orchestrator.getPublisherInformation(),
				Author:           orchestrator.getAuthorInformation(child.MetaData.Author),
				Files:            orchestrator.fileOrchestrator.GetFiles(childRoute),
				Images:           orchestrator.fileOrchestrator.GetImages(childRoute),
				IsRepositoryItem: true,
			}

			// add rft url if rtf conversion is enabled
			if orchestrator.config.Conversion.Rtf.IsEnabled() {
				viewModel.RtfUrl = GetTypedItemUrl(child.Route(), "rtf")
			}

			// store the view model
			viewmodelsByRoute[childRoute.Value()] = viewModel

		}

		endTime := time.Now()
		duration := endTime.Sub(startTime)
		orchestrator.logger.Statistics("Priming the viewModel cache took %f seconds.", duration.Seconds())

		orchestrator.viewmodelsByRoute = viewmodelsByRoute

	})

	return orchestrator.viewmodelsByRoute[itemRoute.Value()]
}

func (orchestrator *ViewModelOrchestrator) getChildModels(itemRoute route.Route) []*viewmodel.Base {

	startTime := time.Now()

	rootItem := orchestrator.rootItem()
	if rootItem == nil {
		orchestrator.logger.Fatal("No root item found")
	}

	pathProvider := orchestrator.relativePather(itemRoute)

	childModels := make([]*viewmodel.Base, 0)

	childItems := orchestrator.getChilds(itemRoute)
	for _, childItem := range childItems {
		baseModel := getBaseModel(rootItem, childItem, pathProvider)
		childModels = append(childModels, &baseModel)
	}

	// sort the models
	viewmodel.SortBaseModelBy(sortBaseModelsByDate).Sort(childModels)

	endTime := time.Now()
	duration := endTime.Sub(startTime)
	orchestrator.logger.Statistics("Getting child models for route %s took %f seconds.", itemRoute, duration.Seconds())

	return childModels
}
<commit_msg>Bug fix for "ViewModel Orchestrator GetViewModel Null Pointer Exception". The viewmodel was not checked before trying to dereference it.<commit_after>// Copyright 2014 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package orchestrator

import (
	"fmt"
	"time"

	"allmark.io/modules/common/route"
	"allmark.io/modules/model"
	converter "allmark.io/modules/services/converter/markdowntohtml"
	"allmark.io/modules/web/view/viewmodel"
)

type ViewModelOrchestrator struct {
	*Orchestrator

	navigationOrchestrator *NavigationOrchestrator
	tagOrchestrator        *TagsOrchestrator
	fileOrchestrator       *FileOrchestrator

	latestByRoute     map[string][]*viewmodel.Model
	viewmodelsByRoute map[string]*viewmodel.Model
}

func (orchestrator *ViewModelOrchestrator) blockingCacheWarmup() {
	orchestrator.getViewModel(orchestrator.rootItem(), false)
	orchestrator.GetLatest(route.New(), 5, 1)
}

func (orchestrator *ViewModelOrchestrator) GetFullViewModel(itemRoute route.Route) (viewModel viewmodel.Model, found bool) {

	startTime := time.Now()

	// get the requested item
	item := orchestrator.getItem(itemRoute)
	if item == nil {
		return viewModel, false
	}

	// get the base view model
	viewModel = *orchestrator.getViewModel(item, false)

	// navigation
	viewModel.ToplevelNavigation = orchestrator.navigationOrchestrator.GetToplevelNavigation()
	viewModel.BreadcrumbNavigation = orchestrator.navigationOrchestrator.GetBreadcrumbNavigation(itemRoute)
	viewModel.ItemNavigation = orchestrator.navigationOrchestrator.GetItemNavigation(itemRoute)

	// childs
	viewModel.Childs = orchestrator.getChildModels(itemRoute)

	// tags
	viewModel.Tags = orchestrator.tagOrchestrator.getItemTags(itemRoute)

	// Geo Coordinates
	viewModel.GeoLocation = getGeoLocation(item)

	// Analytics Settings
	viewModel.Analytics = orchestrator.getAnalyticsSettings()

	// Hash / ETag
	viewModel.Hash = item.Hash

	// special viewmodel attributes
	isRepositoryItem := item.Type == model.TypeRepository
	if isRepositoryItem {

		// tag cloud
		repositoryIsNotEmpty := orchestrator.index().Size() >= 5 // don't bother to create a tag cloud if there aren't enough documents
		if repositoryIsNotEmpty {

			tagCloud := orchestrator.tagOrchestrator.GetTagCloud()
			viewModel.TagCloud = tagCloud

		}

	}

	endTime := time.Now()
	duration := endTime.Sub(startTime)
	orchestrator.logger.Statistics("Getting the full view model %s took %f seconds.", viewModel.Route, duration.Seconds())

	return viewModel, true
}

func (orchestrator *ViewModelOrchestrator) GetViewModel(itemRoute route.Route) (viewModel viewmodel.Model, found bool) {

	// get the requested item
	item := orchestrator.getItem(itemRoute)
	if item == nil {
		return viewModel, false
	}

	vm := orchestrator.getViewModel(item, false)
	if vm == nil {
		return viewModel, false
	}

	return *vm, true
}

func (orchestrator *ViewModelOrchestrator) GetLatest(itemRoute route.Route, pageSize, page int) (latest []*viewmodel.Model, found bool) {

	cacheType := "latest"

	// load from cache
	if orchestrator.latestByRoute != nil {

		// re-prime the cache if it is stale
		if orchestrator.isCacheStale(cacheType) {
			go orchestrator.primeCache(cacheType)
		}

		// return the result
		if models, exists := orchestrator.latestByRoute[itemRoute.Value()]; exists {
			return pagedViewmodels(models, pageSize, page)
		}

		return []*viewmodel.Model{}, false

	}

	orchestrator.setCache(cacheType, func() {

		startTime := time.Now()

		latestModelsByRoute := make(map[string][]*viewmodel.Model)

		for _, childRoute := range orchestrator.repository.Routes() {

			// get the latest items
			latestItems := orchestrator.getLatestItems(childRoute)

			// store the results
			latestModelsByRoute[childRoute.Value()] = orchestrator.getLastesViewModelsFromItemList(latestItems)

		}

		endTime := time.Now()
		duration := endTime.Sub(startTime)
		orchestrator.logger.Statistics("Priming the latest cache took %f seconds.", duration.Seconds())

		// save the result
		orchestrator.latestByRoute = latestModelsByRoute

	})

	// return a result
	if models, exists := orchestrator.latestByRoute[itemRoute.Value()]; exists {
		return pagedViewmodels(models, pageSize, page)
	}

	return []*viewmodel.Model{}, false
}

// Converts a list of model.Item elements into a view models for the latest-items controller
func (orchestrator *ViewModelOrchestrator) getLastesViewModelsFromItemList(items []*model.Item) []*viewmodel.Model {

	// create viewmodels
	models := make([]*viewmodel.Model, 0, len(items))
	for _, item := range items {

		viewModel := orchestrator.getViewModel(item, false)
		if viewModel == nil {
			orchestrator.logger.Error("No view model found for item %q.", item)
			continue
		}

		// prepare lazy loading
		contentWithLazyLoadingEnabled := converter.LazyLoad(viewModel.Content)

		// create a copy (make sure we don't modify the content of the original view model)
		viewmodelCopy := *viewModel
		viewmodelCopy.Content = contentWithLazyLoadingEnabled

		models = append(models, &viewmodelCopy)
	}

	return models
}

func (orchestrator *ViewModelOrchestrator) getViewModel(item *model.Item, skipCache bool) *viewmodel.Model {

	// get the root item
	root := orchestrator.rootItem()
	if root == nil {
		panic(fmt.Sprintf("Cannot get viewmodel for route %q because no root item was found.", item))
	}

	itemRoute := item.Route()
	cacheType := "viewmodels"

	if skipCache {
		// prime the cache synchronously
		orchestrator.primeCache(cacheType)
	}

	// load from cache
	if orchestrator.viewmodelsByRoute != nil {

		// re-prime the cache if it is stale
		if orchestrator.isCacheStale(cacheType) {
			go orchestrator.primeCache(cacheType)
		}

		// return the result
		return orchestrator.viewmodelsByRoute[itemRoute.Value()]
	}

	orchestrator.setCache(cacheType, func() {

		startTime := time.Now()

		viewmodelsByRoute := make(map[string]*viewmodel.Model)

		for _, child := range orchestrator.index().GetAllItems() {

			childRoute := child.Route()

			// convert content
			convertedContent, err := orchestrator.converter.Convert(orchestrator.getItemByAlias, orchestrator.relativePather(childRoute), child)
			if err != nil {
				orchestrator.logger.Warn("Cannot convert content for item %q. Error: %s.", child.String(), err.Error())
				convertedContent = "<!-- Conversion Error -->"
			}

			// create a view model
			viewModel := &viewmodel.Model{
				Base:             getBaseModel(root, child, orchestrator.itemPather()),
				Content:          convertedContent,
				Publisher:        orchestrator.getPublisherInformation(),
				Author:           orchestrator.getAuthorInformation(child.MetaData.Author),
				Files:            orchestrator.fileOrchestrator.GetFiles(childRoute),
				Images:           orchestrator.fileOrchestrator.GetImages(childRoute),
				IsRepositoryItem: true,
			}

			// add rft url if rtf conversion is enabled
			if orchestrator.config.Conversion.Rtf.IsEnabled() {
				viewModel.RtfUrl = GetTypedItemUrl(child.Route(), "rtf")
			}

			// store the view model
			viewmodelsByRoute[childRoute.Value()] = viewModel

		}

		endTime := time.Now()
		duration := endTime.Sub(startTime)
		orchestrator.logger.Statistics("Priming the viewModel cache took %f seconds.", duration.Seconds())

		orchestrator.viewmodelsByRoute = viewmodelsByRoute

	})

	return orchestrator.viewmodelsByRoute[itemRoute.Value()]
}

func (orchestrator *ViewModelOrchestrator) getChildModels(itemRoute route.Route) []*viewmodel.Base {

	startTime := time.Now()

	rootItem := orchestrator.rootItem()
	if rootItem == nil {
		orchestrator.logger.Fatal("No root item found")
	}

	pathProvider := orchestrator.relativePather(itemRoute)

	childModels := make([]*viewmodel.Base, 0)

	childItems := orchestrator.getChilds(itemRoute)
	for _, childItem := range childItems {
		baseModel := getBaseModel(rootItem, childItem, pathProvider)
		childModels = append(childModels, &baseModel)
	}

	// sort the models
	viewmodel.SortBaseModelBy(sortBaseModelsByDate).Sort(childModels)

	endTime := time.Now()
	duration := endTime.Sub(startTime)
	orchestrator.logger.Statistics("Getting child models for route %s took %f seconds.", itemRoute, duration.Seconds())

	return childModels
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net/http"
)

func main() {
	addr := flag.String("addr", ":4000", "HTTP network address")
	certFile := flag.String("certfile", "cert.pem", "certificate PEM file")
	keyFile := flag.String("keyfile", "key.pem", "key PEM file")
	flag.Parse()

	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path != "/" {
			http.NotFound(w, req)
			return
		}
		fmt.Fprintf(w, "Proudly served with Go and HTTPS!")
	})

	srv := &http.Server{
		Addr:    *addr,
		Handler: mux,
		TLSConfig: &tls.Config{
			MinVersion:               tls.VersionTLS13,
			PreferServerCipherSuites: true,
		},
	}

	log.Printf("Starting server on %s", *addr)
	err := srv.ListenAndServeTLS(*certFile, *keyFile)
	log.Fatal(err)
}
<commit_msg>Fix typo<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"net/http"
)

func main() {
	addr := flag.String("addr", ":4000", "HTTPS network address")
	certFile := flag.String("certfile", "cert.pem", "certificate PEM file")
	keyFile := flag.String("keyfile", "key.pem", "key PEM file")
	flag.Parse()

	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		if req.URL.Path != "/" {
			http.NotFound(w, req)
			return
		}
		fmt.Fprintf(w, "Proudly served with Go and HTTPS!")
	})

	srv := &http.Server{
		Addr:    *addr,
		Handler: mux,
		TLSConfig: &tls.Config{
			MinVersion:               tls.VersionTLS13,
			PreferServerCipherSuites: true,
		},
	}

	log.Printf("Starting server on %s", *addr)
	err := srv.ListenAndServeTLS(*certFile, *keyFile)
	log.Fatal(err)
}
<|endoftext|>


<commit_before>package api

import (
	"appengine"
	"appengine/blobstore"
	"appengine/datastore"
	"github.com/gorilla/mux"
	"net/http"
	"time"

	"server/common"
	"server/model"
)

func visualizationsInit(s *mux.Router) {
	s.HandleFunc("/", listVisualizations).Methods("GET")
	s.HandleFunc("/", newVisualization).Methods("POST")
	s.HandleFunc("/{key}", getVisualization).Methods("GET")
	s.HandleFunc("/{key}/uploadurl", getVisualizationFileUploadUrl).Methods("GET")
	s.HandleFunc("/{key}/files", uploadVisualizationFile).Methods("POST")
}

func listVisualizations(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	q := datastore.NewQuery("visualization").Order("-Date");

	// Get visualizations
	var e []model.Visualization
	keys, err := q.GetAll(c, &e)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	// Prepare output
	var output []map[string]interface{}
	for i := range keys {
		output = append(output, map[string]interface{}{"Key": keys[i], "Title": e[i].Title, "Date": e[i].Date})
	}

	common.WriteJson(c, w, output)
}

func newVisualization(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	e := model.Visualization{"Untitled", time.Now(), nil}

	key, err := datastore.Put(c, datastore.NewIncompleteKey(c, "visualization", nil), &e)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	common.WriteJson(c, w, map[string]*datastore.Key{"key": key})
}

func getVisualization(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vars := mux.Vars(r)

	key, err := datastore.DecodeKey(vars["key"])
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	var e model.Visualization
	err = datastore.Get(c, key, &e)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	common.WriteJson(c, w, e)
}

func getVisualizationFileUploadUrl(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vars := mux.Vars(r)

	uploadUrl, err := blobstore.UploadURL(c, "/api/visualizations/" + vars["key"] + "/files", nil)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	common.WriteJson(c, w, uploadUrl.Path)
}

// TODO: delte file from blobstore if not needed anymore
func uploadVisualizationFile(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vars := mux.Vars(r)

	// Check if we have a file uploaded
	blobs, _, err := blobstore.ParseUpload(r)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	files := blobs["file"]
	if len(files) == 0 {
		c.Errorf("no file uploaded")
		return
	}

	key, err := datastore.DecodeKey(vars["key"])
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	// Start a datastore transaction
	var e model.Visualization
	err = datastore.RunInTransaction(c, func(c appengine.Context) error {
		// Get the visualization object
		err = datastore.Get(c, key, &e)
		if err != nil {
			return err
		}

		// Add the new file
		for i := range files {
			nfile := model.File{files[i].Filename, files[i].BlobKey}

			// Check if it already exists
			exists := false
			for j := range e.Files {
				if e.Files[j].Filename == nfile.Filename {
					// Overwrite
					// TODO: delete old file
					e.Files[j] = nfile
					exists = true
					break
				}
			}

			if !exists {
				e.Files = append(e.Files, nfile)
			}
		}

		// Save the visualization object
		key, err = datastore.Put(c, key, &e)
		return err
	}, nil)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	common.WriteJson(c, w, e)
}<commit_msg>Fix no visualizations bug<commit_after>package api

import (
	"appengine"
	"appengine/blobstore"
	"appengine/datastore"
	"github.com/gorilla/mux"
	"net/http"
	"time"

	"server/common"
	"server/model"
)

func visualizationsInit(s *mux.Router) {
	s.HandleFunc("/", listVisualizations).Methods("GET")
	s.HandleFunc("/", newVisualization).Methods("POST")
	s.HandleFunc("/{key}", getVisualization).Methods("GET")
	s.HandleFunc("/{key}/uploadurl", getVisualizationFileUploadUrl).Methods("GET")
	s.HandleFunc("/{key}/files", uploadVisualizationFile).Methods("POST")
}

func listVisualizations(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	q := datastore.NewQuery("visualization").Order("-Date");

	// Get visualizations
	var e []model.Visualization
	keys, err := q.GetAll(c, &e)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	// Prepare output
	output := make([]map[string]interface{}, 0)
	for i := range keys {
		output = append(output, map[string]interface{}{"Key": keys[i], "Title": e[i].Title, "Date": e[i].Date})
	}

	common.WriteJson(c, w, output)
}

func newVisualization(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)

	e := model.Visualization{"Untitled", time.Now(), nil}

	key, err := datastore.Put(c, datastore.NewIncompleteKey(c, "visualization", nil), &e)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	common.WriteJson(c, w, map[string]*datastore.Key{"key": key})
}

func getVisualization(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vars := mux.Vars(r)

	key, err := datastore.DecodeKey(vars["key"])
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	var e model.Visualization
	err = datastore.Get(c, key, &e)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	common.WriteJson(c, w, e)
}

func getVisualizationFileUploadUrl(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vars := mux.Vars(r)

	uploadUrl, err := blobstore.UploadURL(c, "/api/visualizations/" + vars["key"] + "/files", nil)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	common.WriteJson(c, w, uploadUrl.Path)
}

// TODO: delte file from blobstore if not needed anymore
func uploadVisualizationFile(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	vars := mux.Vars(r)

	// Check if we have a file uploaded
	blobs, _, err := blobstore.ParseUpload(r)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	files := blobs["file"]
	if len(files) == 0 {
		c.Errorf("no file uploaded")
		return
	}

	key, err := datastore.DecodeKey(vars["key"])
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	// Start a datastore transaction
	var e model.Visualization
	err = datastore.RunInTransaction(c, func(c appengine.Context) error {
		// Get the visualization object
		err = datastore.Get(c, key, &e)
		if err != nil {
			return err
		}

		// Add the new file
		for i := range files {
			nfile := model.File{files[i].Filename, files[i].BlobKey}

			// Check if it already exists
			exists := false
			for j := range e.Files {
				if e.Files[j].Filename == nfile.Filename {
					// Overwrite
					// TODO: delete old file
					e.Files[j] = nfile
					exists = true
					break
				}
			}

			if !exists {
				e.Files = append(e.Files, nfile)
			}
		}

		// Save the visualization object
		key, err = datastore.Put(c, key, &e)
		return err
	}, nil)
	if err != nil {
		common.ServeError(c, w, err)
		return
	}

	common.WriteJson(c, w, e)
}<|endoftext|>


<commit_before>package path

func simplifyPath(path string) string {
	return path
}
<commit_msg>solve 71 use stack<commit_after>package path

import (
	"bytes"
	"strings"
)

func simplifyPath(path string) string {
	return useStack(path)
}

// useStack time complexity O(N), space complexity O(N)
func useStack(path string) string {
	path = strings.Replace(path, "//", "/", -1)
	dirs := strings.FieldsFunc(path, func(c rune) bool {
		return c == '/'
	})
	st := make([]string, 0, len(dirs))
	var sb bytes.Buffer
	sb.WriteByte('/')
	for _, dir := range dirs {
		if dir == ".." {
			if len(st) > 0 {
				st = st[:len(st)-1]
			}
		} else if dir == "." {
			continue
		} else {
			st = append(st, dir)
		}
	}
	for i, dir := range st {
		sb.WriteString(dir)
		if i < len(st)-1 {
			sb.WriteByte('/')
		}
	}
	return sb.String()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	//    "net/ipv4"
	"golang.org/x/net/ipv4"
	"net"
	"os"
)

func main() {
	manager, _ := NewUDP_Manager()
	c, err := manager.NewUDP(20006, 20005)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println(c.read(41))
	c.write([]byte{'h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0})
	c.close()
}

type UDP_manager struct {
	pl   net.PacketConn
	open bool
	conn *ipv4.RawConn
    buff map[uint16](chan byte)
}

type UDP struct {
    conn      *ipv4.RawConn
    bytes chan byte
    src, dest uint16
}

func NewUDP_Manager() (*UDP_manager, error) {
	p, err := net.ListenPacket("ip4:17", "127.0.0.1")
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	r, err := ipv4.NewRawConn(p)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

    x := &UDP_manager{open: true, conn: r, pl: p, buff: make(map[uint16](chan byte))}

    go x.readAll()

    return x, nil
}

func (x *UDP_manager) readAll() {
    b := make([]byte, 1024)

    for {
        _, payload, _, err := x.conn.ReadFrom(b)
        if err != nil {
            continue
        }

        dest := (((uint16)(payload[2])) << 8) + ((uint16)(payload[3]))

        c, ok := x.buff[dest]
        if ok {
            go func() {
                for _, elem := range(payload) {
                    c <- elem
                }
            }()
        }
    }
}

func (x *UDP_manager) NewUDP(src, dest uint16) (*UDP, error) {
    x.buff[dest] = make(chan byte, 1024)
    return &UDP{src: src, dest: dest, conn: x.conn}, nil
}

func (c *UDP) read(size int) ([]byte, error) {
    data := make([]byte, size)
	for i := 0; i < size; i++ {
        data[i] = <-c.bytes
    }
    return data, nil
}
func (c *UDP) write(x []byte) error {
	UDPHeader := []byte{
		(byte)(c.src >> 8), (byte)(c.src), // Source port in byte slice
		(byte)(c.dest >> 8), (byte)(c.dest), // Dest port in byte slice
		(byte)((8 + len(x)) >> 8), (byte)(8 + len(x)), // Length in bytes of UDP header + data
		0, 0, // Checksum
	}

	x = append(UDPHeader, x...)

	h := &ipv4.Header{
		Version:  ipv4.Version,      // protocol version
		Len:      20,                // header length
		TOS:      0,                 // type-of-service (0 is everything normal)
		TotalLen: len(x) + 20,       // packet total length (octets)
		ID:       0,                 // identification
		Flags:    ipv4.DontFragment, // flags
		FragOff:  0,                 // fragment offset
		TTL:      8,                 // time-to-live (maximum lifespan in seconds)
		Protocol: 17,                // next protocol (17 is UDP)
		Checksum: 0,                 // checksum (apparently autocomputed)
		//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
		Dst: net.IPv4(127, 0, 0, 1), // destination address
		//Options                         // options, extension headers
	}
	err := c.conn.WriteTo(h, x, nil)
	if err != nil {
		fmt.Println(err)
		return err
	}
	return nil
}
func (c *UDP) close() error {
	return c.conn.Close()
}
<commit_msg>Should be finished... hopefully<commit_after>package main

import (
	"fmt"
	"golang.org/x/net/ipv4"
	"net"
)

func main() {
	manager, _ := NewUDP_Manager()
	c := []*UDP{}
	for i := 0; i < 20; i++ {
		udp, _ := manager.NewUDP(((uint16)(20000 + i)), ((uint16)(20000 + i)))
		c = append(c, udp)
		udp.write([]byte{((byte)(100 + i))})
		a, _ := udp.read(1)
		fmt.Println(a)
	}
	for _, element := range c {
		element.close()
	}
}

type UDP_manager struct {
	pl   net.PacketConn
	open bool
	conn *ipv4.RawConn
	buff map[uint16](chan byte)
}

type UDP struct {
	conn      *ipv4.RawConn
	bytes     chan byte
	src, dest uint16
}

func NewUDP_Manager() (*UDP_manager, error) {
	p, err := net.ListenPacket("ip4:17", "127.0.0.1")
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	r, err := ipv4.NewRawConn(p)
	if err != nil {
		fmt.Println(err)
		return nil, err
	}

	x := &UDP_manager{open: true, conn: r, pl: p, buff: make(map[uint16](chan byte))}

	go x.readAll()

	return x, nil
}

func (x *UDP_manager) readAll() {
	b := make([]byte, 1024)

	for {
		_, payload, _, err := x.conn.ReadFrom(b)
		if err != nil {
			continue
		}

		dest := (((uint16)(payload[2])) << 8) + ((uint16)(payload[3]))
		//		fmt.Println(dest)
		//		fmt.Println(payload)
		//
		//		fmt.Println(x.buff)
		c, ok := x.buff[dest]
		//fmt.Println(ok)
		payload = payload[8:]
		if ok {
			go func() {
				for _, elem := range payload {
					//fmt.Println("Writing")
					c <- elem
				}
			}()
		}
	}
}

func (x *UDP_manager) NewUDP(src, dest uint16) (*UDP, error) {
	x.buff[src] = make(chan byte, 1024)
	return &UDP{src: src, dest: dest, conn: x.conn, bytes: x.buff[src]}, nil
}

func (c *UDP) read(size int) ([]byte, error) {
	data := make([]byte, size)
	for i := 0; i < size; i++ {
		//fmt.Println("test")
		data[i] = <-c.bytes
		//fmt.Println(data[i])
	}
	return data, nil
}
func (c *UDP) write(x []byte) error {
	UDPHeader := []byte{
		(byte)(c.src >> 8), (byte)(c.src), // Source port in byte slice
		(byte)(c.dest >> 8), (byte)(c.dest), // Dest port in byte slice
		(byte)((8 + len(x)) >> 8), (byte)(8 + len(x)), // Length in bytes of UDP header + data
		0, 0, // Checksum
	}

	x = append(UDPHeader, x...)

	h := &ipv4.Header{
		Version:  ipv4.Version,      // protocol version
		Len:      20,                // header length
		TOS:      0,                 // type-of-service (0 is everything normal)
		TotalLen: len(x) + 20,       // packet total length (octets)
		ID:       0,                 // identification
		Flags:    ipv4.DontFragment, // flags
		FragOff:  0,                 // fragment offset
		TTL:      8,                 // time-to-live (maximum lifespan in seconds)
		Protocol: 17,                // next protocol (17 is UDP)
		Checksum: 0,                 // checksum (apparently autocomputed)
		//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
		Dst: net.IPv4(127, 0, 0, 1), // destination address
		//Options                         // options, extension headers
	}
	err := c.conn.WriteTo(h, x, nil)
	if err != nil {
		fmt.Println(err)
		return err
	}
	return nil
}
func (c *UDP) close() error {
	return c.conn.Close()
}
<|endoftext|>