<commit_before>package network

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/docker/docker/api/server/httputils"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/network"
	"github.com/docker/libnetwork"
)

func (n *networkRouter) getNetworksList(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	filter := r.Form.Get("filters")
	netFilters, err := filters.FromParam(filter)
	if err != nil {
		return err
	}

	list := []types.NetworkResource{}

	if nr, err := n.clusterProvider.GetNetworks(); err == nil {
		for _, nw := range nr {
			list = append(list, nw)
		}
	}

	// Combine the network list returned by Docker daemon if it is not already
	// returned by the cluster manager
SKIP:
	for _, nw := range n.backend.GetNetworks() {
		for _, nl := range list {
			if nl.ID == nw.ID() {
				continue SKIP
			}
		}
		list = append(list, *n.buildNetworkResource(nw))
	}

	list, err = filterNetworks(list, netFilters)
	if err != nil {
		return err
	}
	return httputils.WriteJSON(w, http.StatusOK, list)
}

func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		if nr, err := n.clusterProvider.GetNetwork(vars["id"]); err == nil {
			return httputils.WriteJSON(w, http.StatusOK, nr)
		}
		return err
	}
	return httputils.WriteJSON(w, http.StatusOK, n.buildNetworkResource(nw))
}

func (n *networkRouter) postNetworkCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var create types.NetworkCreateRequest

	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&create); err != nil {
		return err
	}

	if _, err := n.clusterProvider.GetNetwork(create.Name); err == nil {
		return libnetwork.NetworkNameError(create.Name)
	}

	nw, err := n.backend.CreateNetwork(create)
	if err != nil {
		if _, ok := err.(libnetwork.ManagerRedirectError); !ok {
			return err
		}
		id, err := n.clusterProvider.CreateNetwork(create)
		if err != nil {
			return err
		}
		nw = &types.NetworkCreateResponse{ID: id}
	}

	return httputils.WriteJSON(w, http.StatusCreated, nw)
}

func (n *networkRouter) postNetworkConnect(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var connect types.NetworkConnect
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&connect); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		return err
	}

	return n.backend.ConnectContainerToNetwork(connect.Container, nw.Name(), connect.EndpointConfig)
}

func (n *networkRouter) postNetworkDisconnect(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var disconnect types.NetworkDisconnect
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&disconnect); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		return err
	}

	return n.backend.DisconnectContainerFromNetwork(disconnect.Container, nw, disconnect.Force)
}

func (n *networkRouter) deleteNetwork(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}
	if _, err := n.clusterProvider.GetNetwork(vars["id"]); err == nil {
		return n.clusterProvider.RemoveNetwork(vars["id"])
	}
	if err := n.backend.DeleteNetwork(vars["id"]); err != nil {
		return err
	}
	w.WriteHeader(http.StatusNoContent)
	return nil
}

func (n *networkRouter) buildNetworkResource(nw libnetwork.Network) *types.NetworkResource {
	r := &types.NetworkResource{}
	if nw == nil {
		return r
	}

	info := nw.Info()
	r.Name = nw.Name()
	r.ID = nw.ID()
	r.Scope = info.Scope()
	if n.clusterProvider.IsManager() {
		if _, err := n.clusterProvider.GetNetwork(nw.Name()); err == nil {
			r.Scope = "swarm"
		}
	} else if info.Dynamic() {
		r.Scope = "swarm"
	}
	r.Driver = nw.Type()
	r.EnableIPv6 = info.IPv6Enabled()
	r.Internal = info.Internal()
	r.Options = info.DriverOptions()
	r.Containers = make(map[string]types.EndpointResource)
	buildIpamResources(r, info)
	r.Internal = info.Internal()
	r.Labels = info.Labels()

	epl := nw.Endpoints()
	for _, e := range epl {
		ei := e.Info()
		if ei == nil {
			continue
		}
		sb := ei.Sandbox()
		key := "ep-" + e.ID()
		if sb != nil {
			key = sb.ContainerID()
		}

		r.Containers[key] = buildEndpointResource(e)
	}
	return r
}

func buildIpamResources(r *types.NetworkResource, nwInfo libnetwork.NetworkInfo) {
	id, opts, ipv4conf, ipv6conf := nwInfo.IpamConfig()

	ipv4Info, ipv6Info := nwInfo.IpamInfo()

	r.IPAM.Driver = id

	r.IPAM.Options = opts

	r.IPAM.Config = []network.IPAMConfig{}
	for _, ip4 := range ipv4conf {
		if ip4.PreferredPool == "" {
			continue
		}
		iData := network.IPAMConfig{}
		iData.Subnet = ip4.PreferredPool
		iData.IPRange = ip4.SubPool
		iData.Gateway = ip4.Gateway
		iData.AuxAddress = ip4.AuxAddresses
		r.IPAM.Config = append(r.IPAM.Config, iData)
	}

	if len(r.IPAM.Config) == 0 {
		for _, ip4Info := range ipv4Info {
			iData := network.IPAMConfig{}
			iData.Subnet = ip4Info.IPAMData.Pool.String()
			iData.Gateway = ip4Info.IPAMData.Gateway.String()
			r.IPAM.Config = append(r.IPAM.Config, iData)
		}
	}

	hasIpv6Conf := false
	for _, ip6 := range ipv6conf {
		if ip6.PreferredPool == "" {
			continue
		}
		hasIpv6Conf = true
		iData := network.IPAMConfig{}
		iData.Subnet = ip6.PreferredPool
		iData.IPRange = ip6.SubPool
		iData.Gateway = ip6.Gateway
		iData.AuxAddress = ip6.AuxAddresses
		r.IPAM.Config = append(r.IPAM.Config, iData)
	}

	if !hasIpv6Conf {
		for _, ip6Info := range ipv6Info {
			iData := network.IPAMConfig{}
			iData.Subnet = ip6Info.IPAMData.Pool.String()
			iData.Gateway = ip6Info.IPAMData.Gateway.String()
			r.IPAM.Config = append(r.IPAM.Config, iData)
		}
	}
}

func buildEndpointResource(e libnetwork.Endpoint) types.EndpointResource {
	er := types.EndpointResource{}
	if e == nil {
		return er
	}

	er.EndpointID = e.ID()
	er.Name = e.Name()
	ei := e.Info()
	if ei == nil {
		return er
	}

	if iface := ei.Iface(); iface != nil {
		if mac := iface.MacAddress(); mac != nil {
			er.MacAddress = mac.String()
		}
		if ip := iface.Address(); ip != nil && len(ip.IP) > 0 {
			er.IPv4Address = ip.String()
		}

		if ipv6 := iface.AddressIPv6(); ipv6 != nil && len(ipv6.IP) > 0 {
			er.IPv6Address = ipv6.String()
		}
	}
	return er
}
<commit_msg>Change buildEndpointResource params avoid repeat visits resource<commit_after>package network

import (
	"encoding/json"
	"net/http"

	"golang.org/x/net/context"

	"github.com/docker/docker/api/server/httputils"
	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/filters"
	"github.com/docker/engine-api/types/network"
	"github.com/docker/libnetwork"
)

func (n *networkRouter) getNetworksList(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	filter := r.Form.Get("filters")
	netFilters, err := filters.FromParam(filter)
	if err != nil {
		return err
	}

	list := []types.NetworkResource{}

	if nr, err := n.clusterProvider.GetNetworks(); err == nil {
		for _, nw := range nr {
			list = append(list, nw)
		}
	}

	// Combine the network list returned by Docker daemon if it is not already
	// returned by the cluster manager
SKIP:
	for _, nw := range n.backend.GetNetworks() {
		for _, nl := range list {
			if nl.ID == nw.ID() {
				continue SKIP
			}
		}
		list = append(list, *n.buildNetworkResource(nw))
	}

	list, err = filterNetworks(list, netFilters)
	if err != nil {
		return err
	}
	return httputils.WriteJSON(w, http.StatusOK, list)
}

func (n *networkRouter) getNetwork(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		if nr, err := n.clusterProvider.GetNetwork(vars["id"]); err == nil {
			return httputils.WriteJSON(w, http.StatusOK, nr)
		}
		return err
	}
	return httputils.WriteJSON(w, http.StatusOK, n.buildNetworkResource(nw))
}

func (n *networkRouter) postNetworkCreate(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var create types.NetworkCreateRequest

	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&create); err != nil {
		return err
	}

	if _, err := n.clusterProvider.GetNetwork(create.Name); err == nil {
		return libnetwork.NetworkNameError(create.Name)
	}

	nw, err := n.backend.CreateNetwork(create)
	if err != nil {
		if _, ok := err.(libnetwork.ManagerRedirectError); !ok {
			return err
		}
		id, err := n.clusterProvider.CreateNetwork(create)
		if err != nil {
			return err
		}
		nw = &types.NetworkCreateResponse{ID: id}
	}

	return httputils.WriteJSON(w, http.StatusCreated, nw)
}

func (n *networkRouter) postNetworkConnect(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var connect types.NetworkConnect
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&connect); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		return err
	}

	return n.backend.ConnectContainerToNetwork(connect.Container, nw.Name(), connect.EndpointConfig)
}

func (n *networkRouter) postNetworkDisconnect(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	var disconnect types.NetworkDisconnect
	if err := httputils.ParseForm(r); err != nil {
		return err
	}

	if err := httputils.CheckForJSON(r); err != nil {
		return err
	}

	if err := json.NewDecoder(r.Body).Decode(&disconnect); err != nil {
		return err
	}

	nw, err := n.backend.FindNetwork(vars["id"])
	if err != nil {
		return err
	}

	return n.backend.DisconnectContainerFromNetwork(disconnect.Container, nw, disconnect.Force)
}

func (n *networkRouter) deleteNetwork(ctx context.Context, w http.ResponseWriter, r *http.Request, vars map[string]string) error {
	if err := httputils.ParseForm(r); err != nil {
		return err
	}
	if _, err := n.clusterProvider.GetNetwork(vars["id"]); err == nil {
		return n.clusterProvider.RemoveNetwork(vars["id"])
	}
	if err := n.backend.DeleteNetwork(vars["id"]); err != nil {
		return err
	}
	w.WriteHeader(http.StatusNoContent)
	return nil
}

func (n *networkRouter) buildNetworkResource(nw libnetwork.Network) *types.NetworkResource {
	r := &types.NetworkResource{}
	if nw == nil {
		return r
	}

	info := nw.Info()
	r.Name = nw.Name()
	r.ID = nw.ID()
	r.Scope = info.Scope()
	if n.clusterProvider.IsManager() {
		if _, err := n.clusterProvider.GetNetwork(nw.Name()); err == nil {
			r.Scope = "swarm"
		}
	} else if info.Dynamic() {
		r.Scope = "swarm"
	}
	r.Driver = nw.Type()
	r.EnableIPv6 = info.IPv6Enabled()
	r.Internal = info.Internal()
	r.Options = info.DriverOptions()
	r.Containers = make(map[string]types.EndpointResource)
	buildIpamResources(r, info)
	r.Internal = info.Internal()
	r.Labels = info.Labels()

	epl := nw.Endpoints()
	for _, e := range epl {
		ei := e.Info()
		if ei == nil {
			continue
		}
		sb := ei.Sandbox()
		tmpID := e.ID()
		key := "ep-" + tmpID
		if sb != nil {
			key = sb.ContainerID()
		}

		r.Containers[key] = buildEndpointResource(tmpID, e.Name(), ei)
	}
	return r
}

func buildIpamResources(r *types.NetworkResource, nwInfo libnetwork.NetworkInfo) {
	id, opts, ipv4conf, ipv6conf := nwInfo.IpamConfig()

	ipv4Info, ipv6Info := nwInfo.IpamInfo()

	r.IPAM.Driver = id

	r.IPAM.Options = opts

	r.IPAM.Config = []network.IPAMConfig{}
	for _, ip4 := range ipv4conf {
		if ip4.PreferredPool == "" {
			continue
		}
		iData := network.IPAMConfig{}
		iData.Subnet = ip4.PreferredPool
		iData.IPRange = ip4.SubPool
		iData.Gateway = ip4.Gateway
		iData.AuxAddress = ip4.AuxAddresses
		r.IPAM.Config = append(r.IPAM.Config, iData)
	}

	if len(r.IPAM.Config) == 0 {
		for _, ip4Info := range ipv4Info {
			iData := network.IPAMConfig{}
			iData.Subnet = ip4Info.IPAMData.Pool.String()
			iData.Gateway = ip4Info.IPAMData.Gateway.String()
			r.IPAM.Config = append(r.IPAM.Config, iData)
		}
	}

	hasIpv6Conf := false
	for _, ip6 := range ipv6conf {
		if ip6.PreferredPool == "" {
			continue
		}
		hasIpv6Conf = true
		iData := network.IPAMConfig{}
		iData.Subnet = ip6.PreferredPool
		iData.IPRange = ip6.SubPool
		iData.Gateway = ip6.Gateway
		iData.AuxAddress = ip6.AuxAddresses
		r.IPAM.Config = append(r.IPAM.Config, iData)
	}

	if !hasIpv6Conf {
		for _, ip6Info := range ipv6Info {
			iData := network.IPAMConfig{}
			iData.Subnet = ip6Info.IPAMData.Pool.String()
			iData.Gateway = ip6Info.IPAMData.Gateway.String()
			r.IPAM.Config = append(r.IPAM.Config, iData)
		}
	}
}

func buildEndpointResource(id string, name string, info libnetwork.EndpointInfo) types.EndpointResource {
	er := types.EndpointResource{}

	er.EndpointID = id
	er.Name = name
	ei := info
	if ei == nil {
		return er
	}

	if iface := ei.Iface(); iface != nil {
		if mac := iface.MacAddress(); mac != nil {
			er.MacAddress = mac.String()
		}
		if ip := iface.Address(); ip != nil && len(ip.IP) > 0 {
			er.IPv4Address = ip.String()
		}

		if ipv6 := iface.AddressIPv6(); ipv6 != nil && len(ipv6.IP) > 0 {
			er.IPv6Address = ipv6.String()
		}
	}
	return er
}
<|endoftext|>


<commit_before>package libp2pwebtransport

import (
	"context"
	"crypto/tls"
	"errors"
	"net"
	"net/http"
	"time"

	tpt "github.com/libp2p/go-libp2p-core/transport"

	noise "github.com/libp2p/go-libp2p-noise"

	"github.com/lucas-clemente/quic-go/http3"
	"github.com/marten-seemann/webtransport-go"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr/net"
)

var errClosed = errors.New("closed")

const queueLen = 16
const handshakeTimeout = 10 * time.Second

type listener struct {
	transport   tpt.Transport
	noise       *noise.Transport
	certManager *certManager

	server webtransport.Server

	ctx       context.Context
	ctxCancel context.CancelFunc

	serverClosed chan struct{} // is closed when server.Serve returns

	addr      net.Addr
	multiaddr ma.Multiaddr

	queue chan *webtransport.Conn
}

var _ tpt.Listener = &listener{}

func newListener(laddr ma.Multiaddr, transport tpt.Transport, noise *noise.Transport, certManager *certManager) (tpt.Listener, error) {
	network, addr, err := manet.DialArgs(laddr)
	if err != nil {
		return nil, err
	}
	udpAddr, err := net.ResolveUDPAddr(network, addr)
	if err != nil {
		return nil, err
	}
	udpConn, err := net.ListenUDP(network, udpAddr)
	if err != nil {
		return nil, err
	}
	localMultiaddr, err := toWebtransportMultiaddr(udpConn.LocalAddr())
	if err != nil {
		return nil, err
	}
	ln := &listener{
		transport:    transport,
		noise:        noise,
		certManager:  certManager,
		queue:        make(chan *webtransport.Conn, queueLen),
		serverClosed: make(chan struct{}),
		addr:         udpConn.LocalAddr(),
		multiaddr:    localMultiaddr,
		server: webtransport.Server{
			H3: http3.Server{
				TLSConfig: &tls.Config{GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {
					return certManager.GetConfig(), nil
				}},
			},
		},
	}
	ln.ctx, ln.ctxCancel = context.WithCancel(context.Background())
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello, world!"))
	})
	mux.HandleFunc(webtransportHTTPEndpoint, func(w http.ResponseWriter, r *http.Request) {
		// TODO: check ?type=multistream URL param
		c, err := ln.server.Upgrade(w, r)
		if err != nil {
			w.WriteHeader(500)
			return
		}
		// TODO: handle queue overflow
		ln.queue <- c
		// We need to block until we're done with this WebTransport session.
		<-c.Context().Done()
	})
	ln.server.H3.Handler = mux
	go func() {
		defer close(ln.serverClosed)
		defer func() { udpConn.Close() }()
		if err := ln.server.Serve(udpConn); err != nil {
			// TODO: only output if the server hasn't been closed
			log.Debugw("serving failed", "addr", udpConn.LocalAddr(), "error", err)
		}
	}()
	return ln, nil
}

func (l *listener) Accept() (tpt.CapableConn, error) {
	queue := make(chan tpt.CapableConn, queueLen)
	for {
		select {
		case <-l.ctx.Done():
			return nil, errClosed
		default:
		}

		var c *webtransport.Conn
		select {
		case c = <-l.queue:
			go func(c *webtransport.Conn) {
				ctx, cancel := context.WithTimeout(l.ctx, handshakeTimeout)
				defer cancel()
				conn, err := l.handshake(ctx, c)
				if err != nil {
					log.Debugw("handshake failed", "error", err)
					c.Close()
					return
				}
				// TODO: handle queue overflow
				queue <- conn
			}(c)
		case conn := <-queue:
			return conn, nil
		case <-l.ctx.Done():
			return nil, errClosed
		}
	}
}

func (l *listener) handshake(ctx context.Context, c *webtransport.Conn) (tpt.CapableConn, error) {
	str, err := c.AcceptStream(ctx)
	if err != nil {
		return nil, err
	}
	conn, err := l.noise.SecureInbound(ctx, &webtransportStream{Stream: str, wconn: c}, "")
	if err != nil {
		return nil, err
	}
	return newConn(l.transport, c, conn.LocalPrivateKey(), conn.RemotePublicKey())
}

func (l *listener) Addr() net.Addr {
	return l.addr
}

func (l *listener) Multiaddr() ma.Multiaddr {
	return l.multiaddr.Encapsulate(l.certManager.AddrComponent())
}

func (l *listener) Close() error {
	l.ctxCancel()
	err := l.server.Close()
	<-l.serverClosed
	return err
}
<commit_msg>simplify the listener.Accept logic<commit_after>package libp2pwebtransport

import (
	"context"
	"crypto/tls"
	"errors"
	"net"
	"net/http"
	"time"

	tpt "github.com/libp2p/go-libp2p-core/transport"

	noise "github.com/libp2p/go-libp2p-noise"

	"github.com/lucas-clemente/quic-go/http3"
	"github.com/marten-seemann/webtransport-go"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr/net"
)

var errClosed = errors.New("closed")

const queueLen = 16
const handshakeTimeout = 10 * time.Second

type listener struct {
	transport   tpt.Transport
	noise       *noise.Transport
	certManager *certManager

	server webtransport.Server

	ctx       context.Context
	ctxCancel context.CancelFunc

	serverClosed chan struct{} // is closed when server.Serve returns

	addr      net.Addr
	multiaddr ma.Multiaddr

	queue chan tpt.CapableConn
}

var _ tpt.Listener = &listener{}

func newListener(laddr ma.Multiaddr, transport tpt.Transport, noise *noise.Transport, certManager *certManager) (tpt.Listener, error) {
	network, addr, err := manet.DialArgs(laddr)
	if err != nil {
		return nil, err
	}
	udpAddr, err := net.ResolveUDPAddr(network, addr)
	if err != nil {
		return nil, err
	}
	udpConn, err := net.ListenUDP(network, udpAddr)
	if err != nil {
		return nil, err
	}
	localMultiaddr, err := toWebtransportMultiaddr(udpConn.LocalAddr())
	if err != nil {
		return nil, err
	}
	ln := &listener{
		transport:    transport,
		noise:        noise,
		certManager:  certManager,
		queue:        make(chan tpt.CapableConn, queueLen),
		serverClosed: make(chan struct{}),
		addr:         udpConn.LocalAddr(),
		multiaddr:    localMultiaddr,
		server: webtransport.Server{
			H3: http3.Server{
				TLSConfig: &tls.Config{GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) {
					return certManager.GetConfig(), nil
				}},
			},
		},
	}
	ln.ctx, ln.ctxCancel = context.WithCancel(context.Background())
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello, world!"))
	})
	mux.HandleFunc(webtransportHTTPEndpoint, func(w http.ResponseWriter, r *http.Request) {
		// TODO: check ?type=multistream URL param
		c, err := ln.server.Upgrade(w, r)
		if err != nil {
			w.WriteHeader(500)
			return
		}
		ctx, cancel := context.WithTimeout(ln.ctx, handshakeTimeout)
		conn, err := ln.handshake(ctx, c)
		if err != nil {
			cancel()
			log.Debugw("handshake failed", "error", err)
			c.Close()
			return
		}
		cancel()
		ln.queue <- conn
		// We need to block until we're done with this WebTransport session.
		<-c.Context().Done()
	})
	ln.server.H3.Handler = mux
	go func() {
		defer close(ln.serverClosed)
		defer func() { udpConn.Close() }()
		if err := ln.server.Serve(udpConn); err != nil {
			// TODO: only output if the server hasn't been closed
			log.Debugw("serving failed", "addr", udpConn.LocalAddr(), "error", err)
		}
	}()
	return ln, nil
}

func (l *listener) Accept() (tpt.CapableConn, error) {
	select {
	case <-l.ctx.Done():
		return nil, errClosed
	case c := <-l.queue:
		return c, nil
	}
}

func (l *listener) handshake(ctx context.Context, c *webtransport.Conn) (tpt.CapableConn, error) {
	str, err := c.AcceptStream(ctx)
	if err != nil {
		return nil, err
	}
	conn, err := l.noise.SecureInbound(ctx, &webtransportStream{Stream: str, wconn: c}, "")
	if err != nil {
		return nil, err
	}
	return newConn(l.transport, c, conn.LocalPrivateKey(), conn.RemotePublicKey())
}

func (l *listener) Addr() net.Addr {
	return l.addr
}

func (l *listener) Multiaddr() ma.Multiaddr {
	return l.multiaddr.Encapsulate(l.certManager.AddrComponent())
}

func (l *listener) Close() error {
	l.ctxCancel()
	err := l.server.Close()
	<-l.serverClosed
	return err
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Peter Mattis (peter@cockroachlabs.com)

package driver

import (
	"database/sql"
	"reflect"
	"testing"

	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/testutils"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

var isError = testutils.IsError

func setup(t *testing.T) (*server.TestServer, *sql.DB) {
	s := server.StartTestServer(nil)
	db, err := sql.Open("cockroach", "https://root@"+s.ServingAddr()+"?certs=test_certs")
	if err != nil {
		t.Fatal(err)
	}
	return s, db
}

func cleanup(s *server.TestServer, db *sql.DB) {
	_ = db.Close()
	s.Stop()
}

func readAll(t *testing.T, rows *sql.Rows) [][]string {
	cols, err := rows.Columns()
	if err != nil {
		t.Fatal(err)
	}
	var results [][]string
	results = append(results, cols)

	for rows.Next() {
		strs := make([]string, len(cols))
		vals := make([]interface{}, len(cols))
		for i := range vals {
			vals[i] = &strs[i]
		}
		if err := rows.Scan(vals...); err != nil {
			t.Fatal(err)
		}
		results = append(results, strs)
	}

	if err := rows.Err(); err != nil {
		t.Fatal(err)
	}

	return results
}

func TestCreateDatabase(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	if _, err := db.Exec(`CREATE DATABASE foo`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`CREATE DATABASE foo`); !isError(err, "database .* already exists") {
		t.Fatalf("expected failure, but found success")
	}
	if _, err := db.Exec(`CREATE DATABASE IF NOT EXISTS foo`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`CREATE DATABASE ""`); !isError(err, "empty database name") {
		t.Fatal(err)
	}
}

func TestShowDatabases(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	names := []string{"a", "b", "c", "d", "e", "f", "g", "i"}
	for i, name := range names {
		if _, err := db.Exec("CREATE DATABASE " + name); err != nil {
			t.Fatal(err)
		}

		rows, err := db.Query("SHOW DATABASES")
		if err != nil {
			t.Fatal(err)
		}
		var databases []string
		for rows.Next() {
			var n string
			if err := rows.Scan(&n); err != nil {
				t.Fatal(err)
			}
			databases = append(databases, n)
		}

		expectedDatabases := names[:i+1]
		if !reflect.DeepEqual(expectedDatabases, databases) {
			t.Fatalf("expected %+v, but got %+v", expectedDatabases, databases)
		}
	}
}

func TestCreateTable(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}

	const cols = "(id INT PRIMARY KEY)"
	if _, err := db.Exec("CREATE TABLE users " + cols); !isError(err, "no database specified") {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE t.users " + cols); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE t.users " + cols); !isError(err, "table .* already exists") {
		t.Fatal(err)
	}

	if _, err := db.Exec("SET DATABASE = t"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE users " + cols); !isError(err, "table .* already exists") {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users " + cols); err != nil {
		t.Fatal(err)
	}
}

func TestShowTables(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}

	names := []string{"a", "b", "c", "d", "e", "f", "g", "i"}
	for i, name := range names {
		if _, err := db.Exec("CREATE TABLE t." + name + " (id INT PRIMARY KEY)"); err != nil {
			t.Fatal(err)
		}

		rows, err := db.Query("SHOW TABLES FROM t")
		if err != nil {
			t.Fatal(err)
		}
		var tables []string
		for rows.Next() {
			var n string
			if err := rows.Scan(&n); err != nil {
				t.Fatal(err)
			}
			tables = append(tables, n)
		}

		expectedTables := names[:i+1]
		if !reflect.DeepEqual(expectedTables, tables) {
			t.Fatalf("expected %+v, but got %+v", expectedTables, tables)
		}
	}

	if _, err := db.Query("SHOW TABLES"); !isError(err, "no database specified") {
		t.Fatal(err)
	}
	if _, err := db.Exec("SET DATABASE = t"); err != nil {
		t.Fatal(err)
	}

	rows, err := db.Query("SHOW TABLES")
	if err != nil {
		t.Fatal(err)
	}
	if err := rows.Close(); err != nil {
		t.Fatal(err)
	}
}

func TestShowColumns(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	const schema = `
CREATE TABLE t.users (
  id    INT PRIMARY KEY,
  name  VARCHAR NOT NULL,
  title VARCHAR
)`

	if _, err := db.Query("SHOW COLUMNS FROM t.users"); !isError(err, "database .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Query("SHOW COLUMNS FROM t.users"); !isError(err, "table .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Query("SHOW COLUMNS FROM users"); !isError(err, "no database specified") {
		t.Fatal(err)
	}

	if _, err := db.Exec(schema); err != nil {
		t.Fatal(err)
	}

	rows, err := db.Query("SHOW COLUMNS FROM t.users")
	if err != nil {
		t.Fatal(err)
	}
	results := readAll(t, rows)
	expectedResults := [][]string{
		{"Field", "Type", "Null"},
		{"id", "INT", "true"},
		{"name", "CHAR", "false"},
		{"title", "CHAR", "true"},
	}
	if !reflect.DeepEqual(expectedResults, results) {
		t.Fatalf("expected %s, but got %s", expectedResults, results)
	}
}

func TestShowIndex(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	const schema = `
CREATE TABLE t.users (
  id    INT PRIMARY KEY,
  name  VARCHAR NOT NULL,
  CONSTRAINT foo INDEX (name),
  CONSTRAINT bar UNIQUE (id, name)
)`

	if _, err := db.Query("SHOW INDEX FROM t.users"); !isError(err, "database .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Query("SHOW INDEX FROM t.users"); !isError(err, "table .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Query("SHOW INDEX FROM users"); !isError(err, "no database specified") {
		t.Fatal(err)
	}

	if _, err := db.Exec(schema); err != nil {
		t.Fatal(err)
	}

	rows, err := db.Query("SHOW INDEX FROM t.users")
	if err != nil {
		t.Fatal(err)
	}
	results := readAll(t, rows)
	expectedResults := [][]string{
		{"Table", "Name", "Unique", "Seq", "Column"},
		{"users", "primary", "true", "1", "id"},
		{"users", "foo", "false", "1", "name"},
		{"users", "bar", "true", "1", "id"},
		{"users", "bar", "true", "2", "name"},
	}
	if !reflect.DeepEqual(expectedResults, results) {
		t.Fatalf("expected %s, but got %s", expectedResults, results)
	}
}

func TestInsertSelect(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	schema := `
CREATE TABLE t.kv (
  k CHAR PRIMARY KEY,
  v CHAR
)`

	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`INSERT INTO t.kv VALUES ('a', 'b')`); !isError(err, "table .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Exec(schema); err != nil {
		t.Fatal(err)
	}
	if rows, err := db.Query(`SELECT * FROM t.kv`); err != nil {
		t.Fatal(err)
	} else if results := readAll(t, rows); len(results) > 1 {
		t.Fatalf("non-empty result set from empty table: %s", results)
	}
	if _, err := db.Exec(`INSERT INTO t.kv VALUES ('a')`); !isError(err, "invalid values for columns") {
		t.Fatal(err)
	}
	if _, err := db.Exec(`INSERT INTO t.kv (v) VALUES ('a')`); !isError(err, "missing .* primary key column") {
		t.Fatal(err)
	}
	if _, err := db.Exec(`INSERT INTO t.kv (k,v) VALUES ('a', 'b'), ('c', 'd')`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`INSERT INTO t.kv VALUES ('e', 'f')`); err != nil {
		t.Fatal(err)
	}

	rows, err := db.Query("SELECT * FROM t.kv")
	if err != nil {
		t.Fatal(err)
	}
	results := readAll(t, rows)
	expectedResults := [][]string{
		{"k", "v"},
		{"a", "b"},
		{"c", "d"},
		{"e", "f"},
	}
	if !reflect.DeepEqual(expectedResults, results) {
		t.Fatalf("expected %s, but got %s", expectedResults, results)
	}

	// TODO(pmattis): We need much more testing of WHERE clauses. Need to think
	// through the whole testing story in general.
	rows, err = db.Query("SELECT * FROM t.kv WHERE k IN ('a', 'c')")
	if err != nil {
		t.Fatal(err)
	}
	results = readAll(t, rows)
	expectedResults = [][]string{
		{"k", "v"},
		{"a", "b"},
		{"c", "d"},
	}
	if !reflect.DeepEqual(expectedResults, results) {
		t.Fatalf("expected %s, but got %s", expectedResults, results)
	}
}
<commit_msg>add TODO<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Peter Mattis (peter@cockroachlabs.com)

package driver

import (
	"database/sql"
	"reflect"
	"testing"

	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/testutils"
	"github.com/cockroachdb/cockroach/util/leaktest"
)

var isError = testutils.IsError

// TODO(mberhault): should have one insecure test as well.
func setup(t *testing.T) (*server.TestServer, *sql.DB) {
	s := server.StartTestServer(nil)
	db, err := sql.Open("cockroach", "https://root@"+s.ServingAddr()+"?certs=test_certs")
	if err != nil {
		t.Fatal(err)
	}
	return s, db
}

func cleanup(s *server.TestServer, db *sql.DB) {
	_ = db.Close()
	s.Stop()
}

func readAll(t *testing.T, rows *sql.Rows) [][]string {
	cols, err := rows.Columns()
	if err != nil {
		t.Fatal(err)
	}
	var results [][]string
	results = append(results, cols)

	for rows.Next() {
		strs := make([]string, len(cols))
		vals := make([]interface{}, len(cols))
		for i := range vals {
			vals[i] = &strs[i]
		}
		if err := rows.Scan(vals...); err != nil {
			t.Fatal(err)
		}
		results = append(results, strs)
	}

	if err := rows.Err(); err != nil {
		t.Fatal(err)
	}

	return results
}

func TestCreateDatabase(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	if _, err := db.Exec(`CREATE DATABASE foo`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`CREATE DATABASE foo`); !isError(err, "database .* already exists") {
		t.Fatalf("expected failure, but found success")
	}
	if _, err := db.Exec(`CREATE DATABASE IF NOT EXISTS foo`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`CREATE DATABASE ""`); !isError(err, "empty database name") {
		t.Fatal(err)
	}
}

func TestShowDatabases(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	names := []string{"a", "b", "c", "d", "e", "f", "g", "i"}
	for i, name := range names {
		if _, err := db.Exec("CREATE DATABASE " + name); err != nil {
			t.Fatal(err)
		}

		rows, err := db.Query("SHOW DATABASES")
		if err != nil {
			t.Fatal(err)
		}
		var databases []string
		for rows.Next() {
			var n string
			if err := rows.Scan(&n); err != nil {
				t.Fatal(err)
			}
			databases = append(databases, n)
		}

		expectedDatabases := names[:i+1]
		if !reflect.DeepEqual(expectedDatabases, databases) {
			t.Fatalf("expected %+v, but got %+v", expectedDatabases, databases)
		}
	}
}

func TestCreateTable(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}

	const cols = "(id INT PRIMARY KEY)"
	if _, err := db.Exec("CREATE TABLE users " + cols); !isError(err, "no database specified") {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE t.users " + cols); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE t.users " + cols); !isError(err, "table .* already exists") {
		t.Fatal(err)
	}

	if _, err := db.Exec("SET DATABASE = t"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE users " + cols); !isError(err, "table .* already exists") {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE TABLE IF NOT EXISTS users " + cols); err != nil {
		t.Fatal(err)
	}
}

func TestShowTables(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}

	names := []string{"a", "b", "c", "d", "e", "f", "g", "i"}
	for i, name := range names {
		if _, err := db.Exec("CREATE TABLE t." + name + " (id INT PRIMARY KEY)"); err != nil {
			t.Fatal(err)
		}

		rows, err := db.Query("SHOW TABLES FROM t")
		if err != nil {
			t.Fatal(err)
		}
		var tables []string
		for rows.Next() {
			var n string
			if err := rows.Scan(&n); err != nil {
				t.Fatal(err)
			}
			tables = append(tables, n)
		}

		expectedTables := names[:i+1]
		if !reflect.DeepEqual(expectedTables, tables) {
			t.Fatalf("expected %+v, but got %+v", expectedTables, tables)
		}
	}

	if _, err := db.Query("SHOW TABLES"); !isError(err, "no database specified") {
		t.Fatal(err)
	}
	if _, err := db.Exec("SET DATABASE = t"); err != nil {
		t.Fatal(err)
	}

	rows, err := db.Query("SHOW TABLES")
	if err != nil {
		t.Fatal(err)
	}
	if err := rows.Close(); err != nil {
		t.Fatal(err)
	}
}

func TestShowColumns(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	const schema = `
CREATE TABLE t.users (
  id    INT PRIMARY KEY,
  name  VARCHAR NOT NULL,
  title VARCHAR
)`

	if _, err := db.Query("SHOW COLUMNS FROM t.users"); !isError(err, "database .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Query("SHOW COLUMNS FROM t.users"); !isError(err, "table .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Query("SHOW COLUMNS FROM users"); !isError(err, "no database specified") {
		t.Fatal(err)
	}

	if _, err := db.Exec(schema); err != nil {
		t.Fatal(err)
	}

	rows, err := db.Query("SHOW COLUMNS FROM t.users")
	if err != nil {
		t.Fatal(err)
	}
	results := readAll(t, rows)
	expectedResults := [][]string{
		{"Field", "Type", "Null"},
		{"id", "INT", "true"},
		{"name", "CHAR", "false"},
		{"title", "CHAR", "true"},
	}
	if !reflect.DeepEqual(expectedResults, results) {
		t.Fatalf("expected %s, but got %s", expectedResults, results)
	}
}

func TestShowIndex(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	const schema = `
CREATE TABLE t.users (
  id    INT PRIMARY KEY,
  name  VARCHAR NOT NULL,
  CONSTRAINT foo INDEX (name),
  CONSTRAINT bar UNIQUE (id, name)
)`

	if _, err := db.Query("SHOW INDEX FROM t.users"); !isError(err, "database .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Query("SHOW INDEX FROM t.users"); !isError(err, "table .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Query("SHOW INDEX FROM users"); !isError(err, "no database specified") {
		t.Fatal(err)
	}

	if _, err := db.Exec(schema); err != nil {
		t.Fatal(err)
	}

	rows, err := db.Query("SHOW INDEX FROM t.users")
	if err != nil {
		t.Fatal(err)
	}
	results := readAll(t, rows)
	expectedResults := [][]string{
		{"Table", "Name", "Unique", "Seq", "Column"},
		{"users", "primary", "true", "1", "id"},
		{"users", "foo", "false", "1", "name"},
		{"users", "bar", "true", "1", "id"},
		{"users", "bar", "true", "2", "name"},
	}
	if !reflect.DeepEqual(expectedResults, results) {
		t.Fatalf("expected %s, but got %s", expectedResults, results)
	}
}

func TestInsertSelect(t *testing.T) {
	defer leaktest.AfterTest(t)
	s, db := setup(t)
	defer cleanup(s, db)

	schema := `
CREATE TABLE t.kv (
  k CHAR PRIMARY KEY,
  v CHAR
)`

	if _, err := db.Exec("CREATE DATABASE t"); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`INSERT INTO t.kv VALUES ('a', 'b')`); !isError(err, "table .* does not exist") {
		t.Fatal(err)
	}
	if _, err := db.Exec(schema); err != nil {
		t.Fatal(err)
	}
	if rows, err := db.Query(`SELECT * FROM t.kv`); err != nil {
		t.Fatal(err)
	} else if results := readAll(t, rows); len(results) > 1 {
		t.Fatalf("non-empty result set from empty table: %s", results)
	}
	if _, err := db.Exec(`INSERT INTO t.kv VALUES ('a')`); !isError(err, "invalid values for columns") {
		t.Fatal(err)
	}
	if _, err := db.Exec(`INSERT INTO t.kv (v) VALUES ('a')`); !isError(err, "missing .* primary key column") {
		t.Fatal(err)
	}
	if _, err := db.Exec(`INSERT INTO t.kv (k,v) VALUES ('a', 'b'), ('c', 'd')`); err != nil {
		t.Fatal(err)
	}
	if _, err := db.Exec(`INSERT INTO t.kv VALUES ('e', 'f')`); err != nil {
		t.Fatal(err)
	}

	rows, err := db.Query("SELECT * FROM t.kv")
	if err != nil {
		t.Fatal(err)
	}
	results := readAll(t, rows)
	expectedResults := [][]string{
		{"k", "v"},
		{"a", "b"},
		{"c", "d"},
		{"e", "f"},
	}
	if !reflect.DeepEqual(expectedResults, results) {
		t.Fatalf("expected %s, but got %s", expectedResults, results)
	}

	// TODO(pmattis): We need much more testing of WHERE clauses. Need to think
	// through the whole testing story in general.
	rows, err = db.Query("SELECT * FROM t.kv WHERE k IN ('a', 'c')")
	if err != nil {
		t.Fatal(err)
	}
	results = readAll(t, rows)
	expectedResults = [][]string{
		{"k", "v"},
		{"a", "b"},
		{"c", "d"},
	}
	if !reflect.DeepEqual(expectedResults, results) {
		t.Fatalf("expected %s, but got %s", expectedResults, results)
	}
}
<|endoftext|>


<commit_before>package datad

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
)

var NodeMembershipTTL = 5 * time.Second

// A Node ensures that the provider's keys are registered and coordinates
// distribution of data among the other nodes in the cluster.
type Node struct {
	Name     string
	Provider Provider

	backend  Backend
	registry *Registry

	Log *log.Logger

	stopChan chan struct{}
}

// NewNode creates a new node to publish data from a provider to the cluster.
// The name ("host:port") is advertised to the cluster and therefore must be
// accessible by the other clients and nodes in the cluster. The name should be
// the host and port where the data on this machine is accessible.
//
// Call Start on this node to begin publishing its keys to the cluster.
func NewNode(name string, b Backend, p Provider) *Node {
	name = cleanNodeName(name)
	return &Node{
		Name:     name,
		Provider: p,
		backend:  b,
		registry: NewRegistry(b),
		Log:      log.New(os.Stderr, "", log.Ltime|log.Lshortfile),
		stopChan: make(chan struct{}),
	}
}

func cleanNodeName(name string) string {
	name = strings.TrimPrefix(name, "http://")
	parseName := name
	if !strings.Contains(parseName, ":") {
		parseName += ":80"
	}
	_, _, err := net.SplitHostPort(parseName)
	if err != nil {
		panic("NewNode: bad name '" + name + "': " + err.Error() + " (name should be 'host:port')")
	}
	return name
}

// Start begins advertising this node's provider's keys to the
// cluster.
func (n *Node) Start() error {
	n.logf("Starting node %s.", n.Name)

	err := n.joinCluster()
	if err != nil {
		n.logf("Failed to join cluster: %s", err)
		return err
	}

	err = n.registerExistingKeys()
	if err != nil {
		n.logf("Failed to register existing keys: %s", err)
		return err
	}

	go n.watchRegisteredKeys()
	go n.balancePeriodically()

	return nil
}

// Stop deregisters this node's keys and stops background processes
// for this node.
func (n *Node) Stop() error {
	close(n.stopChan)
	return nil
}

// joinCluster adds this node's provider to the cluster, making it available to
// receive requests for and be assigned keys. It then periodically re-adds this
// node to the cluster before the TTL on the etcd cluster membership key
// elapses.
func (n *Node) joinCluster() error {
	err := n.refreshClusterMembership()
	if err != nil {
		return err
	}

	if NodeMembershipTTL < time.Second {
		panic("NodeMembershipTTL must be at least 1 second")
	}

	go func() {
		t := time.NewTicker(NodeMembershipTTL)
		for {
			select {
			case <-t.C:
				err := n.refreshClusterMembership()
				if err != nil {
					n.logf("Error refreshing node %s cluster membership: %s.", n.Name, err)
				}
			case <-n.stopChan:
				t.Stop()
				return
			}
		}
	}()

	return nil
}

func (n *Node) refreshClusterMembership() error {
	err := n.backend.SetDir(keyPathJoin(nodesPrefix, n.Name), uint64(NodeMembershipTTL/time.Second))
	if isEtcdErrorCode(err, 102) {
		err = n.backend.UpdateDir(keyPathJoin(nodesPrefix, n.Name), uint64(NodeMembershipTTL/time.Second))
	}
	return err
}

// watchRegisteredKeys watches the registry for changes to the list of keys that
// this node is registered for, or for modifications of existing registrations
// (e.g., updates requested).
func (n *Node) watchRegisteredKeys() error {
	watchKey := keysForNodeDir(n.Name)

	recv := make(chan *etcd.Response)
	stopWatch := make(chan bool)

	// Receive watched changes.
	go func() {
		for {
			select {
			case resp := <-recv:
				key := strings.TrimPrefix(resp.Node.Key, watchKey+"/")
				n.logf("Registry changed: %s on key %q.", resp.Action, key)
				if !strings.Contains(strings.ToLower(resp.Action), "delete") {
					n.logf("Updating key %q in data source (in response to registry %s).", key, resp.Action)
					err := n.Provider.Update(key)
					if err != nil {
						n.logf("Error updating key %q in data source: %s.", key, err)
					}
				}
			case <-n.stopChan:
				n.logf("Stopping registry watcher.")
				stopWatch <- true
				return
			}
		}
	}()

	_, err := n.backend.(*EtcdBackend).etcd.Watch(watchKey, 0, true, recv, stopWatch)
	if err != etcd.ErrWatchStoppedByUser {
		return err
	}
	return nil
}

// registerExistingKeys examines this node's provider's local storage for data
// and registers each data key it finds. This means that when the node starts
// up, it's immediately able to receive requests for the data it already has on
// disk. Without this, the cluster would not know that this node's provider has
// these keys.
func (n *Node) registerExistingKeys() error {
	keys, err := n.Provider.Keys("")
	if err != nil {
		return err
	}

	if len(keys) == 0 {
		return nil
	}

	n.logf("Found %d existing keys in provider: %v. Registering existing keys to this node...", len(keys), keys)
	for _, key := range keys {
		err := n.registry.Add(key, n.Name)
		if err != nil {
			return err
		}
	}
	n.logf("Finished registering existing %d keys to this node.", len(keys))

	return nil
}

// startBalancer starts a periodic process that balances the distribution of
// keys to nodes.
func (n *Node) balancePeriodically() {
	// TODO(sqs): allow tweaking the balance interval
	t := time.NewTicker(time.Minute)
	for {
		select {
		case <-t.C:
			err := n.balance()
			if err != nil {
				n.logf("Error balancing: %s. Will retry next balance interval.", err)
			}
		case <-n.stopChan:
			t.Stop()
			return
		}
	}
}

// balance examines all keys and ensures each key has a registered node. If not,
// it registers a node for the key. This lets the cluster heal itself after a
// node goes down (which causes keys to be orphaned).
func (n *Node) balance() error {
	keyMap, err := n.registry.KeyMap()
	if err != nil {
		return err
	}

	if len(keyMap) == 0 {
		return nil
	}

	c := NewClient(n.backend)
	clusterNodes, err := c.NodesInCluster()
	if err != nil {
		return err
	}

	// TODO(sqs): allow tweaking this parameter
	x := rand.Intn(10)
	start := time.Now()

	n.logf("Balancer: starting on %d keys, with known cluster nodes %v.", len(keyMap), clusterNodes)
	actions := 0
	for key, nodes := range keyMap {
		if len(nodes) == 0 {
			regNode := clusterNodes[keyBucket(key, len(clusterNodes))]

			n.logf("Balancer: found unregistered key %q; registering it to node %s.", key, regNode)

			// TODO(sqs): optimize this by only adding if not exists, and then
			// seeing if it exists (to avoid potentially duplicating work).
			err := c.registry.Add(key, regNode)
			if err != nil {
				return err
			}

			actions++
			continue
		}

		// Check liveness of key on each node.
		for _, node := range nodes {
			t, err := c.transportForKey(key, nil, []string{node})
			if err != nil {
				return err
			}
			resp, err := (&http.Client{Transport: t}).Get(slash(key))
			if err != nil {
				actions++
				n.logf("Balancer: liveness check failed for key %q on node %s: %s. Client deregistered key from node.", key, node, err)
			}
			if resp != nil {
				resp.Body.Close()
			}
		}

		// Update keys on this node (but not each time, to avoid overloading the
		// origin servers).
		if x == 0 {
			for _, node := range nodes {
				if node == n.Name {
					n.logf("Balancer: updating key %q in data source on current node.", key)
					err := n.Provider.Update(key)
					if err != nil {
						n.logf("Balancer: error updating key %q in data source on current node: %s. (Continuing to balance other keys.)", key, err)
					}
					actions++
				}
			}
		}
	}
	n.logf("Balancer: completed in %s for %d keys (%d non-read actions performed).", time.Since(start), len(keyMap), actions)

	return nil
}

func (n *Node) logf(format string, a ...interface{}) {
	if n.Log != nil {
		n.Log.Printf(fmt.Sprintf("Node %s: ", n.Name)+format, a...)
	}
}
<commit_msg>fix the key we use to watch (since we use the etcd impl)<commit_after>package datad

import (
	"fmt"
	"log"
	"math/rand"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/coreos/go-etcd/etcd"
)

var NodeMembershipTTL = 5 * time.Second

// A Node ensures that the provider's keys are registered and coordinates
// distribution of data among the other nodes in the cluster.
type Node struct {
	Name     string
	Provider Provider

	backend  Backend
	registry *Registry

	Log *log.Logger

	stopChan chan struct{}
}

// NewNode creates a new node to publish data from a provider to the cluster.
// The name ("host:port") is advertised to the cluster and therefore must be
// accessible by the other clients and nodes in the cluster. The name should be
// the host and port where the data on this machine is accessible.
//
// Call Start on this node to begin publishing its keys to the cluster.
func NewNode(name string, b Backend, p Provider) *Node {
	name = cleanNodeName(name)
	return &Node{
		Name:     name,
		Provider: p,
		backend:  b,
		registry: NewRegistry(b),
		Log:      log.New(os.Stderr, "", log.Ltime|log.Lshortfile),
		stopChan: make(chan struct{}),
	}
}

func cleanNodeName(name string) string {
	name = strings.TrimPrefix(name, "http://")
	parseName := name
	if !strings.Contains(parseName, ":") {
		parseName += ":80"
	}
	_, _, err := net.SplitHostPort(parseName)
	if err != nil {
		panic("NewNode: bad name '" + name + "': " + err.Error() + " (name should be 'host:port')")
	}
	return name
}

// Start begins advertising this node's provider's keys to the
// cluster.
func (n *Node) Start() error {
	n.logf("Starting node %s.", n.Name)

	err := n.joinCluster()
	if err != nil {
		n.logf("Failed to join cluster: %s", err)
		return err
	}

	err = n.registerExistingKeys()
	if err != nil {
		n.logf("Failed to register existing keys: %s", err)
		return err
	}

	go n.watchRegisteredKeys()
	go n.balancePeriodically()

	return nil
}

// Stop deregisters this node's keys and stops background processes
// for this node.
func (n *Node) Stop() error {
	close(n.stopChan)
	return nil
}

// joinCluster adds this node's provider to the cluster, making it available to
// receive requests for and be assigned keys. It then periodically re-adds this
// node to the cluster before the TTL on the etcd cluster membership key
// elapses.
func (n *Node) joinCluster() error {
	err := n.refreshClusterMembership()
	if err != nil {
		return err
	}

	if NodeMembershipTTL < time.Second {
		panic("NodeMembershipTTL must be at least 1 second")
	}

	go func() {
		t := time.NewTicker(NodeMembershipTTL)
		for {
			select {
			case <-t.C:
				err := n.refreshClusterMembership()
				if err != nil {
					n.logf("Error refreshing node %s cluster membership: %s.", n.Name, err)
				}
			case <-n.stopChan:
				t.Stop()
				return
			}
		}
	}()

	return nil
}

func (n *Node) refreshClusterMembership() error {
	err := n.backend.SetDir(keyPathJoin(nodesPrefix, n.Name), uint64(NodeMembershipTTL/time.Second))
	if isEtcdErrorCode(err, 102) {
		err = n.backend.UpdateDir(keyPathJoin(nodesPrefix, n.Name), uint64(NodeMembershipTTL/time.Second))
	}
	return err
}

// watchRegisteredKeys watches the registry for changes to the list of keys that
// this node is registered for, or for modifications of existing registrations
// (e.g., updates requested).
func (n *Node) watchRegisteredKeys() error {
	watchKey := keysForNodeDir(n.Name)
	fullKey := n.backend.(*EtcdBackend).fullKey(watchKey)

	recv := make(chan *etcd.Response, 10)
	stopWatch := make(chan bool, 1)

	// Receive watched changes.
	go func() {
		for {
			select {
			case resp, ok := <-recv:
				if !ok {
					return
				}
				key := strings.TrimPrefix(resp.Node.Key, fullKey+"/")
				n.logf("Registry changed: %s on key %q.", resp.Action, key)
				if !strings.Contains(strings.ToLower(resp.Action), "delete") {
					n.logf("Updating key %q in data source (in response to registry %s).", key, resp.Action)
					err := n.Provider.Update(key)
					if err != nil {
						n.logf("Error updating key %q in data source: %s.", key, err)
					}
				}
			case <-n.stopChan:
				n.logf("Stopping registry watcher.")
				stopWatch <- true
				return
			}
		}
	}()

	_, err := n.backend.(*EtcdBackend).etcd.Watch(fullKey, 0, true, recv, stopWatch)
	if err != etcd.ErrWatchStoppedByUser {
		return err
	}
	return nil
}

// registerExistingKeys examines this node's provider's local storage for data
// and registers each data key it finds. This means that when the node starts
// up, it's immediately able to receive requests for the data it already has on
// disk. Without this, the cluster would not know that this node's provider has
// these keys.
func (n *Node) registerExistingKeys() error {
	keys, err := n.Provider.Keys("")
	if err != nil {
		return err
	}

	if len(keys) == 0 {
		return nil
	}

	n.logf("Found %d existing keys in provider: %v. Registering existing keys to this node...", len(keys), keys)
	for _, key := range keys {
		err := n.registry.Add(key, n.Name)
		if err != nil {
			return err
		}
	}
	n.logf("Finished registering existing %d keys to this node.", len(keys))

	return nil
}

// startBalancer starts a periodic process that balances the distribution of
// keys to nodes.
func (n *Node) balancePeriodically() {
	// TODO(sqs): allow tweaking the balance interval
	t := time.NewTicker(time.Minute)
	for {
		select {
		case <-t.C:
			err := n.balance()
			if err != nil {
				n.logf("Error balancing: %s. Will retry next balance interval.", err)
			}
		case <-n.stopChan:
			t.Stop()
			return
		}
	}
}

// balance examines all keys and ensures each key has a registered node. If not,
// it registers a node for the key. This lets the cluster heal itself after a
// node goes down (which causes keys to be orphaned).
func (n *Node) balance() error {
	keyMap, err := n.registry.KeyMap()
	if err != nil {
		return err
	}

	if len(keyMap) == 0 {
		return nil
	}

	c := NewClient(n.backend)
	clusterNodes, err := c.NodesInCluster()
	if err != nil {
		return err
	}

	// TODO(sqs): allow tweaking this parameter
	x := rand.Intn(10)
	start := time.Now()

	n.logf("Balancer: starting on %d keys, with known cluster nodes %v.", len(keyMap), clusterNodes)
	actions := 0
	for key, nodes := range keyMap {
		if len(nodes) == 0 {
			regNode := clusterNodes[keyBucket(key, len(clusterNodes))]

			n.logf("Balancer: found unregistered key %q; registering it to node %s.", key, regNode)

			// TODO(sqs): optimize this by only adding if not exists, and then
			// seeing if it exists (to avoid potentially duplicating work).
			err := c.registry.Add(key, regNode)
			if err != nil {
				return err
			}

			actions++
			continue
		}

		// Check liveness of key on each node.
		for _, node := range nodes {
			t, err := c.transportForKey(key, nil, []string{node})
			if err != nil {
				return err
			}
			resp, err := (&http.Client{Transport: t}).Get(slash(key))
			if err != nil {
				actions++
				n.logf("Balancer: liveness check failed for key %q on node %s: %s. Client deregistered key from node.", key, node, err)
			}
			if resp != nil {
				resp.Body.Close()
			}
		}

		// Update keys on this node (but not each time, to avoid overloading the
		// origin servers).
		if x == 0 {
			for _, node := range nodes {
				if node == n.Name {
					n.logf("Balancer: updating key %q in data source on current node.", key)
					err := n.Provider.Update(key)
					if err != nil {
						n.logf("Balancer: error updating key %q in data source on current node: %s. (Continuing to balance other keys.)", key, err)
					}
					actions++
				}
			}
		}
	}
	n.logf("Balancer: completed in %s for %d keys (%d non-read actions performed).", time.Since(start), len(keyMap), actions)

	return nil
}

func (n *Node) logf(format string, a ...interface{}) {
	if n.Log != nil {
		n.Log.Printf(fmt.Sprintf("Node %s: ", n.Name)+format, a...)
	}
}
<|endoftext|>


<commit_before>package neurgo

import (
	"encoding/json"
	"log"
)

type Node struct {
	Name      string
	inbound   []*connection
	outbound  []*connection
	processor SignalProcessor
	closing   chan bool
	invisible bool
}

// continually propagate incoming signals -> outgoing signals
func (node *Node) Run() {

	// create the closing channel before goroutine kicked off
	// to solve a potential race condition where someone tries
	// to shutdown a not-yet-running node
	node.closing = make(chan bool)

	go node.runGoroutine()

}

func (node *Node) runGoroutine() {

	if node.processor == nil {
		log.Panicf("%v does not have a signal processor", node)
	}

	for {

		if node.hasBeenShutdown() {
			break
		}

		if node.processor.canPropagate(node) == false {
			log.Panicf("%v cannot propagate any signals", node)
		} else {
			isShutdown := node.processor.propagateSignal(node)
			if isShutdown {
				break
			}
		}

	}

	node.closing <- true // confirm that we are closed

}

func (node *Node) hasBeenShutdown() bool {
	select {
	case <-node.closing:
		return true
	default:
		return false
	}

}

func (node *Node) Shutdown() {
	node.closing <- true // tell node to close
	<-node.closing       // wait for confirmation
	close(node.closing)  // <-- TODO: do we actually need this?
}

func (node *Node) String() string {
	return node.Name
}

func (node *Node) scatterOutput(outputs []float64) {
	for _, outboundConnection := range node.outbound {
		select {
		case outboundConnection.channel <- outputs:
		case <-outboundConnection.closing:
			return
		case <-node.closing:
			return
		}

	}
}

func (node *Node) isInvisible() bool {
	return node.invisible
}

func (node *Node) setInvisible(val bool) {
	node.invisible = val
}

func (node *Node) hasOutboundConnectionTo(other *Node) bool {
	for _, outboundConnection := range node.outbound {
		if outboundConnection.other == other {
			return true
		}
	}
	return false
}

func (node *Node) ConnectBidirectional(target *Node) {
	node.ConnectBidirectionalWeighted(target, nil)
}

func (node *Node) ConnectBidirectionalWeighted(target *Node, weights []float64) {
	channel := make(VectorChannel)
	closing := make(chan bool)
	node.connectOutboundWithChannel(target, channel, closing)
	target.connectInboundWithChannel(node, channel, closing, weights)
}

func (node *Node) connectOutboundWithChannel(target *Node, channel VectorChannel, closing chan bool) {
	connection := &connection{
		channel: channel,
		other:   target,
		closing: closing,
	}
	node.outbound = append(node.outbound, connection)
}

func (node *Node) connectInboundWithChannel(source *Node, channel VectorChannel, closing chan bool, weights []float64) {
	connection := &connection{
		channel: channel,
		weights: weights,
		other:   source,
		closing: closing,
	}
	node.inbound = append(node.inbound, connection) // FIXME: data race #1
}

func (node *Node) DisconnectBidirectional(target *Node) {
	node.disconnectOutbound(target)
	target.disconnectInbound(node)
}

func (node *Node) disconnectOutbound(target *Node) {
	for i, connection := range node.outbound {
		if connection.other == target {
			close(node.outbound[i].closing)
			node.outbound = removeConnection(node.outbound, i)
		}
	}
}

func (node *Node) disconnectInbound(source *Node) {
	for i, connection := range node.inbound {
		if connection.other == source {
			node.inbound = removeConnection(node.inbound, i)
		}
	}
}

func removeConnection(connections []*connection, index int) []*connection {

	newConnections := make([]*connection, len(connections)-1)
	newConnectionsIndex := 0

	for i, connection := range connections {
		if i != index {
			newConnections[newConnectionsIndex] = connection
			newConnectionsIndex += 1
		}
	}
	return newConnections

}

func (node *Node) outboundConnections() []*connection {
	return node.outbound
}

func (node *Node) inboundConnections() []*connection {
	return node.inbound
}

func (node *Node) appendOutboundConnection(target *connection) {
	node.outbound = append(node.outbound, target)
}

func (node *Node) appendInboundConnection(source *connection) {
	node.inbound = append(node.inbound, source)
}

func (node *Node) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		struct {
			Type      string          `json:"type"`
			Name      string          `json:"name"`
			Outbound  []*connection   `json:"outbound"`
			Inbound   []*connection   `json:"inbound"`
			Processor SignalProcessor `json:"processor"`
		}{
			Type:      "Node",
			Name:      node.Name,
			Outbound:  node.outbound,
			Inbound:   node.inbound,
			Processor: node.processor,
		})
}
<commit_msg>remove old comment<commit_after>package neurgo

import (
	"encoding/json"
	"log"
)

type Node struct {
	Name      string
	inbound   []*connection
	outbound  []*connection
	processor SignalProcessor
	closing   chan bool
	invisible bool
}

// continually propagate incoming signals -> outgoing signals
func (node *Node) Run() {

	// create the closing channel before goroutine kicked off
	// to solve a potential race condition where someone tries
	// to shutdown a not-yet-running node
	node.closing = make(chan bool)

	go node.runGoroutine()

}

func (node *Node) runGoroutine() {

	if node.processor == nil {
		log.Panicf("%v does not have a signal processor", node)
	}

	for {

		if node.hasBeenShutdown() {
			break
		}

		if node.processor.canPropagate(node) == false {
			log.Panicf("%v cannot propagate any signals", node)
		} else {
			isShutdown := node.processor.propagateSignal(node)
			if isShutdown {
				break
			}
		}

	}

	node.closing <- true // confirm that we are closed

}

func (node *Node) hasBeenShutdown() bool {
	select {
	case <-node.closing:
		return true
	default:
		return false
	}

}

func (node *Node) Shutdown() {
	node.closing <- true // tell node to close
	<-node.closing       // wait for confirmation
	close(node.closing)  // <-- TODO: do we actually need this?
}

func (node *Node) String() string {
	return node.Name
}

func (node *Node) scatterOutput(outputs []float64) {
	for _, outboundConnection := range node.outbound {
		select {
		case outboundConnection.channel <- outputs:
		case <-outboundConnection.closing:
			return
		case <-node.closing:
			return
		}

	}
}

func (node *Node) isInvisible() bool {
	return node.invisible
}

func (node *Node) setInvisible(val bool) {
	node.invisible = val
}

func (node *Node) hasOutboundConnectionTo(other *Node) bool {
	for _, outboundConnection := range node.outbound {
		if outboundConnection.other == other {
			return true
		}
	}
	return false
}

func (node *Node) ConnectBidirectional(target *Node) {
	node.ConnectBidirectionalWeighted(target, nil)
}

func (node *Node) ConnectBidirectionalWeighted(target *Node, weights []float64) {
	channel := make(VectorChannel)
	closing := make(chan bool)
	node.connectOutboundWithChannel(target, channel, closing)
	target.connectInboundWithChannel(node, channel, closing, weights)
}

func (node *Node) connectOutboundWithChannel(target *Node, channel VectorChannel, closing chan bool) {
	connection := &connection{
		channel: channel,
		other:   target,
		closing: closing,
	}
	node.outbound = append(node.outbound, connection)
}

func (node *Node) connectInboundWithChannel(source *Node, channel VectorChannel, closing chan bool, weights []float64) {
	connection := &connection{
		channel: channel,
		weights: weights,
		other:   source,
		closing: closing,
	}
	node.inbound = append(node.inbound, connection)
}

func (node *Node) DisconnectBidirectional(target *Node) {
	node.disconnectOutbound(target)
	target.disconnectInbound(node)
}

func (node *Node) disconnectOutbound(target *Node) {
	for i, connection := range node.outbound {
		if connection.other == target {
			close(node.outbound[i].closing)
			node.outbound = removeConnection(node.outbound, i)
		}
	}
}

func (node *Node) disconnectInbound(source *Node) {
	for i, connection := range node.inbound {
		if connection.other == source {
			node.inbound = removeConnection(node.inbound, i)
		}
	}
}

func removeConnection(connections []*connection, index int) []*connection {

	newConnections := make([]*connection, len(connections)-1)
	newConnectionsIndex := 0

	for i, connection := range connections {
		if i != index {
			newConnections[newConnectionsIndex] = connection
			newConnectionsIndex += 1
		}
	}
	return newConnections

}

func (node *Node) outboundConnections() []*connection {
	return node.outbound
}

func (node *Node) inboundConnections() []*connection {
	return node.inbound
}

func (node *Node) appendOutboundConnection(target *connection) {
	node.outbound = append(node.outbound, target)
}

func (node *Node) appendInboundConnection(source *connection) {
	node.inbound = append(node.inbound, source)
}

func (node *Node) MarshalJSON() ([]byte, error) {
	return json.Marshal(
		struct {
			Type      string          `json:"type"`
			Name      string          `json:"name"`
			Outbound  []*connection   `json:"outbound"`
			Inbound   []*connection   `json:"inbound"`
			Processor SignalProcessor `json:"processor"`
		}{
			Type:      "Node",
			Name:      node.Name,
			Outbound:  node.outbound,
			Inbound:   node.inbound,
			Processor: node.processor,
		})
}
<|endoftext|>


<commit_before>package pitchfork

type PfDetail struct {
	Type        string
	DisplayName string
}

func (td *PfDetail) ToString() (out string) {
	out = td.Type + " " + td.DisplayName
	return
}

func DetailList() (details []PfDetail, err error) {
	q := "SELECT " +
		"type, " +
		"display_name " +
		"FROM member_detail_types " +
		"ORDER BY type"
	rows, err := DB.Query(q)

	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var detail PfDetail

		err = rows.Scan(&detail.Type, &detail.DisplayName)
		if err != nil {
			details = nil
			return
		}

		details = append(details, detail)
	}
	return
}

func detail_new(ctx PfCtx, args []string) (err error) {
	type_name := args[0]
	type_descr := args[1]

	/* Validate name */
	type_name, err = Chk_ident("Detail Type Name", type_name)
	if err != nil {
		return
	}

	q := "INSERT INTO member_detail_types " +
		"(type, display_name) " +
		"VALUES ($1, $2)"
	err = DB.Exec(ctx,
		"Added new Member Detail Type: ($1, $2)",
		1, q,
		type_name, type_descr)
	return
}

func detail_list(ctx PfCtx, args []string) (err error) {
	details, err := DetailList()
	if err != nil {
		return
	}

	ctx.OutLn("Detail\tDescription\n")

	for _, detail := range details {
		ctx.OutLn("%s", detail.ToString())
	}

	return
}

func detail_menu(ctx PfCtx, args []string) (err error) {
	menu := NewPfMenu([]PfMEntry{
		{"list", detail_list, 0, -1, nil, PERM_USER, "List details types"},
		{"new", detail_new, 2, 2, []string{"type_name", "description"}, PERM_SYS_ADMIN, "Add a new detail type"},
	})

	err = ctx.Menu(args, menu)
	return
}
<commit_msg>Add DetailType() + DetailCheck() to be able to filter detail types better before passing them to SQL<commit_after>package pitchfork

import (
	"errors"
	"strings"
)

type PfDetail struct {
	Type        string
	DisplayName string
}

func (td *PfDetail) ToString() (out string) {
	out = td.Type + " " + td.DisplayName
	return
}

func DetailType(detail string) (out string) {
	out = detail

	idx := strings.Index(detail, " ")
	if idx != -1 {
		out = detail[:idx]
	}

	return
}

func DetailCheck(detail string) (err error) {
	/* Verify that detail is a valid detail */
	details, err := DetailList()
	if err != nil {
		return
	}

	for _, d := range details {
		if d.Type == detail {
			err = nil
			return
		}
	}

	err = errors.New("Invalid detail type")
	return
}

func DetailList() (details []PfDetail, err error) {
	q := "SELECT " +
		"type, " +
		"display_name " +
		"FROM member_detail_types " +
		"ORDER BY type"
	rows, err := DB.Query(q)

	if err != nil {
		return
	}

	defer rows.Close()

	for rows.Next() {
		var detail PfDetail

		err = rows.Scan(&detail.Type, &detail.DisplayName)
		if err != nil {
			details = nil
			return
		}

		details = append(details, detail)
	}
	return
}

func detail_new(ctx PfCtx, args []string) (err error) {
	type_name := args[0]
	type_descr := args[1]

	/* Validate name */
	type_name, err = Chk_ident("Detail Type Name", type_name)
	if err != nil {
		return
	}

	q := "INSERT INTO member_detail_types " +
		"(type, display_name) " +
		"VALUES ($1, $2)"
	err = DB.Exec(ctx,
		"Added new Member Detail Type: ($1, $2)",
		1, q,
		type_name, type_descr)
	return
}

func detail_list(ctx PfCtx, args []string) (err error) {
	details, err := DetailList()
	if err != nil {
		return
	}

	ctx.OutLn("Detail\tDescription\n")

	for _, detail := range details {
		ctx.OutLn("%s", detail.ToString())
	}

	return
}

func detail_menu(ctx PfCtx, args []string) (err error) {
	menu := NewPfMenu([]PfMEntry{
		{"list", detail_list, 0, -1, nil, PERM_USER, "List details types"},
		{"new", detail_new, 2, 2, []string{"type_name", "description"}, PERM_SYS_ADMIN, "Add a new detail type"},
	})

	err = ctx.Menu(args, menu)
	return
}
<|endoftext|>


<commit_before>package goboots

import (
	"bytes"
	"net/http"
	"reflect"
	"text/template"
	//"time"
	"encoding/binary"
)

type IController interface {
	//Setup(app *App)
	Init(app *App)
	Run(w http.ResponseWriter, r *http.Request, params []string) interface{}
	PreFilter(w http.ResponseWriter, r *http.Request, params []string) interface{}
	// rendering
	Render(w http.ResponseWriter, r *http.Request, content interface{})
	_registerMethod(name string, method reflect.Value)
	_getMethod(name string) (reflect.Value, bool)
}

type PageContent struct {
	Title       string
	Content     string
	UserContent string
	HeadInclude string
	FootInclude string
}

type RedirectPageContent struct {
	Title   string
	Message string
	Path    string
}

type ErrorPageContent struct {
	Title        string
	ErrorTitle   string
	ErrorMessage string
	Content      string
}

type Controller struct {
	App           *App
	Path          string
	Name          string
	LayoutName    string
	Action        string
	LayoutAction  string
	Params        []string
	PageTitle     string
	Layout        string
	ContentType   string
	customMethods map[string]reflect.Value
}

func (c *Controller) Init(app *App) {
	c.App = app
	c.PageTitle = ""
	c.Layout = "default"
	c.ContentType = "text/html; charset=utf-8"
}

func (c *Controller) Run(w http.ResponseWriter, r *http.Request, params []string) interface{} {
	// www.example.com/params[0]/params[1]/(...)params[n]
	content := &PageContent{
		Title:   c.App.Config.Name + " - " + c.PageTitle,
		Content: "content goes here",
	}
	return content
}

func (c *Controller) PreFilter(w http.ResponseWriter, r *http.Request, params []string) interface{} {
	return nil
}

func (c *Controller) Render(w http.ResponseWriter, r *http.Request, content interface{}) {
	c.render(w, r, content, "")
}

func (c *Controller) RenderWithLayout(w http.ResponseWriter, r *http.Request, content interface{}, customLayout string) {
	c.render(w, r, content, customLayout)
}

func (c *Controller) render(w http.ResponseWriter, r *http.Request, content interface{}, customLayout string) {
	if len(c.Layout) == 0 && len(customLayout) == 0 {
		// no layout defined!
		var buff bytes.Buffer
		binary.Write(&buff, binary.LittleEndian, content)
		w.Write(buff.Bytes())
		return
	}
	layoutName := c.Layout
	if len(customLayout) > 0 {
		layoutName = customLayout
	}
	var layoutTpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		layoutTpl = c.App.GetLocalizedLayout(layoutName, w, r)
	} else {
		layoutTpl = c.App.GetLayout(layoutName)
	}
	layoutTpl.Execute(w, content)
}

// RenderFromCache renders an entire page to the app's bytecache
func (c *Controller) RenderToCache(w http.ResponseWriter, r *http.Request, name string, content interface{}) {
	if len(c.Layout) == 0 {
		// no layout defined!
		var buff bytes.Buffer
		binary.Write(&buff, binary.LittleEndian, content)
		c.App.ByteCaches.SetCache(name, buff.Bytes())
		return
	}
	var layoutTpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		layoutTpl = c.App.GetLocalizedLayout(c.Layout, w, r)
	} else {
		layoutTpl = c.App.GetLayout(c.Layout)
	}
	var buffer bytes.Buffer
	err := layoutTpl.Execute(&buffer, content)
	__panic(err)
	c.App.ByteCaches.SetCache(name, buffer.Bytes())
}

// RenderFromCache renders an entire page from the app's bytecache
func (c *Controller) RenderFromCache(w http.ResponseWriter, r *http.Request, name string) interface{} {
	w.Write(c.App.ByteCaches.GetCache(name).Content)
	return nil
}

func (c *Controller) ParseContent(w http.ResponseWriter, r *http.Request, locTemplatePath string, content interface{}) string {
	var tpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		tpl = c.App.GetLocalizedViewTemplate(locTemplatePath, w, r)
	} else {
		tpl = c.App.GetViewTemplate(locTemplatePath)
	}

	//TODO: handle error if template is nil
	var buffer bytes.Buffer
	err := tpl.Execute(&buffer, content)
	__panic(err)
	return buffer.String()
}

func (c *Controller) Redirect(w http.ResponseWriter, path string, title string, message string) interface{} {
	tpl := c.App.GetLayout("redirect")
	//TODO: handle error if template is nil
	content := &RedirectPageContent{
		Title:   title,
		Message: message,
		Path:    path,
	}
	tpl.Execute(w, content)
	return nil
}

func (c *Controller) PageError(w http.ResponseWriter, errorTitle string, errorMessage string, extraHTMLContent string) interface{} {
	tpl := c.App.GetLayout("error")
	//TODO: handle error if template is nil
	content := &ErrorPageContent{
		Title:        c.App.Config.Name + " - Error: " + errorTitle,
		ErrorTitle:   errorTitle,
		ErrorMessage: errorMessage,
		Content:      extraHTMLContent,
	}
	tpl.Execute(w, content)
	return nil
}

func (c *Controller) _registerMethod(name string, method reflect.Value) {
	if c.customMethods == nil {
		c.customMethods = make(map[string]reflect.Value, 0)
	}
	c.customMethods[name] = method
}
func (c *Controller) _getMethod(name string) (reflect.Value, bool) {
	if c.customMethods == nil {
		return reflect.Value{}, false
	}
	val, ok := c.customMethods[name]
	return val, ok
}
<commit_msg>ParseContent w and r now optional<commit_after>package goboots

import (
	"bytes"
	"net/http"
	"reflect"
	"text/template"
	//"time"
	"encoding/binary"
)

type IController interface {
	//Setup(app *App)
	Init(app *App)
	Run(w http.ResponseWriter, r *http.Request, params []string) interface{}
	PreFilter(w http.ResponseWriter, r *http.Request, params []string) interface{}
	// rendering
	Render(w http.ResponseWriter, r *http.Request, content interface{})
	_registerMethod(name string, method reflect.Value)
	_getMethod(name string) (reflect.Value, bool)
}

type PageContent struct {
	Title       string
	Content     string
	UserContent string
	HeadInclude string
	FootInclude string
}

type RedirectPageContent struct {
	Title   string
	Message string
	Path    string
}

type ErrorPageContent struct {
	Title        string
	ErrorTitle   string
	ErrorMessage string
	Content      string
}

type Controller struct {
	App           *App
	Path          string
	Name          string
	LayoutName    string
	Action        string
	LayoutAction  string
	Params        []string
	PageTitle     string
	Layout        string
	ContentType   string
	customMethods map[string]reflect.Value
}

func (c *Controller) Init(app *App) {
	c.App = app
	c.PageTitle = ""
	c.Layout = "default"
	c.ContentType = "text/html; charset=utf-8"
}

func (c *Controller) Run(w http.ResponseWriter, r *http.Request, params []string) interface{} {
	// www.example.com/params[0]/params[1]/(...)params[n]
	content := &PageContent{
		Title:   c.App.Config.Name + " - " + c.PageTitle,
		Content: "content goes here",
	}
	return content
}

func (c *Controller) PreFilter(w http.ResponseWriter, r *http.Request, params []string) interface{} {
	return nil
}

func (c *Controller) Render(w http.ResponseWriter, r *http.Request, content interface{}) {
	c.render(w, r, content, "")
}

func (c *Controller) RenderWithLayout(w http.ResponseWriter, r *http.Request, content interface{}, customLayout string) {
	c.render(w, r, content, customLayout)
}

func (c *Controller) render(w http.ResponseWriter, r *http.Request, content interface{}, customLayout string) {
	if len(c.Layout) == 0 && len(customLayout) == 0 {
		// no layout defined!
		var buff bytes.Buffer
		binary.Write(&buff, binary.LittleEndian, content)
		w.Write(buff.Bytes())
		return
	}
	layoutName := c.Layout
	if len(customLayout) > 0 {
		layoutName = customLayout
	}
	var layoutTpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		layoutTpl = c.App.GetLocalizedLayout(layoutName, w, r)
	} else {
		layoutTpl = c.App.GetLayout(layoutName)
	}
	layoutTpl.Execute(w, content)
}

// RenderFromCache renders an entire page to the app's bytecache
func (c *Controller) RenderToCache(w http.ResponseWriter, r *http.Request, name string, content interface{}) {
	if len(c.Layout) == 0 {
		// no layout defined!
		var buff bytes.Buffer
		binary.Write(&buff, binary.LittleEndian, content)
		c.App.ByteCaches.SetCache(name, buff.Bytes())
		return
	}
	var layoutTpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 {
		layoutTpl = c.App.GetLocalizedLayout(c.Layout, w, r)
	} else {
		layoutTpl = c.App.GetLayout(c.Layout)
	}
	var buffer bytes.Buffer
	err := layoutTpl.Execute(&buffer, content)
	__panic(err)
	c.App.ByteCaches.SetCache(name, buffer.Bytes())
}

// RenderFromCache renders an entire page from the app's bytecache
func (c *Controller) RenderFromCache(w http.ResponseWriter, r *http.Request, name string) interface{} {
	w.Write(c.App.ByteCaches.GetCache(name).Content)
	return nil
}

func (c *Controller) ParseContent(w http.ResponseWriter, r *http.Request, locTemplatePath string, content interface{}) string {
	var tpl *template.Template
	if len(c.App.Config.DefaultLanguage) > 0 && w != nil && r != nil {
		tpl = c.App.GetLocalizedViewTemplate(locTemplatePath, w, r)
	} else {
		tpl = c.App.GetViewTemplate(locTemplatePath)
	}

	//TODO: handle error if template is nil
	var buffer bytes.Buffer
	err := tpl.Execute(&buffer, content)
	__panic(err)
	return buffer.String()
}

func (c *Controller) Redirect(w http.ResponseWriter, path string, title string, message string) interface{} {
	tpl := c.App.GetLayout("redirect")
	//TODO: handle error if template is nil
	content := &RedirectPageContent{
		Title:   title,
		Message: message,
		Path:    path,
	}
	tpl.Execute(w, content)
	return nil
}

func (c *Controller) PageError(w http.ResponseWriter, errorTitle string, errorMessage string, extraHTMLContent string) interface{} {
	tpl := c.App.GetLayout("error")
	//TODO: handle error if template is nil
	content := &ErrorPageContent{
		Title:        c.App.Config.Name + " - Error: " + errorTitle,
		ErrorTitle:   errorTitle,
		ErrorMessage: errorMessage,
		Content:      extraHTMLContent,
	}
	tpl.Execute(w, content)
	return nil
}

func (c *Controller) _registerMethod(name string, method reflect.Value) {
	if c.customMethods == nil {
		c.customMethods = make(map[string]reflect.Value, 0)
	}
	c.customMethods[name] = method
}
func (c *Controller) _getMethod(name string) (reflect.Value, bool) {
	if c.customMethods == nil {
		return reflect.Value{}, false
	}
	val, ok := c.customMethods[name]
	return val, ok
}
<|endoftext|>


<commit_before>package triggerbuild

import (
	"log"
	"net/http"

	"github.com/winston-ci/winston/builder"
	"github.com/winston-ci/winston/jobs"
)

type handler struct {
	jobs    map[string]jobs.Job
	builder builder.Builder
}

func NewHandler(jobs map[string]jobs.Job, builder builder.Builder) http.Handler {
	return &handler{
		jobs:    jobs,
		builder: builder,
	}
}

func (handler *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	job, found := handler.jobs[r.FormValue(":job")]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	log.Println("triggering", job)

	_, err := handler.builder.Build(job)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	w.WriteHeader(http.StatusCreated)
}
<commit_msg>redirect to new build when triggered<commit_after>package triggerbuild

import (
	"fmt"
	"log"
	"net/http"

	"github.com/tedsuo/router"

	"github.com/winston-ci/winston/builder"
	"github.com/winston-ci/winston/jobs"
	"github.com/winston-ci/winston/server/routes"
)

type handler struct {
	jobs    map[string]jobs.Job
	builder builder.Builder
}

func NewHandler(jobs map[string]jobs.Job, builder builder.Builder) http.Handler {
	return &handler{
		jobs:    jobs,
		builder: builder,
	}
}

func (handler *handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	job, found := handler.jobs[r.FormValue(":job")]
	if !found {
		w.WriteHeader(http.StatusNotFound)
		return
	}

	log.Println("triggering", job)

	build, err := handler.builder.Build(job)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	redirectPath, err := routes.Routes.PathForHandler(routes.GetBuild, router.Params{
		"job":   job.Name,
		"build": fmt.Sprintf("%d", build.ID),
	})
	if err != nil {
		log.Fatalln("failed to construct redirect uri:", err)
	}

	http.Redirect(w, r, redirectPath, 302)
}
<|endoftext|>


<commit_before>package simplestreams

import (
	"fmt"
	"strings"
	"time"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/osarch"
)

// Products represents the base of download.json
type Products struct {
	ContentID string             `json:"content_id"`
	DataType  string             `json:"datatype"`
	Format    string             `json:"format"`
	License   string             `json:"license,omitempty"`
	Products  map[string]Product `json:"products"`
	Updated   string             `json:"updated,omitempty"`
}

// Product represents a single product inside download.json
type Product struct {
	Aliases         string                    `json:"aliases"`
	Architecture    string                    `json:"arch"`
	OperatingSystem string                    `json:"os"`
	Release         string                    `json:"release"`
	ReleaseCodename string                    `json:"release_codename,omitempty"`
	ReleaseTitle    string                    `json:"release_title"`
	Supported       bool                      `json:"supported,omitempty"`
	SupportedEOL    string                    `json:"support_eol,omitempty"`
	Version         string                    `json:"version,omitempty"`
	Versions        map[string]ProductVersion `json:"versions"`

	// Non-standard fields (only used on some image servers).
	Variant string `json:"variant,omitempty"`
}

// ProductVersion represents a particular version of a product
type ProductVersion struct {
	Items      map[string]ProductVersionItem `json:"items"`
	Label      string                        `json:"label,omitempty"`
	PublicName string                        `json:"pubname,omitempty"`
}

// ProductVersionItem represents a file/item of a particular ProductVersion
type ProductVersionItem struct {
	LXDHashSha256DiskImg     string `json:"combined_disk1-img_sha256,omitempty"`
	LXDHashSha256DiskKvmImg  string `json:"combined_disk-kvm-img_sha256,omitempty"`
	LXDHashSha256DiskUefiImg string `json:"combined_uefi1-img_sha256,omitempty"`
	LXDHashSha256RootXz      string `json:"combined_rootxz_sha256,omitempty"`
	LXDHashSha256            string `json:"combined_sha256,omitempty"`
	LXDHashSha256SquashFs    string `json:"combined_squashfs_sha256,omitempty"`
	FileType                 string `json:"ftype"`
	HashMd5                  string `json:"md5,omitempty"`
	Path                     string `json:"path"`
	HashSha256               string `json:"sha256,omitempty"`
	Size                     int64  `json:"size"`
	DeltaBase                string `json:"delta_base,omitempty"`
}

// ToLXD converts the products data into a list of LXD images and associated downloadable files
func (s *Products) ToLXD() ([]api.Image, map[string][][]string) {
	downloads := map[string][][]string{}

	images := []api.Image{}
	nameLayout := "20060102"
	eolLayout := "2006-01-02"

	for _, product := range s.Products {
		// Skip unsupported architectures
		architecture, err := osarch.ArchitectureId(product.Architecture)
		if err != nil {
			continue
		}

		architectureName, err := osarch.ArchitectureName(architecture)
		if err != nil {
			continue
		}

		for name, version := range product.Versions {
			// Short of anything better, use the name as date (see format above)
			if len(name) < 8 {
				continue
			}

			creationDate, err := time.Parse(nameLayout, name[0:8])
			if err != nil {
				continue
			}

			// Image processing function
			addImage := func(meta *ProductVersionItem, root *ProductVersionItem) error {
				// Look for deltas (only on squashfs)
				deltas := []ProductVersionItem{}
				if root != nil && root.FileType == "squashfs" {
					for _, item := range version.Items {
						if item.FileType == "squashfs.vcdiff" {
							deltas = append(deltas, item)
						}
					}
				}

				// Figure out the fingerprint
				fingerprint := ""
				if root != nil {
					if root.FileType == "root.tar.xz" {
						if meta.LXDHashSha256RootXz != "" {
							fingerprint = meta.LXDHashSha256RootXz
						} else {
							fingerprint = meta.LXDHashSha256
						}
					} else if root.FileType == "squashfs" {
						fingerprint = meta.LXDHashSha256SquashFs
					} else if root.FileType == "disk-kvm.img" {
						fingerprint = meta.LXDHashSha256DiskKvmImg
					} else if root.FileType == "disk1.img" {
						fingerprint = meta.LXDHashSha256DiskImg
					} else if root.FileType == "uefi1.img" {
						fingerprint = meta.LXDHashSha256DiskUefiImg
					}
				} else {
					fingerprint = meta.HashSha256
				}

				if fingerprint == "" {
					return fmt.Errorf("No LXD image fingerprint found")
				}

				// Figure out the size
				size := meta.Size
				if root != nil {
					size += root.Size
				}

				// Determine filename
				if meta.Path == "" {
					return fmt.Errorf("Missing path field on metadata entry")
				}

				fields := strings.Split(meta.Path, "/")
				filename := fields[len(fields)-1]

				// Generate the actual image entry
				description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture)
				if version.Label != "" {
					description = fmt.Sprintf("%s (%s)", description, version.Label)
				}
				description = fmt.Sprintf("%s (%s)", description, name)

				image := api.Image{}
				image.Architecture = architectureName
				image.Public = true
				image.Size = size
				image.CreatedAt = creationDate
				image.UploadedAt = creationDate
				image.Filename = filename
				image.Fingerprint = fingerprint
				image.Properties = map[string]string{
					"os":           product.OperatingSystem,
					"release":      product.Release,
					"version":      product.Version,
					"architecture": product.Architecture,
					"label":        version.Label,
					"serial":       name,
					"description":  description,
				}

				if product.Variant != "" {
					image.Properties["variant"] = product.Variant
				}

				image.Type = "container"

				if root != nil {
					image.Properties["type"] = root.FileType
					if root.FileType == "disk1.img" || root.FileType == "disk-kvm.img" || root.FileType == "uefi1.img" {
						image.Type = "virtual-machine"
					}
				} else {
					image.Properties["type"] = "tar.gz"
				}

				// Clear unset properties
				for k, v := range image.Properties {
					if v == "" {
						delete(image.Properties, k)
					}
				}

				// Add the provided aliases
				if product.Aliases != "" {
					image.Aliases = []api.ImageAlias{}
					for _, entry := range strings.Split(product.Aliases, ",") {
						image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry})
					}
				}

				// Attempt to parse the EOL
				image.ExpiresAt = time.Unix(0, 0).UTC()
				if product.SupportedEOL != "" {
					eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
					if err == nil {
						image.ExpiresAt = eolDate
					}
				}

				// Set the file list
				var imgDownloads [][]string
				if root == nil {
					imgDownloads = [][]string{{meta.Path, meta.HashSha256, "meta", fmt.Sprintf("%d", meta.Size)}}
				} else {
					imgDownloads = [][]string{
						{meta.Path, meta.HashSha256, "meta", fmt.Sprintf("%d", meta.Size)},
						{root.Path, root.HashSha256, "root", fmt.Sprintf("%d", root.Size)}}
				}

				// Add the deltas
				for _, delta := range deltas {
					srcImage, ok := product.Versions[delta.DeltaBase]
					if !ok {
						// Delta for a since expired image
						continue
					}

					// Locate source image fingerprint
					var srcFingerprint string
					for _, item := range srcImage.Items {
						if item.FileType != "lxd.tar.xz" {
							continue
						}

						srcFingerprint = item.LXDHashSha256SquashFs
						break
					}

					if srcFingerprint == "" {
						// Couldn't find the image
						continue
					}

					// Add the delta
					imgDownloads = append(imgDownloads, []string{
						delta.Path,
						delta.HashSha256,
						fmt.Sprintf("root.delta-%s", srcFingerprint),
						fmt.Sprintf("%d", delta.Size)})
				}

				// Add the image
				downloads[fingerprint] = imgDownloads
				images = append(images, image)

				return nil
			}

			// Locate a valid LXD image
			for _, item := range version.Items {
				if item.FileType == "lxd_combined.tar.gz" {
					err := addImage(&item, nil)
					if err != nil {
						continue
					}
				}

				if item.FileType == "lxd.tar.xz" {
					// Locate the root files
					for _, subItem := range version.Items {
						if shared.StringInSlice(subItem.FileType, []string{"disk1.img", "disk-kvm.img", "uefi1.img", "root.tar.xz", "squashfs"}) {
							err := addImage(&item, &subItem)
							if err != nil {
								continue
							}
						}
					}
				}
			}
		}
	}

	return images, downloads
}
<commit_msg>shared/simplestreams: Support image reqs metadata<commit_after>package simplestreams

import (
	"fmt"
	"strings"
	"time"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/osarch"
)

// Products represents the base of download.json
type Products struct {
	ContentID string             `json:"content_id"`
	DataType  string             `json:"datatype"`
	Format    string             `json:"format"`
	License   string             `json:"license,omitempty"`
	Products  map[string]Product `json:"products"`
	Updated   string             `json:"updated,omitempty"`
}

// Product represents a single product inside download.json
type Product struct {
	Aliases         string                    `json:"aliases"`
	Architecture    string                    `json:"arch"`
	OperatingSystem string                    `json:"os"`
	LXDRequirements map[string]string         `json:"lxd_requirements,omitempty"`
	Release         string                    `json:"release"`
	ReleaseCodename string                    `json:"release_codename,omitempty"`
	ReleaseTitle    string                    `json:"release_title"`
	Supported       bool                      `json:"supported,omitempty"`
	SupportedEOL    string                    `json:"support_eol,omitempty"`
	Version         string                    `json:"version,omitempty"`
	Versions        map[string]ProductVersion `json:"versions"`

	// Non-standard fields (only used on some image servers).
	Variant string `json:"variant,omitempty"`
}

// ProductVersion represents a particular version of a product
type ProductVersion struct {
	Items      map[string]ProductVersionItem `json:"items"`
	Label      string                        `json:"label,omitempty"`
	PublicName string                        `json:"pubname,omitempty"`
}

// ProductVersionItem represents a file/item of a particular ProductVersion
type ProductVersionItem struct {
	LXDHashSha256DiskImg     string `json:"combined_disk1-img_sha256,omitempty"`
	LXDHashSha256DiskKvmImg  string `json:"combined_disk-kvm-img_sha256,omitempty"`
	LXDHashSha256DiskUefiImg string `json:"combined_uefi1-img_sha256,omitempty"`
	LXDHashSha256RootXz      string `json:"combined_rootxz_sha256,omitempty"`
	LXDHashSha256            string `json:"combined_sha256,omitempty"`
	LXDHashSha256SquashFs    string `json:"combined_squashfs_sha256,omitempty"`
	FileType                 string `json:"ftype"`
	HashMd5                  string `json:"md5,omitempty"`
	Path                     string `json:"path"`
	HashSha256               string `json:"sha256,omitempty"`
	Size                     int64  `json:"size"`
	DeltaBase                string `json:"delta_base,omitempty"`
}

// ToLXD converts the products data into a list of LXD images and associated downloadable files
func (s *Products) ToLXD() ([]api.Image, map[string][][]string) {
	downloads := map[string][][]string{}

	images := []api.Image{}
	nameLayout := "20060102"
	eolLayout := "2006-01-02"

	for _, product := range s.Products {
		// Skip unsupported architectures
		architecture, err := osarch.ArchitectureId(product.Architecture)
		if err != nil {
			continue
		}

		architectureName, err := osarch.ArchitectureName(architecture)
		if err != nil {
			continue
		}

		for name, version := range product.Versions {
			// Short of anything better, use the name as date (see format above)
			if len(name) < 8 {
				continue
			}

			creationDate, err := time.Parse(nameLayout, name[0:8])
			if err != nil {
				continue
			}

			// Image processing function
			addImage := func(meta *ProductVersionItem, root *ProductVersionItem) error {
				// Look for deltas (only on squashfs)
				deltas := []ProductVersionItem{}
				if root != nil && root.FileType == "squashfs" {
					for _, item := range version.Items {
						if item.FileType == "squashfs.vcdiff" {
							deltas = append(deltas, item)
						}
					}
				}

				// Figure out the fingerprint
				fingerprint := ""
				if root != nil {
					if root.FileType == "root.tar.xz" {
						if meta.LXDHashSha256RootXz != "" {
							fingerprint = meta.LXDHashSha256RootXz
						} else {
							fingerprint = meta.LXDHashSha256
						}
					} else if root.FileType == "squashfs" {
						fingerprint = meta.LXDHashSha256SquashFs
					} else if root.FileType == "disk-kvm.img" {
						fingerprint = meta.LXDHashSha256DiskKvmImg
					} else if root.FileType == "disk1.img" {
						fingerprint = meta.LXDHashSha256DiskImg
					} else if root.FileType == "uefi1.img" {
						fingerprint = meta.LXDHashSha256DiskUefiImg
					}
				} else {
					fingerprint = meta.HashSha256
				}

				if fingerprint == "" {
					return fmt.Errorf("No LXD image fingerprint found")
				}

				// Figure out the size
				size := meta.Size
				if root != nil {
					size += root.Size
				}

				// Determine filename
				if meta.Path == "" {
					return fmt.Errorf("Missing path field on metadata entry")
				}

				fields := strings.Split(meta.Path, "/")
				filename := fields[len(fields)-1]

				// Generate the actual image entry
				description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture)
				if version.Label != "" {
					description = fmt.Sprintf("%s (%s)", description, version.Label)
				}
				description = fmt.Sprintf("%s (%s)", description, name)

				image := api.Image{}
				image.Architecture = architectureName
				image.Public = true
				image.Size = size
				image.CreatedAt = creationDate
				image.UploadedAt = creationDate
				image.Filename = filename
				image.Fingerprint = fingerprint
				image.Properties = map[string]string{
					"os":           product.OperatingSystem,
					"release":      product.Release,
					"version":      product.Version,
					"architecture": product.Architecture,
					"label":        version.Label,
					"serial":       name,
					"description":  description,
				}

				for lxdReq, lxdReqVal := range product.LXDRequirements {
					image.Properties["requirements."+lxdReq] = lxdReqVal
				}

				if product.Variant != "" {
					image.Properties["variant"] = product.Variant
				}

				image.Type = "container"

				if root != nil {
					image.Properties["type"] = root.FileType
					if root.FileType == "disk1.img" || root.FileType == "disk-kvm.img" || root.FileType == "uefi1.img" {
						image.Type = "virtual-machine"
					}
				} else {
					image.Properties["type"] = "tar.gz"
				}

				// Clear unset properties
				for k, v := range image.Properties {
					if v == "" {
						delete(image.Properties, k)
					}
				}

				// Add the provided aliases
				if product.Aliases != "" {
					image.Aliases = []api.ImageAlias{}
					for _, entry := range strings.Split(product.Aliases, ",") {
						image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry})
					}
				}

				// Attempt to parse the EOL
				image.ExpiresAt = time.Unix(0, 0).UTC()
				if product.SupportedEOL != "" {
					eolDate, err := time.Parse(eolLayout, product.SupportedEOL)
					if err == nil {
						image.ExpiresAt = eolDate
					}
				}

				// Set the file list
				var imgDownloads [][]string
				if root == nil {
					imgDownloads = [][]string{{meta.Path, meta.HashSha256, "meta", fmt.Sprintf("%d", meta.Size)}}
				} else {
					imgDownloads = [][]string{
						{meta.Path, meta.HashSha256, "meta", fmt.Sprintf("%d", meta.Size)},
						{root.Path, root.HashSha256, "root", fmt.Sprintf("%d", root.Size)}}
				}

				// Add the deltas
				for _, delta := range deltas {
					srcImage, ok := product.Versions[delta.DeltaBase]
					if !ok {
						// Delta for a since expired image
						continue
					}

					// Locate source image fingerprint
					var srcFingerprint string
					for _, item := range srcImage.Items {
						if item.FileType != "lxd.tar.xz" {
							continue
						}

						srcFingerprint = item.LXDHashSha256SquashFs
						break
					}

					if srcFingerprint == "" {
						// Couldn't find the image
						continue
					}

					// Add the delta
					imgDownloads = append(imgDownloads, []string{
						delta.Path,
						delta.HashSha256,
						fmt.Sprintf("root.delta-%s", srcFingerprint),
						fmt.Sprintf("%d", delta.Size)})
				}

				// Add the image
				downloads[fingerprint] = imgDownloads
				images = append(images, image)

				return nil
			}

			// Locate a valid LXD image
			for _, item := range version.Items {
				if item.FileType == "lxd_combined.tar.gz" {
					err := addImage(&item, nil)
					if err != nil {
						continue
					}
				}

				if item.FileType == "lxd.tar.xz" {
					// Locate the root files
					for _, subItem := range version.Items {
						if shared.StringInSlice(subItem.FileType, []string{"disk1.img", "disk-kvm.img", "uefi1.img", "root.tar.xz", "squashfs"}) {
							err := addImage(&item, &subItem)
							if err != nil {
								continue
							}
						}
					}
				}
			}
		}
	}

	return images, downloads
}
<|endoftext|>