<commit_before>// +build linux

/*
 * Copyright (C) 2018 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package lldp

import (
	"fmt"
	"net"
	"sync"
	"sync/atomic"
	"syscall"
	"unsafe"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/safchain/ethtool"
	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/flow/probes"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/topology"
	"github.com/skydive-project/skydive/topology/graph"
	"golang.org/x/sys/unix"
)

/*
#include <linux/if_packet.h>  // packet_mreq
#include <linux/if_ether.h>  // ETH_ALEN
*/
import "C"

const lldpBPFFilter = `ether[0] & 1 = 1 and
  ether proto 0x88cc and
  !(ether src %s) and
  (ether dst 01:80:c2:00:00:0e or
   ether dst 01:80:c2:00:00:03 or
   ether dst 01:80:c2:00:00:00)`

// Capture 8192 bytes so that we have the full Ethernet frame
const lldpSnapLen = 8192

// Probe describes the probe that is in charge of listening for
// LLDP packets on interfaces and create the corresponding chassis and port nodes
type Probe struct {
	sync.RWMutex
	graph.DefaultGraphListener
	g             *graph.Graph
	hostNode      *graph.Node     // graph node of the running host
	interfaceMap  map[string]bool // map interface names to the capturing state
	state         int64           // state of the probe (running or stopped)
	wg            sync.WaitGroup  // capture goroutines wait group
	autoDiscovery bool            // capture LLDP traffic on all capable interfaces
}

type ifreq struct {
	ifrName   [ethtool.IFNAMSIZ]byte
	ifrHwaddr syscall.RawSockaddr
}

// addMulticastAddr adds a multicast address to an interface using an ioctl call
func addMulticastAddr(intf string, addr string) error {
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0)
	if err != nil {
		return err
	}
	defer syscall.Close(fd)

	var name [ethtool.IFNAMSIZ]byte
	copy(name[:], []byte(intf))

	mac, _ := net.ParseMAC(addr)

	var sockaddr syscall.RawSockaddr
	// RawSockaddr.Data is []int8 on amd64 and []uint8 on ppc64le
	switch data := interface{}(sockaddr.Data).(type) {
	case [14]int8:
		for i, n := range mac {
			data[i] = int8(n)
		}
	case [14]uint8:
		for i, n := range mac {
			data[i] = uint8(n)
		}
	}

	ifr := &ifreq{
		ifrName:   name,
		ifrHwaddr: sockaddr,
	}

	_, _, ep := unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
		unix.SIOCADDMULTI, uintptr(unsafe.Pointer(ifr)))

	if ep != 0 {
		return syscall.Errno(ep)
	}
	return nil
}

func (p *Probe) handlePacket(n *graph.Node, packet gopacket.Packet) {
	lldpLayer := packet.Layer(layers.LayerTypeLinkLayerDiscovery)
	if lldpLayer != nil {
		lldpLayer := lldpLayer.(*layers.LinkLayerDiscovery)

		chassisMetadata := graph.Metadata{
			"LLDP": map[string]interface{}{
				"ChassisIDType": lldpLayer.ChassisID.Subtype.String(),
			},
			"Type":  "switch",
			"Probe": "lldp",
		}

		var chassisID string
		switch lldpLayer.ChassisID.Subtype {
		case layers.LLDPChassisIDSubTypeMACAddr:
			chassisID = net.HardwareAddr(lldpLayer.ChassisID.ID).String()
		default:
			chassisID = string(lldpLayer.ChassisID.ID)
		}
		common.SetField(chassisMetadata, "LLDP.ChassisID", chassisID)
		common.SetField(chassisMetadata, "Name", chassisID)

		portMetadata := graph.Metadata{
			"LLDP": map[string]interface{}{
				"PortIDType": lldpLayer.PortID.Subtype.String(),
			},
			"Type":  "switchport",
			"Probe": "lldp",
		}

		var portID string
		switch lldpLayer.PortID.Subtype {
		case layers.LLDPPortIDSubtypeMACAddr:
			portID = net.HardwareAddr(lldpLayer.PortID.ID).String()
		default:
			portID = string(lldpLayer.PortID.ID)
		}
		common.SetField(portMetadata, "LLDP.PortID", portID)
		common.SetField(portMetadata, "Name", portID)

		if lldpLayerInfo := packet.Layer(layers.LayerTypeLinkLayerDiscoveryInfo); lldpLayerInfo != nil {
			lldpLayerInfo := lldpLayerInfo.(*layers.LinkLayerDiscoveryInfo)

			if lldpLayerInfo.PortDescription != "" {
				common.SetField(portMetadata, "LLDP.Description", lldpLayerInfo.PortDescription)
				portMetadata["Name"] = lldpLayerInfo.PortDescription
			}

			if lldpLayerInfo.SysDescription != "" {
				common.SetField(chassisMetadata, "LLDP.Description", lldpLayerInfo.SysDescription)
			}

			if lldpLayerInfo.SysName != "" {
				common.SetField(chassisMetadata, "LLDP.SysName", lldpLayerInfo.SysName)
				chassisMetadata["Name"] = lldpLayerInfo.SysName
			}

			if lldp8201Q, err := lldpLayerInfo.Decode8021(); err == nil {
				if lldp8201Q.LinkAggregation.Supported {
					common.SetField(portMetadata, "LLDP.LinkAggregation", map[string]interface{}{
						"Enabled":   lldp8201Q.LinkAggregation.Enabled,
						"PortID":    int64(lldp8201Q.LinkAggregation.PortID),
						"Supported": lldp8201Q.LinkAggregation.Supported,
					})
				}

				if lldp8201Q.PVID != 0 {
					common.SetField(portMetadata, "LLDP.PVID", int64(lldp8201Q.PVID))
				}

				if lldp8201Q.VIDUsageDigest != 0 {
					common.SetField(portMetadata, "LLDP.VIDUsageDigest", int64(lldp8201Q.VIDUsageDigest))
				}

				if lldp8201Q.ManagementVID != 0 {
					common.SetField(portMetadata, "LLDP.ManagementVID", int64(lldp8201Q.ManagementVID))
				}

				if len(lldp8201Q.VLANNames) != 0 {
					vlanNames := make([]interface{}, len(lldp8201Q.VLANNames))
					for i, vlan := range lldp8201Q.VLANNames {
						vlanNames[i] = map[string]interface{}{
							"ID":   vlan.ID,
							"Name": vlan.Name,
						}
					}
					common.SetField(portMetadata, "LLDP.VLANNames", vlanNames)
				}

				if len(lldp8201Q.PPVIDs) != 0 {
					ppvids := make([]interface{}, len(lldp8201Q.PPVIDs))
					for i, ppvid := range lldp8201Q.PPVIDs {
						ppvids[i] = map[string]interface{}{
							"Enabled":   ppvid.Enabled,
							"ID":        int64(ppvid.ID),
							"Supported": ppvid.Supported,
						}
					}
					common.SetField(portMetadata, "LLDP.PPVIDs", ppvids)
				}
			}

			if lldp8023, err := lldpLayerInfo.Decode8023(); err == nil {
				if lldp8023.MTU != 0 {
					portMetadata["MTU"] = int64(lldp8023.MTU)
				}
			}
		}

		// TODO: Handle TTL (set port to down when timer expires ?)

		p.g.Lock()

		// Create a node for the sending chassis with a predictable ID
		chassis := p.getOrCreate(graph.GenID(chassisID, lldpLayer.ChassisID.Subtype.String()), chassisMetadata)

		// Create a port with a predicatable ID
		port := p.getOrCreate(graph.GenID(
			chassisID, lldpLayer.ChassisID.Subtype.String(),
			portID, lldpLayer.PortID.Subtype.String(),
		), portMetadata)

		if !topology.HaveOwnershipLink(p.g, chassis, port) {
			topology.AddOwnershipLink(p.g, chassis, port, nil)
			topology.AddLayer2Link(p.g, chassis, port, nil)
		}

		if !topology.HaveLayer2Link(p.g, port, n) {
			topology.AddLayer2Link(p.g, port, n, nil)
		}

		p.g.Unlock()
	}
}

func (p *Probe) startCapture(ifName, mac string, n *graph.Node) error {
	lldpPrefix := "01:80:c2:00:00"
	for _, lastByte := range []byte{0x00, 0x03, 0x0e} {
		// Add multicast address so that the kernel does not discard it
		if err := addMulticastAddr(ifName, fmt.Sprintf("%s:%02X", lldpPrefix, lastByte)); err != nil {
			return fmt.Errorf("Failed to add multicast address: %s", err)
		}
	}

	// Set BPF filter to only capture LLDP packets
	bpfFilter := fmt.Sprintf(lldpBPFFilter, mac)

	packetProbe, err := probes.NewGoPacketProbe(p.g, n, probes.AFPacket, bpfFilter, lldpSnapLen)
	if err != nil {
		return err
	}

	// Set capturing state to true
	p.interfaceMap[ifName] = true

	p.wg.Add(1)

	go func() {
		defer func() {
			logging.GetLogger().Infof("Stopping LLDP capture on %s", ifName)

			p.Lock()
			p.interfaceMap[ifName] = false
			p.Unlock()

			p.wg.Done()
		}()

		packetProbe.Run(func(packet gopacket.Packet) {
			p.handlePacket(n, packet)
		}, nil)
	}()

	return err
}

func (p *Probe) getOrCreate(id graph.Identifier, m graph.Metadata) *graph.Node {
	node := p.g.GetNode(id)
	if node == nil {
		node = p.g.NewNode(id, m)
	} else {
		tr := p.g.StartMetadataTransaction(node)
		for k, v := range m {
			tr.AddMetadata(k, v)
		}
		tr.Commit()
	}
	return node
}

// handleNode checks if LLDP capture should be started on the passed interface node, ie:
// - when no LLDP capture is running for this interface
// - when its first packet layer is Ethernet and it has a MAC address
// - when the interface is listed in the configuration file or we are in auto discovery mode
func (p *Probe) handleNode(n *graph.Node) {
	firstLayerType, _ := probes.GoPacketFirstLayerType(n)
	mac, _ := n.GetFieldString("MAC")
	name, _ := n.GetFieldString("Name")

	if name != "" && mac != "" && firstLayerType == layers.LayerTypeEthernet {
		if active, found := p.interfaceMap[name]; (found || p.autoDiscovery) && !active {
			logging.GetLogger().Infof("Starting LLDP capture on %s", name)
			if err := p.startCapture(name, mac, n); err != nil {
				logging.GetLogger().Error(err)
			}
		}
	}
}

// OnEdgeAdded is called when a new edge was created on the graph
func (p *Probe) OnEdgeAdded(e *graph.Edge) {
	p.Lock()
	defer p.Unlock()

	// Only consider nodes that are owned by the host node
	if e.GetParent() == p.hostNode.ID {
		if relationType, _ := e.GetFieldString("RelationType"); relationType == topology.OwnershipLink {
			n := p.g.GetNode(e.GetChild())
			p.handleNode(n)
		}
	}
}

// OnNodeUpdated is called when a new node was updated on the graph
func (p *Probe) OnNodeUpdated(n *graph.Node) {
	p.Lock()
	defer p.Unlock()

	// If the interface was modified from down to up
	if state, _ := n.GetFieldString("State"); state == "UP" {
		if p.g.AreLinked(p.hostNode, n, topology.OwnershipMetadata()) {
			p.handleNode(n)
		}
	}
}

// Start capturing LLDP packets
func (p *Probe) Start() {
	atomic.StoreInt64(&p.state, common.RunningState)
	p.g.AddEventListener(p)

	p.g.RLock()
	defer p.g.RUnlock()
	p.Lock()
	defer p.Unlock()

	// The nodes may have already been created
	children := p.g.LookupChildren(p.hostNode, nil, topology.OwnershipMetadata())
	for _, intfNode := range children {
		p.handleNode(intfNode)
	}
}

// Stop capturing LLDP packets
func (p *Probe) Stop() {
	p.g.RemoveEventListener(p)
	atomic.StoreInt64(&p.state, common.StoppingState)
	p.wg.Wait()
}

// NewProbe returns a new LLDP probe
func NewProbe(g *graph.Graph, hostNode *graph.Node, interfaces []string) (*Probe, error) {
	interfaceMap := make(map[string]bool)
	for _, intf := range interfaces {
		interfaceMap[intf] = false
	}

	return &Probe{
		g:             g,
		hostNode:      hostNode,
		interfaceMap:  interfaceMap,
		state:         common.StoppedState,
		autoDiscovery: len(interfaces) == 0,
	}, nil
}
<commit_msg>lldp: trim zero bytes from string<commit_after>// +build linux

/*
 * Copyright (C) 2018 Red Hat, Inc.
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 *
 */

package lldp

import (
	"bytes"
	"fmt"
	"net"
	"sync"
	"sync/atomic"
	"syscall"
	"unsafe"

	"github.com/google/gopacket"
	"github.com/google/gopacket/layers"
	"github.com/safchain/ethtool"
	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/flow/probes"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/topology"
	"github.com/skydive-project/skydive/topology/graph"
	"golang.org/x/sys/unix"
)

/*
#include <linux/if_packet.h>  // packet_mreq
#include <linux/if_ether.h>  // ETH_ALEN
*/
import "C"

const lldpBPFFilter = `ether[0] & 1 = 1 and
  ether proto 0x88cc and
  !(ether src %s) and
  (ether dst 01:80:c2:00:00:0e or
   ether dst 01:80:c2:00:00:03 or
   ether dst 01:80:c2:00:00:00)`

// Capture 8192 bytes so that we have the full Ethernet frame
const lldpSnapLen = 8192

// Probe describes the probe that is in charge of listening for
// LLDP packets on interfaces and create the corresponding chassis and port nodes
type Probe struct {
	sync.RWMutex
	graph.DefaultGraphListener
	g             *graph.Graph
	hostNode      *graph.Node     // graph node of the running host
	interfaceMap  map[string]bool // map interface names to the capturing state
	state         int64           // state of the probe (running or stopped)
	wg            sync.WaitGroup  // capture goroutines wait group
	autoDiscovery bool            // capture LLDP traffic on all capable interfaces
}

type ifreq struct {
	ifrName   [ethtool.IFNAMSIZ]byte
	ifrHwaddr syscall.RawSockaddr
}

// addMulticastAddr adds a multicast address to an interface using an ioctl call
func addMulticastAddr(intf string, addr string) error {
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, 0)
	if err != nil {
		return err
	}
	defer syscall.Close(fd)

	var name [ethtool.IFNAMSIZ]byte
	copy(name[:], []byte(intf))

	mac, _ := net.ParseMAC(addr)

	var sockaddr syscall.RawSockaddr
	// RawSockaddr.Data is []int8 on amd64 and []uint8 on ppc64le
	switch data := interface{}(sockaddr.Data).(type) {
	case [14]int8:
		for i, n := range mac {
			data[i] = int8(n)
		}
	case [14]uint8:
		for i, n := range mac {
			data[i] = uint8(n)
		}
	}

	ifr := &ifreq{
		ifrName:   name,
		ifrHwaddr: sockaddr,
	}

	_, _, ep := unix.Syscall(unix.SYS_IOCTL, uintptr(fd),
		unix.SIOCADDMULTI, uintptr(unsafe.Pointer(ifr)))

	if ep != 0 {
		return syscall.Errno(ep)
	}
	return nil
}

func (p *Probe) handlePacket(n *graph.Node, packet gopacket.Packet) {
	lldpLayer := packet.Layer(layers.LayerTypeLinkLayerDiscovery)
	if lldpLayer != nil {
		lldpLayer := lldpLayer.(*layers.LinkLayerDiscovery)

		bytesToString := func(b []byte) string {
			return string(bytes.Trim(b, "\x00"))
		}

		chassisMetadata := graph.Metadata{
			"LLDP": map[string]interface{}{
				"ChassisIDType": lldpLayer.ChassisID.Subtype.String(),
			},
			"Type":  "switch",
			"Probe": "lldp",
		}

		var chassisID string
		switch lldpLayer.ChassisID.Subtype {
		case layers.LLDPChassisIDSubTypeMACAddr:
			chassisID = net.HardwareAddr(lldpLayer.ChassisID.ID).String()
		default:
			chassisID = bytesToString(lldpLayer.ChassisID.ID)
		}
		common.SetField(chassisMetadata, "LLDP.ChassisID", chassisID)
		common.SetField(chassisMetadata, "Name", chassisID)

		portMetadata := graph.Metadata{
			"LLDP": map[string]interface{}{
				"PortIDType": lldpLayer.PortID.Subtype.String(),
			},
			"Type":  "switchport",
			"Probe": "lldp",
		}

		var portID string
		switch lldpLayer.PortID.Subtype {
		case layers.LLDPPortIDSubtypeMACAddr:
			portID = net.HardwareAddr(lldpLayer.PortID.ID).String()
		default:
			portID = bytesToString(lldpLayer.PortID.ID)
		}
		common.SetField(portMetadata, "LLDP.PortID", portID)
		common.SetField(portMetadata, "Name", portID)

		if lldpLayerInfo := packet.Layer(layers.LayerTypeLinkLayerDiscoveryInfo); lldpLayerInfo != nil {
			lldpLayerInfo := lldpLayerInfo.(*layers.LinkLayerDiscoveryInfo)

			if lldpLayerInfo.PortDescription != "" {
				common.SetField(portMetadata, "LLDP.Description", lldpLayerInfo.PortDescription)
				portMetadata["Name"] = bytesToString([]byte(lldpLayerInfo.PortDescription))
			}

			if lldpLayerInfo.SysDescription != "" {
				common.SetField(chassisMetadata, "LLDP.Description", bytesToString([]byte(lldpLayerInfo.SysDescription)))
			}

			if sysName := bytesToString([]byte(lldpLayerInfo.SysName)); sysName != "" {
				common.SetField(chassisMetadata, "LLDP.SysName", sysName)
				chassisMetadata["Name"] = sysName
			}

			if lldp8201Q, err := lldpLayerInfo.Decode8021(); err == nil {
				if lldp8201Q.LinkAggregation.Supported {
					common.SetField(portMetadata, "LLDP.LinkAggregation", map[string]interface{}{
						"Enabled":   lldp8201Q.LinkAggregation.Enabled,
						"PortID":    int64(lldp8201Q.LinkAggregation.PortID),
						"Supported": lldp8201Q.LinkAggregation.Supported,
					})
				}

				if lldp8201Q.PVID != 0 {
					common.SetField(portMetadata, "LLDP.PVID", int64(lldp8201Q.PVID))
				}

				if lldp8201Q.VIDUsageDigest != 0 {
					common.SetField(portMetadata, "LLDP.VIDUsageDigest", int64(lldp8201Q.VIDUsageDigest))
				}

				if lldp8201Q.ManagementVID != 0 {
					common.SetField(portMetadata, "LLDP.ManagementVID", int64(lldp8201Q.ManagementVID))
				}

				if len(lldp8201Q.VLANNames) != 0 {
					vlanNames := make([]interface{}, len(lldp8201Q.VLANNames))
					for i, vlan := range lldp8201Q.VLANNames {
						vlanNames[i] = map[string]interface{}{
							"ID":   vlan.ID,
							"Name": bytesToString([]byte(vlan.Name)),
						}
					}
					common.SetField(portMetadata, "LLDP.VLANNames", vlanNames)
				}

				if len(lldp8201Q.PPVIDs) != 0 {
					ppvids := make([]interface{}, len(lldp8201Q.PPVIDs))
					for i, ppvid := range lldp8201Q.PPVIDs {
						ppvids[i] = map[string]interface{}{
							"Enabled":   ppvid.Enabled,
							"ID":        int64(ppvid.ID),
							"Supported": ppvid.Supported,
						}
					}
					common.SetField(portMetadata, "LLDP.PPVIDs", ppvids)
				}
			}

			if lldp8023, err := lldpLayerInfo.Decode8023(); err == nil {
				if lldp8023.MTU != 0 {
					portMetadata["MTU"] = int64(lldp8023.MTU)
				}
			}
		}

		// TODO: Handle TTL (set port to down when timer expires ?)

		p.g.Lock()

		// Create a node for the sending chassis with a predictable ID
		chassis := p.getOrCreate(graph.GenID(chassisID, lldpLayer.ChassisID.Subtype.String()), chassisMetadata)

		// Create a port with a predicatable ID
		port := p.getOrCreate(graph.GenID(
			chassisID, lldpLayer.ChassisID.Subtype.String(),
			portID, lldpLayer.PortID.Subtype.String(),
		), portMetadata)

		if !topology.HaveOwnershipLink(p.g, chassis, port) {
			topology.AddOwnershipLink(p.g, chassis, port, nil)
			topology.AddLayer2Link(p.g, chassis, port, nil)
		}

		if !topology.HaveLayer2Link(p.g, port, n) {
			topology.AddLayer2Link(p.g, port, n, nil)
		}

		p.g.Unlock()
	}
}

func (p *Probe) startCapture(ifName, mac string, n *graph.Node) error {
	lldpPrefix := "01:80:c2:00:00"
	for _, lastByte := range []byte{0x00, 0x03, 0x0e} {
		// Add multicast address so that the kernel does not discard it
		if err := addMulticastAddr(ifName, fmt.Sprintf("%s:%02X", lldpPrefix, lastByte)); err != nil {
			return fmt.Errorf("Failed to add multicast address: %s", err)
		}
	}

	// Set BPF filter to only capture LLDP packets
	bpfFilter := fmt.Sprintf(lldpBPFFilter, mac)

	packetProbe, err := probes.NewGoPacketProbe(p.g, n, probes.AFPacket, bpfFilter, lldpSnapLen)
	if err != nil {
		return err
	}

	// Set capturing state to true
	p.interfaceMap[ifName] = true

	p.wg.Add(1)

	go func() {
		defer func() {
			logging.GetLogger().Infof("Stopping LLDP capture on %s", ifName)

			p.Lock()
			p.interfaceMap[ifName] = false
			p.Unlock()

			p.wg.Done()
		}()

		packetProbe.Run(func(packet gopacket.Packet) {
			p.handlePacket(n, packet)
		}, nil)
	}()

	return err
}

func (p *Probe) getOrCreate(id graph.Identifier, m graph.Metadata) *graph.Node {
	node := p.g.GetNode(id)
	if node == nil {
		node = p.g.NewNode(id, m)
	} else {
		tr := p.g.StartMetadataTransaction(node)
		for k, v := range m {
			tr.AddMetadata(k, v)
		}
		tr.Commit()
	}
	return node
}

// handleNode checks if LLDP capture should be started on the passed interface node, ie:
// - when no LLDP capture is running for this interface
// - when its first packet layer is Ethernet and it has a MAC address
// - when the interface is listed in the configuration file or we are in auto discovery mode
func (p *Probe) handleNode(n *graph.Node) {
	firstLayerType, _ := probes.GoPacketFirstLayerType(n)
	mac, _ := n.GetFieldString("MAC")
	name, _ := n.GetFieldString("Name")

	if name != "" && mac != "" && firstLayerType == layers.LayerTypeEthernet {
		if active, found := p.interfaceMap[name]; (found || p.autoDiscovery) && !active {
			logging.GetLogger().Infof("Starting LLDP capture on %s", name)
			if err := p.startCapture(name, mac, n); err != nil {
				logging.GetLogger().Error(err)
			}
		}
	}
}

// OnEdgeAdded is called when a new edge was created on the graph
func (p *Probe) OnEdgeAdded(e *graph.Edge) {
	p.Lock()
	defer p.Unlock()

	// Only consider nodes that are owned by the host node
	if e.GetParent() == p.hostNode.ID {
		if relationType, _ := e.GetFieldString("RelationType"); relationType == topology.OwnershipLink {
			n := p.g.GetNode(e.GetChild())
			p.handleNode(n)
		}
	}
}

// OnNodeUpdated is called when a new node was updated on the graph
func (p *Probe) OnNodeUpdated(n *graph.Node) {
	p.Lock()
	defer p.Unlock()

	// If the interface was modified from down to up
	if state, _ := n.GetFieldString("State"); state == "UP" {
		if p.g.AreLinked(p.hostNode, n, topology.OwnershipMetadata()) {
			p.handleNode(n)
		}
	}
}

// Start capturing LLDP packets
func (p *Probe) Start() {
	atomic.StoreInt64(&p.state, common.RunningState)
	p.g.AddEventListener(p)

	p.g.RLock()
	defer p.g.RUnlock()
	p.Lock()
	defer p.Unlock()

	// The nodes may have already been created
	children := p.g.LookupChildren(p.hostNode, nil, topology.OwnershipMetadata())
	for _, intfNode := range children {
		p.handleNode(intfNode)
	}
}

// Stop capturing LLDP packets
func (p *Probe) Stop() {
	p.g.RemoveEventListener(p)
	atomic.StoreInt64(&p.state, common.StoppingState)
	p.wg.Wait()
}

// NewProbe returns a new LLDP probe
func NewProbe(g *graph.Graph, hostNode *graph.Node, interfaces []string) (*Probe, error) {
	interfaceMap := make(map[string]bool)
	for _, intf := range interfaces {
		interfaceMap[intf] = false
	}

	return &Probe{
		g:             g,
		hostNode:      hostNode,
		interfaceMap:  interfaceMap,
		state:         common.StoppedState,
		autoDiscovery: len(interfaces) == 0,
	}, nil
}
<|endoftext|>


<commit_before>package internal

import (
	"go/ast"
	"go/importer"
	// "go/internal/gcimporter"
	"go/token"
	"go/types"
	"os"
	"os/exec"
	"strings"
)

func GetTypeInfo(pkgDir, importPath, tempGoSrcDir string, fset *token.FileSet, files []*ast.File) (*types.Info, error) {
	typesConfig := types.Config{}
	// I gave up to loading pkg from source.(by using "code.google.com/p/go.tools/go/loader")
	// 	typesConfig.Import = func(imports map[string]*types.Package, path string) (*types.Package, error) {
	// 		// Import from source if fail to import from binary
	// 		pkg, err := gcimporter.Import(imports, path)
	// 		if err == nil {
	// 			return pkg, nil
	// 		}
	//
	// 		lConfig := loader.Config{}
	// 		lConfig.TypeChecker = typesConfig
	// 		lConfig.Build = &build.Default
	// 		lConfig.SourceImports = true
	// 		lConfig.Import(path)
	//
	// 		prog, err := lConfig.Load()
	// 		if err != nil {
	// 			return nil, err
	// 		}
	// 		fmt.Println(prog.Imported[path].Types)
	// 		return prog.Imported[path].Pkg, nil
	// 	}
	// Because
	//   - it is slow.
	//   - i met strange errors.
	//       (e.g. github.com/ToQoz/gopwt/translator.go:201:29: cannot pass argument token.NewFileSet() (value of type *go/token.FileSet) to parameter of type *go/token.FileSet)

	// Install binaries
	deps, err := findDeps(importPath, tempGoSrcDir)
	if err != nil {
		return nil, err
	}
	if IsBuildableFileSet(fset) {
		// install self
		deps = append(deps, ".")
	}
	if len(deps) > 0 {
		install := exec.Command("go", "install")
		install.Dir = pkgDir
		install.Stdout = os.Stdout
		install.Stderr = os.Stderr
		if Verbose {
			install.Args = append(install.Args, "-v")
		}
		install.Args = append(install.Args, deps...)
		if err := install.Run(); err != nil {
			return nil, err
		}
	}

	// Assume types from ast.Node
	// typesConfig.Import = gcimporter.Import
	typesConfig.Importer = importer.Default()
	pkg := types.NewPackage(importPath, "")
	info := &types.Info{
		Types:      map[ast.Expr]types.TypeAndValue{},
		Defs:       map[*ast.Ident]types.Object{},
		Uses:       map[*ast.Ident]types.Object{},
		Implicits:  map[ast.Node]types.Object{},
		Selections: map[*ast.SelectorExpr]*types.Selection{},
		Scopes:     map[ast.Node]*types.Scope{},
		InitOrder:  []*types.Initializer{},
	}

	// WORKARROUND(xtest): if the tests is xtest like a `{pkg}_test`, check types only in `{pkg}_test`
	// NOTE: if your check types `package {pkg}_test` and `package {pkg}`, you'll get `package {pkg}_test; expected {pkg}`
	isXtest := false
	xtests := []*ast.File{}
	for _, f := range files {
		if strings.HasSuffix(f.Name.Name, "_test") {
			xtests = append(xtests, f)
		}
	}
	isXtest = len(xtests) > 0 && len(xtests) != len(files)

	if isXtest {
		err = types.NewChecker(&typesConfig, fset, pkg, info).Files(xtests)
	} else {
		err = types.NewChecker(&typesConfig, fset, pkg, info).Files(files)
	}
	if err != nil {
		return nil, err
	}

	return info, nil
}

func DeterminantExprOfIsTypeConversion(e ast.Expr) ast.Expr {
	switch e.(type) {
	case *ast.ParenExpr:
		return DeterminantExprOfIsTypeConversion(e.(*ast.ParenExpr).X)
	case *ast.StarExpr:
		return DeterminantExprOfIsTypeConversion(e.(*ast.StarExpr).X)
	case *ast.CallExpr:
		return DeterminantExprOfIsTypeConversion(e.(*ast.CallExpr).Fun)
	case *ast.SelectorExpr:
		return e.(*ast.SelectorExpr).Sel
	default:
		return e
	}
}

func IsTypeConversion(info *types.Info, e *ast.CallExpr) bool {
	if info == nil {
		return false
	}

	funcOrType := DeterminantExprOfIsTypeConversion(e)

	switch funcOrType.(type) {
	case *ast.ChanType, *ast.FuncType, *ast.MapType, *ast.ArrayType, *ast.StructType, *ast.InterfaceType:
		return true
	case *ast.Ident:
		id := funcOrType.(*ast.Ident)

		if t, ok := info.Types[id]; ok {
			return t.IsType()
		}

		if o := info.ObjectOf(id); o != nil {
			switch o.(type) {
			case *types.TypeName:
				return true
			default:
				return false
			}
		}
	}

	panic("unexpected error")
}
<commit_msg>chore(gopwt/translator): make errros on `go install` understandable.<commit_after>package internal

import (
	"bytes"
	"fmt"
	"go/ast"
	"go/importer"
	// "go/internal/gcimporter"
	"go/token"
	"go/types"
	"os"
	"os/exec"
	"strings"
)

func GetTypeInfo(pkgDir, importPath, tempGoSrcDir string, fset *token.FileSet, files []*ast.File) (*types.Info, error) {
	typesConfig := types.Config{}
	// I gave up to loading pkg from source.(by using "code.google.com/p/go.tools/go/loader")
	// 	typesConfig.Import = func(imports map[string]*types.Package, path string) (*types.Package, error) {
	// 		// Import from source if fail to import from binary
	// 		pkg, err := gcimporter.Import(imports, path)
	// 		if err == nil {
	// 			return pkg, nil
	// 		}
	//
	// 		lConfig := loader.Config{}
	// 		lConfig.TypeChecker = typesConfig
	// 		lConfig.Build = &build.Default
	// 		lConfig.SourceImports = true
	// 		lConfig.Import(path)
	//
	// 		prog, err := lConfig.Load()
	// 		if err != nil {
	// 			return nil, err
	// 		}
	// 		fmt.Println(prog.Imported[path].Types)
	// 		return prog.Imported[path].Pkg, nil
	// 	}
	// Because
	//   - it is slow.
	//   - i met strange errors.
	//       (e.g. github.com/ToQoz/gopwt/translator.go:201:29: cannot pass argument token.NewFileSet() (value of type *go/token.FileSet) to parameter of type *go/token.FileSet)

	// Install binaries
	deps, err := findDeps(importPath, tempGoSrcDir)
	if err != nil {
		return nil, err
	}
	if IsBuildableFileSet(fset) {
		// install self
		deps = append(deps, ".")
	}
	if len(deps) > 0 {
		install := exec.Command("go", "install")
		install.Dir = pkgDir
		install.Stdout = os.Stdout
		b := []byte{}
		buf := bytes.NewBuffer(b)
		install.Stderr = buf
		if Verbose {
			install.Args = append(install.Args, "-v")
		}
		install.Args = append(install.Args, deps...)
		if err := install.Run(); err != nil {
			return nil, fmt.Errorf("[ERROR] go install %s\n\n%s", strings.Join(deps, " "), buf.String())
		}
	}

	// Assume types from ast.Node
	// typesConfig.Import = gcimporter.Import
	typesConfig.Importer = importer.Default()
	pkg := types.NewPackage(importPath, "")
	info := &types.Info{
		Types:      map[ast.Expr]types.TypeAndValue{},
		Defs:       map[*ast.Ident]types.Object{},
		Uses:       map[*ast.Ident]types.Object{},
		Implicits:  map[ast.Node]types.Object{},
		Selections: map[*ast.SelectorExpr]*types.Selection{},
		Scopes:     map[ast.Node]*types.Scope{},
		InitOrder:  []*types.Initializer{},
	}

	// WORKARROUND(xtest): if the tests is xtest like a `{pkg}_test`, check types only in `{pkg}_test`
	// NOTE: if your check types `package {pkg}_test` and `package {pkg}`, you'll get `package {pkg}_test; expected {pkg}`
	isXtest := false
	xtests := []*ast.File{}
	for _, f := range files {
		if strings.HasSuffix(f.Name.Name, "_test") {
			xtests = append(xtests, f)
		}
	}
	isXtest = len(xtests) > 0 && len(xtests) != len(files)

	if isXtest {
		err = types.NewChecker(&typesConfig, fset, pkg, info).Files(xtests)
	} else {
		err = types.NewChecker(&typesConfig, fset, pkg, info).Files(files)
	}
	if err != nil {
		return nil, err
	}

	return info, nil
}

func DeterminantExprOfIsTypeConversion(e ast.Expr) ast.Expr {
	switch e.(type) {
	case *ast.ParenExpr:
		return DeterminantExprOfIsTypeConversion(e.(*ast.ParenExpr).X)
	case *ast.StarExpr:
		return DeterminantExprOfIsTypeConversion(e.(*ast.StarExpr).X)
	case *ast.CallExpr:
		return DeterminantExprOfIsTypeConversion(e.(*ast.CallExpr).Fun)
	case *ast.SelectorExpr:
		return e.(*ast.SelectorExpr).Sel
	default:
		return e
	}
}

func IsTypeConversion(info *types.Info, e *ast.CallExpr) bool {
	if info == nil {
		return false
	}

	funcOrType := DeterminantExprOfIsTypeConversion(e)

	switch funcOrType.(type) {
	case *ast.ChanType, *ast.FuncType, *ast.MapType, *ast.ArrayType, *ast.StructType, *ast.InterfaceType:
		return true
	case *ast.Ident:
		id := funcOrType.(*ast.Ident)

		if t, ok := info.Types[id]; ok {
			return t.IsType()
		}

		if o := info.ObjectOf(id); o != nil {
			switch o.(type) {
			case *types.TypeName:
				return true
			default:
				return false
			}
		}
	}

	panic("unexpected error")
}
<|endoftext|>


<commit_before>// Copyright 2015 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

// +build libax25

package ax25

/*
#cgo LDFLAGS: -lax25
#include <sys/socket.h>
#include <netax25/ax25.h>
#include <netax25/axlib.h>
#include <netax25/axconfig.h>
#include <fcntl.h>
*/
import "C"

import (
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"syscall"
	"time"
	"unsafe"
)

type ax25Addr C.struct_full_sockaddr_ax25

var numAXPorts int

// bug(martinhpedersen): The AX.25 stack does not support SOCK_STREAM, so any write to the connection
// that is larger than maximum packet length will fail. The b2f impl. requires 125 bytes long packets.
var ErrMessageTooLong = errors.New("Write: Message too long. Consider increasing maximum packet length to >= 125.")

type fd uintptr

type ax25Listener struct {
	sock      fd
	localAddr AX25Addr
}

func loadPorts() (int, error) {
	if numAXPorts > 0 {
		return numAXPorts, nil
	}

	n, err := C.ax25_config_load_ports()
	if err != nil {
		return int(n), err
	} else if n == 0 {
		return 0, fmt.Errorf("No AX.25 ports configured")
	}

	numAXPorts = int(n)
	return numAXPorts, err
}

// Addr returns the listener's network address, an AX25Addr.
func (ln ax25Listener) Addr() net.Addr { return ln.localAddr }

// Close stops listening on the AX.25 port. Already Accepted connections are not closed.
func (ln ax25Listener) Close() error { return ln.sock.close() } //TODO: Should make sure any Accept() calls returns with an error!

// Accept waits for the next call and returns a generic Conn.
//
// See net.Listener for more information.
func (ln ax25Listener) Accept() (net.Conn, error) {
	nfd, addr, err := ln.sock.accept()
	if err != nil {
		return nil, err
	}

	conn := &Conn{
		localAddr:       ln.localAddr,
		remoteAddr:      AX25Addr{addr},
		ReadWriteCloser: os.NewFile(uintptr(nfd), ""),
	}

	return conn, nil
}

// ListenAX25 announces on the local port axPort using mycall as the local address.
func ListenAX25(axPort, mycall string) (net.Listener, error) {
	if _, err := loadPorts(); err != nil {
		return nil, err
	}

	// Setup local address (via callsign of supplied axPort)
	localAddr := newAX25Addr(mycall)
	if err := localAddr.setPort(axPort); err != nil {
		return nil, err
	}

	// Create file descriptor
	var socket fd
	if f, err := syscall.Socket(syscall.AF_AX25, syscall.SOCK_SEQPACKET, 0); err != nil {
		return nil, err
	} else {
		socket = fd(f)
	}

	if err := socket.bind(localAddr); err != nil {
		return nil, err
	}
	if err := syscall.Listen(int(socket), syscall.SOMAXCONN); err != nil {
		return nil, err
	}

	return ax25Listener{
		sock:      fd(socket),
		localAddr: AX25Addr{localAddr},
	}, nil
}

// DialAX25Timeout acts like DialAX25 but takes a timeout.
func DialAX25Timeout(axPort, mycall, targetcall string, timeout time.Duration) (*Conn, error) {
	if _, err := loadPorts(); err != nil {
		return nil, err
	}

	// Setup local address (via callsign of supplied axPort)
	localAddr := newAX25Addr(mycall)
	if err := localAddr.setPort(axPort); err != nil {
		return nil, err
	}
	remoteAddr := newAX25Addr(targetcall)

	// Create file descriptor
	var socket fd
	if f, err := syscall.Socket(syscall.AF_AX25, syscall.SOCK_SEQPACKET, 0); err != nil {
		return nil, err
	} else {
		socket = fd(f)
	}

	// Bind
	if err := socket.bind(localAddr); err != nil {
		return nil, err
	}

	// Connect
	err := socket.connectTimeout(remoteAddr, timeout)
	if err != nil {
		socket.close()
		return nil, err
	}

	return &Conn{
		ReadWriteCloser: os.NewFile(uintptr(socket), axPort),
		localAddr:       AX25Addr{localAddr},
		remoteAddr:      AX25Addr{remoteAddr},
	}, nil
}

func (c *Conn) Close() error {
	if !c.ok() {
		return syscall.EINVAL
	}

	return c.ReadWriteCloser.Close()
}

func (c *Conn) Write(p []byte) (n int, err error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}

	n, err = c.ReadWriteCloser.Write(p)
	perr, ok := err.(*os.PathError)
	if !ok {
		return
	}

	switch perr.Err.Error() {
	case "message too long":
		return n, ErrMessageTooLong
	default:
		return
	}
}

func (c *Conn) Read(p []byte) (n int, err error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}

	n, err = c.ReadWriteCloser.Read(p)
	perr, ok := err.(*os.PathError)
	if !ok {
		return
	}

	//TODO: These errors should not be checked using string comparison!
	// The weird error handling here is needed because of how the *os.File treats
	// the underlying fd. This should be fixed the same way as net.FileConn does.
	switch perr.Err.Error() {
	case "transport endpoint is not connected": // We get this error when the remote hangs up
		return n, io.EOF
	default:
		return
	}
}

// DialAX25 connects to the remote station targetcall using the named axport and mycall.
func DialAX25(axPort, mycall, targetcall string) (*Conn, error) {
	return DialAX25Timeout(axPort, mycall, targetcall, 0)
}

func (sock fd) connectTimeout(addr ax25Addr, timeout time.Duration) (err error) {
	if timeout == 0 {
		return sock.connect(addr)
	}
	if err = syscall.SetNonblock(int(sock), true); err != nil {
		return err
	}

	err = sock.connect(addr)
	if err == nil {
		return nil // Connected
	} else if err != syscall.EINPROGRESS {
		return fmt.Errorf("Unable to connect: %s", err)
	}

	// Shamelessly stolen from src/pkg/exp/inotify/inotify_linux.go:
	//
	// Create fdSet, taking into consideration that
	// 64-bit OS uses Bits: [16]int64, while 32-bit OS uses Bits: [32]int32.
	// This only support File Descriptors up to 1024
	//
	if sock > 1024 {
		panic(fmt.Errorf("connectTimeout: File Descriptor >= 1024: %v", sock))
	}
	fdset := new(syscall.FdSet)
	fElemSize := 32 * 32 / len(fdset.Bits)
	fdset.Bits[int(sock)/fElemSize] |= 1 << uint(int(sock)%fElemSize)
	//
	// Thanks!
	//

	// Wait or timeout
	var n int
	var tv syscall.Timeval
	for {
		tv = syscall.NsecToTimeval(int64(timeout))
		n, err = syscall.Select(int(sock)+1, nil, fdset, nil, &tv)
		if n < 0 && err != syscall.EINTR {
			sock.close()
			return fmt.Errorf("Unable to connect: %s", err)
		} else if n > 0 {
			// Verify that connection is OK
			nerr, err := syscall.GetsockoptInt(int(sock), syscall.SOL_SOCKET, syscall.SO_ERROR)
			if err != nil {
				sock.close()
				return err
			}
			err = syscall.Errno(nerr)
			if nerr != 0 && err != syscall.EINPROGRESS && err != syscall.EALREADY && err != syscall.EINTR {
				sock.close()
				return err
			} else {
				break // Connected
			}
		} else {
			sock.close()
			return fmt.Errorf("Unable to connect: timeout")
		}
	}

	syscall.SetNonblock(int(sock), false)
	return
}

func (sock fd) close() error {
	return syscall.Close(int(sock))
}

func (sock fd) accept() (nfd fd, addr ax25Addr, err error) {
	addrLen := C.socklen_t(unsafe.Sizeof(addr))

	n, err := C.accept(
		C.int(sock),
		(*C.struct_sockaddr)(unsafe.Pointer(&addr)),
		&addrLen)

	if addrLen != C.socklen_t(unsafe.Sizeof(addr)) {
		panic("unexpected socklet_t")
	}

	return fd(n), addr, err
}

func (sock fd) connect(addr ax25Addr) (err error) {
	_, err = C.connect(
		C.int(sock),
		(*C.struct_sockaddr)(unsafe.Pointer(&addr)),
		C.socklen_t(unsafe.Sizeof(addr)))

	return
}

func (sock fd) bind(addr ax25Addr) (err error) {
	_, err = C.bind(
		C.int(sock),
		(*C.struct_sockaddr)(unsafe.Pointer(&addr)),
		C.socklen_t(unsafe.Sizeof(addr)))

	return
}

type ax25_address *C.ax25_address

func (a ax25Addr) Address() Address {
	return AddressFromString(
		C.GoString(C.ax25_ntoa(a.ax25_address())),
	)
}

func (a ax25Addr) Digis() []Address {
	digis := make([]Address, a.numDigis())
	for i, digi := range a.digis() {
		digis[i] = AddressFromString(C.GoString(C.ax25_ntoa(digi)))
	}
	return digis
}

func (a *ax25Addr) numDigis() int {
	return int(a.fsa_ax25.sax25_ndigis)
}

func (a *ax25Addr) digis() []ax25_address {
	digis := make([]ax25_address, a.numDigis())
	for i, _ := range digis {
		digis[i] = (*C.ax25_address)(unsafe.Pointer(&a.fsa_digipeater[i]))
	}
	return digis
}

func (a *ax25Addr) ax25_address() ax25_address {
	return (*C.ax25_address)(unsafe.Pointer(&a.fsa_ax25.sax25_call.ax25_call))
}

func (a *ax25Addr) setPort(port string) (err error) {
	C.ax25_aton_entry(
		C.ax25_config_get_addr(C.CString(port)),
		&a.fsa_digipeater[0].ax25_call[0],
	)
	a.fsa_ax25.sax25_ndigis = 1
	return
}

func newAX25Addr(address string) ax25Addr {
	var addr C.struct_full_sockaddr_ax25

	if C.ax25_aton(C.CString(address), &addr) < 0 {
		panic("ax25_aton")
	}
	addr.fsa_ax25.sax25_family = syscall.AF_AX25

	return ax25Addr(addr)
}
<commit_msg>ax25: Return error on empty axport from Dial and Listen<commit_after>// Copyright 2015 Martin Hebnes Pedersen (LA5NTA). All rights reserved.
// Use of this source code is governed by the MIT-license that can be
// found in the LICENSE file.

// +build libax25

package ax25

/*
#cgo LDFLAGS: -lax25
#include <sys/socket.h>
#include <netax25/ax25.h>
#include <netax25/axlib.h>
#include <netax25/axconfig.h>
#include <fcntl.h>
*/
import "C"

import (
	"errors"
	"fmt"
	"io"
	"net"
	"os"
	"syscall"
	"time"
	"unsafe"
)

type ax25Addr C.struct_full_sockaddr_ax25

var numAXPorts int

// bug(martinhpedersen): The AX.25 stack does not support SOCK_STREAM, so any write to the connection
// that is larger than maximum packet length will fail. The b2f impl. requires 125 bytes long packets.
var ErrMessageTooLong = errors.New("Write: Message too long. Consider increasing maximum packet length to >= 125.")

type fd uintptr

type ax25Listener struct {
	sock      fd
	localAddr AX25Addr
}

func loadPorts() (int, error) {
	if numAXPorts > 0 {
		return numAXPorts, nil
	}

	n, err := C.ax25_config_load_ports()
	if err != nil {
		return int(n), err
	} else if n == 0 {
		return 0, fmt.Errorf("No AX.25 ports configured")
	}

	numAXPorts = int(n)
	return numAXPorts, err
}

// Addr returns the listener's network address, an AX25Addr.
func (ln ax25Listener) Addr() net.Addr { return ln.localAddr }

// Close stops listening on the AX.25 port. Already Accepted connections are not closed.
func (ln ax25Listener) Close() error { return ln.sock.close() } //TODO: Should make sure any Accept() calls returns with an error!

// Accept waits for the next call and returns a generic Conn.
//
// See net.Listener for more information.
func (ln ax25Listener) Accept() (net.Conn, error) {
	nfd, addr, err := ln.sock.accept()
	if err != nil {
		return nil, err
	}

	conn := &Conn{
		localAddr:       ln.localAddr,
		remoteAddr:      AX25Addr{addr},
		ReadWriteCloser: os.NewFile(uintptr(nfd), ""),
	}

	return conn, nil
}

// ListenAX25 announces on the local port axPort using mycall as the local address.
//
// An error will be returned if axPort is empty.
func ListenAX25(axPort, mycall string) (net.Listener, error) {
	if axPort == "" {
		return nil, errors.New("Invalid empty axport")
	}

	if _, err := loadPorts(); err != nil {
		return nil, err
	}

	// Setup local address (via callsign of supplied axPort)
	localAddr := newAX25Addr(mycall)
	if err := localAddr.setPort(axPort); err != nil {
		return nil, err
	}

	// Create file descriptor
	var socket fd
	if f, err := syscall.Socket(syscall.AF_AX25, syscall.SOCK_SEQPACKET, 0); err != nil {
		return nil, err
	} else {
		socket = fd(f)
	}

	if err := socket.bind(localAddr); err != nil {
		return nil, err
	}
	if err := syscall.Listen(int(socket), syscall.SOMAXCONN); err != nil {
		return nil, err
	}

	return ax25Listener{
		sock:      fd(socket),
		localAddr: AX25Addr{localAddr},
	}, nil
}

// DialAX25Timeout acts like DialAX25 but takes a timeout.
func DialAX25Timeout(axPort, mycall, targetcall string, timeout time.Duration) (*Conn, error) {
	if axPort == "" {
		return nil, errors.New("Invalid empty axport")
	}

	if _, err := loadPorts(); err != nil {
		return nil, err
	}

	// Setup local address (via callsign of supplied axPort)
	localAddr := newAX25Addr(mycall)
	if err := localAddr.setPort(axPort); err != nil {
		return nil, err
	}
	remoteAddr := newAX25Addr(targetcall)

	// Create file descriptor
	var socket fd
	if f, err := syscall.Socket(syscall.AF_AX25, syscall.SOCK_SEQPACKET, 0); err != nil {
		return nil, err
	} else {
		socket = fd(f)
	}

	// Bind
	if err := socket.bind(localAddr); err != nil {
		return nil, err
	}

	// Connect
	err := socket.connectTimeout(remoteAddr, timeout)
	if err != nil {
		socket.close()
		return nil, err
	}

	return &Conn{
		ReadWriteCloser: os.NewFile(uintptr(socket), axPort),
		localAddr:       AX25Addr{localAddr},
		remoteAddr:      AX25Addr{remoteAddr},
	}, nil
}

func (c *Conn) Close() error {
	if !c.ok() {
		return syscall.EINVAL
	}

	return c.ReadWriteCloser.Close()
}

func (c *Conn) Write(p []byte) (n int, err error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}

	n, err = c.ReadWriteCloser.Write(p)
	perr, ok := err.(*os.PathError)
	if !ok {
		return
	}

	switch perr.Err.Error() {
	case "message too long":
		return n, ErrMessageTooLong
	default:
		return
	}
}

func (c *Conn) Read(p []byte) (n int, err error) {
	if !c.ok() {
		return 0, syscall.EINVAL
	}

	n, err = c.ReadWriteCloser.Read(p)
	perr, ok := err.(*os.PathError)
	if !ok {
		return
	}

	//TODO: These errors should not be checked using string comparison!
	// The weird error handling here is needed because of how the *os.File treats
	// the underlying fd. This should be fixed the same way as net.FileConn does.
	switch perr.Err.Error() {
	case "transport endpoint is not connected": // We get this error when the remote hangs up
		return n, io.EOF
	default:
		return
	}
}

// DialAX25 connects to the remote station targetcall using the named axport and mycall.
//
// An error will be returned if axPort is empty.
func DialAX25(axPort, mycall, targetcall string) (*Conn, error) {
	return DialAX25Timeout(axPort, mycall, targetcall, 0)
}

func (sock fd) connectTimeout(addr ax25Addr, timeout time.Duration) (err error) {
	if timeout == 0 {
		return sock.connect(addr)
	}
	if err = syscall.SetNonblock(int(sock), true); err != nil {
		return err
	}

	err = sock.connect(addr)
	if err == nil {
		return nil // Connected
	} else if err != syscall.EINPROGRESS {
		return fmt.Errorf("Unable to connect: %s", err)
	}

	// Shamelessly stolen from src/pkg/exp/inotify/inotify_linux.go:
	//
	// Create fdSet, taking into consideration that
	// 64-bit OS uses Bits: [16]int64, while 32-bit OS uses Bits: [32]int32.
	// This only support File Descriptors up to 1024
	//
	if sock > 1024 {
		panic(fmt.Errorf("connectTimeout: File Descriptor >= 1024: %v", sock))
	}
	fdset := new(syscall.FdSet)
	fElemSize := 32 * 32 / len(fdset.Bits)
	fdset.Bits[int(sock)/fElemSize] |= 1 << uint(int(sock)%fElemSize)
	//
	// Thanks!
	//

	// Wait or timeout
	var n int
	var tv syscall.Timeval
	for {
		tv = syscall.NsecToTimeval(int64(timeout))
		n, err = syscall.Select(int(sock)+1, nil, fdset, nil, &tv)
		if n < 0 && err != syscall.EINTR {
			sock.close()
			return fmt.Errorf("Unable to connect: %s", err)
		} else if n > 0 {
			// Verify that connection is OK
			nerr, err := syscall.GetsockoptInt(int(sock), syscall.SOL_SOCKET, syscall.SO_ERROR)
			if err != nil {
				sock.close()
				return err
			}
			err = syscall.Errno(nerr)
			if nerr != 0 && err != syscall.EINPROGRESS && err != syscall.EALREADY && err != syscall.EINTR {
				sock.close()
				return err
			} else {
				break // Connected
			}
		} else {
			sock.close()
			return fmt.Errorf("Unable to connect: timeout")
		}
	}

	syscall.SetNonblock(int(sock), false)
	return
}

func (sock fd) close() error {
	return syscall.Close(int(sock))
}

func (sock fd) accept() (nfd fd, addr ax25Addr, err error) {
	addrLen := C.socklen_t(unsafe.Sizeof(addr))

	n, err := C.accept(
		C.int(sock),
		(*C.struct_sockaddr)(unsafe.Pointer(&addr)),
		&addrLen)

	if addrLen != C.socklen_t(unsafe.Sizeof(addr)) {
		panic("unexpected socklet_t")
	}

	return fd(n), addr, err
}

func (sock fd) connect(addr ax25Addr) (err error) {
	_, err = C.connect(
		C.int(sock),
		(*C.struct_sockaddr)(unsafe.Pointer(&addr)),
		C.socklen_t(unsafe.Sizeof(addr)))

	return
}

func (sock fd) bind(addr ax25Addr) (err error) {
	_, err = C.bind(
		C.int(sock),
		(*C.struct_sockaddr)(unsafe.Pointer(&addr)),
		C.socklen_t(unsafe.Sizeof(addr)))

	return
}

type ax25_address *C.ax25_address

func (a ax25Addr) Address() Address {
	return AddressFromString(
		C.GoString(C.ax25_ntoa(a.ax25_address())),
	)
}

func (a ax25Addr) Digis() []Address {
	digis := make([]Address, a.numDigis())
	for i, digi := range a.digis() {
		digis[i] = AddressFromString(C.GoString(C.ax25_ntoa(digi)))
	}
	return digis
}

func (a *ax25Addr) numDigis() int {
	return int(a.fsa_ax25.sax25_ndigis)
}

func (a *ax25Addr) digis() []ax25_address {
	digis := make([]ax25_address, a.numDigis())
	for i, _ := range digis {
		digis[i] = (*C.ax25_address)(unsafe.Pointer(&a.fsa_digipeater[i]))
	}
	return digis
}

func (a *ax25Addr) ax25_address() ax25_address {
	return (*C.ax25_address)(unsafe.Pointer(&a.fsa_ax25.sax25_call.ax25_call))
}

func (a *ax25Addr) setPort(port string) (err error) {
	C.ax25_aton_entry(
		C.ax25_config_get_addr(C.CString(port)),
		&a.fsa_digipeater[0].ax25_call[0],
	)
	a.fsa_ax25.sax25_ndigis = 1
	return
}

func newAX25Addr(address string) ax25Addr {
	var addr C.struct_full_sockaddr_ax25

	if C.ax25_aton(C.CString(address), &addr) < 0 {
		panic("ax25_aton")
	}
	addr.fsa_ax25.sax25_family = syscall.AF_AX25

	return ax25Addr(addr)
}
<|endoftext|>


<commit_before>package internet

import (
	"context"

	"v2ray.com/core/common/net"
)

type Dialer func(ctx context.Context, dest net.Destination) (Connection, error)

var (
	transportDialerCache = make(map[string]Dialer)
)

func RegisterTransportDialer(protocol string, dialer Dialer) error {
	if _, found := transportDialerCache[protocol]; found {
		return newError(protocol, " dialer already registered").AtError()
	}
	transportDialerCache[protocol] = dialer
	return nil
}

// Dial dials a internet connection towards the given destination.
func Dial(ctx context.Context, dest net.Destination) (Connection, error) {
	if dest.Network == net.Network_TCP {
		streamSettings := StreamSettingsFromContext(ctx)
		if streamSettings == nil {
			s, err := ToMemoryStreamConfig(nil)
			if err != nil {
				return nil, newError("failed to create default stream settings").Base(err)
			}
			streamSettings = s
			ctx = ContextWithStreamSettings(ctx, streamSettings)
		}

		protocol = streamSettings.ProtocolName
		dialer := transportDialerCache[protocol]
		if dialer == nil {
			return nil, newError(protocol, " dialer not registered").AtError()
		}
		return dialer(ctx, dest)
	}

	udpDialer := transportDialerCache["udp"]
	if udpDialer == nil {
		return nil, newError("UDP dialer not registered").AtError()
	}
	return udpDialer(ctx, dest)
}

// DialSystem calls system dialer to create a network connection.
func DialSystem(ctx context.Context, src net.Address, dest net.Destination) (net.Conn, error) {
	return effectiveSystemDialer.Dial(ctx, src, dest)
}
<commit_msg>fix build error<commit_after>package internet

import (
	"context"

	"v2ray.com/core/common/net"
)

type Dialer func(ctx context.Context, dest net.Destination) (Connection, error)

var (
	transportDialerCache = make(map[string]Dialer)
)

func RegisterTransportDialer(protocol string, dialer Dialer) error {
	if _, found := transportDialerCache[protocol]; found {
		return newError(protocol, " dialer already registered").AtError()
	}
	transportDialerCache[protocol] = dialer
	return nil
}

// Dial dials a internet connection towards the given destination.
func Dial(ctx context.Context, dest net.Destination) (Connection, error) {
	if dest.Network == net.Network_TCP {
		streamSettings := StreamSettingsFromContext(ctx)
		if streamSettings == nil {
			s, err := ToMemoryStreamConfig(nil)
			if err != nil {
				return nil, newError("failed to create default stream settings").Base(err)
			}
			streamSettings = s
			ctx = ContextWithStreamSettings(ctx, streamSettings)
		}

		protocol := streamSettings.ProtocolName
		dialer := transportDialerCache[protocol]
		if dialer == nil {
			return nil, newError(protocol, " dialer not registered").AtError()
		}
		return dialer(ctx, dest)
	}

	udpDialer := transportDialerCache["udp"]
	if udpDialer == nil {
		return nil, newError("UDP dialer not registered").AtError()
	}
	return udpDialer(ctx, dest)
}

// DialSystem calls system dialer to create a network connection.
func DialSystem(ctx context.Context, src net.Address, dest net.Destination) (net.Conn, error) {
	return effectiveSystemDialer.Dial(ctx, src, dest)
}
<|endoftext|>


<commit_before>package ttl_hash_set

import (
	"time"

	"github.com/fzzy/radix/redis"
)

type TTLHashSet struct {
	client *redis.Client
	prefix string
}

func NewTTLHashSet(prefix string, address string) (*TTLHashSet, error) {
	client, err := redis.Dial("tcp", address)
	if err != nil {
		return nil, err
	}

	return &TTLHashSet{
		client: client,
		prefix: prefix,
	}, nil
}

func (t *TTLHashSet) Add(key string) (bool, error) {
	localKey := prefixKey(t.prefix, key)

	// Use pipelining to set the key and set expiry in one go.
	t.client.Append("SET", localKey, 1)
	t.client.Append("EXPIRE", localKey, (5 * time.Hour).Seconds())

	return t.client.GetReply().Bool()
}

func (t *TTLHashSet) Close() error {
	return t.client.Close()
}

func (t *TTLHashSet) Exists(key string) (bool, error) {
	localKey := prefixKey(t.prefix, key)
	return t.client.Cmd("EXISTS", localKey).Bool()
}

func (t *TTLHashSet) TTL(key string) (int, error) {
	localKey := prefixKey(t.prefix, key)
	return t.client.Cmd("TTL", localKey).Int()
}

func prefixKey(prefix string, key string) string {
	return prefix + ":" + key
}
<commit_msg>Increase expiry time to 24 hours<commit_after>package ttl_hash_set

import (
	"time"

	"github.com/fzzy/radix/redis"
)

type TTLHashSet struct {
	client *redis.Client
	prefix string
}

func NewTTLHashSet(prefix string, address string) (*TTLHashSet, error) {
	client, err := redis.Dial("tcp", address)
	if err != nil {
		return nil, err
	}

	return &TTLHashSet{
		client: client,
		prefix: prefix,
	}, nil
}

func (t *TTLHashSet) Add(key string) (bool, error) {
	localKey := prefixKey(t.prefix, key)

	// Use pipelining to set the key and set expiry in one go.
	t.client.Append("SET", localKey, 1)
	t.client.Append("EXPIRE", localKey, (24 * time.Hour).Seconds())

	return t.client.GetReply().Bool()
}

func (t *TTLHashSet) Close() error {
	return t.client.Close()
}

func (t *TTLHashSet) Exists(key string) (bool, error) {
	localKey := prefixKey(t.prefix, key)
	return t.client.Cmd("EXISTS", localKey).Bool()
}

func (t *TTLHashSet) TTL(key string) (int, error) {
	localKey := prefixKey(t.prefix, key)
	return t.client.Cmd("TTL", localKey).Int()
}

func prefixKey(prefix string, key string) string {
	return prefix + ":" + key
}
<|endoftext|>


<commit_before>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"

	"github.com/TheThingsNetwork/ttn/api"
	"github.com/TheThingsNetwork/ttn/ttnctl/util"
	"github.com/spf13/cobra"
)

var gatewaysTokenCmd = &cobra.Command{
	Use:    "token [Type] [gatewayID]",
	Hidden: true,
	Short:  "Get the token for a gateway.",
	Long:   `gateways token gets a signed token for the gateway.`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 1 {
			cmd.UsageFunc()(cmd)
			return
		}

		gatewayID := args[0]
		if !api.ValidID(gatewayID) {
			ctx.Fatal("Invalid Gateway ID")
		}
		ctx = ctx.WithField("id", gatewayID)

		account := util.GetAccount(ctx)

		token, err := account.GetGatewayToken(gatewayID)
		if err != nil {
			ctx.WithError(err).Fatal("Could not get gateway token")
		}
		if token.Token == "" {
			ctx.Fatal("Gateway token was empty")
		}

		ctx.Info("Got gateway token")

		fmt.Println()
		fmt.Println(token.Token)
		fmt.Println()
		if !token.Expires.IsZero() {
			fmt.Printf("Expires %s\n", token.Expires)
			fmt.Println()
		}
	},
}

func init() {
	gatewaysCmd.AddCommand(gatewaysTokenCmd)
}
<commit_msg>Fix help for ttnctl gateway token<commit_after>// Copyright © 2016 The Things Network
// Use of this source code is governed by the MIT license that can be found in the LICENSE file.

package cmd

import (
	"fmt"

	"github.com/TheThingsNetwork/ttn/api"
	"github.com/TheThingsNetwork/ttn/ttnctl/util"
	"github.com/spf13/cobra"
)

var gatewaysTokenCmd = &cobra.Command{
	Use:    "token [GatewayID]",
	Hidden: true,
	Short:  "Get the token for a gateway.",
	Long:   `gateways token gets a signed token for the gateway.`,
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 1 {
			cmd.UsageFunc()(cmd)
			return
		}

		gatewayID := args[0]
		if !api.ValidID(gatewayID) {
			ctx.Fatal("Invalid Gateway ID")
		}
		ctx = ctx.WithField("id", gatewayID)

		account := util.GetAccount(ctx)

		token, err := account.GetGatewayToken(gatewayID)
		if err != nil {
			ctx.WithError(err).Fatal("Could not get gateway token")
		}
		if token.Token == "" {
			ctx.Fatal("Gateway token was empty")
		}

		ctx.Info("Got gateway token")

		fmt.Println()
		fmt.Println(token.Token)
		fmt.Println()
		if !token.Expires.IsZero() {
			fmt.Printf("Expires %s\n", token.Expires)
			fmt.Println()
		}
	},
}

func init() {
	gatewaysCmd.AddCommand(gatewaysTokenCmd)
}
<|endoftext|>


<commit_before>package dnsd

import (
	"context"
	"strings"
	"sync"

	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/lalog"
)

const (
	/*
		MaxNameEntriesToExtract is the maximum number of entries to be extracted from one instance of hosts file.
		The limit prevents an exceedingly long third party host file from taking too much memory.
	*/
	MaxNameEntriesToExtract = 50000
)

// HostsFileURLs is a collection of URLs where up-to-date ad/malware/spyware blacklist hosts files are published.
var HostsFileURLs = []string{
	"http://winhelp2002.mvps.org/hosts.txt",
	"http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext",
	"http://www.malwaredomainlist.com/hostslist/hosts.txt",
	"http://someonewhocares.org/hosts/hosts",
	"https://hosts.oisd.nl/light",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/ransomware.txt",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/scam.txt",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/tracking.txt",
}

/*
Whitelist is an array of domain names that often appear in black lists, but cause inconvenience when blocked. These
names are removed from downloaded black lists.
*/
var Whitelist = []string{
	/*
		2018-06-24 - youtube app on iPhone fails to save watch history, some sources suggest that this domain name is
		the culprit.
	*/
	"s.youtube.com",
	/*
		2019-05-23 - facebook app and instagram app take ~1 minute to load due to blocked connectivity to their graph
		API domains.
	*/
	"graph.facebook.com",
	"graph.instagram.com",
	// "Common whitelisted domains"(https://discourse.pi-hole.net/t/commonly-whitelisted-domains/212) collected on 2021-02-24
	"106c06cd218b007d-b1e8a1331f68446599e96a4b46a050f5.ams.plex.services", "79423.analytics.edgekey.net", "activity.windows.com", "android.clients.google.com", "api.facebook.com", "api-tv.spotify.com",
	"appleid.apple.com", "app.plex.tv", "appspot-preview.l.google.com", "apps.skype.com", "apresolve.spotify.com", "apt.sonarr.tv", "arc.msn.com", "assets.adobedtm.com", "attestation.xboxlive.com",
	"b-api.facebook.com", "b-graph.facebook.com", "bigzipfiles.facebook.com", "captive.apple.com", "cdn.fbsbx.com", "cdn.vidible.tv", "cert.mgt.xboxlive.com", "chevrolet.com", "clientconfig.passport.net",
	"clients2.google.com", "clients3.google.com", "clients4.google.com", "client-s.gateway.messenger.live.com", "connect.facebook.com", "connectivitycheck.android.com", "connectivitycheck.gstatic.com",
	"cpms35.spop10.ams.plex.bz", "cpms.spop10.ams.plex.bz", "creative.ak.fbcdn.net", "c.s-microsoft.com", "ctldl.windowsupdate.com", "dashboard.plex.tv", "def-vef.xboxlive.com", "delivery.vidible.tv",
	"device.auth.xboxlive.com", "dev.virtualearth.net", "directvapplications.hb.omtrdc.net", "directvnow.com", "displaycatalog.mp.microsoft.com", "dl.delivery.mp.microsoft.com", "dl.dropboxusercontent.com",
	"dl.google.com", "download.sonarr.tv", "ecn.dev.virtualearth.net", "edge.api.brightcove.com", "edge-mqtt.facebook.com", "eds.xboxlive.com", "entitlement.auth.adobe.com", "external-lhr0-1.xx.fbcdn.net",
	"external-lhr10-1.xx.fbcdn.net", "external-lhr1-1.xx.fbcdn.net", "external-lhr2-1.xx.fbcdn.net", "external-lhr3-1.xx.fbcdn.net", "external-lhr4-1.xx.fbcdn.net", "external-lhr5-1.xx.fbcdn.net", "external-lhr6-1.xx.fbcdn.net",
	"external-lhr7-1.xx.fbcdn.net", "external-lhr8-1.xx.fbcdn.net", "external-lhr9-1.xx.fbcdn.net", "fbcdn-creative-a.akamaihd.net", "fb.me", "fe3.delivery.dsp.mp.microsoft.com.nsatc.net", "firestore.googleapis.com",
	"forums.sonarr.tv", "fpdownload.adobe.com", "geo-prod.do.dsp.mp.microsoft.com", "gfwsl.geforce.com", "g.live.com", "googleapis.l.google.com", "graph.facebook.com", "gravatar.com", "gsp1.apple.com",
	"help.ui.xboxlive.com", "imagesak.secureserver.net", "img.vidible.tv", "ipv6.msftncsi.com", "i.s-microsoft.com", "itunes.apple.com", "js.maxmind.com", "l.facebook.com", "licensing.xboxlive.com", "livepassdl.conviva.com",
	"login.live.com", "login.microsoftonline.com", "meta-db-worker02.pop.ric.plex.bz", "meta.plex.bz", "meta.plex.tv", "metrics.plex.tv", "m.hotmail.com", "mobile.pipe.aria.microsoft.com", "mqtt.c10r.facebook.com",
	"msftncsi.com", "m.weeklyad.target.com", "nexus.ensighten.com", "nine.plugins.plexapp.com", "node.plexapp.com", "notify.xboxlive.com", "ns1.dropbox.com", "ns2.dropbox.com", "o1.email.plex.tv", "o2.sg0.plex.tv",
	"officeclient.microsoft.com", "outlook.office365.com", "placehold.it", "placeholdit.imgix.net", "plex.tv", "portal.fb.com", "pricelist.skype.com", "prod.telemetry.ros.rockstargames.com", "products.office.com",
	"proxy02.pop.ord.plex.bz", "proxy.plex.bz", "proxy.plex.tv", "pubsub.plex.bz", "pubsub.plex.tv", "redirector.gvt1.com", "reminders-pa.googleapis.com", "s1.symcb.com", "s2.symcb.com", "s3.symcb.com", "s4.symcb.com",
	"s5.symcb.com", "sa.symcb.com", "scontent.fgdl5-1.fna.fbcdn.net", "scontent-lhr3-1.xx.fbcdn.net", "scontent.xx.fbcdn.net", "services.sonarr.tv", "settings-win.data.microsoft.com", "sfdataservice.microsoft.com",
	"s.gateway.messenger.live.com", "skyhook.sonarr.tv", "sls.update.microsoft.com", "sls.update.microsoft.com.akadns.net", "s.marketwatch.com", "s.mzstatic.com", "spclient.wg.spotify.com", "ssl.google-analytics.com",
	"staging.plex.tv", "star.c10r.facebook.com", "star-mini.c10r.facebook.com", "status.plex.tv", "s.youtube.com", "s.zkcdn.net", "t0.ssl.ak.dynamic.tiles.virtualearth.net", "t0.ssl.ak.tiles.virtualearth.net", "tags.tiqcdn.com",
	"telemetry-console.api.playstation.com", "themoviedb.com", "thetvdb.com", "title.auth.xboxlive.com", "title.mgt.xboxlive.com", "tracking.epicgames.com", "tracking-protection.cdn.mozilla.net", "tsfe.trafficshaping.dsp.mp.microsoft.com",
	"tvdb2.plex.tv", "tvthemes.plexapp.com", "tvthemes.plexapp.com.cdn.cloudflare.net", "ui.skype.com", "upload.facebook.com", "v10.events.data.microsoft.com", "v10.vortex-win.data.microsoft.com", "v20.events.data.microsoft.com",
	"v20.vortex-win.data.microsoft.com", "video-stats.l.google.com", "videos.vidible.tv", "wdcpalt.microsoft.com", "wdcp.microsoft.com", "weeklyad.target.com", "weeklyad.target.com.edgesuite.net", "widget-cdn.rpxnow.com",
	"www.apple.com", "www.appleiphonecell.com", "www.asadcdn.com", "www.google-analytics.com", "www.msftncsi.com", "www.plex.tv", "www.xboxlive.com", "xboxexperiencesprod.experimentation.xboxlive.com", "xbox.ipv6.microsoft.com",
	"xflight.xboxlive.com", "xkms.xboxlive.com", "xsts.auth.xboxlive.com",
}

/*
DownloadAllBlacklists attempts to download all hosts files and return combined list of domain names to block.
The special cases of white listed names are removed from return value.
*/
func DownloadAllBlacklists(maxEntries int, logger lalog.Logger) []string {
	wg := new(sync.WaitGroup)
	wg.Add(len(HostsFileURLs))

	// Download all lists in parallel
	lists := make([][]string, len(HostsFileURLs))
	for i, url := range HostsFileURLs {
		go func(i int, url string) {
			resp, err := inet.DoHTTP(context.Background(), inet.HTTPRequest{TimeoutSec: BlackListDownloadTimeoutSec}, url)
			if err == nil {
				names := ExtractNamesFromHostsContent(string(resp.Body))
				logger.Info("DownloadAllBlacklists", url, err, "downloaded %d names, please obey the license the list author uses to publish the data.", len(names))
				lists[i] = names
			} else {
				logger.Warning("DownloadAllBlacklists", url, err, "failed to download blacklist")
				lists[i] = []string{}
			}
			defer wg.Done()
		}(i, url)
	}
	wg.Wait()
	// Calculate unique set of domain names
	set := map[string]struct{}{}
	for _, list := range lists {
		for _, str := range list {
			if _, exists := set[str]; !exists && len(set) < maxEntries {
				set[str] = struct{}{}
			}
		}
	}
	// Remove white listed names
	for _, toRemove := range Whitelist {
		delete(set, toRemove)
	}

	ret := make([]string, 0, len(set))
	for str := range set {
		ret = append(ret, str)
	}
	logger.Info("DownloadAllBlacklists", "", nil, "downloaded %d unique names in total", len(ret))
	return ret
}

/*
ExtractNamesFromHostsContent extracts domain names from hosts file content. It will not return empty lines, comments, and potentially
illegal domain names.
*/
func ExtractNamesFromHostsContent(content string) []string {
	ret := make([]string, 0, 16384)
	for _, line := range strings.Split(content, "\n") {
		if strings.ContainsRune(line, 0) {
			/*
				If attempting to resolve this name that contains NULL byte on Windows, it will unfortunately trigger an
				internal panic in Go's DNS resolution routine.
			*/
			continue
		}
		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			// Skip blank and comments
			continue
		}
		// Find the second field
		space := strings.IndexRune(line, ' ')
		if space == -1 {
			// Skip malformed line
			continue
		}
		line = strings.TrimSpace(line[space:])
		nameEnd := strings.IndexRune(line, '#')
		// Name may be followed by a comment
		if nameEnd == -1 {
			nameEnd = len(line)
		}
		// Extract the name itself. Matching of black list name always takes place in lower case.
		aName := strings.ToLower(strings.TrimSpace(line[:nameEnd]))
		if aName == "" || strings.HasSuffix(aName, "localhost") || strings.HasSuffix(aName, "localdomain") ||
			len(aName) < 4 || len(aName) > 253 {
			// Skip empty names, local names, and overly short names
			// Also, domain name length may not exceed 253 characters according to various technical documents in the public domain.
			continue
		}
		ret = append(ret, aName)
		if len(ret) > MaxNameEntriesToExtract {
			// Avoid taking in too many names
			break
		}
	}
	return ret
}
<commit_msg>dnsd: unblock t.co and bit.ly<commit_after>package dnsd

import (
	"context"
	"strings"
	"sync"

	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/lalog"
)

const (
	/*
		MaxNameEntriesToExtract is the maximum number of entries to be extracted from one instance of hosts file.
		The limit prevents an exceedingly long third party host file from taking too much memory.
	*/
	MaxNameEntriesToExtract = 50000
)

// HostsFileURLs is a collection of URLs where up-to-date ad/malware/spyware blacklist hosts files are published.
var HostsFileURLs = []string{
	"http://winhelp2002.mvps.org/hosts.txt",
	"http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext",
	"http://www.malwaredomainlist.com/hostslist/hosts.txt",
	"http://someonewhocares.org/hosts/hosts",
	"https://hosts.oisd.nl/light",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/ransomware.txt",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/scam.txt",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/tracking.txt",
}

/*
Whitelist is an array of domain names that often appear in black lists, but cause inconvenience when blocked. These
names are removed from downloaded black lists.
*/
var Whitelist = []string{
	/*
		2018-06-24 - youtube app on iPhone fails to save watch history, some sources suggest that this domain name is
		the culprit.
	*/
	"s.youtube.com",
	/*
		2019-05-23 - facebook app and instagram app take ~1 minute to load due to blocked connectivity to their graph
		API domains.
	*/
	"graph.facebook.com",
	"graph.instagram.com",
	// 2022-07-05 - unblock URL shortening services.
	"t.co", "bit.ly",
	// "Common whitelisted domains"(https://discourse.pi-hole.net/t/commonly-whitelisted-domains/212) collected on 2021-02-24
	"106c06cd218b007d-b1e8a1331f68446599e96a4b46a050f5.ams.plex.services", "79423.analytics.edgekey.net", "activity.windows.com", "android.clients.google.com", "api.facebook.com", "api-tv.spotify.com",
	"appleid.apple.com", "app.plex.tv", "appspot-preview.l.google.com", "apps.skype.com", "apresolve.spotify.com", "apt.sonarr.tv", "arc.msn.com", "assets.adobedtm.com", "attestation.xboxlive.com",
	"b-api.facebook.com", "b-graph.facebook.com", "bigzipfiles.facebook.com", "captive.apple.com", "cdn.fbsbx.com", "cdn.vidible.tv", "cert.mgt.xboxlive.com", "chevrolet.com", "clientconfig.passport.net",
	"clients2.google.com", "clients3.google.com", "clients4.google.com", "client-s.gateway.messenger.live.com", "connect.facebook.com", "connectivitycheck.android.com", "connectivitycheck.gstatic.com",
	"cpms35.spop10.ams.plex.bz", "cpms.spop10.ams.plex.bz", "creative.ak.fbcdn.net", "c.s-microsoft.com", "ctldl.windowsupdate.com", "dashboard.plex.tv", "def-vef.xboxlive.com", "delivery.vidible.tv",
	"device.auth.xboxlive.com", "dev.virtualearth.net", "directvapplications.hb.omtrdc.net", "directvnow.com", "displaycatalog.mp.microsoft.com", "dl.delivery.mp.microsoft.com", "dl.dropboxusercontent.com",
	"dl.google.com", "download.sonarr.tv", "ecn.dev.virtualearth.net", "edge.api.brightcove.com", "edge-mqtt.facebook.com", "eds.xboxlive.com", "entitlement.auth.adobe.com", "external-lhr0-1.xx.fbcdn.net",
	"external-lhr10-1.xx.fbcdn.net", "external-lhr1-1.xx.fbcdn.net", "external-lhr2-1.xx.fbcdn.net", "external-lhr3-1.xx.fbcdn.net", "external-lhr4-1.xx.fbcdn.net", "external-lhr5-1.xx.fbcdn.net", "external-lhr6-1.xx.fbcdn.net",
	"external-lhr7-1.xx.fbcdn.net", "external-lhr8-1.xx.fbcdn.net", "external-lhr9-1.xx.fbcdn.net", "fbcdn-creative-a.akamaihd.net", "fb.me", "fe3.delivery.dsp.mp.microsoft.com.nsatc.net", "firestore.googleapis.com",
	"forums.sonarr.tv", "fpdownload.adobe.com", "geo-prod.do.dsp.mp.microsoft.com", "gfwsl.geforce.com", "g.live.com", "googleapis.l.google.com", "graph.facebook.com", "gravatar.com", "gsp1.apple.com",
	"help.ui.xboxlive.com", "imagesak.secureserver.net", "img.vidible.tv", "ipv6.msftncsi.com", "i.s-microsoft.com", "itunes.apple.com", "js.maxmind.com", "l.facebook.com", "licensing.xboxlive.com", "livepassdl.conviva.com",
	"login.live.com", "login.microsoftonline.com", "meta-db-worker02.pop.ric.plex.bz", "meta.plex.bz", "meta.plex.tv", "metrics.plex.tv", "m.hotmail.com", "mobile.pipe.aria.microsoft.com", "mqtt.c10r.facebook.com",
	"msftncsi.com", "m.weeklyad.target.com", "nexus.ensighten.com", "nine.plugins.plexapp.com", "node.plexapp.com", "notify.xboxlive.com", "ns1.dropbox.com", "ns2.dropbox.com", "o1.email.plex.tv", "o2.sg0.plex.tv",
	"officeclient.microsoft.com", "outlook.office365.com", "placehold.it", "placeholdit.imgix.net", "plex.tv", "portal.fb.com", "pricelist.skype.com", "prod.telemetry.ros.rockstargames.com", "products.office.com",
	"proxy02.pop.ord.plex.bz", "proxy.plex.bz", "proxy.plex.tv", "pubsub.plex.bz", "pubsub.plex.tv", "redirector.gvt1.com", "reminders-pa.googleapis.com", "s1.symcb.com", "s2.symcb.com", "s3.symcb.com", "s4.symcb.com",
	"s5.symcb.com", "sa.symcb.com", "scontent.fgdl5-1.fna.fbcdn.net", "scontent-lhr3-1.xx.fbcdn.net", "scontent.xx.fbcdn.net", "services.sonarr.tv", "settings-win.data.microsoft.com", "sfdataservice.microsoft.com",
	"s.gateway.messenger.live.com", "skyhook.sonarr.tv", "sls.update.microsoft.com", "sls.update.microsoft.com.akadns.net", "s.marketwatch.com", "s.mzstatic.com", "spclient.wg.spotify.com", "ssl.google-analytics.com",
	"staging.plex.tv", "star.c10r.facebook.com", "star-mini.c10r.facebook.com", "status.plex.tv", "s.youtube.com", "s.zkcdn.net", "t0.ssl.ak.dynamic.tiles.virtualearth.net", "t0.ssl.ak.tiles.virtualearth.net", "tags.tiqcdn.com",
	"telemetry-console.api.playstation.com", "themoviedb.com", "thetvdb.com", "title.auth.xboxlive.com", "title.mgt.xboxlive.com", "tracking.epicgames.com", "tracking-protection.cdn.mozilla.net", "tsfe.trafficshaping.dsp.mp.microsoft.com",
	"tvdb2.plex.tv", "tvthemes.plexapp.com", "tvthemes.plexapp.com.cdn.cloudflare.net", "ui.skype.com", "upload.facebook.com", "v10.events.data.microsoft.com", "v10.vortex-win.data.microsoft.com", "v20.events.data.microsoft.com",
	"v20.vortex-win.data.microsoft.com", "video-stats.l.google.com", "videos.vidible.tv", "wdcpalt.microsoft.com", "wdcp.microsoft.com", "weeklyad.target.com", "weeklyad.target.com.edgesuite.net", "widget-cdn.rpxnow.com",
	"www.apple.com", "www.appleiphonecell.com", "www.asadcdn.com", "www.google-analytics.com", "www.msftncsi.com", "www.plex.tv", "www.xboxlive.com", "xboxexperiencesprod.experimentation.xboxlive.com", "xbox.ipv6.microsoft.com",
	"xflight.xboxlive.com", "xkms.xboxlive.com", "xsts.auth.xboxlive.com",
}

/*
DownloadAllBlacklists attempts to download all hosts files and return combined list of domain names to block.
The special cases of white listed names are removed from return value.
*/
func DownloadAllBlacklists(maxEntries int, logger lalog.Logger) []string {
	wg := new(sync.WaitGroup)
	wg.Add(len(HostsFileURLs))

	// Download all lists in parallel
	lists := make([][]string, len(HostsFileURLs))
	for i, url := range HostsFileURLs {
		go func(i int, url string) {
			resp, err := inet.DoHTTP(context.Background(), inet.HTTPRequest{TimeoutSec: BlackListDownloadTimeoutSec}, url)
			if err == nil {
				names := ExtractNamesFromHostsContent(string(resp.Body))
				logger.Info("DownloadAllBlacklists", url, err, "downloaded %d names, please obey the license the list author uses to publish the data.", len(names))
				lists[i] = names
			} else {
				logger.Warning("DownloadAllBlacklists", url, err, "failed to download blacklist")
				lists[i] = []string{}
			}
			defer wg.Done()
		}(i, url)
	}
	wg.Wait()
	// Calculate unique set of domain names
	set := map[string]struct{}{}
	for _, list := range lists {
		for _, str := range list {
			if _, exists := set[str]; !exists && len(set) < maxEntries {
				set[str] = struct{}{}
			}
		}
	}
	// Remove white listed names
	for _, toRemove := range Whitelist {
		delete(set, toRemove)
	}

	ret := make([]string, 0, len(set))
	for str := range set {
		ret = append(ret, str)
	}
	logger.Info("DownloadAllBlacklists", "", nil, "downloaded %d unique names in total", len(ret))
	return ret
}

/*
ExtractNamesFromHostsContent extracts domain names from hosts file content. It will not return empty lines, comments, and potentially
illegal domain names.
*/
func ExtractNamesFromHostsContent(content string) []string {
	ret := make([]string, 0, 16384)
	for _, line := range strings.Split(content, "\n") {
		if strings.ContainsRune(line, 0) {
			/*
				If attempting to resolve this name that contains NULL byte on Windows, it will unfortunately trigger an
				internal panic in Go's DNS resolution routine.
			*/
			continue
		}
		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			// Skip blank and comments
			continue
		}
		// Find the second field
		space := strings.IndexRune(line, ' ')
		if space == -1 {
			// Skip malformed line
			continue
		}
		line = strings.TrimSpace(line[space:])
		nameEnd := strings.IndexRune(line, '#')
		// Name may be followed by a comment
		if nameEnd == -1 {
			nameEnd = len(line)
		}
		// Extract the name itself. Matching of black list name always takes place in lower case.
		aName := strings.ToLower(strings.TrimSpace(line[:nameEnd]))
		if aName == "" || strings.HasSuffix(aName, "localhost") || strings.HasSuffix(aName, "localdomain") ||
			len(aName) < 4 || len(aName) > 253 {
			// Skip empty names, local names, and overly short names
			// Also, domain name length may not exceed 253 characters according to various technical documents in the public domain.
			continue
		}
		ret = append(ret, aName)
		if len(ret) > MaxNameEntriesToExtract {
			// Avoid taking in too many names
			break
		}
	}
	return ret
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bytes"
	"encoding/csv"
	"errors"
	"io"
	"net"
	"path"
	"sort"
	"strconv"
)

type geoNameId uint32

func ParseGeoNameId(n string) (geoNameId, error) {
	i, err := strconv.ParseUint(n, 10, 32)
	if err != nil {
		return 0, err
	}
	return geoNameId(i), nil
}

type continent struct {
	Code string
	Name string
}

type country struct {
	Code string
	Name string
}

type GeoDBEntry struct {
	Continent continent
	Country   country
}

type geoNames map[geoNameId]GeoDBEntry

func readGeoNames(r io.Reader) (geoNames, error) {
	g := geoNames{}
	c := csv.NewReader(r)
	for i := 0; ; i++ {
		record, err := c.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(record) < 6 {
			return nil, errors.New("unexpected line")
		}
		if i == 0 {
			continue
		}
		id, err := ParseGeoNameId(record[0])
		if err != nil {
			return nil, err
		}
		g[id] = GeoDBEntry{
			Continent: continent{
				Code: record[2],
				Name: record[3],
			},
			Country: country{
				Code: record[4],
				Name: record[5],
			},
		}
	}
	return g, nil
}

func (g geoNames) Get(id geoNameId) (GeoDBEntry, bool) {
	e, ok := g[id]
	return e, ok
}

func less(left, right net.IP) bool {
	leftLen := len(left)
	rightLen := len(right)
	if leftLen != rightLen {
		return leftLen < rightLen
	}
	for idx, b := range right {
		if left[idx] != b {
			return left[idx] < b
		}
	}
	return false
}

type network struct {
	net.IPNet
	NameId geoNameId
}

type networks []network

func readNetworks(r io.Reader) (networks, error) {
	n := networks{}

	c := csv.NewReader(r)
	for i := 0; ; i++ {
		record, err := c.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(record) < 2 {
			return nil, errors.New("unexpected line")
		}
		if i == 0 || record[1] == "" {
			continue
		}

		_, nw, err := net.ParseCIDR(record[0])
		if err != nil {
			return nil, err
		}
		id, err := ParseGeoNameId(record[1])
		if err != nil {
			return nil, err
		}
		n = append(n, network{IPNet: *nw, NameId: id})
	}

	sort.Sort(n)
	return n, nil
}

func (n networks) Len() int {
	return len(n)
}

func (n networks) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

func (n networks) Less(i, j int) bool {
	return less(n[i].IP, n[j].IP)
}

func (n networks) Get(ip net.IP) (geoNameId, bool) {
	idx := sort.Search(len(n), func(i int) bool {
		return less(ip, n[i].IP)
	})
	if idx == 0 {
		return 0, false
	}

	if idx == -1 {
		idx = len(n)
	}
	idx -= 1
	if idx < len(n) && n[idx].IPNet.Contains(ip) {
		return n[idx].NameId, true
	}
	return 0, false
}

type GeoDB struct {
	names geoNames
	ipv4  networks
	ipv6  networks
}

func withEachFile(z *zip.Reader, fn func(string, io.Reader) error) error {
	for _, file := range z.File {
		err := func(f *zip.File) error {
			r, err := f.Open()
			if err != nil {
				return err
			}
			defer r.Close()

			return fn(f.Name, r)
		}(file)

		if err != nil {
			return err
		}
	}
	return nil
}

func NewGeoDB(b []byte) (*GeoDB, error) {
	z, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
	if err != nil {
		return nil, err
	}

	var ipv4 networks
	var ipv6 networks
	var names geoNames
	withEachFile(z, func(filename string, reader io.Reader) (err error) {
		switch path.Base(filename) {
		case "GeoLite2-Country-Blocks-IPv4.csv":
			ipv4, err = readNetworks(reader)
		case "GeoLite2-Country-Blocks-IPv6.csv":
			ipv6, err = readNetworks(reader)
		case "GeoLite2-Country-Locations-en.csv":
			names, err = readGeoNames(reader)
		}
		return
	})

	if ipv4 == nil || ipv6 == nil || names == nil {
		return nil, errors.New("couldn't find all sections")
	}
	return &GeoDB{names: names, ipv4: ipv4, ipv6: ipv6}, nil
}

func (d *GeoDB) Get(s string) (GeoDBEntry, bool) {
	ip := net.ParseIP(s)

	var id geoNameId
	var ok bool
	if ipv4 := ip.To4(); ipv4 != nil {
		id, ok = d.ipv4.Get(ipv4)
	} else if ipv6 := ip.To16(); ipv6 != nil {
		id, ok = d.ipv6.Get(ipv6)
	}

	if !ok {
		return GeoDBEntry{}, false
	}
	e, ok := d.names.Get(id)
	return e, ok
}
<commit_msg>Lowercase JSON output keys<commit_after>package main

import (
	"archive/zip"
	"bytes"
	"encoding/csv"
	"errors"
	"io"
	"net"
	"path"
	"sort"
	"strconv"
)

type geoNameId uint32

func ParseGeoNameId(n string) (geoNameId, error) {
	i, err := strconv.ParseUint(n, 10, 32)
	if err != nil {
		return 0, err
	}
	return geoNameId(i), nil
}

type continent struct {
	Code string `json:"code,omitempty"`
	Name string `json:"name,omitempty"`
}

type country struct {
	Code string `json:"code,omitempty"`
	Name string `json:"name,omitempty"`
}

type GeoDBEntry struct {
	Continent continent `json:"continent"`
	Country   country   `json:"country"`
}

type geoNames map[geoNameId]GeoDBEntry

func readGeoNames(r io.Reader) (geoNames, error) {
	g := geoNames{}
	c := csv.NewReader(r)
	for i := 0; ; i++ {
		record, err := c.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(record) < 6 {
			return nil, errors.New("unexpected line")
		}
		if i == 0 {
			continue
		}
		id, err := ParseGeoNameId(record[0])
		if err != nil {
			return nil, err
		}
		g[id] = GeoDBEntry{
			Continent: continent{
				Code: record[2],
				Name: record[3],
			},
			Country: country{
				Code: record[4],
				Name: record[5],
			},
		}
	}
	return g, nil
}

func (g geoNames) Get(id geoNameId) (GeoDBEntry, bool) {
	e, ok := g[id]
	return e, ok
}

func less(left, right net.IP) bool {
	leftLen := len(left)
	rightLen := len(right)
	if leftLen != rightLen {
		return leftLen < rightLen
	}
	for idx, b := range right {
		if left[idx] != b {
			return left[idx] < b
		}
	}
	return false
}

type network struct {
	net.IPNet
	NameId geoNameId
}

type networks []network

func readNetworks(r io.Reader) (networks, error) {
	n := networks{}

	c := csv.NewReader(r)
	for i := 0; ; i++ {
		record, err := c.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		if len(record) < 2 {
			return nil, errors.New("unexpected line")
		}
		if i == 0 || record[1] == "" {
			continue
		}

		_, nw, err := net.ParseCIDR(record[0])
		if err != nil {
			return nil, err
		}
		id, err := ParseGeoNameId(record[1])
		if err != nil {
			return nil, err
		}
		n = append(n, network{IPNet: *nw, NameId: id})
	}

	sort.Sort(n)
	return n, nil
}

func (n networks) Len() int {
	return len(n)
}

func (n networks) Swap(i, j int) {
	n[i], n[j] = n[j], n[i]
}

func (n networks) Less(i, j int) bool {
	return less(n[i].IP, n[j].IP)
}

func (n networks) Get(ip net.IP) (geoNameId, bool) {
	idx := sort.Search(len(n), func(i int) bool {
		return less(ip, n[i].IP)
	})
	if idx == 0 {
		return 0, false
	}

	if idx == -1 {
		idx = len(n)
	}
	idx -= 1
	if idx < len(n) && n[idx].IPNet.Contains(ip) {
		return n[idx].NameId, true
	}
	return 0, false
}

type GeoDB struct {
	names geoNames
	ipv4  networks
	ipv6  networks
}

func withEachFile(z *zip.Reader, fn func(string, io.Reader) error) error {
	for _, file := range z.File {
		err := func(f *zip.File) error {
			r, err := f.Open()
			if err != nil {
				return err
			}
			defer r.Close()

			return fn(f.Name, r)
		}(file)

		if err != nil {
			return err
		}
	}
	return nil
}

func NewGeoDB(b []byte) (*GeoDB, error) {
	z, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
	if err != nil {
		return nil, err
	}

	var ipv4 networks
	var ipv6 networks
	var names geoNames
	withEachFile(z, func(filename string, reader io.Reader) (err error) {
		switch path.Base(filename) {
		case "GeoLite2-Country-Blocks-IPv4.csv":
			ipv4, err = readNetworks(reader)
		case "GeoLite2-Country-Blocks-IPv6.csv":
			ipv6, err = readNetworks(reader)
		case "GeoLite2-Country-Locations-en.csv":
			names, err = readGeoNames(reader)
		}
		return
	})

	if ipv4 == nil || ipv6 == nil || names == nil {
		return nil, errors.New("couldn't find all sections")
	}
	return &GeoDB{names: names, ipv4: ipv4, ipv6: ipv6}, nil
}

func (d *GeoDB) Get(s string) (GeoDBEntry, bool) {
	ip := net.ParseIP(s)

	var id geoNameId
	var ok bool
	if ipv4 := ip.To4(); ipv4 != nil {
		id, ok = d.ipv4.Get(ipv4)
	} else if ipv6 := ip.To16(); ipv6 != nil {
		id, ok = d.ipv6.Get(ipv6)
	}

	if !ok {
		return GeoDBEntry{}, false
	}
	e, ok := d.names.Get(id)
	return e, ok
}
<|endoftext|>


<commit_before>package nv

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"reflect"
	"sort"

	xdr "github.com/davecgh/go-xdr/xdr2"
)

func Encode(i interface{}) ([]byte, error) {
	if i == nil {
		return nil, errors.New("can not encode a nil pointer")
	}

	v := reflect.ValueOf(i)
	if !v.IsValid() {
		return nil, fmt.Errorf("type '%s' is invalid", v.Kind().String())
	}

	var err error
	buff := bytes.NewBuffer(nil)
	if err = binary.Write(buff, binary.BigEndian, encoding{Encoding: 1, Endianess: 1}); err != nil {
		return nil, err
	}

	if err = encodeList(buff, v); err != nil {
		return nil, err
	}

	return buff.Bytes(), nil
}

func encodeList(w io.Writer, v reflect.Value) error {
	var err error
	if err = binary.Write(w, binary.BigEndian, header{Flag: UNIQUE_NAME}); err != nil {
		return err
	}

	v = deref(v)
	switch v.Kind() {
	case reflect.Struct:
		_, err = encodeStruct(v, w)
	case reflect.Map:
		keys := make([]string, len(v.MapKeys()))
		for i, k := range v.MapKeys() {
			keys[i] = k.Interface().(string)
		}
		sort.Strings(keys)

		for _, k := range keys {
			_, err = encodeItem(w, k, v.MapIndex(reflect.ValueOf(k)))
			if err != nil {
				return err
			}
		}
		err = binary.Write(w, binary.BigEndian, uint64(0))
	default:
		return fmt.Errorf("invalid type '%s', must be a struct", v.Kind().String())
	}

	return err
}

func encodeStruct(v reflect.Value, w io.Writer) (int, error) {
	var err error
	size := 0

	forEachField(v, func(i int, field reflect.Value) {
		name := v.Type().Field(i).Name
		if tags := getTags(i, v); len(tags) > 0 && tags[0] != "" {
			name = tags[0]
		}
		encodeItem(w, name, field)
	})

	if err = binary.Write(w, binary.BigEndian, uint64(0)); err != nil {
		return 0, err
	}
	return size + 8, nil
}

func encodeItem(w io.Writer, name string, field reflect.Value) ([]byte, error) {
	p := pair{
		Name:      name,
		NElements: 1,
	}

	switch t := field.Kind(); t {
	case reflect.String:
		p.Type = STRING
	case reflect.Uint64:
		p.Type = UINT64
	case reflect.Int32, reflect.Int:
		p.Type = INT32
	case reflect.Struct:
		p.Type = NVLIST
	default:
		return nil, fmt.Errorf("unknown type:", t)
	}

	if field.Type().String() == "nv.mVal" {
		p.Type = field.FieldByName("Type").Interface().(dataType)
		p.Name = field.FieldByName("Name").Interface().(string)
		p.data = field.FieldByName("Value").Interface()
	} else {
		if !field.CanSet() {
			panic("can't Set")
			return nil, nil
		}
		p.data = field.Interface()
	}

	if p.Type == UNKNOWN || p.Type > DOUBLE {
		return nil, fmt.Errorf("invalid Type '%v'", p.Type)
	}

	value := p.data
	vbuf := &bytes.Buffer{}
	switch p.Type {
	case BYTE:
		value = int8(value.(uint8))
	case UINT8:
		value = int(int8(value.(uint8)))
	case BYTE_ARRAY:
		p.NElements = uint32(len(value.([]byte)))
		n := int(p.NElements)
		arrType := reflect.ArrayOf(n, reflect.TypeOf(byte(0)))
		arr := reflect.New(arrType).Elem()
		for i, b := range value.([]byte) {
			arr.Index(i).SetUint(uint64(b))
		}
		value = arr.Interface()
	case BOOLEAN_ARRAY:
		p.NElements = uint32(len(value.([]bool)))
	case INT8_ARRAY:
		p.NElements = uint32(len(value.([]int8)))
	case INT16_ARRAY:
		p.NElements = uint32(len(value.([]int16)))
	case INT32_ARRAY:
		p.NElements = uint32(len(value.([]int32)))
	case INT64_ARRAY:
		p.NElements = uint32(len(value.([]int64)))
	case UINT8_ARRAY:
		// this one is weird since UINT8s are encoded as char
		// aka int32s... :(
		p.NElements = uint32(len(value.([]uint8)))
		n := int(p.NElements)
		sliceType := reflect.SliceOf(reflect.TypeOf(int32(0)))
		slice := reflect.MakeSlice(sliceType, n, n)
		for i, b := range value.([]uint8) {
			slice.Index(i).SetInt(int64(int8(b)))
		}
		value = slice.Interface()
	case UINT16_ARRAY:
		p.NElements = uint32(len(value.([]uint16)))
	case UINT32_ARRAY:
		p.NElements = uint32(len(value.([]uint32)))
	case UINT64_ARRAY:
		p.NElements = uint32(len(value.([]uint64)))
	case STRING_ARRAY:
		p.NElements = uint32(len(value.([]string)))
		arrType := reflect.ArrayOf(int(p.NElements), reflect.TypeOf(""))
		arr := reflect.New(arrType).Elem()
		for i, b := range value.([]string) {
			arr.Index(i).SetString(b)
		}
		value = arr.Interface()
	case NVLIST:
		if err := encodeList(vbuf, reflect.ValueOf(value)); err != nil {
			return nil, err
		}
		p.data = vbuf.Bytes()
	case NVLIST_ARRAY:
		p.NElements = uint32(len(value.([]mList)))
		for _, l := range value.([]mList) {
			if err := encodeList(vbuf, reflect.ValueOf(l)); err != nil {
				return nil, err
			}
		}
		p.data = vbuf.Bytes()
	}

	if vbuf.Len() == 0 {
		_, err := xdr.NewEncoder(vbuf).Encode(value)
		if err != nil {
			return nil, err
		}
	}

	p.EncodedSize = uint32(p.encodedSize())
	p.DecodedSize = uint32(p.decodedSize())

	pbuf := &bytes.Buffer{}
	_, err := xdr.NewEncoder(pbuf).Encode(p)
	if err != nil {
		return nil, err
	}

	_, err = pbuf.WriteTo(w)
	if err != nil {
		return nil, err
	}
	_, err = vbuf.WriteTo(w)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

func deref(v reflect.Value) reflect.Value {
	for v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	return v
}
<commit_msg>nv/encode: skip unexported fields<commit_after>package nv

import (
	"bytes"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"reflect"
	"sort"

	xdr "github.com/davecgh/go-xdr/xdr2"
)

func Encode(i interface{}) ([]byte, error) {
	if i == nil {
		return nil, errors.New("can not encode a nil pointer")
	}

	v := reflect.ValueOf(i)
	if !v.IsValid() {
		return nil, fmt.Errorf("type '%s' is invalid", v.Kind().String())
	}

	var err error
	buff := bytes.NewBuffer(nil)
	if err = binary.Write(buff, binary.BigEndian, encoding{Encoding: 1, Endianess: 1}); err != nil {
		return nil, err
	}

	if err = encodeList(buff, v); err != nil {
		return nil, err
	}

	return buff.Bytes(), nil
}

func encodeList(w io.Writer, v reflect.Value) error {
	var err error
	if err = binary.Write(w, binary.BigEndian, header{Flag: UNIQUE_NAME}); err != nil {
		return err
	}

	v = deref(v)
	switch v.Kind() {
	case reflect.Struct:
		_, err = encodeStruct(v, w)
	case reflect.Map:
		keys := make([]string, len(v.MapKeys()))
		for i, k := range v.MapKeys() {
			keys[i] = k.Interface().(string)
		}
		sort.Strings(keys)

		for _, k := range keys {
			_, err = encodeItem(w, k, v.MapIndex(reflect.ValueOf(k)))
			if err != nil {
				return err
			}
		}
		err = binary.Write(w, binary.BigEndian, uint64(0))
	default:
		return fmt.Errorf("invalid type '%s', must be a struct", v.Kind().String())
	}

	return err
}

func encodeStruct(v reflect.Value, w io.Writer) (int, error) {
	var err error
	size := 0

	forEachField(v, func(i int, field reflect.Value) {
		// Skip fields that can't be set (e.g. unexported)
		if !field.CanSet() {
			return
		}
		name := v.Type().Field(i).Name
		if tags := getTags(i, v); len(tags) > 0 && tags[0] != "" {
			name = tags[0]
		}
		encodeItem(w, name, field)
	})

	if err = binary.Write(w, binary.BigEndian, uint64(0)); err != nil {
		return 0, err
	}
	return size + 8, nil
}

func encodeItem(w io.Writer, name string, field reflect.Value) ([]byte, error) {
	p := pair{
		Name:      name,
		NElements: 1,
	}

	switch t := field.Kind(); t {
	case reflect.String:
		p.Type = STRING
	case reflect.Uint64:
		p.Type = UINT64
	case reflect.Int32, reflect.Int:
		p.Type = INT32
	case reflect.Struct:
		p.Type = NVLIST
	default:
		return nil, fmt.Errorf("unknown type:", t)
	}

	if field.Type().String() == "nv.mVal" {
		p.Type = field.FieldByName("Type").Interface().(dataType)
		p.Name = field.FieldByName("Name").Interface().(string)
		p.data = field.FieldByName("Value").Interface()
	} else {
		if !field.CanSet() {
			panic("can't Set")
			return nil, nil
		}
		p.data = field.Interface()
	}

	if p.Type == UNKNOWN || p.Type > DOUBLE {
		return nil, fmt.Errorf("invalid Type '%v'", p.Type)
	}

	value := p.data
	vbuf := &bytes.Buffer{}
	switch p.Type {
	case BYTE:
		value = int8(value.(uint8))
	case UINT8:
		value = int(int8(value.(uint8)))
	case BYTE_ARRAY:
		p.NElements = uint32(len(value.([]byte)))
		n := int(p.NElements)
		arrType := reflect.ArrayOf(n, reflect.TypeOf(byte(0)))
		arr := reflect.New(arrType).Elem()
		for i, b := range value.([]byte) {
			arr.Index(i).SetUint(uint64(b))
		}
		value = arr.Interface()
	case BOOLEAN_ARRAY:
		p.NElements = uint32(len(value.([]bool)))
	case INT8_ARRAY:
		p.NElements = uint32(len(value.([]int8)))
	case INT16_ARRAY:
		p.NElements = uint32(len(value.([]int16)))
	case INT32_ARRAY:
		p.NElements = uint32(len(value.([]int32)))
	case INT64_ARRAY:
		p.NElements = uint32(len(value.([]int64)))
	case UINT8_ARRAY:
		// this one is weird since UINT8s are encoded as char
		// aka int32s... :(
		p.NElements = uint32(len(value.([]uint8)))
		n := int(p.NElements)
		sliceType := reflect.SliceOf(reflect.TypeOf(int32(0)))
		slice := reflect.MakeSlice(sliceType, n, n)
		for i, b := range value.([]uint8) {
			slice.Index(i).SetInt(int64(int8(b)))
		}
		value = slice.Interface()
	case UINT16_ARRAY:
		p.NElements = uint32(len(value.([]uint16)))
	case UINT32_ARRAY:
		p.NElements = uint32(len(value.([]uint32)))
	case UINT64_ARRAY:
		p.NElements = uint32(len(value.([]uint64)))
	case STRING_ARRAY:
		p.NElements = uint32(len(value.([]string)))
		arrType := reflect.ArrayOf(int(p.NElements), reflect.TypeOf(""))
		arr := reflect.New(arrType).Elem()
		for i, b := range value.([]string) {
			arr.Index(i).SetString(b)
		}
		value = arr.Interface()
	case NVLIST:
		if err := encodeList(vbuf, reflect.ValueOf(value)); err != nil {
			return nil, err
		}
		p.data = vbuf.Bytes()
	case NVLIST_ARRAY:
		p.NElements = uint32(len(value.([]mList)))
		for _, l := range value.([]mList) {
			if err := encodeList(vbuf, reflect.ValueOf(l)); err != nil {
				return nil, err
			}
		}
		p.data = vbuf.Bytes()
	}

	if vbuf.Len() == 0 {
		_, err := xdr.NewEncoder(vbuf).Encode(value)
		if err != nil {
			return nil, err
		}
	}

	p.EncodedSize = uint32(p.encodedSize())
	p.DecodedSize = uint32(p.decodedSize())

	pbuf := &bytes.Buffer{}
	_, err := xdr.NewEncoder(pbuf).Encode(p)
	if err != nil {
		return nil, err
	}

	_, err = pbuf.WriteTo(w)
	if err != nil {
		return nil, err
	}
	_, err = vbuf.WriteTo(w)
	if err != nil {
		return nil, err
	}

	return nil, nil
}

func deref(v reflect.Value) reflect.Value {
	for v.Kind() == reflect.Ptr {
		v = v.Elem()
	}
	return v
}
<|endoftext|>