Merge pull request #2613 from Slenderman00/masteruninett

Palo Alto ARP table plugin

NOTES.rst

@@ -22,6 +22,30 @@ Dependency changes
 
 .. IMPORTANT:: NAV 5.10 requires PostgreSQL to be at least version *11*.
 
+New dependencies
+~~~~~~~~~~~~~~~~
+
+Dependencies to these Python modules have been added in order to support
+communicating with Palo Alto firewall APIs:
+
+* :mod:`PyOpenSSL` (``==23.3.0``)
+* :mod:`service-identity` (``==21.1.0``)
+
+Support for fetching ARP cache data from Palo Alto firewalls
+------------------------------------------------------------
+
+Palo Alto firewalls do support SNMP.  They do not, however, support fetching
+ARP cache data using SNMP.  A new ipdevpoll plugin, ``paloaltoarp``, has been
+added to fetch ARP cache data using the REST API built in to these firewall
+products.
+
+Access credentials for Palo Alto firewalls need to be configured in
+:file:`ipdevpoll.conf`, but a later NAV release should move to providing
+management profiles also for this.
+
+Please read more in :doc:`the ipdevpoll reference documentation
+</reference/ipdevpoll>` for configuration details.
+
 
 NAV 5.9
 =======

changelog.d/2613.added.md

@@ -0,0 +1 @@
+New ipdevpoll plugin to fetch ARP cache data from Palo Alto firewall APIs

doc/reference/ipdevpoll.rst

@@ -106,6 +106,36 @@ Section [linkstate]
   The value ``any`` will generate alerts for all link state changes, but
   **this is not recommended** for performance reasons.
 
+Section [paloaltoarp]
+---------------------
+
+This section configures the Palo Alto ARP plugin.  Palo Alto firewalls do
+support SNMP.  They do not, however, support fetching ARP cache data using
+SNMP.  This plugin enables fetching ARP records from Palo Alto firewalls using
+their built-in REST API.
+
+Currently, there is no management profile type for this type of REST APIs, so
+credentials to access a Palo Alto firewall's API must be configured in this
+section.
+
+If you have a Palo Alto firewall named ``example-fw.example.org``, with an IP
+address of ``10.0.42.42`` and a secret API token of
+``762e87e0ec051a1c5211a08dd48e7a93720eee63``, you can configure this in this
+section by adding::
+
+  example-fw.example.org = 762e87e0ec051a1c5211a08dd48e7a93720eee63
+
+Or, alternatively::
+
+  10.0.42.42 = 762e87e0ec051a1c5211a08dd48e7a93720eee63
+
+
+.. warning:: The Palo Alto ARP plugin does not currently verify TLS
+             certificates when accessing a Palo Alto API.  This will be changed
+             at a later date, but if it worries you, you should not use the
+             plugin yet.
+
+
 Job sections
 ------------
 

python/nav/etc/ipdevpoll.conf

@@ -79,6 +79,7 @@ linkaggregate=
 bgp=
 poe=
 juniperalarm=
+paloaltoarp=
 
 [job_inventory]
 #
@@ -132,9 +133,9 @@ description: Checks for changes in the reverse DNS records of devices
 interval: 30m
 intensity: 0
 plugins:
- arp
+ arp paloaltoarp
 description:
- The ip2mac job logs IP to MAC address mappings from routers
+ The ip2mac job logs IP to MAC address mappings from routers and firewalls
  (i.e. from IPv4 ARP and IPv6 Neighbor caches)
 
 
@@ -225,6 +226,15 @@ filter = topology
 # using the staticroutes plugins. Value is a number of seconds between requests.
 #throttle-delay=0.0
 
+[paloaltoarp]
+# Until a management profile type for (Palo Alto) REST API credentials
+# exist in NAV, this section is used to configure API tokens/keys per
+# Palo Alto firewall.  Identify each Palo Alto firewall with either its
+# NAV sysname or management IP address:
+
+#hostname = secret-API-key
+#ip = another-secret-API-key
+
 [sensors]
 # A space-separated list of Python modules to load into ipdevpoll as the
 # sensors plugin is loaded. An asterisk suffix will cause all modules in that

python/nav/ipdevpoll/plugins/paloaltoarp.py

@@ -0,0 +1,150 @@
+#
+# Copyright (C) 2023, 2024 University of Tromsø
+# Copyright (C) 2024 Sikt
+#
+# This file is part of Network Administration Visualized (NAV).
+#
+# NAV is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License version 3 as published by the Free
+# Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.  You should have received a copy of the GNU General Public
+# License along with NAV. If not, see <http://www.gnu.org/licenses/>.
+#
+
+"""ipdevpoll plugin for fetching arp mappings from Palo Alto firewalls
+
+Add [paloaltoarp] section to ipdevpoll.conf
+add hostname = key to [paloaltoarp] section
+for example:
+[paloaltoarp]
+10.0.0.0 = abcdefghijklmnopqrstuvwxyz1234567890
+
+"""
+
+import xml.etree.ElementTree as ET
+from typing import Dict
+
+from IPy import IP
+from twisted.internet import defer, reactor, ssl
+from twisted.internet.defer import returnValue
+from twisted.web import client
+from twisted.web.client import Agent
+from twisted.web.http_headers import Headers
+
+from nav import buildconf
+from nav.ipdevpoll.plugins.arp import Arp
+
+
+class PaloaltoArp(Arp):
+    configured_devices: Dict[str, str] = {}
+
+    @classmethod
+    def on_plugin_load(cls):
+        """Loads the list of PaloAlto access keys from ipdevpoll.conf into the plugin
+        class instance, so that `can_handle` will be able to answer which devices
+        this plugin can run for.
+        """
+        from nav.ipdevpoll.config import ipdevpoll_conf
+
+        cls._logger.debug("loading paloaltoarp configuration")
+        if 'paloaltoarp' not in ipdevpoll_conf:
+            cls._logger.debug("PaloaltoArp config section NOT found")
+            return
+        cls._logger.debug("PaloaltoArp config section found")
+        cls.configured_devices = dict(ipdevpoll_conf['paloaltoarp'])
+
+    @classmethod
+    def can_handle(cls, netbox):
+        """Return True if this plugin can handle the given netbox."""
+        return (
+            netbox.sysname in cls.configured_devices
+            or str(netbox.ip) in cls.configured_devices
+        )
+
+    @defer.inlineCallbacks
+    def handle(self):
+        """Handle plugin business, return a deferred."""
+
+        api_key = self.configured_devices.get(
+            str(self.netbox.ip), self.configured_devices.get(self.netbox.sysname, "")
+        )
+        self._logger.debug("Collecting IP/MAC mappings for Paloalto device")
+
+        mappings = yield self._get_paloalto_arp_mappings(self.netbox.ip, api_key)
+        if mappings is None:
+            self._logger.info("No mappings found for Paloalto device")
+            returnValue(None)
+
+        yield self._process_data(mappings)
+
+        returnValue(None)
+
+    @defer.inlineCallbacks
+    def _get_paloalto_arp_mappings(self, address: str, key: str):
+        """Get mappings from Paloalto device"""
+
+        arptable = yield self._do_request(address, key)
+        if arptable is None:
+            returnValue(None)
+
+        # process arpdata into an array of mappings
+        mappings = parse_arp(arptable.decode('utf-8'))
+        returnValue(mappings)
+
+    @defer.inlineCallbacks
+    def _do_request(self, address: str, key: str):
+        """Make request to Paloalto device"""
+
+        class SslPolicy(client.BrowserLikePolicyForHTTPS):
+            def creatorForNetloc(self, hostname, port):
+                return ssl.CertificateOptions(verify=False)
+
+        url = f"https://{address}/api/?type=op&cmd=<show><arp><entry+name+=+'all'/></arp></show>&key={key}"
+        self._logger.debug("making request: %s", url)
+
+        agent = Agent(reactor, contextFactory=SslPolicy())
+
+        try:
+            response = yield agent.request(
+                b'GET',
+                url.encode('utf-8'),
+                Headers(
+                    {'User-Agent': [f'NAV/PaloaltoArp; version {buildconf.VERSION}']}
+                ),
+                None,
+            )
+        except Exception:  # noqa
+            self._logger.exception(
+                "Error when talking to PaloAlto API. "
+                "Make sure the device is reachable and the API key is correct."
+            )
+            returnValue(None)
+
+        response = yield client.readBody(response)
+        returnValue(response)
+
+
+def parse_arp(arp):
+    """
+    Create mappings from arp table
+    xml.etree.ElementTree is considered insecure: https://docs.python.org/3/library/xml.html#xml-vulnerabilities
+    However, since we are not parsing untrusted data, this should not be a problem.
+    """
+
+    arps = []
+
+    root = ET.fromstring(arp)
+    entries = root[0][4]
+    for entry in entries:
+        status = entry[0].text
+        ip = entry[1].text
+        mac = entry[2].text
+        if status.strip() != "i":
+            if mac != "(incomplete)":
+                arps.append(('ifindex', IP(ip), mac))
+
+    return arps

requirements/base.txt

@@ -39,3 +39,12 @@ backports.zoneinfo ; python_version < '3.9'
 importlib_metadata ; python_version < '3.8'
 importlib_resources ; python_version < '3.9'
 git+https://github.com/Uninett/[email protected]#egg=drf-oidc-auth
+
+# The following modules are really sub-requirements of Twisted, not of
+# NAV directly.  They may be optional from Twisted's point of view,
+# but they are required for parts of the Twisted library that NAV uses:
+#
+# PyOpenSSL is required for TLS verification during PaloAlto API GET operations
+PyOpenSSL==23.3.0
+# service-identity is required to make TLS communication libraries shut up about potential MITM attacks
+service-identity==21.1.0

tests/unittests/ipdevpoll/plugins_paloaltoarp_test.py

@@ -0,0 +1,109 @@
+from unittest.mock import patch, Mock
+
+from IPy import IP
+from nav.ipdevpoll.plugins.paloaltoarp import PaloaltoArp, parse_arp
+from twisted.internet import defer
+from twisted.internet.defer import inlineCallbacks, succeed
+from twisted.web.client import Agent, Response
+
+mock_data = b'''
+    <response status="success">
+    <result>
+            <max>132000</max>
+            <total>3</total>
+            <timeout>1800</timeout>
+            <dp>s3dp1</dp>
+            <entries>
+                <entry>
+                    <status>  s  </status>
+                    <ip>192.168.0.1</ip>
+                    <mac>00:00:00:00:00:01</mac>
+                    <ttl>100</ttl>
+                    <interface>ae2</interface>
+                    <port>ae2</port>
+                </entry>
+                <entry>
+                    <status>  e  </status>
+                    <ip>192.168.0.2</ip>
+                    <mac>00:00:00:00:00:02</mac>
+                    <ttl>200</ttl>
+                    <interface>ae2</interface>
+                    <port>ae2</port>
+                </entry>
+                <entry>
+                    <status>  c  </status>
+                    <ip>192.168.0.3</ip>
+                    <mac>00:00:00:00:00:03</mac>
+                    <ttl>300</ttl>
+                    <interface>ae3.61</interface>
+                    <port>ae3</port>
+                </entry>
+                <entry>
+                    <status>  i  </status>
+                    <ip>192.168.0.4</ip>
+                    <mac>00:00:00:00:00:04</mac>
+                    <ttl>400</ttl>
+                    <interface>ae3.61</interface>
+                    <port>ae3</port>
+                </entry>
+            </entries>
+        </result>
+    </response>
+    '''
+
+
+def test_parse_mappings():
+    assert parse_arp(mock_data) == [
+        ('ifindex', IP('192.168.0.1'), '00:00:00:00:00:01'),
+        ('ifindex', IP('192.168.0.2'), '00:00:00:00:00:02'),
+        ('ifindex', IP('192.168.0.3'), '00:00:00:00:00:03'),
+    ]
+
+
+@inlineCallbacks
+def test_get_mappings():
+    # Mocking the __init__ method
+    with patch.object(PaloaltoArp, "__init__", lambda x: None):
+        instance = PaloaltoArp()
+        instance.config = {'paloaltoarp': {'abcdefghijklmnop': '0.0.0.0'}}
+
+        # Mocking _do_request to return the mock_data when called
+        with patch.object(
+            PaloaltoArp, "_do_request", return_value=defer.succeed(mock_data)
+        ):
+            mappings = yield instance._get_paloalto_arp_mappings(
+                "0.0.0.0", "abcdefghijklmnop"
+            )
+
+            assert mappings == [
+                ('ifindex', IP('192.168.0.1'), '00:00:00:00:00:01'),
+                ('ifindex', IP('192.168.0.2'), '00:00:00:00:00:02'),
+                ('ifindex', IP('192.168.0.3'), '00:00:00:00:00:03'),
+            ]
+
+
+@inlineCallbacks
+def test_do_request():
+    mock_response = Mock(spec=Response)
+    mock_agent = Mock(spec=Agent)
+    mock_agent.request.return_value = succeed(mock_response)
+
+    with patch(
+        'nav.ipdevpoll.plugins.paloaltoarp.Agent', return_value=mock_agent
+    ), patch('twisted.web.client.readBody', return_value="test content"):
+        mock_address = "paloalto.example.org"
+        mock_key = "secret"
+
+        mock_netbox = Mock(sysname=mock_address, ip="127.0.0.1")
+
+        plugin = PaloaltoArp(netbox=mock_netbox, agent=Mock(), containers=Mock())
+        result = yield plugin._do_request(mock_address, mock_key)
+
+        expected_url = f"https://{mock_address}/api/?type=op&cmd=<show><arp><entry+name+=+'all'/></arp></show>&key={mock_key}".encode(
+            "utf-8"
+        )
+        mock_agent.request.assert_called()
+        args, kwargs = mock_agent.request.call_args
+        assert expected_url in args
+
+        assert result == "test content"