Merge pull request #2743 from lunkwill42/feature/snmpv3-ipdevpoll

Add initial SNMPv3 support to ipdevpoll

python/nav/ipdevpoll/__init__.py

@@ -77,7 +77,7 @@ def can_handle(cls, netbox):
         """
         snmp_up = getattr(netbox, 'snmp_up', True)
 
-        basic_req = netbox.is_up() and snmp_up and bool(netbox.read_only)
+        basic_req = netbox.is_up() and snmp_up and bool(netbox.snmp_parameters)
         vendor_check = cls._verify_vendor_restriction(netbox)
         return basic_req and vendor_check
 

python/nav/ipdevpoll/dataloader.py

@@ -176,10 +176,9 @@ def is_netbox_changed(netbox1, netbox2):
     for attr in (
         'ip',
         'type',
-        'read_only',
-        'snmp_version',
         'up',
         'snmp_up',
+        'snmp_parameters',
         'deleted_at',
     ):
         if getattr(netbox1, attr) != getattr(netbox2, attr):

python/nav/ipdevpoll/jobs.py

@@ -37,7 +37,6 @@
 from .plugins import plugin_registry
 from . import storage, shadows, dataloader
 from .utils import log_unhandled_failure
-from .snmp.common import snmp_parameter_factory
 
 _logger = logging.getLogger(__name__)
 ports = cycle([snmpprotocol.port() for i in range(50)])
@@ -86,7 +85,7 @@ class JobHandler(object):
     def __init__(self, name, netbox, plugins=None, interval=None):
         self.name = name
         self.netbox_id = netbox
-        self.netbox = None
+        self.netbox: shadows.Netbox = None
         self.cancelled = threading.Event()
         self.interval = interval
 
@@ -101,18 +100,16 @@ def _create_agentproxy(self):
         if self.agent:
             self._destroy_agentproxy()
 
-        if not self.netbox.read_only:
+        if not self.netbox.snmp_parameters:
             self.agent = None
             return
 
         port = next(ports)
         self.agent = AgentProxy(
             self.netbox.ip,
             161,
-            community=self.netbox.read_only,
-            snmpVersion='v%s' % self.netbox.snmp_version,
             protocol=port.protocol,
-            snmp_parameters=snmp_parameter_factory(self.netbox),
+            snmp_parameters=self.netbox.snmp_parameters,
         )
         try:
             self.agent.open()

python/nav/ipdevpoll/plugins/snmpcheck.py

@@ -14,7 +14,7 @@
 # License along with NAV. If not, see <http://www.gnu.org/licenses/>.
 #
 """snmp check plugin"""
-
+from pynetsnmp.netsnmp import SnmpTimeoutError
 from twisted.internet import error, defer
 from twisted.internet.defer import returnValue
 
@@ -44,14 +44,16 @@ class SnmpCheck(Plugin):
 
     @classmethod
     def can_handle(cls, netbox):
-        return netbox.is_up() and bool(netbox.read_only)
+        return netbox.is_up() and bool(netbox.snmp_parameters)
 
     def __init__(self, *args, **kwargs):
         super(SnmpCheck, self).__init__(*args, **kwargs)
 
     @defer.inlineCallbacks
     def handle(self):
-        self._logger.debug("snmp version from db: %s", self.netbox.snmp_version)
+        self._logger.debug(
+            "snmp version from db: %s", self.netbox.snmp_parameters.version
+        )
         was_down = yield db.run_in_thread(self._currently_down)
         is_ok = yield self._do_check()
 
@@ -64,11 +66,11 @@ def handle(self):
 
     @defer.inlineCallbacks
     def _do_check(self):
-        self._logger.debug("checking SNMP%s availability", self.agent.snmpVersion)
+        self._logger.debug("checking SNMP v%s availability", self.agent.snmpVersion)
         try:
             result = yield self.agent.walk(SYSTEM_OID)
-        except (defer.TimeoutError, error.TimeoutError):
-            self._logger.debug("SNMP%s timed out", self.agent.snmpVersion)
+        except (defer.TimeoutError, error.TimeoutError, SnmpTimeoutError):
+            self._logger.debug("SNMP v%s timed out", self.agent.snmpVersion)
             returnValue(False)
 
         self._logger.debug("SNMP response: %r", result)

python/nav/ipdevpoll/shadows/netbox.py

@@ -14,9 +14,12 @@
 # License along with NAV. If not, see <http://www.gnu.org/licenses/>.
 #
 """netbox related shadow classes"""
+from typing import Union
+
 from django.db.models import Q
 from django.db import transaction
 
+from nav.ipdevpoll.snmp.common import SNMPParameters
 from nav.models import manage
 from nav.ipdevpoll.storage import Shadow
 
@@ -29,13 +32,14 @@ class Netbox(Shadow):
     def __init__(self, *args, **kwargs):
         super(Netbox, self).__init__(*args, **kwargs)
         if args:
-            obj = args[0]
+            obj: Union[Netbox, manage.Netbox] = args[0]
             self.snmp_up = getattr(obj, 'snmp_up', not obj.is_snmp_down())
             self.last_updated = getattr(
                 obj, 'last_updated', self._translate_last_jobs(obj)
             )
-            self.read_only = getattr(obj, 'read_only')
-            self.snmp_version = getattr(obj, 'snmp_version')
+            self.snmp_parameters = getattr(obj, "snmp_parameters", None)
+            if not self.snmp_parameters:
+                self.snmp_parameters = SNMPParameters.factory(obj)
 
     @staticmethod
     def _translate_last_jobs(netbox):
@@ -58,13 +62,15 @@ def is_up(self):
         return self.up == manage.Netbox.UP_UP
 
     def copy(self, other):
+        """In addition to copying the 'official' attrs of another Netbox object,
+        this also copies 'computed'/'internal' attributes that are only part of the
+        Netbox shadow class definition.
+        """
         super(Netbox, self).copy(other)
         for attr in (
             "snmp_up",
             "last_updated",
-            "snmp_version",
-            "read_only",
-            "read_write",
+            "snmp_parameters",
         ):
             if hasattr(other, attr):
                 setattr(self, attr, getattr(other, attr))

python/nav/ipdevpoll/snmp/common.py

@@ -16,13 +16,17 @@
 "common AgentProxy mixin"
 import time
 import logging
+from dataclasses import dataclass
 from functools import wraps
-from collections import namedtuple
+from typing import Optional, Any, Dict
 
 from twisted.internet import reactor
 from twisted.internet.defer import succeed
 from twisted.internet.task import deferLater
 
+from nav.Snmp.defines import SecurityLevel, AuthenticationProtocol, PrivacyProtocol
+from nav.models.manage import Netbox
+
 _logger = logging.getLogger(__name__)
 
 
@@ -89,12 +93,14 @@ def __init__(self, *args, **kwargs):
             self.snmp_parameters = kwargs['snmp_parameters']
             del kwargs['snmp_parameters']
         else:
-            self.snmp_parameters = SNMP_DEFAULTS
+            self.snmp_parameters = SNMPParameters()
         self._result_cache = {}
         self._last_request = 0
         self.throttle_delay = self.snmp_parameters.throttle_delay
 
-        super(AgentProxyMixIn, self).__init__(*args, **kwargs)
+        kwargs_out = self.snmp_parameters.as_agentproxy_args()
+        kwargs_out.update(kwargs)
+        super(AgentProxyMixIn, self).__init__(*args, **kwargs_out)
         # If we're mixed in with a pure twistedsnmp AgentProxy, the timeout
         # parameter will have no effect, since it is an argument to individual
         # method calls.
@@ -134,10 +140,128 @@ def _getbulk(self, *args, **kwargs):
         return super(AgentProxyMixIn, self)._getbulk(*args, **kwargs)
 
 
-# pylint: disable=C0103
-SNMPParameters = namedtuple('SNMPParameters', 'timeout max_repetitions throttle_delay')
-
-SNMP_DEFAULTS = SNMPParameters(timeout=1.5, max_repetitions=50, throttle_delay=0)
+@dataclass
+class SNMPParameters:
+    """SNMP session parameters common to all SNMP protocol versions"""
+
+    # Common for all SNMP sessions
+    version: int = 1
+    timeout: float = 1.5
+    tries: int = 3
+
+    # Common for v1 and v2 only
+    community: str = "public"
+
+    # Common for v2c +
+    max_repetitions: int = 50
+
+    # SNMPv3 only
+    sec_level: Optional[SecurityLevel] = None
+    auth_protocol: Optional[AuthenticationProtocol] = None
+    sec_name: str = None
+    auth_password: Optional[str] = None
+    priv_protocol: Optional[PrivacyProtocol] = None
+    priv_password: Optional[str] = None
+
+    # Specific to ipdevpoll-derived implementations
+    throttle_delay: int = 0
+
+    def __post_init__(self):
+        """Enforces Enum types on init"""
+        if self.sec_level and not isinstance(self.sec_level, SecurityLevel):
+            self.sec_level = SecurityLevel(self.sec_level)
+        if self.auth_protocol and not isinstance(
+            self.auth_protocol, AuthenticationProtocol
+        ):
+            self.auth_protocol = AuthenticationProtocol(self.auth_protocol)
+        if self.priv_protocol and not isinstance(self.priv_protocol, PrivacyProtocol):
+            self.priv_protocol = PrivacyProtocol(self.priv_protocol)
+
+    @property
+    def version_string(self):
+        """Returns the SNMP protocol version as a command line compatible string"""
+        return "2c" if self.version == 2 else str(self.version)
+
+    @classmethod
+    def factory(
+        cls, netbox: Optional[Netbox] = None, **kwargs
+    ) -> Optional["SNMPParameters"]:
+        """Creates and returns a set of SNMP parameters based on three sources, in
+        reverse order of precedence:
+
+        1. Given a Netbox, adds the parameters from its preferred SNMP profile.
+        2. SNMP parameters from ipdevpoll.conf.
+        3. SNMP parameters given as keyword arguments to the factory method.
+
+        Beware that this method will synchronously fetch management profiles from the
+        database using the Django ORM, and should not be called from async code
+        unless deferred to a worker thread.
+
+        If the netbox argument is a Netbox without a configured SNMP profile, None will
+        be returned.
+        """
+        kwargs_out = {}
+        if netbox:
+            profile = netbox.get_preferred_snmp_management_profile(writeable=False)
+            if profile:
+                if profile.protocol == profile.PROTOCOL_SNMPV3:
+                    kwargs["version"] = 3
+                kwargs_out.update(
+                    {k: v for k, v in profile.configuration.items() if hasattr(cls, k)}
+                )
+            else:
+                _logger.debug("%r has no snmp profile", netbox)
+                return None
+
+        kwargs_out.update(cls.get_params_from_ipdevpoll_config())
+        kwargs_out.update(kwargs)
+        return cls(**kwargs_out)
+
+    @classmethod
+    def get_params_from_ipdevpoll_config(cls, section: str = "snmp") -> Dict[str, Any]:
+        """Reads and returns global SNMP parameters from ipdevpoll configuration as a
+        simple dict.
+        """
+        from nav.ipdevpoll.config import ipdevpoll_conf as config
+
+        params = {}
+        for var, getter in [
+            ('max-repetitions', config.getint),
+            ('timeout', config.getfloat),
+            ('throttle-delay', config.getfloat),
+        ]:
+            if config.has_option(section, var):
+                key = var.replace('-', '_')
+                params[key] = getter(section, var)
+
+        return params
+
+    def as_agentproxy_args(self) -> Dict[str, Any]:
+        """Returns the SNMP session parameters in a dict format compatible with
+        pynetsnmp.twistedsnmp.AgentProxy() keyword arguments.
+        """
+        kwargs = {"snmpVersion": self.version_string}
+        if self.version in (1, 2):
+            kwargs["community"] = self.community
+        if self.timeout:
+            kwargs["timeout"] = self.timeout
+        if self.tries:
+            kwargs["tries"] = self.tries
+
+        if self.version == 3:
+            params = []
+            params.extend(["-l", self.sec_level.value, "-u", self.sec_name])
+            if self.auth_protocol:
+                params.extend(["-a", self.auth_protocol.value])
+            if self.auth_password:
+                params.extend(["-A", self.auth_password])
+            if self.priv_protocol:
+                params.extend(["-x", self.priv_protocol.value])
+            if self.priv_password:
+                params.extend(["-X", self.priv_password])
+            kwargs["cmdLineArgs"] = tuple(params)
+
+        return kwargs
 
 
 # pylint: disable=W0212
@@ -148,22 +272,7 @@ def snmp_parameter_factory(host=None):
     :returns: An SNMPParameters namedtuple.
 
     """
-    section = 'snmp'
-
-    from nav.ipdevpoll.config import ipdevpoll_conf as config
-
-    params = SNMP_DEFAULTS._asdict()
-
-    for var, getter in [
-        ('max-repetitions', config.getint),
-        ('timeout', config.getfloat),
-        ('throttle-delay', config.getfloat),
-    ]:
-        if config.has_option(section, var):
-            key = var.replace('-', '_')
-            params[key] = getter(section, var)
-
-    return SNMPParameters(**params)
+    return SNMPParameters.factory(netbox=host)
 
 
 class SnmpError(Exception):

python/nav/mibs/mibretriever.py

@@ -33,6 +33,7 @@
 
 import logging
 
+from pynetsnmp.netsnmp import SnmpTimeoutError
 from twisted.internet import defer, reactor
 from twisted.internet.defer import returnValue
 from twisted.internet.error import TimeoutError
@@ -427,6 +428,11 @@ def _result_formatter(result):
 
             return formatted_result
 
+        def _snmp_timeout_handler(failure: defer.failure.Failure):
+            """Transforms SnmpTimeoutErrors into "regular" TimeoutErrors"""
+            failure.trap(SnmpTimeoutError)
+            raise TimeoutError(failure.value)
+
         def _valueerror_handler(failure):
             failure.trap(ValueError)
             self._logger.warning(
@@ -439,6 +445,7 @@ def _valueerror_handler(failure):
             return {}  # alternative is to retry or raise a Timeout exception
 
         deferred = self.agent_proxy.getTable([str(node.oid)])
+        deferred.addErrback(_snmp_timeout_handler)
         deferred.addCallbacks(_result_formatter, _valueerror_handler)
         return deferred
 

python/nav/models/manage.py

@@ -28,7 +28,7 @@
 import logging
 import math
 import re
-from typing import Set
+from typing import Set, Optional
 
 import IPy
 from django.conf import settings
@@ -358,7 +358,9 @@ def _get_snmp_config(self, variable='community', writeable=None):
         if profiles:
             return profiles[0].configuration.get(variable)
 
-    def get_preferred_snmp_management_profile(self, writeable=None):
+    def get_preferred_snmp_management_profile(
+        self, writeable=None
+    ) -> Optional[ManagementProfile]:
         """
         Returns the snmp management profile with the highest available
         SNMP version.