Gunicorn: handle NoSuchProcess exceptions during scanning

[adrianriepl]

During the Gunicorn check sometimes a process may disappear and won't be accessible anymore, resulting in an unhandled exception.

Formatted by ChatGPT:

2025-02-18 12:19:02 UTC | CORE | ERROR | (pkg/collector/worker/check_logger.go:71 in Error) 
| check:gunicorn | Error running check: 
[
    {
        "message": "process no longer exists (pid=1574791)",
        "traceback": "Traceback (most recent call last):\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", 
            line 1717, in wrapper\n    
                return fun(self, *args, **kwargs)\n           
                ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", 
            line 508, in wrapper\n    
                raise raise_from(err, None)\n          
                ^^^^^^^^^^^^^^^^^^^^^\n  
            File \"<string>\", line 3, in raise_from\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", 
            line 506, in wrapper\n    
                return fun(self)\n           
                ^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", 
            line 1780, in _parse_stat_file\n    
                data = bcat(\"%s/%s/stat\" % (self._procfs_path, self.pid))\n           
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", 
            line 851, in bcat\n    
                return cat(fname, fallback=fallback, _open=open_binary)\n           
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", 
            line 839, in cat\n    
                with _open(fname) as f:\n         
                ^^^^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", 
            line 799, in open_binary\n    
                return open(fname, \"rb\", buffering=FILE_READ_BUFFER_SIZE)\n           
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n
            FileNotFoundError: [Errno 2] No such file or directory: '/host/proc/1574791/stat'\n\n
            During handling of the above exception, another exception occurred:\n\n
            Traceback (most recent call last):\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/datadog_checks/base/checks/base.py\", 
            line 1301, in run\n    
                self.check(instance)\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/datadog_checks/gunicorn/gunicorn.py\", 
            line 65, in check\n    
                master_procs = self._get_master_proc_by_name(proc_name)\n                   
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/datadog_checks/gunicorn/gunicorn.py\", 
            line 155, in _get_master_proc_by_name\n    
                self.log.debug(\"Cannot read information from process %s: %s\", p.name(), e, exc_info=True)\n                                                                  
                ^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/__init__.py\", 
            line 656, in name\n    
                name = self._proc.name()\n           
                ^^^^^^^^^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", 
            line 1717, in wrapper\n    
                return fun(self, *args, **kwargs)\n           
                ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", 
            line 1831, in name\n    
                name = self._parse_stat_file()['name']\n           
                ^^^^^^^^^^^^^^^^^^^^^^^\n  
            File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", 
            line 1726, in wrapper\n    
                raise NoSuchProcess(self.pid, self._name)\n
            psutil.NoSuchProcess: process no longer exists (pid=1574791)\n"
    }
]

Copied directly from Agent logs:

2025-02-18 12:19:02 UTC | CORE | ERROR | (pkg/collector/worker/check_logger.go:71 in Error) | check:gunicorn | Error running check: [{"message":"process no longer exists (pid=1574791)","traceback":"Traceback (most recent call last):\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", line 1717, in wrapper\n    return fun(self, *args, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", line 508, in wrapper\n    raise raise_from(err, None)\n          ^^^^^^^^^^^^^^^^^^^^^\n  File \"<string>\", line 3, in raise_from\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", line 506, in wrapper\n    return fun(self)\n           ^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", line 1780, in _parse_stat_file\n    data = bcat(\"%s/%s/stat\" % (self._procfs_path, self.pid))\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", line 851, in bcat\n    return cat(fname, fallback=fallback, _open=open_binary)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", line 839, in cat\n    with _open(fname) as f:\n         ^^^^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_common.py\", line 799, in open_binary\n    return open(fname, \"rb\", buffering=FILE_READ_BUFFER_SIZE)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\nFileNotFoundError: [Errno 2] No such file or directory: '/host/proc/1574791/stat'\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/datadog_checks/base/checks/base.py\", line 1301, in run\n    self.check(instance)\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/datadog_checks/gunicorn/gunicorn.py\", line 65, in check\n    master_procs = self._get_master_proc_by_name(proc_name)\n                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/datadog_checks/gunicorn/gunicorn.py\", line 155, in _get_master_proc_by_name\n    self.log.debug(\"Cannot read information from process %s: %s\", p.name(), e, exc_info=True)\n                                                                  ^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/__init__.py\", line 656, in name\n    name = self._proc.name()\n           ^^^^^^^^^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", line 1717, in wrapper\n    return fun(self, *args, **kwargs)\n           ^^^^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", line 1831, in name\n    name = self._parse_stat_file()['name']\n           ^^^^^^^^^^^^^^^^^^^^^^^\n  File \"/opt/datadog-agent/embedded/lib/python3.12/site-packages/psutil/_pslinux.py\", line 1726, in wrapper\n    raise NoSuchProcess(self.pid, self._name)\npsutil.NoSuchProcess: process no longer exists (pid=XXX)\n"}]

Can probably be fixed with something like here:

integrations-core/process/datadog_checks/process/process.py

Lines 169 to 173 in 54735b6

	except psutil.NoSuchProcess:
	# As the process list isn't necessarily scanned right after it's created
	# (since we're using a shared cache), there can be cases where processes
	# in the list are dead when an instance of the check tries to scan them.
	self.log.debug('Process disappeared while scanning')