diff --git a/sdcm/coredump.py b/sdcm/coredump.py index 85d152975f..668b0fc9ef 100644 --- a/sdcm/coredump.py +++ b/sdcm/coredump.py @@ -14,6 +14,7 @@ import os import re import time +import json from abc import abstractmethod from typing import List, Optional, Dict from datetime import datetime @@ -24,6 +25,7 @@ from sdcm.log import SDCMAdapter from sdcm.remote import NETWORK_EXCEPTIONS from sdcm.utils.decorators import timeout +from sdcm.utils.version_utils import get_systemd_version from sdcm.sct_events.system import CoreDumpEvent from sdcm.sct_events.decorators import raise_event_on_failure @@ -287,7 +289,45 @@ class CoredumpExportSystemdThread(CoredumpThreadBase): Relay on coredumpctl on the host-side to do all things """ + @cached_property + def systemd_version(self): + systemd_version = 0 + try: + systemd_version = get_systemd_version(self.node.remoter.run( + "systemctl --version", ignore_status=True).stdout) + except Exception: # pylint: disable=broad-except + self.log.warning("failed to get systemd version:", exc_info=True) + return systemd_version + + def get_list_of_cores_json(self) -> Optional[List[CoreDumpInfo]]: + result = self.node.remoter.run( + 'sudo coredumpctl -q --json=short', verbose=False, ignore_status=True) + if not result.ok: + return [] + + # example of json output + # [{"time":1669548359983740,"pid":11218,"uid":0,"gid":0,"sig":11, + # "corefile":"missing","exe":"/usr/bin/bash","size":null}, + # {"time":1669585690901244,"pid":6755,"uid":112,"gid":118,"sig":6, + # "corefile":"present","exe":"/opt/scylladb/libexec/scylla","size":61750136832}] + coredump_infos = [] + try: + coredump_infos = json.loads(result.stdout) + except json.JSONDecodeError: + self.log.warning("couldn't parse:\n %s", result.stdout) + + pid_list = [] + for dump in coredump_infos: + if "bash" not in dump['exe'] and "fwupd" not in dump['exe']: + pid_list.append(CoreDumpInfo(pid=str(dump['pid']), node=self.node)) + return pid_list + def get_list_of_cores(self) -> Optional[List[CoreDumpInfo]]: + if self.systemd_version >= 248: + # since systemd/systemd@0689cfd we have option to get + # the coredump information in json format + return self.get_list_of_cores_json() + result = self.node.remoter.run( 'sudo coredumpctl --no-pager --no-legend 2>&1', verbose=False, ignore_status=True) if "No coredumps found" in result.stdout or not result.ok: @@ -367,6 +407,7 @@ def update_coredump_info_with_more_information(self, core_info: CoreDumpInfo): # Storage: /var/lib/systemd/coredump/core.vi.1000.6c4de4c206a0476e88444e5ebaaac482.18554.1578994298000000.lz4 (inaccessible) if "inaccessible" in line: continue + line = line.replace('(present)', '') corefile = line[line.find(':') + 1:].strip() elif line.startswith('Timestamp:'): timestring = None diff --git a/unit_tests/test_coredump.py b/unit_tests/test_coredump.py index a8e7392e3f..7d4705e27f 100644 --- a/unit_tests/test_coredump.py +++ b/unit_tests/test_coredump.py @@ -161,6 +161,9 @@ def _init_target_coredump_class(self, test_name: str) -> CoredumpExportSystemdTe def test_success_test(self): self._run_coredump_with_fake_remoter('success_test') + def test_success_test_systemd_248(self): + self._run_coredump_with_fake_remoter('success_test_systemd_248') + def test_fail_upload_test(self): self._run_coredump_with_fake_remoter('fail_upload_test') diff --git a/unit_tests/test_data/test_coredump/systemd/success_test_systemd_248_remoter.json b/unit_tests/test_data/test_coredump/systemd/success_test_systemd_248_remoter.json new file mode 100644 index 0000000000..7dd4035ed6 --- /dev/null +++ b/unit_tests/test_data/test_coredump/systemd/success_test_systemd_248_remoter.json @@ -0,0 +1,126 @@ +{ + "cat /etc/os-release": [ + { + "__instance__": "fabric.runners.Result", + "stdout": "NAME=\"CentOS Linux\"\nVERSION=\"7 (Core)\"\nID=\"centos\"\nID_LIKE=\"rhel fedora\"\nVERSION_ID=\"7\"\nPRETTY_NAME=\"CentOS Linux 7 (Core)\"\nANSI_COLOR=\"0;31\"\nCPE_NAME=\"cpe:/o:centos:centos:7\"\nHOME_URL=\"https://www.centos.org/\"\nBUG_REPORT_URL=\"https://bugs.centos.org/\"\n\nCENTOS_MANTISBT_PROJECT=\"CentOS-7\"\nCENTOS_MANTISBT_PROJECT_VERSION=\"7\"\nREDHAT_SUPPORT_PRODUCT=\"centos\"\nREDHAT_SUPPORT_PRODUCT_VERSION=\"7\"\n", + "stderr": "", + "exited": 0, + "exit_status": 0 + } + ], + "systemctl --version": [ + { + "__instance__": "fabric.runners.Result", + "stdout": "systemd 249 (249.11-0ubuntu3.6)\n+PAM +AUDIT +SELINUX +APPARMOR +IMA +SMACK +SECCOMP +GCRYPT +GNUTLS +OPENSSL +ACL +BLKID +CURL +ELFUTILS +FIDO2 +IDN2 -IDN +IPTC +KMOD +LIBCRYPTSETUP +LIBFDISK +PCRE2 -PWQUALITY -P11KIT -QRENCODE +BZIP2 +LZ4 +XZ +ZLIB +ZSTD -XKBCOMMON +UTMP +SYSVINIT default-hierarchy=unified", + "stderr": "", + "exited": 0, + "exit_status": 0 + } + ], + "sudo coredumpctl -q --json=short": [ + { + "__instance__": "invoke.exceptions.UnexpectedExit", + "result": { + "__instance__": "fabric.runners.Result", + "stdout": "", + "stderr": "Fake error, just to check what will coredump exporter thread can handle it properly", + "exited": 1 + }, + "reason": null + }, + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": "[{\"time\":1669548359983740,\"pid\":150,\"uid\":0,\"gid\":0,\"sig\":11,\"corefile\":\"missing\",\"exe\":\"/usr/bin/bash\",\"size\":null},{\"time\":1669585690901244,\"pid\":24393,\"uid\":112,\"gid\":118,\"sig\":6,\"corefile\":\"present\",\"exe\":\"/opt/scylladb/libexec/scylla\",\"size\":61750136832}]\n\n", + "exited": 0, + "exit_status": 0 + }, + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": "", + "exited": 0, + "exit_status": 0 + }, + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": "[{\"time\":1669548359983740,\"pid\":150,\"uid\":0,\"gid\":0,\"sig\":11,\"corefile\":\"missing\",\"exe\":\"/usr/bin/bash\",\"size\":null},{\"time\":1669585690901244,\"pid\":5348,\"uid\":112,\"gid\":118,\"sig\":6,\"corefile\":\"present\",\"exe\":\"/opt/scylladb/libexec/scylla\",\"size\":61750136832}]\n\n", + "exited": 0, + "exit_status": 0 + } + ], + "sudo coredumpctl info --no-pager --no-legend 24393": [ + { + "__instance__": "invoke.exceptions.UnexpectedExit", + "result": { + "__instance__": "fabric.runners.Result", + "stdout": "", + "stderr": "Fake error, just to check what will coredump exporter thread can handle it properly", + "exited": 1 + }, + "reason": null + }, + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": " PID: 24393 (python)\n UID: 1000 (dkropachev)\n GID: 1000 (dkropachev)\n Signal: 11 (SEGV)\n Timestamp: Thu 2020-08-13 13:51:14 +07 (2 weeks 1 days ago)\n Command Line: python -m unittest tests.test_multithreading\n Executable: /usr/bin/python3.8\n Control Group: /user.slice/user-1000.slice/user@1000.service/gnome-launched-pycharm-professional_pycharm-professional.desktop-13551.scope\n Unit: user@1000.service\n User Unit: gnome-launched-pycharm-professional_pycharm-professional.desktop-13551.scope\n Slice: user-1000.slice\n Owner UID: 1000 (dkropachev)\n Boot ID: 16da0914feb542e9bdb89a2976e8ff2e\n Machine ID: a72dad55f1754c44ad63a008ad3a60a5\n Hostname: dkropahev-pc\n Storage: /var/lib/systemd/coredump/core.python.1000.16da0914feb542e9bdb89a2976e8ff2e.24393.1597301474000000000000.lz4 (inaccessible)\n Message: Process 24393 (python) of user 1000 dumped core.\n \n Stack trace of thread 24411:\n #0 0x00007f0f0df4a552 __memmove_avx_unaligned_erms (libc.so.6 + 0x18e552)\n #1 0x00007f0f0bf33a27 _libssh2_transport_send (libssh2.so.1 + 0x3fa27)\n #2 0x00007f0f0bf086d8 _libssh2_channel_open (libssh2.so.1 + 0x146d8)\n #3 0x00007f0f0bf08c36 libssh2_channel_open_ex (libssh2.so.1 + 0x14c36)\n #4 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #5 0x000000000050112b n/a (python3.8 + 0x10112b)\n #6 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #7 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #12 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #13 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #14 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x000000000050722c n/a (python3.8 + 0x10722c)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #21 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #22 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #23 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24414:\n #0 0x00007f0f0c81bada HMAC_Init_ex (libcrypto.so.1.1 + 0x176ada)\n #1 0x00007f0f0bf1ea52 mac_method_hmac_sha2_256_hash (libssh2.so.1 + 0x2aa52)\n #2 0x00007f0f0bf32bb1 fullpacket (libssh2.so.1 + 0x3ebb1)\n #3 0x00007f0f0bf33547 _libssh2_transport_read (libssh2.so.1 + 0x3f547)\n #4 0x00007f0f0bf21f71 _libssh2_packet_requirev (libssh2.so.1 + 0x2df71)\n #5 0x00007f0f0bf0ac9e _libssh2_channel_process_startup (libssh2.so.1 + 0x16c9e)\n #6 0x00007f0f0bf0ada4 libssh2_channel_process_startup (libssh2.so.1 + 0x16da4)\n #7 0x00007f0f0beb904e __pyx_pf_4ssh2_7channel_7Channel_6execute (channel.cpython-38-x86_64-linux-gnu.so + 0x904e)\n #8 0x0000000000500e06 n/a (python3.8 + 0x100e06)\n #9 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24415:\n #0 0x00007f0f0c872f86 n/a (libcrypto.so.1.1 + 0x1cdf86)\n #1 0x00007f0f0c8740b4 SHA256_Final (libcrypto.so.1.1 + 0x1cf0b4)\n #2 0x00007f0f0c8017da EVP_DigestFinal_ex (libcrypto.so.1.1 + 0x15c7da)\n #3 0x00007f0f0c81bc83 HMAC_Final (libcrypto.so.1.1 + 0x176c83)\n #4 0x00007f0f0bf1eabb mac_method_hmac_sha2_256_hash (libssh2.so.1 + 0x2aabb)\n #5 0x00007f0f0bf33b9a _libssh2_transport_send (libssh2.so.1 + 0x3fb9a)\n #6 0x00007f0f0bf086d8 _libssh2_channel_open (libssh2.so.1 + 0x146d8)\n #7 0x00007f0f0bf08c36 libssh2_channel_open_ex (libssh2.so.1 + 0x14c36)\n #8 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #9 0x000000000050112b n/a (python3.8 + 0x10112b)\n #10 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #11 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #12 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #13 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #14 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #15 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #16 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #19 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #20 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #21 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #22 0x000000000050722c n/a (python3.8 + 0x10722c)\n #23 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #24 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #25 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #26 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #27 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24413:\n #0 0x00007f0f0ded196f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x00007f0f0bf29931 _libssh2_wait_socket (libssh2.so.1 + 0x35931)\n #2 0x00007f0f0bf0c3bd libssh2_channel_wait_eof (libssh2.so.1 + 0x183bd)\n #3 0x00007f0f0beb8b4f __pyx_pf_4ssh2_7channel_7Channel_22wait_eof (channel.cpython-38-x86_64-linux-gnu.so + 0x8b4f)\n #4 0x000000000050112b n/a (python3.8 + 0x10112b)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #10 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #11 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24393:\n #0 0x00007f0f0ddac3f4 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x133f4)\n #1 0x00007f0f0ddac4e8 __new_sem_wait_slow (libpthread.so.0 + 0x134e8)\n #2 0x0000000000540ecf PyThread_acquire_lock_timed (python3.8 + 0x140ecf)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #14 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #15 0x000000000050729f n/a (python3.8 + 0x10729f)\n #16 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #19 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #20 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #24 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #25 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #26 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #27 0x0000000000598037 n/a (python3.8 + 0x198037)\n #28 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #29 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #30 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #31 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #32 0x000000000050722c n/a (python3.8 + 0x10722c)\n #33 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #34 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #35 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #36 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #37 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #38 0x0000000000598037 n/a (python3.8 + 0x198037)\n #39 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #40 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #41 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #42 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #43 0x000000000050722c n/a (python3.8 + 0x10722c)\n #44 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #45 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #46 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #47 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #48 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #49 0x0000000000598037 n/a (python3.8 + 0x198037)\n #50 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #51 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x000000000050722c n/a (python3.8 + 0x10722c)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #59 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #60 0x0000000000598037 n/a (python3.8 + 0x198037)\n #61 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #62 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #63 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n \n Stack trace of thread 24412:\n #0 0x00007f0f0ded196f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x00007f0f0bf29931 _libssh2_wait_socket (libssh2.so.1 + 0x35931)\n #2 0x00007f0f0bf08c77 libssh2_channel_open_ex (libssh2.so.1 + 0x14c77)\n #3 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #4 0x000000000050112b n/a (python3.8 + 0x10112b)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #10 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #11 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n", + "exited": 0, + "exit_status": 0 + } + ], + "sudo coredumpctl info --no-pager --no-legend 5348": [ + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": " PID: 5348 (scylla)\n UID: 112 (scylla)\n GID: 118 (scylla)\n Signal: 6 (ABRT)\n Timestamp: Mon 2022-11-28 12:19:40 UTC (15min ago)\n Command Line: /usr/bin/scylla --blocked-reactor-notify-ms 25 --abort-on-lsa-bad-alloc 1 --abort-on-seastar-bad-alloc --abort-on-internal-error 1 --abort-on-ebadf 1 --enable-sstable-key-validation 1 --smp 5 --log-to-syslog 1 --log-to-stdout 0 --default-log-level info --network-stack posix --io-properties-file=/etc/scylla.d/io_properties.yaml --cpuset 1-7 --lock-memory=1\n Executable: /opt/scylladb/libexec/scylla\n Control Group: /scylla.slice/scylla-server.slice/scylla-server.service\n Unit: scylla-server.service\n Slice: scylla-server.slice\n Boot ID: ff0f302ef93d4366812e6c5bcca67a90\n Machine ID: 4f7707a0ac3e44c5894dd86302dfd3b6\n Hostname: longevity-large-partitions-3h-fix-5-db-node-8b477908-1\n Storage: /var/lib/systemd/coredump/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4 (present)\n Disk Size: 55.6G\n Message: Process 5348 (scylla) of user 112 dumped core.\n \n Found module linux-vdso.so.1 with build-id: f30349c58a7a11a5813ea9673dadfad2c022a35d\n Found module libzstd.so.1 with build-id: 2b3060a28a992246fb32e1412965c3195dc15558\n Metadata for module libzstd.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"zstd\",\n \t\"version\" : \"1.5.2-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module liblzma.so.5 with build-id: 223b6bb0a820acf0da9d0d4712a9bc417ad98228\n Metadata for module liblzma.so.5 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"xz\",\n \t\"version\" : \"5.2.5-10.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcap.so.2 with build-id: f3f24bb6e411e3885602e753eda1e764fc1f5bfb\n Metadata for module libcap.so.2 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libcap\",\n \t\"version\" : \"2.48-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicudata.so.71 with build-id: 47276216e74b87c4b9bd01ac85c7bf332e6f2aed\n Metadata for module libicudata.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libffi.so.8 with build-id: bfbaa691e04ac523b4ce7970b3d2c9dc9b2aa296\n Metadata for module libffi.so.8 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libffi\",\n \t\"version\" : \"3.4.2-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcrypto.so.3 with build-id: 7470c53208efe3360d43315b439f03d64569b15b\n Metadata for module libcrypto.so.3 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"openssl\",\n \t\"version\" : \"3.0.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libssl.so.3 with build-id: d3fc0855dc0632f42fda3b02b4035f4245633408\n Metadata for module libssl.so.3 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"openssl\",\n \t\"version\" : \"3.0.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module ld.so with build-id: 41678be289bf6cf83ffc264138296b89b42db9e5\n Found module libc.so.6 with build-id: db8b8949801cc5d89afa366440dcc44c9c9ca76a\n Found module libgcc_s.so.1 with build-id: cb8492d3d140719087c752176d9c9f9a3cc24f55\n Found module libstdc++.so.6 with build-id: 3ef8edaa608a7bd7e2d8165665b7874284cf5f20\n Found module libsystemd.so.0 with build-id: 77a8cd6d743722bcba10ddd73af35dad325c5d70\n Metadata for module libsystemd.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"systemd\",\n \t\"version\" : \"251.8-586.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module liblua-5.4.so with build-id: 7337c06e288dc3cc740072b5ed926d68422b544b\n Metadata for module liblua-5.4.so owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"lua\",\n \t\"version\" : \"5.4.4-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libdeflate.so.0 with build-id: 4a9b9e6426b09c7727eca5906705cd47d6356b52\n Metadata for module libdeflate.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libdeflate\",\n \t\"version\" : \"1.9-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libxxhash.so.0 with build-id: e9d597d29fee60aa936d42919d5478b1489fcfcd\n Metadata for module libxxhash.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"xxhash\",\n \t\"version\" : \"0.8.1-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicui18n.so.71 with build-id: f4748e4011f7e0fd328ade31e62bc9167e417b74\n Metadata for module libicui18n.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicuuc.so.71 with build-id: f0a97a601f057dd0739c60482ff1b33c5e054456\n Metadata for module libicuuc.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libboost_regex.so.1.78.0 with build-id: 72db12b143d11d0a1f6b4d087ba66b0bed1b143a\n Metadata for module libboost_regex.so.1.78.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"boost\",\n \t\"version\" : \"1.78.0-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libboost_date_time.so.1.78.0 with build-id: eb65c4205b9d9fd037e8c33e34d2f0c529041ff2\n Metadata for module libboost_date_time.so.1.78.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"boost\",\n \t\"version\" : \"1.78.0-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcrypt.so.2 with build-id: 328e629b151ea0ec40b4e85fdd4f2859a88b69b4\n Metadata for module libcrypt.so.2 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libxcrypt\",\n \t\"version\" : \"4.4.30-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libjsoncpp.so.25 with build-id: 81664b30fef3b5f5552d5127aa7df65a5540b19e\n Metadata for module libjsoncpp.so.25 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"jsoncpp\",\n \t\"version\" : \"1.9.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libsnappy.so.1 with build-id: a7aec4ee674baa81a2a0f445be25f068d676fe4b\n Metadata for module libsnappy.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"snappy\",\n \t\"version\" : \"1.1.9-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libz.so.1 with build-id: d94299d7572e23295ceaf7674110eb6f2689cd91\n Metadata for module libz.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"zlib\",\n \t\"version\" : \"1.2.12-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libyaml-cpp.so.0.6 with build-id: 4b5879027e25c21db9008a81ce6936b85aeb856e\n Metadata for module libyaml-cpp.so.0.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"yaml-cpp\",\n \t\"version\" : \"0.6.3-7.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libm.so.6 with build-id: 4fd16b4c77b8cec3b6d6d76f58e4c25f247fab0e\n Found module libhwloc.so.15 with build-id: dc3323c96c1be89f9270a8b8e4e9623181a59e84\n Metadata for module libhwloc.so.15 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"hwloc\",\n \t\"version\" : \"2.5.0-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libp11-kit.so.0 with build-id: eab90567b10ce2fa21b2a7eb4e5a3e097e63b4af\n Metadata for module libp11-kit.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"p11-kit\",\n \t\"version\" : \"0.24.1-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libidn2.so.0 with build-id: 220865bcdf15c803f8dd897650798ecb5c2ef1e6\n Metadata for module libidn2.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libidn2\",\n \t\"version\" : \"2.3.4-1.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libtasn1.so.6 with build-id: fca2bdc5861a580a09853fbc80c9f63591bf0ea6\n Metadata for module libtasn1.so.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libtasn1\",\n \t\"version\" : \"4.18.0-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libhogweed.so.6 with build-id: 6f32d090a6de6b372ab9258d2f9656c808aba96d\n Metadata for module libhogweed.so.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"nettle\",\n \t\"version\" : \"3.8-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libnettle.so.8 with build-id: c0f18d9e28d93220296008fe3bc58b22347b1835\n Metadata for module libnettle.so.8 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"nettle\",\n \t\"version\" : \"3.8-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libunistring.so.2 with build-id: 0c89f0c5be4a239b6ab72dc3aa015b95fc8ede0d\n Stack trace of thread 5348:\n #0 0x00007f233e312e7c __pthread_kill_implementation (libc.so.6 + 0x8ce7c)\n #1 0x00007f233e2c2aa6 raise (libc.so.6 + 0x3caa6)\n #2 0x00007f233e2ac7fc abort (libc.so.6 + 0x267fc)\n #3 0x00007f233e2ac71b __assert_fail_base.cold (libc.so.6 + 0x2671b)\n #4 0x00007f233e2bb656 __assert_fail (libc.so.6 + 0x35656)\n #5 0x0000000003c867fe _ZN28reader_concurrency_semaphore19maybe_admit_waitersEv (scylla + 0x3a867fe)\n #6 0x0000000003c88476 _ZN28reader_concurrency_semaphore5evictERNS_13inactive_readENS_12evict_reasonE (scylla + 0x3a88476)\n #7 0x0000000004f5078c _ZN7seastar7reactor23do_expire_lowres_timersEv (scylla + 0x4d5078c)\n #8 0x0000000004f6e9b9 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b9)\n #9 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #10 0x0000000004f51bda _ZN7seastar7reactor3runEv (scylla + 0x4d51bda)\n #11 0x0000000004ef9306 _ZN7seastar12app_template14run_deprecatedEiPPcOSt8functionIFvvEE (scylla + 0x4cf9306)\n #12 0x0000000004ef8479 _ZN7seastar12app_template3runEiPPcOSt8functionIFNS_6futureIiEEvEE (scylla + 0x4cf8479)\n #13 0x00000000010d2b55 _ZL11scylla_mainiPPc (scylla + 0xed2b55)\n #14 0x00000000010d4661 _ZNKSt8functionIFiiPPcEEclEiS1_ (scylla + 0xed4661)\n #15 0x00000000010d12b6 main (scylla + 0xed12b6)\n #16 0x00007f233e2ad510 __libc_start_call_main (libc.so.6 + 0x27510)\n #17 0x00007f233e2ad5c9 __libc_start_main@@GLIBC_2.34 (libc.so.6 + 0x275c9)\n #18 0x00000000010cf425 _start (scylla + 0xecf425)\n \n Stack trace of thread 5357:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5353:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5355:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5354:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5356:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5349:\n #0 0x0000000004f94d41 _ZN7seastar8internalL15try_reap_eventsEmllPNS0_9linux_abi8io_eventEPK8timespecb (scylla + 0x4d94d41)\n #1 0x0000000004f94ecb _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94ecb)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f910bf _ZN7seastar19reactor_backend_aio23reap_kernel_completionsEv (scylla + 0x4d910bf)\n #4 0x0000000004f6e9b9 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b9)\n #5 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #6 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #7 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #8 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #9 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5351:\n #0 0x00007f233e38a6bd syscall (libc.so.6 + 0x1046bd)\n #1 0x0000000004f94f01 _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94f01)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f91390 _ZN7seastar19reactor_backend_aio23wait_and_process_eventsEPK10__sigset_t (scylla + 0x4d91390)\n #4 0x0000000004f52b7d _ZN7seastar7reactor6do_runEv (scylla + 0x4d52b7d)\n #5 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #6 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #7 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #8 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5350:\n #0 0x0000000004f6e9b0 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b0)\n #1 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #2 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5352:\n #0 0x00007f233e38a6bd syscall (libc.so.6 + 0x1046bd)\n #1 0x0000000004f94f01 _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94f01)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f91390 _ZN7seastar19reactor_backend_aio23wait_and_process_eventsEPK10__sigset_t (scylla + 0x4d91390)\n #4 0x0000000004f52b7d _ZN7seastar7reactor6do_runEv (scylla + 0x4d52b7d)\n #5 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #6 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #7 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #8 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)", + "exited": 0, + "exit_status": 0 + } + ], + "sudo coredumpctl info --no-pager --no-legend 307283": [ + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": " PID: 307283 (sshd)\n UID: 1000 (dkropachev)\n GID: 1000 (dkropachev)\n Signal: 31 (SYS)\n Timestamp: Mon 2020-08-24 10:31:01 +07 (4 days ago)\n Command Line: sshd: dkropachev [net]\n Executable: /usr/sbin/sshd\n Control Group: /user.slice/user-1000.slice/user@1000.service/gnome-launched-pycharm-professional_pycharm-professional.desktop-6641.scope\n Unit: user@1000.service\n User Unit: gnome-launched-pycharm-professional_pycharm-professional.desktop-6641.scope\n Slice: user-1000.slice\n Owner UID: 1000 (dkropachev)\n Boot ID: 3ee441d8238246e79d2c30f6619ceeac\n Machine ID: a72dad55f1754c44ad63a008ad3a60a5\n Hostname: dkropahev-pc\n Storage: /var/lib/systemd/coredump/core.sshd.1000.3ee441d8238246e79d2c30f6619ceeac.307283.1598239861000000000000.lz4\n Message: Process 307283 (sshd) of user 1000 dumped core.\n \n Stack trace of thread 307283:\n #0 0x00007fbdaf06177b __socket (libc.so.6 + 0x12377b)\n #1 0x00007fbdaf058b43 openlog_internal (libc.so.6 + 0x11ab43)\n #2 0x00007fbdaf05901f __vsyslog_internal (libc.so.6 + 0x11b01f)\n #3 0x00007fbdaf059333 __syslog_chk (libc.so.6 + 0x11b333)\n #4 0x000055a4e7c12b18 n/a (sshd + 0x5ab18)\n #5 0x000055a4e7c10a2a n/a (sshd + 0x58a2a)\n #6 0x000055a4e7be7f96 n/a (sshd + 0x2ff96)\n #7 0x000055a4e7c12981 n/a (sshd + 0x5a981)\n #8 0x000055a4e7c10a2a n/a (sshd + 0x58a2a)\n #9 0x000055a4e7be80c8 n/a (sshd + 0x300c8)\n #10 0x000055a4e7be97e9 n/a (sshd + 0x317e9)\n #11 0x000055a4e7bc957b n/a (sshd + 0x1157b)\n #12 0x00007fbdaef650b3 __libc_start_main (libc.so.6 + 0x270b3)\n #13 0x000055a4e7bc9b7e n/a (sshd + 0x11b7e)\n", + "exited": 0, + "exit_status": 0 + } + ], + "sudo coredumpctl info --no-pager --no-legend 1245911": [ + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": " PID: 1245911 (python)\n UID: 1000 (dkropachev)\n GID: 1000 (dkropachev)\n Signal: 6 (ABRT)\n Timestamp: Mon 2020-08-24 15:51:51 +07 (3 days ago)\n Command Line: /usr/src/scylladb/scylla-cluster-tests/venv/bin/python /snap/pycharm-professional/211/plugins/python/helpers/pycharm/_jb_unittest_runner.py --path /usr/src/scylladb/scylla-cluster-tests/dkropachev/unit_tests/test_remoter.py\n Executable: /usr/bin/python3.8\n Control Group: /user.slice/user-1000.slice/user@1000.service/gnome-launched-pycharm-professional_pycharm-professional.desktop-6641.scope\n Unit: user@1000.service\n User Unit: gnome-launched-pycharm-professional_pycharm-professional.desktop-6641.scope\n Slice: user-1000.slice\n Owner UID: 1000 (dkropachev)\n Boot ID: 3ee441d8238246e79d2c30f6619ceeac\n Machine ID: a72dad55f1754c44ad63a008ad3a60a5\n Hostname: dkropahev-pc\n Storage: /var/lib/systemd/coredump/core.python.1000.3ee441d8238246e79d2c30f6619ceeac.1245911.1598259111000000000000.lz4\n Message: Process 1245911 (python) of user 1000 dumped core.\n \n Stack trace of thread 1403737:\n #0 0x00007f549c60b18b __GI_raise (libc.so.6 + 0x4618b)\n #1 0x00007f549c5ea859 __GI_abort (libc.so.6 + 0x25859)\n #2 0x00007f549c6553ee __libc_message (libc.so.6 + 0x903ee)\n #3 0x00007f549c65d47c malloc_printerr (libc.so.6 + 0x9847c)\n #4 0x00007f549c65f120 _int_free (libc.so.6 + 0x9a120)\n #5 0x00007f5483b3fe79 libssh2_default_free (libssh2.so.1 + 0x34e79)\n #6 0x00007f5483b239b2 _libssh2_channel_free (libssh2.so.1 + 0x189b2)\n #7 0x00007f5483b23a05 libssh2_channel_free (libssh2.so.1 + 0x18a05)\n #8 0x00007f5496caa351 __pyx_pf_4ssh2_7channel_7Channel_2__dealloc__ (channel.cpython-38-x86_64-linux-gnu.so + 0x6351)\n #9 0x00000000005e8308 n/a (python3.8 + 0x1e8308)\n #10 0x0000000000540bc8 n/a (python3.8 + 0x140bc8)\n #11 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #12 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #13 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #14 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #15 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #16 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #17 0x00000000005ecf72 n/a (python3.8 + 0x1ecf72)\n #18 0x00000000004ec8b9 n/a (python3.8 + 0xec8b9)\n #19 0x00000000005f8523 n/a (python3.8 + 0x1f8523)\n #20 0x00000000005a78c5 n/a (python3.8 + 0x1a78c5)\n #21 0x000000000050ead4 PyObject_GetIter (python3.8 + 0x10ead4)\n #22 0x0000000000568466 _PyEval_EvalFrameDefault (python3.8 + 0x168466)\n #23 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #24 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #25 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #26 0x00000000004ff90f n/a (python3.8 + 0xff90f)\n #27 0x00000000005bdbe7 PyObject_GetAttr (python3.8 + 0x1bdbe7)\n #28 0x00000000005675d2 _PyEval_EvalFrameDefault (python3.8 + 0x1675d2)\n #29 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #30 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #31 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #32 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #33 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #34 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #35 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #36 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #37 0x000000000050722c n/a (python3.8 + 0x10722c)\n #38 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #39 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #40 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #41 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #42 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1245911:\n #0 0x00007f549c5b53f4 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x133f4)\n #1 0x00007f549c5b54e8 __new_sem_wait_slow (libpthread.so.0 + 0x134e8)\n #2 0x0000000000540ecf PyThread_acquire_lock_timed (python3.8 + 0x140ecf)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #17 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #18 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #19 0x000000000050729f n/a (python3.8 + 0x10729f)\n #20 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #21 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #22 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #23 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #24 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #25 0x000000000050722c n/a (python3.8 + 0x10722c)\n #26 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #27 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #28 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #29 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #30 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #31 0x0000000000598037 n/a (python3.8 + 0x198037)\n #32 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #33 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x000000000050722c n/a (python3.8 + 0x10722c)\n #37 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #38 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #41 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #42 0x0000000000598037 n/a (python3.8 + 0x198037)\n #43 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #44 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #45 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #46 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #47 0x000000000050722c n/a (python3.8 + 0x10722c)\n #48 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #49 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #50 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #51 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #52 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #53 0x0000000000598037 n/a (python3.8 + 0x198037)\n #54 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #55 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #56 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #57 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #58 0x000000000050722c n/a (python3.8 + 0x10722c)\n #59 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #60 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #61 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #62 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #63 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n \n Stack trace of thread 1403362:\n #0 0x00007f549c5b27b1 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x107b1)\n #1 0x0000000000570679 PyEval_RestoreThread (python3.8 + 0x170679)\n #2 0x0000000000527712 n/a (python3.8 + 0x127712)\n #3 0x00000000005009da n/a (python3.8 + 0x1009da)\n #4 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #5 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #6 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #10 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #11 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #12 0x000000000050729f n/a (python3.8 + 0x10729f)\n #13 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #14 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #15 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #16 0x0000000000507729 n/a (python3.8 + 0x107729)\n #17 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #18 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #19 0x000000000050712e n/a (python3.8 + 0x10712e)\n #20 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #21 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #22 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #23 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #24 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #25 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #26 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #27 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #28 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #29 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #30 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #31 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #32 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #33 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #34 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #35 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #36 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #37 0x000000000050722c n/a (python3.8 + 0x10722c)\n #38 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #39 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #40 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #41 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #42 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403364:\n #0 0x00007f549c5b5618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007f549c5b5743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x000000000050729f n/a (python3.8 + 0x10729f)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #16 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #17 0x0000000000507729 n/a (python3.8 + 0x107729)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #20 0x000000000050712e n/a (python3.8 + 0x10712e)\n #21 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #22 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #23 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #24 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #25 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #26 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #27 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #28 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #29 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #30 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #31 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #32 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #33 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #34 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #35 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #36 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #37 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #38 0x000000000050722c n/a (python3.8 + 0x10722c)\n #39 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #40 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #41 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #42 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #43 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403363:\n #0 0x00007f549c5b5618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007f549c5b5743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x000000000050729f n/a (python3.8 + 0x10729f)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #16 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #17 0x0000000000507729 n/a (python3.8 + 0x107729)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #20 0x000000000050712e n/a (python3.8 + 0x10712e)\n #21 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #22 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #23 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #24 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #25 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #26 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #27 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #28 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #29 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #30 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #31 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #32 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #33 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #34 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #35 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #36 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #37 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #38 0x000000000050722c n/a (python3.8 + 0x10722c)\n #39 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #40 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #41 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #42 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #43 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403365:\n #0 0x00007f549c5b5618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007f549c5b5743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403742:\n #0 0x00007f549c5b5618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007f549c5b5743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403553:\n #0 0x00007f549c5b27b1 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x107b1)\n #1 0x0000000000570679 PyEval_RestoreThread (python3.8 + 0x170679)\n #2 0x000000000061caad n/a (python3.8 + 0x21caad)\n #3 0x0000000000622955 n/a (python3.8 + 0x222955)\n #4 0x0000000000623364 n/a (python3.8 + 0x223364)\n #5 0x0000000000623489 n/a (python3.8 + 0x223489)\n #6 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x000000000050729f n/a (python3.8 + 0x10729f)\n #10 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #11 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #12 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #13 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #14 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x000000000050722c n/a (python3.8 + 0x10722c)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #21 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #22 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #23 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403366:\n #0 0x00007f549c5b5618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007f549c5b5743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403752:\n #0 0x00007f549c5b5618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007f549c5b5743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403745:\n #0 0x00007f549c5b27b1 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x107b1)\n #1 0x0000000000570679 PyEval_RestoreThread (python3.8 + 0x170679)\n #2 0x000000000061caad n/a (python3.8 + 0x21caad)\n #3 0x0000000000622955 n/a (python3.8 + 0x222955)\n #4 0x0000000000623364 n/a (python3.8 + 0x223364)\n #5 0x0000000000623489 n/a (python3.8 + 0x223489)\n #6 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x000000000050729f n/a (python3.8 + 0x10729f)\n #10 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #11 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #12 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #13 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #14 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x000000000050722c n/a (python3.8 + 0x10722c)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #21 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #22 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #23 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403751:\n #0 0x00007f549c5b27b1 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x107b1)\n #1 0x0000000000570679 PyEval_RestoreThread (python3.8 + 0x170679)\n #2 0x000000000061caad n/a (python3.8 + 0x21caad)\n #3 0x0000000000622955 n/a (python3.8 + 0x222955)\n #4 0x0000000000623364 n/a (python3.8 + 0x223364)\n #5 0x0000000000623489 n/a (python3.8 + 0x223489)\n #6 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #9 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x000000000050722c n/a (python3.8 + 0x10722c)\n #20 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #21 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #22 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #23 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #24 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403566:\n #0 0x00007f549c6da96f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x000000000061caa3 n/a (python3.8 + 0x21caa3)\n #2 0x0000000000622955 n/a (python3.8 + 0x222955)\n #3 0x0000000000623364 n/a (python3.8 + 0x223364)\n #4 0x0000000000623489 n/a (python3.8 + 0x223489)\n #5 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #6 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #7 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #8 0x000000000050729f n/a (python3.8 + 0x10729f)\n #9 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403562:\n #0 0x00007f549c5b27b1 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x107b1)\n #1 0x0000000000570679 PyEval_RestoreThread (python3.8 + 0x170679)\n #2 0x000000000061caad n/a (python3.8 + 0x21caad)\n #3 0x0000000000622955 n/a (python3.8 + 0x222955)\n #4 0x0000000000623364 n/a (python3.8 + 0x223364)\n #5 0x0000000000623489 n/a (python3.8 + 0x223489)\n #6 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x000000000050729f n/a (python3.8 + 0x10729f)\n #10 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #11 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #12 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #13 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #14 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x000000000050722c n/a (python3.8 + 0x10722c)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #21 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #22 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #23 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1403369:\n #0 0x00007f549c5b5618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007f549c5b5743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007f549c5ab609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007f549c6e7103 __clone (libc.so.6 + 0x122103)\n", + "exited": 0, + "exit_status": 0 + } + ], + "sudo coredumpctl info --no-pager --no-legend 1404017": [ + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": " PID: 1404017 (python)\n UID: 1000 (dkropachev)\n GID: 1000 (dkropachev)\n Signal: 11 (SEGV)\n Timestamp: Mon 2020-08-24 16:07:10 +07 (3 days ago)\n Command Line: /usr/src/scylladb/scylla-cluster-tests/venv/bin/python /snap/pycharm-professional/211/plugins/python/helpers/pycharm/_jb_unittest_runner.py --path /usr/src/scylladb/scylla-cluster-tests/dkropachev/unit_tests/test_remoter.py\n Executable: /usr/bin/python3.8\n Control Group: /user.slice/user-1000.slice/user@1000.service/gnome-launched-pycharm-professional_pycharm-professional.desktop-6641.scope\n Unit: user@1000.service\n User Unit: gnome-launched-pycharm-professional_pycharm-professional.desktop-6641.scope\n Slice: user-1000.slice\n Owner UID: 1000 (dkropachev)\n Boot ID: 3ee441d8238246e79d2c30f6619ceeac\n Machine ID: a72dad55f1754c44ad63a008ad3a60a5\n Hostname: dkropahev-pc\n Storage: /var/lib/systemd/coredump/core.python.1000.3ee441d8238246e79d2c30f6619ceeac.1404017.1598260030000000000000.lz4\n Message: Process 1404017 (python) of user 1000 dumped core.\n \n Stack trace of thread 1560584:\n #0 0x0000000000000000 n/a (n/a + 0x0)\n #1 0x00007fdb6a301a05 libssh2_channel_free (libssh2.so.1 + 0x18a05)\n #2 0x00007fdb7dcc7351 __pyx_pf_4ssh2_7channel_7Channel_2__dealloc__ (channel.cpython-38-x86_64-linux-gnu.so + 0x6351)\n #3 0x00000000005e8308 n/a (python3.8 + 0x1e8308)\n #4 0x0000000000540bc8 n/a (python3.8 + 0x140bc8)\n #5 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #6 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #7 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #8 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #9 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #10 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #11 0x00000000005ecf72 n/a (python3.8 + 0x1ecf72)\n #12 0x00000000004ec8b9 n/a (python3.8 + 0xec8b9)\n #13 0x00000000005f8523 n/a (python3.8 + 0x1f8523)\n #14 0x000000000053880f _PyObject_GC_New (python3.8 + 0x13880f)\n #15 0x00000000005cb59a n/a (python3.8 + 0x1cb59a)\n #16 0x00000000005bbb16 PyObject_SetAttr (python3.8 + 0x1bbb16)\n #17 0x0000000000567daf _PyEval_EvalFrameDefault (python3.8 + 0x167daf)\n #18 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #19 0x00000000005986f4 n/a (python3.8 + 0x1986f4)\n #20 0x00000000005f2367 _PyObject_MakeTpCall (python3.8 + 0x1f2367)\n #21 0x000000000056cb23 _PyEval_EvalFrameDefault (python3.8 + 0x16cb23)\n #22 0x000000000059832b n/a (python3.8 + 0x19832b)\n #23 0x00000000005f2367 _PyObject_MakeTpCall (python3.8 + 0x1f2367)\n #24 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #25 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #26 0x000000000050729f n/a (python3.8 + 0x10729f)\n #27 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #28 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #29 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #30 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #31 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #32 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #33 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #37 0x000000000050712e n/a (python3.8 + 0x10712e)\n #38 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #41 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #42 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #43 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #44 0x0000000000507729 n/a (python3.8 + 0x107729)\n #45 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #46 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #47 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #48 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #49 0x0000000000507729 n/a (python3.8 + 0x107729)\n #50 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #51 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x0000000000507729 n/a (python3.8 + 0x107729)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #59 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #60 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #61 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #62 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #63 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n \n Stack trace of thread 1404017:\n #0 0x00007fdb82dcd3f4 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x133f4)\n #1 0x00007fdb82dcd4e8 __new_sem_wait_slow (libpthread.so.0 + 0x134e8)\n #2 0x0000000000540ecf PyThread_acquire_lock_timed (python3.8 + 0x140ecf)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #17 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #18 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #19 0x000000000050729f n/a (python3.8 + 0x10729f)\n #20 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #21 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #22 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #23 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #24 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #25 0x000000000050722c n/a (python3.8 + 0x10722c)\n #26 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #27 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #28 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #29 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #30 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #31 0x0000000000598037 n/a (python3.8 + 0x198037)\n #32 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #33 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x000000000050722c n/a (python3.8 + 0x10722c)\n #37 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #38 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #41 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #42 0x0000000000598037 n/a (python3.8 + 0x198037)\n #43 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #44 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #45 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #46 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #47 0x000000000050722c n/a (python3.8 + 0x10722c)\n #48 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #49 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #50 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #51 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #52 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #53 0x0000000000598037 n/a (python3.8 + 0x198037)\n #54 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #55 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #56 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #57 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #58 0x000000000050722c n/a (python3.8 + 0x10722c)\n #59 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #60 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #61 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #62 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #63 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n \n Stack trace of thread 1560582:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x000000000050729f n/a (python3.8 + 0x10729f)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #16 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #21 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #22 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #23 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #24 0x000000000050712e n/a (python3.8 + 0x10712e)\n #25 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #26 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #27 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #28 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #29 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #30 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #31 0x0000000000507729 n/a (python3.8 + 0x107729)\n #32 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #33 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x0000000000507729 n/a (python3.8 + 0x107729)\n #37 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #38 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #41 0x0000000000507729 n/a (python3.8 + 0x107729)\n #42 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #43 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #44 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #45 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #46 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #47 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #48 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #49 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #50 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #51 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x0000000000507729 n/a (python3.8 + 0x107729)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #59 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #60 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #61 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #62 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #63 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n \n Stack trace of thread 1560583:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x000000000050729f n/a (python3.8 + 0x10729f)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #16 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #21 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #22 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #23 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #24 0x000000000050712e n/a (python3.8 + 0x10712e)\n #25 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #26 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #27 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #28 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #29 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #30 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #31 0x0000000000507729 n/a (python3.8 + 0x107729)\n #32 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #33 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x0000000000507729 n/a (python3.8 + 0x107729)\n #37 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #38 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #41 0x0000000000507729 n/a (python3.8 + 0x107729)\n #42 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #43 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #44 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #45 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #46 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #47 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #48 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #49 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #50 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #51 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x0000000000507729 n/a (python3.8 + 0x107729)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #59 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #60 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #61 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #62 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #63 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n \n Stack trace of thread 1560789:\n #0 0x00007fdb82dca7b1 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x107b1)\n #1 0x0000000000570679 PyEval_RestoreThread (python3.8 + 0x170679)\n #2 0x000000000061caad n/a (python3.8 + 0x21caad)\n #3 0x0000000000622955 n/a (python3.8 + 0x222955)\n #4 0x0000000000623364 n/a (python3.8 + 0x223364)\n #5 0x0000000000623489 n/a (python3.8 + 0x223489)\n #6 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x000000000050729f n/a (python3.8 + 0x10729f)\n #10 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #11 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #12 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #13 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #14 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x000000000050722c n/a (python3.8 + 0x10722c)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #21 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #22 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #23 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560587:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560586:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560782:\n #0 0x00007fdb82ef296f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x000000000061caa3 n/a (python3.8 + 0x21caa3)\n #2 0x0000000000622955 n/a (python3.8 + 0x222955)\n #3 0x0000000000623364 n/a (python3.8 + 0x223364)\n #4 0x0000000000623489 n/a (python3.8 + 0x223489)\n #5 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #6 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #7 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #8 0x000000000050729f n/a (python3.8 + 0x10729f)\n #9 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560768:\n #0 0x00007fdb82ef296f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x000000000061caa3 n/a (python3.8 + 0x21caa3)\n #2 0x0000000000622955 n/a (python3.8 + 0x222955)\n #3 0x0000000000623364 n/a (python3.8 + 0x223364)\n #4 0x0000000000623489 n/a (python3.8 + 0x223489)\n #5 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #6 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #7 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #8 0x000000000050729f n/a (python3.8 + 0x10729f)\n #9 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560585:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n", + "exited": 0, + "exit_status": 0 + } + ], + "sudo coredumpctl info --no-pager --no-legend 1404018": [ + { + "__instance__": "fabric.runners.Result", + "stderr": "", + "stdout": " PID: 1404018 (python)\n UID: 1000 (dkropachev)\n GID: 1000 (dkropachev)\n Signal: 11 (SEGV)\n Timestamp: Mon 2020-08-24 16:07:10 UTC (3 days ago)\n Command Line: /usr/src/scylladb/scylla-cluster-tests/venv/bin/python /snap/pycharm-professional/211/plugins/python/helpers/pycharm/_jb_unittest_runner.py --path /usr/src/scylladb/scylla-cluster-tests/dkropachev/unit_tests/test_remoter.py\n Executable: /usr/bin/python3.8\n Control Group: /user.slice/user-1000.slice/user@1000.service/gnome-launched-pycharm-professional_pycharm-professional.desktop-6641.scope\n Unit: user@1000.service\n User Unit: gnome-launched-pycharm-professional_pycharm-professional.desktop-6641.scope\n Slice: user-1000.slice\n Owner UID: 1000 (dkropachev)\n Boot ID: 3ee441d8238246e79d2c30f6619ceeac\n Machine ID: a72dad55f1754c44ad63a008ad3a60a5\n Hostname: dkropahev-pc\n Storage: /var/lib/systemd/coredump/core.python.1000.3ee441d8238246e79d2c30f6619ceeac.1404017.1598260030000000000000.lz4\n Message: Process 1404017 (python) of user 1000 dumped core.\n \n Stack trace of thread 1560584:\n #0 0x0000000000000000 n/a (n/a + 0x0)\n #1 0x00007fdb6a301a05 libssh2_channel_free (libssh2.so.1 + 0x18a05)\n #2 0x00007fdb7dcc7351 __pyx_pf_4ssh2_7channel_7Channel_2__dealloc__ (channel.cpython-38-x86_64-linux-gnu.so + 0x6351)\n #3 0x00000000005e8308 n/a (python3.8 + 0x1e8308)\n #4 0x0000000000540bc8 n/a (python3.8 + 0x140bc8)\n #5 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #6 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #7 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #8 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #9 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #10 0x0000000000540c1a n/a (python3.8 + 0x140c1a)\n #11 0x00000000005ecf72 n/a (python3.8 + 0x1ecf72)\n #12 0x00000000004ec8b9 n/a (python3.8 + 0xec8b9)\n #13 0x00000000005f8523 n/a (python3.8 + 0x1f8523)\n #14 0x000000000053880f _PyObject_GC_New (python3.8 + 0x13880f)\n #15 0x00000000005cb59a n/a (python3.8 + 0x1cb59a)\n #16 0x00000000005bbb16 PyObject_SetAttr (python3.8 + 0x1bbb16)\n #17 0x0000000000567daf _PyEval_EvalFrameDefault (python3.8 + 0x167daf)\n #18 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #19 0x00000000005986f4 n/a (python3.8 + 0x1986f4)\n #20 0x00000000005f2367 _PyObject_MakeTpCall (python3.8 + 0x1f2367)\n #21 0x000000000056cb23 _PyEval_EvalFrameDefault (python3.8 + 0x16cb23)\n #22 0x000000000059832b n/a (python3.8 + 0x19832b)\n #23 0x00000000005f2367 _PyObject_MakeTpCall (python3.8 + 0x1f2367)\n #24 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #25 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #26 0x000000000050729f n/a (python3.8 + 0x10729f)\n #27 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #28 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #29 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #30 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #31 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #32 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #33 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #37 0x000000000050712e n/a (python3.8 + 0x10712e)\n #38 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #41 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #42 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #43 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #44 0x0000000000507729 n/a (python3.8 + 0x107729)\n #45 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #46 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #47 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #48 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #49 0x0000000000507729 n/a (python3.8 + 0x107729)\n #50 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #51 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x0000000000507729 n/a (python3.8 + 0x107729)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #59 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #60 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #61 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #62 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #63 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n \n Stack trace of thread 1404017:\n #0 0x00007fdb82dcd3f4 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x133f4)\n #1 0x00007fdb82dcd4e8 __new_sem_wait_slow (libpthread.so.0 + 0x134e8)\n #2 0x0000000000540ecf PyThread_acquire_lock_timed (python3.8 + 0x140ecf)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #17 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #18 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #19 0x000000000050729f n/a (python3.8 + 0x10729f)\n #20 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #21 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #22 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #23 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #24 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #25 0x000000000050722c n/a (python3.8 + 0x10722c)\n #26 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #27 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #28 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #29 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #30 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #31 0x0000000000598037 n/a (python3.8 + 0x198037)\n #32 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #33 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x000000000050722c n/a (python3.8 + 0x10722c)\n #37 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #38 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #41 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #42 0x0000000000598037 n/a (python3.8 + 0x198037)\n #43 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #44 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #45 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #46 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #47 0x000000000050722c n/a (python3.8 + 0x10722c)\n #48 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #49 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #50 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #51 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #52 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #53 0x0000000000598037 n/a (python3.8 + 0x198037)\n #54 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #55 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #56 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #57 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #58 0x000000000050722c n/a (python3.8 + 0x10722c)\n #59 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #60 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #61 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #62 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #63 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n \n Stack trace of thread 1560582:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x000000000050729f n/a (python3.8 + 0x10729f)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #16 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #21 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #22 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #23 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #24 0x000000000050712e n/a (python3.8 + 0x10712e)\n #25 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #26 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #27 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #28 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #29 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #30 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #31 0x0000000000507729 n/a (python3.8 + 0x107729)\n #32 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #33 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x0000000000507729 n/a (python3.8 + 0x107729)\n #37 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #38 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #41 0x0000000000507729 n/a (python3.8 + 0x107729)\n #42 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #43 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #44 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #45 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #46 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #47 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #48 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #49 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #50 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #51 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x0000000000507729 n/a (python3.8 + 0x107729)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #59 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #60 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #61 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #62 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #63 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n \n Stack trace of thread 1560583:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x000000000050729f n/a (python3.8 + 0x10729f)\n #14 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #15 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #16 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #21 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #22 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #23 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #24 0x000000000050712e n/a (python3.8 + 0x10712e)\n #25 0x000000000056c475 _PyEval_EvalFrameDefault (python3.8 + 0x16c475)\n #26 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #27 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #28 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #29 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #30 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #31 0x0000000000507729 n/a (python3.8 + 0x107729)\n #32 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #33 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #34 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #35 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #36 0x0000000000507729 n/a (python3.8 + 0x107729)\n #37 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #38 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #39 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #40 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #41 0x0000000000507729 n/a (python3.8 + 0x107729)\n #42 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #43 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #44 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #45 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #46 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #47 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #48 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #49 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #50 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #51 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x0000000000507729 n/a (python3.8 + 0x107729)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #59 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #60 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #61 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #62 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #63 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n \n Stack trace of thread 1560789:\n #0 0x00007fdb82dca7b1 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x107b1)\n #1 0x0000000000570679 PyEval_RestoreThread (python3.8 + 0x170679)\n #2 0x000000000061caad n/a (python3.8 + 0x21caad)\n #3 0x0000000000622955 n/a (python3.8 + 0x222955)\n #4 0x0000000000623364 n/a (python3.8 + 0x223364)\n #5 0x0000000000623489 n/a (python3.8 + 0x223489)\n #6 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x000000000050729f n/a (python3.8 + 0x10729f)\n #10 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #11 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #12 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #13 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #14 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x000000000050722c n/a (python3.8 + 0x10722c)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #21 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #22 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #23 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560587:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560586:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560782:\n #0 0x00007fdb82ef296f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x000000000061caa3 n/a (python3.8 + 0x21caa3)\n #2 0x0000000000622955 n/a (python3.8 + 0x222955)\n #3 0x0000000000623364 n/a (python3.8 + 0x223364)\n #4 0x0000000000623489 n/a (python3.8 + 0x223489)\n #5 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #6 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #7 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #8 0x000000000050729f n/a (python3.8 + 0x10729f)\n #9 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560768:\n #0 0x00007fdb82ef296f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x000000000061caa3 n/a (python3.8 + 0x21caa3)\n #2 0x0000000000622955 n/a (python3.8 + 0x222955)\n #3 0x0000000000623364 n/a (python3.8 + 0x223364)\n #4 0x0000000000623489 n/a (python3.8 + 0x223489)\n #5 0x0000000000501ade n/a (python3.8 + 0x101ade)\n #6 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #7 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #8 0x000000000050729f n/a (python3.8 + 0x10729f)\n #9 0x0000000000568819 _PyEval_EvalFrameDefault (python3.8 + 0x168819)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 1560585:\n #0 0x00007fdb82dcd618 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x13618)\n #1 0x00007fdb82dcd743 __new_sem_wait_slow (libpthread.so.0 + 0x13743)\n #2 0x0000000000540f2d PyThread_acquire_lock_timed (python3.8 + 0x140f2d)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000050722c n/a (python3.8 + 0x10722c)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007fdb82dc3609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007fdb82eff103 __clone (libc.so.6 + 0x122103)\n", + "exited": 0, + "exit_status": 0 + } + ], + "sudo curl --request PUT --fail --show-error --upload-file '/var/lib/systemd/coredump/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4' 'https://upload.scylladb.com/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000./core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4'": [ + { + "__instance__": "fabric.runners.Result", + "stderr": " % Total % Received % Xferd Average Speed Time Time Time Current\n Dload Upload Total Spent Left Speed\n\r 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0\r 0 60.5M 0 0 0 65536 0 153k 0:06:43 --:--:-- 0:06:43 153k\r 5 60.5M 0 0 5 3520k 0 2703k 0:00:22 0:00:01 0:00:21 2701k\r 11 60.5M 0 0 11 7232k 0 3187k 0:00:19 0:00:02 0:00:17 3185k\r 15 60.5M 0 0 15 9856k 0 3062k 0:00:20 0:00:03 0:00:17 3061k\r 20 60.5M 0 0 20 12.3M 0 2993k 0:00:20 0:00:04 0:00:16 2992k\r 25 60.5M 0 0 25 15.4M 0 2997k 0:00:20 0:00:05 0:00:15 3241k\r 30 60.5M 0 0 30 18.3M 0 3017k 0:00:20 0:00:06 0:00:14 3099k\r 35 60.5M 0 0 35 21.4M 0 3027k 0:00:20 0:00:07 0:00:13 2954k\r 40 60.5M 0 0 40 24.5M 0 3050k 0:00:20 0:00:08 0:00:12 3042k\r 45 60.5M 0 0 45 27.4M 0 3045k 0:00:20 0:00:09 0:00:11 3089k\r 50 60.5M 0 0 50 30.6M 0 3061k 0:00:20 0:00:10 0:00:10 3128k\r 55 60.5M 0 0 55 33.8M 0 3086k 0:00:20 0:00:11 0:00:09 3172k\r 61 60.5M 0 0 61 37.1M 0 3112k 0:00:19 0:00:12 0:00:07 3235k\r 67 60.5M 0 0 67 41.0M 0 3170k 0:00:19 0:00:13 0:00:06 3366k\r 74 60.5M 0 0 74 45.1M 0 3249k 0:00:19 0:00:14 0:00:05 3626k\r 82 60.5M 0 0 82 50.0M 0 3362k 0:00:18 0:00:15 0:00:03 3983k\r 92 60.5M 0 0 92 55.7M 0 3520k 0:00:17 0:00:16 0:00:01 4495k\r100 60.5M 100 381 100 60.5M 21 3510k 0:00:18 0:00:17 0:00:01 4408k\r100 60.5M 100 381 100 60.5M 21 3509k 0:00:18 0:00:17 0:00:01 4531k\n", + "stdout": "AccessDeniedAccess denied.
Anonymous caller does not have storage.objects.delete access to upload.scylladb.com/core.python.1000.3ee441d8238246e79d2c30f6619ceeac.1245911.1598259111000000000000./core.python.1000.3ee441d8238246e79d2c30f6619ceeac.1245911.1598259111000000000000.lz4.
", + "exited": 0, + "exit_status": 0 + } + ] +} diff --git a/unit_tests/test_data/test_coredump/systemd/success_test_systemd_248_results.json b/unit_tests/test_data/test_coredump/systemd/success_test_systemd_248_results.json new file mode 100644 index 0000000000..6ffc1c9367 --- /dev/null +++ b/unit_tests/test_data/test_coredump/systemd/success_test_systemd_248_results.json @@ -0,0 +1,52 @@ +{ + "found": [ + { + "__instance__": "sdcm.coredump.CoreDumpInfo", + "pid": "24393", + "corefile": "", + "source_timestamp": 1597301474.0, + "coredump_info": " PID: 24393 (python)\n UID: 1000 (dkropachev)\n GID: 1000 (dkropachev)\n Signal: 11 (SEGV)\n Timestamp: Thu 2020-08-13 13:51:14 +07 (2 weeks 1 days ago)\n Command Line: python -m unittest tests.test_multithreading\n Executable: /usr/bin/python3.8\n Control Group: /user.slice/user-1000.slice/user@1000.service/gnome-launched-pycharm-professional_pycharm-professional.desktop-13551.scope\n Unit: user@1000.service\n User Unit: gnome-launched-pycharm-professional_pycharm-professional.desktop-13551.scope\n Slice: user-1000.slice\n Owner UID: 1000 (dkropachev)\n Boot ID: 16da0914feb542e9bdb89a2976e8ff2e\n Machine ID: a72dad55f1754c44ad63a008ad3a60a5\n Hostname: dkropahev-pc\n Storage: /var/lib/systemd/coredump/core.python.1000.16da0914feb542e9bdb89a2976e8ff2e.24393.1597301474000000000000.lz4 (inaccessible)\n Message: Process 24393 (python) of user 1000 dumped core.\n \n Stack trace of thread 24411:\n #0 0x00007f0f0df4a552 __memmove_avx_unaligned_erms (libc.so.6 + 0x18e552)\n #1 0x00007f0f0bf33a27 _libssh2_transport_send (libssh2.so.1 + 0x3fa27)\n #2 0x00007f0f0bf086d8 _libssh2_channel_open (libssh2.so.1 + 0x146d8)\n #3 0x00007f0f0bf08c36 libssh2_channel_open_ex (libssh2.so.1 + 0x14c36)\n #4 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #5 0x000000000050112b n/a (python3.8 + 0x10112b)\n #6 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #7 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #12 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #13 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #14 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x000000000050722c n/a (python3.8 + 0x10722c)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #21 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #22 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #23 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24414:\n #0 0x00007f0f0c81bada HMAC_Init_ex (libcrypto.so.1.1 + 0x176ada)\n #1 0x00007f0f0bf1ea52 mac_method_hmac_sha2_256_hash (libssh2.so.1 + 0x2aa52)\n #2 0x00007f0f0bf32bb1 fullpacket (libssh2.so.1 + 0x3ebb1)\n #3 0x00007f0f0bf33547 _libssh2_transport_read (libssh2.so.1 + 0x3f547)\n #4 0x00007f0f0bf21f71 _libssh2_packet_requirev (libssh2.so.1 + 0x2df71)\n #5 0x00007f0f0bf0ac9e _libssh2_channel_process_startup (libssh2.so.1 + 0x16c9e)\n #6 0x00007f0f0bf0ada4 libssh2_channel_process_startup (libssh2.so.1 + 0x16da4)\n #7 0x00007f0f0beb904e __pyx_pf_4ssh2_7channel_7Channel_6execute (channel.cpython-38-x86_64-linux-gnu.so + 0x904e)\n #8 0x0000000000500e06 n/a (python3.8 + 0x100e06)\n #9 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24415:\n #0 0x00007f0f0c872f86 n/a (libcrypto.so.1.1 + 0x1cdf86)\n #1 0x00007f0f0c8740b4 SHA256_Final (libcrypto.so.1.1 + 0x1cf0b4)\n #2 0x00007f0f0c8017da EVP_DigestFinal_ex (libcrypto.so.1.1 + 0x15c7da)\n #3 0x00007f0f0c81bc83 HMAC_Final (libcrypto.so.1.1 + 0x176c83)\n #4 0x00007f0f0bf1eabb mac_method_hmac_sha2_256_hash (libssh2.so.1 + 0x2aabb)\n #5 0x00007f0f0bf33b9a _libssh2_transport_send (libssh2.so.1 + 0x3fb9a)\n #6 0x00007f0f0bf086d8 _libssh2_channel_open (libssh2.so.1 + 0x146d8)\n #7 0x00007f0f0bf08c36 libssh2_channel_open_ex (libssh2.so.1 + 0x14c36)\n #8 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #9 0x000000000050112b n/a (python3.8 + 0x10112b)\n #10 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #11 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #12 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #13 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #14 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #15 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #16 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #19 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #20 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #21 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #22 0x000000000050722c n/a (python3.8 + 0x10722c)\n #23 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #24 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #25 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #26 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #27 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24413:\n #0 0x00007f0f0ded196f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x00007f0f0bf29931 _libssh2_wait_socket (libssh2.so.1 + 0x35931)\n #2 0x00007f0f0bf0c3bd libssh2_channel_wait_eof (libssh2.so.1 + 0x183bd)\n #3 0x00007f0f0beb8b4f __pyx_pf_4ssh2_7channel_7Channel_22wait_eof (channel.cpython-38-x86_64-linux-gnu.so + 0x8b4f)\n #4 0x000000000050112b n/a (python3.8 + 0x10112b)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #10 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #11 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24393:\n #0 0x00007f0f0ddac3f4 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x133f4)\n #1 0x00007f0f0ddac4e8 __new_sem_wait_slow (libpthread.so.0 + 0x134e8)\n #2 0x0000000000540ecf PyThread_acquire_lock_timed (python3.8 + 0x140ecf)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #14 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #15 0x000000000050729f n/a (python3.8 + 0x10729f)\n #16 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #19 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #20 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #24 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #25 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #26 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #27 0x0000000000598037 n/a (python3.8 + 0x198037)\n #28 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #29 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #30 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #31 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #32 0x000000000050722c n/a (python3.8 + 0x10722c)\n #33 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #34 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #35 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #36 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #37 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #38 0x0000000000598037 n/a (python3.8 + 0x198037)\n #39 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #40 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #41 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #42 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #43 0x000000000050722c n/a (python3.8 + 0x10722c)\n #44 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #45 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #46 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #47 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #48 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #49 0x0000000000598037 n/a (python3.8 + 0x198037)\n #50 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #51 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x000000000050722c n/a (python3.8 + 0x10722c)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #59 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #60 0x0000000000598037 n/a (python3.8 + 0x198037)\n #61 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #62 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #63 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n \n Stack trace of thread 24412:\n #0 0x00007f0f0ded196f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x00007f0f0bf29931 _libssh2_wait_socket (libssh2.so.1 + 0x35931)\n #2 0x00007f0f0bf08c77 libssh2_channel_open_ex (libssh2.so.1 + 0x14c77)\n #3 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #4 0x000000000050112b n/a (python3.8 + 0x10112b)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #10 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #11 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n", + "download_instructions": "", + "download_url": "", + "command_line": "python -m unittest tests.test_multithreading", + "executable": "/usr/bin/python3.8" + }, + { + "__instance__": "sdcm.coredump.CoreDumpInfo", + "pid": "5348", + "corefile": "/var/lib/systemd/coredump/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4", + "source_timestamp": 1669637980.0, + "coredump_info": " PID: 5348 (scylla)\n UID: 112 (scylla)\n GID: 118 (scylla)\n Signal: 6 (ABRT)\n Timestamp: Mon 2022-11-28 12:19:40 UTC (15min ago)\n Command Line: /usr/bin/scylla --blocked-reactor-notify-ms 25 --abort-on-lsa-bad-alloc 1 --abort-on-seastar-bad-alloc --abort-on-internal-error 1 --abort-on-ebadf 1 --enable-sstable-key-validation 1 --smp 5 --log-to-syslog 1 --log-to-stdout 0 --default-log-level info --network-stack posix --io-properties-file=/etc/scylla.d/io_properties.yaml --cpuset 1-7 --lock-memory=1\n Executable: /opt/scylladb/libexec/scylla\n Control Group: /scylla.slice/scylla-server.slice/scylla-server.service\n Unit: scylla-server.service\n Slice: scylla-server.slice\n Boot ID: ff0f302ef93d4366812e6c5bcca67a90\n Machine ID: 4f7707a0ac3e44c5894dd86302dfd3b6\n Hostname: longevity-large-partitions-3h-fix-5-db-node-8b477908-1\n Storage: /var/lib/systemd/coredump/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4 (present)\n Disk Size: 55.6G\n Message: Process 5348 (scylla) of user 112 dumped core.\n \n Found module linux-vdso.so.1 with build-id: f30349c58a7a11a5813ea9673dadfad2c022a35d\n Found module libzstd.so.1 with build-id: 2b3060a28a992246fb32e1412965c3195dc15558\n Metadata for module libzstd.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"zstd\",\n \t\"version\" : \"1.5.2-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module liblzma.so.5 with build-id: 223b6bb0a820acf0da9d0d4712a9bc417ad98228\n Metadata for module liblzma.so.5 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"xz\",\n \t\"version\" : \"5.2.5-10.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcap.so.2 with build-id: f3f24bb6e411e3885602e753eda1e764fc1f5bfb\n Metadata for module libcap.so.2 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libcap\",\n \t\"version\" : \"2.48-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicudata.so.71 with build-id: 47276216e74b87c4b9bd01ac85c7bf332e6f2aed\n Metadata for module libicudata.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libffi.so.8 with build-id: bfbaa691e04ac523b4ce7970b3d2c9dc9b2aa296\n Metadata for module libffi.so.8 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libffi\",\n \t\"version\" : \"3.4.2-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcrypto.so.3 with build-id: 7470c53208efe3360d43315b439f03d64569b15b\n Metadata for module libcrypto.so.3 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"openssl\",\n \t\"version\" : \"3.0.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libssl.so.3 with build-id: d3fc0855dc0632f42fda3b02b4035f4245633408\n Metadata for module libssl.so.3 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"openssl\",\n \t\"version\" : \"3.0.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module ld.so with build-id: 41678be289bf6cf83ffc264138296b89b42db9e5\n Found module libc.so.6 with build-id: db8b8949801cc5d89afa366440dcc44c9c9ca76a\n Found module libgcc_s.so.1 with build-id: cb8492d3d140719087c752176d9c9f9a3cc24f55\n Found module libstdc++.so.6 with build-id: 3ef8edaa608a7bd7e2d8165665b7874284cf5f20\n Found module libsystemd.so.0 with build-id: 77a8cd6d743722bcba10ddd73af35dad325c5d70\n Metadata for module libsystemd.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"systemd\",\n \t\"version\" : \"251.8-586.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module liblua-5.4.so with build-id: 7337c06e288dc3cc740072b5ed926d68422b544b\n Metadata for module liblua-5.4.so owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"lua\",\n \t\"version\" : \"5.4.4-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libdeflate.so.0 with build-id: 4a9b9e6426b09c7727eca5906705cd47d6356b52\n Metadata for module libdeflate.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libdeflate\",\n \t\"version\" : \"1.9-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libxxhash.so.0 with build-id: e9d597d29fee60aa936d42919d5478b1489fcfcd\n Metadata for module libxxhash.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"xxhash\",\n \t\"version\" : \"0.8.1-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicui18n.so.71 with build-id: f4748e4011f7e0fd328ade31e62bc9167e417b74\n Metadata for module libicui18n.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicuuc.so.71 with build-id: f0a97a601f057dd0739c60482ff1b33c5e054456\n Metadata for module libicuuc.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libboost_regex.so.1.78.0 with build-id: 72db12b143d11d0a1f6b4d087ba66b0bed1b143a\n Metadata for module libboost_regex.so.1.78.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"boost\",\n \t\"version\" : \"1.78.0-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libboost_date_time.so.1.78.0 with build-id: eb65c4205b9d9fd037e8c33e34d2f0c529041ff2\n Metadata for module libboost_date_time.so.1.78.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"boost\",\n \t\"version\" : \"1.78.0-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcrypt.so.2 with build-id: 328e629b151ea0ec40b4e85fdd4f2859a88b69b4\n Metadata for module libcrypt.so.2 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libxcrypt\",\n \t\"version\" : \"4.4.30-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libjsoncpp.so.25 with build-id: 81664b30fef3b5f5552d5127aa7df65a5540b19e\n Metadata for module libjsoncpp.so.25 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"jsoncpp\",\n \t\"version\" : \"1.9.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libsnappy.so.1 with build-id: a7aec4ee674baa81a2a0f445be25f068d676fe4b\n Metadata for module libsnappy.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"snappy\",\n \t\"version\" : \"1.1.9-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libz.so.1 with build-id: d94299d7572e23295ceaf7674110eb6f2689cd91\n Metadata for module libz.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"zlib\",\n \t\"version\" : \"1.2.12-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libyaml-cpp.so.0.6 with build-id: 4b5879027e25c21db9008a81ce6936b85aeb856e\n Metadata for module libyaml-cpp.so.0.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"yaml-cpp\",\n \t\"version\" : \"0.6.3-7.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libm.so.6 with build-id: 4fd16b4c77b8cec3b6d6d76f58e4c25f247fab0e\n Found module libhwloc.so.15 with build-id: dc3323c96c1be89f9270a8b8e4e9623181a59e84\n Metadata for module libhwloc.so.15 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"hwloc\",\n \t\"version\" : \"2.5.0-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libp11-kit.so.0 with build-id: eab90567b10ce2fa21b2a7eb4e5a3e097e63b4af\n Metadata for module libp11-kit.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"p11-kit\",\n \t\"version\" : \"0.24.1-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libidn2.so.0 with build-id: 220865bcdf15c803f8dd897650798ecb5c2ef1e6\n Metadata for module libidn2.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libidn2\",\n \t\"version\" : \"2.3.4-1.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libtasn1.so.6 with build-id: fca2bdc5861a580a09853fbc80c9f63591bf0ea6\n Metadata for module libtasn1.so.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libtasn1\",\n \t\"version\" : \"4.18.0-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libhogweed.so.6 with build-id: 6f32d090a6de6b372ab9258d2f9656c808aba96d\n Metadata for module libhogweed.so.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"nettle\",\n \t\"version\" : \"3.8-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libnettle.so.8 with build-id: c0f18d9e28d93220296008fe3bc58b22347b1835\n Metadata for module libnettle.so.8 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"nettle\",\n \t\"version\" : \"3.8-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libunistring.so.2 with build-id: 0c89f0c5be4a239b6ab72dc3aa015b95fc8ede0d\n Stack trace of thread 5348:\n #0 0x00007f233e312e7c __pthread_kill_implementation (libc.so.6 + 0x8ce7c)\n #1 0x00007f233e2c2aa6 raise (libc.so.6 + 0x3caa6)\n #2 0x00007f233e2ac7fc abort (libc.so.6 + 0x267fc)\n #3 0x00007f233e2ac71b __assert_fail_base.cold (libc.so.6 + 0x2671b)\n #4 0x00007f233e2bb656 __assert_fail (libc.so.6 + 0x35656)\n #5 0x0000000003c867fe _ZN28reader_concurrency_semaphore19maybe_admit_waitersEv (scylla + 0x3a867fe)\n #6 0x0000000003c88476 _ZN28reader_concurrency_semaphore5evictERNS_13inactive_readENS_12evict_reasonE (scylla + 0x3a88476)\n #7 0x0000000004f5078c _ZN7seastar7reactor23do_expire_lowres_timersEv (scylla + 0x4d5078c)\n #8 0x0000000004f6e9b9 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b9)\n #9 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #10 0x0000000004f51bda _ZN7seastar7reactor3runEv (scylla + 0x4d51bda)\n #11 0x0000000004ef9306 _ZN7seastar12app_template14run_deprecatedEiPPcOSt8functionIFvvEE (scylla + 0x4cf9306)\n #12 0x0000000004ef8479 _ZN7seastar12app_template3runEiPPcOSt8functionIFNS_6futureIiEEvEE (scylla + 0x4cf8479)\n #13 0x00000000010d2b55 _ZL11scylla_mainiPPc (scylla + 0xed2b55)\n #14 0x00000000010d4661 _ZNKSt8functionIFiiPPcEEclEiS1_ (scylla + 0xed4661)\n #15 0x00000000010d12b6 main (scylla + 0xed12b6)\n #16 0x00007f233e2ad510 __libc_start_call_main (libc.so.6 + 0x27510)\n #17 0x00007f233e2ad5c9 __libc_start_main@@GLIBC_2.34 (libc.so.6 + 0x275c9)\n #18 0x00000000010cf425 _start (scylla + 0xecf425)\n \n Stack trace of thread 5357:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5353:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5355:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5354:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5356:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5349:\n #0 0x0000000004f94d41 _ZN7seastar8internalL15try_reap_eventsEmllPNS0_9linux_abi8io_eventEPK8timespecb (scylla + 0x4d94d41)\n #1 0x0000000004f94ecb _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94ecb)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f910bf _ZN7seastar19reactor_backend_aio23reap_kernel_completionsEv (scylla + 0x4d910bf)\n #4 0x0000000004f6e9b9 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b9)\n #5 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #6 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #7 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #8 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #9 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5351:\n #0 0x00007f233e38a6bd syscall (libc.so.6 + 0x1046bd)\n #1 0x0000000004f94f01 _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94f01)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f91390 _ZN7seastar19reactor_backend_aio23wait_and_process_eventsEPK10__sigset_t (scylla + 0x4d91390)\n #4 0x0000000004f52b7d _ZN7seastar7reactor6do_runEv (scylla + 0x4d52b7d)\n #5 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #6 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #7 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #8 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5350:\n #0 0x0000000004f6e9b0 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b0)\n #1 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #2 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5352:\n #0 0x00007f233e38a6bd syscall (libc.so.6 + 0x1046bd)\n #1 0x0000000004f94f01 _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94f01)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f91390 _ZN7seastar19reactor_backend_aio23wait_and_process_eventsEPK10__sigset_t (scylla + 0x4d91390)\n #4 0x0000000004f52b7d _ZN7seastar7reactor6do_runEv (scylla + 0x4d52b7d)\n #5 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #6 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #7 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #8 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)", + "download_instructions": "gsutil cp gs://upload.scylladb.com/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000./core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4 .\ngunzip /var/lib/systemd/coredump/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4", + "download_url": "https://storage.cloud.google.com/upload.scylladb.com/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000./core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4", + "command_line": "/usr/bin/scylla --blocked-reactor-notify-ms 25 --abort-on-lsa-bad-alloc 1 --abort-on-seastar-bad-alloc --abort-on-internal-error 1 --abort-on-ebadf 1 --enable-sstable-key-validation 1 --smp 5 --log-to-syslog 1 --log-to-stdout 0 --default-log-level info --network-stack posix --io-properties-file=/etc/scylla.d/io_properties.yaml --cpuset 1-7 --lock-memory=1", + "executable": "/opt/scylladb/libexec/scylla" + } + ], + "in_progress": [], + "completed": [ + { + "__instance__": "sdcm.coredump.CoreDumpInfo", + "pid": "24393", + "corefile": "", + "source_timestamp": 1597301474.0, + "coredump_info": " PID: 24393 (python)\n UID: 1000 (dkropachev)\n GID: 1000 (dkropachev)\n Signal: 11 (SEGV)\n Timestamp: Thu 2020-08-13 13:51:14 +07 (2 weeks 1 days ago)\n Command Line: python -m unittest tests.test_multithreading\n Executable: /usr/bin/python3.8\n Control Group: /user.slice/user-1000.slice/user@1000.service/gnome-launched-pycharm-professional_pycharm-professional.desktop-13551.scope\n Unit: user@1000.service\n User Unit: gnome-launched-pycharm-professional_pycharm-professional.desktop-13551.scope\n Slice: user-1000.slice\n Owner UID: 1000 (dkropachev)\n Boot ID: 16da0914feb542e9bdb89a2976e8ff2e\n Machine ID: a72dad55f1754c44ad63a008ad3a60a5\n Hostname: dkropahev-pc\n Storage: /var/lib/systemd/coredump/core.python.1000.16da0914feb542e9bdb89a2976e8ff2e.24393.1597301474000000000000.lz4 (inaccessible)\n Message: Process 24393 (python) of user 1000 dumped core.\n \n Stack trace of thread 24411:\n #0 0x00007f0f0df4a552 __memmove_avx_unaligned_erms (libc.so.6 + 0x18e552)\n #1 0x00007f0f0bf33a27 _libssh2_transport_send (libssh2.so.1 + 0x3fa27)\n #2 0x00007f0f0bf086d8 _libssh2_channel_open (libssh2.so.1 + 0x146d8)\n #3 0x00007f0f0bf08c36 libssh2_channel_open_ex (libssh2.so.1 + 0x14c36)\n #4 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #5 0x000000000050112b n/a (python3.8 + 0x10112b)\n #6 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #7 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #12 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #13 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #14 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #15 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #16 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x000000000050722c n/a (python3.8 + 0x10722c)\n #19 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #20 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #21 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #22 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #23 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24414:\n #0 0x00007f0f0c81bada HMAC_Init_ex (libcrypto.so.1.1 + 0x176ada)\n #1 0x00007f0f0bf1ea52 mac_method_hmac_sha2_256_hash (libssh2.so.1 + 0x2aa52)\n #2 0x00007f0f0bf32bb1 fullpacket (libssh2.so.1 + 0x3ebb1)\n #3 0x00007f0f0bf33547 _libssh2_transport_read (libssh2.so.1 + 0x3f547)\n #4 0x00007f0f0bf21f71 _libssh2_packet_requirev (libssh2.so.1 + 0x2df71)\n #5 0x00007f0f0bf0ac9e _libssh2_channel_process_startup (libssh2.so.1 + 0x16c9e)\n #6 0x00007f0f0bf0ada4 libssh2_channel_process_startup (libssh2.so.1 + 0x16da4)\n #7 0x00007f0f0beb904e __pyx_pf_4ssh2_7channel_7Channel_6execute (channel.cpython-38-x86_64-linux-gnu.so + 0x904e)\n #8 0x0000000000500e06 n/a (python3.8 + 0x100e06)\n #9 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #10 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #13 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #14 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #15 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #18 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #19 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #20 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #24 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #25 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #26 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24415:\n #0 0x00007f0f0c872f86 n/a (libcrypto.so.1.1 + 0x1cdf86)\n #1 0x00007f0f0c8740b4 SHA256_Final (libcrypto.so.1.1 + 0x1cf0b4)\n #2 0x00007f0f0c8017da EVP_DigestFinal_ex (libcrypto.so.1.1 + 0x15c7da)\n #3 0x00007f0f0c81bc83 HMAC_Final (libcrypto.so.1.1 + 0x176c83)\n #4 0x00007f0f0bf1eabb mac_method_hmac_sha2_256_hash (libssh2.so.1 + 0x2aabb)\n #5 0x00007f0f0bf33b9a _libssh2_transport_send (libssh2.so.1 + 0x3fb9a)\n #6 0x00007f0f0bf086d8 _libssh2_channel_open (libssh2.so.1 + 0x146d8)\n #7 0x00007f0f0bf08c36 libssh2_channel_open_ex (libssh2.so.1 + 0x14c36)\n #8 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #9 0x000000000050112b n/a (python3.8 + 0x10112b)\n #10 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #11 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #12 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #13 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #14 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #15 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #16 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #19 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #20 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #21 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #22 0x000000000050722c n/a (python3.8 + 0x10722c)\n #23 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #24 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #25 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #26 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #27 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24413:\n #0 0x00007f0f0ded196f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x00007f0f0bf29931 _libssh2_wait_socket (libssh2.so.1 + 0x35931)\n #2 0x00007f0f0bf0c3bd libssh2_channel_wait_eof (libssh2.so.1 + 0x183bd)\n #3 0x00007f0f0beb8b4f __pyx_pf_4ssh2_7channel_7Channel_22wait_eof (channel.cpython-38-x86_64-linux-gnu.so + 0x8b4f)\n #4 0x000000000050112b n/a (python3.8 + 0x10112b)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #10 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #11 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n \n Stack trace of thread 24393:\n #0 0x00007f0f0ddac3f4 futex_abstimed_wait_cancelable (libpthread.so.0 + 0x133f4)\n #1 0x00007f0f0ddac4e8 __new_sem_wait_slow (libpthread.so.0 + 0x134e8)\n #2 0x0000000000540ecf PyThread_acquire_lock_timed (python3.8 + 0x140ecf)\n #3 0x0000000000527704 n/a (python3.8 + 0x127704)\n #4 0x00000000005009da n/a (python3.8 + 0x1009da)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #7 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #8 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #9 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #10 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #11 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #14 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #15 0x000000000050729f n/a (python3.8 + 0x10729f)\n #16 0x000000000056769f _PyEval_EvalFrameDefault (python3.8 + 0x16769f)\n #17 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #18 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #19 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #20 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #21 0x000000000050722c n/a (python3.8 + 0x10722c)\n #22 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #23 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #24 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #25 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #26 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #27 0x0000000000598037 n/a (python3.8 + 0x198037)\n #28 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #29 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #30 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #31 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #32 0x000000000050722c n/a (python3.8 + 0x10722c)\n #33 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #34 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #35 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #36 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #37 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #38 0x0000000000598037 n/a (python3.8 + 0x198037)\n #39 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #40 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #41 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #42 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #43 0x000000000050722c n/a (python3.8 + 0x10722c)\n #44 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #45 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #46 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #47 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #48 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #49 0x0000000000598037 n/a (python3.8 + 0x198037)\n #50 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #51 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #52 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #53 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #54 0x000000000050722c n/a (python3.8 + 0x10722c)\n #55 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #56 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #57 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #58 0x00000000005f2b03 _PyObject_FastCallDict (python3.8 + 0x1f2b03)\n #59 0x00000000005f0b1d _PyObject_Call_Prepend (python3.8 + 0x1f0b1d)\n #60 0x0000000000598037 n/a (python3.8 + 0x198037)\n #61 0x00000000005f2406 _PyObject_MakeTpCall (python3.8 + 0x1f2406)\n #62 0x000000000056c6a6 _PyEval_EvalFrameDefault (python3.8 + 0x16c6a6)\n #63 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n \n Stack trace of thread 24412:\n #0 0x00007f0f0ded196f __GI___poll (libc.so.6 + 0x11596f)\n #1 0x00007f0f0bf29931 _libssh2_wait_socket (libssh2.so.1 + 0x35931)\n #2 0x00007f0f0bf08c77 libssh2_channel_open_ex (libssh2.so.1 + 0x14c77)\n #3 0x00007f0f0bf5070d __pyx_pf_4ssh2_7session_7Session_36open_session (session.cpython-38-x86_64-linux-gnu.so + 0x970d)\n #4 0x000000000050112b n/a (python3.8 + 0x10112b)\n #5 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #6 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #7 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #8 0x0000000000565972 _PyEval_EvalCodeWithName (python3.8 + 0x165972)\n #9 0x00000000005f1d85 _PyFunction_Vectorcall (python3.8 + 0x1f1d85)\n #10 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #11 0x0000000000568e1f _PyEval_EvalFrameDefault (python3.8 + 0x168e1f)\n #12 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #13 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #14 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #15 0x00000000005677c7 _PyEval_EvalFrameDefault (python3.8 + 0x1677c7)\n #16 0x00000000005f1b8b _PyFunction_Vectorcall (python3.8 + 0x1f1b8b)\n #17 0x000000000050722c n/a (python3.8 + 0x10722c)\n #18 0x00000000005f1107 PyObject_Call (python3.8 + 0x1f1107)\n #19 0x000000000064fb98 n/a (python3.8 + 0x24fb98)\n #20 0x000000000066ee14 n/a (python3.8 + 0x26ee14)\n #21 0x00007f0f0dda2609 start_thread (libpthread.so.0 + 0x9609)\n #22 0x00007f0f0dede103 __clone (libc.so.6 + 0x122103)\n", + "download_instructions": "", + "download_url": "", + "command_line": "python -m unittest tests.test_multithreading", + "executable": "/usr/bin/python3.8" + }, + { + "__instance__": "sdcm.coredump.CoreDumpInfo", + "pid": "5348", + "corefile": "/var/lib/systemd/coredump/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4", + "source_timestamp": 1669637980.0, + "coredump_info": " PID: 5348 (scylla)\n UID: 112 (scylla)\n GID: 118 (scylla)\n Signal: 6 (ABRT)\n Timestamp: Mon 2022-11-28 12:19:40 UTC (15min ago)\n Command Line: /usr/bin/scylla --blocked-reactor-notify-ms 25 --abort-on-lsa-bad-alloc 1 --abort-on-seastar-bad-alloc --abort-on-internal-error 1 --abort-on-ebadf 1 --enable-sstable-key-validation 1 --smp 5 --log-to-syslog 1 --log-to-stdout 0 --default-log-level info --network-stack posix --io-properties-file=/etc/scylla.d/io_properties.yaml --cpuset 1-7 --lock-memory=1\n Executable: /opt/scylladb/libexec/scylla\n Control Group: /scylla.slice/scylla-server.slice/scylla-server.service\n Unit: scylla-server.service\n Slice: scylla-server.slice\n Boot ID: ff0f302ef93d4366812e6c5bcca67a90\n Machine ID: 4f7707a0ac3e44c5894dd86302dfd3b6\n Hostname: longevity-large-partitions-3h-fix-5-db-node-8b477908-1\n Storage: /var/lib/systemd/coredump/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4 (present)\n Disk Size: 55.6G\n Message: Process 5348 (scylla) of user 112 dumped core.\n \n Found module linux-vdso.so.1 with build-id: f30349c58a7a11a5813ea9673dadfad2c022a35d\n Found module libzstd.so.1 with build-id: 2b3060a28a992246fb32e1412965c3195dc15558\n Metadata for module libzstd.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"zstd\",\n \t\"version\" : \"1.5.2-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module liblzma.so.5 with build-id: 223b6bb0a820acf0da9d0d4712a9bc417ad98228\n Metadata for module liblzma.so.5 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"xz\",\n \t\"version\" : \"5.2.5-10.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcap.so.2 with build-id: f3f24bb6e411e3885602e753eda1e764fc1f5bfb\n Metadata for module libcap.so.2 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libcap\",\n \t\"version\" : \"2.48-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicudata.so.71 with build-id: 47276216e74b87c4b9bd01ac85c7bf332e6f2aed\n Metadata for module libicudata.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libffi.so.8 with build-id: bfbaa691e04ac523b4ce7970b3d2c9dc9b2aa296\n Metadata for module libffi.so.8 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libffi\",\n \t\"version\" : \"3.4.2-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcrypto.so.3 with build-id: 7470c53208efe3360d43315b439f03d64569b15b\n Metadata for module libcrypto.so.3 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"openssl\",\n \t\"version\" : \"3.0.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libssl.so.3 with build-id: d3fc0855dc0632f42fda3b02b4035f4245633408\n Metadata for module libssl.so.3 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"openssl\",\n \t\"version\" : \"3.0.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module ld.so with build-id: 41678be289bf6cf83ffc264138296b89b42db9e5\n Found module libc.so.6 with build-id: db8b8949801cc5d89afa366440dcc44c9c9ca76a\n Found module libgcc_s.so.1 with build-id: cb8492d3d140719087c752176d9c9f9a3cc24f55\n Found module libstdc++.so.6 with build-id: 3ef8edaa608a7bd7e2d8165665b7874284cf5f20\n Found module libsystemd.so.0 with build-id: 77a8cd6d743722bcba10ddd73af35dad325c5d70\n Metadata for module libsystemd.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"systemd\",\n \t\"version\" : \"251.8-586.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module liblua-5.4.so with build-id: 7337c06e288dc3cc740072b5ed926d68422b544b\n Metadata for module liblua-5.4.so owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"lua\",\n \t\"version\" : \"5.4.4-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libdeflate.so.0 with build-id: 4a9b9e6426b09c7727eca5906705cd47d6356b52\n Metadata for module libdeflate.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libdeflate\",\n \t\"version\" : \"1.9-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libxxhash.so.0 with build-id: e9d597d29fee60aa936d42919d5478b1489fcfcd\n Metadata for module libxxhash.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"xxhash\",\n \t\"version\" : \"0.8.1-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicui18n.so.71 with build-id: f4748e4011f7e0fd328ade31e62bc9167e417b74\n Metadata for module libicui18n.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libicuuc.so.71 with build-id: f0a97a601f057dd0739c60482ff1b33c5e054456\n Metadata for module libicuuc.so.71 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"icu\",\n \t\"version\" : \"71.1-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libboost_regex.so.1.78.0 with build-id: 72db12b143d11d0a1f6b4d087ba66b0bed1b143a\n Metadata for module libboost_regex.so.1.78.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"boost\",\n \t\"version\" : \"1.78.0-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libboost_date_time.so.1.78.0 with build-id: eb65c4205b9d9fd037e8c33e34d2f0c529041ff2\n Metadata for module libboost_date_time.so.1.78.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"boost\",\n \t\"version\" : \"1.78.0-9.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libcrypt.so.2 with build-id: 328e629b151ea0ec40b4e85fdd4f2859a88b69b4\n Metadata for module libcrypt.so.2 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libxcrypt\",\n \t\"version\" : \"4.4.30-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libjsoncpp.so.25 with build-id: 81664b30fef3b5f5552d5127aa7df65a5540b19e\n Metadata for module libjsoncpp.so.25 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"jsoncpp\",\n \t\"version\" : \"1.9.5-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libsnappy.so.1 with build-id: a7aec4ee674baa81a2a0f445be25f068d676fe4b\n Metadata for module libsnappy.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"snappy\",\n \t\"version\" : \"1.1.9-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libz.so.1 with build-id: d94299d7572e23295ceaf7674110eb6f2689cd91\n Metadata for module libz.so.1 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"zlib\",\n \t\"version\" : \"1.2.12-5.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libyaml-cpp.so.0.6 with build-id: 4b5879027e25c21db9008a81ce6936b85aeb856e\n Metadata for module libyaml-cpp.so.0.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"yaml-cpp\",\n \t\"version\" : \"0.6.3-7.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libm.so.6 with build-id: 4fd16b4c77b8cec3b6d6d76f58e4c25f247fab0e\n Found module libhwloc.so.15 with build-id: dc3323c96c1be89f9270a8b8e4e9623181a59e84\n Metadata for module libhwloc.so.15 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"hwloc\",\n \t\"version\" : \"2.5.0-4.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libp11-kit.so.0 with build-id: eab90567b10ce2fa21b2a7eb4e5a3e097e63b4af\n Metadata for module libp11-kit.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"p11-kit\",\n \t\"version\" : \"0.24.1-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libidn2.so.0 with build-id: 220865bcdf15c803f8dd897650798ecb5c2ef1e6\n Metadata for module libidn2.so.0 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libidn2\",\n \t\"version\" : \"2.3.4-1.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libtasn1.so.6 with build-id: fca2bdc5861a580a09853fbc80c9f63591bf0ea6\n Metadata for module libtasn1.so.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"libtasn1\",\n \t\"version\" : \"4.18.0-3.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libhogweed.so.6 with build-id: 6f32d090a6de6b372ab9258d2f9656c808aba96d\n Metadata for module libhogweed.so.6 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"nettle\",\n \t\"version\" : \"3.8-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libnettle.so.8 with build-id: c0f18d9e28d93220296008fe3bc58b22347b1835\n Metadata for module libnettle.so.8 owned by FDO found: {\n \t\"type\" : \"rpm\",\n \t\"name\" : \"nettle\",\n \t\"version\" : \"3.8-2.fc37\",\n \t\"architecture\" : \"x86_64\",\n \t\"osCpe\" : \"cpe:/o:fedoraproject:fedora:37\"\n }\n \n Found module libunistring.so.2 with build-id: 0c89f0c5be4a239b6ab72dc3aa015b95fc8ede0d\n Stack trace of thread 5348:\n #0 0x00007f233e312e7c __pthread_kill_implementation (libc.so.6 + 0x8ce7c)\n #1 0x00007f233e2c2aa6 raise (libc.so.6 + 0x3caa6)\n #2 0x00007f233e2ac7fc abort (libc.so.6 + 0x267fc)\n #3 0x00007f233e2ac71b __assert_fail_base.cold (libc.so.6 + 0x2671b)\n #4 0x00007f233e2bb656 __assert_fail (libc.so.6 + 0x35656)\n #5 0x0000000003c867fe _ZN28reader_concurrency_semaphore19maybe_admit_waitersEv (scylla + 0x3a867fe)\n #6 0x0000000003c88476 _ZN28reader_concurrency_semaphore5evictERNS_13inactive_readENS_12evict_reasonE (scylla + 0x3a88476)\n #7 0x0000000004f5078c _ZN7seastar7reactor23do_expire_lowres_timersEv (scylla + 0x4d5078c)\n #8 0x0000000004f6e9b9 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b9)\n #9 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #10 0x0000000004f51bda _ZN7seastar7reactor3runEv (scylla + 0x4d51bda)\n #11 0x0000000004ef9306 _ZN7seastar12app_template14run_deprecatedEiPPcOSt8functionIFvvEE (scylla + 0x4cf9306)\n #12 0x0000000004ef8479 _ZN7seastar12app_template3runEiPPcOSt8functionIFNS_6futureIiEEvEE (scylla + 0x4cf8479)\n #13 0x00000000010d2b55 _ZL11scylla_mainiPPc (scylla + 0xed2b55)\n #14 0x00000000010d4661 _ZNKSt8functionIFiiPPcEEclEiS1_ (scylla + 0xed4661)\n #15 0x00000000010d12b6 main (scylla + 0xed12b6)\n #16 0x00007f233e2ad510 __libc_start_call_main (libc.so.6 + 0x27510)\n #17 0x00007f233e2ad5c9 __libc_start_main@@GLIBC_2.34 (libc.so.6 + 0x275c9)\n #18 0x00000000010cf425 _start (scylla + 0xecf425)\n \n Stack trace of thread 5357:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5353:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5355:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5354:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5356:\n #0 0x00007f233e380fac read (libc.so.6 + 0xfafac)\n #1 0x0000000004f8f075 _ZN7seastar11thread_pool4workENS_13basic_sstringIcjLj15ELb1EEE (scylla + 0x4d8f075)\n #2 0x0000000004f8f422 _ZNSt17_Function_handlerIFvvEZN7seastar11thread_poolC1EPNS1_7reactorENS1_13basic_sstringIcjLj15ELb1EEEE3$_0E9_M_invokeERKSt9_Any_data (scylla + 0x4d8f422)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5349:\n #0 0x0000000004f94d41 _ZN7seastar8internalL15try_reap_eventsEmllPNS0_9linux_abi8io_eventEPK8timespecb (scylla + 0x4d94d41)\n #1 0x0000000004f94ecb _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94ecb)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f910bf _ZN7seastar19reactor_backend_aio23reap_kernel_completionsEv (scylla + 0x4d910bf)\n #4 0x0000000004f6e9b9 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b9)\n #5 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #6 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #7 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #8 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #9 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5351:\n #0 0x00007f233e38a6bd syscall (libc.so.6 + 0x1046bd)\n #1 0x0000000004f94f01 _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94f01)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f91390 _ZN7seastar19reactor_backend_aio23wait_and_process_eventsEPK10__sigset_t (scylla + 0x4d91390)\n #4 0x0000000004f52b7d _ZN7seastar7reactor6do_runEv (scylla + 0x4d52b7d)\n #5 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #6 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #7 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #8 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5350:\n #0 0x0000000004f6e9b0 _ZNSt17_Function_handlerIFbvEZN7seastar7reactor6do_runEvE4$_43E9_M_invokeERKSt9_Any_data (scylla + 0x4d6e9b0)\n #1 0x0000000004f528c2 _ZN7seastar7reactor6do_runEv (scylla + 0x4d528c2)\n #2 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #3 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #4 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #5 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)\n \n Stack trace of thread 5352:\n #0 0x00007f233e38a6bd syscall (libc.so.6 + 0x1046bd)\n #1 0x0000000004f94f01 _ZN7seastar8internal13io_pgeteventsEmllPNS0_9linux_abi8io_eventEPK8timespecPK10__sigset_tb (scylla + 0x4d94f01)\n #2 0x0000000004f90c75 _ZN7seastar19reactor_backend_aio12await_eventsEiPK10__sigset_t (scylla + 0x4d90c75)\n #3 0x0000000004f91390 _ZN7seastar19reactor_backend_aio23wait_and_process_eventsEPK10__sigset_t (scylla + 0x4d91390)\n #4 0x0000000004f52b7d _ZN7seastar7reactor6do_runEv (scylla + 0x4d52b7d)\n #5 0x0000000004f6f892 _ZNSt17_Function_handlerIFvvEZN7seastar3smp9configureERKNS1_11smp_optionsERKNS1_15reactor_optionsEE4$_90E9_M_invokeERKSt9_Any_data (scylla + 0x4d6f892)\n #6 0x0000000004f2484b _ZN7seastar12posix_thread13start_routineEPv (scylla + 0x4d2484b)\n #7 0x00007f233e31114d start_thread (libc.so.6 + 0x8b14d)\n #8 0x00007f233e392950 __clone3 (libc.so.6 + 0x10c950)", + "download_instructions": "gsutil cp gs://upload.scylladb.com/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000./core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4 .\ngunzip /var/lib/systemd/coredump/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4", + "download_url": "https://storage.cloud.google.com/upload.scylladb.com/core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000./core.scylla.112.ff0f302ef93d4366812e6c5bcca67a90.5348.1669637980000000.lz4", + "command_line": "/usr/bin/scylla --blocked-reactor-notify-ms 25 --abort-on-lsa-bad-alloc 1 --abort-on-seastar-bad-alloc --abort-on-internal-error 1 --abort-on-ebadf 1 --enable-sstable-key-validation 1 --smp 5 --log-to-syslog 1 --log-to-stdout 0 --default-log-level info --network-stack posix --io-properties-file=/etc/scylla.d/io_properties.yaml --cpuset 1-7 --lock-memory=1", + "executable": "/opt/scylladb/libexec/scylla" + } + ], + "exception": null +}