From 933521b7d10665a76a64158fb86e4f169f182032 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 8 Jul 2024 16:05:13 -0600 Subject: [PATCH 1/5] fix: set usedforsecurity=False for md5 operations To work on machines with FIPS enfored, md5 can't be used for security, and Python enforces this by default, but can be used with an extra function argument. Re-use md5_hexdigest as it wasn't be used so we can determine if this function argument is available, as its only available on Python 3.9 and newer. Bug: #4479 --- suricata/update/main.py | 11 +++++------ suricata/update/util.py | 11 +++++++---- tests/test_util.py | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/suricata/update/main.py b/suricata/update/main.py index 78145184..099d591d 100644 --- a/suricata/update/main.py +++ b/suricata/update/main.py @@ -114,8 +114,7 @@ def check_checksum(self, tmp_filename, url, checksum_url=None): if not isinstance(checksum_url, str): checksum_url = url[0] + ".md5" net_arg=(checksum_url,url[1]) - local_checksum = hashlib.md5( - open(tmp_filename, "rb").read()).hexdigest().strip() + local_checksum = util.md5_hexdigest(open(tmp_filename, "rb").read()) remote_checksum_buf = io.BytesIO() logger.info("Checking %s." % (checksum_url)) net.get(net_arg, remote_checksum_buf) @@ -154,7 +153,7 @@ def url_basename(self, url): return filename def get_tmp_filename(self, url): - url_hash = hashlib.md5(url.encode("utf-8")).hexdigest() + url_hash = util.md5_hexdigest(url.encode("utf-8")) return os.path.join( config.get_cache_dir(), "%s-%s" % (url_hash, self.url_basename(url))) @@ -470,7 +469,7 @@ def handle_dataset_files(rule, dep_files): return dataset_contents = dep_files[source_filename] - source_filename_hash = hashlib.md5(source_filename.encode()).hexdigest() + source_filename_hash = util.md5_hexdigest(source_filename.encode()) new_rule = re.sub(r"(dataset.*?load\s+){}".format(dataset_filename), r"\g<1>datasets/{}".format(source_filename_hash), rule.format()) dest_filename = os.path.join(config.get_output_dir(), "datasets", source_filename_hash) dest_dir = os.path.dirname(dest_filename) @@ -783,7 +782,7 @@ def md5(self, filename): if not os.path.exists(filename): return "" else: - return hashlib.md5(open(filename, "rb").read()).hexdigest() + return util.md5_hexdigest(open(filename, "rb").read()) def any_modified(self): for filename in self.hashes: @@ -1000,7 +999,7 @@ def load_sources(suricata_version): for url in urls: # To de-duplicate filenames, add a prefix that is a hash of the URL. - prefix = hashlib.md5(url[0].encode()).hexdigest() + prefix = util.md5_hexdigest(url[0].encode()) source_files = Fetch().run(url) for key in source_files: content = source_files[key] diff --git a/suricata/update/util.py b/suricata/update/util.py index 50788d8d..c2a73d0b 100644 --- a/suricata/update/util.py +++ b/suricata/update/util.py @@ -22,15 +22,18 @@ import atexit import shutil import zipfile +import sys -def md5_hexdigest(filename): - """ Compute the MD5 checksum for the contents of the provided filename. - :param filename: Filename to computer MD5 checksum of. +def md5_hexdigest(buf): + """ Compute the MD5 checksum for the provided buffer. :returns: A string representing the hex value of the computed MD5. """ - return hashlib.md5(open(filename).read().encode()).hexdigest() + if sys.version_info.major < 3 or (sys.version_info.major == 3 and sys.version_info.minor < 9): + return hashlib.md5(buf).hexdigest().strip() + else: + return hashlib.md5(buf, usedforsecurity=False).hexdigest().strip() def mktempdir(delete_on_exit=True): """ Create a temporary directory that is removed on exit. """ diff --git a/tests/test_util.py b/tests/test_util.py index 8862f471..ed13c81d 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -30,4 +30,4 @@ def test_hexdigest(self): test_file.flush() self.assertEqual( "120ea8a25e5d487bf68b5f7096440019", - util.md5_hexdigest(test_file.name)) + util.md5_hexdigest(open(test_file.name).read().encode())) From e002edadcfe2d9eaf8c86ce1ea57ae9e51777484 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 8 Jul 2024 16:07:44 -0600 Subject: [PATCH 2/5] github-ci: remove centos 7 build (eol) --- .github/workflows/tests.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 22a19f3e..6569aec5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,30 +52,6 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py - centos-7: - name: CentOS 7 - runs-on: ubuntu-latest - container: centos:7 - steps: - - run: yum -y install epel-release - - run: | - yum -y install \ - python2-pytest \ - python2-pyyaml \ - python36-pytest \ - python36-yaml - - uses: actions/checkout@v1 - - - name: Python 2 unit tests - run: PYTHONPATH=. py.test-2.7 - - name: Python 2 integration tests - run: PYTHONPATH=. python2 ./tests/integration_tests.py - - - name: Python 3 unit tests - run: PYTHONPATH=. py.test-3 - - name: Python 3 integration tests - run: PYTHONPATH=. python3 ./tests/integration_tests.py - fedora-39: name: Fedora 39 runs-on: ubuntu-latest From 5d9750232a898e5456c8b8cce2feac423be0592f Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 8 Jul 2024 16:08:53 -0600 Subject: [PATCH 3/5] github-ci: add fedora 40; remove fedora 38 --- .github/workflows/tests.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6569aec5..f3df24bc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -52,10 +52,10 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py - fedora-39: - name: Fedora 39 + fedora-40: + name: Fedora 40 runs-on: ubuntu-latest - container: fedora:39 + container: fedora:40 steps: - run: | dnf -y install \ @@ -68,17 +68,17 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py - fedora-38: - name: Fedora 38 + fedora-39: + name: Fedora 39 runs-on: ubuntu-latest - container: fedora:38 + container: fedora:39 steps: - run: | dnf -y install \ python3 \ python3-pytest \ python3-pyyaml - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Python 3 unit tests run: PYTHONPATH=. pytest-3 - name: Python 3 integration tests From 31e662790cc81a22d0570fdfd84e5dc6da1d184c Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 8 Jul 2024 16:10:03 -0600 Subject: [PATCH 4/5] github-ci: add Ubuntu 24.04 build --- .github/workflows/tests.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f3df24bc..da410c46 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -84,6 +84,22 @@ jobs: - name: Python 3 integration tests run: PYTHONPATH=. python3 ./tests/integration_tests.py + ubuntu-2404: + name: Ubuntu 24.04 + runs-on: ubuntu-latest + container: ubuntu:24.04 + steps: + - run: apt update + - run: | + apt -y install \ + python3-pytest \ + python3-yaml + - uses: actions/checkout@v1 + - name: Python 3 unit tests + run: PYTHONPATH=. pytest-3 + - name: Python 3 integration tests + run: PYTHONPATH=. python3 ./tests/integration_tests.py + ubuntu-2204: name: Ubuntu 22.04 runs-on: ubuntu-latest From 61b278bbc087ed99b155a403062a9e20cee51032 Mon Sep 17 00:00:00 2001 From: Jason Ish Date: Mon, 8 Jul 2024 16:10:51 -0600 Subject: [PATCH 5/5] github-ci: fix MacOS test --- .github/workflows/tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index da410c46..970fa9a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -226,9 +226,10 @@ jobs: name: MacOS Latest runs-on: macos-latest steps: + - run: PATH="/usr/local/opt/python/libexec/bin:$PATH" >> $GITHUB_ENV - run: brew install python - - run: pip3 install PyYAML - - run: pip3 install pytest + - run: brew install PyYAML + - run: pip3 install --break-system-packages pytest - uses: actions/checkout@v1 - run: PYTHONPATH=. python3 -m pytest - run: PYTHONPATH=. python3 ./tests/integration_tests.py