discover: add ReleaseParser for up-to-date Ubuntu releases

Though the url http://cloud-images.ubuntu.com/query/released.latest.txt
still exists, it is not up-to-date and does not contain recent data
about ubuntu releases and as a result no recent ubuntu images can be
found.

Instead we should look try and find the images by the directory:
https://cloud-images.ubuntu.com/releases/

Signed-off-by: Kyr Shatskyy <[email protected]>

downburst/discover.py

@@ -1,11 +1,14 @@
-import os
-import requests
-import re
 import csv
 import json
+import logging
+import os
+import re
+import requests
 
 from html.parser import HTMLParser
 
+log = logging.getLogger(__name__)
+
 URL=os.environ.get("DOWNBURST_DISCOVER_URL", "http://download.ceph.com/cloudinit/")
 
 class Parser(HTMLParser):
@@ -19,6 +22,17 @@ def handle_starttag(self, tag, attrs):
                 if  key == 'href' and (val.endswith('.img') or val.endswith('.raw')):
                     self.filenames.append(val)
 
+class ReleaseParser(HTMLParser):
+    def __init__(self):
+        self.dirs = []
+        HTMLParser.__init__(self)
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'a':
+            for key, val in attrs:
+                if  key == 'href' and val.startswith('release-'):
+                    self.dirs.append(val.rstrip('/'))
+
 class UbuntuHandler:
     URL = 'http://cloud-images.ubuntu.com'
 
@@ -49,6 +63,8 @@ class UbuntuHandler:
         '16.04': 'xenial',
         '18.04': 'bionic',
         '20.04': 'focal',
+        '22.04': 'jammy',
+        '24.04': 'noble',
     }
 
     RELEASE_TO_VERSION = {v:k for k, v in VERSION_TO_RELEASE.items()}
@@ -70,26 +86,29 @@ def get_version(self, distroversion):
             pass
         return distroversion
 
-    def get_serial(self, release):
-        url = self.URL + '/query/released.latest.txt'
+    def get_latest_release_serial(self, release):
+        url = self.URL + f"/releases/{release}"
         r = requests.get(url)
         r.raise_for_status()
-        serial = None
-        for row in csv.DictReader(r.content.decode().strip().split("\n"),
-                                  delimiter="\t",
-                                  fieldnames=('release', 'flavour', 'stability',
-                                              'serial')):
-
-            if row['release'] == release and row['flavour'] == 'server':
-                return row['serial'], row['stability']
+        parser = ReleaseParser()
+        parser.feed(r.content.decode())
+        parser.close()
+        latest_release_directory = sorted(parser.dirs)[-1]
+        if latest_release_directory:
+            serial = latest_release_directory.split('-')[1]
+            return serial, 'release'
+
         raise NameError('Image not found on server at ' + url)
 
     def get_filename(self, arch, version, state):
         if state == 'release':
             state = ''
         else:
             state = '-' + state
-        if version == '20.04':
+        major, minor = version.split('.')
+        if int(major) >= 23 and int(minor) >= 10:
+            return 'ubuntu-' + version + state + '-server-cloudimg-'+ arch + '.img'
+        elif int(major) >= 20:
             return 'ubuntu-' + version + state + '-server-cloudimg-'+ arch + '-disk-kvm.img'
         else:
             return 'ubuntu-' + version + state + '-server-cloudimg-'+ arch + '-disk1.img'
@@ -130,8 +149,9 @@ def __call__(self, distroversion, arch):
         if arch == "x86_64":
             arch = "amd64"
         release = self.get_release(distroversion)
+        log.debug(f"Found release: {release}")
         version = self.get_version(distroversion)
-        serial, state = self.get_serial(release)
+        serial, state = self.get_latest_release_serial(release)
         filename = self.get_filename(arch, version, state)
         base_url = self.get_base_url(release, serial, state)
         sha256 = self.get_sha256(base_url, filename)

downburst/test/test_discover.py

@@ -8,14 +8,19 @@ def test_ubuntu_handler(m_requests_get):
     assert '18.04' == h.get_version('bionic')
     m_request = Mock()
     m_request.content = (
-            b"focal	server	release	20230302\n"
-            b"jammy	server	release	20230303\n")
+        b"<html>\n<body>\n<div>\n<div>\n<pre><hr>\n"
+        b'<a href="/releases/">Parent Directory</a>\n'
+        b'<a href="release-20241216/">release-20241216/</a>\n'
+        b'<a href="release-20250109/">release-20250109/</a>\n'
+        b'<a href="release/">release/</a>\n'
+        b"<hr></pre>\n"
+        b"</div></div></body></html>\n")
     m_requests_get.return_value = m_request
-    assert ('20230302','release') == h.get_serial('focal')
+    assert ('20250109','release') == h.get_latest_release_serial('focal')
 
-@patch('downburst.discover.UbuntuHandler.get_serial')
-def test_get(m_get_serial):
-    m_get_serial.return_value = ('20230420', 'release')
+@patch('downburst.discover.UbuntuHandler.get_latest_release_serial')
+def test_get(m_get_latest_release_serial):
+    m_get_latest_release_serial.return_value = ('20230420', 'release')
     checksum = 'cd824b19795e8a6b9ae993b0b5157de0275e952a7a9e8b9717ca07acec22f51b'
     res = discover.get('ubuntu', '20.04', 'x86_64')
     assert checksum == res['checksum']