Support beta in tests (#80)

* Support an attribute that includes the v prefix in Redpanda versions * Support for beta in Docker Compose files * Bump version * Ignore Console betas * Apply suggestions from code review * Fetch raw tags * minor improvements
redpanda-data · Oct 31, 2024 · 63deb72 · 63deb72
1 parent c37f7c6
commit 63deb72
Show file tree

Hide file tree

Showing 9 changed files with 159 additions and 108 deletions.
diff --git a/extensions/replace-attributes-in-attachments.js b/extensions/replace-attributes-in-attachments.js
@@ -1,7 +1,7 @@
 'use strict';
+const semver = require('semver');
 
 module.exports.register = function () {
-  const sanitizeAttributeValue = (value) => String(value).replace("@", "");
   this.on('contentClassified', ({ contentCatalog }) => {
     const componentVersionTable = contentCatalog.getComponents().reduce((componentMap, component) => {
       componentMap[component.name] = component.versions.reduce((versionMap, componentVersion) => {
@@ -12,39 +12,66 @@ module.exports.register = function () {
     }, {});
 
     contentCatalog.findBy({ family: 'attachment' }).forEach((attachment) => {
-      const componentVersion = componentVersionTable[attachment.src.component][attachment.src.version];
-      let attributes = componentVersion.asciidoc?.attributes;
-      if (!attributes) return;
-      attributes = Object.entries(attributes).reduce((accum, [name, val]) => {
-        const stringValue = String(val); // Ensure val is a string
-        accum[name] = stringValue.endsWith('@') ? stringValue.slice(0, stringValue.length - 1) : stringValue;
+      const componentVersion = componentVersionTable[attachment.src.component]?.[attachment.src.version];
+      if (!componentVersion?.asciidoc?.attributes) return;
+
+      const attributes = Object.entries(componentVersion.asciidoc.attributes).reduce((accum, [name, val]) => {
+        const stringValue = String(val);
+        accum[name] = stringValue.endsWith('@') ? sanitizeAttributeValue(stringValue) : stringValue;
         return accum;
       }, {});
-      let modified;
+
       let contentString = attachment.contents.toString();
-      // Specific replacements for YAML files
+      let modified = false;
+
+    // Determine if we're using the tag or version attributes
+    // We introduced tag attributes in Self-Managed 24.3
+    const isPrerelease = attributes['page-component-version-is-prerelease'];
+    const componentVersionNumber = formatVersion(componentVersion.version || '');
+    const useTagAttributes = isPrerelease || (componentVersionNumber && semver.gte(componentVersionNumber, '24.3.0') && componentVersion.title === 'Self-Managed');
+
+    // Set replacements based on the condition
+    const redpandaVersion = isPrerelease
+      ? sanitizeAttributeValue(attributes['redpanda-beta-tag'] || '')
+      : (useTagAttributes
+      ? sanitizeAttributeValue(attributes['latest-redpanda-tag'] || '')
+      : sanitizeAttributeValue(attributes['full-version'] || ''));
+
+      const consoleVersion = useTagAttributes
+        ? sanitizeAttributeValue(attributes['latest-console-tag'] || '')
+        : sanitizeAttributeValue(attributes['latest-console-version'] || '');
+      const redpandaRepo = isPrerelease ? 'redpanda-unstable' : 'redpanda';
+      const consoleRepo = 'console';
+
+      // YAML-specific replacements
       if (attachment.out.path.endsWith('.yaml') || attachment.out.path.endsWith('.yml')) {
-        const redpandaVersionRegex = /(\$\{REDPANDA_VERSION[^\}]*\})/g;
-        const redpandaConsoleVersionRegex = /(\$\{REDPANDA_CONSOLE_VERSION[^\}]*\})/g;
-        let fullVersion = attributes['full-version'] ? sanitizeAttributeValue(attributes['full-version']) : '';
-        const latestConsoleVersion = attributes['latest-console-version'] ? sanitizeAttributeValue(attributes['latest-console-version']) : '';
-        if (attributes['page-component-version-is-prerelease']) {
-          fullVersion = attributes['redpanda-beta-version'] ? sanitizeAttributeValue(attributes['redpanda-beta-version']) : fullVersion;
-        }
-        contentString = contentString.replace(redpandaVersionRegex, fullVersion);
-        contentString = contentString.replace(redpandaConsoleVersionRegex, latestConsoleVersion);
+        contentString = replacePlaceholder(contentString, /\$\{REDPANDA_DOCKER_REPO:[^\}]*\}/g, redpandaRepo);
+        contentString = replacePlaceholder(contentString, /\$\{CONSOLE_DOCKER_REPO:[^\}]*\}/g, consoleRepo);
+        contentString = replacePlaceholder(contentString, /\$\{REDPANDA_VERSION[^\}]*\}/g, redpandaVersion);
+        contentString = replacePlaceholder(contentString, /\$\{REDPANDA_CONSOLE_VERSION[^\}]*\}/g, consoleVersion);
         modified = true;
       }
 
-      const result = contentString.replace(/\{([\p{Alpha}\d_][\p{Alpha}\d_-]*)\}/gu, (match, name) => {
+      // General attribute replacements (excluding uppercase with underscores)
+      const result = contentString.replace(/\{([a-z][\p{Alpha}\d_-]*)\}/gu, (match, name) => {
         if (!(name in attributes)) return match;
         modified = true;
-        let value = attributes[name];
-        if (value.endsWith('@')) value = value.slice(0, value.length - 1);
-        return value;
+        return attributes[name];
       });
 
       if (modified) attachment.contents = Buffer.from(result);
     });
   });
+
+  // Helper function to replace placeholders with attribute values
+  function replacePlaceholder(content, regex, replacement) {
+    return content.replace(regex, replacement);
+  }
+
+  const sanitizeAttributeValue = (value) => String(value).replace('@', '');
+
+  const formatVersion = (version) => {
+    if (!version) return null;
+    return semver.valid(version) ? version : `${version}.0`;
+  };
 };
diff --git a/extensions/version-fetcher/get-latest-connect.js b/extensions/version-fetcher/get-latest-connect.js
@@ -2,8 +2,7 @@ module.exports = async (github, owner, repo) => {
 
   try {
     const release = await github.rest.repos.getLatestRelease({ owner, repo });
-    tag = release.data.tag_name.replace(/^v/, '');
-    return tag;
+    return release.data.tag_name
   } catch (error) {
     console.error(error);
     return null;

diff --git a/extensions/version-fetcher/get-latest-console-version.js b/extensions/version-fetcher/get-latest-console-version.js
@@ -1,5 +1,5 @@
 module.exports = async (github, owner, repo) => {
-  const semver = require('semver')
+  const semver = require('semver');
   try {
     const releases = await github.rest.repos.listReleases({
       owner,
@@ -8,15 +8,14 @@ module.exports = async (github, owner, repo) => {
       per_page: 50
     });
 
-    // Filter valid semver tags and sort them
+    // Filter tags with valid semver format
     const sortedReleases = releases.data
-      .map(release => release.tag_name.replace(/^v/, ''))
-      .filter(tag => semver.valid(tag))
-      // Sort in descending order to get the highest version first
-      .sort(semver.rcompare);
+      .map(release => release.tag_name)
+      .filter(tag => semver.valid(tag.replace(/^v/, '')))
+      .sort((a, b) => semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, '')));
 
     if (sortedReleases.length > 0) {
-      // Return the highest version
+      // Return the highest version with "v" prefix
       return sortedReleases[0];
     } else {
       console.log("No valid semver releases found.");

diff --git a/extensions/version-fetcher/get-latest-redpanda-version.js b/extensions/version-fetcher/get-latest-redpanda-version.js
@@ -11,21 +11,20 @@ module.exports = async (github, owner, repo) => {
 
     // Filter valid semver tags, exclude drafts, and sort them to find the highest version
     const sortedReleases = releases.data
-      .filter(release => !release.draft)
-      .map(release => release.tag_name.replace(/^v/, ''))
-      .filter(tag => semver.valid(tag))
-      // Sort in descending order to get the highest version first
-      .sort(semver.rcompare);
-
+    .filter(release => !release.draft)
+    .map(release => release.tag_name)
+    .filter(tag => semver.valid(tag.replace(/^v/, '')))
+    .sort((a, b) => semver.rcompare(a.replace(/^v/, ''), b.replace(/^v/, '')));
+
     if (sortedReleases.length > 0) {
-      const latestRedpandaReleaseVersion = sortedReleases.find(tag => !tag.includes('rc'));
-      const latestRcReleaseVersion = sortedReleases.find(tag => tag.includes('rc'));
+      const latestRedpandaReleaseVersion = sortedReleases.find(tag => !tag.includes('-rc'));
+      const latestRcReleaseVersion = sortedReleases.find(tag => tag.includes('-rc'));
 
       // Get the commit hash for the highest version tag
       const commitData = await github.rest.git.getRef({
         owner,
         repo,
-        ref: `tags/v${latestRedpandaReleaseVersion}`
+        ref: `tags/${latestRedpandaReleaseVersion}`
       });
       const latestRedpandaReleaseCommitHash = commitData.data.object.sha;
 
@@ -34,7 +33,7 @@ module.exports = async (github, owner, repo) => {
         const rcCommitData = await github.rest.git.getRef({
           owner,
           repo,
-          ref: `tags/v${latestRcReleaseVersion}`
+          ref: `tags/${latestRcReleaseVersion}`
         });
         latestRcReleaseCommitHash = rcCommitData.data.object.sha;
       }

diff --git a/extensions/version-fetcher/set-latest-version.js b/extensions/version-fetcher/set-latest-version.js
@@ -1,13 +1,16 @@
+'use strict';
+
 module.exports.register = function ({ config }) {
-  const GetLatestRedpandaVersion = require('./get-latest-redpanda-version')
-  const GetLatestConsoleVersion = require('./get-latest-console-version')
-  const GetLatestOperatorVersion = require('./get-latest-operator-version')
-  const GetLatestHelmChartVersion = require('./get-latest-redpanda-helm-version')
-  const GetLatestConnectVersion = require('./get-latest-connect')
-  const chalk = require('chalk')
-  const logger = this.getLogger('set-latest-version-extension')
+  const GetLatestRedpandaVersion = require('./get-latest-redpanda-version');
+  const GetLatestConsoleVersion = require('./get-latest-console-version');
+  const GetLatestOperatorVersion = require('./get-latest-operator-version');
+  const GetLatestHelmChartVersion = require('./get-latest-redpanda-helm-version');
+  const GetLatestConnectVersion = require('./get-latest-connect');
+  const chalk = require('chalk');
+  const logger = this.getLogger('set-latest-version-extension');
+
   if (!process.env.REDPANDA_GITHUB_TOKEN) {
-    logger.warn('REDPANDA_GITHUB_TOKEN environment variable not set. Attempting unauthenticated request.')
+    logger.warn('REDPANDA_GITHUB_TOKEN environment variable not set. Attempting unauthenticated request.');
   }
 
   this.on('contentClassified', async ({ contentCatalog }) => {
@@ -17,83 +20,105 @@ module.exports.register = function ({ config }) {
     const OctokitWithRetries = Octokit.plugin(retry);
 
     const owner = 'redpanda-data';
-
-    let githubOptions = {
+    const githubOptions = {
       userAgent: 'Redpanda Docs',
       baseUrl: 'https://api.github.com',
+      auth: process.env.REDPANDA_GITHUB_TOKEN || undefined,
     };
-
-    if (process.env.REDPANDA_GITHUB_TOKEN) {
-      githubOptions.auth = process.env.REDPANDA_GITHUB_TOKEN;
-    }
-
     const github = new OctokitWithRetries(githubOptions);
+
     try {
-      const results = await Promise.allSettled([
+      const [
+        latestRedpandaResult,
+        latestConsoleResult,
+        latestOperatorResult,
+        latestHelmChartResult,
+        latestConnectResult,
+      ] = await Promise.allSettled([
         GetLatestRedpandaVersion(github, owner, 'redpanda'),
         GetLatestConsoleVersion(github, owner, 'console'),
         GetLatestOperatorVersion(github, owner, 'redpanda-operator'),
         GetLatestHelmChartVersion(github, owner, 'helm-charts', 'charts/redpanda/Chart.yaml'),
-        GetLatestConnectVersion(github, owner, 'connect')
-      ])
+        GetLatestConnectVersion(github, owner, 'connect'),
+      ]);
 
-      const LatestRedpandaVersion = results[0].status === 'fulfilled' ? results[0].value : null
-      const LatestConsoleVersion = results[1].status === 'fulfilled' ? results[1].value : null
-      const LatestOperatorVersion = results[2].status === 'fulfilled' ? results[2].value : null
-      const LatestHelmChartVersion = results[3].status === 'fulfilled' ? results[3].value : null
-      const LatestConnectVersion = results[4].status === 'fulfilled' ? results[4].value : null
+      const latestVersions = {
+        redpanda: latestRedpandaResult.status === 'fulfilled' ? latestRedpandaResult.value : {},
+        console: latestConsoleResult.status === 'fulfilled' ? latestConsoleResult.value : undefined,
+        operator: latestOperatorResult.status === 'fulfilled' ? latestOperatorResult.value : undefined,
+        helmChart: latestHelmChartResult.status === 'fulfilled' ? latestHelmChartResult.value : undefined,
+        connect: latestConnectResult.status === 'fulfilled' ? latestConnectResult.value : undefined,
+      };
 
-      const components = await contentCatalog.getComponents()
+      const components = await contentCatalog.getComponents();
       components.forEach(component => {
-        let prerelease = component.latestPrerelease;
+        const prerelease = component.latestPrerelease;
+
         component.versions.forEach(({ name, version, asciidoc }) => {
-          // This attribute is used for conditionally rendering content for beta releases.
-          // It is also used in the `unpublish-pages` extension to unpublish beta pages that aren't part of a beta version.
-          if (prerelease && prerelease.version === version) {
-            asciidoc.attributes['page-component-version-is-prerelease'] = 'true'
+          if (prerelease?.version === version) {
+            asciidoc.attributes['page-component-version-is-prerelease'] = 'true';
           }
-          if (LatestConsoleVersion) {
-            asciidoc.attributes['latest-console-version'] = `${LatestConsoleVersion}@`
-            logger.info(`Set Redpanda Console version to ${LatestConsoleVersion} in ${name} ${version}`)
+
+          // Set operator and helm chart attributes
+          if (latestVersions.operator) {
+            asciidoc.attributes['latest-operator-version'] = latestVersions.operator;
           }
-          if (LatestConnectVersion) {
-            asciidoc.attributes['latest-connect-version'] = `${LatestConnectVersion}@`
-            logger.info(`Set Redpanda Connect version to ${LatestConnectVersion} in ${name} ${version}`)
+          if (latestVersions.helmChart) {
+            asciidoc.attributes['latest-redpanda-helm-chart-version'] = latestVersions.helmChart;
           }
-          if (LatestRedpandaVersion && LatestRedpandaVersion.latestRcRelease && LatestRedpandaVersion.latestRcRelease.version) {
-            asciidoc.attributes['redpanda-beta-version'] = `${LatestRedpandaVersion.latestRcRelease.version}@`
-            asciidoc.attributes['redpanda-beta-commit'] = `${LatestRedpandaVersion.latestRcRelease.commitHash}@`
-            logger.info(`Updated to latest Redpanda RC version: ${LatestRedpandaVersion.latestRcRelease.version} with commit: ${LatestRedpandaVersion.latestRcRelease.commitHash}`)
-        }
-        })
-
-        if (!component.latest.asciidoc) {
-          component.latest.asciidoc = { attributes: {} }
-        }
 
-        if (LatestRedpandaVersion && LatestRedpandaVersion.latestRedpandaRelease && semver.valid(LatestRedpandaVersion.latestRedpandaRelease.version)) {
-          let currentVersion = component.latest.asciidoc.attributes['full-version'] || '0.0.0'
-          if (semver.gt(LatestRedpandaVersion.latestRedpandaRelease.version, currentVersion)) {
-            component.latest.asciidoc.attributes['full-version'] = `${LatestRedpandaVersion.latestRedpandaRelease.version}@`
-            component.latest.asciidoc.attributes['latest-release-commit'] = `${LatestRedpandaVersion.latestRedpandaRelease.commitHash}@`
-            logger.info(`Updated to latest Redpanda version: ${LatestRedpandaVersion.latestRedpandaRelease.version} with commit: ${LatestRedpandaVersion.latestRedpandaRelease.commitHash}`)
+          // Set attributes for console and connect versions
+          if (latestVersions.console) {
+            setVersionAndTagAttributes(asciidoc, 'latest-console', latestVersions.console, name, version);
           }
-        }
+          if (latestVersions.connect) {
+            setVersionAndTagAttributes(asciidoc, 'latest-connect', latestVersions.connect, name, version);
+          }
+          // Special handling for Redpanda RC versions if in beta
+          if (latestVersions.redpanda?.latestRcRelease?.version) {
+            setVersionAndTagAttributes(asciidoc, 'redpanda-beta', latestVersions.redpanda.latestRcRelease.version, name, version)
+            asciidoc.attributes['redpanda-beta-commit'] = latestVersions.redpanda.latestRcRelease.commitHash;
+          }
+        });
 
-        if (LatestOperatorVersion) {
-          component.latest.asciidoc.attributes['latest-operator-version'] = `${LatestOperatorVersion}@`
-          logger.info(`Updated to latest Redpanda Operator version: ${LatestOperatorVersion}`)
-        }
+        if (!component.latest.asciidoc) component.latest.asciidoc = { attributes: {} };
 
-        if (LatestHelmChartVersion) {
-          component.latest.asciidoc.attributes['latest-redpanda-helm-chart-version'] = `${LatestHelmChartVersion}@`
-          logger.info(`Updated to latest Redpanda Helm chart version: ${LatestHelmChartVersion}`)
+        // For Redpanda GA version, set both latest-redpanda-version and latest-redpanda-tag if available
+        if (semver.valid(latestVersions.redpanda?.latestRedpandaRelease?.version)) {
+          const currentVersion = component.latest.asciidoc.attributes['full-version'] || '0.0.0';
+          if (semver.gt(latestVersions.redpanda.latestRedpandaRelease.version, currentVersion)) {
+            // Required for backwards compatibility. Some docs still use full-version
+            component.latest.asciidoc.attributes['full-version'] = sanitizeVersion(latestVersions.redpanda.latestRedpandaRelease.version);
+            setVersionAndTagAttributes(component.latest.asciidoc, 'latest-redpanda', latestVersions.redpanda.latestRedpandaRelease.version, component.latest.name, component.latest.version);
+            component.latest.asciidoc.attributes['latest-release-commit'] = latestVersions.redpanda.latestRedpandaRelease.commitHash;
+            logger.info(`Updated Redpanda release version to ${latestVersions.redpanda.latestRedpandaRelease.version}`);
+          }
         }
-      })
+      });
 
-      console.log(`${chalk.green('Updated Redpanda documentation versions successfully.')}`)
+      console.log(chalk.green('Updated Redpanda documentation versions successfully.'));
     } catch (error) {
-      logger.error(`Error updating versions: ${error}`)
+      logger.error(`Error updating versions: ${error}`);
     }
-  })
-}
+  });
+
+  // Helper function to set both latest-*version and latest-*tag attributes
+  function setVersionAndTagAttributes(asciidoc, baseName, versionData, name = '', version = '') {
+    if (versionData) {
+      const versionWithoutPrefix = sanitizeVersion(versionData);
+      asciidoc.attributes[`${baseName}-version`] = versionWithoutPrefix; // Without "v" prefix
+      asciidoc.attributes[`${baseName}-tag`] = `${versionData}`;
+
+      if (name && version) {
+        logger.info(`Set ${baseName}-version to ${versionWithoutPrefix} and ${baseName}-tag to ${versionData} in ${name} ${version}`);
+      } else {
+        logger.info(`Updated ${baseName}-version to ${versionWithoutPrefix} and ${baseName}-tag to ${versionData}`);
+      }
+    }
+  }
+
+  // Helper function to sanitize version by removing "v" prefix
+  function sanitizeVersion(version) {
+    return version.replace(/^v/, '');
+  }
+};
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@redpanda-data/docs-extensions-and-macros",
-  "version": "3.7.2",
+  "version": "3.7.3",
   "description": "Antora extensions and macros developed for Redpanda documentation.",
   "keywords": [
     "antora",

diff --git a/preview/extensions-and-macros/antora.yml b/preview/extensions-and-macros/antora.yml
@@ -9,4 +9,5 @@ asciidoc:
     replace-attributes-in-attachments: true
     test-attribute: This attribute was replaced by the `replace-attributes-in-attachments` extension.
     full-version: 23.2.1
+    latest-redpanda-version: v23.2.1
     latest-release-commit: 'This should get replaced'
diff --git a/preview/extensions-and-macros/modules/ROOT/pages/test.adoc b/preview/extensions-and-macros/modules/ROOT/pages/test.adoc
@@ -74,6 +74,7 @@ helm_ref:storage[]
 The `version fetcher` extension gets the latest version of Redpanda and Redpanda Console and assigns them to the following attributes:
 
 - `\{full-version}`: {full-version}
+- `\{latest-redpanda-version}`: {latest-redpanda-version}
 - `\{latest-release-commit}`: {latest-release-commit}
 - `\{latest-console-version}`: {latest-console-version}
 - `\{latest-operator-version}`: {latest-operator-version}