diff --git a/extensions/antora/antora-modify-sitemaps/modify-sitemaps.js b/extensions/antora/antora-modify-sitemaps/modify-sitemaps.js index dcfdac0..69d9ace 100644 --- a/extensions/antora/antora-modify-sitemaps/modify-sitemaps.js +++ b/extensions/antora/antora-modify-sitemaps/modify-sitemaps.js @@ -1,4 +1,5 @@ const { posix: path } = require('path') +const semver = require('semver') module.exports.register = function ({ config }) { @@ -14,58 +15,119 @@ module.exports.register = function ({ config }) { const SITEMAP_EXT = '.xml' const logger = this.getLogger('modify-sitemaps') - let componentVersions, mappableComponentVersions = {} - let excludeVersions = {} - - let latestVersions + let componentVersions, mappedComponentVersions = {} + let mappedVersions = {} + let unMappableComponents = [] this - .on('navigationBuilt', ({ contentCatalog }) => { + .on('navigationBuilt', ({ playbook, contentCatalog }) => { + const files = contentCatalog.findBy({ family: 'nav' }) + componentVersions = files.reduce((v, file) => { - const latestComponentVersion = contentCatalog.getComponent(file.src.component).latest.version - v[file.src.component] = latestComponentVersion + v.hasOwnProperty(file.src.component) ? null : v[file.src.component] = { latest: '', versions: [] } + v[file.src.component].versions.indexOf(file.src.version) === -1 ? v[file.src.component].versions.push(file.src.version) : null return v; }, {}); - defaultSiteMapVersion = contentCatalog.getComponent(files[0].src.component).latest.version + // derive a default component from site startPage if possible + const defaultComponent = playbook.site.startPage ? contentCatalog.resolvePage(playbook.site.startPage).src.origin.descriptor.name : Object.keys(componentVersions)[0] ; + + // check latest is not a prerelease and revert to latest actual release if it is + for (const component of Object.keys(componentVersions)) { + const latest = contentCatalog.getComponent(component).latest.version + if (latest === '') { + componentVersions[component].latest = '~' + continue + } + const latestCheck = latest === ('current' || '') ? latest : semver.coerce(latest, { loose: true, includePrerelease: true }) + if (latestCheck && semver.prerelease(latestCheck)) { + const releases = componentVersions[component].versions + + // turn releases into semver objects + const semverList = [] + const semverObj = {} + for (const r of releases) { + const s = r === 'current' ? 'current' : semver.valid(semver.coerce(r, { loose: true, includePrerelease: true })) + if (s) { + semverList.push(s) + semverObj[s] = r + } + } + + // ignore prereleases + for (s of semver.rsort(semverList)) { + if (!semver.prerelease(s)) { + componentVersions[component].latest = semverObj[s] + logger.info({ }, '%s version %s is the latest version according to semantic versioning rules', component, semverObj[s]) + break + } else { + logger.info({ }, '%s version %s is a prerelease', component, semverObj[s]) + } + } + + } else { + componentVersions[component].latest = latest + } + } - const { sitemapVersion, data = { components: componentVersions }, sitemapLocVersion = 'current' } = config + const defaultSiteMapVersion = componentVersions[defaultComponent].latest + + const { sitemapVersion, data = { components: {}}, sitemapLocVersion = 'current' } = config if (!sitemapVersion && data.components.length == 0) { logger.error({ }, 'sitemap_version is required but has not been specified in the playbook. Default sitemap generation will be used') return } + // check for each component if we can make a sitemap for the version specified + for (const c of Object.keys(componentVersions)) { + if (data.components[c]) logger.info({ }, '%s sitemap will be generated from version %s (specified by playbook data)', c, data.components[c]) + else if (sitemapVersion) logger.info({ }, '%s sitemap will be generated from version %s (specified by playbook)', c, sitemapVersion) + else if (componentVersions[c].latest) logger.info({ }, '%s sitemap will be generated from version %s (specified by semantic versioning rules)', c, componentVersions[c].latest) + + const versionToMap = data.components[c] || sitemapVersion || componentVersions[c].latest || defaultSiteMapVersion || '' + if (versionToMap && versionToMap != '~' && !componentVersions[c].versions.includes(versionToMap)) { + logger.warn({ }, 'Component \'%s\' does not include version \'%s\'. Available versions are \'%s\'', c, versionToMap, componentVersions[c].versions.join(', ') ) + unMappableComponents.push(c) + } + } + const delegate = this.getFunctions().mapSite this.replaceFunctions({ mapSite (playbook, pages) { - const mappablePages = pages.reduce((mappable, file) => { - - const mappableVersion = sitemapVersion || data.components[file.src.component] || '' - const mappableFile = ( file.src.version == mappableVersion || ( !file.src.version && !mappableVersion ) ) - - if (mappableFile) { - logger.debug({ file: file.src }, 'Adding file in %s %s to sitemap', file.src.component, file.src.version || '(versionless)') - } else { - logger.debug({ file: file.src }, 'NOT adding file in %s %s to sitemap', file.src.component, file.src.version || '(versionlesscontent)') - } + const publishablePages = contentCatalog.getPages((page) => page.out) + const mappablePages = publishablePages.reduce((mappable, file) => { + + // is this component mappable? + // const mappableComponent = !unMappableComponents.includes(file.src.component) + // what version of this file's component are we trying to add to the sitemap? + let mappableVersion = data.components[file.src.component] || sitemapVersion || componentVersions[file.src.component].latest || defaultSiteMapVersion || '' + if (mappableVersion === '~') mappableVersion = '' + // is this file in that version of the component? + const mappableFile = ( file.src.version == mappableVersion || ( !file.src.version && !mappableVersion ) ) + + if (mappableFile) { + logger.debug({ file: file.src }, 'Adding file in %s %s to sitemap', file.src.component, file.src.version || '(versionless)') + } else { + logger.debug({ file: file.src }, 'NOT adding file in %s %s to sitemap', file.src.component, file.src.version || '(versionlesscontent)') + } - excludeVersions[file.src.component] = ( typeof excludeVersions[file.src.component] != 'undefined' && excludeVersions[file.src.component] instanceof Array ) ? excludeVersions[file.src.component] : [] + mappedVersions[file.src.component] = ( typeof mappedVersions[file.src.component] != 'undefined' && mappedVersions[file.src.component] instanceof Array ) ? mappedVersions[file.src.component] : [] - if ( mappableVersion ) file.pub.url = file.pub.url.replace(mappableVersion,sitemapLocVersion) - if ( mappableFile) { - - mappable.push(file); - mappableComponentVersions[file.src.component] = mappableVersion; + if ( mappableVersion ) file.pub.url = file.pub.url.replace(mappableVersion,sitemapLocVersion) + if ( mappableFile) { + mappable.push(file); + mappedComponentVersions[file.src.component] = mappableVersion; } else { - if (!excludeVersions[file.src.component].includes(file.src.version)) excludeVersions[file.src.component].push(file.src.version) + if (!mappedVersions[file.src.component].includes(file.src.version)) mappedVersions[file.src.component].push(file.src.version) + } return mappable; }, []); - logger.info({ }, 'Adding %d %s to the %s', mappablePages.length, pluralize(mappablePages.length, 'page'), pluralize(mappableComponentVersions.length, 'sitemap')) + + logger.info({ }, 'Adding %d %s to the %s', mappablePages.length, pluralize(mappablePages.length, 'page'), pluralize(mappedComponentVersions.length, 'sitemap')) return delegate.call(this, playbook, mappablePages) - } }) }) @@ -84,7 +146,7 @@ module.exports.register = function ({ config }) { return } - logger.info({ }, '%s generated', pluralize(mappableComponentVersions.length, 'Sitemap')) + logger.info({ }, '%d %s generated', Object.keys(mappedComponentVersions).length, pluralize(Object.keys(mappedComponentVersions).length, 'Sitemap')) const siteFiles = siteCatalog.getFiles((page) => page.out) @@ -97,8 +159,8 @@ module.exports.register = function ({ config }) { let dirname, versionDir, path_ if (file.out.path.startsWith(SITEMAP_STEM) && sitemapFiles.length == 1) { - dirname = Object.keys(mappableComponentVersions)[0] - versionDir = mappableComponentVersions[dirname] != '' ? mappableComponentVersions[dirname] : '' ; + dirname = Object.keys(mappedComponentVersions)[0] + versionDir = mappedComponentVersions[dirname] != '' ? mappedComponentVersions[dirname] : '' ; path_ = path.join(dirname, versionDir, SITEMAP_STEM+SITEMAP_EXT) } @@ -112,7 +174,7 @@ module.exports.register = function ({ config }) { if (file.out.path.startsWith(SITEMAP_PREFIX)) { dirname = file.out.path.replace(SITEMAP_PREFIX,'').replace(SITEMAP_EXT,'') - versionDir = mappableComponentVersions[dirname] != '' ? mappableComponentVersions[dirname] : '' ; + versionDir = mappedComponentVersions[dirname] != '' ? mappedComponentVersions[dirname] : '' ; path_ = path.join(dirname, versionDir, SITEMAP_STEM+SITEMAP_EXT) } @@ -125,16 +187,16 @@ module.exports.register = function ({ config }) { }) // components without sitemaps - for (const component in excludeVersions) { - if (!Object.keys(mappableComponentVersions).includes(component)) { - const mappableVersion = sitemapVersion || data.components[component] - logger.warn({ }, 'Could not create sitemap for \'%s\' version \'%s\'. Available versions are \'%s\'', component, mappableVersion, excludeVersions[component].join(', ') ) + for (const component in mappedVersions) { + if (!Object.keys(mappedComponentVersions).includes(component)) { + const mappableVersion = sitemapVersion || componentVersions[file.src.component].latest || defaultSiteMapVersion + logger.warn({ }, 'Could not create sitemap for \'%s\' version \'%s\'. Available versions are \'%s\'', component, mappableVersion, componentVersions[component].versions.join(', ') ) } } // components without sitemaps for (const component in data.components) { - if (!Object.keys(excludeVersions).includes(component)) { + if (!Object.keys(mappedVersions).includes(component)) { logger.warn({ }, 'Sitemap generation for \'%s\' version \'%s\' specified, but no files for this component were found in the site catalog', component, data.components[component] ) } } diff --git a/extensions/antora/antora-modify-sitemaps/package.json b/extensions/antora/antora-modify-sitemaps/package.json index 88b7df9..5b431e8 100644 --- a/extensions/antora/antora-modify-sitemaps/package.json +++ b/extensions/antora/antora-modify-sitemaps/package.json @@ -1,6 +1,6 @@ { "name": "@neo4j-antora/antora-modify-sitemaps", - "version": "0.5.0", + "version": "0.6.0", "description": "Override default Antora sitemap generator to include only pages for the current versions of components, and optionally move sitemaps into the component folders for the current versions", "main": "modify-sitemaps.js", "scripts": { @@ -10,5 +10,8 @@ "antora" ], "author": "Neo4j", - "license": "MIT" + "license": "MIT", + "dependencies": { + "semver": "^7.6.3" + } } diff --git a/package-lock.json b/package-lock.json index a66b42f..f000f47 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6880,4 +6880,4 @@ } } } -} +} \ No newline at end of file diff --git a/scripts/surge-teardown/package.json b/scripts/surge-teardown/package.json new file mode 100644 index 0000000..f558729 --- /dev/null +++ b/scripts/surge-teardown/package.json @@ -0,0 +1,16 @@ +{ + "name": "surge-teardown", + "version": "1.0.0", + "description": "", + "main": "surge-teardown.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Neo4j", + "license": "MIT", + "dependencies": { + "dotenv": "^16.4.7", + "octokit": "^3.2.1", + "strip-ansi": "^6.0.0" + } +} diff --git a/scripts/surge-teardown/surge-teardown.js b/scripts/surge-teardown/surge-teardown.js new file mode 100644 index 0000000..61c461a --- /dev/null +++ b/scripts/surge-teardown/surge-teardown.js @@ -0,0 +1,89 @@ +const util = require('util'); +const exec = util.promisify(require('child_process').exec); +const fs = require('fs'); +const stripAnsi = require('strip-ansi'); +const { Octokit } = require("octokit"); +const { env } = require('process'); + +require('dotenv').config() +const { GH_TOKEN, SKIP_NEO_TECHNOLOGY } = process.env; + +const octokit = new Octokit({ + auth: GH_TOKEN +}) + +async function teardownDeploy(deploy) { + try { + const { stdout, stderr } = await exec(`surge teardown ${deploy}`); + console.log('stdout:', stdout); + console.log('stderr:', stderr); + }catch (err) { + console.error(err); + }; +} + +async function surgeList() { + try { + const { stdout, stderr } = await exec('surge list'); + console.log('stdout:', stdout); + console.log('stderr:', stderr); + + const deploys = stripAnsi(stdout).split('\n'); + + // const deploys = fs.readFileSync('deploys.txt','utf-8').split(/\r?\n|\r|\n/g); + + return deploys; + }catch (err) { + console.error(err); + }; +}; + +const getPRStatus = async(org, repo, prNumber) => { + try { + const { data } = await octokit.request('GET /repos/{owner}/{repo}/pulls/{pull_number}', { + owner: org, + repo: repo, + pull_number: prNumber, + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + return data.state; + }catch (err) { + console.error(err); + }; +} + +surgeList().then((deploys) => { + + for (let deploy of deploys) { + const deployDetails = deploy.replace(/[ \s\t]+/g,' ').trim().split(' '); + + // ge the deploy url + const deployUrl = deployDetails[0] + if (!deployUrl) continue; + console.log('checking PR:', deployUrl) + // derive the pr details from the deploy url + const prDetails = deployUrl.replace('.surge.sh','').split(/[.-]/); + const prNumber = prDetails.pop(); + if (isNaN(prNumber)) continue; + + const org = prDetails[0] === 'neo4j' ? 'neo4j' : prDetails.slice(0,2).join('-'); + + // neo-technology is protected by SAML + if (org === 'neo-technology' && SKIP_NEO_TECHNOLOGY) continue; + + const repo = prDetails.join('-').replace(org+'-',''); + // console.log(deployUrl) + // check the pr details to see if the pr is closed + getPRStatus(org, repo, prNumber).then((prStatus) => { + // if the pr is closed, teardown the deploy + if (prStatus === 'closed') { + console.log(`${deployUrl} - PR is closed`); + if (env.CI) teardownDeploy(deployUrl); + } else { + // console.log(`${deployUrl} - PR is not closed`); + } + }) + } +}); \ No newline at end of file