From 89c28d22f60d4902d7829ac5c5a88b05869c752d Mon Sep 17 00:00:00 2001 From: Andy Scherzinger Date: Sun, 28 Jul 2024 12:07:45 +0200 Subject: [PATCH] ci: compile license files Signed-off-by: Andy Scherzinger --- build-js/WebpackSPDXPlugin.js | 221 ++++++++++++++++++++++++++++++++++ build-js/npm-post-build.sh | 24 ++++ package.json | 1 + webpack.js | 14 +++ 4 files changed, 260 insertions(+) create mode 100644 build-js/WebpackSPDXPlugin.js create mode 100755 build-js/npm-post-build.sh diff --git a/build-js/WebpackSPDXPlugin.js b/build-js/WebpackSPDXPlugin.js new file mode 100644 index 000000000..eeb338c03 --- /dev/null +++ b/build-js/WebpackSPDXPlugin.js @@ -0,0 +1,221 @@ +'use strict' + +/** + * Party inspired by https://github.com/FormidableLabs/webpack-stats-plugin + * + * SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors + * SPDX-License-Identifier: MIT + */ + +const { constants } = require('node:fs') +const fs = require('node:fs/promises') +const path = require('node:path') +const webpack = require('webpack') + +class WebpackSPDXPlugin { + + #options + + /** + * @param {object} opts Parameters + * @param {Record} opts.override Override licenses for packages + */ + constructor(opts = {}) { + this.#options = { override: {}, ...opts } + } + + apply(compiler) { + compiler.hooks.thisCompilation.tap('spdx-plugin', (compilation) => { + // `processAssets` is one of the last hooks before frozen assets. + // We choose `PROCESS_ASSETS_STAGE_REPORT` which is the last possible + // stage after which to emit. + compilation.hooks.processAssets.tapPromise( + { + name: 'spdx-plugin', + stage: compilation.constructor.PROCESS_ASSETS_STAGE_REPORT, + }, + () => this.emitLicenses(compilation), + ) + }) + } + + /** + * Find the nearest package.json + * @param {string} dir Directory to start checking + */ + async #findPackage(dir) { + if (!dir || dir === '/' || dir === '.') { + return null + } + + const packageJson = `${dir}/package.json` + try { + await fs.access(packageJson, constants.F_OK) + } catch (e) { + return await this.#findPackage(path.dirname(dir)) + } + + const { private: isPrivatePacket, name } = JSON.parse(await fs.readFile(packageJson)) + // "private" is set in internal package.json which should not be resolved but the parent package.json + // Same if no name is set in package.json + if (isPrivatePacket === true || !name) { + return (await this.#findPackage(path.dirname(dir))) ?? packageJson + } + return packageJson + } + + /** + * Emit licenses found in compilation to '.license' files + * @param {webpack.Compilation} compilation Webpack compilation object + * @param {*} callback Callback for old webpack versions + */ + async emitLicenses(compilation, callback) { + const logger = compilation.getLogger('spdx-plugin') + // cache the node packages + const packageInformation = new Map() + + const warnings = new Set() + /** @type {Map>} */ + const sourceMap = new Map() + + for (const chunk of compilation.chunks) { + for (const file of chunk.files) { + if (sourceMap.has(file)) { + sourceMap.get(file).add(chunk) + } else { + sourceMap.set(file, new Set([chunk])) + } + } + } + + for (const [asset, chunks] of sourceMap.entries()) { + /** @type {Set} */ + const modules = new Set() + /** + * @param {webpack.Module} module + */ + const addModule = (module) => { + if (module && !modules.has(module)) { + modules.add(module) + for (const dep of module.dependencies) { + addModule(compilation.moduleGraph.getModule(dep)) + } + } + } + chunks.forEach((chunk) => chunk.getModules().forEach(addModule)) + + const sources = [...modules].map((module) => module.identifier()) + .map((source) => { + const skipped = [ + 'delegated', + 'external', + 'container entry', + 'ignored', + 'remote', + 'data:', + ] + // Webpack sources that we can not infer license information or that is not included (external modules) + if (skipped.some((prefix) => source.startsWith(prefix))) { + return '' + } + // Internal webpack sources + if (source.startsWith('webpack/runtime')) { + return require.resolve('webpack') + } + // Handle webpack loaders + if (source.includes('!')) { + return source.split('!').at(-1) + } + if (source.includes('|')) { + return source + .split('|') + .filter((s) => s.startsWith(path.sep)) + .at(0) + } + return source + }) + .filter((s) => !!s) + .map((s) => s.split('?', 2)[0]) + + // Skip assets without modules, these are emitted by webpack plugins + if (sources.length === 0) { + logger.warn(`Skipping ${asset} because it does not contain any source information`) + continue + } + + /** packages used by the current asset + * @type {Set} + */ + const packages = new Set() + + // packages is the list of packages used by the asset + for (const sourcePath of sources) { + const pkg = await this.#findPackage(path.dirname(sourcePath)) + if (!pkg) { + logger.warn(`No package for source found (${sourcePath})`) + continue + } + + if (!packageInformation.has(pkg)) { + // Get the information from the package + const { author: packageAuthor, name, version, license: packageLicense, licenses } = JSON.parse(await fs.readFile(pkg)) + // Handle legacy packages + let license = !packageLicense && licenses + ? licenses.map((entry) => entry.type ?? entry).join(' OR ') + : packageLicense + if (license?.includes(' ') && !license?.startsWith('(')) { + license = `(${license})` + } + // Handle both object style and string style author + const author = typeof packageAuthor === 'object' + ? `${packageAuthor.name}` + (packageAuthor.mail ? ` <${packageAuthor.mail}>` : '') + : packageAuthor ?? `${name} developers` + + packageInformation.set(pkg, { + version, + // Fallback to directory name if name is not set + name: name ?? path.basename(path.dirname(pkg)), + author, + license, + }) + } + packages.add(pkg) + } + + let output = 'This file is generated from multiple sources. Included packages:\n' + const authors = new Set() + const licenses = new Set() + for (const packageName of [...packages].sort()) { + const pkg = packageInformation.get(packageName) + const license = this.#options.override[pkg.name] ?? pkg.license + // Emit warning if not already done + if (!license && !warnings.has(pkg.name)) { + logger.warn(`Missing license information for package ${pkg.name}, you should add it to the 'override' option.`) + warnings.add(pkg.name) + } + licenses.add(license || 'unknown') + authors.add(pkg.author) + output += `- ${pkg.name}\n\t- version: ${pkg.version}\n\t- license: ${license}\n` + } + output = `\n\n${output}` + for (const author of [...authors].sort()) { + output = `SPDX-FileCopyrightText: ${author}\n${output}` + } + for (const license of [...licenses].sort()) { + output = `SPDX-License-Identifier: ${license}\n${output}` + } + + compilation.emitAsset( + asset.split('?', 2)[0] + '.license', + new webpack.sources.RawSource(output), + ) + } + + if (callback) { + return callback() + } + } + +} + +module.exports = WebpackSPDXPlugin diff --git a/build-js/npm-post-build.sh b/build-js/npm-post-build.sh new file mode 100755 index 000000000..c1c69a277 --- /dev/null +++ b/build-js/npm-post-build.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: 2024 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +# Add licenses for source maps +if [ -d "js" ]; then + for f in js/*.js; do + # If license file and source map exists copy license for the source map + if [ -f "$f.license" ] && [ -f "$f.map" ]; then + # Remove existing link + [ -e "$f.map.license" ] || [ -L "$f.map.license" ] && rm "$f.map.license" + # Create a new link + ln -s "$(basename "$f.license")" "$f.map.license" + fi + done + echo "Copying licenses for sourcemaps done" +else + echo "This script needs to be executed from the root of the repository" + exit 1 +fi + diff --git a/package.json b/package.json index decfce1ff..85754ac08 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "private": true, "scripts": { "build": "NODE_ENV=production webpack --progress --config webpack.js", + "postbuild": "build/npm-post-build.sh", "dev": "NODE_ENV=development webpack --progress --config webpack.js", "watch": "NODE_ENV=development webpack --progress --watch --config webpack.js", "serve": "NODE_ENV=development webpack serve --progress --config webpack.js", diff --git a/webpack.js b/webpack.js index 7c5502604..e79c62dd1 100644 --- a/webpack.js +++ b/webpack.js @@ -1,6 +1,7 @@ const path = require('path') const webpack = require('webpack') const webpackConfig = require('@nextcloud/webpack-vue-config') +const WebpackSPDXPlugin = require('./build-js/WebpackSPDXPlugin.js') const webpackRules = require('@nextcloud/webpack-vue-config/rules') const SassGridConfig = require('./src/utils/SassGridConfig') @@ -87,4 +88,17 @@ webpackConfig.plugins.push( }) ) +webpackConfig.plugins = [ + ...webpackConfig.plugins, + // Generate reuse license files + new WebpackSPDXPlugin({ + override: { + // TODO: Remove if they fixed the license in the package.json + '@nextcloud/axios': 'GPL-3.0-or-later', + '@nextcloud/vue': 'AGPL-3.0-or-later', + 'nextcloud-vue-collections': 'AGPL-3.0-or-later', + } + }), +] + module.exports = webpackConfig