feat: automatically generate docspec for Python projects

.eslintrc.js

@@ -23,5 +23,7 @@ module.exports = {
 
 		// We import from the default theme but its not a dep
 		'import/no-extraneous-dependencies': 'off',
+		'unicorn/prefer-ternary': 'off',
+		'unicorn/no-abusive-eslint-disable': 'off',
 	},
 };

packages/plugin/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apify/docusaurus-plugin-typedoc-api",
-  "version": "4.2.2",
+  "version": "4.2.3-0",
   "description": "Docusaurus plugin that provides source code API documentation powered by TypeDoc. ",
   "keywords": [
     "docusaurus",
@@ -48,13 +48,15 @@
     "@vscode/codicons": "^0.0.35",
     "marked": "^9.1.6",
     "marked-smartypants": "^1.1.5",
-    "typedoc": "^0.25.7"
+    "typedoc": "^0.25.7",
+    "zx": "^8.1.4"
   },
   "devDependencies": {
     "@docusaurus/module-type-aliases": "^3.5.2",
     "@docusaurus/tsconfig": "^3.5.2",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "typescript": "^5.3.3"
-  }
+  },
+  "stableVersion": "4.2.2"
 }

packages/plugin/src/index.ts

@@ -13,6 +13,7 @@ import {
 	generateJson,
 	loadPackageJsonAndDocs,
 } from './plugin/data';
+import { generateJsonFromPythonProject } from './plugin/python-generator';
 import { extractSidebar } from './plugin/sidebar';
 import { getVersionedDocsDirPath, readVersionsMetadata } from './plugin/version';
 import type {
@@ -55,6 +56,7 @@ const DEFAULT_OPTIONS: Required<DocusaurusPluginTypeDocApiOptions> = {
 	remarkPlugins: [],
 	rehypePlugins: [],
 	versions: {},
+	python: false,
 };
 
 async function importFile<T>(file: string): Promise<T> {
@@ -147,12 +149,19 @@ export default function typedocApiPlugin(
 
 					console.log(`[${prefix}]:`, 'Generating docs...');
 
-					await generateJson(
-						projectRoot,
-						entryPoints,
-						path.join(outDir, 'api-typedoc.json'),
-						options,
-					);
+					if (options.python) {
+						await generateJsonFromPythonProject({
+							projectRoot,
+							outFile: path.join(outDir, 'api-typedoc.json'),
+						});
+					} else {
+						await generateJson(
+							projectRoot,
+							entryPoints,
+							path.join(outDir, 'api-typedoc.json'),
+							options,
+						);
+					}
 
 					console.log(`[${prefix}]:`, 'Persisting packages...');
 
@@ -198,6 +207,11 @@ export default function typedocApiPlugin(
 									fs.mkdirSync(context.generatedFilesDir, { recursive: true });
 								}
 								fs.copyFileSync(options.pathToCurrentVersionTypedocJSON, outFile);
+							} else if (options.python) {
+								await generateJsonFromPythonProject({
+									projectRoot,
+									outFile,
+								});
 							} else {
 								await generateJson(projectRoot, entryPoints, outFile, options);
 							}

packages/plugin/src/plugin/python-generator/index.ts

@@ -0,0 +1,15 @@
+import { parseWithPydocMarkdown } from "./pydoc-markdown";
+import { pydocToTypedoc } from "./transform-docs";
+
+export async function generateJsonFromPythonProject({
+    outFile,
+    projectRoot,
+} : { outFile: string, projectRoot: string }): Promise<void> {
+    const pydocJson = await parseWithPydocMarkdown({ projectRoot });
+
+    await pydocToTypedoc({
+        moduleName: 'python', // TODO: get from project config files or passed options
+        outFile,
+        pydocJson,
+    });
+}

packages/plugin/src/plugin/python-generator/pydoc-markdown.ts

@@ -0,0 +1,65 @@
+import { rmSync, writeFileSync } from 'fs';
+import path from 'path';
+import { $ } from 'zx';
+
+/**
+ * Generates the pydoc-markdown configuration file
+ * @returns The pydoc-markdown configuration file as a string
+ */
+function getConfigYml({
+    projectRoot
+}: { projectRoot: string }): string {
+    return `
+loaders:
+  - type: python
+    search_path: ["${projectRoot}"]
+processors:
+  - type: filter
+    skip_empty_modules: true
+  - type: crossref
+renderer:
+  type: docusaurus
+  docs_base_path: docs
+  relative_output_path: reference
+  relative_sidebar_path: sidebar.json
+  sidebar_top_level_label: null
+`
+}
+
+export async function parseWithPydocMarkdown({
+    projectRoot,
+}: {
+    projectRoot: string,
+}
+): Promise<string> {
+    // Check whether the user has Python and pydoc-markdown installed
+    for (const cmd of ['python', 'pydoc-markdown']) {
+        try {
+            // eslint-disable-next-line no-await-in-loop
+            await $`${cmd} --version`;
+        } catch {
+            throw new Error(`Please install ${cmd} to use this plugin with Python projects.`);
+        }
+    };
+
+    // Generate the JSON file
+    try {
+        const configYml = getConfigYml({ projectRoot });
+        const configPath = path.join(__dirname, 'pydoc-markdown.temp.yml');
+        writeFileSync(configPath, configYml);
+
+        const pydoc = await $`pydoc-markdown --quiet --dump ${configPath}`;
+
+        rmSync(configPath);
+
+        let json = await pydoc.text();
+
+        json = json.replaceAll(path.resolve(projectRoot), 'REPO_ROOT_PLACEHOLDER');
+
+        return json;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    } catch (error: any) {
+        // eslint-disable-next-line 
+        throw new Error(`Failed to generate JSON file from Python project:\n\t${error.stderr.split('\n').slice(-2).join('\n')}`);
+    }
+}