PCC-1026 Reorganize importer code. Implement smart component replacement for drupal import.

.changeset/afraid-maps-smell.md

@@ -0,0 +1,7 @@
+---
+"@pantheon-systems/pcc-cli": patch
+---
+
+Added --publish flag to Drupal import. Fixed drupal import bugs. Added link to
+folder after import completes. Fixed createSmartComponent function call using
+incorrect endpoint.

.changeset/chatty-singers-scream.md

@@ -0,0 +1,5 @@
+---
+"@pantheon-systems/pcc-cli": minor
+---
+
+Internal refactoring of 3rd-party file imports.

packages/cli/package.json

@@ -49,6 +49,7 @@
     "google-auth-library": "^9.6.3",
     "googleapis": "^129.0.0",
     "inquirer": "^8.2.6",
+    "node-html-parser": "^6.1.12",
     "nunjucks": "^3.2.4",
     "octokit": "^3.1.2",
     "open": "^9.1.0",

packages/cli/src/cli/commands/import/common.ts

@@ -0,0 +1,62 @@
+import { exit } from "process";
+import chalk from "chalk";
+import type { GaxiosResponse } from "gaxios";
+import { OAuth2Client } from "google-auth-library";
+import { drive_v3, google } from "googleapis";
+import ora from "ora";
+import AddOnApiHelper from "../../../lib/addonApiHelper";
+import { getLocalAuthDetails } from "../../../lib/localStorage";
+import { Logger } from "../../../lib/logger";
+
+export async function createFileOnDrive({
+  requestBody,
+  body,
+  drive,
+}: {
+  requestBody: Partial<drive_v3.Schema$File>;
+  body: string;
+  drive?: drive_v3.Drive;
+}) {
+  const logger = new Logger();
+
+  if (!drive) {
+    // Check user has required permission to create drive file
+    await AddOnApiHelper.getIdToken([
+      "https://www.googleapis.com/auth/drive.file",
+    ]);
+    const authDetails = await getLocalAuthDetails();
+    if (!authDetails) {
+      logger.error(chalk.red(`ERROR: Failed to retrieve login details.`));
+      exit(1);
+    }
+
+    const oauth2Client = new OAuth2Client();
+    oauth2Client.setCredentials(authDetails);
+    drive = google.drive({
+      version: "v3",
+      auth: oauth2Client,
+    });
+  }
+
+  // Create Google Doc
+  const spinner = ora("Creating document on the Google Drive...").start();
+  const res = (await drive.files.create({
+    requestBody: {
+      ...requestBody,
+      mimeType: "application/vnd.google-apps.document",
+    },
+    media: {
+      mimeType: "text/html",
+      body,
+    },
+  })) as GaxiosResponse<drive_v3.Schema$File>;
+  const fileId = res.data.id;
+  const fileUrl = `https://docs.google.com/document/d/${fileId}`;
+
+  if (!fileId) {
+    spinner.fail("Failed to create document on the Google Drive.");
+    exit(1);
+  }
+
+  return { fileId, fileUrl, drive, spinner };
+}

packages/cli/src/cli/commands/import.ts → packages/cli/src/cli/commands/import/drupal.ts

@@ -1,27 +1,24 @@
 import { randomUUID } from "crypto";
-import * as fs from "fs";
 import { exit } from "process";
 import axios, { AxiosError } from "axios";
 import Promise from "bluebird";
 import chalk from "chalk";
-import { parseFromString } from "dom-parser";
 import type { GaxiosResponse } from "gaxios";
 import { OAuth2Client } from "google-auth-library";
 import { drive_v3, google } from "googleapis";
-import ora from "ora";
+import { HTMLElement, parse } from "node-html-parser";
 import queryString from "query-string";
-import showdown from "showdown";
-import AddOnApiHelper from "../../lib/addonApiHelper";
-import { getLocalAuthDetails } from "../../lib/localStorage";
-import { Logger } from "../../lib/logger";
-import { errorHandler } from "../exceptions";
-
-const HEADING_TAGS = ["h1", "h2", "h3", "title"];
+import AddOnApiHelper from "../../../lib/addonApiHelper";
+import { getLocalAuthDetails } from "../../../lib/localStorage";
+import { Logger } from "../../../lib/logger";
+import { errorHandler } from "../../exceptions";
+import { createFileOnDrive } from "./common";
 
 type DrupalImportParams = {
   baseUrl: string;
   siteId: string;
   verbose: boolean;
+  automaticallyPublish: boolean;
 };
 
 interface DrupalPost {
@@ -80,7 +77,12 @@ async function getDrupalPosts(url: string) {
 }
 
 export const importFromDrupal = errorHandler<DrupalImportParams>(
-  async ({ baseUrl, siteId, verbose }: DrupalImportParams) => {
+  async ({
+    baseUrl,
+    siteId,
+    verbose,
+    automaticallyPublish,
+  }: DrupalImportParams) => {
     const logger = new Logger();
 
     if (baseUrl) {
@@ -104,7 +106,7 @@ export const importFromDrupal = errorHandler<DrupalImportParams>(
       }
     }
 
-    await AddOnApiHelper.getIdToken([
+    const idToken = await AddOnApiHelper.getIdToken([
       "https://www.googleapis.com/auth/drive.file",
     ]);
 
@@ -122,11 +124,12 @@ export const importFromDrupal = errorHandler<DrupalImportParams>(
       auth: oauth2Client,
     });
 
+    const folderName = `PCC Import from Drupal on ${new Date().toLocaleDateString()} unique id: ${randomUUID()}`;
     const folderRes = (await drive.files
       .create({
         fields: "id,name",
         requestBody: {
-          name: `PCC Import from Drupal on ${new Date().toLocaleDateString()} unique id: ${randomUUID()}`,
+          name: folderName,
           mimeType: "application/vnd.google-apps.folder",
         },
       })
@@ -190,26 +193,36 @@ export const importFromDrupal = errorHandler<DrupalImportParams>(
           (x) => x.id === post.relationships.field_author.data.id,
         )?.attributes?.title;
 
-        const res = (await drive.files.create({
+        // Initially create a blank document, just to get an article id
+        // that we can work with for further steps, such as adding smart components.
+        const { fileId, spinner } = await createFileOnDrive({
           requestBody: {
-            // Name from the article.
             name: post.attributes.title,
-            mimeType: "application/vnd.google-apps.document",
+
             parents: [folderId],
           },
+          body: "",
+          drive,
+        });
+        spinner.succeed();
+
+        // Add it to the PCC site.
+        await AddOnApiHelper.getDocument(fileId, true, undefined, idToken);
+
+        // Set the document's content.
+        (await drive.files.update({
+          fileId,
+          requestBody: {
+            mimeType: "application/vnd.google-apps.document",
+          },
           media: {
             mimeType: "text/html",
-            body: post.attributes.body.processed,
+            body: await processHTMLForSmartComponents(
+              post.attributes.body.processed,
+              fileId,
+            ),
           },
         })) as GaxiosResponse<drive_v3.Schema$File>;
-        const fileId = res.data.id;
-
-        if (!fileId) {
-          throw new Error(`Failed to create file for ${post.attributes.title}`);
-        }
-
-        // Add it to the PCC site.
-        await AddOnApiHelper.getDocument(fileId, true);
 
         try {
           await AddOnApiHelper.updateDocument(
@@ -230,7 +243,9 @@ export const importFromDrupal = errorHandler<DrupalImportParams>(
             verbose,
           );
 
-          await AddOnApiHelper.publishDocument(fileId);
+          if (automaticallyPublish) {
+            await AddOnApiHelper.publishDocument(fileId);
+          }
         } catch (e) {
           console.error(e instanceof AxiosError ? e.response?.data : e);
           throw e;
@@ -243,109 +258,44 @@ export const importFromDrupal = errorHandler<DrupalImportParams>(
 
     logger.log(
       chalk.green(
-        `Successfully imported ${allPosts.length} documents into ${folderRes.data.name}`,
+        `Successfully imported ${allPosts.length} documents into ${folderName} (https://drive.google.com/drive/u/0/folders/${folderRes.data.id})`,
       ),
     );
   },
 );
 
-type MarkdownImportParams = {
-  filePath: string;
-  siteId: string;
-  verbose: boolean;
-  publish: boolean;
-};
-
-export const importFromMarkdown = errorHandler<MarkdownImportParams>(
-  async ({ filePath, siteId, verbose, publish }: MarkdownImportParams) => {
-    const logger = new Logger();
+async function processHTMLForSmartComponents(html: string, articleId: string) {
+  const root = parse(html);
+  const iframeNodes: HTMLElement[] =
+    (root.querySelector("iframe")?.childNodes as HTMLElement[]) ?? [];
 
-    if (!fs.existsSync(filePath)) {
-      logger.error(
-        chalk.red(
-          `ERROR: Could not find markdown file at given path (${filePath})`,
-        ),
-      );
-      exit(1);
-    }
+  await Promise.all(
+    iframeNodes.map(async (node) => {
+      let src = node.getAttribute("src");
 
-    // Prepare article content and title
-    const content = fs.readFileSync(filePath).toString();
+      if (src == null) return;
 
-    // Check user has required permission to create drive file
-    await AddOnApiHelper.getIdToken([
-      "https://www.googleapis.com/auth/drive.file",
-    ]);
-    const authDetails = await getLocalAuthDetails();
-    if (!authDetails) {
-      logger.error(chalk.red(`ERROR: Failed to retrieve login details.`));
-      exit(1);
-    }
-
-    // Create Google Doc
-    const spinner = ora("Creating document on the Google Drive...").start();
-    const oauth2Client = new OAuth2Client();
-    oauth2Client.setCredentials(authDetails);
-    const drive = google.drive({
-      version: "v3",
-      auth: oauth2Client,
-    });
-    const converter = new showdown.Converter();
-    const html = converter.makeHtml(content);
-    const dom = parseFromString(html);
-
-    // Derive document's title
-    let title: string | undefined = undefined;
-    for (const item of HEADING_TAGS) {
-      const element = dom.getElementsByTagName(item)[0];
-      if (element) {
-        title = element.textContent;
-        break;
+      if (src.includes("oembed?url=")) {
+        src = decodeURIComponent(src.split("oembed?url=")[1]);
       }
-    }
-    title = title || "Untitled Document";
 
-    const res = (await drive.files.create({
-      requestBody: {
-        name: title,
-        mimeType: "application/vnd.google-apps.document",
-      },
-      media: {
-        mimeType: "text/html",
-        body: html,
-      },
-    })) as GaxiosResponse<drive_v3.Schema$File>;
-    const fileId = res.data.id;
-    const fileUrl = `https://docs.google.com/document/d/${fileId}`;
-
-    if (!fileId) {
-      spinner.fail("Failed to create document on the Google Drive.");
-      exit(1);
-    }
+      const componentType = "MEDIA_PREVIEW";
+      const componentId = await AddOnApiHelper.createSmartComponent(
+        articleId,
+        {
+          url: src,
+          canUsePlainIframe: true,
+        },
+        componentType,
+      );
 
-    // Create PCC document
-    await AddOnApiHelper.getDocument(fileId, true, title);
-    // Cannot set metadataFields(title,slug) in the same request since we reset metadataFields
-    //  when changing the siteId.
-    await AddOnApiHelper.updateDocument(
-      fileId,
-      siteId,
-      title,
-      [],
-      null,
-      verbose,
-    );
-    await AddOnApiHelper.getDocument(fileId, false, title);
+      node.replaceWith(
+        parse(
+          `<a href="https://pcc.media/${componentType}/${componentId}">MEDIA_PREVIEW: ${src}</a>`,
+        ),
+      );
+    }),
+  );
 
-    // Publish PCC document
-    if (publish) {
-      await AddOnApiHelper.publishDocument(fileId);
-    }
-    spinner.succeed(
-      `Successfully created document at below path${
-        publish ? " and published it on the PCC." : ":"
-      }`,
-    );
-    logger.log(chalk.green(fileUrl, "\n"));
-  },
-);
+  return root.toString();
+}

packages/cli/src/cli/commands/import/index.ts

@@ -0,0 +1,4 @@
+import { importFromDrupal } from "./drupal";
+import { importFromMarkdown } from "./markdown";
+
+export { importFromDrupal, importFromMarkdown };