Merge pull request #25 from MichiganDaily/staging

Staging

README.md

@@ -2,8 +2,7 @@
 
 > Everything and the kitchen sink
 
-A collection of helper scripts that are used across The Michigan Daily's
-projects.
+A collection of helper scripts that are used across The Michigan Daily's projects.
 
 ## Installation
 
@@ -16,8 +15,8 @@ It's really easy, just run
 ```sh
 sink gsheet  # fetch Google Sheets
 sink gdoc    # fetch Google Docs
-sink fetch # fetch Google Sheets and Docs
+sink json    # fetch JSON files
+sink fetch   # fetch everything
 ```
 
-Like before, you still need a `config.json` (usually comes with the template
-that is using `sink`) and an `auth.json` (usually comes with your MOE).
+Like before, you still need a `config.json` (usually comes with the template that is using `sink`) and an `auth.json` (usually comes with your MOE).

config.example.json

@@ -1,15 +1,23 @@
 {
   "fetch": [
     {
+      "type": "doc",
       "id": "",
       "output": "",
       "auth": "~/.daily-google-services.json"
     },
     {
+      "type": "sheet",
       "id": "",
       "sheetId": "0",
       "output": "",
       "auth": "~/.daily-google-services.json"
+    },
+    {
+      "type": "json",
+      "id": "",
+      "output": "",
+      "auth": "~/.daily-google-services.json"
     }
   ]
 }

package.json

@@ -19,15 +19,15 @@
   "private": true,
   "type": "module",
   "dependencies": {
-    "@googleapis/drive": "^2.1.0",
-    "@googleapis/sheets": "^0.3.0",
+    "@googleapis/drive": "^3.0.1",
+    "@googleapis/sheets": "^3.0.2",
     "archieml": "^0.5.0",
-    "chalk": "^4.1.2",
-    "commander": "^8.2.0",
+    "chalk": "^5.0.1",
+    "commander": "^9.4.0",
     "d3-dsv": "^3.0.1",
-    "find-up": "^6.2.0",
-    "google-auth-library": "^7.11.0",
-    "html-entities": "^2.3.2",
-    "htmlparser2": "^7.1.2"
+    "find-up": "^6.3.0",
+    "google-auth-library": "^8.4.0",
+    "html-entities": "^2.3.3",
+    "htmlparser2": "^8.0.1"
   }
 }

src/_utils.js

@@ -29,7 +29,7 @@ export const get_auth = (path, scopes) => {
   if (!existsSync(file)) {
     fatal_error(`
   Could not open service account credentials at ${file}.
-  Reconfigure fetch.sheets.auth in config.json or download the credentials file.
+  Reconfigure your auth properties in config.json or download the credentials file.
   `);
   }
 

src/sink-fetch.js

@@ -1,17 +1,24 @@
 import { fileURLToPath } from "node:url";
 
-import { program } from "commander/esm.mjs";
+import { program } from "commander";
 
 import { fetchDoc } from "./sink-gdoc.js";
 import { fetchSheet } from "./sink-gsheet.js";
+import { fetchJson } from "./sink-json.js";
 import { load_config } from "./_utils.js";
 
 const main = async (opts) => {
+  const typeToFunction = {
+    doc: fetchDoc,
+    sheet: fetchSheet,
+    json: fetchJson,
+  };
+
   const { config } = await load_config(opts.config);
   config.fetch
     .filter((d) => d.id.length && d.output.length)
     .forEach((file) => {
-      const func = file.sheetId == null ? fetchDoc : fetchSheet;
+      const func = typeToFunction[file.type];
       func(file);
     });
 };

src/sink-gdoc.js

@@ -8,15 +8,14 @@
 import { fileURLToPath } from "node:url";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 
-import { program } from "commander/esm.mjs";
+import { program } from "commander";
 import { drive } from "@googleapis/drive";
 import { decode } from "html-entities";
 
 import archieml from "archieml";
 const { load } = archieml;
 
-import htmlparser2 from "htmlparser2";
-const { DomHandler, Parser } = htmlparser2;
+import { DomHandler, Parser } from "htmlparser2";
 
 import { load_config, success, get_auth } from "./_utils.js";
 
@@ -125,7 +124,7 @@ export const fetchDoc = async ({ id, output, auth }) => {
 const main = async (opts) => {
   const { config } = await load_config(opts.config);
   const files = config.fetch.filter(
-    (d) => d.sheetId == null && d.id.length && d.output.length
+    (d) => d.type === "doc" && d.id.length && d.output.length
   );
   files.forEach(fetchDoc);
 };

src/sink-gsheet.js

@@ -3,7 +3,7 @@
 import { fileURLToPath } from "node:url";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 
-import { program } from "commander/esm.mjs";
+import { program } from "commander";
 import { sheets } from "@googleapis/sheets";
 import { csvFormat } from "d3-dsv";
 
@@ -52,7 +52,8 @@ export const fetchSheet = async ({ id, sheetId, output, auth }) => {
 async function main(opts) {
   const { config } = await load_config(opts.config);
   const files = config.fetch.filter(
-    (d) => d.sheetId !== undefined && d.id.length && d.output.length
+    (d) =>
+      d.type === "sheet" && d.id.length && d.output.length && d.sheetId.length
   );
   files.forEach(fetchSheet);
 }

src/sink-json.js

@@ -0,0 +1,40 @@
+import { fileURLToPath } from "node:url";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+
+import { program } from "commander";
+import { drive } from "@googleapis/drive";
+
+import { load_config, success, get_auth } from "./_utils.js";
+
+export const fetchJson = async ({ id, output, auth }) => {
+  const scopes = ["https://www.googleapis.com/auth/drive"];
+  const authObject = get_auth(auth, scopes);
+
+  const gdrive = drive({ version: "v3", auth: authObject });
+
+  const { data } = await gdrive.files.get({ fileId: id, alt: "media" });
+
+  const dir = output.substring(0, output.lastIndexOf("/"));
+  !existsSync(dir.length > 0 ? dir : ".") &&
+    mkdirSync(dir, { recursive: true });
+  writeFileSync(output, JSON.stringify(data));
+  success(`Wrote output to ${output}`);
+};
+
+const main = async (opts) => {
+  const { config } = await load_config(opts.config);
+  const files = config.fetch.filter(
+    (d) => d.type === "json" && d.id.length && d.output.length
+  );
+  files.forEach(fetchJson);
+};
+
+const self = fileURLToPath(import.meta.url);
+if (process.argv[1] === self) {
+  program
+    .version("1.3.0")
+    .option("-c, --config <path>", "path to config file")
+    .parse();
+
+  main(program.opts());
+}

src/sink.js

@@ -1,13 +1,14 @@
 #!/usr/bin/env node
 
-import { program } from "commander/esm.mjs";
+import { program } from "commander";
 
 program
   .version("1.3.0")
   .name("sink")
   .description("Utility scripts")
   .command("gdoc", "fetch ArchieML Google Doc into JSON file")
   .command("gsheet", "fetch Google Sheet into CSV file")
+  .command("json", "fetch JSON files from Google Drive")
   .command("fetch", "fetch all Google Docs and Sheets");
 
 program.parse(process.argv);