Tool to find unused variables (#50103)

package.json

@@ -35,6 +35,7 @@
     "find-orphaned-assets": "node src/assets/scripts/find-orphaned-assets.js",
     "find-orphaned-features": "tsx src/data-directory/scripts/find-orphaned-features/index.ts",
     "find-past-built-pr": "tsx src/workflows/find-past-built-pr.ts",
+    "find-unused-variables": "tsx src/content-linter/scripts/find-unsed-variables.ts",
     "fixture-dev": "cross-env ROOT=src/fixtures/fixtures npm start",
     "fixture-test": "cross-env ROOT=src/fixtures/fixtures npm test -- src/fixtures/tests",
     "index": "tsx src/search/scripts/index/index.ts",

src/content-linter/scripts/find-unsed-variables.ts

@@ -0,0 +1,149 @@
+/**
+ * This script iterates over all pages and all reusables and looks for
+ * mentions of variables in Liquid syntax. For example,
+ *
+ *    ---
+ *    title: '{% data variables.product.prodname_mobile %} is cool'
+ *    shortTitle: '{% data variables.product.prodname_mobile %}'
+ *    ---
+ *
+ *    This also mentions {% data variables.product.prodname_ios %}
+ *
+ * So in this case, we *know* that `prodname_mobile` and
+ * `prodname_ios` inside `data/variables/product.yml` is definitely used.
+ * So that variable won't be mentioned as unused.
+ *
+ */
+import fs from 'fs'
+import yaml from 'js-yaml'
+
+import { program } from 'commander'
+
+import { loadPages, loadUnversionedTree } from '@/frame/lib/page-data.js'
+import { TokenizationError } from 'liquidjs'
+
+import readFrontmatter from '@/frame/lib/read-frontmatter.js'
+import { getLiquidTokens } from '@/content-linter/lib/helpers/liquid-utils.js'
+import walkFiles from '@/workflows/walk-files.js'
+
+program
+  .description('Finds unused variables in frontmatter, content, and reusables')
+  .option('-o, --output-file <path>', 'path to output file', 'stdout')
+  .option('--json', 'serialize output in JSON')
+  .option('--markdown', 'serialize output as a Markdown comment')
+  .parse(process.argv)
+
+type Options = {
+  outputFile: string
+  json?: boolean
+  markdown?: boolean
+}
+main(program.opts())
+
+async function main(options: Options) {
+  const variables = getVariables()
+  const pages = await getPages()
+  for (const page of pages) {
+    try {
+      const filePath = page.fullPath
+      const fileContent = fs.readFileSync(filePath, 'utf-8')
+      const { content, data } = readFrontmatter(fileContent)
+      const title = (data && data.title) || ''
+      const shortTitle = (data && data.shortTitle) || ''
+      const intro = (data && data.intro) || ''
+      for (const string of [content, title, shortTitle, intro]) {
+        checkString(string, variables)
+      }
+    } catch (err) {
+      if (err instanceof Error && 'code' in err && err.code === 'ENOENT') continue
+      throw err
+    }
+  }
+  for (const filePath of getReusableFiles()) {
+    const fileContent = fs.readFileSync(filePath, 'utf-8')
+    checkString(fileContent, variables)
+  }
+
+  const { outputFile, json } = options
+  if (!outputFile || outputFile === 'stdout') {
+    if (json) {
+      console.log(JSON.stringify(Object.fromEntries(variables), null, 2))
+    } else {
+      console.log(variables)
+    }
+  } else if (options.markdown) {
+    let output = ''
+    const keys = Array.from(variables.values()).sort()
+    if (keys.length > 0) {
+      output += `There are ${variables.size} unused variables.\n\n`
+      output += '| Variable | File |\n'
+      output += '| --- | --- |\n'
+      for (const key of keys) {
+        output += `| ${key} | ${variables.get(key)} |\n`
+      }
+      output += `\nThis comment was generated by the \`find-unused-variables\` script.\n`
+    }
+    if (outputFile && output) {
+      fs.writeFileSync(outputFile, output, 'utf-8')
+    } else if (output) {
+      console.log(output)
+    }
+  } else {
+    if (json || outputFile.endsWith('.json')) {
+      fs.writeFileSync(outputFile, JSON.stringify(Object.fromEntries(variables), null, 2), 'utf-8')
+    } else {
+      let output = ''
+      for (const [key, value] of variables) {
+        output += `${key} in ${value}\n`
+      }
+      fs.writeFileSync(outputFile, output, 'utf-8')
+    }
+  }
+}
+
+function getVariables(): Map<string, string> {
+  const variables = new Map<string, string>()
+  for (const filePath of walkFiles('data/variables', '.yml')) {
+    const dottedPathBase =
+      'variables.' + filePath.replace('data/variables/', '').replace('.yml', '').replace(/\//g, '.')
+    const data = yaml.load(fs.readFileSync(filePath, 'utf-8')) as Record<string, unknown>
+    for (const key of Object.keys(data)) {
+      const dottedPath = dottedPathBase + '.' + key
+      variables.set(dottedPath, filePath)
+    }
+  }
+  return variables
+}
+
+async function getPages() {
+  const unversionedTree = await loadUnversionedTree([])
+  const pageList = await loadPages(unversionedTree)
+  return pageList
+}
+
+function getReusableFiles(root = 'data') {
+  const here: string[] = []
+  for (const file of fs.readdirSync(root)) {
+    const filePath = `${root}/${file}`
+    if (fs.statSync(filePath).isDirectory()) {
+      here.push(...getReusableFiles(filePath))
+    } else if (file.endsWith('.md') && file !== 'README.md') {
+      here.push(filePath)
+    }
+  }
+  return here
+}
+
+function checkString(string: string, variables: Map<string, string>) {
+  try {
+    for (const token of getLiquidTokens(string)) {
+      if (token.name === 'data') {
+        const { args } = token
+        variables.delete(args)
+      }
+    }
+  } catch (err) {
+    if (err instanceof TokenizationError) return
+    throw err
+  }
+}