Merge branch 'main' into education-add-blackboard-lms

.devcontainer/Dockerfile

@@ -1,7 +1,7 @@
 # To find available Node images, see https://mcr.microsoft.com/en-us/product/devcontainers/javascript-node/tags
 
 # [Choice] Node.js version
-ARG VARIANT="dev-20-bullseye"
+ARG VARIANT="dev-22-bullseye"
 FROM mcr.microsoft.com/devcontainers/javascript-node:${VARIANT}
 
 # [Optional] Uncomment this section to install additional OS packages.

.devcontainer/devcontainer.json

@@ -2,70 +2,67 @@
 // https://github.com/microsoft/vscode-dev-containers/tree/v0.177.0/containers/javascript-node
 // -
 {
-    "name": "docs.github.com",
-    "build": {
-        "dockerfile": "Dockerfile",
-        // Update 'VARIANT' to pick a Node version
-        "args": { "VARIANT": "20" }
-    },
+  "name": "docs.github.com",
+  "build": {
+    "dockerfile": "Dockerfile",
+    // Update 'VARIANT' to pick a Node version
+    "args": { "VARIANT": "22" }
+  },
 
-    // Install features. Type 'feature' in the VS Code command palette for a full list.
-    "features": {
-        "sshd": "latest"
-    },
+  // Install features. Type 'feature' in the VS Code command palette for a full list.
+  "features": {
+    "sshd": "latest"
+  },
 
-    "customizations": {
-        "vscode": {
-            // Set *default* container specific settings.json values on container create.
-            "settings": {
-                "terminal.integrated.shell.linux": "/bin/bash",
-                "cSpell.language": ",en"
-            },
-            // Visual Studio Code extensions which help authoring for docs.github.com.
-            "extensions": [
-                "dbaeumer.vscode-eslint",
-                "sissel.shopify-liquid",
-                "davidanson.vscode-markdownlint",
-                "bierner.markdown-preview-github-styles",
-                "streetsidesoftware.code-spell-checker",
-                "alistairchristie.open-reusables",
-                "AlistairChristie.version-identifier",
-                "peterbe.ghdocs-goer",
-                "GitHub.copilot",
-                "GitHub.copilot-chat"
-            ]
-        },
-        "codespaces": {
-            "repositories": {
-                // allow Codespaces to pull from separate repo when user has access
-                "github/docs-early-access": {
-                    "permissions": {
-                        "contents": "write"
-                    }
-                }
-            }
-        }
+  "customizations": {
+    "vscode": {
+      // Set *default* container specific settings.json values on container create.
+      "settings": {
+        "terminal.integrated.shell.linux": "/bin/bash",
+        "cSpell.language": ",en"
+      },
+      // Visual Studio Code extensions which help authoring for docs.github.com.
+      "extensions": [
+        "dbaeumer.vscode-eslint",
+        "sissel.shopify-liquid",
+        "davidanson.vscode-markdownlint",
+        "bierner.markdown-preview-github-styles",
+        "streetsidesoftware.code-spell-checker",
+        "alistairchristie.open-reusables",
+        "AlistairChristie.version-identifier",
+        "peterbe.ghdocs-goer",
+        "GitHub.copilot",
+        "GitHub.copilot-chat"
+      ]
     },
-
-    // Use 'forwardPorts' to make a list of ports inside the container available locally.
-    "forwardPorts": [4000],
-
-    "portsAttributes": {
-        "4000": {
-            "label": "Preview"
+    "codespaces": {
+      "repositories": {
+        // allow Codespaces to pull from separate repo when user has access
+        "github/docs-early-access": {
+          "permissions": {
+            "contents": "write"
+          }
         }
-    },
+      }
+    }
+  },
 
-    // Use 'postCreateCommand' to run commands after the container is created.
-    "postCreateCommand": "npm ci",
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  "forwardPorts": [4000],
 
-    // Use 'updateContentCommand' to run commands to be included in Codespace pre-builds
-    "updateContentCommand": "git clone https://github.com/github/rest-api-description.git",
+  "portsAttributes": {
+    "4000": {
+      "label": "Preview"
+    }
+  },
 
-    // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
-    "remoteUser": "node",
+  // Use 'postCreateCommand' to run commands after the container is created.
+  "postCreateCommand": "npm ci",
 
-    "hostRequirements": {
-        "memory": "8gb"
-    }
+  // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+  "remoteUser": "node",
+
+  "hostRequirements": {
+    "memory": "8gb"
+  }
 }

.env.example

@@ -0,0 +1,23 @@
+# This file is a template for what your untracked .env file might look like for local development.
+# Please copy this to a new .env file and fill in the values as needed.
+
+# Requires a running local Elasticsearch service. Can be started via Docker, see https://github.com/github/docs-engineering/blob/main/docs/elasticsearch/elasticsearch-locally.md
+# When this value is unset searches will be proxied to the production Elasticsearch endpoint
+ELASTICSEARCH_URL=http://localhost:9200
+
+# Set for sending events in local development. See https://github.com/github/docs-engineering/blob/main/docs/analytics/hydro-mock.md
+HYDRO_ENDPOINT=
+HYDRO_SECRET=
+
+# Localization variables
+# See https://github.com/github/docs-internal/tree/main/src/languages#working-with-translated-content-locally
+ENABLED_LANGUAGES=
+TRANSLATIONS_ROOT=
+
+# For running the src/search/scripts/scrape script
+# You may want a lower value depending on your CPU
+BUILD_RECORDS_MAX_CONCURRENT=100
+BUILD_RECORDS_MIN_TIME=
+
+# Set to true to enable the /fastly-cache-test route for debugging Fastly headers
+ENABLE_FASTLY_TESTING=

.github/workflows/azure-preview-env-deploy-public.yml

@@ -37,7 +37,15 @@ jobs:
   build-and-deploy-azure-preview-public:
     name: Build and deploy Azure preview environment (public)
     runs-on: ubuntu-latest
-    if: github.repository == 'github/docs'
+    # Ensure this is actually a pull request and not a merge group
+    # If its a merge group, report success without doing anything
+    # See https://bit.ly/3qB9nZW > If a job in a workflow is skipped due to a conditional, it will report its status as "Success".
+    if: |
+      (
+        (github.event.pull_request.head.sha)
+         && (github.event.number || github.run_id)
+       )
+       && github.repository == 'github/docs'
     timeout-minutes: 15
     environment:
       name: preview-env-${{ github.event.number }}

.github/workflows/azure-prod-build-deploy.yml

@@ -61,12 +61,11 @@ jobs:
 
       # Create a temporary token for the Azure Container Registry
       # and set it as a GitHub Actions environment variable
-      # Then clean up by deleting the temp token.
       # Created token are viewable in the ACR resource UI
       # under Repository permissions > Tokens
       - name: 'Create Azure Container Registry Token'
         env:
-          PROD_REGISTRY_SERVER: ${{ secrets.PROD_REGISTRY_SERVER }}
+          CONTAINER_REGISTRY_SERVER: ${{ secrets.PROD_REGISTRY_SERVER }}
         run: npm run create-acr-token
 
       - name: 'Docker login'
@@ -119,7 +118,7 @@ jobs:
 
       - name: 'Apply updated docker-compose.prod.yaml config to canary slot'
         run: |
-          az webapp config container set --multicontainer-config-type COMPOSE --multicontainer-config-file docker-compose.prod.yaml --slot ${{ env.SLOT_NAME }} -n ${{ env.APP_SERVICE_NAME }} -g ${{ env.RESOURCE_GROUP_NAME }} --container-registry-url ${{ secrets.PROD_REGISTRY_SERVER }} --container-registry-user ${{ env.ACR_TOKEN_NAME }} --container-registry-password ${{ env.ACR_TOKEN_VALUE }}
+          az webapp config container set --multicontainer-config-type COMPOSE --multicontainer-config-file docker-compose.prod.yaml --slot ${{ env.SLOT_NAME }} -n ${{ env.APP_SERVICE_NAME }} -g ${{ env.RESOURCE_GROUP_NAME }}
 
       # Watch canary slot instances to see when all the instances are ready
       - name: Check that canary slot is ready

.github/workflows/index-autocomplete-elasticsearch.yml → .github/workflows/index-autocomplete-search.yml

@@ -1,7 +1,7 @@
-name: Index autocomplete Elasticsearch
+name: Index autocomplete search in Elasticsearch
 
-# **What it does**: Indexes autocomplete data into Elasticsearch.
-# **Why we have it**: So we can power the API for autocomplete.
+# **What it does**: Indexes autocomplete data (general and AI search) into Elasticsearch.
+# **Why we have it**: So we can power the APIs for autocomplete.
 # **Who does it impact**: docs-engineering
 
 on:
@@ -10,7 +10,7 @@ on:
     - cron: '20 16 * * *' # Run every day at 16:20 UTC / 8:20 PST
   pull_request:
     paths:
-      - .github/workflows/index-autocomplete-elasticsearch.yml
+      - .github/workflows/index-autocomplete-search.yml
       - 'src/search/scripts/index/**'
       - 'package*.json'
 
@@ -40,10 +40,15 @@ jobs:
         if: ${{ github.event_name == 'pull_request' }}
         run: curl --fail --retry-connrefused --retry 5 -I http://localhost:9200
 
-      - name: Run indexing
+      - name: Run general auto-complete indexing
         env:
           ELASTICSEARCH_URL: ${{ github.event_name == 'pull_request' && 'http://localhost:9200' || secrets.ELASTICSEARCH_URL }}
-        run: npm run index -- autocomplete docs-internal-data
+        run: npm run index-general-autocomplete -- docs-internal-data
+
+      - name: Run AI search auto-complete indexing
+        env:
+          ELASTICSEARCH_URL: ${{ github.event_name == 'pull_request' && 'http://localhost:9200' || secrets.ELASTICSEARCH_URL }}
+        run: npm run index-ai-search-autocomplete -- docs-internal-data
 
       - uses: ./.github/actions/slack-alert
         if: ${{ failure() && github.event_name == 'schedule' }}

.github/workflows/sync-search-pr.yml → .github/workflows/index-general-search-pr.yml

@@ -1,6 +1,6 @@
-name: Sync search - PR
+name: Index general search in Elasticsearch on PR
 
-# **What it does**: This does what `sync-sarch-elasticsearch.yml` does but
+# **What it does**: This does what `index-general-search-elasticsearch.yml` does but
 #                   with a localhost Elasticsearch and only for English.
 # **Why we have it**: To test that the script works and the popular pages json is valid.
 # **Who does it impact**: Docs engineering
@@ -11,8 +11,8 @@ on:
     paths:
       - 'src/search/**'
       - 'package*.json'
-      # Ultimately, for debugging this workflow itself
-      - .github/workflows/sync-search-pr.yml
+      # For debugging this workflow
+      - .github/workflows/index-general-search-pr.yml
       # Make sure we run this if the composite action changes
       - .github/actions/setup-elasticsearch/action.yml
 
@@ -25,9 +25,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  # Yes, it's hardcoded but it makes all the steps look exactly the same
-  # as they do in `sync-search-elasticsearch.yml` where it uses
-  # that `${{ env.ELASTICSEARCH_URL }}`
   ELASTICSEARCH_URL: http://localhost:9200
   # Since we'll run in NDOE_ENV=production, we need to be explicit that
   # we don't want Hydro configured.
@@ -63,7 +60,7 @@ jobs:
         env:
           ENABLE_DEV_LOGGING: false
         run: |
-          npm run sync-search-server > /tmp/stdout.log 2> /tmp/stderr.log &
+          npm run general-search-scrape-server > /tmp/stdout.log 2> /tmp/stderr.log &
 
           # first sleep to give it a chance to start
           sleep 6
@@ -88,15 +85,13 @@ jobs:
           # let's just accept an empty string instead.
           THROW_ON_EMPTY: false
 
-          # The sync-search-index recognizes this env var if you don't
-          # use the `--docs-internal-data <PATH>` option.
           DOCS_INTERNAL_DATA: docs-internal-data
 
         run: |
           mkdir /tmp/records
-          npm run sync-search-indices -- /tmp/records \
+          npm run general-search-scrape -- /tmp/records \
             --language en \
-            --version dotcom
+            --version fpt
 
           ls -lh /tmp/records
 
@@ -106,9 +101,9 @@ jobs:
 
       - name: Index into Elasticsearch
         run: |
-          npm run index-elasticsearch -- /tmp/records \
+          npm run index-general-search -- /tmp/records \
             --language en \
-            --version dotcom
+            --version fpt
 
       - name: Check created indexes and aliases
         run: |

.github/workflows/sync-search-elasticsearch.yml → .github/workflows/index-general-search.yml

@@ -1,4 +1,4 @@
-name: Sync search Elasticsearch
+name: Index general search in Elasticsearch
 
 # **What it does**: It scrapes the whole site and dumps the records in a
 #                   temp directory. Then it indexes that into Elasticsearch.
@@ -140,7 +140,7 @@ jobs:
         env:
           ENABLE_DEV_LOGGING: false
         run: |
-          npm run sync-search-server > /tmp/stdout.log 2> /tmp/stderr.log &
+          npm run general-search-scrape-server > /tmp/stdout.log 2> /tmp/stderr.log &
 
           # first sleep to give it a chance to start
           sleep 6
@@ -169,13 +169,11 @@ jobs:
           # the same as not set within the script.
           VERSION: ${{ inputs.version }}
 
-          # The sync-search-index recognizes this env var if you don't
-          # use the `--docs-internal-data <PATH>` option.
           DOCS_INTERNAL_DATA: docs-internal-data
 
         run: |
           mkdir /tmp/records
-          npm run sync-search-indices -- /tmp/records \
+          npm run general-search-scrape -- /tmp/records \
             --language ${{ matrix.language }}
 
           ls -lh /tmp/records
@@ -186,12 +184,12 @@ jobs:
 
       - name: Index into Elasticsearch
         env:
-          # Must match what we used when scraping (npm run sync-search-indices)
+          # Must match what we used when scraping (npm run general-search-scrape)
           # otherwise the script will seek other versions from disk that might
           # not exist.
           VERSION: ${{ inputs.version }}
         run: |
-          npm run index-elasticsearch -- /tmp/records \
+          npm run index-general-search -- /tmp/records \
             --language ${{ matrix.language }} \
             --stagger-seconds 5 \
             --retries 5

.github/workflows/link-check-daily.yml

@@ -86,7 +86,7 @@ jobs:
           # treat it as a warning and not as a broken link.
           EXTERNAL_SERVER_ERRORS_AS_WARNINGS: true
           FAIL_ON_FLAW: false
-        timeout-minutes: 60
+        timeout-minutes: 120
         run: npm run rendered-content-link-checker
 
       - name: Insight into external link checker DB json file (after)

.github/workflows/local-dev.yml

@@ -21,8 +21,7 @@ jobs:
       - name: Check out repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - name: Install dependencies
-        run: npm install
+      - uses: ./.github/actions/node-npm-setup
 
       - uses: ./.github/actions/get-docs-early-access
         if: ${{ github.repository == 'github/docs-internal' }}

.github/workflows/orphaned-files-check.yml

@@ -24,7 +24,7 @@ permissions:
   contents: read
 
 jobs:
-  orphaned-assets-check:
+  orphaned-files-check:
     if: ${{ github.repository == 'github/docs-internal' }}
     runs-on: ubuntu-latest
     steps: