Merge branch 'master' into BUDI-9011

.github/workflows/budibase_ci.yml

@@ -202,6 +202,9 @@ jobs:
 
       - run: yarn --frozen-lockfile
 
+      - name: Build client library - necessary for component tests
+        run: yarn build:client
+
       - name: Set up PostgreSQL 16
         if: matrix.datasource == 'postgres'
         run: |

hosting/letsencrypt/nginx-ssl.conf

@@ -41,6 +41,11 @@ server {
     }
 
     location ~ ^/api/(system|admin|global)/ {
+        # Enable buffering for potentially large OIDC configs
+        proxy_buffering on;
+        proxy_buffer_size 16k;
+        proxy_buffers 4 32k;
+
         proxy_pass      http://127.0.0.1:4002;
     }
 

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "3.4.4",
+  "version": "3.4.7",
   "npmClient": "yarn",
   "concurrency": 20,
   "command": {

package.json

@@ -18,6 +18,7 @@
     "eslint-plugin-jest": "28.9.0",
     "eslint-plugin-local-rules": "3.0.2",
     "eslint-plugin-svelte": "2.46.1",
+    "svelte-preprocess": "^6.0.3",
     "husky": "^8.0.3",
     "kill-port": "^1.6.1",
     "lerna": "7.4.2",
@@ -67,6 +68,7 @@
     "lint:fix:eslint": "eslint --fix --max-warnings=0 packages",
     "lint:fix:prettier": "prettier --write \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
     "lint:fix": "yarn run lint:fix:eslint && yarn run lint:fix:prettier",
+    "build:client": "lerna run --stream build --scope @budibase/client",
     "build:specs": "lerna run --stream specs",
     "build:docker:airgap": "node hosting/scripts/airgapped/airgappedDockerBuild",
     "build:docker:airgap:single": "SINGLE_IMAGE=1 node hosting/scripts/airgapped/airgappedDockerBuild",

packages/backend-core/__mocks__/@aws-sdk/client-s3.ts

@@ -0,0 +1,28 @@
+export class S3 {
+  headBucket() {
+    return jest.fn().mockReturnThis()
+  }
+  deleteObject() {
+    return jest.fn().mockReturnThis()
+  }
+  deleteObjects() {
+    return jest.fn().mockReturnThis()
+  }
+  createBucket() {
+    return jest.fn().mockReturnThis()
+  }
+  getObject() {
+    return jest.fn().mockReturnThis()
+  }
+  listObject() {
+    return jest.fn().mockReturnThis()
+  }
+  promise() {
+    return jest.fn().mockReturnThis()
+  }
+  catch() {
+    return jest.fn()
+  }
+}
+
+export const GetObjectCommand = jest.fn(inputs => ({ inputs }))

packages/backend-core/__mocks__/@aws-sdk/s3-request-presigner.ts

@@ -0,0 +1,4 @@
+export const getSignedUrl = jest.fn((_, cmd) => {
+  const { inputs } = cmd
+  return `http://s3.example.com/${inputs?.Bucket}/${inputs?.Key}`
+})

packages/backend-core/package.json

@@ -30,6 +30,9 @@
     "test:watch": "jest --watchAll"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "3.709.0",
+    "@aws-sdk/lib-storage": "3.709.0",
+    "@aws-sdk/s3-request-presigner": "3.709.0",
     "@budibase/nano": "10.1.5",
     "@budibase/pouchdb-replication-stream": "1.2.11",
     "@budibase/shared-core": "*",
@@ -71,19 +74,20 @@
   "devDependencies": {
     "@jest/types": "^29.6.3",
     "@shopify/jest-koa-mocks": "5.1.1",
+    "@smithy/types": "4.0.0",
     "@swc/core": "1.3.71",
     "@swc/jest": "0.2.27",
     "@types/chance": "1.1.3",
     "@types/cookies": "0.7.8",
     "@types/jest": "29.5.5",
+    "@types/koa": "2.13.4",
     "@types/lodash": "4.14.200",
     "@types/node-fetch": "2.6.4",
     "@types/pouchdb": "6.4.2",
     "@types/redlock": "4.0.7",
     "@types/semver": "7.3.7",
     "@types/tar-fs": "2.0.1",
     "@types/uuid": "8.3.4",
-    "@types/koa": "2.13.4",
     "chance": "1.1.8",
     "ioredis-mock": "8.9.0",
     "jest": "29.7.0",

packages/backend-core/src/docIds/params.ts

@@ -8,6 +8,10 @@ import {
 import { getProdAppID } from "./conversions"
 import { DatabaseQueryOpts, VirtualDocumentType } from "@budibase/types"
 
+const EXTERNAL_TABLE_ID_REGEX = new RegExp(
+  `^${DocumentType.DATASOURCE_PLUS}_(.+)__(.+)$`
+)
+
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this
  * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
@@ -64,6 +68,11 @@ export function getQueryIndex(viewName: ViewName) {
   return `database/${viewName}`
 }
 
+export const isExternalTableId = (id: string): boolean => {
+  const matches = id.match(EXTERNAL_TABLE_ID_REGEX)
+  return !!id && matches !== null
+}
+
 /**
  * Check if a given ID is that of a table.
  */
@@ -72,7 +81,7 @@ export const isTableId = (id: string): boolean => {
   return (
     !!id &&
     (id.startsWith(`${DocumentType.TABLE}${SEPARATOR}`) ||
-      id.startsWith(`${DocumentType.DATASOURCE_PLUS}${SEPARATOR}`))
+      isExternalTableId(id))
   )
 }
 

packages/backend-core/src/environment.ts

@@ -154,7 +154,7 @@ const environment = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   AWS_SESSION_TOKEN: process.env.AWS_SESSION_TOKEN,
-  AWS_REGION: process.env.AWS_REGION,
+  AWS_REGION: process.env.AWS_REGION || "eu-west-1",
   MINIO_URL: process.env.MINIO_URL,
   MINIO_ENABLED: process.env.MINIO_ENABLED || 1,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

packages/backend-core/src/objectStore/buckets/app.ts

@@ -13,7 +13,7 @@ export function clientLibraryPath(appId: string) {
  * due to issues with the domain we were unable to continue doing this - keeping
  * incase we are able to switch back to CDN path again in future.
  */
-export function clientLibraryCDNUrl(appId: string, version: string) {
+export async function clientLibraryCDNUrl(appId: string, version: string) {
   let file = clientLibraryPath(appId)
   if (env.CLOUDFRONT_CDN) {
     // append app version to bust the cache
@@ -24,7 +24,7 @@ export function clientLibraryCDNUrl(appId: string, version: string) {
     // file is public
     return cloudfront.getUrl(file)
   } else {
-    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
+    return await objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file)
   }
 }
 
@@ -44,10 +44,10 @@ export function clientLibraryUrl(appId: string, version: string) {
   return `/api/assets/client?${qs.encode(qsParams)}`
 }
 
-export function getAppFileUrl(s3Key: string) {
+export async function getAppFileUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
-    return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, s3Key)
+    return await objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, s3Key)
   }
 }

packages/backend-core/src/objectStore/buckets/global.ts

@@ -5,15 +5,19 @@ import * as cloudfront from "../cloudfront"
 
 // URLs
 
-export const getGlobalFileUrl = (type: string, name: string, etag?: string) => {
+export const getGlobalFileUrl = async (
+  type: string,
+  name: string,
+  etag?: string
+) => {
   let file = getGlobalFileS3Key(type, name)
   if (env.CLOUDFRONT_CDN) {
     if (etag) {
       file = `${file}?etag=${etag}`
     }
     return cloudfront.getPresignedUrl(file)
   } else {
-    return objectStore.getPresignedUrl(env.GLOBAL_BUCKET_NAME, file)
+    return await objectStore.getPresignedUrl(env.GLOBAL_BUCKET_NAME, file)
   }
 }
 

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -6,35 +6,37 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export function enrichPluginURLs(plugins?: Plugin[]): Plugin[] {
+export async function enrichPluginURLs(plugins?: Plugin[]): Promise<Plugin[]> {
   if (!plugins || !plugins.length) {
     return []
   }
-  return plugins.map(plugin => {
-    const jsUrl = getPluginJSUrl(plugin)
-    const iconUrl = getPluginIconUrl(plugin)
-    return { ...plugin, jsUrl, iconUrl }
-  })
+  return await Promise.all(
+    plugins.map(async plugin => {
+      const jsUrl = await getPluginJSUrl(plugin)
+      const iconUrl = await getPluginIconUrl(plugin)
+      return { ...plugin, jsUrl, iconUrl }
+    })
+  )
 }
 
-function getPluginJSUrl(plugin: Plugin) {
+async function getPluginJSUrl(plugin: Plugin) {
   const s3Key = getPluginJSKey(plugin)
   return getPluginUrl(s3Key)
 }
 
-function getPluginIconUrl(plugin: Plugin): string | undefined {
+async function getPluginIconUrl(plugin: Plugin) {
   const s3Key = getPluginIconKey(plugin)
   if (!s3Key) {
     return
   }
   return getPluginUrl(s3Key)
 }
 
-function getPluginUrl(s3Key: string) {
+async function getPluginUrl(s3Key: string) {
   if (env.CLOUDFRONT_CDN) {
     return cloudfront.getPresignedUrl(s3Key)
   } else {
-    return objectStore.getPresignedUrl(env.PLUGIN_BUCKET_NAME, s3Key)
+    return await objectStore.getPresignedUrl(env.PLUGIN_BUCKET_NAME, s3Key)
   }
 }
 

packages/backend-core/src/objectStore/buckets/tests/app.spec.ts

@@ -93,25 +93,25 @@ describe("app", () => {
         testEnv.multiTenant()
       })
 
-      it("gets url with embedded minio", () => {
+      it("gets url with embedded minio", async () => {
         testEnv.withMinio()
-        const url = getAppFileUrl()
+        const url = await getAppFileUrl()
         expect(url).toBe(
           "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
         )
       })
 
-      it("gets url with custom S3", () => {
+      it("gets url with custom S3", async () => {
         testEnv.withS3()
-        const url = getAppFileUrl()
+        const url = await getAppFileUrl()
         expect(url).toBe(
           "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
         )
       })
 
-      it("gets url with cloudfront + s3", () => {
+      it("gets url with cloudfront + s3", async () => {
         testEnv.withCloudfront()
-        const url = getAppFileUrl()
+        const url = await getAppFileUrl()
         // omit rest of signed params
         expect(
           url.includes("http://cf.example.com/app_123/attachments/image.jpeg?")
@@ -126,8 +126,8 @@ describe("app", () => {
 
       it("gets url with embedded minio", async () => {
         testEnv.withMinio()
-        await testEnv.withTenant(() => {
-          const url = getAppFileUrl()
+        await testEnv.withTenant(async () => {
+          const url = await getAppFileUrl()
           expect(url).toBe(
             "/files/signed/prod-budi-app-assets/app_123/attachments/image.jpeg"
           )
@@ -136,8 +136,8 @@ describe("app", () => {
 
       it("gets url with custom S3", async () => {
         testEnv.withS3()
-        await testEnv.withTenant(() => {
-          const url = getAppFileUrl()
+        await testEnv.withTenant(async () => {
+          const url = await getAppFileUrl()
           expect(url).toBe(
             "http://s3.example.com/prod-budi-app-assets/app_123/attachments/image.jpeg"
           )
@@ -146,8 +146,8 @@ describe("app", () => {
 
       it("gets url with cloudfront + s3", async () => {
         testEnv.withCloudfront()
-        await testEnv.withTenant(() => {
-          const url = getAppFileUrl()
+        await testEnv.withTenant(async () => {
+          const url = await getAppFileUrl()
           // omit rest of signed params
           expect(
             url.includes(