Form service backend implemented (#297)

* Refactor form service to use createService utility

* Use injected db gateway in form service functions, without changes to the existing localStorage approach.

* Add form db gateway functions

* Wire form db gateways to application.

* Wire form service to frontend via a client-side proxy and API routes

* Fix a couple build issues

* Move db gateway functions into the service that consumes it.
There are failing tests, due to describe blocks defined in the shared `describeDatabase` helper not getting registered with Vitest. Pushing up to CI to run test there, and rule out a local config issue.

* Get describeDatabase working in the auth and forms packages; mark integration tests of demo apps as "expect failure" due to dependency issues that need to be worked out.

* Temp: deploy on this branch

* Update lockfile

* Always use ssl with demo apps

* Trying to isolate db dependencies; get CLI working with NodeNext modules

* Tweak imports for NodeNext

* Testing

* Builds working with refactored database structure and module settings.
Includes an upgrade of Turborepo and new clean tasks.

* Remove pnpm version in Github Actions script, in lieu of the packageManager key in package.json

* Switch design package to NodeNext module resolution

* Add tests for all the forms services (todo: submitForm).
This necessitated giving the forms library the Rollup treatment, and moving some dependencies around to avoid intermixing nodejs deps with the design lib.

* use createTestFormServiceContext in submitForm test

* Fix build break in spotlight app

* Use NodeNext in the base tsconfig.json

* docassemble updated for nodenext

* Wire existing data flow to submitForm. to be updated for server environment

* Do one more test deploy

* Remove test deploy

.github/workflows/_terraform-apply.yml

@@ -31,7 +31,6 @@ jobs:
         uses: pnpm/action-setup@v4
         id: pnpm-install
         with:
-          version: 8
           run_install: false
 
       - name: Get pnpm store directory

.github/workflows/_terraform-plan-pr-comment.yml

@@ -33,7 +33,6 @@ jobs:
         uses: pnpm/action-setup@v4
         id: pnpm-install
         with:
-          version: 8
           run_install: false
 
       - name: Get pnpm store directory

.github/workflows/_validate.yml

@@ -25,7 +25,6 @@ jobs:
         uses: pnpm/action-setup@v4
         id: pnpm-install
         with:
-          version: 9
           run_install: false
 
       - name: Get pnpm store directory

apps/cli/package.json

@@ -2,11 +2,12 @@
   "name": "@atj/cli-app",
   "version": "1.0.0",
   "description": "10x ATJ command-line interface",
-  "type": "commonjs",
+  "type": "module",
   "license": "CC0",
   "main": "src/index.ts",
   "scripts": {
-    "build": "tsup src/* --format cjs",
+    "build": "tsup src/* --format esm",
+    "clean": "rimraf dist tsconfig.tsbuildinfo coverage",
     "cli": "node dist/index.js",
     "dev": "tsup src/* --watch",
     "test": "vitest run --coverage"

apps/cli/src/cli-controller/cli-controller.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it, vi } from 'vitest';
 import { mock } from 'vitest-mock-extended';
 
-import { CliController } from '.';
+import { CliController } from './index.js';
 
 describe('cli controller', () => {
   it('works', async () => {

apps/cli/src/cli-controller/index.ts

@@ -1,8 +1,8 @@
 import { Command } from 'commander';
 
 import { createDependencyGraph } from '@atj/dependency-graph';
-import type { Context } from './types';
-import { addSecretCommands } from './secrets';
+import type { Context } from './types.js';
+import { addSecretCommands } from './secrets.js';
 
 export const CliController = (ctx: Context) => {
   const cli = new Command().description(

apps/cli/src/cli-controller/secrets.ts

@@ -1,9 +1,8 @@
+import path from 'path';
 import { Command } from 'commander';
 
-import { commands, getSecretsVault } from '@atj/infra-core';
-import { Context } from './types';
-import { type DeployEnv } from '@atj/infra-core/src/values';
-import path from 'path';
+import { type DeployEnv, commands, getSecretsVault } from '@atj/infra-core';
+import { type Context } from './types.js';
 
 export const addSecretCommands = (ctx: Context, cli: Command) => {
   const cmd = cli

apps/cli/src/index.ts

@@ -1,5 +1,5 @@
-const { join } = require('path');
-const { CliController } = require('./cli-controller');
+import { join } from 'path';
+import { CliController } from './cli-controller/index.js';
 
 // This should map to the directory containing the package.json.
 // By convention, assume that the originating process was run from the root

apps/cli/tsconfig.json

@@ -1,7 +1,8 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "module": "ESNext",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
     "outDir": "./dist",
     "emitDeclarationOnly": true
   },

apps/rest-api/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "build": "esbuild src/index.ts --bundle --minify --sourcemap --platform=node --target=es2020 --outfile=dist/index.js",
     "build:client": "tsup src/* --env.NODE_ENV production --dts-resolve",
-    "clean": "rm -rf dist tsconfig.tsbuildinfo",
+    "clean": "rimraf dist tsconfig.tsbuildinfo",
     "dev": "tsup src/* --watch"
   },
   "dependencies": {

apps/server-doj/package.json

@@ -6,9 +6,10 @@
   "license": "CC0",
   "main": "src/index.ts",
   "scripts": {
-    "start": "node dist/index.js",
     "build": "tsup src/* --format esm",
+    "clean": "rimraf dist tsconfig.tsbuildinfo coverage",
     "dev": "tsup src/* --watch  --format esm",
+    "start": "VCAP_SERVICES='{\"aws-rds\": [{ \"credentials\": { \"uri\": \"\" }}]}' node dist/index.js",
     "test": "vitest run --coverage"
   },
   "dependencies": {

apps/server-doj/src/index.ts

@@ -1,3 +1,4 @@
+import { createPostgresDatabaseContext } from '@atj/database/context';
 import { createCustomServer } from './server.js';
 
 const port = process.env.PORT || 4321;
@@ -14,8 +15,8 @@ const getCloudGovServerSecrets = () => {
 };
 
 const secrets = getCloudGovServerSecrets();
-createCustomServer({ dbUri: secrets?.dbUri }).then((server: any) =>
-  server.listen(port, () => {
-    console.log(`Server running on http://localhost:${port}`);
-  })
-);
+const db = await createPostgresDatabaseContext(secrets.dbUri, true);
+const server = await createCustomServer(db);
+server.listen(port, () => {
+  console.log(`Server running on http://localhost:${port}`);
+});

apps/server-doj/src/server.ts

@@ -1,16 +1,7 @@
-import {
-  createDatabaseGateway,
-  createPostgresDatabaseContext,
-} from '@atj/database';
+import { type DatabaseContext } from '@atj/database';
 import { createServer } from '@atj/server';
 
-export const createCustomServer = async (ctx: {
-  dbUri: string;
-}): Promise<any> => {
-  const db = createDatabaseGateway(
-    await createPostgresDatabaseContext(ctx.dbUri, true)
-  );
-
+export const createCustomServer = async (db: DatabaseContext): Promise<any> => {
   return createServer({
     title: 'DOJ Form Service',
     db,

apps/server-doj/tests/integration.test.ts

@@ -1,18 +1,14 @@
 import request from 'supertest';
 import { describe, expect, test } from 'vitest';
-import { describeDatabase } from '@atj/database/testing';
+
+import { createInMemoryDatabaseContext } from '@atj/database/context';
 
 import { createCustomServer } from '../src/server';
 
 describe('DOJ Form Service', () => {
-  test('avoid "No test suite found in file" error', async () => {
-    expect(true).toBe(true);
-  });
-});
-
-describeDatabase('DOJ Form Service', () => {
-  test('renders the home page', async ({ db }) => {
-    const app = await createCustomServer({ dbUri: db.ctx.connectionUri });
+  test('renders the home page', async () => {
+    const db = await createInMemoryDatabaseContext();
+    const app = await createCustomServer(db);
     const response = await request(app).get('/');
     expect(response.ok).toBe(true);
     expect(response.text).toMatch(/DOJ Form Service/);

apps/server-doj/tsconfig.json

@@ -1,7 +1,7 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "module": "CommonJS",
+    "module": "NodeNext",
     "moduleResolution": "NodeNext",
     "outDir": "./dist",
     "emitDeclarationOnly": true

apps/server-doj/vitest.config.ts

@@ -1,12 +1,10 @@
 import { defineConfig } from 'vitest/config';
-import { getDatabaseTestContainerGlobalSetupPath } from '@atj/database';
 
 import sharedTestConfig from '../../vitest.shared';
 
 export default defineConfig({
   ...sharedTestConfig,
   test: {
     ...sharedTestConfig.test,
-    globalSetup: [getDatabaseTestContainerGlobalSetupPath()],
   },
 });

apps/server-kansas/package.json

@@ -6,9 +6,10 @@
   "license": "CC0",
   "main": "src/index.ts",
   "scripts": {
-    "start": "node dist/index.js",
     "build": "tsup src/* --format esm",
+    "clean": "rimraf dist tsconfig.tsbuildinfo coverage",
     "dev": "tsup src/* --watch  --format esm",
+    "start": "VCAP_SERVICES='{\"aws-rds\": [{ \"credentials\": { \"uri\": \"\" }}]}' node dist/index.js",
     "test": "vitest run --coverage"
   },
   "dependencies": {

apps/server-kansas/src/index.ts

@@ -1,3 +1,4 @@
+import { createPostgresDatabaseContext } from '@atj/database/context';
 import { createCustomServer } from './server.js';
 
 const port = process.env.PORT || 4321;
@@ -14,8 +15,8 @@ const getCloudGovServerSecrets = () => {
 };
 
 const secrets = getCloudGovServerSecrets();
-createCustomServer({ dbUri: secrets?.dbUri }).then((server: any) =>
-  server.listen(port, () => {
-    console.log(`Server running on http://localhost:${port}`);
-  })
-);
+const db = await createPostgresDatabaseContext(secrets.dbUri, true);
+const server = await createCustomServer(db);
+server.listen(port, () => {
+  console.log(`Server running on http://localhost:${port}`);
+});

apps/server-kansas/src/server.ts

@@ -1,16 +1,7 @@
-import {
-  createDatabaseGateway,
-  createPostgresDatabaseContext,
-} from '@atj/database';
+import { type DatabaseContext } from '@atj/database';
 import { createServer } from '@atj/server';
 
-export const createCustomServer = async (ctx: {
-  dbUri: string;
-}): Promise<any> => {
-  const db = createDatabaseGateway(
-    await createPostgresDatabaseContext(ctx.dbUri, true)
-  );
-
+export const createCustomServer = async (db: DatabaseContext): Promise<any> => {
   return createServer({
     title: 'DOJ Form Service',
     db,
@@ -22,6 +13,7 @@ export const createCustomServer = async (ctx: {
     },
     isUserAuthorized: async (email: string) => {
       return [
+        // 10x team members
         '[email protected]',
         '[email protected]',
         '[email protected]',

apps/server-kansas/tests/integration.test.ts

@@ -1,18 +1,13 @@
 import request from 'supertest';
 import { describe, expect, test } from 'vitest';
-import { describeDatabase } from '@atj/database/testing';
 
+import { createInMemoryDatabaseContext } from '@atj/database/context';
 import { createCustomServer } from '../src/server';
 
-describe('DOJ Form Service', () => {
-  test('avoid "No test suite found in file" error', async () => {
-    expect(true).toBe(true);
-  });
-});
-
-describeDatabase('Kansas State Courts Form Service', () => {
-  test('renders the home page', async ({ db }) => {
-    const app = await createCustomServer({ dbUri: db.ctx.connectionUri });
+describe('Kansas State Courts Form Service', () => {
+  test('renders the home page', async () => {
+    const db = await createInMemoryDatabaseContext();
+    const app = await createCustomServer(db);
     const response = await request(app).get('/');
     expect(response.ok).toBe(true);
     expect(response.text).toMatch(/DOJ Form Service/);

apps/server-kansas/tsconfig.json

@@ -1,7 +1,8 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "module": "ESNext",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
     "emitDeclarationOnly": true,
     "outDir": "./dist"
   },

apps/spotlight/package.json

@@ -5,7 +5,7 @@
   "scripts": {
     "astro": "astro",
     "build": "astro build",
-    "clean": "rm -rf dist",
+    "clean": "rimraf dist tsconfig.tsbuildinfo coverage",
     "dev": "astro dev",
     "preview": "astro preview"
   },

apps/spotlight/src/context.ts

@@ -1,14 +1,19 @@
-import { FormConfig } from '@atj/forms';
+import {
+  type FormConfig,
+  type FormService,
+  createFormService,
+} from '@atj/forms';
 import { defaultFormConfig } from '@atj/forms';
-import { service } from '@atj/forms';
+import { BrowserFormRepository } from '@atj/forms/context';
 
 import { type GithubRepository } from './lib/github';
+import { createTestBrowserFormService } from '@atj/forms/context';
 
 export type AppContext = {
   baseUrl: `${string}/`;
   github: GithubRepository;
   formConfig: FormConfig;
-  formService: service.FormService;
+  formService: FormService;
   uswdsRoot: `${string}/`;
 };
 
@@ -33,8 +38,13 @@ const createAppContext = (env: any): AppContext => {
 
 const createAppFormService = () => {
   if (globalThis.window) {
-    return service.createBrowserFormService();
+    const repository = new BrowserFormRepository(window.localStorage);
+    return createFormService({
+      repository,
+      config: defaultFormConfig,
+      isUserLoggedIn: () => true,
+    });
   } else {
-    return service.createTestFormService();
+    return createTestBrowserFormService();
   }
 };

apps/spotlight/tsconfig.json

@@ -3,7 +3,8 @@
   // https://github.com/withastro/astro/blob/main/packages/astro/tsconfigs/base.json
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "module": "ESNext",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
     "jsx": "react",
     "resolveJsonModule": true
   },

infra/cdktf/package.json

@@ -10,7 +10,8 @@
     "build:synth:main": "DEPLOY_ENV=main cdktf synth",
     "build:synth:staging": "DEPLOY_ENV=staging cdktf synth",
     "build:tsc": "tsc --pretty",
-    "clean": "rm -rf .gen cdktf.out",
+    "clean": "rimraf cdktf.out dist tsconfig.tsbuildinfo",
+    "clean:gen": "rimraf .gen",
     "deploy:main": "DEPLOY_ENV=main cdktf deploy",
     "deploy:staging": "DEPLOY_ENV=staging cdktf deploy",
     "dev": "tsc -w",

infra/core/package.json

@@ -8,6 +8,7 @@
   "types": "dist/index.d.js",
   "scripts": {
     "build": "tsc",
+    "clean": "rimraf dist tsconfig.tsbuildinfo coverage",
     "dev": "tsc --watch",
     "test": "vitest run --coverage"
   },

infra/core/src/commands/delete-secret.test.ts

@@ -1,8 +1,7 @@
 import { describe, expect, it } from 'vitest';
 
-import { createInMemorySecretsVault } from '../lib';
-import { getSecretKeyList } from './get-secret-key-list';
-import { deleteSecret } from './delete-secret';
+import { createInMemorySecretsVault } from '../lib/index.js';
+import { deleteSecret } from './delete-secret.js';
 
 const getTestVault = (vaultData: any) => {
   const result = createInMemorySecretsVault(JSON.stringify(vaultData));

infra/core/src/commands/delete-secret.ts

@@ -1,4 +1,4 @@
-import type { SecretKey, SecretsVault } from '../lib/types';
+import type { SecretKey, SecretsVault } from '../lib/types.js';
 
 export const deleteSecret = async (vault: SecretsVault, key: SecretKey) => {
   return await vault.deleteSecret(key);

infra/core/src/commands/get-secret-key-list.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from 'vitest';
 
-import { createInMemorySecretsVault } from '../lib';
-import { getSecretKeyList } from './get-secret-key-list';
+import { createInMemorySecretsVault } from '../lib/index.js';
+import { getSecretKeyList } from './get-secret-key-list.js';
 
 const getTestVault = (vaultData: any) => {
   const result = createInMemorySecretsVault(JSON.stringify(vaultData));

infra/core/src/commands/get-secret-key-list.ts

@@ -1,4 +1,4 @@
-import { type SecretsVault } from '../lib/types';
+import { type SecretsVault } from '../lib/types.js';
 
 export const getSecretKeyList = async (vault: SecretsVault) => {
   return await vault.getSecretKeys();