Fio-8316 invalid data submitted in nested form

.gitignore

@@ -1,6 +1,10 @@
+.DS_Store
 .vscode
 node_modules
 docs
 lib
 dist
 .dccache
+# ignore nyc output
+.nyc_output
+coverage

package.json

@@ -13,7 +13,7 @@
     "./dist/formio.core.min.js": "./dist/formio.core.min.js"
   },
   "scripts": {
-    "test": "TEST=1 mocha -r ts-node/register -r tsconfig-paths/register -r mock-local-storage -r jsdom-global/register -b -t 0 'src/**/__tests__/*.test.ts'",
+    "test": "TEST=1 nyc --reporter=lcov --reporter=text --reporter=text-summary mocha -r ts-node/register -r tsconfig-paths/register -r mock-local-storage -r jsdom-global/register -t 0 'src/**/__tests__/*.test.ts'",
     "lib": "tsc --project tsconfig.json && tsc-alias -p tsconfig.json",
     "replace": "node -r tsconfig-paths/register -r ts-node/register ./lib/base/array/ArrayComponent.js",
     "test:debug": "mocha -r ts-node/register -r tsconfig-paths/register -r mock-local-storage -r jsdom-global/register --debug-brk --inspect '**/*.spec.ts'",
@@ -72,6 +72,7 @@
     "mocha": "^10.0.0",
     "mocha-jsdom": "^2.0.0",
     "mock-local-storage": "^1.1.20",
+    "nyc": "^15.1.0",
     "power-assert": "^1.6.1",
     "sinon": "^17.0.1",
     "ts-loader": "^9.5.0",
@@ -97,5 +98,23 @@
     "json-logic-js": "^2.0.2",
     "lodash": "^4.17.21",
     "moment": "^2.29.4"
+  },
+  "nyc": {
+    "check-coverage": true,
+    "statements": 62,
+    "branches": 52,
+    "functions": 57,
+    "lines": 60,
+    "include": [
+      "src/**/*.ts",
+      "src/**/*.js"
+    ],
+    "exclude": [
+      "src/**/*.test.ts",
+      "src/**/__tests__/",
+      "src/experimental/**/*.ts",
+      "src/types/**/*.ts"
+    ],
+    "all": true
   }
-}
+}

src/process/__tests__/process.test.ts

@@ -1050,9 +1050,78 @@ describe('Process Tests', () => {
         submission.data = context.data;
         context.processors = ProcessTargets.evaluator;
         processSync(context);
+        console.log('errors:', context.scope.errors);
+
         assert.equal(context.scope.errors.length, 0);
     });
+    it('should remove submission data not in a nested form definition', async function () {
+        const form = {
+            _id: {},
+            title: 'parent',
+            name: 'parent',
+            type: 'form',
+            components: [
+                {
+                    type: "checkbox",
+                    label: "A",
+                    key: "A",
+                    input: true,
+                },
+                {
+                    type: "checkbox",
+                    label: "B",
+                    key: "B",
+                    input: true,
+                },
+                {
+                    key: 'child',
+                    label: 'child',
+                    form: 'child form',
+                    type: 'form',
+                    input: true,
+                    reference: false,
+                    clearOnHide: false,
+                    components: [
+                        {
+                            label: "Input",
+                            key: 'input',
+                            type: 'textfield',
+                            input: true
+                        }
+                    ]
+                }
+            ]
+        }
+        const submission = {
+            data: {
+                A: true,
+                B: true,
+                child: {
+                    data: {
+                        input: 'test',
+                        invalid: 'invalid submission data'
+                    }
+                }
+            }
+        }
+        const context = {
+            form,
+            submission,
+            data: submission.data,
+            components: form.components,
+            processors: ProcessTargets.submission,
+            scope: {},
+            config: {
+                server: true
+            }
+        };
+        processSync(context);
+
 
+        expect(context.data.child.data).to.not.have.property('invalid');
+
+
+    })
     it('Should process nested form data correctly', async () => {
         const submission = {
             data: {

src/process/filter/index.ts

@@ -15,7 +15,7 @@ export const filterProcessSync: ProcessorFnSync<FilterScope> = (context: FilterC
                 scope.filter[absolutePath] = {
                     compModelType: modelType,
                     include: true,
-                    value: {data: {}}
+                    value: { data: {} }
                 };
                 break;
             case 'array':
@@ -47,19 +47,28 @@ export const filterProcess: ProcessorFn<FilterScope> = async (context: FilterCon
 };
 
 export const filterPostProcess: ProcessorFnSync<FilterScope> = (context: FilterContext) => {
+
     const { scope, submission } = context;
+    console.log('filterPostProcess: context', Object.keys(context));
+    console.dir(context.components)
+
     const filtered = {};
     for (const path in scope.filter) {
-        if (scope.filter[path].include) {
-            let value = get(submission?.data, path);
-            if (isObject(value) && isObject(scope.filter[path].value)) {
-                if (scope.filter[path].compModelType === 'dataObject') {
-                    value = {...value, ...scope.filter[path].value, data: (value as any)?.data}
-                } else {
-                    value = {...value, ...scope.filter[path].value}
-                }
+        console.log('filterPostProcess path:', path);
+        const pathFilter = scope.filter[path];
+        if (pathFilter) {
+
+            let value = get(submission?.data, path) as any;
+            console.log('path filter:', scope.filter[path]);
+            console.log('path value:', value);
+
+            // dataObject types (nested form) paths are recursively added to the scope.filter
+            // excluding those from this and setting their components onto the submission directly
+            if (pathFilter.compModelType !== 'dataObject') {
+                set(filtered, path, value);
+            } else {
+                console.log('skipping path:', path, 'value:', value)
             }
-            set(filtered, path, value);
         }
     }
     context.data = filtered;

src/process/process.ts

@@ -16,20 +16,24 @@ import { clearHiddenProcessInfo } from "./clearHidden";
 
 export async function process<ProcessScope>(context: ProcessContext<ProcessScope>): Promise<ProcessScope> {
     const { instances, components, data, scope, flat, processors } = context;
+
+
     await eachComponentDataAsync(components, data, async (component, compData, row, path, components, index) => {
         // Skip processing if row is null or undefined
         if (!row) {
             return;
         }
-        await processOne<ProcessScope>({...context, ...{
-            data: compData,
-            component,
-            components,
-            path,
-            row,
-            index,
-            instance: instances ? instances[path] : undefined
-        }});
+        await processOne<ProcessScope>({
+            ...context, ...{
+                data: compData,
+                component,
+                components,
+                path,
+                row,
+                index,
+                instance: instances ? instances[path] : undefined
+            }
+        });
         if (flat) {
             return true;
         }
@@ -49,12 +53,14 @@ export async function process<ProcessScope>(context: ProcessContext<ProcessScope
 
 export function processSync<ProcessScope>(context: ProcessContext<ProcessScope>): ProcessScope {
     const { instances, components, data, scope, flat, processors } = context;
+
     eachComponentData(components, data, (component, compData, row, path, components, index) => {
         // Skip processing if row is null or undefined
         if (!row) {
             return;
         }
-        processOneSync<ProcessScope>({...context,
+        processOneSync<ProcessScope>({
+            ...context,
             data: compData,
             component,
             components,