Matter 1.4

This includes application-level upgrades to Matter 1.4.  Upgrades the model, cluster definitions, endpoint types and behaviors.  Some protocol-level features are still TODO.

One notable change is that Basic Information "unique ID" is mandatory in 1.4.  We now generate one unconditionally unless provided as input.  This required tweaking current and deprecated codepaths and numerous tests but should be largely transparent to users.

Extends conformance and constraint parsers with new flags and syntax, plus a couple of bits that were used previously but we didn't support.  Some of these were complicated and consisted of ad-hoc syntax only used in one or two places...  But validation of the final model now sits at 100%.

Includes many upgrades to spec ingestion to handle special cases, mostly due to malformatting.  Replaces some hard-coded fixes with automatic detection/repair of errors that have become more prevalent as the spec grows.

Tweaks codegen for derived enums to generate union types that accept values from extended enums.  This appears to have been unnecessary because the specification was changed to include the redundant values, possibly because the equivalent fix was deemed infeasible in CHIP and/or alchemy.  However it does improve consistency so leaving in.

Includes a few minor CHIP test tweaks.  The new changes appear to have been transparent to CHIP but some test edits were no longer necessary and I fixed a few tests that had been broken previously.

Finally, includes a large chunk of code to handle ANSI text processing.  A bit of a rathole...  Came about because I wanted to make test error reporting more legible, which requires complex nested merging of styled text from multiple sources.  And I'd gotten tired of the mix of hand-coded styling and use of a 3rd-party library that has typing issues.  Over time we can migrate to this new code and drop the 3rd-party dependency, but will require some package reorganization because it currently lives in "tooling" which is not available everywhere.  So just approaching on an "as needed" basis for now.

chip-testing/src/AllClustersTestInstance.ts

@@ -399,7 +399,7 @@ export class AllClustersTestInstance extends NodeTestInstance {
                     levelValue: ConcentrationMeasurement.LevelValue.Critical,
                 },
                 colorControl: {
-                    coupleColorTempToLevelMinMireds: 0,
+                    coupleColorTempToLevelMinMireds: 1,
                     remainingTime: 0,
                     driftCompensation: 0,
                     compensationText: "foo",

chip-testing/test/core/BINFO.test.ts

@@ -5,22 +5,21 @@
  */
 
 import { OccurrenceManager } from "@matter/main/protocol";
-import { edit } from "@matter/testing";
 import { NodeTestInstance } from "../../src/NodeTestInstance.js";
 
 describe("BINFO", () => {
-    // TODO - remove when we're on 1.4
-    before(() =>
-        chip
-            .testFor("BINFO/2.1")
-            .edit(
-                edit.sed(
-                    "s/value: 18/value: 17/",
-                    "s/minValue: 0x01040000/minValue: 0x01030000/",
-                    "s/maxValue: 0x0104FF00/maxValue: 0x0103FF00/",
-                ),
-            ),
-    );
+    // When CHIP moves to a newer protocol that we don't support we can move CHIP's version back here
+    // before(() =>
+    //     chip
+    //         .testFor("BINFO/2.1")
+    //         .edit(
+    //             edit.sed(
+    //                 "s/value: 18/value: 17/",
+    //                 "s/minValue: 0x01040000/minValue: 0x01030000/",
+    //                 "s/maxValue: 0x0104FF00/maxValue: 0x0103FF00/",
+    //             ),
+    //         ),
+    // );
 
     chip("BINFO/*").exclude("BINFO/2.2");
 

chip-testing/test/core/BRBINFO.test.ts

@@ -4,6 +4,8 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
+import { BridgeApp } from "../support.js";
+
 describe("BRBINFO", () => {
-    chip("BRBINFO/*").exclude("BRBINFO/4.1"); // Exclude ICD; test doesn't specify PICS
+    chip("BRBINFO/*").subject(BridgeApp).exclude("BRBINFO/4.1"); // Exclude ICD; test doesn't specify PICS
 });

chip-testing/test/core/CADMIN.test.ts

@@ -22,7 +22,7 @@ describe("CADMIN", () => {
     //before(() => chip.testFor("CADMIN/1.19").edit(edit.sed("s/0x0000000B/0x00000587/")));
 
     // TODO - temporarily disabled other tests while working out 1.19 issues
-    //chip("CADMIN/*").exclude("CADMIN/1.19");
+    chip("CADMIN/*").exclude("CADMIN/1.19");
 
     // TODO - need to resolve issues w/ 1.19
     chip("CADMIN/1.19").beforeTest(subject => {

chip-testing/test/core/SC.test.ts

@@ -38,7 +38,7 @@ describe("SC", () => {
     });
 
     // Exclude 5.1 and 5.2 because our GroupKeyManagement is too limited
-    //chip("SC/*").exclude("SC/5.1", "SC/5.2", "SC/7.1");
+    chip("SC/*").exclude("SC/5.1", "SC/5.2", "SC/7.1");
 
     // 7.1 must start factory fresh
     chip("SC/7.1").uncommissioned();

chip/Dockerfile

@@ -162,3 +162,6 @@ RUN mkdir /run/dbus
 # Summarize tests for efficient metadata load at runtime
 COPY generate-test-descriptor /bin/generate-test-descriptor
 RUN generate-test-descriptor > /lib/test-descriptor.json
+
+# Include CHIP SHA for diagnostic purposes
+COPY --from=build /connectedhomeip/.git/refs/heads/master /etc/chip-version

codegen/src/clusters/GeneratorScope.ts

@@ -46,17 +46,19 @@ export interface GeneratorScope {
      *
      * @param model the model to name
      * @param tlv structs have both Name and TlvName defined; this selects the latter
+     * @param specific if true retrieves the name for this specific model rather than any canonical override
      */
-    nameFor(model: Model, tlv?: boolean): string;
+    nameFor(model: Model, tlv?: boolean, specific?: boolean): string;
 
     /**
      * Obtain the location of a model's definition.
      *
      * Throws an error if the model is not referenced in this scope.
      *
      * @param model the model whose location we load
+     * @param specific if true retrieves the name for this specific model rather than any canonical override
      */
-    locationOf(model: Model): GeneratorScope.Location;
+    locationOf(model: Model, specific?: boolean): GeneratorScope.Location;
 
     /**
      * Obtain the canonical model for a definition.
@@ -133,8 +135,8 @@ function allocateScope(scope: Scope): GeneratorScope {
         owner,
         scope,
 
-        nameFor(model: Model, tlv: boolean) {
-            const { definition: definer } = this.locationOf(model);
+        nameFor(model: Model, tlv: boolean, specific = false) {
+            const { definition: definer } = this.locationOf(model, specific);
             const name = names.get(definer);
             if (name === undefined) {
                 throw new InternalError(`No name assigned to ${model} in scope ${owner}`);
@@ -145,8 +147,11 @@ function allocateScope(scope: Scope): GeneratorScope {
             return name;
         },
 
-        locationOf(model: Model) {
-            const location = locations.get(scope.modelFor(model));
+        locationOf(model: Model, specific = false) {
+            if (!specific) {
+                model = scope.modelFor(model);
+            }
+            const location = locations.get(model);
             if (location === undefined) {
                 throw new InternalError(`No location identified for ${model} from scope ${owner}`);
             }
@@ -242,7 +247,7 @@ function identifyNamedModels(scope: Scope): Locations {
     /**
      * Set the location for a model and, if the model is not local, its scope
      */
-    function define(model: Model) {
+    function define(model: Model, specific = false) {
         if (!(model instanceof ValueModel)) {
             return;
         }
@@ -273,14 +278,16 @@ function identifyNamedModels(scope: Scope): Locations {
             return;
         }
 
-        const extension = scope.extensionOf(definer);
-        if (extension === model) {
-            // The model is the definer even if it doesn't provide fields of its own
-            definer = model;
-        } else if (extension) {
-            // Only the extension appears in the file
-            define(extension);
-            return;
+        if (!specific) {
+            const extension = scope.extensionOf(definer);
+            if (extension === model) {
+                // The model is the definer even if it doesn't provide fields of its own
+                definer = model;
+            } else if (extension) {
+                // Only the extension appears in the file
+                define(extension);
+                return;
+            }
         }
 
         if (locations.has(definer)) {
@@ -303,12 +310,12 @@ function identifyNamedModels(scope: Scope): Locations {
         locations.set(definer, location);
 
         // The model is not defined locally but we need to import any subtree that references shadowed elements.
-        if (definer instanceof ValueModel && referencesShadows(definer)) {
+        if (definer instanceof ValueModel && !specific && referencesShadows(definer)) {
             location.isLocal = true;
             modelScope = owner;
 
             // Now localize any sub-references
-            definer.members.forEach(define);
+            definer.members.forEach(m => define(m));
         }
 
         if (modelScope instanceof ClusterModel && modelScope !== owner) {
@@ -319,6 +326,14 @@ function identifyNamedModels(scope: Scope): Locations {
                 isGlobal: true,
             });
         }
+
+        // For enums any base that contributes values must be defined as well as it will be referenced in the type
+        if (metatype === Metatype.enum) {
+            const { base } = definer;
+            if (base && base.metatype === undefined) {
+                define(base, true);
+            }
+        }
     }
 
     /**

codegen/src/clusters/TlvGenerator.ts

@@ -155,10 +155,19 @@ export class TlvGenerator {
 
             case Metatype.enum:
                 {
-                    const dt = this.defineDatatype(model);
-                    if (dt) {
+                    const dts = new Set<string>();
+                    for (let m = this.#scope.definingModelFor(model); m; m = m.base) {
+                        if (!m.children.length) {
+                            continue;
+                        }
+                        const dt = this.#defineSpecificDatatype(m);
+                        if (dt !== undefined) {
+                            dts.add(dt);
+                        }
+                    }
+                    if (dts.size) {
                         this.importTlv("number", "TlvEnum");
-                        tlv = `TlvEnum<${dt}>()`;
+                        tlv = `TlvEnum<${[...dts].join(" | ")}>()`;
                     } else {
                         // No fields; revert to the primitive type the enum derives from
                         tlv = this.#primitiveTlv(metabase, model);
@@ -260,7 +269,7 @@ export class TlvGenerator {
         const enumBlock = this.definitions.expressions(`export enum ${name} {`, "}");
 
         this.definitions.insertingBefore(enumBlock, () => {
-            model.members.forEach(child => {
+            model.children.forEach(child => {
                 let name = child.name;
                 if (name.match(/^\d+$/)) {
                     // Typescript doesn't allow numeric enum keys
@@ -352,14 +361,17 @@ export class TlvGenerator {
     defineDatatype(model: ValueModel) {
         // Obtain the defining model.  This is the actual datatype definition
         const defining = this.#scope.definingModelFor(model);
-        if (defining) {
-            model = defining;
-        } else {
+
+        if (!defining) {
             // If there's no defining model, the datatype is empty.  Use either the base or the model directly for
             // naming.  Handling of this is context specific
             return;
         }
 
+        return this.#defineSpecificDatatype(defining);
+    }
+
+    #defineSpecificDatatype(model: ValueModel) {
         // Special case for status codes.  "Status code" in door lock cluster seems to be the global status codes
         // instead of the local one.  So always reference the global one until we see something different
         if (model.isGlobal && model.name === status.name && this.file.scope.owner !== model) {
@@ -375,9 +387,9 @@ export class TlvGenerator {
 
         // If the model is not local, import rather than define
         const isObject = model.effectiveMetatype === Metatype.object;
-        const location = this.#scope.locationOf(model);
+        const location = this.#scope.locationOf(model, true);
         if (!location.isLocal) {
-            return this.#file.reference(model, isObject);
+            return this.#file.reference(model, isObject, true);
         }
 
         // If the type is already defined we reference the existing definition

codegen/src/mom/spec/header-detection-heuristics.ts

@@ -0,0 +1,19 @@
+/**
+ * @license
+ * Copyright 2022-2024 Matter.js Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export function looksLikeField(text?: string) {
+    return !!text?.match(/^[a-z0-9]+(?: Field| Value)$/i);
+}
+
+export function looksLikeDatatype(text?: string) {
+    return !!(
+        text?.match(/^[a-z0-9]+(?:Enum|Struct| Attribute| Command| Event| Type| Field| Value| Bits?)$/i) ||
+        // This is a real winner.  Joint Fabric Datastore cluster in 1.4 manages to follow no convention for indicating
+        // these are structs *and* innovatively sticks spaces in type names.  We don't add to the pattern above because
+        // the spaces make this pretty aggressive so we execute with case sensitivity
+        text?.match(/^(?:[A-Z][?:a-zA-Z0-9]* )+Entry Type$/)
+    );
+}