first steps towards data-plane-gateway deprecation

crates/agent/src/api/create_data_plane.rs

@@ -147,9 +147,10 @@ async fn do_create_data_plane(
             ops_l2_stats_transform,
             broker_address,
             reactor_address,
-            hmac_keys
+            hmac_keys,
+            enable_l2
         ) values (
-            $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11
+            $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12
         )
         on conflict (data_plane_name) do update set
             broker_address = $9,
@@ -171,6 +172,7 @@ async fn do_create_data_plane(
         broker_address,
         reactor_address,
         hmac_keys.as_slice(),
+        !hmac_keys.is_empty(), // Enable L2 if HMAC keys are defined at creation.
     )
     .fetch_one(pg_pool)
     .await?;

crates/agent/src/api/update_l2_reporting.rs

@@ -108,11 +108,9 @@ export class Derivation extends Types.IDerivation {"#
             ops_l1_inferred_name  as "ops_l1_inferred_name: models::Collection",
             ops_l2_inferred_transform,
             ops_l1_stats_name     as "ops_l1_stats_name:    models::Collection",
-            ops_l2_stats_transform
+            ops_l2_stats_transform,
+            enable_l2
         from data_planes
-        -- Data-planes without configured HMAC keys are presumed to not be ready,
-        -- and we hold back from processing their L1 derivations.
-        where hmac_keys != '{}'
         order by data_plane_name asc;
         "#,
     )
@@ -123,6 +121,7 @@ export class Derivation extends Types.IDerivation {"#
         l2_inferred_transforms.push(models::TransformDef {
             name: models::Transform::new(&data_plane.ops_l2_inferred_transform),
             source: models::Source::Collection(data_plane.ops_l1_inferred_name.clone()),
+            disable: !data_plane.enable_l2,
 
             shuffle: models::Shuffle::Key(models::CompositeKey::new([models::JsonPointer::new(
                 "/collection_name",
@@ -132,23 +131,25 @@ export class Derivation extends Types.IDerivation {"#
             )),
 
             backfill: 0,
-            disable: false,
             priority: 0,
             read_delay: None,
         });
 
         l2_stats_transforms.push(models::TransformDef {
             name: models::Transform::new(&data_plane.ops_l2_stats_transform),
             source: models::Source::Collection(data_plane.ops_l1_stats_name.clone()),
+            disable: !data_plane.enable_l2,
 
             backfill: 0,
-            disable: false,
             lambda: models::RawValue::default(),
             priority: 0,
             read_delay: None,
             shuffle: models::Shuffle::Any,
         });
 
+        if !data_plane.enable_l2 {
+            l2_stats_module.push_str("\n/*");
+        }
         l2_stats_module.push_str(&format!(
             r#"
     {method_name}(read: {{ doc: Types.{type_name}}}): Types.Document[] {{
@@ -160,6 +161,9 @@ export class Derivation extends Types.IDerivation {"#
                 camel_case(&data_plane.ops_l2_stats_transform, true)
             )
         ));
+        if !data_plane.enable_l2 {
+            l2_stats_module.push_str("\n*/");
+        }
     }
 
     l2_stats_module.push_str("\n}\n");

crates/agent/src/integration_tests/harness.rs

@@ -181,7 +181,8 @@ impl TestHarness {
                     ops_l2_stats_transform,
                     broker_address,
                     reactor_address,
-                    hmac_keys
+                    hmac_keys,
+                    enable_l2
                 ) values (
                     'ops/dp/public/test',
                     'test.dp.estuary-data.com',
@@ -193,7 +194,8 @@ impl TestHarness {
                     'from-L1-stats',
                     'broker:address',
                     'reactor:address',
-                    '{secret-key}'
+                    '{secret-key}',
+                    false
                 ) on conflict do nothing
             )
             select 1 as "something: bool";

supabase/migrations/63_new_data_plane_columns.sql

@@ -0,0 +1,19 @@
+begin;
+
+alter table data_planes add column aws_iam_user_arn text;
+alter table data_planes add column cidr_blocks cidr[] not null default '{}';
+alter table data_planes add column enable_l2 boolean not null default false;
+alter table data_planes add column gcp_service_account_email text;
+alter table data_planes add column ssh_private_key text;
+
+-- Must be provided explicitly.
+alter table data_planes alter column enable_l2 drop default;
+
+-- Users may read out details of applied data-plane configuration.
+grant select (
+    aws_iam_user_arn,
+    cidr_blocks,
+    gcp_service_account_email
+) on data_planes to authenticated;
+
+commit;