go.mod/Tiltfile: update for grpc-web and connector networking

Bring in Gazette updates for grpc-web gateways and net.Listener customization. Deeply rework Tiltfile to remove data-plane-gateway, by creating a self-signed TLS CA and Certificate that are used by the broker and reactor (and may be used by other services if desired). Use a naming strategy of `thing.flow.localhost`, because many libraries like rustls don't accept wildcard certs of a toplevel `*.localhost` but will happily accept `*.flow.localhost`. Update for changes to --broker.allow-origin, --consumer.allow-origin, and --flow.dashboard, as well as extracting explicit arguments into environment variables.
estuary · Sep 20, 2024 · 0034fa3 · 0034fa3
1 parent bd26e42
commit 0034fa3
Show file tree

Hide file tree

Showing 4 changed files with 154 additions and 90 deletions.
diff --git a/Tiltfile b/Tiltfile
@@ -1,38 +1,79 @@
 # This file is interpreted by `tilt`, and describes how to get a local flow environment running.
-DATABASE_URL="postgresql://postgres:postgres@localhost:5432/postgres"
-os.putenv("DATABASE_URL", DATABASE_URL)
-os.putenv("RUST_LOG", "info")
-os.putenv("DOCKER_DEFAULT_PLATFORM", "linux/amd64")
+DATABASE_URL="postgresql://postgres:[email protected]:5432/postgres"
 
 # Secret used to sign Authorizations within a local data plane, as base64("supersecret").
 # Also allow requests without an Authorization (to not break data-plane-gateway just yet).
 AUTH_KEYS="c3VwZXJzZWNyZXQ=,AA=="
-os.putenv("CONSUMER_AUTH_KEYS", AUTH_KEYS)
-os.putenv("BROKER_AUTH_KEYS", AUTH_KEYS)
-
 
 REPO_BASE= '%s/..' % os.getcwd()
 TEST_KMS_KEY="projects/helpful-kingdom-273219/locations/us-central1/keyRings/dev/cryptoKeys/testing"
 
-HOME_DIR=os.getenv("HOME")
-FLOW_DIR=os.getenv("FLOW_DIR", os.path.join(HOME_DIR, "flow-local"))
+FLOW_DIR=os.getenv("FLOW_DIR", os.path.join(os.getenv("HOME"), "flow-local"))
 ETCD_DATA_DIR=os.path.join(FLOW_DIR, "etcd")
 
 FLOW_BUILDS_ROOT="file://"+os.path.join(FLOW_DIR, "builds")+"/"
-# Or alternatively, use an actual bucket when testing with external data-planes:
-# FLOW_BUILDS_ROOT="gs://example/builds/"
 
 # A token for the local-stack system user signed against the local-stack
 # supabase secret (super-secret-jwt-token-with-at-least-32-characters-long).
 SYSTEM_USER_TOKEN="eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJodHRwOi8vMTI3LjAuMC4xOjU0MzEvYXV0aC92MSIsInN1YiI6ImZmZmZmZmZmLWZmZmYtZmZmZi1mZmZmLWZmZmZmZmZmZmZmZiIsImF1ZCI6ImF1dGhlbnRpY2F0ZWQiLCJleHAiOjI3MDAwMDAwMDAsImlhdCI6MTcwMDAwMDAwMCwiZW1haWwiOiJzdXBwb3J0QGVzdHVhcnkuZGV2Iiwicm9sZSI6ImF1dGhlbnRpY2F0ZWQiLCJpc19hbm9ueW1vdXMiOmZhbHNlfQ.Nb-N4s_YnObBHGivSTe_8FEniVUUpehzrRkF5JgNWWU"
 
-# Start supabase, which is needed in order to compile the agent
-local_resource('supabase', cmd='supabase start', links='http://localhost:5433')
+# Paths for CA and server certificates
+CA_KEY_PATH = "%s/ca.key" % FLOW_DIR
+CA_CERT_PATH = "%s/ca.crt" % FLOW_DIR
+TLS_KEY_PATH = "%s/server.key" % FLOW_DIR
+TLS_CERT_PATH = "%s/server.crt" % FLOW_DIR
+
+local_resource(
+    'supabase',
+    cmd='supabase start',
+    links='http://db.flow.localhost:5433',
+)
 
-# Builds many of the binaries that we'll need
-local_resource('make', cmd='make', resource_deps=['supabase'])
+local_resource(
+    'make',
+    cmd='make',
+    resource_deps=['supabase'],
+)
 
-local_resource('etcd', serve_cmd='%s/flow/.build/package/bin/etcd \
+local_resource(
+    'self-signed-tls-cert',
+    dir=REPO_BASE,
+    cmd='[ -f "%s" ] && [ -f "%s" ] || (\
+        openssl req -x509 -nodes -days 3650 \
+            -subj "/C=US/ST=QC/O=Estuary/CN=Estuary Root CA" \
+            -addext basicConstraints=critical,CA:TRUE,pathlen:0 \
+            -newkey ec -pkeyopt ec_paramgen_curve:P-256 \
+            -keyout "%s" \
+            -out "%s" \
+    ) && (\
+        openssl req -nodes -newkey ec -pkeyopt ec_paramgen_curve:P-256 \
+            -subj "/C=US/ST=QC/O=Estuary/CN=flow.localhost" \
+            -addext "subjectAltName=DNS:flow.localhost,DNS:*.flow.localhost,IP:127.0.0.1" \
+            -keyout "%s" -out server.csr \
+    ) && (\
+        echo "subjectAltName=DNS:flow.localhost,DNS:*.flow.localhost,IP:127.0.0.1" > extfile.txt && \
+        echo "basicConstraints=CA:FALSE" >> extfile.txt && \
+        openssl x509 -req -days 365 \
+            -in server.csr -CA "%s" -CAkey "%s" -CAcreateserial \
+            -out "%s" \
+            -extfile extfile.txt \
+    ) && (\
+        rm server.csr extfile.txt \
+    )' % (
+        TLS_CERT_PATH,   # Check if server certificate already exists
+        TLS_KEY_PATH,    # Check if server key already exists
+        CA_KEY_PATH,     # CA key output path (ECDSA)
+        CA_CERT_PATH,    # CA certificate output path
+        TLS_KEY_PATH,    # Server key output path (ECDSA)
+        CA_CERT_PATH,    # CA certificate input path
+        CA_KEY_PATH,     # CA key input path
+        TLS_CERT_PATH    # Server certificate output path
+    )
+)
+
+local_resource(
+    'etcd',
+    serve_cmd='%s/flow/.build/package/bin/etcd \
     --data-dir %s \
     --log-level info \
     --logger zap' % (REPO_BASE, ETCD_DATA_DIR),
@@ -43,90 +84,126 @@ local_resource('etcd', serve_cmd='%s/flow/.build/package/bin/etcd \
     )
 )
 
-local_resource('gazette', serve_cmd='%s/flow/.build/package/bin/gazette serve \
-    --broker.port=8080 \
-    --broker.host=localhost \
+local_resource(
+    'gazette',
+    serve_cmd='%s/flow/.build/package/bin/gazette serve \
+    --broker.allow-origin http://localhost:3000 \
     --broker.disable-stores \
-    --broker.max-replication=1 \
-    --log.level=info' % REPO_BASE,
-    links='http://localhost:8080/debug/pprof',
+    --broker.host gazette.flow.localhost \
+    --broker.max-replication 1 \
+    --broker.port 8080 \
+    --etcd.address http://etcd.flow.localhost:2379 \
+    --log.level=info \
+    ' % REPO_BASE,
+    serve_env={
+        "BROKER_AUTH_KEYS": AUTH_KEYS,
+        "BROKER_PEER_CA_FILE": CA_CERT_PATH,
+        "BROKER_SERVER_CERT_FILE": TLS_CERT_PATH,
+        "BROKER_SERVER_CERT_KEY_FILE": TLS_KEY_PATH,
+    },
+    links='https://gazette.flow.localhost:8080/debug/pprof',
     resource_deps=['etcd'],
     readiness_probe=probe(
         initial_delay_secs=5,
-        http_get=http_get_action(port=8080, path='/debug/ready')
+        http_get=http_get_action(port=8080, path='/debug/ready', scheme='https')
     )
 )
 
-local_resource('reactor', serve_cmd='%s/flow/.build/package/bin/flowctl-go serve consumer \
-    --flow.allow-local \
-    --broker.address http://localhost:8080 \
+local_resource(
+    'reactor',
+    serve_cmd='%s/flow/.build/package/bin/flowctl-go serve consumer \
+    --broker.address https://gazette.flow.localhost:8080 \
     --broker.cache.size 128 \
-    --consumer.host localhost \
+    --consumer.allow-origin http://localhost:3000 \
+    --consumer.host reactor.flow.localhost \
     --consumer.limit 1024 \
     --consumer.max-hot-standbys 0 \
     --consumer.port 9000 \
-    --etcd.address http://localhost:2379 \
-    --flow.builds-root %s \
-    --flow.network supabase_network_flow \
-    --flow.control-api http://localhost:8675 \
+    --etcd.address http://etcd.flow.localhost:2379 \
+    --flow.allow-local \
+    --flow.control-api http://agent.flow.localhost:8675 \
+    --flow.dashboard   http://localhost:3000 \
     --flow.data-plane-fqdn local-cluster.dp.estuary-data.com \
-    --log.format text \
-    --log.level info' % (REPO_BASE, FLOW_BUILDS_ROOT),
-    links='http://localhost:9000/debug/pprof',
+    --flow.network supabase_network_flow \
+    --log.level info \
+    ' % (REPO_BASE),
+    serve_env={
+        "BROKER_AUTH_KEYS": AUTH_KEYS,
+        "BROKER_TRUSTED_CA_FILE": CA_CERT_PATH,
+        "CONSUMER_AUTH_KEYS": AUTH_KEYS,
+        "CONSUMER_PEER_CA_FILE": CA_CERT_PATH,
+        "CONSUMER_SERVER_CERT_FILE": TLS_CERT_PATH,
+        "CONSUMER_SERVER_CERT_KEY_FILE": TLS_KEY_PATH,
+        "DOCKER_DEFAULT_PLATFORM": "linux/amd64",
+        "FLOW_BUILDS_ROOT": FLOW_BUILDS_ROOT,
+    },
+    links='https://reactor.flow.localhost:9000/debug/pprof',
     resource_deps=['etcd'],
     readiness_probe=probe(
         initial_delay_secs=5,
-        http_get=http_get_action(port=9000, path='/debug/ready')
-    )
+        http_get=http_get_action(port=9000, path='/debug/ready', scheme='https')
+    ),
 )
 
-local_resource('agent', serve_cmd='%s/flow/.build/package/bin/agent \
+local_resource(
+    'agent',
+    serve_cmd='%s/flow/.build/package/bin/agent \
     --connector-network supabase_network_flow \
     --allow-local \
     --allow-origin http://localhost:3000 \
     --api-port 8675 \
-    --builds-root %s \
     --serve-handlers \
-    --bin-dir %s/flow/.build/package/bin' % (REPO_BASE, FLOW_BUILDS_ROOT, REPO_BASE),
+    ' % (REPO_BASE),
+    serve_env={
+        "BIN_DIR": '%s/flow/.build/package/bin' % REPO_BASE,
+        "BUILDS_ROOT": FLOW_BUILDS_ROOT,
+        "DATABASE_URL": DATABASE_URL,
+        "RUST_LOG": "info",
+        "SSL_CERT_FILE": CA_CERT_PATH,
+    },
     resource_deps=['reactor', 'gazette']
 )
 
-local_resource('create-data-plane-local-cluster',
-    cmd='sleep 5 && curl -v \
+local_resource(
+    'create-data-plane-local-cluster',
+    cmd='sleep 2 && curl -v \
         -X POST \
         -H "content-type: application/json" \
         -H "authorization: bearer %s" \
         --data-binary \'{ \
             "name":"local-cluster",\
             "category": {\
                 "manual": {\
-                    "brokerAddress": "http://localhost:8080",\
-                    "reactorAddress": "http://localhost:9000",\
+                    "brokerAddress": "https://gazette.flow.localhost:8080",\
+                    "reactorAddress": "https://reactor.flow.localhost:9000",\
                     "hmacKeys": ["c3VwZXJzZWNyZXQ="]\
                 }\
             }\
-        }\' http://localhost:8675/admin/create-data-plane' % SYSTEM_USER_TOKEN,
+        }\' http://agent.flow.localhost:8675/admin/create-data-plane' % SYSTEM_USER_TOKEN,
     resource_deps=['agent']
 )
 
-local_resource('update-l2-reporting',
+local_resource(
+    'update-l2-reporting',
     cmd='curl -v \
         -X POST \
         -H "content-type: application/json" \
         -H "authorization: bearer %s" \
         --data-binary \'{ \
             "defaultDataPlane":"ops/dp/public/local-cluster",\
             "dryRun":false\
-        }\' http://localhost:8675/admin/update-l2-reporting' % SYSTEM_USER_TOKEN,
+        }\' http://agent.flow.localhost:8675/admin/update-l2-reporting' % SYSTEM_USER_TOKEN,
     resource_deps=['create-data-plane-local-cluster']
 )
 
-local_resource('local-ops-view',
+local_resource(
+    'local-ops-view',
     cmd='./local/ops-publication.sh ops-catalog/local-view.bundle.json | psql "%s"' % DATABASE_URL,
     resource_deps=['update-l2-reporting']
 )
 
-local_resource('config-encryption',
+local_resource(
+    'config-encryption',
     serve_cmd='%s/config-encryption/target/debug/flow-config-encryption --gcp-kms %s' % (REPO_BASE, TEST_KMS_KEY)
 )
 
@@ -137,45 +214,8 @@ local_resource(
 )
 
 local_resource(
-    'ui',
+    'dashboard',
     serve_dir='%s/ui' % REPO_BASE,
     serve_cmd='BROWSER=none npm start',
     links='http://localhost:3000'
-)
-
-DPG_REPO='%s/data-plane-gateway' % REPO_BASE
-DPG_TLS_CERT_PATH='%s/local-tls-cert.pem' % DPG_REPO
-DPG_TLS_KEY_PATH='%s/local-tls-private-key.pem' % DPG_REPO
-
-local_resource('dpg-tls-cert',
-    dir='%s/data-plane-gateway' % REPO_BASE,
-    # These incantations create a non-CA self-signed certificate which is
-    # valid for localhost and its subdomains. rustls is quite fiddly about
-    # accepting self-signed certificates so all of these are required.
-    cmd='[ -f %s ] || openssl req -x509 -nodes -days 365 \
-        -subj  "/ST=QC/O=Estuary/CN=localhost" \
-        -addext basicConstraints=critical,CA:FALSE,pathlen:1 \
-        -addext "subjectAltName=DNS:localhost,DNS:*.localhost,IP:127.0.0.1" \
-        -newkey rsa:2048 -keyout "%s" \
-        -out "%s"' % (DPG_TLS_KEY_PATH, DPG_TLS_KEY_PATH, DPG_TLS_CERT_PATH)
-)
-
-local_resource('data-plane-gateway',
-    dir=DPG_REPO,
-    serve_dir=DPG_REPO,
-    cmd='go build .',
-    serve_cmd='./data-plane-gateway \
-        --tls-private-key=%s \
-        --tls-certificate=%s \
-        --broker-address=localhost:8080 \
-        --consumer-address=localhost:9000 \
-        --log.level=debug \
-        --inference-address=localhost:9090 \
-        --control-plane-auth-url=http://localhost:3000' % (
-            DPG_TLS_KEY_PATH,
-            DPG_TLS_CERT_PATH
-        ),
-    links='https://localhost:28318/',
-    resource_deps=['gazette', 'reactor', 'dpg-tls-cert']
-)
-
+)
diff --git a/go.mod b/go.mod
@@ -12,6 +12,7 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
+	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/jessevdk/go-flags v1.5.0
 	github.com/jgraettinger/gorocksdb v0.0.0-20240221161858-8f4873ee26e0
 	github.com/mattn/go-sqlite3 v2.0.3+incompatible
@@ -22,7 +23,7 @@ require (
 	github.com/stretchr/testify v1.9.0
 	go.etcd.io/etcd/api/v3 v3.5.0
 	go.etcd.io/etcd/client/v3 v3.5.0
-	go.gazette.dev/core v0.99.1-0.20240908182145-d0c9d56a6796
+	go.gazette.dev/core v0.99.1-0.20240920050446-fc9749109d92
 	golang.org/x/net v0.26.0
 	google.golang.org/api v0.126.0
 	google.golang.org/grpc v1.65.0
@@ -49,6 +50,9 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/dustinkirkland/golang-petname v0.0.0-20191129215211-8e5a1ed0cff0 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/gogo/gateway v1.1.0 // indirect
+	github.com/golang/glog v1.2.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
@@ -57,6 +61,7 @@ require (
 	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
 	github.com/googleapis/gax-go/v2 v2.11.0 // indirect
 	github.com/gorilla/schema v1.4.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
 	github.com/jgraettinger/cockroach-encoding v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect