Merge branch 'develop' into nasdf/refactor/subscription-events

.github/workflows/check-cli-documentation.yml

@@ -35,15 +35,6 @@ jobs:
       - name: Checkout code into the directory
         uses: actions/checkout@v3
 
-      # This check is there as a safety to ensure we start clean (without any changes).
-      # If there are ever changes here, the rest of the job will output false result.
-      - name: Check no changes exist initially
-        uses: tj-actions/verify-changed-files@v20
-        with:
-          fail-if-changed: true
-          files: |
-             docs/website/references/cli
-
       - name: Setup Go environment explicitly
         uses: actions/setup-go@v3
         with:

.github/workflows/check-http-documentation.yml

@@ -0,0 +1,52 @@
+# Copyright 2024 Democratized Data Foundation
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+# This workflow checks that all HTTP documentation is up to date.
+# If the documentation is not up to date then this action will fail.
+name: Check HTTP Documentation Workflow
+
+on:
+  pull_request:
+    branches:
+      - master
+      - develop
+
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+    branches:
+      - master
+      - develop
+
+jobs:
+  check-http-documentation:
+    name: Check http documentation job
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code into the directory
+        uses: actions/checkout@v3
+
+      - name: Setup Go environment explicitly
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.21"
+          check-latest: true
+
+      - name: Try generating http documentation
+        run: make docs:http
+
+      - name: Check no new changes exist
+        uses: tj-actions/verify-changed-files@v20
+        with:
+          fail-if-changed: true
+          files: |
+             docs/website/references/http

.github/workflows/check-mocks.yml

@@ -0,0 +1,50 @@
+# Copyright 2024 Democratized Data Foundation
+#
+# Use of this software is governed by the Business Source License
+# included in the file licenses/BSL.txt.
+#
+# As of the Change Date specified in that file, in accordance with
+# the Business Source License, use of this software will be governed
+# by the Apache License, Version 2.0, included in the file
+# licenses/APL.txt.
+
+# This workflow checks that all mocks are up to date.
+# If the mocks are not up to date then this action will fail.
+name: Check Mocks Workflow
+
+on:
+  pull_request:
+    branches:
+      - master
+      - develop
+
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+    branches:
+      - master
+      - develop
+
+jobs:
+  check-mocks:
+    name: Check mocks job
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code into the directory
+        uses: actions/checkout@v3
+
+      - name: Setup Go environment explicitly
+        uses: actions/setup-go@v3
+        with:
+          go-version: "1.21"
+          check-latest: true
+
+      - name: Try generating mocks
+        run: make mocks
+
+      - name: Check no new changes exist
+        uses: tj-actions/verify-changed-files@v20
+        with:
+          fail-if-changed: true

.github/workflows/test-and-upload-coverage.yml

@@ -34,25 +34,29 @@ jobs:
         client-type: [go, http, cli]
         database-type: [badger-file, badger-memory]
         mutation-type: [gql, collection-named, collection-save]
+        lens-type: [wasm-time]
         detect-changes: [false]
         database-encryption: [false]
         include:
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
+            lens-type: wasm-time
             detect-changes: true
             database-encryption: false
           - os: ubuntu-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
+            lens-type: wasm-time
             detect-changes: false
             database-encryption: true
           - os: macos-latest
             client-type: go
             database-type: badger-memory
             mutation-type: collection-save
+            lens-type: wasm-time
             detect-changes: false
             database-encryption: false
 ## TODO: https://github.com/sourcenetwork/defradb/issues/2080
@@ -63,6 +67,20 @@ jobs:
 ##          mutation-type: collection-save
 ##          detect-changes: false
 ##          database-encryption: false
+          - os: ubuntu-latest
+            client-type: go
+            database-type: badger-memory
+            mutation-type: collection-save
+            lens-type: wazero
+            detect-changes: false
+            database-encryption: false
+          - os: ubuntu-latest
+            client-type: go
+            database-type: badger-memory
+            mutation-type: collection-save
+            lens-type: wasmer
+            detect-changes: false
+            database-encryption: false
 
     runs-on: ${{ matrix.os }}
 
@@ -80,6 +98,7 @@ jobs:
       DEFRA_BADGER_FILE: ${{ matrix.database-type == 'badger-file' }}
       DEFRA_BADGER_ENCRYPTION: ${{ matrix.database-encryption }}
       DEFRA_MUTATION_TYPE: ${{ matrix.mutation-type }}
+      DEFRA_LENS_TYPE: ${{ matrix.lens-type }}
 
     steps:
       - name: Checkout code into the directory

Makefile

@@ -158,8 +158,8 @@ deps\:chglog:
 deps\:modules:
 	go mod download
 
-.PHONY: deps\:mock
-deps\:mock:
+.PHONY: deps\:mocks
+deps\:mocks:
 	go install github.com/vektra/mockery/[email protected]
 
 .PHONY: deps\:playground
@@ -173,11 +173,11 @@ deps:
 	$(MAKE) deps:chglog && \
 	$(MAKE) deps:lint && \
 	$(MAKE) deps:test && \
-	$(MAKE) deps:mock
+	$(MAKE) deps:mocks
 
-.PHONY: mock
-mock:
-	@$(MAKE) deps:mock
+.PHONY: mocks
+mocks:
+	@$(MAKE) deps:mocks
 	mockery --config="tools/configs/mockery.yaml"
 
 .PHONY: dev\:start
@@ -356,12 +356,17 @@ chglog:
 docs:
 	@$(MAKE) docs\:cli
 	@$(MAKE) docs\:manpages
+	@$(MAKE) docs\:http
 
 .PHONY: docs\:cli
 docs\:cli:
 	rm -f docs/website/references/cli/*.md
 	go run cmd/genclidocs/main.go -o docs/website/references/cli
 
+.PHONY: docs\:http
+docs\:http:
+	go run cmd/genopenapi/main.go | python -m json.tool > docs/website/references/http/openapi.json
+
 .PHONY: docs\:manpages
 docs\:manpages:
 	go run cmd/genmanpages/main.go -o build/man/

cli/server_dump.go

@@ -37,7 +37,7 @@ func MakeServerDumpCmd() *cobra.Command {
 			if err != nil {
 				return err
 			}
-			db, err := db.NewDB(cmd.Context(), rootstore, acp.NoACP)
+			db, err := db.NewDB(cmd.Context(), rootstore, acp.NoACP, nil)
 			if err != nil {
 				return errors.Wrap("failed to initialize database", err)
 			}

cli/start.go

@@ -126,6 +126,7 @@ func MakeStartCommand() *cobra.Command {
 				http.WithAllowedOrigins(cfg.GetStringSlice("api.allowed-origins")...),
 				http.WithTLSCertPath(cfg.GetString("api.pubKeyPath")),
 				http.WithTLSKeyPath(cfg.GetString("api.privKeyPath")),
+				node.WithLensRuntime(node.LensRuntimeType(cfg.GetString("lens.runtime"))),
 			}
 
 			if cfg.GetString("datastore.store") != configStoreMemory {

client/lens.go

@@ -15,6 +15,8 @@ import (
 
 	"github.com/lens-vm/lens/host-go/config/model"
 	"github.com/sourcenetwork/immutable/enumerable"
+
+	"github.com/sourcenetwork/defradb/datastore"
 )
 
 // LensConfig represents the configuration of a Lens migration in Defra.
@@ -38,9 +40,18 @@ type LensConfig struct {
 	model.Lens
 }
 
+// TxnSource represents an object capable of constructing the transactions that
+// implicit-transaction registries need internally.
+type TxnSource interface {
+	NewTxn(context.Context, bool) (datastore.Txn, error)
+}
+
 // LensRegistry exposes several useful thread-safe migration related functions which may
 // be used to manage migrations.
 type LensRegistry interface {
+	// Init initializes the registry with the provided transaction source.
+	Init(TxnSource)
+
 	// SetMigration caches the migration for the given collection ID. It does not persist the migration in long
 	// term storage, for that one should call [Store.SetMigration(ctx, cfg)].
 	//

docs/config.md

@@ -111,3 +111,12 @@ Keyring backend to use. Defaults to `file`.
 
 - `file` Stores keys in encrypted files
 - `system` Stores keys in the OS managed keyring
+
+## `lens.runtime`
+
+The LensVM wasm runtime to run lens modules in.
+
+Possible values:
+- `wasm-time` (default): https://github.com/bytecodealliance/wasmtime-go
+- `wasmer` (windows not supported): https://github.com/wasmerio/wasmer-go
+- `wazero`: https://github.com/tetratelabs/wazero