From 7dc07e74b13e8b6a1ca7c1a93c27f4cab46c62ab Mon Sep 17 00:00:00 2001 From: Shahzad Lone Date: Tue, 4 Jun 2024 19:17:54 -0400 Subject: [PATCH 1/3] docs: Add http/openapi documentation & ci workflow (#2678) ## Relevant issue(s) Related #510 Resolve #2677 ## Description - Detect OpenAPI / HTTP documentation is always up to date. - Generate open-api docs in the appropriate dir. ## How has this been tested? - using `act` tool - manually introducing a change and seeing the action fail: https://github.com/sourcenetwork/defradb/actions/runs/9359749777/job/25763961265?pr=2678 Specify the platform(s) on which this was tested: - WSL2 instance --- .github/workflows/check-cli-documentation.yml | 9 - .../workflows/check-http-documentation.yml | 52 + Makefile | 5 + docs/website/references/http/openapi.json | 2094 +++++++++++++++++ 4 files changed, 2151 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/check-http-documentation.yml create mode 100644 docs/website/references/http/openapi.json diff --git a/.github/workflows/check-cli-documentation.yml b/.github/workflows/check-cli-documentation.yml index 06a7bd7bdc..be50351518 100644 --- a/.github/workflows/check-cli-documentation.yml +++ b/.github/workflows/check-cli-documentation.yml @@ -35,15 +35,6 @@ jobs: - name: Checkout code into the directory uses: actions/checkout@v3 - # This check is there as a safety to ensure we start clean (without any changes). - # If there are ever changes here, the rest of the job will output false result. - - name: Check no changes exist initially - uses: tj-actions/verify-changed-files@v20 - with: - fail-if-changed: true - files: | - docs/website/references/cli - - name: Setup Go environment explicitly uses: actions/setup-go@v3 with: diff --git a/.github/workflows/check-http-documentation.yml b/.github/workflows/check-http-documentation.yml new file mode 100644 index 0000000000..e68471c162 --- /dev/null +++ b/.github/workflows/check-http-documentation.yml @@ -0,0 +1,52 @@ +# Copyright 2024 Democratized Data Foundation +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +# This workflow checks that all HTTP documentation is up to date. +# If the documentation is not up to date then this action will fail. +name: Check HTTP Documentation Workflow + +on: + pull_request: + branches: + - master + - develop + + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + branches: + - master + - develop + +jobs: + check-http-documentation: + name: Check http documentation job + + runs-on: ubuntu-latest + + steps: + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Setup Go environment explicitly + uses: actions/setup-go@v3 + with: + go-version: "1.21" + check-latest: true + + - name: Try generating http documentation + run: make docs:http + + - name: Check no new changes exist + uses: tj-actions/verify-changed-files@v20 + with: + fail-if-changed: true + files: | + docs/website/references/http diff --git a/Makefile b/Makefile index bc4093ddbf..1f50ab816c 100644 --- a/Makefile +++ b/Makefile @@ -356,12 +356,17 @@ chglog: docs: @$(MAKE) docs\:cli @$(MAKE) docs\:manpages + @$(MAKE) docs\:http .PHONY: docs\:cli docs\:cli: rm -f docs/website/references/cli/*.md go run cmd/genclidocs/main.go -o docs/website/references/cli +.PHONY: docs\:http +docs\:http: + go run cmd/genopenapi/main.go | python -m json.tool > docs/website/references/http/openapi.json + .PHONY: docs\:manpages docs\:manpages: go run cmd/genmanpages/main.go -o build/man/ diff --git a/docs/website/references/http/openapi.json b/docs/website/references/http/openapi.json new file mode 100644 index 0000000000..a8c02e72bf --- /dev/null +++ b/docs/website/references/http/openapi.json @@ -0,0 +1,2094 @@ +{ + "components": { + "parameters": { + "txn": { + "description": "Transaction id", + "in": "header", + "name": "x-defradb-tx", + "schema": { + "format": "int64", + "type": "integer" + } + } + }, + "responses": { + "error": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/error" + } + } + }, + "description": "error" + }, + "success": { + "description": "ok" + } + }, + "schemas": { + "add_view_request": { + "properties": { + "Query": { + "type": "string" + }, + "SDL": { + "type": "string" + }, + "Transform": {} + }, + "type": "object" + }, + "backup_config": { + "properties": { + "collections": { + "items": { + "type": "string" + }, + "type": "array" + }, + "filepath": { + "type": "string" + }, + "format": { + "type": "string" + }, + "pretty": { + "type": "boolean" + } + }, + "type": "object" + }, + "ccip_request": { + "properties": { + "data": { + "type": "string" + }, + "sender": { + "type": "string" + } + }, + "type": "object" + }, + "ccip_response": { + "properties": { + "data": { + "type": "string" + } + }, + "type": "object" + }, + "collection": { + "properties": { + "Fields": { + "items": { + "properties": { + "ID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Kind": {}, + "Name": { + "type": "string" + }, + "RelationName": {} + }, + "type": "object" + }, + "type": "array" + }, + "ID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Indexes": { + "items": { + "properties": { + "Fields": { + "items": { + "properties": { + "Descending": { + "type": "boolean" + }, + "Name": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "ID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Name": { + "type": "string" + }, + "Unique": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": "array" + }, + "Name": {}, + "Policy": {}, + "RootID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "SchemaVersionID": { + "type": "string" + }, + "Sources": { + "items": {}, + "type": "array" + } + }, + "type": "object" + }, + "collection_definition": { + "properties": { + "description": { + "properties": { + "Fields": { + "items": { + "properties": { + "ID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Kind": {}, + "Name": { + "type": "string" + }, + "RelationName": {} + }, + "type": "object" + }, + "type": "array" + }, + "ID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Indexes": { + "items": { + "properties": { + "Fields": { + "items": { + "properties": { + "Descending": { + "type": "boolean" + }, + "Name": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "ID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Name": { + "type": "string" + }, + "Unique": { + "type": "boolean" + } + }, + "type": "object" + }, + "type": "array" + }, + "Name": {}, + "Policy": {}, + "RootID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "SchemaVersionID": { + "type": "string" + }, + "Sources": { + "items": {}, + "type": "array" + } + }, + "type": "object" + }, + "schema": { + "properties": { + "Fields": { + "items": { + "properties": { + "Kind": {}, + "Name": { + "type": "string" + }, + "Typ": { + "maximum": 255, + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + }, + "Name": { + "type": "string" + }, + "Root": { + "type": "string" + }, + "VersionID": { + "type": "string" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "collection_delete": { + "properties": { + "filter": {} + }, + "type": "object" + }, + "collection_update": { + "properties": { + "filter": {}, + "updater": { + "type": "string" + } + }, + "type": "object" + }, + "create_tx": { + "properties": { + "id": { + "maximum": 18446744073709552000, + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "delete_result": { + "properties": { + "Count": { + "format": "int64", + "type": "integer" + }, + "DocIDs": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "document": { + "additionalProperties": true, + "type": "object" + }, + "error": { + "properties": { + "error": {} + }, + "type": "object" + }, + "graphql_request": { + "properties": { + "query": { + "type": "string" + } + }, + "type": "object" + }, + "graphql_response": { + "properties": { + "data": {}, + "errors": { + "items": {}, + "type": "array" + } + }, + "type": "object" + }, + "index": { + "properties": { + "Fields": { + "items": { + "properties": { + "Descending": { + "type": "boolean" + }, + "Name": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "ID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Name": { + "type": "string" + }, + "Unique": { + "type": "boolean" + } + }, + "type": "object" + }, + "lens_config": { + "properties": { + "DestinationSchemaVersionID": { + "type": "string" + }, + "Lenses": { + "items": { + "properties": { + "Arguments": { + "additionalProperties": {}, + "type": "object" + }, + "Inverse": { + "type": "boolean" + }, + "Path": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + }, + "SourceSchemaVersionID": { + "type": "string" + } + }, + "type": "object" + }, + "migrate_request": { + "properties": { + "CollectionID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Data": { + "items": { + "additionalProperties": {}, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "patch_schema_request": { + "properties": { + "Migration": {}, + "Patch": { + "type": "string" + }, + "SetAsDefaultVersion": { + "type": "boolean" + } + }, + "type": "object" + }, + "peer_info": { + "properties": { + "Addrs": { + "items": {}, + "type": "array" + }, + "ID": { + "type": "string" + } + }, + "type": "object" + }, + "replicator": { + "properties": { + "Info": { + "properties": { + "Addrs": { + "items": {}, + "type": "array" + }, + "ID": { + "type": "string" + } + }, + "type": "object" + }, + "Schemas": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "schema": { + "properties": { + "Fields": { + "items": { + "properties": { + "Kind": {}, + "Name": { + "type": "string" + }, + "Typ": { + "maximum": 255, + "minimum": 0, + "type": "integer" + } + }, + "type": "object" + }, + "type": "array" + }, + "Name": { + "type": "string" + }, + "Root": { + "type": "string" + }, + "VersionID": { + "type": "string" + } + }, + "type": "object" + }, + "set_migration_request": { + "properties": { + "CollectionID": { + "maximum": 4294967295, + "minimum": 0, + "type": "integer" + }, + "Config": { + "properties": { + "Lenses": { + "items": { + "properties": { + "Arguments": { + "additionalProperties": {}, + "type": "object" + }, + "Inverse": { + "type": "boolean" + }, + "Path": { + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + } + }, + "type": "object" + }, + "update_result": { + "properties": { + "Count": { + "format": "int64", + "type": "integer" + }, + "DocIDs": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + } + }, + "securitySchemes": { + "bearerToken": { + "bearerFormat": "JWT", + "scheme": "bearer", + "type": "http" + } + } + }, + "externalDocs": { + "description": "Learn more about DefraDB", + "url": "https://docs.source.network" + }, + "info": { + "title": "DefraDB API", + "version": "0" + }, + "openapi": "3.0.3", + "paths": { + "/acp/policy": { + "post": { + "description": "Add a policy using acp system", + "operationId": "add policy", + "requestBody": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "acp_policy" + ] + } + }, + "/backup/export": { + "post": { + "description": "Export a database backup to file", + "operationId": "backup_export", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/backup_config" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "backup" + ] + } + }, + "/backup/import": { + "post": { + "description": "Import a database backup from file", + "operationId": "backup_import", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/backup_config" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "backup" + ] + } + }, + "/ccip": { + "post": { + "description": "CCIP POST endpoint", + "operationId": "ccip_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ccip_request" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ccip_response" + } + } + }, + "description": "GraphQL response" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "ccip" + ] + } + }, + "/ccip/{sender}/{data}": { + "get": { + "description": "CCIP GET endpoint", + "operationId": "ccip_get", + "parameters": [ + { + "description": "Hex encoded request data", + "in": "path", + "name": "data", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Hex encoded sender address", + "in": "path", + "name": "sender", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ccip_response" + } + } + }, + "description": "GraphQL response" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "ccip" + ] + } + }, + "/collections": { + "get": { + "description": "Introspect collection(s) by name, schema id, or version id.", + "operationId": "collection_describe", + "parameters": [ + { + "description": "Collection name", + "in": "query", + "name": "name", + "schema": { + "type": "string" + } + }, + { + "description": "Collection schema root", + "in": "query", + "name": "schema_root", + "schema": { + "type": "string" + } + }, + { + "description": "Collection schema version id", + "in": "query", + "name": "version_id", + "schema": { + "type": "string" + } + }, + { + "description": "If true, inactive collections will be returned in addition to active ones", + "in": "query", + "name": "get_inactive", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/collection" + }, + { + "items": { + "$ref": "#/components/schemas/collection" + }, + "type": "array" + } + ] + } + } + }, + "description": "Collection(s) with matching name, schema id, or version id." + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + }, + "patch": { + "description": "Update collection definitions", + "operationId": "patch_collection", + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "string" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + } + }, + "/collections/{name}": { + "delete": { + "description": "Delete document(s) from a collection", + "operationId": "collection_delete_with_filter", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/collection_delete" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/delete_result" + } + } + }, + "description": "Delete results" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + }, + "get": { + "description": "Get all document IDs", + "operationId": "collection_keys", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + }, + "patch": { + "description": "Update document(s) in a collection", + "operationId": "collection_update_with_filter", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/collection_update" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/update_result" + } + } + }, + "description": "Update results" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + }, + "post": { + "description": "Create document(s) in a collection", + "operationId": "collection_create", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/document" + }, + { + "items": { + "$ref": "#/components/schemas/document" + }, + "type": "array" + } + ] + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + } + }, + "/collections/{name}/indexes": { + "get": { + "description": "List secondary indexes", + "operationId": "index_list", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/index" + }, + "type": "array" + } + } + }, + "description": "List of indexes" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "index" + ] + }, + "post": { + "description": "Create a secondary index", + "operationId": "index_create", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/index" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/index" + } + } + }, + "description": "Index description" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "index" + ] + } + }, + "/collections/{name}/indexes/{index}": { + "delete": { + "description": "Delete a secondary index", + "operationId": "index_drop", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "index", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "index" + ] + } + }, + "/collections/{name}/{docID}": { + "delete": { + "description": "Delete a document by docID", + "operationId": "collection_delete", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "docID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + }, + "get": { + "description": "Get a document by docID", + "operationId": "collection_get", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "docID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/document" + } + } + }, + "description": "Document value" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + }, + "patch": { + "description": "Update a document by docID", + "operationId": "collection_update", + "parameters": [ + { + "description": "Collection name", + "in": "path", + "name": "name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "path", + "name": "docID", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "collection" + ] + } + }, + "/debug/dump": { + "get": { + "description": "Dump database", + "operationId": "debug_dump", + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "debug" + ] + } + }, + "/graphql": { + "get": { + "description": "GraphQL GET endpoint", + "operationId": "graphql_get", + "parameters": [ + { + "in": "query", + "name": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/graphql_response" + } + } + }, + "description": "GraphQL response" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "graphql" + ] + }, + "post": { + "description": "GraphQL POST endpoint", + "operationId": "graphql_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/graphql_request" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/graphql_response" + } + } + }, + "description": "GraphQL response" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "graphql" + ] + } + }, + "/lens": { + "post": { + "description": "Add a new lens migration", + "operationId": "lens_set_migration", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/lens_config" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "lens" + ] + } + }, + "/lens/registry": { + "post": { + "description": "Add a new lens migration to registry", + "operationId": "lens_registry_set_migration", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/set_migration_request" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "lens" + ] + } + }, + "/lens/registry/reload": { + "post": { + "description": "Reload lens migrations", + "operationId": "lens_registry_reload", + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "lens" + ] + } + }, + "/lens/registry/{version}/down": { + "post": { + "description": "Migrate documents from a collection", + "operationId": "lens_registry_migrate_down", + "parameters": [ + { + "in": "path", + "name": "version", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/migrate_request" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "lens" + ] + } + }, + "/lens/registry/{version}/up": { + "post": { + "description": "Migrate documents to a collection", + "operationId": "lens_registry_migrate_up", + "parameters": [ + { + "in": "path", + "name": "version", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/migrate_request" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "lens" + ] + } + }, + "/p2p/collections": { + "delete": { + "description": "Remove peer collections", + "operationId": "peer_collection_remove", + "requestBody": { + "content": { + "application/json": { + "schema": { + "items": { + "type": "string" + }, + "type": "array" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "p2p" + ] + }, + "get": { + "description": "List peer collections", + "operationId": "peer_collection_list", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "type": "string" + }, + "type": "array" + } + } + }, + "description": "Peer collections" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "p2p" + ] + }, + "post": { + "description": "Add peer collections", + "operationId": "peer_collection_add", + "requestBody": { + "content": { + "application/json": { + "schema": { + "items": { + "type": "string" + }, + "type": "array" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "p2p" + ] + } + }, + "/p2p/info": { + "get": { + "operationId": "peer_info", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/peer_info" + } + } + }, + "description": "Peer network info" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "p2p" + ] + } + }, + "/p2p/replicators": { + "delete": { + "description": "Delete peer replicators", + "operationId": "peer_replicator_delete", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/replicator" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "p2p" + ] + }, + "get": { + "description": "List peer replicators", + "operationId": "peer_replicator_list", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/replicator" + }, + "type": "array" + } + } + }, + "description": "Replicators" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "p2p" + ] + }, + "post": { + "description": "Add peer replicators", + "operationId": "peer_replicator_set", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/replicator" + } + } + }, + "required": true + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "p2p" + ] + } + }, + "/schema": { + "get": { + "description": "Introspect schema(s) by name, schema root, or version id.", + "operationId": "schema_describe", + "parameters": [ + { + "description": "Schema name", + "in": "query", + "name": "name", + "schema": { + "type": "string" + } + }, + { + "description": "Schema root", + "in": "query", + "name": "root", + "schema": { + "type": "string" + } + }, + { + "description": "Schema version id", + "in": "query", + "name": "version_id", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/schema" + }, + { + "items": { + "$ref": "#/components/schemas/schema" + }, + "type": "array" + } + ] + } + } + }, + "description": "Schema(s) with matching name, schema id, or version id." + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "schema" + ] + }, + "patch": { + "description": "Update a schema definition", + "operationId": "patch_schema", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/patch_schema_request" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "schema" + ] + }, + "post": { + "description": "Add a new schema definition", + "operationId": "add_schema", + "requestBody": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/collection" + }, + "type": "array" + } + } + }, + "description": "Collection(s)" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "schema" + ] + } + }, + "/schema/default": { + "post": { + "description": "Set the default schema version for a collection", + "operationId": "set_default_schema_version", + "requestBody": { + "content": { + "text/plain": { + "schema": { + "type": "string" + } + } + } + }, + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "schema" + ] + } + }, + "/tx": { + "post": { + "description": "Create a new transaction", + "operationId": "new_transaction", + "parameters": [ + { + "description": "Read only transaction", + "in": "query", + "name": "read_only", + "schema": { + "default": false, + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/create_tx" + } + } + }, + "description": "Transaction info" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "transaction" + ] + } + }, + "/tx/concurrent": { + "post": { + "description": "Create a new concurrent transaction", + "operationId": "new_concurrent_transaction", + "parameters": [ + { + "description": "Read only transaction", + "in": "query", + "name": "read_only", + "schema": { + "default": false, + "type": "boolean" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/create_tx" + } + } + }, + "description": "Transaction info" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "transaction" + ] + } + }, + "/tx/{id}": { + "delete": { + "description": "Discard a transaction", + "operationId": "transaction_discard", + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "transaction" + ] + }, + "post": { + "description": "Commit a transaction", + "operationId": "transaction_commit", + "parameters": [ + { + "in": "path", + "name": "id", + "required": true, + "schema": { + "format": "int64", + "type": "integer" + } + } + ], + "responses": { + "200": { + "$ref": "#/components/responses/success" + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "transaction" + ] + } + }, + "/view": { + "post": { + "description": "Manage database views.", + "operationId": "view", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/add_view_request" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "oneOf": [ + { + "$ref": "#/components/schemas/collection_definition" + }, + { + "items": { + "$ref": "#/components/schemas/collection_definition" + }, + "type": "array" + } + ] + } + } + }, + "description": "The created collection and embedded schemas for the added view." + }, + "400": { + "$ref": "#/components/responses/error" + }, + "default": { + "description": "" + } + }, + "tags": [ + "view" + ] + } + } + }, + "servers": [ + { + "description": "Local DefraDB instance", + "url": "http://localhost:9181/api/v0" + } + ], + "tags": [ + { + "description": "Add or update schema definitions", + "name": "schema" + }, + { + "description": "Add, remove, or update documents", + "name": "collection" + }, + { + "description": "Add views", + "name": "view" + }, + { + "description": "Add, update, or remove indexes", + "name": "index" + }, + { + "description": "Migrate documents to and from schema versions", + "name": "lens" + }, + { + "description": "Peer-to-peer network operations", + "name": "p2p" + }, + { + "description": "Access control policy operations", + "name": "acp" + }, + { + "description": "Database transaction operations", + "name": "transaction" + }, + { + "description": "Database backup operations", + "name": "backup" + }, + { + "description": "GraphQL query endpoints", + "name": "graphql" + }, + { + "externalDocs": { + "description": "EIP-3668", + "url": "https://eips.ethereum.org/EIPS/eip-3668" + }, + "name": "ccip" + } + ] +} From 25fc07da90988194ca3d52b41c00a8ea0da2e0eb Mon Sep 17 00:00:00 2001 From: Shahzad Lone Date: Tue, 4 Jun 2024 19:49:53 -0400 Subject: [PATCH 2/3] ci(i): Add workflow to detect out of date mocks (#2679) ## Relevant issue(s) Resolves #1616 ## Description - Add github action that will fail if there are any out of date mocks - Proposed a year ago here: https://github.com/sourcenetwork/defradb/pull/1612#pullrequestreview-1516813391 ## How has this been tested? - using `act` tool - manually through introducing a mock change in this commit: [`4b20f8f` (#2679)](https://github.com/sourcenetwork/defradb/pull/2679/commits/4b20f8f992866b58839c2f23c4c2dc1ac3d7f5d7) and then seeing the action fail here: https://github.com/sourcenetwork/defradb/actions/runs/9361512099/job/25768647150?pr=2679 - the last commit reverts the commit that was introduce the test the mock detection works, I didn't drop the commit and used the revert to have it documented better. Specify the platform(s) on which this was tested: - WSL2 --- .github/workflows/check-mocks.yml | 50 +++++++++++++++++++++++++++++++ Makefile | 12 ++++---- tests/README.md | 2 +- 3 files changed, 57 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/check-mocks.yml diff --git a/.github/workflows/check-mocks.yml b/.github/workflows/check-mocks.yml new file mode 100644 index 0000000000..80d9dbee20 --- /dev/null +++ b/.github/workflows/check-mocks.yml @@ -0,0 +1,50 @@ +# Copyright 2024 Democratized Data Foundation +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +# This workflow checks that all mocks are up to date. +# If the mocks are not up to date then this action will fail. +name: Check Mocks Workflow + +on: + pull_request: + branches: + - master + - develop + + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + branches: + - master + - develop + +jobs: + check-mocks: + name: Check mocks job + + runs-on: ubuntu-latest + + steps: + - name: Checkout code into the directory + uses: actions/checkout@v3 + + - name: Setup Go environment explicitly + uses: actions/setup-go@v3 + with: + go-version: "1.21" + check-latest: true + + - name: Try generating mocks + run: make mocks + + - name: Check no new changes exist + uses: tj-actions/verify-changed-files@v20 + with: + fail-if-changed: true diff --git a/Makefile b/Makefile index 1f50ab816c..c4c02da3d7 100644 --- a/Makefile +++ b/Makefile @@ -158,8 +158,8 @@ deps\:chglog: deps\:modules: go mod download -.PHONY: deps\:mock -deps\:mock: +.PHONY: deps\:mocks +deps\:mocks: go install github.com/vektra/mockery/v2@v2.32.0 .PHONY: deps\:playground @@ -173,11 +173,11 @@ deps: $(MAKE) deps:chglog && \ $(MAKE) deps:lint && \ $(MAKE) deps:test && \ - $(MAKE) deps:mock + $(MAKE) deps:mocks -.PHONY: mock -mock: - @$(MAKE) deps:mock +.PHONY: mocks +mocks: + @$(MAKE) deps:mocks mockery --config="tools/configs/mockery.yaml" .PHONY: dev\:start diff --git a/tests/README.md b/tests/README.md index a17fbcaf08..2dfe2d17c7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -23,7 +23,7 @@ These tests focus on small, isolated parts of the code to ensure each part is wo For unit tests, we sometimes use mocks. Mocks are automatically generated from Go interfaces using the mockery tool. This helps to isolate the code being tested and provide more focused and reliable tests. -To regenerate the mocks, run `make mock`. +To regenerate the mocks, run `make mocks`. The mocks are typically generated into a separate mocks directory. From dd0e5af48f19ee58724256563f4beb4a888bdd36 Mon Sep 17 00:00:00 2001 From: AndrewSisley Date: Wed, 5 Jun 2024 17:51:41 -0400 Subject: [PATCH 3/3] feat: Allow lens runtime selection via config (#2684) ## Relevant issue(s) Resolves #2683 ## Description Allows lens runtime selection via config/cli param. Also adds CI jobs to the matrix to test wasmer and wazero. --- .../workflows/test-and-upload-coverage.yml | 19 ++++ cli/server_dump.go | 2 +- cli/start.go | 1 + client/lens.go | 11 +++ docs/config.md | 9 ++ go.mod | 2 + go.sum | 4 + http/client_lens.go | 2 + http/handler_ccip_test.go | 2 +- internal/db/config.go | 17 ---- internal/db/config_test.go | 13 --- internal/db/db.go | 27 +++--- internal/db/db_test.go | 4 +- internal/lens/registry.go | 25 +----- internal/lens/txn_registry.go | 11 ++- net/node_test.go | 20 ++--- net/peer_test.go | 18 ++-- node/errors.go | 25 ++++++ node/lens.go | 90 +++++++++++++++++++ node/lens_wasmer.go | 24 +++++ node/lens_wasmtime.go | 25 ++++++ node/lens_wazero.go | 24 +++++ node/node.go | 11 ++- tests/clients/cli/wrapper_lens.go | 2 + tests/gen/cli/util_test.go | 2 +- tests/integration/db.go | 3 +- tests/integration/lens.go | 15 ++++ tools/defradb.containerfile | 5 ++ 28 files changed, 315 insertions(+), 98 deletions(-) create mode 100644 node/errors.go create mode 100644 node/lens.go create mode 100644 node/lens_wasmer.go create mode 100644 node/lens_wasmtime.go create mode 100644 node/lens_wazero.go diff --git a/.github/workflows/test-and-upload-coverage.yml b/.github/workflows/test-and-upload-coverage.yml index d613f32f98..f20860bb7c 100644 --- a/.github/workflows/test-and-upload-coverage.yml +++ b/.github/workflows/test-and-upload-coverage.yml @@ -34,6 +34,7 @@ jobs: client-type: [go, http, cli] database-type: [badger-file, badger-memory] mutation-type: [gql, collection-named, collection-save] + lens-type: [wasm-time] detect-changes: [false] database-encryption: [false] include: @@ -41,18 +42,21 @@ jobs: client-type: go database-type: badger-memory mutation-type: collection-save + lens-type: wasm-time detect-changes: true database-encryption: false - os: ubuntu-latest client-type: go database-type: badger-memory mutation-type: collection-save + lens-type: wasm-time detect-changes: false database-encryption: true - os: macos-latest client-type: go database-type: badger-memory mutation-type: collection-save + lens-type: wasm-time detect-changes: false database-encryption: false ## TODO: https://github.com/sourcenetwork/defradb/issues/2080 @@ -63,6 +67,20 @@ jobs: ## mutation-type: collection-save ## detect-changes: false ## database-encryption: false + - os: ubuntu-latest + client-type: go + database-type: badger-memory + mutation-type: collection-save + lens-type: wazero + detect-changes: false + database-encryption: false + - os: ubuntu-latest + client-type: go + database-type: badger-memory + mutation-type: collection-save + lens-type: wasmer + detect-changes: false + database-encryption: false runs-on: ${{ matrix.os }} @@ -80,6 +98,7 @@ jobs: DEFRA_BADGER_FILE: ${{ matrix.database-type == 'badger-file' }} DEFRA_BADGER_ENCRYPTION: ${{ matrix.database-encryption }} DEFRA_MUTATION_TYPE: ${{ matrix.mutation-type }} + DEFRA_LENS_TYPE: ${{ matrix.lens-type }} steps: - name: Checkout code into the directory diff --git a/cli/server_dump.go b/cli/server_dump.go index 1d3c68e54a..9008c81730 100644 --- a/cli/server_dump.go +++ b/cli/server_dump.go @@ -37,7 +37,7 @@ func MakeServerDumpCmd() *cobra.Command { if err != nil { return err } - db, err := db.NewDB(cmd.Context(), rootstore, acp.NoACP) + db, err := db.NewDB(cmd.Context(), rootstore, acp.NoACP, nil) if err != nil { return errors.Wrap("failed to initialize database", err) } diff --git a/cli/start.go b/cli/start.go index 9b863a1f07..9505fd7fff 100644 --- a/cli/start.go +++ b/cli/start.go @@ -126,6 +126,7 @@ func MakeStartCommand() *cobra.Command { http.WithAllowedOrigins(cfg.GetStringSlice("api.allowed-origins")...), http.WithTLSCertPath(cfg.GetString("api.pubKeyPath")), http.WithTLSKeyPath(cfg.GetString("api.privKeyPath")), + node.WithLensRuntime(node.LensRuntimeType(cfg.GetString("lens.runtime"))), } if cfg.GetString("datastore.store") != configStoreMemory { diff --git a/client/lens.go b/client/lens.go index 3f5befc604..997ddb4831 100644 --- a/client/lens.go +++ b/client/lens.go @@ -15,6 +15,8 @@ import ( "github.com/lens-vm/lens/host-go/config/model" "github.com/sourcenetwork/immutable/enumerable" + + "github.com/sourcenetwork/defradb/datastore" ) // LensConfig represents the configuration of a Lens migration in Defra. @@ -38,9 +40,18 @@ type LensConfig struct { model.Lens } +// TxnSource represents an object capable of constructing the transactions that +// implicit-transaction registries need internally. +type TxnSource interface { + NewTxn(context.Context, bool) (datastore.Txn, error) +} + // LensRegistry exposes several useful thread-safe migration related functions which may // be used to manage migrations. type LensRegistry interface { + // Init initializes the registry with the provided transaction source. + Init(TxnSource) + // SetMigration caches the migration for the given collection ID. It does not persist the migration in long // term storage, for that one should call [Store.SetMigration(ctx, cfg)]. // diff --git a/docs/config.md b/docs/config.md index 0981ad9a2e..ca69d6afd2 100644 --- a/docs/config.md +++ b/docs/config.md @@ -111,3 +111,12 @@ Keyring backend to use. Defaults to `file`. - `file` Stores keys in encrypted files - `system` Stores keys in the OS managed keyring + +## `lens.runtime` + +The LensVM wasm runtime to run lens modules in. + +Possible values: +- `wasm-time` (default): https://github.com/bytecodealliance/wasmtime-go +- `wasmer` (windows not supported): https://github.com/wasmerio/wasmer-go +- `wazero`: https://github.com/tetratelabs/wazero diff --git a/go.mod b/go.mod index 5e47eeadf2..f9bac2daa8 100644 --- a/go.mod +++ b/go.mod @@ -282,8 +282,10 @@ require ( github.com/tendermint/go-amino v0.16.0 // indirect github.com/tendermint/tm-db v0.6.7 // indirect github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8 // indirect + github.com/tetratelabs/wazero v1.5.0 // indirect github.com/textileio/go-log/v2 v2.1.3-gke-2 // indirect github.com/ugorji/go/codec v1.2.12 // indirect + github.com/wasmerio/wasmer-go v1.0.4 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect diff --git a/go.sum b/go.sum index 64f0b7b56b..3203ff8b9c 100644 --- a/go.sum +++ b/go.sum @@ -1139,6 +1139,8 @@ github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8 h1:RBkacARv7qY5laaXGlF4wFB/tk5rnthhPb8oIBGoagY= github.com/teserakt-io/golang-ed25519 v0.0.0-20210104091850-3888c087a4c8/go.mod h1:9PdLyPiZIiW3UopXyRnPYyjUXSpiQNHRLu8fOsR3o8M= +github.com/tetratelabs/wazero v1.5.0 h1:Yz3fZHivfDiZFUXnWMPUoiW7s8tC1sjdBtlJn08qYa0= +github.com/tetratelabs/wazero v1.5.0/go.mod h1:0U0G41+ochRKoPKCJlh0jMg1CHkyfK8kDqiirMmKY8A= github.com/textileio/go-datastore-extensions v1.0.1 h1:qIJGqJaigQ1wD4TdwS/hf73u0HChhXvvUSJuxBEKS+c= github.com/textileio/go-datastore-extensions v1.0.1/go.mod h1:Pzj9FDRkb55910dr/FX8M7WywvnS26gBgEDez1ZBuLE= github.com/textileio/go-ds-badger3 v0.1.0 h1:q0kBuBmAcRUR3ClMSYlyw0224XeuzjjGinU53Qz1uXI= @@ -1168,6 +1170,8 @@ github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsX github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/wasmerio/wasmer-go v1.0.4 h1:MnqHoOGfiQ8MMq2RF6wyCeebKOe84G88h5yv+vmxJgs= +github.com/wasmerio/wasmer-go v1.0.4/go.mod h1:0gzVdSfg6pysA6QVp6iVRPTagC6Wq9pOE8J86WKb2Fk= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4= github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= diff --git a/http/client_lens.go b/http/client_lens.go index 34945a41d6..249eb99984 100644 --- a/http/client_lens.go +++ b/http/client_lens.go @@ -35,6 +35,8 @@ type setMigrationRequest struct { Config model.Lens } +func (w *LensRegistry) Init(txnSource client.TxnSource) {} + func (c *LensRegistry) SetMigration(ctx context.Context, collectionID uint32, config model.Lens) error { methodURL := c.http.baseURL.JoinPath("lens", "registry") diff --git a/http/handler_ccip_test.go b/http/handler_ccip_test.go index ab8381565a..e17d8a882a 100644 --- a/http/handler_ccip_test.go +++ b/http/handler_ccip_test.go @@ -193,7 +193,7 @@ func TestCCIPPost_WithInvalidBody(t *testing.T) { func setupDatabase(t *testing.T) client.DB { ctx := context.Background() - cdb, err := db.NewDB(ctx, memory.NewDatastore(ctx), acp.NoACP, db.WithUpdateEvents()) + cdb, err := db.NewDB(ctx, memory.NewDatastore(ctx), acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) _, err = cdb.AddSchema(ctx, `type User { diff --git a/internal/db/config.go b/internal/db/config.go index a655647df7..1364cab09b 100644 --- a/internal/db/config.go +++ b/internal/db/config.go @@ -11,7 +11,6 @@ package db import ( - "github.com/lens-vm/lens/host-go/engine/module" "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/events" @@ -40,19 +39,3 @@ func WithMaxRetries(num int) Option { db.maxTxnRetries = immutable.Some(num) } } - -// WithLensPoolSize sets the maximum number of cached migrations instances to preserve per schema version. -// -// Will default to `5` if not set. -func WithLensPoolSize(size int) Option { - return func(db *db) { - db.lensPoolSize = immutable.Some(size) - } -} - -// WithLensRuntime returns an option that sets the lens registry runtime. -func WithLensRuntime(runtime module.Runtime) Option { - return func(db *db) { - db.lensRuntime = immutable.Some(runtime) - } -} diff --git a/internal/db/config_test.go b/internal/db/config_test.go index d4dbadaec6..f80e538b4f 100644 --- a/internal/db/config_test.go +++ b/internal/db/config_test.go @@ -13,7 +13,6 @@ package db import ( "testing" - "github.com/lens-vm/lens/host-go/runtimes/wasmtime" "github.com/stretchr/testify/assert" ) @@ -29,15 +28,3 @@ func TestWithMaxRetries(t *testing.T) { assert.True(t, d.maxTxnRetries.HasValue()) assert.Equal(t, 10, d.maxTxnRetries.Value()) } - -func TestWithLensPoolSize(t *testing.T) { - d := &db{} - WithLensPoolSize(10)(d) - assert.Equal(t, 10, d.lensPoolSize.Value()) -} - -func TestWithLensRuntime(t *testing.T) { - d := &db{} - WithLensRuntime(wasmtime.New())(d) - assert.NotNil(t, d.lensRuntime.Value()) -} diff --git a/internal/db/db.go b/internal/db/db.go index 4379928c82..979626034c 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -21,7 +21,6 @@ import ( ds "github.com/ipfs/go-datastore" dsq "github.com/ipfs/go-datastore/query" - "github.com/lens-vm/lens/host-go/engine/module" "github.com/sourcenetwork/corelog" "github.com/sourcenetwork/immutable" @@ -32,7 +31,6 @@ import ( "github.com/sourcenetwork/defradb/errors" "github.com/sourcenetwork/defradb/events" "github.com/sourcenetwork/defradb/internal/core" - "github.com/sourcenetwork/defradb/internal/lens" "github.com/sourcenetwork/defradb/internal/request/graphql" ) @@ -57,10 +55,6 @@ type db struct { parser core.Parser - // The maximum number of cached migrations instances to preserve per schema version. - lensPoolSize immutable.Option[int] - lensRuntime immutable.Option[module.Runtime] - lensRegistry client.LensRegistry // The maximum number of retries per transaction. @@ -81,15 +75,17 @@ func NewDB( ctx context.Context, rootstore datastore.RootStore, acp immutable.Option[acp.ACP], + lens client.LensRegistry, options ...Option, ) (client.DB, error) { - return newDB(ctx, rootstore, acp, options...) + return newDB(ctx, rootstore, acp, lens, options...) } func newDB( ctx context.Context, rootstore datastore.RootStore, acp immutable.Option[acp.ACP], + lens client.LensRegistry, options ...Option, ) (*db, error) { multistore := datastore.MultiStoreFrom(rootstore) @@ -100,11 +96,12 @@ func newDB( } db := &db{ - rootstore: rootstore, - multistore: multistore, - acp: acp, - parser: parser, - options: options, + rootstore: rootstore, + multistore: multistore, + acp: acp, + lensRegistry: lens, + parser: parser, + options: options, } // apply options @@ -112,9 +109,9 @@ func newDB( opt(db) } - // lens options may be set by `WithLens` funcs, and because they are funcs on db - // we have to mutate `db` here to set the registry. - db.lensRegistry = lens.NewRegistry(db, db.lensPoolSize, db.lensRuntime) + if lens != nil { + lens.Init(db) + } err = db.initialize(ctx) if err != nil { diff --git a/internal/db/db_test.go b/internal/db/db_test.go index fe60449cc2..6f5a03e809 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -26,7 +26,7 @@ func newMemoryDB(ctx context.Context) (*db, error) { if err != nil { return nil, err } - return newDB(ctx, rootstore, acp.NoACP) + return newDB(ctx, rootstore, acp.NoACP, nil) } func TestNewDB(t *testing.T) { @@ -38,7 +38,7 @@ func TestNewDB(t *testing.T) { return } - _, err = NewDB(ctx, rootstore, acp.NoACP) + _, err = NewDB(ctx, rootstore, acp.NoACP, nil) if err != nil { t.Error(err) } diff --git a/internal/lens/registry.go b/internal/lens/registry.go index 1d9c51ab46..c0fc87a14f 100644 --- a/internal/lens/registry.go +++ b/internal/lens/registry.go @@ -17,8 +17,6 @@ import ( "github.com/lens-vm/lens/host-go/config" "github.com/lens-vm/lens/host-go/config/model" "github.com/lens-vm/lens/host-go/engine/module" - "github.com/lens-vm/lens/host-go/runtimes/wasmtime" - "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/immutable/enumerable" "github.com/sourcenetwork/defradb/client" @@ -72,12 +70,6 @@ func newTxnCtx(txn datastore.Txn) *txnContext { } } -// TxnSource represents an object capable of constructing the transactions that -// implicit-transaction registries need internally. -type TxnSource interface { - NewTxn(context.Context, bool) (datastore.Txn, error) -} - // DefaultPoolSize is the default size of the lens pool for each schema version. const DefaultPoolSize int = 5 @@ -85,28 +77,19 @@ const DefaultPoolSize int = 5 // // It will be of size 5 (per schema version) if a size is not provided. func NewRegistry( - db TxnSource, - poolSize immutable.Option[int], - runtime immutable.Option[module.Runtime], + poolSize int, + runtime module.Runtime, ) client.LensRegistry { registry := &lensRegistry{ - poolSize: DefaultPoolSize, - runtime: wasmtime.New(), + poolSize: poolSize, + runtime: runtime, modulesByPath: map[string]module.Module{}, lensPoolsByCollectionID: map[uint32]*lensPool{}, reversedPoolsByCollectionID: map[uint32]*lensPool{}, txnCtxs: map[uint64]*txnContext{}, } - if poolSize.HasValue() { - registry.poolSize = poolSize.Value() - } - if runtime.HasValue() { - registry.runtime = runtime.Value() - } - return &implicitTxnLensRegistry{ - db: db, registry: registry, } } diff --git a/internal/lens/txn_registry.go b/internal/lens/txn_registry.go index 8093dedbdd..65ad12cf2b 100644 --- a/internal/lens/txn_registry.go +++ b/internal/lens/txn_registry.go @@ -22,7 +22,7 @@ import ( type implicitTxnLensRegistry struct { registry *lensRegistry - db TxnSource + db client.TxnSource } type explicitTxnLensRegistry struct { @@ -33,13 +33,12 @@ type explicitTxnLensRegistry struct { var _ client.LensRegistry = (*implicitTxnLensRegistry)(nil) var _ client.LensRegistry = (*explicitTxnLensRegistry)(nil) -func (r *implicitTxnLensRegistry) WithTxn(txn datastore.Txn) client.LensRegistry { - return &explicitTxnLensRegistry{ - registry: r.registry, - txn: txn, - } +func (r *implicitTxnLensRegistry) Init(txnSource client.TxnSource) { + r.db = txnSource } +func (r *explicitTxnLensRegistry) Init(txnSource client.TxnSource) {} + func (r *explicitTxnLensRegistry) WithTxn(txn datastore.Txn) client.LensRegistry { return &explicitTxnLensRegistry{ registry: r.registry, diff --git a/net/node_test.go b/net/node_test.go index 5e0b30570f..55b0573474 100644 --- a/net/node_test.go +++ b/net/node_test.go @@ -36,7 +36,7 @@ func FixtureNewMemoryDBWithBroadcaster(t *testing.T) client.DB { opts := badgerds.Options{Options: badger.DefaultOptions("").WithInMemory(true)} rootstore, err := badgerds.NewDatastore("", &opts) require.NoError(t, err) - database, err = db.NewDB(ctx, rootstore, acp.NoACP, db.WithUpdateEvents()) + database, err = db.NewDB(ctx, rootstore, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) return database } @@ -44,7 +44,7 @@ func FixtureNewMemoryDBWithBroadcaster(t *testing.T) client.DB { func TestNewNode_WithEnableRelay_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n, err := NewNode( context.Background(), @@ -59,7 +59,7 @@ func TestNewNode_WithDBClosed_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) db.Close() @@ -73,7 +73,7 @@ func TestNewNode_WithDBClosed_NoError(t *testing.T) { func TestNewNode_NoPubSub_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n, err := NewNode( context.Background(), @@ -88,7 +88,7 @@ func TestNewNode_NoPubSub_NoError(t *testing.T) { func TestNewNode_WithEnablePubSub_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n, err := NewNode( @@ -106,7 +106,7 @@ func TestNewNode_WithEnablePubSub_NoError(t *testing.T) { func TestNodeClose_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n, err := NewNode( context.Background(), @@ -119,7 +119,7 @@ func TestNodeClose_NoError(t *testing.T) { func TestNewNode_BootstrapWithNoPeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n1, err := NewNode( @@ -135,7 +135,7 @@ func TestNewNode_BootstrapWithNoPeer_NoError(t *testing.T) { func TestNewNode_BootstrapWithOnePeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n1, err := NewNode( @@ -162,7 +162,7 @@ func TestNewNode_BootstrapWithOnePeer_NoError(t *testing.T) { func TestNewNode_BootstrapWithOneValidPeerAndManyInvalidPeers_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n1, err := NewNode( @@ -192,7 +192,7 @@ func TestNewNode_BootstrapWithOneValidPeerAndManyInvalidPeers_NoError(t *testing func TestListenAddrs_WithListenAddresses_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n, err := NewNode( context.Background(), diff --git a/net/peer_test.go b/net/peer_test.go index e708ff0708..dca864a1e3 100644 --- a/net/peer_test.go +++ b/net/peer_test.go @@ -75,7 +75,7 @@ func newTestNode(ctx context.Context, t *testing.T) (client.DB, *Node) { store := memory.NewDatastore(ctx) acpLocal := acp.NewLocalACP() acpLocal.Init(context.Background(), "") - db, err := db.NewDB(ctx, store, immutable.Some[acp.ACP](acpLocal), db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, immutable.Some[acp.ACP](acpLocal), nil, db.WithUpdateEvents()) require.NoError(t, err) n, err := NewNode( @@ -91,7 +91,7 @@ func newTestNode(ctx context.Context, t *testing.T) (client.DB, *Node) { func TestNewPeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) h, err := libp2p.New() @@ -114,7 +114,7 @@ func TestNewPeer_NoDB_NilDBError(t *testing.T) { func TestNewPeer_WithExistingTopic_TopicAlreadyExistsError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) _, err = db.AddSchema(ctx, `type User { @@ -164,11 +164,11 @@ func TestStartAndClose_NoError(t *testing.T) { func TestStart_WithKnownPeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db1, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db1, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) store2 := memory.NewDatastore(ctx) - db2, err := db.NewDB(ctx, store2, acp.NoACP, db.WithUpdateEvents()) + db2, err := db.NewDB(ctx, store2, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n1, err := NewNode( @@ -200,11 +200,11 @@ func TestStart_WithKnownPeer_NoError(t *testing.T) { func TestStart_WithOfflineKnownPeer_NoError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db1, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db1, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) store2 := memory.NewDatastore(ctx) - db2, err := db.NewDB(ctx, store2, acp.NoACP, db.WithUpdateEvents()) + db2, err := db.NewDB(ctx, store2, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n1, err := NewNode( @@ -240,7 +240,7 @@ func TestStart_WithOfflineKnownPeer_NoError(t *testing.T) { func TestStart_WithNoUpdateChannel_NilUpdateChannelError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP) + db, err := db.NewDB(ctx, store, acp.NoACP, nil) require.NoError(t, err) n, err := NewNode( @@ -259,7 +259,7 @@ func TestStart_WithNoUpdateChannel_NilUpdateChannelError(t *testing.T) { func TestStart_WitClosedUpdateChannel_ClosedChannelError(t *testing.T) { ctx := context.Background() store := memory.NewDatastore(ctx) - db, err := db.NewDB(ctx, store, acp.NoACP, db.WithUpdateEvents()) + db, err := db.NewDB(ctx, store, acp.NoACP, nil, db.WithUpdateEvents()) require.NoError(t, err) n, err := NewNode( diff --git a/node/errors.go b/node/errors.go new file mode 100644 index 0000000000..d19b53359b --- /dev/null +++ b/node/errors.go @@ -0,0 +1,25 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package node + +import ( + "github.com/sourcenetwork/defradb/errors" +) + +const ( + errLensRuntimeNotSupported string = "the selected lens runtime is not supported by this build" +) + +var ErrLensRuntimeNotSupported = errors.New(errLensRuntimeNotSupported) + +func NewErrLensRuntimeNotSupported(lens LensRuntimeType) error { + return errors.New(errLensRuntimeNotSupported, errors.NewKV("Lens", lens)) +} diff --git a/node/lens.go b/node/lens.go new file mode 100644 index 0000000000..aa8bfcc8d0 --- /dev/null +++ b/node/lens.go @@ -0,0 +1,90 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +package node + +import ( + "context" + + "github.com/lens-vm/lens/host-go/engine/module" + + "github.com/sourcenetwork/defradb/client" + "github.com/sourcenetwork/defradb/internal/lens" +) + +type LensRuntimeType string + +const ( + // The Go-enum default LensRuntimeType. + // + // The actual runtime type that this resolves to depends on the build target. + DefaultLens LensRuntimeType = "" +) + +// runtimeConstructors is a map of [LensRuntimeType]s to lens runtimes. +// +// Is is populated by the `init` functions in the runtime-specific files - this +// allows it's population to be managed by build flags. +var runtimeConstructors = map[LensRuntimeType]func() module.Runtime{} + +// LensOptions contains Lens configuration values. +type LensOptions struct { + lensRuntime LensRuntimeType + + // The maximum number of cached migrations instances to preserve per schema version. + lensPoolSize int +} + +// DefaultACPOptions returns new options with default values. +func DefaultLensOptions() *LensOptions { + return &LensOptions{ + lensPoolSize: lens.DefaultPoolSize, + } +} + +type LenOpt func(*LensOptions) + +// WithLensRuntime returns an option that sets the lens registry runtime. +func WithLensRuntime(runtime LensRuntimeType) Option { + return func(o *LensOptions) { + o.lensRuntime = runtime + } +} + +// WithLensPoolSize sets the maximum number of cached migrations instances to preserve per schema version. +// +// Will default to `5` if not set. +func WithLensPoolSize(size int) Option { + return func(o *LensOptions) { + o.lensPoolSize = size + } +} + +func NewLens( + ctx context.Context, + opts ...LenOpt, +) (client.LensRegistry, error) { + options := DefaultLensOptions() + for _, opt := range opts { + opt(options) + } + + var runtime module.Runtime + if runtimeConstructor, ok := runtimeConstructors[options.lensRuntime]; ok { + runtime = runtimeConstructor() + } else { + return nil, NewErrLensRuntimeNotSupported(options.lensRuntime) + } + + return lens.NewRegistry( + options.lensPoolSize, + runtime, + ), nil +} diff --git a/node/lens_wasmer.go b/node/lens_wasmer.go new file mode 100644 index 0000000000..fd99357ab7 --- /dev/null +++ b/node/lens_wasmer.go @@ -0,0 +1,24 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +//go:build !windows && !js + +package node + +import ( + "github.com/lens-vm/lens/host-go/engine/module" + "github.com/lens-vm/lens/host-go/runtimes/wasmer" +) + +const Wasmer LensRuntimeType = "wasmer" + +func init() { + runtimeConstructors[Wasmer] = func() module.Runtime { return wasmer.New() } +} diff --git a/node/lens_wasmtime.go b/node/lens_wasmtime.go new file mode 100644 index 0000000000..9f0070f3bf --- /dev/null +++ b/node/lens_wasmtime.go @@ -0,0 +1,25 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +//go:build !js + +package node + +import ( + "github.com/lens-vm/lens/host-go/engine/module" + "github.com/lens-vm/lens/host-go/runtimes/wasmtime" +) + +const WasmTime LensRuntimeType = "wasm-time" + +func init() { + runtimeConstructors[DefaultLens] = func() module.Runtime { return wasmtime.New() } + runtimeConstructors[WasmTime] = func() module.Runtime { return wasmtime.New() } +} diff --git a/node/lens_wazero.go b/node/lens_wazero.go new file mode 100644 index 0000000000..40d3f1b056 --- /dev/null +++ b/node/lens_wazero.go @@ -0,0 +1,24 @@ +// Copyright 2024 Democratized Data Foundation +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + +//go:build !js + +package node + +import ( + "github.com/lens-vm/lens/host-go/engine/module" + "github.com/lens-vm/lens/host-go/runtimes/wazero" +) + +const Wazero LensRuntimeType = "wazero" + +func init() { + runtimeConstructors[Wazero] = func() module.Runtime { return wazero.New() } +} diff --git a/node/node.go b/node/node.go index bb3163834c..215cf05fc7 100644 --- a/node/node.go +++ b/node/node.go @@ -89,6 +89,7 @@ func NewNode(ctx context.Context, opts ...Option) (*Node, error) { netOpts []net.NodeOpt storeOpts []StoreOpt serverOpts []http.ServerOpt + lensOpts []LenOpt ) options := DefaultOptions() @@ -111,6 +112,9 @@ func NewNode(ctx context.Context, opts ...Option) (*Node, error) { case net.NodeOpt: netOpts = append(netOpts, t) + + case LenOpt: + lensOpts = append(lensOpts, t) } } @@ -124,7 +128,12 @@ func NewNode(ctx context.Context, opts ...Option) (*Node, error) { return nil, err } - db, err := db.NewDB(ctx, rootstore, acp, dbOpts...) + lens, err := NewLens(ctx, lensOpts...) + if err != nil { + return nil, err + } + + db, err := db.NewDB(ctx, rootstore, acp, lens, dbOpts...) if err != nil { return nil, err } diff --git a/tests/clients/cli/wrapper_lens.go b/tests/clients/cli/wrapper_lens.go index a9f3e20bd1..3aac1ae392 100644 --- a/tests/clients/cli/wrapper_lens.go +++ b/tests/clients/cli/wrapper_lens.go @@ -28,6 +28,8 @@ type LensRegistry struct { cmd *cliWrapper } +func (w *LensRegistry) Init(txnSource client.TxnSource) {} + func (w *LensRegistry) SetMigration(ctx context.Context, collectionID uint32, config model.Lens) error { args := []string{"client", "schema", "migration", "set-registry"} diff --git a/tests/gen/cli/util_test.go b/tests/gen/cli/util_test.go index dbfef99524..6f5535e6c6 100644 --- a/tests/gen/cli/util_test.go +++ b/tests/gen/cli/util_test.go @@ -50,7 +50,7 @@ func start(ctx context.Context) (*defraInstance, error) { return nil, errors.Wrap("failed to open datastore", err) } - db, err := db.NewDB(ctx, rootstore, acp.NoACP) + db, err := db.NewDB(ctx, rootstore, acp.NoACP, nil) if err != nil { return nil, errors.Wrap("failed to create a database", err) } diff --git a/tests/integration/db.go b/tests/integration/db.go index db2217a04d..c473e4cdd0 100644 --- a/tests/integration/db.go +++ b/tests/integration/db.go @@ -105,13 +105,14 @@ func NewBadgerFileDB(ctx context.Context, t testing.TB) (client.DB, error) { func setupDatabase(s *state) (client.DB, string, error) { opts := []node.Option{ db.WithUpdateEvents(), - db.WithLensPoolSize(lensPoolSize), + node.WithLensPoolSize(lensPoolSize), // The test framework sets this up elsewhere when required so that it may be wrapped // into a [client.DB]. node.WithDisableAPI(true), // The p2p is configured in the tests by [ConfigureNode] actions, we disable it here // to keep the tests as lightweight as possible. node.WithDisableP2P(true), + node.WithLensRuntime(lensType), } if badgerEncryption && encryptionKey == nil { diff --git a/tests/integration/lens.go b/tests/integration/lens.go index c99cc3f5b7..61ece97d73 100644 --- a/tests/integration/lens.go +++ b/tests/integration/lens.go @@ -11,12 +11,27 @@ package tests import ( + "os" + "github.com/sourcenetwork/immutable" "github.com/sourcenetwork/defradb/client" "github.com/sourcenetwork/defradb/internal/db" + "github.com/sourcenetwork/defradb/node" +) + +const ( + lensTypeEnvName = "DEFRA_LENS_TYPE" ) +var ( + lensType node.LensRuntimeType +) + +func init() { + lensType = node.LensRuntimeType(os.Getenv(lensTypeEnvName)) +} + // ConfigureMigration is a test action which will configure a Lens migration using the // provided configuration. type ConfigureMigration struct { diff --git a/tools/defradb.containerfile b/tools/defradb.containerfile index 53a849b6c6..7e07bf887d 100644 --- a/tools/defradb.containerfile +++ b/tools/defradb.containerfile @@ -18,11 +18,16 @@ RUN make deps:modules COPY . . COPY --from=PLAYGROUND_BUILD /repo/dist /repo/playground/dist/ ENV BUILD_TAGS=playground +# manually copy libwasmer.so to fix linking issue https://github.com/wasmerio/wasmer-go/issues/281 +RUN export WASMER_ARCH=$(go env GOHOSTARCH | sed "s/arm64/aarch64/") && \ + export WASMER_PATH=$(go env GOMODCACHE)/github.com/wasmerio/wasmer-go@v1.0.4/wasmer/packaged/lib/linux-$WASMER_ARCH/libwasmer.so && \ + cp $WASMER_PATH /lib/libwasmer.so RUN make build # Stage: RUN FROM debian:bookworm-slim COPY --from=BUILD /repo/build/defradb /defradb +COPY --from=BUILD /lib/libwasmer.so /lib/libwasmer.so # Documents which ports are normally used. # To publish the ports: `docker run -p 9181:9181` ...