Merge branch 'master' into frifri/rbac-iceberg-v2

airbyte-ci/connectors/pipelines/README.md

@@ -854,6 +854,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only
 
 | Version | PR                                                         | Description                                                                                                                  |
 |---------|------------------------------------------------------------| ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.48.3  | [#50988](https://github.com/airbytehq/airbyte/pull/50988)  | Remove deprecated `--no-update` flag from poetry commands |
 | 4.48.2  | [#50871](https://github.com/airbytehq/airbyte/pull/50871)  | Speed up connector modification detection.                                                                                   |
 | 4.48.1  | [#50410](https://github.com/airbytehq/airbyte/pull/50410)  | Java connector build: give ownership of built artifacts to the current image user.                                                                                          |
 | 4.48.0  | [#49960](https://github.com/airbytehq/airbyte/pull/49960)  | Deprecate airbyte-ci format command                                                                                          |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/generate_erd/pipeline.py

@@ -100,9 +100,7 @@ def _build_erd_container(self, connector_directory: Directory, discovered_catalo
             .with_workdir("/app")
         )
 
-        return container.with_exec(["poetry", "lock", "--no-update"], use_entrypoint=True).with_exec(
-            ["poetry", "install"], use_entrypoint=True
-        )
+        return container.with_exec(["poetry", "lock"], use_entrypoint=True).with_exec(["poetry", "install"], use_entrypoint=True)
 
 
 class UploadDbmlSchema(Step):

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/test/steps/common.py

@@ -828,7 +828,5 @@ async def _build_test_container(self, target_container_id: str) -> Container:
                 )
             )
 
-        container = container.with_exec(["poetry", "lock", "--no-update"], use_entrypoint=True).with_exec(
-            ["poetry", "install"], use_entrypoint=True
-        )
+        container = container.with_exec(["poetry", "lock"], use_entrypoint=True).with_exec(["poetry", "install"], use_entrypoint=True)
         return container

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/upgrade_cdk/pipeline.py

@@ -139,7 +139,7 @@ async def upgrade_cdk_version_for_python_connector(self, og_connector_dir: Direc
         connector_container = base_container.with_mounted_directory("/connector", updated_connector_dir).with_workdir("/connector")
 
         poetry_lock_file = await connector_container.file(POETRY_LOCK_FILENAME).contents()
-        updated_container = await connector_container.with_exec(["poetry", "lock", "--no-update"], use_entrypoint=True)
+        updated_container = await connector_container.with_exec(["poetry", "lock"], use_entrypoint=True)
         updated_poetry_lock_file = await updated_container.file(POETRY_LOCK_FILENAME).contents()
 
         if poetry_lock_file != updated_poetry_lock_file:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py

@@ -239,7 +239,7 @@ def prepare_container_for_poe_tasks(
                 ],
                 use_entrypoint=True,
             )
-            .with_exec(["poetry", "lock", "--no-update"], use_entrypoint=True)
+            .with_exec(["poetry", "lock"], use_entrypoint=True)
         )
 
     # Install the poetry package

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.48.2"
+version = "4.48.3"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-integrations/connectors/source-mysql/build.gradle

@@ -9,7 +9,7 @@ application {
 airbyteBulkConnector {
     core = 'extract'
     toolkits = ['extract-jdbc', 'extract-cdc']
-    cdk = '0.255'
+    cdk = '0.257'
 }
 
 dependencies {

airbyte-integrations/connectors/source-mysql/metadata.yaml

@@ -9,7 +9,7 @@ data:
   connectorSubtype: database
   connectorType: source
   definitionId: 435bb9a5-7887-4809-aa58-28c27df0d7ad
-  dockerImageTag: 3.10.0-rc.8
+  dockerImageTag: 3.10.0-rc.9
   dockerRepository: airbyte/source-mysql
   documentationUrl: https://docs.airbyte.com/integrations/sources/mysql
   githubIssueLabel: source-mysql

docs/api-documentation.md

@@ -21,6 +21,6 @@ Navigate to our full API documentation to learn how to retrieve your access toke
 Our full API documentation is located here: [api.airbyte.com](https://reference.airbyte.com/reference/getting-started).
 
 ## Configuration API (Deprecated)
-The configuration API is now deprecated and no longer supported. It is an internal API that is designed for communications between different Airbyte components ratther than managing your Airbyte workspace.
+The configuration API is now deprecated and no longer supported. It is an internal API that is designed for communications between different Airbyte components rather than managing your Airbyte workspace.
 
-Users utilize the Config API at their own risk. This API is utilized internally by the Airbyte Engineering team and may be modified in the future if the need arises. 
+Users utilize the Config API at their own risk. This API is utilized internally by the Airbyte Engineering team and may be modified in the future if the need arises. 

docs/connector-development/writing-connector-docs.md

@@ -1,29 +1,105 @@
 # Writing Connector Documentation
 
-This document provides guidance on tools and techniques specifically for writing documentation for Airbyte connectors.
+This topic guides you through writing documentation for Airbyte connectors. The systems and practices described in [Updating Documentation](../contributing-to-airbyte/writing-docs.md) apply here as well. However, there are several features and restrictions that only apply to connectors.
 
-For more information about writing documentation at Airbyte generally, please also see the [Contributing Guide for Airbyte Documentation](../contributing-to-airbyte/writing-docs.md).
+## QA checks
 
-## Custom markdown extensions for connector docs
+If you're writing docs for a new connector, your docs must pass our [QA checks](../contributing-to-airbyte/resources/qa-checks).
 
-Airbyte's markdown documentation—particularly connector-specific documentation—needs to gracefully support multiple different contexts: key details may differ between open-source builds and Airbyte Cloud, and the more exhaustive explanations appropriate for https://docs.airbyte.com may bury key details when rendered as inline documentation within the Airbyte application. In order to support all these different contexts without resorting to multiple overlapping files that must be maintained in parallel, Airbyte's documentation tooling supports multiple nonstandard features.
+## Custom Markdown extensions for connector docs
 
-Please familiarize yourself with all the tools available to you when writing documentation for a connector, so that you can provide appropriately tailored information to your readers in whichever context they see it.
+Airbyte's connector documentation must gracefully support different contexts in a way platform documentation doesn't.
 
-:::note
-As a general rule, features that introduce new behavior or prevent certain content from rendering will affect how the Airbyte UI displays markdown content, but have no impact on https://docs.airbyte.com. If you want to test out these in-app features in [a local Airbyte build](https://docs.airbyte.com/contributing-to-airbyte/developing-locally/#develop-on-airbyte-webapp), ensure that you have the `airbyte` git repository checked out to the same parent directory as the airbyte platform repository: if so, development builds will by default fetch connector documentation from your local filesystem, allowing you to freely edit their content and view the rendered output.
-:::
+- https://docs.airbyte.com
+- In-app documentation in self-managed versions of Airbyte
+- In-app documentation in the Cloud version of Airbyte
+
+Key details about setting up a connection may differ between Cloud and Self-Managed. We created custom Markdown extensions that can show or hide different pieces of content based on the reader's environment. This is a rudimentary form of a concept called single-sourcing: writing content once, but using it in multiple contexts. This prevents us from having to maintain multiple highly similar pools of content.
+
+The following features for single-sourcing are available. You can combine them to produce a more meaningful result.
+
+### Hide content from the Airbyte UI
+
+Some content is important to document, but unhelpful in Airbyte's UI. This could be:
+
+- Background information that helps people understand a connector but doesn't affect the configuration process
+- Edge cases with complex solutions
+- Context about each environment, which doesn't need to be seen if you're in that environment
 
-## Mapping UI to Associated Docs
+Wrapping content in `<HideInUI>...</HideInUI>` tags prevents Airbyte from rendering that content in-app, but https://docs.airbyte.com renders it normally.
 
-Sometimes a connector's configuration UI may have a field that requires more explanation than can be provided in the UI itself. In these cases, the connector developer can use the `FieldAnchor` syntax below to link documentation to a specific UI component.
+### Hide content from Cloud
 
-When a user selects the associated field in the UI, the documentation will automatically scroll to the related documentation section the right-hand panel.
+You can hide content from Airbyte Cloud, while still rendering it in Self-Managed and https://docs.airbyte.com.
 
-The `FieldAnchor` syntax accepts a modified `jsonpath` expression, as follows:
+```md
+<!-- env:oss -->
+
+Only Self-Managed builds of the Airbyte UI will render this content.
+
+<!-- /env:oss -->
+```
+
+### Hide content from Self-Managed
+
+You can hide content from Airbyte Self-Managed, while still rendering it in Cloud and https://docs.airbyte.com.
 
-`example-a.md`:
 ```md
+<!-- env:cloud -->
+
+Only Cloud builds of the Airbyte UI will render this content.
+
+<!-- /env:cloud -->
+```
+
+### Example
+
+Here's an example where the configuration steps are different in Cloud and Self-Managed. In this case, you want to render everything on https://docs.airbyte.com, but you want in-app content reflect only the environment the user is running.
+
+```markdown title="connector-doc.md"
+# My connector
+
+This content is rendered everywhere.
+
+<!-- env:oss -->
+
+<HideInUI>
+
+## For open source: 
+
+</HideInUI>
+
+Only self-managed builds of the Airbyte UI will render this content.
+
+<!-- /env:oss -->
+
+<!-- env:cloud -->
+<HideInUI>
+
+## For Airbyte Cloud:
+
+</HideInUI>
+
+Only Cloud builds of the Airbyte UI will render this content.
+
+<!-- /env:oss -->
+```
+
+### Testing your content
+
+To test in-app content in [a local Airbyte build](https://docs.airbyte.com/contributing-to-airbyte/developing-locally/#develop-on-airbyte-webapp), check out the `airbyte` git repository to the same branch and directory as the airbyte platform repository. Development builds fetch connector documentation from your local filesystem, so you can edit their content and view the rendered output in Airbyte.
+
+To test https://docs.airbyte.com content, [build Docusaurus locally](../contributing-to-airbyte/writing-docs.md#set-up-your-environment).
+
+## Map the UI to associated content
+
+Sometimes a field requires more explanation than can be provided in a tooltip. In these cases, use the `<FieldAnchor>` tag to link documentation to a specific UI component.
+
+When a user selects that field in the UI, the in-app documentation panel automatically scrolls to the related documentation, highlighting all content contained inside the `<FieldAnchor></FieldAnchor>` tag.
+
+The `FieldAnchor` syntax accepts a modified version of `jsonpath`, without the conventional `$.` prefix. It looks like this:
+
+```md title="example-a.md"
 ## Configuring Widgets
 
 <FieldAnchor field="widget_option">
@@ -35,9 +111,8 @@ The `FieldAnchor` syntax accepts a modified `jsonpath` expression, as follows:
 
 Taking a more complex example, you can access deeper-nested fields using `jsonpath` expressions syntax:
 
-`example-b.md`:
-```md
-## Configuring Unstructures Streams
+```md title="example-b.md"
+## Configuring Unstructured Streams
 
 <FieldAnchor field="streams.0.format[unstructured],streams.1.format[unstructured],streams.2.format[unstructured]">
 
@@ -46,66 +121,81 @@ Taking a more complex example, you can access deeper-nested fields using `jsonpa
 </FieldAnchor>
 ```
 
-Note:
-- The syntax expected is a modified version of jsonpath, without the conventional `$.` prefix.
-- The `FieldAnchor` syntax is only supported in the context of connector documentation, and will not render on the documentation site.
+:::note
+The `FieldAnchor` tag only affects in-app content for sources and destinations. It has no effect on https://docs.airbyte.com or any platform content.
+:::
 
 How it works:
 
-- When a user focuses the field identified by the `field` attribute in the connector setup UI, the documentation pane will automatically scroll to the associated section of the documentation, highlighting all content contained inside the `<FieldAnchor></FieldAnchor>` tag.
-- These are rendered as regular divs in the documentation site, so they have no effect in places other than the in-app documentation panel.
-- There must be blank lines between a custom tag like `FieldAnchor` the content it wraps for the documentation site to render markdown syntax inside the custom tag to html.
+- There must be blank lines between a custom tag like `FieldAnchor` the content it wraps.
 - The `field` attribute must be a valid `jsonpath` expression to one of the properties nested under `connectionSpecification.properties` in that connector's `spec.json` or `spec.yaml` file. For example, if the connector spec contains a `connectionSpecification.properties.replication_method.replication_slot`, you would mark the start of the related documentation section with `<FieldAnchor field="replication_method.replication_slot">` and its end with `</FieldAnchor>`.
-- It's also possible to highlight the same section for multiple fields by separating them with commas, like `<FieldAnchor field="replication_method.replication_slot,replication_method.queue_size">`.
-- To mark a section as highlighted after the user picks an option from a `oneOf`: use a `field` prop like `path.to.field[value-of-selection-key]`, where the `value-of-selection-key` is the value of a `const` field nested inside that `oneOf`. For example, if the specification of the `oneOf` field is:
-
-```json
-"replication_method": {
-  "type": "object",
-  "title": "Update Method",
-  "oneOf": [
-    {
-      "title": "Read Changes using Binary Log (CDC)",
-      "required": ["method"],
-      "properties": {
-        "method": {
-          "type": "string",
-          "const": "CDC",
-          "order": 0
-        },
-        "initial_waiting_seconds": {
-          "type": "integer",
-          "title": "Initial Waiting Time in Seconds (Advanced)",
+- Highlight the same section for multiple fields by separating them with commas, like this: `<FieldAnchor field="replication_method.replication_slot,replication_method.queue_size">`.
+- To highlight a section after the user picks an option from a `oneOf`: use a `field` prop like `path.to.field[value-of-selection-key]`, where the `value-of-selection-key` is the value of a `const` field nested inside that `oneOf`.
+
+    For example, if the specification of the `oneOf` field is:
+
+    ```json
+    "replication_method": {
+      "type": "object",
+      "title": "Update Method",
+      "oneOf": [
+        {
+          "title": "Read Changes using Binary Log (CDC)",
+          "required": ["method"],
+          "properties": {
+            "method": {
+              "type": "string",
+              <!-- highlight-next-line -->
+              "const": "CDC",
+              "order": 0
+            },
+            "initial_waiting_seconds": {
+              "type": "integer",
+              "title": "Initial Waiting Time in Seconds (Advanced)",
+            },
+          }
         },
-      }
-    },
-    {
-      "title": "Scan Changes with User Defined Cursor",
-      "required": ["method"],
-      "properties": {
-        "method": {
-          "type": "string",
-          "const": "STANDARD",
-          "order": 0
+        {
+          "title": "Scan Changes with User Defined Cursor",
+          "required": ["method"],
+          "properties": {
+            "method": {
+              "type": "string",
+              <!-- highlight-next-line -->
+              "const": "STANDARD",
+              "order": 0
+            }
+          }
         }
-      }
+      ]
     }
-  ]
-}
-```
+    ```
 
-The selection keys are `CDC` and `STANDARD`, so you can wrap a specific replication method's documentation section with a `<FieldAnchor field="replication_method[CDC]">...</FieldAnchor>` tag, and it will be highlighted if the user selects CDC replication in the UI.
+    The selection keys are `CDC` and `STANDARD`. Wrap a specific replication method's documentation section with a `<FieldAnchor field="replication_method[CDC]">...</FieldAnchor>` tag to highlight it if the user selects CDC replication in the UI.
 
-:::tip
-Because of their close connection with the connector setup form fields, `<FieldAnchor>` tags are only enabled for the source and destination setup pages.
-:::
+### Documenting PyAirbyte usage
+
+PyAirbyte is a Python library that allows you to run syncs within a Python script for a subset of Airbyte's connectors. Documentation around PyAirbyte connectors is automatically generated from the connector's JSON schema spec. There are a few approaches to combine full control over the documentation with automatic generation for common cases:
 
-## Prevent specific docs from rendering in the Config UI with `<HideInUI>`
+- If a connector:
+
+    1. Is PyAirbyte-enabled (`remoteRegistries.pypi.enabled` is set in the `metadata.yaml` file of the connector), and 
+    2. Has no second-level heading `Usage with PyAirbyte` in the documentation 
+
+  The documentation will be automatically generated and placed above the `Changelog` section.
 
-Certain content is important to document, but unhelpful in the context of the Airbyte UI's inline documentation views:
+- By manually specifying a `Usage with PyAirbyte` section, this is disabled. The following is a good starting point for this section:
 
-- background information that helps users understand a connector but doesn't affect configuration
-- edge cases that are unusual but time-consuming to solve
-- context for readers on the documentation site about environment-specific content (see [below](#environment-specific-in-app-content-with-magic-html-comments))
+```md
+<HideInUI>
+
+## Usage with PyAirbyte
+
+<PyAirbyteExample connector="source-google-sheets" />
+
+<SpecSchema connector="source-google-sheets" />
+
+</HideInUI>
+```
 
-Wrapping such content in a pair of `<HideInUI>...</HideInUI>` tags will prevent it from being rendered within the Airbyte UI without affecting its presentation on https://docs.airbyte.com. This allows a single markdown file to be the source of truth for both a streamlined in-app reference and a more thorough treatment on the documentation website.
+The `PyAirbyteExample` component will generate a code example that can be run with PyAirbyte, excluding an auto-generated sample configuration based on the configuration schema. The `SpecSchema` component will generate a reference table with the connector's JSON schema spec, like a non-interactive version of the connector form in the UI. It can be used on any docs page.