From 87c75fca57b225a130aa1d733ea9c99f68189b93 Mon Sep 17 00:00:00 2001 From: Alex Birdsall Date: Mon, 11 Dec 2023 14:58:47 -0800 Subject: [PATCH 01/27] =?UTF-8?q?:window:=20=F0=9F=9B=A0=EF=B8=8F=20Make?= =?UTF-8?q?=20user=20type=20workspace=20name=20in=20confirmation=20modal?= =?UTF-8?q?=20before=20deleting=20(#9697)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ConfirmWorkspaceDeletionModal.module.scss | 7 ++ .../useConfirmWorkspaceDeletionModal.tsx | 93 +++++++++++++++++++ airbyte-webapp/src/locales/en.json | 2 + .../components/DeleteWorkspaceSection.tsx | 40 +------- .../components/DeleteWorkspace.tsx | 49 +--------- 5 files changed, 110 insertions(+), 81 deletions(-) create mode 100644 airbyte-webapp/src/area/workspace/utils/ConfirmWorkspaceDeletionModal.module.scss create mode 100644 airbyte-webapp/src/area/workspace/utils/useConfirmWorkspaceDeletionModal.tsx diff --git a/airbyte-webapp/src/area/workspace/utils/ConfirmWorkspaceDeletionModal.module.scss b/airbyte-webapp/src/area/workspace/utils/ConfirmWorkspaceDeletionModal.module.scss new file mode 100644 index 00000000000..063c73a2b44 --- /dev/null +++ b/airbyte-webapp/src/area/workspace/utils/ConfirmWorkspaceDeletionModal.module.scss @@ -0,0 +1,7 @@ +@use "scss/variables"; + +.form { + display: flex; + margin-top: variables.$spacing-xl; + gap: variables.$spacing-sm; +} diff --git a/airbyte-webapp/src/area/workspace/utils/useConfirmWorkspaceDeletionModal.tsx b/airbyte-webapp/src/area/workspace/utils/useConfirmWorkspaceDeletionModal.tsx new file mode 100644 index 00000000000..14b732f95ba --- /dev/null +++ b/airbyte-webapp/src/area/workspace/utils/useConfirmWorkspaceDeletionModal.tsx @@ -0,0 +1,93 @@ +import { UseMutateAsyncFunction } from "@tanstack/react-query"; +import { useState } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; +import { useNavigate } from "react-router-dom"; + +import { Box } from "components/ui/Box"; +import { Button } from "components/ui/Button"; +import { Input } from "components/ui/Input"; +import { Text } from "components/ui/Text"; + +import { WorkspaceRead } from "core/request/AirbyteClient"; +import { useModalService } from "hooks/services/Modal"; +import { useNotificationService } from "hooks/services/Notification"; +import { RoutePaths } from "pages/routePaths"; + +import styles from "./ConfirmWorkspaceDeletionModal.module.scss"; + +const WorkspaceDeletionModalContent = ({ onSubmit, workspace }: { onSubmit: () => void; workspace: WorkspaceRead }) => { + const [confirmationInput, setConfirmationInput] = useState(""); + const isConfirmationValid = confirmationInput === workspace.name; + return ( +
+ + + + +
+ setConfirmationInput(event.target.value)} + /> + +
+ +
+ ); +}; + +/** + * Returns a function that can be used to open a confirmation modal for deleting a + * workspace. The user must type the workspace name in a confirmation input in order to + * proceed with the deletion. + * + * @param workspace - the workspace to delete + * @param deleteWorkspace - the API function which will actually delete the workspace upon successful confirmation + */ +export const useConfirmWorkspaceDeletionModal = ( + workspace: WorkspaceRead, + deleteWorkspace: UseMutateAsyncFunction +) => { + const { formatMessage } = useIntl(); + const { registerNotification } = useNotificationService(); + const navigate = useNavigate(); + const { openModal } = useModalService(); + + return async () => { + const result = await openModal<"confirm">({ + title: formatMessage( + { + id: "settings.workspaceSettings.deleteWorkspace.confirmation.title", + }, + { name: workspace.name } + ), + content: ({ onClose }) => ( + onClose("confirm")} /> + ), + size: "md", + }); + + // "closed" indicates a successful confirmation; "canceled" [sic] is its counterpart + // when the user backs out + if (result.type === "closed") { + try { + await deleteWorkspace(workspace.workspaceId); + registerNotification({ + id: "settings.workspace.delete.success", + text: formatMessage({ id: "settings.workspaceSettings.delete.success" }), + type: "success", + }); + navigate(`/${RoutePaths.Workspaces}`); + } catch { + registerNotification({ + id: "settings.workspace.delete.error", + text: formatMessage({ id: "settings.workspaceSettings.delete.error" }), + type: "error", + }); + } + } + }; +}; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 47102543f9a..d5ba0bd706f 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -1469,6 +1469,8 @@ "settings.workspaceSettings.delete.confirmation.title": "Delete workspace", "settings.workspaceSettings.delete.confirmation.text": "Deleting this workspace will remove it for all users and cancel all pending syncs. Do you want to proceed?", "settings.workspaceSettings.delete.confirmation.submitButtonText": "Delete workspace", + "settings.workspaceSettings.deleteWorkspace.confirmation.title": "Delete workspace: {name}", + "settings.workspaceSettings.deleteWorkspace.confirmation.text": "Deleting this workspace will remove it for all users and cancel all pending syncs. To confirm deletion, type the name of the workspace in the input field.", "settings.workspaceSettings.delete.permissionsError": "You do not have sufficient permissions to delete this workspace. Please consult with the workspace owner.", "settings.integrationSettings": "Integration settings", diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/DeleteWorkspaceSection.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/DeleteWorkspaceSection.tsx index 1744a16df27..953cdf740ae 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/DeleteWorkspaceSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspaceSettingsView/components/DeleteWorkspaceSection.tsx @@ -1,57 +1,25 @@ import React from "react"; -import { FormattedMessage, useIntl } from "react-intl"; -import { useNavigate } from "react-router-dom"; +import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; +import { useConfirmWorkspaceDeletionModal } from "area/workspace/utils/useConfirmWorkspaceDeletionModal"; import { useCurrentWorkspace } from "core/api"; import { useRemoveCloudWorkspace } from "core/api/cloud"; -import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; -import { useNotificationService } from "hooks/services/Notification"; -import { RoutePaths } from "pages/routePaths"; export const DeleteWorkspaceSection: React.FC = () => { - const { formatMessage } = useIntl(); - const navigate = useNavigate(); const workspace = useCurrentWorkspace(); - const { registerNotification } = useNotificationService(); - const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const { mutateAsync: removeCloudWorkspace, isLoading: isRemovingCloudWorkspace } = useRemoveCloudWorkspace(); - const deleteCurrentWorkspace = () => { - openConfirmationModal({ - title: "settings.workspaceSettings.delete.confirmation.title", - text: "settings.workspaceSettings.delete.confirmation.text", - submitButtonText: "settings.workspaceSettings.delete.confirmation.submitButtonText", - onSubmit() { - closeConfirmationModal(); - removeCloudWorkspace(workspace.workspaceId) - .then(() => { - registerNotification({ - id: "settings.workspace.delete.success", - text: formatMessage({ id: "settings.workspaceSettings.delete.success" }), - type: "success", - }); - navigate(`/${RoutePaths.Workspaces}`); - }) - .catch(() => { - registerNotification({ - id: "settings.workspace.delete.error", - text: formatMessage({ id: "settings.workspaceSettings.delete.error" }), - type: "error", - }); - }); - }, - }); - }; + const confirmWorkspaceDeletion = useConfirmWorkspaceDeletionModal(workspace, removeCloudWorkspace); return ( - diff --git a/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx b/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx index a05a329b33b..d656582cfd0 100644 --- a/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/components/DeleteWorkspace.tsx @@ -1,62 +1,21 @@ -import { FormattedMessage, useIntl } from "react-intl"; -import { useNavigate } from "react-router-dom"; +import { FormattedMessage } from "react-intl"; import { Box } from "components/ui/Box"; import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; -import { Text } from "components/ui/Text"; +import { useConfirmWorkspaceDeletionModal } from "area/workspace/utils/useConfirmWorkspaceDeletionModal"; import { useCurrentWorkspace, useDeleteWorkspace } from "core/api"; -import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; -import { useNotificationService } from "hooks/services/Notification"; -import { RoutePaths } from "pages/routePaths"; export const DeleteWorkspace: React.FC = () => { - const { formatMessage } = useIntl(); const workspace = useCurrentWorkspace(); - const navigate = useNavigate(); - const { registerNotification } = useNotificationService(); - const { openConfirmationModal, closeConfirmationModal } = useConfirmationModalService(); const { mutateAsync: deleteWorkspace, isLoading: isDeletingWorkspace } = useDeleteWorkspace(); - - const onDeleteClick = () => { - openConfirmationModal({ - title: formatMessage({ id: "settings.workspaceSettings.deleteLabel" }), - text: formatMessage({ id: "settings.workspaceSettings.deleteModal.proceed" }), - additionalContent: ( - - - - - - ), - submitButtonText: formatMessage({ id: "settings.workspaceSettings.deleteModal.confirm" }), - onSubmit() { - closeConfirmationModal(); - deleteWorkspace(workspace.workspaceId) - .then(() => { - registerNotification({ - id: "settings.workspace.delete.success", - text: formatMessage({ id: "settings.workspaceSettings.delete.success" }), - type: "success", - }); - navigate(`/${RoutePaths.Workspaces}`); - }) - .catch(() => { - registerNotification({ - id: "settings.workspace.delete.error", - text: formatMessage({ id: "settings.workspaceSettings.delete.error" }), - type: "error", - }); - }); - }, - }); - }; + const confirmWorkspaceDeletion = useConfirmWorkspaceDeletionModal(workspace, deleteWorkspace); return ( - From ecac5f6614540d1d73a89cf263124f4e3a953b1d Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 12 Dec 2023 14:59:31 +0100 Subject: [PATCH 02/27] =?UTF-8?q?=F0=9F=AA=9F=F0=9F=94=A7=20Refactor=20sou?= =?UTF-8?q?rce/destination=20APIs=20to=20new=20infrastructure=20(#10284)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.eslintLegacyFolderStructure.js | 3 - .../utils/useGetDestinationFromParams.tsx | 2 +- .../utils/useGetSourceFromParams.tsx | 2 +- .../CreateConnection/CreateNewDestination.tsx | 3 +- .../CreateConnection/CreateNewSource.tsx | 3 +- .../CreateConnection/SelectDestination.tsx | 3 +- .../CreateConnection/SelectSource.tsx | 3 +- .../CreateConnectionForm.test.tsx | 40 +++---- .../CreateConnectionForm.tsx | 3 +- .../CreateConnectionForm/SchemaError.tsx | 2 +- .../useAnalyticsTrackFunctions.ts | 2 +- .../api/hooks/destinations.tsx} | 86 +++++++------ airbyte-webapp/src/core/api/hooks/index.ts | 2 + .../api/hooks/sources.tsx} | 113 ++++++++++-------- .../core/api/hooks/upgradeConnectorVersion.ts | 3 +- .../api}/useRequestErrorHandler.tsx | 2 +- .../domain/connector/DestinationService.ts | 33 ----- .../core/domain/connector/SourceService.ts | 57 --------- .../src/core/request/AirbyteRequestService.ts | 12 -- .../src/core/services/i18n/I18nProvider.tsx | 5 +- airbyte-webapp/src/core/utils/useMemoDebug.ts | 26 ++++ .../ConnectionEdit/ConnectionEditService.tsx | 3 +- .../ConnectionForm/ConnectionFormService.tsx | 3 +- .../cloud/services/auth/CloudAuthService.tsx | 24 ++-- .../pages/ConnectorsPage/DestinationsPage.tsx | 3 +- .../pages/ConnectorsPage/SourcesPage.tsx | 3 +- .../CreateConnectionPage.tsx | 3 +- .../CreateConnectionTitleBlock.tsx | 4 +- .../AllDestinationsPage.tsx | 2 +- .../CreateDestinationPage.tsx | 3 +- .../DestinationConnectionsPage.tsx | 3 +- .../DestinationSettingsPage.tsx | 8 +- .../source/AllSourcesPage/AllSourcesPage.tsx | 2 +- .../CreateSourcePage/CreateSourcePage.tsx | 3 +- .../SourceConnectionsPage.tsx | 3 +- .../SourceSettingsPage/SourceSettingsPage.tsx | 4 +- .../src/services/useInitService.tsx | 19 --- .../ConnectorForm/ConnectorForm.test.tsx | 5 +- 38 files changed, 200 insertions(+), 300 deletions(-) rename airbyte-webapp/src/{hooks/services/useDestinationHook.tsx => core/api/hooks/destinations.tsx} (70%) rename airbyte-webapp/src/{hooks/services/useSourceHook.tsx => core/api/hooks/sources.tsx} (66%) rename airbyte-webapp/src/{hooks/services => core/api}/useRequestErrorHandler.tsx (92%) delete mode 100644 airbyte-webapp/src/core/domain/connector/DestinationService.ts delete mode 100644 airbyte-webapp/src/core/domain/connector/SourceService.ts delete mode 100644 airbyte-webapp/src/core/request/AirbyteRequestService.ts create mode 100644 airbyte-webapp/src/core/utils/useMemoDebug.ts delete mode 100644 airbyte-webapp/src/services/useInitService.tsx diff --git a/airbyte-webapp/.eslintLegacyFolderStructure.js b/airbyte-webapp/.eslintLegacyFolderStructure.js index 9be1c6a99a0..98ae7e877b5 100644 --- a/airbyte-webapp/.eslintLegacyFolderStructure.js +++ b/airbyte-webapp/.eslintLegacyFolderStructure.js @@ -5,7 +5,6 @@ module.exports = [ "src/services/connectorBuilder/ConnectorBuilderStateService.tsx", "src/services/connectorBuilder/ConnectorBuilderLocalStorageService.tsx", "src/services/connectorBuilder/SchemaWorker.ts", - "src/services/useInitService.tsx", "src/services/Scope.ts", // src/hooks "src/hooks/useDeleteModal.tsx", @@ -22,10 +21,8 @@ module.exports = [ "src/hooks/services/ConfirmationModal/ConfirmationModalService.tsx", "src/hooks/services/ConfirmationModal/index.ts", "src/hooks/services/ConfirmationModal/reducer.ts", - "src/hooks/services/useSourceHook.tsx", "src/hooks/services/useConnector.tsx", "src/hooks/services/useWorkspace.tsx", - "src/hooks/services/useDestinationHook.tsx", "src/hooks/services/Modal/ModalService.tsx", "src/hooks/services/Modal/types.ts", "src/hooks/services/Modal/ModalService.test.tsx", diff --git a/airbyte-webapp/src/area/connector/utils/useGetDestinationFromParams.tsx b/airbyte-webapp/src/area/connector/utils/useGetDestinationFromParams.tsx index d446d1b547f..a11f2e5c8de 100644 --- a/airbyte-webapp/src/area/connector/utils/useGetDestinationFromParams.tsx +++ b/airbyte-webapp/src/area/connector/utils/useGetDestinationFromParams.tsx @@ -2,7 +2,7 @@ import { useParams, useSearchParams } from "react-router-dom"; import { StepsTypes } from "components/ConnectorBlocks"; -import { useGetDestination } from "hooks/services/useDestinationHook"; +import { useGetDestination } from "core/api"; export const useGetDestinationFromParams = () => { const params = useParams<{ "*": StepsTypes | "" | undefined; destinationId: string }>(); diff --git a/airbyte-webapp/src/area/connector/utils/useGetSourceFromParams.tsx b/airbyte-webapp/src/area/connector/utils/useGetSourceFromParams.tsx index 7f919628c58..c68edda9450 100644 --- a/airbyte-webapp/src/area/connector/utils/useGetSourceFromParams.tsx +++ b/airbyte-webapp/src/area/connector/utils/useGetSourceFromParams.tsx @@ -2,7 +2,7 @@ import { useParams, useSearchParams } from "react-router-dom"; import { StepsTypes } from "components/ConnectorBlocks"; -import { useGetSource } from "hooks/services/useSourceHook"; +import { useGetSource } from "core/api"; export const useGetSourceFromParams = () => { const params = useParams<{ "*": StepsTypes | "" | undefined; sourceId: string }>(); diff --git a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx index feedb394302..d734a9d2045 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewDestination.tsx @@ -10,10 +10,9 @@ import { Button } from "components/ui/Button"; import { Icon } from "components/ui/Icon"; import { useSuggestedDestinations } from "area/connector/utils"; -import { useDestinationDefinitionList } from "core/api"; +import { useDestinationDefinitionList, useCreateDestination } from "core/api"; import { AppActionCodes, useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; -import { useCreateDestination } from "hooks/services/useDestinationHook"; import { DESTINATION_ID_PARAM, DESTINATION_TYPE_PARAM } from "./SelectDestination"; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx index f301054199d..2fbb6b5f2fe 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/CreateNewSource.tsx @@ -8,10 +8,9 @@ import { Button } from "components/ui/Button"; import { Icon } from "components/ui/Icon"; import { useSuggestedSources } from "area/connector/utils"; -import { useSourceDefinitionList } from "core/api"; +import { useSourceDefinitionList, useCreateSource } from "core/api"; import { AppActionCodes, useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; -import { useCreateSource } from "hooks/services/useSourceHook"; import { SourceForm, SourceFormValues } from "pages/source/CreateSourcePage/SourceForm"; import { SOURCE_ID_PARAM, SOURCE_TYPE_PARAM } from "./SelectSource"; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx b/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx index 84a504ca17e..3d4d5a60859 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/SelectDestination.tsx @@ -9,8 +9,7 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; -import { useConnectionList } from "core/api"; -import { useDestinationList } from "hooks/services/useDestinationHook"; +import { useConnectionList, useDestinationList } from "core/api"; import { CreateNewDestination, DESTINATION_DEFINITION_PARAM } from "./CreateNewDestination"; import { RadioButtonTiles } from "./RadioButtonTiles"; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx b/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx index 58f85871d0c..d27d971151d 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/SelectSource.tsx @@ -9,8 +9,7 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; -import { useSourceDefinitionList } from "core/api"; -import { useSourceList } from "hooks/services/useSourceHook"; +import { useSourceDefinitionList, useSourceList } from "core/api"; import { CreateNewSource, SOURCE_DEFINITION_PARAM } from "./CreateNewSource"; import { RadioButtonTiles } from "./RadioButtonTiles"; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx index a8d4ca7aad7..e1950cc2205 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx @@ -15,13 +15,22 @@ import { mockSourceDefinitionVersion, } from "test-utils/mock-data/mockSource"; import { mockTheme } from "test-utils/mock-data/mockTheme"; -import { TestWrapper, useMockIntersectionObserver } from "test-utils/testutils"; +import { mocked, TestWrapper, useMockIntersectionObserver } from "test-utils/testutils"; +import type { SchemaError } from "core/api"; +import { useDiscoverSchema } from "core/api"; import { defaultOssFeatures, FeatureItem } from "core/services/features"; -import * as sourceHook from "hooks/services/useSourceHook"; import { CreateConnectionForm } from "./CreateConnectionForm"; +const mockBaseUseDiscoverSchema = { + schemaErrorStatus: null, + isLoading: false, + schema: mockConnection.syncCatalog, + catalogId: "", + onDiscoverSchema: () => Promise.resolve(), +}; + jest.mock("area/workspace/utils", () => ({ useCurrentWorkspaceId: () => "workspace-id", })); @@ -36,6 +45,7 @@ jest.mock("core/api", () => ({ useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, useSourceDefinition: () => mockSourceDefinition, useDestinationDefinition: () => mockDestinationDefinition, + useDiscoverSchema: jest.fn(() => mockBaseUseDiscoverSchema), })); jest.mock("area/connector/utils", () => ({ @@ -71,38 +81,26 @@ describe("CreateConnectionForm", () => { return renderResult!; }; - const baseUseDiscoverSchema = { - schemaErrorStatus: null, - isLoading: false, - schema: mockConnection.syncCatalog, - catalogId: "", - onDiscoverSchema: () => Promise.resolve(), - }; - beforeEach(() => { useMockIntersectionObserver(); }); it("should render", async () => { - jest.spyOn(sourceHook, "useDiscoverSchema").mockImplementationOnce(() => baseUseDiscoverSchema); const renderResult = await render(); expect(renderResult).toMatchSnapshot(); expect(renderResult.queryByText("Please wait a little bit more…")).toBeFalsy(); }); it("should render when loading", async () => { - jest - .spyOn(sourceHook, "useDiscoverSchema") - .mockImplementationOnce(() => ({ ...baseUseDiscoverSchema, isLoading: true })); - + mocked(useDiscoverSchema).mockImplementationOnce(() => ({ ...mockBaseUseDiscoverSchema, isLoading: true })); const renderResult = await render(); expect(renderResult).toMatchSnapshot(); }); it("should render with an error", async () => { - jest.spyOn(sourceHook, "useDiscoverSchema").mockImplementationOnce(() => ({ - ...baseUseDiscoverSchema, - schemaErrorStatus: new Error("Test Error") as sourceHook.SchemaError, + mocked(useDiscoverSchema).mockImplementationOnce(() => ({ + ...mockBaseUseDiscoverSchema, + schemaErrorStatus: new Error("Test Error") as SchemaError, })); const renderResult = await render(); @@ -114,8 +112,6 @@ describe("CreateConnectionForm", () => { const CRON_EXPRESSION_EVERY_MINUTE = "* * * * * * ?"; it("should display an error for an invalid cron expression", async () => { - jest.spyOn(sourceHook, "useDiscoverSchema").mockImplementationOnce(() => baseUseDiscoverSchema); - const container = tlr( @@ -136,8 +132,6 @@ describe("CreateConnectionForm", () => { }); it("should allow cron expressions under one hour when feature enabled", async () => { - jest.spyOn(sourceHook, "useDiscoverSchema").mockImplementationOnce(() => baseUseDiscoverSchema); - const container = tlr( @@ -158,8 +152,6 @@ describe("CreateConnectionForm", () => { }); it("should not allow cron expressions under one hour when feature not enabled", async () => { - jest.spyOn(sourceHook, "useDiscoverSchema").mockImplementationOnce(() => baseUseDiscoverSchema); - const featuresToInject = defaultOssFeatures.filter((f) => f !== FeatureItem.AllowSyncSubOneHourCronExpressions); const container = tlr( diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx index 914c4fbad4f..3aa36e1c691 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.tsx @@ -7,7 +7,7 @@ import { FlexContainer } from "components/ui/Flex"; import { useGetDestinationFromSearchParams, useGetSourceFromSearchParams } from "area/connector/utils"; import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { useCreateConnection } from "core/api"; +import { useCreateConnection, useDiscoverSchema } from "core/api"; import { FeatureItem, useFeature } from "core/services/features"; import { ConnectionFormServiceProvider, @@ -15,7 +15,6 @@ import { } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useExperimentContext } from "hooks/services/Experiment"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; -import { useDiscoverSchema } from "hooks/services/useSourceHook"; import { ConnectionNameCard } from "./ConnectionNameCard"; import styles from "./CreateConnectionForm.module.scss"; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx index a9a607896fc..3dc70454d07 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx @@ -6,9 +6,9 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Icon } from "components/ui/Icon"; +import { SchemaError as SchemaErrorType } from "core/api"; import { LogsRequestError } from "core/request/LogsRequestError"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; -import { SchemaError as SchemaErrorType } from "hooks/services/useSourceHook"; import styles from "./SchemaError.module.scss"; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/useAnalyticsTrackFunctions.ts b/airbyte-webapp/src/components/connection/CreateConnectionForm/useAnalyticsTrackFunctions.ts index dd305d45bde..767e54ddfd5 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/useAnalyticsTrackFunctions.ts +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/useAnalyticsTrackFunctions.ts @@ -1,8 +1,8 @@ import { useCallback } from "react"; +import { SchemaError } from "core/api"; import { DestinationRead, SourceRead } from "core/api/types/AirbyteClient"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; -import { SchemaError } from "hooks/services/useSourceHook"; /** * track discover schema failure action diff --git a/airbyte-webapp/src/hooks/services/useDestinationHook.tsx b/airbyte-webapp/src/core/api/hooks/destinations.tsx similarity index 70% rename from airbyte-webapp/src/hooks/services/useDestinationHook.tsx rename to airbyte-webapp/src/core/api/hooks/destinations.tsx index 1c0efcdd8ce..22443cb89ae 100644 --- a/airbyte-webapp/src/hooks/services/useDestinationHook.tsx +++ b/airbyte-webapp/src/core/api/hooks/destinations.tsx @@ -2,19 +2,23 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useCallback } from "react"; import { ConnectionConfiguration } from "area/connector/types"; -import { useRemoveConnectionsFromList, useSuspenseQuery } from "core/api"; -// eslint-disable-next-line import/no-restricted-paths -import { useRequestOptions } from "core/api/useRequestOptions"; -import { DestinationService } from "core/domain/connector/DestinationService"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { isDefined } from "core/utils/common"; -import { useInitService } from "services/useInitService"; - -import { useRequestErrorHandler } from "./useRequestErrorHandler"; -import { useCurrentWorkspace } from "./useWorkspace"; -import { useConfig } from "../../config"; -import { DestinationRead, WebBackendConnectionListItem } from "../../core/request/AirbyteClient"; -import { SCOPE_WORKSPACE } from "../../services/Scope"; +import { SCOPE_WORKSPACE } from "services/Scope"; + +import { useRemoveConnectionsFromList } from "./connections"; +import { useCurrentWorkspace } from "./workspaces"; +import { + createDestination, + deleteDestination, + getDestination, + listDestinationsForWorkspace, + updateDestination, +} from "../generated/AirbyteClient"; +import { DestinationRead, WebBackendConnectionListItem } from "../types/AirbyteClient"; +import { useRequestErrorHandler } from "../useRequestErrorHandler"; +import { useRequestOptions } from "../useRequestOptions"; +import { useSuspenseQuery } from "../useSuspenseQuery"; export const destinationsKeys = { all: [SCOPE_WORKSPACE, "destinations"] as const, @@ -34,31 +38,31 @@ interface ConnectorProps { destinationDefinitionId: string; } -function useDestinationService() { - const { apiUrl } = useConfig(); - const requestOptions = useRequestOptions(); - return useInitService(() => new DestinationService(requestOptions), [apiUrl, requestOptions]); -} - interface DestinationList { destinations: DestinationRead[]; } const useDestinationList = (): DestinationList => { + const requestOptions = useRequestOptions(); const workspace = useCurrentWorkspace(); - const service = useDestinationService(); - return useSuspenseQuery(destinationsKeys.lists(), () => service.list(workspace.workspaceId)); + return useSuspenseQuery(destinationsKeys.lists(), () => + listDestinationsForWorkspace({ workspaceId: workspace.workspaceId }, requestOptions) + ); }; const useGetDestination = ( destinationId: T ): T extends string ? DestinationRead : DestinationRead | undefined => { - const service = useDestinationService(); + const requestOptions = useRequestOptions(); - return useSuspenseQuery(destinationsKeys.detail(destinationId ?? ""), () => service.get(destinationId ?? ""), { - enabled: isDefined(destinationId), - }); + return useSuspenseQuery( + destinationsKeys.detail(destinationId ?? ""), + () => getDestination({ destinationId: destinationId ?? "" }, requestOptions), + { + enabled: isDefined(destinationId), + } + ); }; export const useInvalidateDestination = (destinationId: T): (() => void) => { @@ -70,7 +74,7 @@ export const useInvalidateDestination = (de }; const useCreateDestination = () => { - const service = useDestinationService(); + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); const workspace = useCurrentWorkspace(); const onError = useRequestErrorHandler("destinations.createError"); @@ -83,12 +87,15 @@ const useCreateDestination = () => { throw new Error("No Destination Definition Provided"); } - return service.create({ - name: values.name, - destinationDefinitionId: destinationConnector?.destinationDefinitionId, - workspaceId: workspace.workspaceId, - connectionConfiguration: values.connectionConfiguration, - }); + return createDestination( + { + name: values.name, + destinationDefinitionId: destinationConnector?.destinationDefinitionId, + workspaceId: workspace.workspaceId, + connectionConfiguration: values.connectionConfiguration, + }, + requestOptions + ); }, { onSuccess: (data) => { @@ -102,7 +109,7 @@ const useCreateDestination = () => { }; const useDeleteDestination = () => { - const service = useDestinationService(); + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); const analyticsService = useAnalyticsService(); const removeConnectionsFromList = useRemoveConnectionsFromList(); @@ -110,7 +117,7 @@ const useDeleteDestination = () => { return useMutation( (payload: { destination: DestinationRead; connectionsWithDestination: WebBackendConnectionListItem[] }) => - service.delete(payload.destination.destinationId), + deleteDestination({ destinationId: payload.destination.destinationId }, requestOptions), { onSuccess: (_data, ctx) => { analyticsService.track(Namespace.DESTINATION, Action.DELETE, { @@ -138,17 +145,20 @@ const useDeleteDestination = () => { }; const useUpdateDestination = () => { - const service = useDestinationService(); + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); const onError = useRequestErrorHandler("destinations.updateError"); return useMutation( (updateDestinationPayload: { values: ValuesProps; destinationId: string }) => { - return service.update({ - name: updateDestinationPayload.values.name, - destinationId: updateDestinationPayload.destinationId, - connectionConfiguration: updateDestinationPayload.values.connectionConfiguration, - }); + return updateDestination( + { + name: updateDestinationPayload.values.name, + destinationId: updateDestinationPayload.destinationId, + connectionConfiguration: updateDestinationPayload.values.connectionConfiguration, + }, + requestOptions + ); }, { onSuccess: (data) => { diff --git a/airbyte-webapp/src/core/api/hooks/index.ts b/airbyte-webapp/src/core/api/hooks/index.ts index 9f30dfb49bb..abb3a107550 100644 --- a/airbyte-webapp/src/core/api/hooks/index.ts +++ b/airbyte-webapp/src/core/api/hooks/index.ts @@ -7,6 +7,7 @@ export * from "./connectorDefinitionSpecification"; export * from "./connectorDocumentation"; export * from "./connectorOAuth"; export * from "./connectorUpdates"; +export * from "./destinations"; export * from "./destinationDefinitions"; export * from "./geographies"; export * from "./health"; @@ -17,6 +18,7 @@ export * from "./operations"; export * from "./organizations"; export * from "./permissions"; export * from "./security"; +export * from "./sources"; export * from "./sourceDefinitions"; export * from "./streams"; export * from "./permissions"; diff --git a/airbyte-webapp/src/hooks/services/useSourceHook.tsx b/airbyte-webapp/src/core/api/hooks/sources.tsx similarity index 66% rename from airbyte-webapp/src/hooks/services/useSourceHook.tsx rename to airbyte-webapp/src/core/api/hooks/sources.tsx index 2d8fbb68a64..50817f4f91a 100644 --- a/airbyte-webapp/src/hooks/services/useSourceHook.tsx +++ b/airbyte-webapp/src/core/api/hooks/sources.tsx @@ -3,24 +3,24 @@ import { useCallback, useEffect, useState } from "react"; import { flushSync } from "react-dom"; import { ConnectionConfiguration } from "area/connector/types"; -import { useConfig } from "config"; -import { useSuspenseQuery, useRemoveConnectionsFromList } from "core/api"; -// eslint-disable-next-line import/no-restricted-paths -import { useRequestOptions } from "core/api/useRequestOptions"; -import { SourceService } from "core/domain/connector/SourceService"; -import { - AirbyteCatalog, - SourceRead, - SynchronousJobRead, - WebBackendConnectionListItem, -} from "core/request/AirbyteClient"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { isDefined } from "core/utils/common"; -import { useInitService } from "services/useInitService"; +import { SCOPE_WORKSPACE } from "services/Scope"; -import { useRequestErrorHandler } from "./useRequestErrorHandler"; -import { useCurrentWorkspace } from "./useWorkspace"; -import { SCOPE_WORKSPACE } from "../../services/Scope"; +import { useRemoveConnectionsFromList } from "./connections"; +import { useCurrentWorkspace } from "./workspaces"; +import { + createSource, + deleteSource, + discoverSchemaForSource, + getSource, + listSourcesForWorkspace, + updateSource, +} from "../generated/AirbyteClient"; +import { AirbyteCatalog, SourceRead, SynchronousJobRead, WebBackendConnectionListItem } from "../types/AirbyteClient"; +import { useRequestErrorHandler } from "../useRequestErrorHandler"; +import { useRequestOptions } from "../useRequestOptions"; +import { useSuspenseQuery } from "../useSuspenseQuery"; export const sourcesKeys = { all: [SCOPE_WORKSPACE, "sources"] as const, @@ -41,31 +41,31 @@ interface ConnectorProps { sourceDefinitionId: string; } -function useSourceService() { - const { apiUrl } = useConfig(); - const requestOptions = useRequestOptions(); - return useInitService(() => new SourceService(requestOptions), [apiUrl, requestOptions]); -} - interface SourceList { sources: SourceRead[]; } const useSourceList = (): SourceList => { + const requestOptions = useRequestOptions(); const workspace = useCurrentWorkspace(); - const service = useSourceService(); - return useSuspenseQuery(sourcesKeys.lists(), () => service.list(workspace.workspaceId)); + return useSuspenseQuery(sourcesKeys.lists(), () => + listSourcesForWorkspace({ workspaceId: workspace.workspaceId }, requestOptions) + ); }; const useGetSource = ( sourceId: T ): T extends string ? SourceRead : SourceRead | undefined => { - const service = useSourceService(); + const requestOptions = useRequestOptions(); - return useSuspenseQuery(sourcesKeys.detail(sourceId ?? ""), () => service.get(sourceId ?? ""), { - enabled: isDefined(sourceId), - }); + return useSuspenseQuery( + sourcesKeys.detail(sourceId ?? ""), + () => getSource({ sourceId: sourceId ?? "" }, requestOptions), + { + enabled: isDefined(sourceId), + } + ); }; export const useInvalidateSource = (sourceId: T): (() => void) => { @@ -77,7 +77,7 @@ export const useInvalidateSource = (sourceI }; const useCreateSource = () => { - const service = useSourceService(); + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); const workspace = useCurrentWorkspace(); const onError = useRequestErrorHandler("sources.createError"); @@ -87,12 +87,15 @@ const useCreateSource = () => { const { values, sourceConnector } = createSourcePayload; try { // Try to create source - const result = await service.create({ - name: values.name, - sourceDefinitionId: sourceConnector?.sourceDefinitionId, - workspaceId: workspace.workspaceId, - connectionConfiguration: values.connectionConfiguration, - }); + const result = await createSource( + { + name: values.name, + sourceDefinitionId: sourceConnector?.sourceDefinitionId, + workspaceId: workspace.workspaceId, + connectionConfiguration: values.connectionConfiguration, + }, + requestOptions + ); return result; } catch (e) { @@ -111,7 +114,7 @@ const useCreateSource = () => { }; const useDeleteSource = () => { - const service = useSourceService(); + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); const analyticsService = useAnalyticsService(); const removeConnectionsFromList = useRemoveConnectionsFromList(); @@ -119,7 +122,7 @@ const useDeleteSource = () => { return useMutation( (payload: { source: SourceRead; connectionsWithSource: WebBackendConnectionListItem[] }) => - service.delete(payload.source.sourceId), + deleteSource({ sourceId: payload.source.sourceId }, requestOptions), { onSuccess: (_data, ctx) => { analyticsService.track(Namespace.SOURCE, Action.DELETE, { @@ -146,17 +149,20 @@ const useDeleteSource = () => { }; const useUpdateSource = () => { - const service = useSourceService(); + const requestOptions = useRequestOptions(); const queryClient = useQueryClient(); const onError = useRequestErrorHandler("sources.updateError"); return useMutation( (updateSourcePayload: { values: ValuesProps; sourceId: string }) => { - return service.update({ - name: updateSourcePayload.values.name, - sourceId: updateSourcePayload.sourceId, - connectionConfiguration: updateSourcePayload.values.connectionConfiguration, - }); + return updateSource( + { + name: updateSourcePayload.values.name, + sourceId: updateSourcePayload.sourceId, + connectionConfiguration: updateSourcePayload.values.connectionConfiguration, + }, + requestOptions + ); }, { onSuccess: (data) => { @@ -179,7 +185,7 @@ const useDiscoverSchema = ( catalogId: string | undefined; onDiscoverSchema: () => Promise; } => { - const service = useSourceService(); + const requestOptions = useRequestOptions(); const [schema, setSchema] = useState(undefined); const [catalogId, setCatalogId] = useState(""); const [isLoading, setIsLoading] = useState(false); @@ -189,17 +195,30 @@ const useDiscoverSchema = ( setIsLoading(true); setSchemaErrorStatus(null); try { - const data = await service.discoverSchema(sourceId || "", disableCache); + const result = await discoverSchemaForSource( + { sourceId: sourceId || "", disable_cache: disableCache }, + requestOptions + ); + + if (!result.jobInfo?.succeeded || !result.catalog) { + // @ts-expect-error TODO: address this case + const e = result.jobInfo?.logs ? new LogsRequestError(result.jobInfo) : new CommonRequestError(result); + // Generate error with failed status and received logs + e._status = 400; + e.response = result.jobInfo; + throw e; + } + flushSync(() => { - setSchema(data.catalog); - setCatalogId(data.catalogId); + setSchema(result.catalog); + setCatalogId(result.catalogId); }); } catch (e) { setSchemaErrorStatus(e); } finally { setIsLoading(false); } - }, [disableCache, service, sourceId]); + }, [disableCache, requestOptions, sourceId]); useEffect(() => { if (sourceId) { diff --git a/airbyte-webapp/src/core/api/hooks/upgradeConnectorVersion.ts b/airbyte-webapp/src/core/api/hooks/upgradeConnectorVersion.ts index d70ee4060fa..b76f5c23a1d 100644 --- a/airbyte-webapp/src/core/api/hooks/upgradeConnectorVersion.ts +++ b/airbyte-webapp/src/core/api/hooks/upgradeConnectorVersion.ts @@ -1,9 +1,8 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; +import { destinationsKeys, sourcesKeys } from "core/api"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; -import { destinationsKeys } from "hooks/services/useDestinationHook"; -import { sourcesKeys } from "hooks/services/useSourceHook"; import { definitionKeys } from "./actorDefinitionVersions"; import { diff --git a/airbyte-webapp/src/hooks/services/useRequestErrorHandler.tsx b/airbyte-webapp/src/core/api/useRequestErrorHandler.tsx similarity index 92% rename from airbyte-webapp/src/hooks/services/useRequestErrorHandler.tsx rename to airbyte-webapp/src/core/api/useRequestErrorHandler.tsx index 5817815e53c..73833b937c9 100644 --- a/airbyte-webapp/src/hooks/services/useRequestErrorHandler.tsx +++ b/airbyte-webapp/src/core/api/useRequestErrorHandler.tsx @@ -1,7 +1,7 @@ import { useCallback, useEffect } from "react"; import { useIntl } from "react-intl"; -import { useNotificationService } from "./Notification"; +import { useNotificationService } from "../../hooks/services/Notification"; const NOTIFICATION_ID = "unexpected-request-error"; diff --git a/airbyte-webapp/src/core/domain/connector/DestinationService.ts b/airbyte-webapp/src/core/domain/connector/DestinationService.ts deleted file mode 100644 index f663e5ee85c..00000000000 --- a/airbyte-webapp/src/core/domain/connector/DestinationService.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { AirbyteRequestService } from "core/request/AirbyteRequestService"; - -import { - createDestination, - deleteDestination, - DestinationCreate, - DestinationUpdate, - getDestination, - listDestinationsForWorkspace, - updateDestination, -} from "../../request/AirbyteClient"; - -export class DestinationService extends AirbyteRequestService { - public get(destinationId: string) { - return getDestination({ destinationId }, this.requestOptions); - } - - public list(workspaceId: string) { - return listDestinationsForWorkspace({ workspaceId }, this.requestOptions); - } - - public create(body: DestinationCreate) { - return createDestination(body, this.requestOptions); - } - - public update(body: DestinationUpdate) { - return updateDestination(body, this.requestOptions); - } - - public delete(destinationId: string) { - return deleteDestination({ destinationId }, this.requestOptions); - } -} diff --git a/airbyte-webapp/src/core/domain/connector/SourceService.ts b/airbyte-webapp/src/core/domain/connector/SourceService.ts deleted file mode 100644 index dd395ea0454..00000000000 --- a/airbyte-webapp/src/core/domain/connector/SourceService.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { AirbyteRequestService } from "core/request/AirbyteRequestService"; -import { CommonRequestError } from "core/request/CommonRequestError"; -import { LogsRequestError } from "core/request/LogsRequestError"; - -import { - createSource, - deleteSource, - discoverSchemaForSource, - getSource, - listSourcesForWorkspace, - SourceCreate, - SourceUpdate, - updateSource, -} from "../../request/AirbyteClient"; - -export class SourceService extends AirbyteRequestService { - public get(sourceId: string) { - return getSource({ sourceId }, this.requestOptions); - } - - public list(workspaceId: string) { - return listSourcesForWorkspace({ workspaceId }, this.requestOptions); - } - - public create(body: SourceCreate) { - return createSource(body, this.requestOptions); - } - - public update(body: SourceUpdate) { - return updateSource(body, this.requestOptions); - } - - public delete(sourceId: string) { - return deleteSource({ sourceId }, this.requestOptions); - } - - public async discoverSchema(sourceId: string, disableCache?: boolean) { - const result = await discoverSchemaForSource({ sourceId, disable_cache: disableCache }, this.requestOptions); - - if (!result.jobInfo?.succeeded || !result.catalog) { - // @ts-expect-error TODO: address this case - const e = result.jobInfo?.logs ? new LogsRequestError(result.jobInfo) : new CommonRequestError(result); - // Generate error with failed status and received logs - e._status = 400; - // @ts-expect-error address this case - e.response = result.jobInfo; - throw e; - } - - return { - catalog: result.catalog, - jobInfo: result.jobInfo, - catalogId: result.catalogId, - id: sourceId, - }; - } -} diff --git a/airbyte-webapp/src/core/request/AirbyteRequestService.ts b/airbyte-webapp/src/core/request/AirbyteRequestService.ts deleted file mode 100644 index 273363e2425..00000000000 --- a/airbyte-webapp/src/core/request/AirbyteRequestService.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { ApiCallOptions } from "../api"; - -/** - * @deprecated This class will be removed soon and should no longer be used or extended. - */ -export abstract class AirbyteRequestService { - public readonly requestOptions: ApiCallOptions; - - constructor(requestOptions: ApiCallOptions) { - this.requestOptions = requestOptions; - } -} diff --git a/airbyte-webapp/src/core/services/i18n/I18nProvider.tsx b/airbyte-webapp/src/core/services/i18n/I18nProvider.tsx index 7540fa50430..f5eb1264ab9 100644 --- a/airbyte-webapp/src/core/services/i18n/I18nProvider.tsx +++ b/airbyte-webapp/src/core/services/i18n/I18nProvider.tsx @@ -1,5 +1,6 @@ import type { IntlConfig } from "react-intl"; +import isEqual from "lodash/isEqual"; import React, { useContext, useMemo, useState } from "react"; import { IntlProvider } from "react-intl"; @@ -21,12 +22,12 @@ interface I18nProviderProps { } export const I18nProvider: React.FC> = ({ children, messages, locale }) => { - const [overwrittenMessages, setOvewrittenMessages] = useState(); + const [overwrittenMessages, setOvewrittenMessages] = useState({}); const i18nOverwriteContext = useMemo( () => ({ setMessageOverwrite: (messages) => { - setOvewrittenMessages(messages); + setOvewrittenMessages((prevMessages) => (isEqual(prevMessages, messages) ? prevMessages : messages)); }, }), [] diff --git a/airbyte-webapp/src/core/utils/useMemoDebug.ts b/airbyte-webapp/src/core/utils/useMemoDebug.ts new file mode 100644 index 00000000000..b0fc4f48edd --- /dev/null +++ b/airbyte-webapp/src/core/utils/useMemoDebug.ts @@ -0,0 +1,26 @@ +import { useEffect, useMemo, useRef } from "react"; + +const usePrevious = (dependencies: readonly unknown[]) => { + const ref = useRef(dependencies); + useEffect(() => { + ref.current = dependencies; + }); + return ref.current; +}; + +export const useMemoDebug = (factory: () => T, deps: readonly unknown[], hookName?: string): T => { + const previousDeps = usePrevious(deps); + return useMemo(() => { + console.log(`%c${hookName || "useMemoDebug"} calculated new value`, "background-color: #0f766e; color: #FFF"); + const changes = deps.map((dep, index) => { + return { + "Changed?": dep !== previousDeps[index] ? "🆕" : "⬜", + "Old Value": previousDeps[index], + "New Value": dep, + }; + }); + console.table(changes); + return factory(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, deps); +}; diff --git a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx index 1ce6134815c..76fdd3e4f68 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionEdit/ConnectionEditService.tsx @@ -3,7 +3,7 @@ import { useContext, useState, createContext, useCallback } from "react"; import { useIntl } from "react-intl"; import { useAsyncFn } from "react-use"; -import { useGetConnection, useGetConnectionQuery, useUpdateConnection } from "core/api"; +import { SchemaError, useGetConnection, useGetConnectionQuery, useUpdateConnection } from "core/api"; import { AirbyteCatalog, ConnectionStatus, @@ -13,7 +13,6 @@ import { import { ConnectionFormServiceProvider } from "../ConnectionForm/ConnectionFormService"; import { useNotificationService } from "../Notification"; -import { SchemaError } from "../useSourceHook"; interface ConnectionEditProps { connectionId: string; diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx index 81c3fa155db..5db4fdda93c 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx @@ -11,6 +11,7 @@ import { useGetDestinationDefinitionSpecification, useSourceDefinition, useDestinationDefinition, + SchemaError, } from "core/api"; import { ActorDefinitionVersionRead, @@ -22,8 +23,6 @@ import { } from "core/request/AirbyteClient"; import { FormError, generateMessageFromError } from "core/utils/errorStatusMessage"; -import { SchemaError } from "../useSourceHook"; - export type ConnectionFormMode = "create" | "edit" | "readonly"; export type ConnectionOrPartialConnection = diff --git a/airbyte-webapp/src/packages/cloud/services/auth/CloudAuthService.tsx b/airbyte-webapp/src/packages/cloud/services/auth/CloudAuthService.tsx index 293412f0952..47377514531 100644 --- a/airbyte-webapp/src/packages/cloud/services/auth/CloudAuthService.tsx +++ b/airbyte-webapp/src/packages/cloud/services/auth/CloudAuthService.tsx @@ -20,7 +20,7 @@ import { sendPasswordResetEmail, } from "firebase/auth"; import React, { PropsWithChildren, useCallback, useEffect, useMemo, useState } from "react"; -import { useIntl } from "react-intl"; +import { FormattedMessage } from "react-intl"; import { useNavigate } from "react-router-dom"; import { Observable, Subject } from "rxjs"; @@ -54,7 +54,6 @@ export enum FirebaseAuthMessageId { export const CloudAuthService: React.FC = ({ children }) => { const [, setSpeedyConnectionTimestamp] = useLocalStorage("exp-speedy-connection-timestamp", ""); const [logoutInProgress, setLogoutInProgress] = useState(false); - const { formatMessage } = useIntl(); const queryClient = useQueryClient(); const { registerNotification } = useNotificationService(); const { mutateAsync: updateAirbyteUser } = useUpdateUser(); @@ -85,19 +84,19 @@ export const CloudAuthService: React.FC = ({ children }) => { } registerNotification({ id: "workspace.emailVerificationSuccess", - text: formatMessage({ id: "verifyEmail.success" }), + text: , type: "success", }); }) .catch(() => { registerNotification({ id: "workspace.emailVerificationError", - text: formatMessage({ id: "verifyEmail.error" }), + text: , type: "error", }); }); }, - [firebaseAuth, registerNotification, formatMessage, firebaseUser, emailDidVerify] + [firebaseAuth, registerNotification, firebaseUser, emailDidVerify] ); const logout = useCallback(async () => { @@ -179,7 +178,7 @@ export const CloudAuthService: React.FC = ({ children }) => { .then(() => { registerNotification({ id: "workspace.emailVerificationResendSuccess", - text: formatMessage({ id: "credits.emailVerification.resendConfirmation" }), + text: , type: "success", }); }) @@ -188,27 +187,21 @@ export const CloudAuthService: React.FC = ({ children }) => { case AuthErrorCodes.NETWORK_REQUEST_FAILED: registerNotification({ id: error.code, - text: formatMessage({ - id: FirebaseAuthMessageId.NetworkFailure, - }), + text: , type: "error", }); break; case AuthErrorCodes.TOO_MANY_ATTEMPTS_TRY_LATER: registerNotification({ id: error.code, - text: formatMessage({ - id: FirebaseAuthMessageId.TooManyRequests, - }), + text: , type: "warning", }); break; default: registerNotification({ id: error.code, - text: formatMessage({ - id: FirebaseAuthMessageId.DefaultError, - }), + text: , type: "error", }); } @@ -358,7 +351,6 @@ export const CloudAuthService: React.FC = ({ children }) => { firebaseAuth, firebaseInitialized, firebaseUser, - formatMessage, getAirbyteUser, keycloakAuth, logout, diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx index be11036a1db..229254f7f32 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx @@ -1,13 +1,12 @@ import React, { useCallback, useMemo, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { useDestinationDefinitionList, useUpdateDestinationDefinition } from "core/api"; +import { useDestinationDefinitionList, useUpdateDestinationDefinition, useDestinationList } from "core/api"; import { DestinationDefinitionRead } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useNotificationService } from "hooks/services/Notification"; import ConnectorsView from "./components/ConnectorsView"; -import { useDestinationList } from "../../../../hooks/services/useDestinationHook"; const DestinationsPage: React.FC = () => { useTrackPage(PageTrackingCodes.SETTINGS_DESTINATION); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx index d0bb271e9ad..bdc1e4b21aa 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx @@ -1,11 +1,10 @@ import React, { useCallback, useMemo, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; -import { useListBuilderProjects, useSourceDefinitionList, useUpdateSourceDefinition } from "core/api"; +import { useListBuilderProjects, useSourceDefinitionList, useUpdateSourceDefinition, useSourceList } from "core/api"; import { SourceDefinitionRead } from "core/request/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useNotificationService } from "hooks/services/Notification"; -import { useSourceList } from "hooks/services/useSourceHook"; import ConnectorsView, { ConnectorsViewProps } from "./components/ConnectorsView"; diff --git a/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionPage.tsx b/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionPage.tsx index b72c85e0d3a..5563eba4ab4 100644 --- a/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionPage.tsx +++ b/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionPage.tsx @@ -9,11 +9,10 @@ import { SelectSource } from "components/connection/CreateConnection/SelectSourc import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { useCurrentWorkspaceId } from "area/workspace/utils"; +import { useGetDestination, useGetSource } from "core/api"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { trackAction } from "core/utils/datadog"; import { AppActionCodes } from "hooks/services/AppMonitoringService"; -import { useGetDestination } from "hooks/services/useDestinationHook"; -import { useGetSource } from "hooks/services/useSourceHook"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocumentationLayout"; diff --git a/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx b/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx index e256cdac2dc..c25a57c0122 100644 --- a/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx +++ b/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx @@ -16,10 +16,10 @@ import { useSourceDefinitionVersion, useSourceDefinition, useDestinationDefinition, + useGetDestination, + useGetSource, } from "core/api"; import { SupportLevel } from "core/request/AirbyteClient"; -import { useGetDestination } from "hooks/services/useDestinationHook"; -import { useGetSource } from "hooks/services/useSourceHook"; import { RoutePaths } from "pages/routePaths"; import styles from "./CreateConnectionTitleBlock.module.scss"; diff --git a/airbyte-webapp/src/pages/destination/AllDestinationsPage/AllDestinationsPage.tsx b/airbyte-webapp/src/pages/destination/AllDestinationsPage/AllDestinationsPage.tsx index 47c73085d21..9b40a1d7a3d 100644 --- a/airbyte-webapp/src/pages/destination/AllDestinationsPage/AllDestinationsPage.tsx +++ b/airbyte-webapp/src/pages/destination/AllDestinationsPage/AllDestinationsPage.tsx @@ -10,8 +10,8 @@ import { Heading } from "components/ui/Heading"; import { Icon } from "components/ui/Icon"; import { PageHeader } from "components/ui/PageHeader"; +import { useDestinationList } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; -import { useDestinationList } from "hooks/services/useDestinationHook"; import { DestinationPaths } from "../../routePaths"; diff --git a/airbyte-webapp/src/pages/destination/CreateDestinationPage/CreateDestinationPage.tsx b/airbyte-webapp/src/pages/destination/CreateDestinationPage/CreateDestinationPage.tsx index 469dd3e4c30..048712658d1 100644 --- a/airbyte-webapp/src/pages/destination/CreateDestinationPage/CreateDestinationPage.tsx +++ b/airbyte-webapp/src/pages/destination/CreateDestinationPage/CreateDestinationPage.tsx @@ -13,10 +13,9 @@ import { Icon } from "components/ui/Icon"; import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { ConnectionConfiguration } from "area/connector/types"; -import { useDestinationDefinitionList } from "core/api"; +import { useDestinationDefinitionList, useCreateDestination } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; -import { useCreateDestination } from "hooks/services/useDestinationHook"; import { DestinationPaths, RoutePaths } from "pages/routePaths"; import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocumentationLayout"; diff --git a/airbyte-webapp/src/pages/destination/DestinationConnectionsPage/DestinationConnectionsPage.tsx b/airbyte-webapp/src/pages/destination/DestinationConnectionsPage/DestinationConnectionsPage.tsx index 70e8d4b6a11..b8595e8b427 100644 --- a/airbyte-webapp/src/pages/destination/DestinationConnectionsPage/DestinationConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationConnectionsPage/DestinationConnectionsPage.tsx @@ -9,8 +9,7 @@ import { DropdownMenuOptionType } from "components/ui/DropdownMenu"; import { FlexContainer } from "components/ui/Flex"; import { useGetDestinationFromParams } from "area/connector/utils"; -import { useCurrentWorkspace, useConnectionList } from "core/api"; -import { useSourceList } from "hooks/services/useSourceHook"; +import { useCurrentWorkspace, useConnectionList, useSourceList } from "core/api"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; export const DestinationConnectionsPage = () => { diff --git a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx index ee5b1f5b987..a4b55b44969 100644 --- a/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/destination/DestinationSettingsPage/DestinationSettingsPage.tsx @@ -10,14 +10,12 @@ import { useDestinationDefinitionVersion, useGetDestinationDefinitionSpecification, useDestinationDefinition, -} from "core/api"; -import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; -import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; -import { useDeleteDestination, useInvalidateDestination, useUpdateDestination, -} from "hooks/services/useDestinationHook"; +} from "core/api"; +import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; +import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; import { useDeleteModal } from "hooks/useDeleteModal"; import { ConnectorCard } from "views/Connector/ConnectorCard"; import { ConnectorCardValues } from "views/Connector/ConnectorForm/types"; diff --git a/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx b/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx index 51284035e67..c32202c1b31 100644 --- a/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx +++ b/airbyte-webapp/src/pages/source/AllSourcesPage/AllSourcesPage.tsx @@ -9,8 +9,8 @@ import { Heading } from "components/ui/Heading"; import { Icon } from "components/ui/Icon"; import { PageHeader } from "components/ui/PageHeader"; +import { useSourceList } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; -import { useSourceList } from "hooks/services/useSourceHook"; import { SourcesTable } from "./SourcesTable"; import { SourcePaths } from "../../routePaths"; diff --git a/airbyte-webapp/src/pages/source/CreateSourcePage/CreateSourcePage.tsx b/airbyte-webapp/src/pages/source/CreateSourcePage/CreateSourcePage.tsx index ed2fbebc482..4d48b072390 100644 --- a/airbyte-webapp/src/pages/source/CreateSourcePage/CreateSourcePage.tsx +++ b/airbyte-webapp/src/pages/source/CreateSourcePage/CreateSourcePage.tsx @@ -12,10 +12,9 @@ import { Icon } from "components/ui/Icon"; import { PageHeaderWithNavigation } from "components/ui/PageHeader"; import { ConnectionConfiguration } from "area/connector/types"; -import { useSourceDefinitionList } from "core/api"; +import { useSourceDefinitionList, useCreateSource } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useFormChangeTrackerService } from "hooks/services/FormChangeTracker"; -import { useCreateSource } from "hooks/services/useSourceHook"; import { SourcePaths, RoutePaths } from "pages/routePaths"; import { ConnectorDocumentationWrapper } from "views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationWrapper"; diff --git a/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionsPage.tsx b/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionsPage.tsx index 5a8b35bbd6d..0bb5a89a8bd 100644 --- a/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionsPage.tsx @@ -8,8 +8,7 @@ import { DropdownMenuOptionType } from "components/ui/DropdownMenu"; import { FlexContainer } from "components/ui/Flex/FlexContainer"; import { useGetSourceFromParams } from "area/connector/utils"; -import { useCurrentWorkspace, useConnectionList } from "core/api"; -import { useDestinationList } from "hooks/services/useDestinationHook"; +import { useCurrentWorkspace, useConnectionList, useDestinationList } from "core/api"; import { ConnectionRoutePaths, RoutePaths } from "pages/routePaths"; const SourceConnectionTable = React.lazy(() => import("./SourceConnectionTable")); diff --git a/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx b/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx index a4cefc3bdff..fdc07f404a7 100644 --- a/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx +++ b/airbyte-webapp/src/pages/source/SourceSettingsPage/SourceSettingsPage.tsx @@ -10,10 +10,12 @@ import { useSourceDefinitionVersion, useGetSourceDefinitionSpecification, useSourceDefinition, + useDeleteSource, + useInvalidateSource, + useUpdateSource, } from "core/api"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useFormChangeTrackerService, useUniqueFormId } from "hooks/services/FormChangeTracker"; -import { useDeleteSource, useInvalidateSource, useUpdateSource } from "hooks/services/useSourceHook"; import { useDeleteModal } from "hooks/useDeleteModal"; import { ConnectorCard } from "views/Connector/ConnectorCard"; import { ConnectorCardValues } from "views/Connector/ConnectorForm"; diff --git a/airbyte-webapp/src/services/useInitService.tsx b/airbyte-webapp/src/services/useInitService.tsx deleted file mode 100644 index e46f2c956ac..00000000000 --- a/airbyte-webapp/src/services/useInitService.tsx +++ /dev/null @@ -1,19 +0,0 @@ -import { DependencyList, useRef } from "react"; -import { useUpdateEffect } from "react-use"; - -export function useInitService(f: () => T, deps: DependencyList): T { - const service = useRef(null); - - useUpdateEffect(() => { - if (service.current !== null) { - service.current = f(); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, deps); - - if (service.current === null) { - service.current = f(); - } - - return service.current; -} diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx index e307a0b6bed..490b416479f 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx @@ -21,11 +21,8 @@ import { DocumentationPanelContext } from "../ConnectorDocumentationLayout/Docum // hack to fix tests. https://github.com/remarkjs/react-markdown/issues/635 jest.mock("components/ui/Markdown", () => ({ children }: React.PropsWithChildren) => <>{children}); -jest.mock("../../../hooks/services/useDestinationHook", () => ({ - useDestinationList: () => ({ destinations: [] }), -})); - jest.mock("core/api", () => ({ + useDestinationList: () => ({ destinations: [] }), useConsentUrls: () => ({ getSourceConsentUrl: () => "http://example.com" }), useCompleteOAuth: jest.fn(() => ({ completeSourceOAuth: () => Promise.resolve({}), From 6f9e01a802eb9a633a3e806395858ef0b8bf0ae9 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 12 Dec 2023 16:19:31 +0100 Subject: [PATCH 03/27] =?UTF-8?q?=F0=9F=AA=9F=F0=9F=A7=B9=20Remove=20core/?= =?UTF-8?q?request=20module=20(#10316)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- airbyte-webapp/.eslintLegacyFolderStructure.js | 1 - .../components/AttemptDetails/AttemptDetails.tsx | 2 +- .../components/JobHistoryItem/JobStatusIcon.tsx | 2 +- .../components/JobHistoryItem/JobStatusLabel.tsx | 2 +- .../components/JobHistoryItem/useCleanLogs.tsx | 2 +- .../components/JobLogsModal/AttemptStatusIcon.tsx | 2 +- .../JobLogsModal/JobLogsModalFailureMessage.tsx | 2 +- airbyte-webapp/src/area/connection/types/jobs.ts | 2 +- .../src/area/connection/utils/attemptLink.ts | 2 +- .../connection/utils/computeStreamStatus.test.ts | 2 +- .../area/connection/utils/computeStreamStatus.ts | 2 +- .../src/area/connection/utils/jobs.test.ts | 2 +- airbyte-webapp/src/area/connection/utils/jobs.ts | 2 +- .../components/NoWorkspacesPermissionWarning.tsx | 2 +- .../utils/useConfirmWorkspaceDeletionModal.tsx | 2 +- .../src/components/EntityTable/ConnectionTable.tsx | 2 +- .../EntityTable/components/ChangesStatusIcon.tsx | 2 +- .../EntityTable/components/FrequencyCell.tsx | 2 +- .../EntityTable/components/StatusCell.tsx | 2 +- airbyte-webapp/src/components/EntityTable/types.ts | 2 +- airbyte-webapp/src/components/EntityTable/utils.tsx | 2 +- .../src/components/JobFailure/JobFailure.tsx | 2 +- .../NotificationSettingsForm.tsx | 2 +- .../NotificationSettingsForm/TestWebhookButton.tsx | 2 +- .../formValuesToNotificationSettings.test.ts | 2 +- .../formValuesToNotificationSettings.ts | 2 +- .../notificationSettingsToFormValues.test.ts | 2 +- .../notificationSettingsToFormValues.ts | 2 +- .../common/ApiErrorBoundary/ApiErrorBoundary.tsx | 3 +-- .../CatalogDiffModal/CatalogDiffModal.stories.tsx | 2 +- .../CatalogDiffModal/CatalogDiffModal.test.tsx | 2 +- .../CatalogDiffModal/CatalogDiffModal.tsx | 2 +- .../connection/CatalogDiffModal/DiffAccordion.tsx | 2 +- .../CatalogDiffModal/DiffAccordionHeader.tsx | 2 +- .../connection/CatalogDiffModal/DiffFieldTable.tsx | 2 +- .../connection/CatalogDiffModal/DiffSection.tsx | 2 +- .../connection/CatalogDiffModal/FieldRow.tsx | 2 +- .../connection/CatalogDiffModal/FieldSection.tsx | 2 +- .../connection/CatalogDiffModal/StreamRow.tsx | 2 +- .../components/connection/CatalogDiffModal/types.ts | 2 +- .../connection/CatalogDiffModal/utils.tsx | 2 +- .../ConnectionConfigurationPreview.tsx | 2 +- .../ConnectionForm/NamespaceDefinitionFormField.tsx | 2 +- .../NonBreakingChangesPreferenceFormField.tsx | 2 +- .../ScheduleFormField/BasicScheduleFormControl.tsx | 2 +- .../ScheduleFormField/CronScheduleFormControl.tsx | 2 +- .../ScheduleFormField/ScheduleFormField.tsx | 2 +- .../ScheduleFormField/ScheduleTypeFormControl.tsx | 2 +- .../useBasicFrequencyDropdownData.test.tsx | 2 +- .../useBasicFrequencyDropdownData.tsx | 2 +- .../useTrackConnectionFrequency.ts | 2 +- .../ConnectionForm/SchemaChangesBackdrop.test.tsx | 2 +- .../ConnectionForm/calculateInitialCatalog.test.ts | 2 +- .../ConnectionForm/calculateInitialCatalog.ts | 2 +- .../connection/ConnectionForm/formConfig.tsx | 4 +--- .../connection/ConnectionForm/frequencyConfig.ts | 2 +- .../components/connection/ConnectionForm/schema.ts | 4 ++-- .../components/connection/ConnectionForm/types.ts | 2 +- .../components/connection/ConnectionForm/utils.ts | 2 +- .../ConnectionOnboarding/ConnectionOnboarding.tsx | 2 +- .../ConnectionSync/ConnectionSyncButtons.tsx | 2 +- .../CreateConnection/SelectExistingConnector.tsx | 2 +- .../CreateConnectionForm.test.tsx | 1 + .../connection/CreateConnectionForm/SchemaError.tsx | 3 +-- .../DestinationNamespaceDescription.tsx | 2 +- .../DestinationNamespaceModal.tsx | 2 +- .../ExampleSettingsTable.tsx | 2 +- .../useExampleSettingsTable.ts | 2 +- .../connection/EnabledControl/EnabledControl.tsx | 2 +- .../connection/JobProgress/JobProgress.tsx | 2 +- .../connection/JobProgress/StreamProgress.tsx | 2 +- .../components/connection/JobProgress/utils.test.ts | 2 +- .../src/components/connection/JobProgress/utils.ts | 2 +- .../connection/StreamStatus/streamStatusUtils.tsx | 2 +- .../connection/TransformationForm/types.ts | 2 +- .../StreamDetailsPanel/StreamDetailsPanel.tsx | 2 +- .../StreamDetailsPanel/StreamPanelHeader.tsx | 2 +- .../StreamFieldsTable/StreamFieldsTable.test.tsx | 2 +- .../StreamFieldsTable/StreamFieldsTable.tsx | 2 +- .../syncCatalog/StreamFieldsTable/SyncFieldCell.tsx | 2 +- .../StreamsConfigTable/StreamsConfigTableHeader.tsx | 2 +- .../useStreamsConfigTableRowProps.test.tsx | 2 +- .../syncCatalog/SyncCatalog/SyncCatalogRow.tsx | 2 +- .../SyncCatalog/streamConfigHelpers.test.ts | 2 +- .../syncCatalog/SyncCatalog/streamConfigHelpers.ts | 2 +- .../syncCatalog/SyncCatalog/updateStreamSyncMode.ts | 2 +- .../syncCatalog/SyncCatalog/useStreamFilters.tsx | 2 +- .../syncCatalog/SyncModeSelect/SyncModeSelect.tsx | 2 +- .../connector/ConnectorNavigationTabs.tsx | 2 +- .../components/connector/ConnectorTitleBlock.tsx | 2 +- .../StreamTestingPanel/StreamTester.tsx | 2 +- .../DestinationConnectionTable.tsx | 2 +- .../destination/DestinationForm/DestinationForm.tsx | 3 +-- .../src/components/forms/DataResidencyDropdown.tsx | 2 +- .../source/SelectConnector/FilterSupportLevel.tsx | 2 +- .../source/SelectConnector/SelectConnector.tsx | 2 +- .../ConnectorDefinitionBranding.tsx | 2 +- .../ui/SupportLevelBadge/SupportLevelBadge.tsx | 2 +- .../components/workspace/WorkspacesPickerList.tsx | 2 +- airbyte-webapp/src/core/api/apiCall.ts | 4 ++-- .../{request => api/errors}/CommonRequestError.ts | 0 .../{request => api/errors}/LogsRequestError.ts | 3 ++- .../src/core/{request => api/errors}/ServerError.ts | 0 .../core/{request => api/errors}/VersionError.ts | 0 airbyte-webapp/src/core/api/errors/index.ts | 4 ++++ .../src/core/api/hooks/actorDefinitionVersions.ts | 2 +- .../src/core/api/hooks/cloud/cloudWorkspaces.ts | 2 +- airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts | 4 ++-- airbyte-webapp/src/core/api/hooks/cloud/users.ts | 6 +++--- airbyte-webapp/src/core/api/hooks/connections.tsx | 2 +- .../src/core/api/hooks/connectorBuilderProject.ts | 2 +- airbyte-webapp/src/core/api/hooks/connectorCheck.ts | 2 +- .../api/hooks/connectorDefinitionSpecification.ts | 2 +- .../src/core/api/hooks/connectorUpdates.ts | 2 +- .../src/core/api/hooks/destinationDefinitions.ts | 2 +- airbyte-webapp/src/core/api/hooks/destinations.tsx | 2 +- airbyte-webapp/src/core/api/hooks/geographies.ts | 3 +-- airbyte-webapp/src/core/api/hooks/jobs.ts | 2 +- airbyte-webapp/src/core/api/hooks/organizations.ts | 2 +- airbyte-webapp/src/core/api/hooks/permissions.ts | 2 +- .../src/core/api/hooks/sourceDefinitions.ts | 2 +- airbyte-webapp/src/core/api/hooks/sources.tsx | 2 +- airbyte-webapp/src/core/api/hooks/streams.ts | 2 +- airbyte-webapp/src/core/api/hooks/workspaces.ts | 2 +- airbyte-webapp/src/core/api/index.ts | 2 ++ .../src/{services/Scope.ts => core/api/scopes.ts} | 0 .../src/core/domain/connector/connector.ts | 2 +- airbyte-webapp/src/core/domain/connector/source.ts | 13 +++++++------ airbyte-webapp/src/core/domain/connector/types.ts | 2 +- airbyte-webapp/src/core/request/AirbyteClient.ts | 8 -------- airbyte-webapp/src/core/services/analytics/utils.ts | 2 +- .../core/services/auth/EnterpriseAuthService.tsx | 2 +- airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx | 2 +- airbyte-webapp/src/core/utils/rbac/rbac.test.ts | 2 +- .../src/core/utils/rbac/rbacPermissionsQuery.ts | 2 +- airbyte-webapp/src/core/utils/useLocalStorage.ts | 2 +- .../src/hooks/connection/useDestinationNamespace.ts | 2 +- .../src/hooks/connection/useSchemaChanges.ts | 2 +- .../ConnectionForm/ConnectionFormService.tsx | 2 +- .../src/hooks/services/useConnectorAuth.tsx | 9 ++++----- airbyte-webapp/src/hooks/services/useWorkspace.tsx | 2 +- .../auth/KeycloakService/KeycloakService.tsx | 2 +- .../BillingPage/components/CreditsUsageContext.tsx | 2 +- .../BillingPage/components/CreditsUsageFilters.tsx | 2 +- .../components/UsagePerConnectionTable.tsx | 2 +- .../InsufficientPermissionsErrorBoundary.tsx | 2 +- .../DataResidencyView/DataResidencyView.tsx | 2 +- .../WorkspacesPage/CloudWorkspacesCreateControl.tsx | 2 +- .../GeneralOrganizationSettingsPage.tsx | 2 +- .../components/AccessManagementCard.tsx | 2 +- .../components/AccessManagementTable.tsx | 2 +- .../components/AddUserControl.tsx | 2 +- .../components/RoleManagementControl.tsx | 2 +- .../components/useGetAccessManagementData.tsx | 2 +- .../pages/ConnectorsPage/DestinationsPage.tsx | 2 +- .../pages/ConnectorsPage/SourcesPage.tsx | 2 +- .../ConnectorsPage/components/ConnectorCell.tsx | 2 +- .../ConnectorsPage/components/ConnectorsView.tsx | 2 +- .../AllConnectionsPage/AllConnectionsPage.tsx | 2 +- .../AllConnectionsPage/ConnectionsTable.tsx | 2 +- .../ConnectionJobHistoryPage/JobsList.tsx | 2 +- .../ConnectionPage/ConnectionTitleBlock.tsx | 2 +- .../ConnectionReplicationPage.test.tsx | 1 + .../ConnectionReplicationPage/ResetWarningModal.tsx | 2 +- .../ConnectionSettingsPage.tsx | 2 +- .../ConnectionSettingsPage/StateBlock.tsx | 2 +- .../CreateConnectionTitleBlock.tsx | 2 +- .../StreamStatusPage/ConnectionStatusMessages.tsx | 4 ++-- .../StreamStatusPage/ConnectionStatusOverview.tsx | 2 +- .../connections/StreamStatusPage/StreamsList.tsx | 2 +- airbyte-webapp/src/pages/routes.tsx | 2 +- .../pages/source/AllSourcesPage/SourcesTable.tsx | 2 +- .../pages/source/CreateSourcePage/SourceForm.tsx | 3 +-- .../SourceConnectionsPage/SourceConnectionTable.tsx | 2 +- .../components/WorkspacesCreateControl.tsx | 2 +- .../pages/workspaces/components/WorkspacesList.tsx | 2 +- .../ConnectorBuilderStateService.tsx | 2 +- .../src/test-utils/mock-data/mockAirbyteStream.ts | 2 +- .../mock-data/mockAirbyteStreamConfiguration.ts | 2 +- .../src/test-utils/mock-data/mockAttempt.ts | 2 +- .../src/test-utils/mock-data/mockCatalogDiff.ts | 2 +- .../src/test-utils/mock-data/mockConnection.ts | 2 +- .../src/test-utils/mock-data/mockDestination.ts | 2 +- .../mockDestinationDefinitionSpecification.ts | 2 +- .../src/test-utils/mock-data/mockInstanceConfig.ts | 2 +- airbyte-webapp/src/test-utils/mock-data/mockJob.ts | 2 +- .../src/test-utils/mock-data/mockSource.ts | 2 +- .../test-utils/mock-data/mockSourceDefinition.ts | 2 +- .../test-utils/mock-data/mockStreamStatusRead.ts | 2 +- .../src/test-utils/mock-data/mockWorkspace.ts | 2 +- airbyte-webapp/src/test-utils/testutils.tsx | 2 +- .../views/Connector/ConnectorCard/ConnectorCard.tsx | 4 ++-- .../Connector/ConnectorCard/components/Controls.tsx | 2 +- .../Connector/ConnectorCard/components/TestCard.tsx | 2 +- .../ConnectorCard/useAnalyticsTrackFunctions.tsx | 2 +- .../Connector/ConnectorCard/useTestConnector.tsx | 2 +- .../Connector/ConnectorForm/ConnectorForm.test.tsx | 2 +- .../Sections/auth/useOauthFlowAdapter.tsx | 2 +- .../Sections/auth/useOauthRevocationAdapter.tsx | 2 +- .../ConnectorForm/components/WarningMessage.tsx | 2 +- .../ConnectorForm/useAuthentication.test.tsx | 2 +- .../src/views/Connector/ConnectorForm/utils.ts | 2 +- .../views/common/ResourceNotFoundErrorBoundary.tsx | 2 +- 203 files changed, 218 insertions(+), 225 deletions(-) rename airbyte-webapp/src/core/{request => api/errors}/CommonRequestError.ts (100%) rename airbyte-webapp/src/core/{request => api/errors}/LogsRequestError.ts (89%) rename airbyte-webapp/src/core/{request => api/errors}/ServerError.ts (100%) rename airbyte-webapp/src/core/{request => api/errors}/VersionError.ts (100%) create mode 100644 airbyte-webapp/src/core/api/errors/index.ts rename airbyte-webapp/src/{services/Scope.ts => core/api/scopes.ts} (100%) delete mode 100644 airbyte-webapp/src/core/request/AirbyteClient.ts diff --git a/airbyte-webapp/.eslintLegacyFolderStructure.js b/airbyte-webapp/.eslintLegacyFolderStructure.js index 98ae7e877b5..2c6d93e195e 100644 --- a/airbyte-webapp/.eslintLegacyFolderStructure.js +++ b/airbyte-webapp/.eslintLegacyFolderStructure.js @@ -5,7 +5,6 @@ module.exports = [ "src/services/connectorBuilder/ConnectorBuilderStateService.tsx", "src/services/connectorBuilder/ConnectorBuilderLocalStorageService.tsx", "src/services/connectorBuilder/SchemaWorker.ts", - "src/services/Scope.ts", // src/hooks "src/hooks/useDeleteModal.tsx", "src/hooks/useTypesafeReducer.ts", diff --git a/airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.tsx b/airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.tsx index cefc3133222..d0e4e61b57a 100644 --- a/airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.tsx +++ b/airbyte-webapp/src/area/connection/components/AttemptDetails/AttemptDetails.tsx @@ -5,7 +5,7 @@ import { FormattedDate, FormattedMessage, FormattedTimeParts, useIntl } from "re import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; -import { AttemptRead, AttemptStatus, FailureReason, FailureType } from "core/request/AirbyteClient"; +import { AttemptRead, AttemptStatus, FailureReason, FailureType } from "core/api/types/AirbyteClient"; import { formatBytes } from "core/utils/numberHelper"; import styles from "./AttemptDetails.module.scss"; diff --git a/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusIcon.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusIcon.tsx index bcc800752b8..c14ed74d9e0 100644 --- a/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusIcon.tsx +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusIcon.tsx @@ -2,7 +2,7 @@ import { StatusIcon } from "components/ui/StatusIcon"; import { JobWithAttempts } from "area/connection/types/jobs"; import { isJobPartialSuccess, didJobSucceed, getJobStatus } from "area/connection/utils/jobs"; -import { JobStatus } from "core/request/AirbyteClient"; +import { JobStatus } from "core/api/types/AirbyteClient"; interface JobStatusIconProps { job: JobWithAttempts; diff --git a/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusLabel.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusLabel.tsx index 1004f1f71c4..d33efde3823 100644 --- a/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusLabel.tsx +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/JobStatusLabel.tsx @@ -4,7 +4,7 @@ import { Text } from "components/ui/Text"; import { JobWithAttempts } from "area/connection/types/jobs"; import { isJobPartialSuccess, getJobAttempts, getJobStatus } from "area/connection/utils/jobs"; -import { JobStatus } from "core/request/AirbyteClient"; +import { JobStatus } from "core/api/types/AirbyteClient"; interface JobStatusLabelProps { jobWithAttempts: JobWithAttempts; diff --git a/airbyte-webapp/src/area/connection/components/JobHistoryItem/useCleanLogs.tsx b/airbyte-webapp/src/area/connection/components/JobHistoryItem/useCleanLogs.tsx index 789fba71c0f..a04d509dde5 100644 --- a/airbyte-webapp/src/area/connection/components/JobHistoryItem/useCleanLogs.tsx +++ b/airbyte-webapp/src/area/connection/components/JobHistoryItem/useCleanLogs.tsx @@ -1,7 +1,7 @@ import Anser from "anser"; import { useMemo } from "react"; -import { AttemptInfoRead } from "core/request/AirbyteClient"; +import { AttemptInfoRead } from "core/api/types/AirbyteClient"; export type CleanedLogLines = Array<{ original: string; diff --git a/airbyte-webapp/src/area/connection/components/JobLogsModal/AttemptStatusIcon.tsx b/airbyte-webapp/src/area/connection/components/JobLogsModal/AttemptStatusIcon.tsx index 5474980710b..34903d88258 100644 --- a/airbyte-webapp/src/area/connection/components/JobLogsModal/AttemptStatusIcon.tsx +++ b/airbyte-webapp/src/area/connection/components/JobLogsModal/AttemptStatusIcon.tsx @@ -1,6 +1,6 @@ import { StatusIcon } from "components/ui/StatusIcon"; -import { AttemptInfoRead, AttemptStatus } from "core/request/AirbyteClient"; +import { AttemptInfoRead, AttemptStatus } from "core/api/types/AirbyteClient"; interface AttemptStatusIconProps { attempt: AttemptInfoRead; diff --git a/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModalFailureMessage.tsx b/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModalFailureMessage.tsx index 4c194b512e6..35a87386acd 100644 --- a/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModalFailureMessage.tsx +++ b/airbyte-webapp/src/area/connection/components/JobLogsModal/JobLogsModalFailureMessage.tsx @@ -6,7 +6,7 @@ import { Button } from "components/ui/Button"; import { FlexContainer } from "components/ui/Flex"; import { Message } from "components/ui/Message"; -import { AttemptFailureSummary, FailureType } from "core/request/AirbyteClient"; +import { AttemptFailureSummary, FailureType } from "core/api/types/AirbyteClient"; import { copyToClipboard } from "core/utils/clipboard"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/area/connection/types/jobs.ts b/airbyte-webapp/src/area/connection/types/jobs.ts index b87605c0ea0..f2da4fe5e5d 100644 --- a/airbyte-webapp/src/area/connection/types/jobs.ts +++ b/airbyte-webapp/src/area/connection/types/jobs.ts @@ -1,4 +1,4 @@ -import { JobWithAttemptsRead } from "core/request/AirbyteClient"; +import { JobWithAttemptsRead } from "core/api/types/AirbyteClient"; // JobWithAttemptsRead has an optional job property, but we really want it to be required export interface JobWithAttempts extends JobWithAttemptsRead { diff --git a/airbyte-webapp/src/area/connection/utils/attemptLink.ts b/airbyte-webapp/src/area/connection/utils/attemptLink.ts index d4d22bb3e23..d71243b985f 100644 --- a/airbyte-webapp/src/area/connection/utils/attemptLink.ts +++ b/airbyte-webapp/src/area/connection/utils/attemptLink.ts @@ -1,6 +1,6 @@ import { useLocation } from "react-router-dom"; -import { AttemptRead } from "core/request/AirbyteClient"; +import { AttemptRead } from "core/api/types/AirbyteClient"; const PARSE_REGEXP = /^#(?\w*)::(?\w*)$/; diff --git a/airbyte-webapp/src/area/connection/utils/computeStreamStatus.test.ts b/airbyte-webapp/src/area/connection/utils/computeStreamStatus.test.ts index 147adf4c38d..b830f932440 100644 --- a/airbyte-webapp/src/area/connection/utils/computeStreamStatus.test.ts +++ b/airbyte-webapp/src/area/connection/utils/computeStreamStatus.test.ts @@ -11,7 +11,7 @@ import { StreamStatusJobType, StreamStatusRead, StreamStatusRunState, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { computeStreamStatus, getStreamKey } from "./computeStreamStatus"; diff --git a/airbyte-webapp/src/area/connection/utils/computeStreamStatus.ts b/airbyte-webapp/src/area/connection/utils/computeStreamStatus.ts index a4b34ad348b..b1348910fde 100644 --- a/airbyte-webapp/src/area/connection/utils/computeStreamStatus.ts +++ b/airbyte-webapp/src/area/connection/utils/computeStreamStatus.ts @@ -11,7 +11,7 @@ import { StreamStatusJobType, StreamStatusRead, StreamStatusRunState, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; export type AirbyteStreamAndConfigurationWithEnforcedStream = AirbyteStreamAndConfiguration & { stream: AirbyteStream }; diff --git a/airbyte-webapp/src/area/connection/utils/jobs.test.ts b/airbyte-webapp/src/area/connection/utils/jobs.test.ts index 4cd69bbd936..2a0367f5f05 100644 --- a/airbyte-webapp/src/area/connection/utils/jobs.test.ts +++ b/airbyte-webapp/src/area/connection/utils/jobs.test.ts @@ -1,6 +1,6 @@ import { mockAttempt } from "test-utils/mock-data/mockAttempt"; -import { AttemptRead, JobStatus } from "core/request/AirbyteClient"; +import { AttemptRead, JobStatus } from "core/api/types/AirbyteClient"; import { isJobPartialSuccess } from "./jobs"; diff --git a/airbyte-webapp/src/area/connection/utils/jobs.ts b/airbyte-webapp/src/area/connection/utils/jobs.ts index d8f4241a851..92ac78193a4 100644 --- a/airbyte-webapp/src/area/connection/utils/jobs.ts +++ b/airbyte-webapp/src/area/connection/utils/jobs.ts @@ -1,4 +1,4 @@ -import { AttemptRead, JobStatus, SynchronousJobRead } from "core/request/AirbyteClient"; +import { AttemptRead, JobStatus, SynchronousJobRead } from "core/api/types/AirbyteClient"; import { JobWithAttempts } from "../types/jobs"; diff --git a/airbyte-webapp/src/area/workspace/components/NoWorkspacesPermissionWarning.tsx b/airbyte-webapp/src/area/workspace/components/NoWorkspacesPermissionWarning.tsx index a534f404f79..0b90ca0b7bb 100644 --- a/airbyte-webapp/src/area/workspace/components/NoWorkspacesPermissionWarning.tsx +++ b/airbyte-webapp/src/area/workspace/components/NoWorkspacesPermissionWarning.tsx @@ -5,7 +5,7 @@ import { FlexContainer } from "components/ui/Flex"; import { ExternalLink } from "components/ui/Link"; import { Text } from "components/ui/Text"; -import { OrganizationRead } from "core/request/AirbyteClient"; +import { OrganizationRead } from "core/api/types/AirbyteClient"; import styles from "./NoWorkspacesPermissionWarning.module.scss"; import OctaviaThinking from "./octavia-thinking-no-gears.svg?react"; diff --git a/airbyte-webapp/src/area/workspace/utils/useConfirmWorkspaceDeletionModal.tsx b/airbyte-webapp/src/area/workspace/utils/useConfirmWorkspaceDeletionModal.tsx index 14b732f95ba..d38de7334d1 100644 --- a/airbyte-webapp/src/area/workspace/utils/useConfirmWorkspaceDeletionModal.tsx +++ b/airbyte-webapp/src/area/workspace/utils/useConfirmWorkspaceDeletionModal.tsx @@ -8,7 +8,7 @@ import { Button } from "components/ui/Button"; import { Input } from "components/ui/Input"; import { Text } from "components/ui/Text"; -import { WorkspaceRead } from "core/request/AirbyteClient"; +import { WorkspaceRead } from "core/api/types/AirbyteClient"; import { useModalService } from "hooks/services/Modal"; import { useNotificationService } from "hooks/services/Notification"; import { RoutePaths } from "pages/routePaths"; diff --git a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx index b49246b67ef..b223d37e4e7 100644 --- a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx +++ b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx @@ -4,7 +4,7 @@ import { FormattedMessage } from "react-intl"; import { Table } from "components/ui/Table"; -import { ConnectionScheduleType, SchemaChange } from "core/request/AirbyteClient"; +import { ConnectionScheduleType, SchemaChange } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import ConnectionSettingsCell from "./components/ConnectionSettingsCell"; diff --git a/airbyte-webapp/src/components/EntityTable/components/ChangesStatusIcon.tsx b/airbyte-webapp/src/components/EntityTable/components/ChangesStatusIcon.tsx index 26dfc6b1f34..99b95589986 100644 --- a/airbyte-webapp/src/components/EntityTable/components/ChangesStatusIcon.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/ChangesStatusIcon.tsx @@ -5,7 +5,7 @@ import { FormattedMessage } from "react-intl"; import { Icon } from "components/ui/Icon"; import { Tooltip } from "components/ui/Tooltip"; -import { SchemaChange } from "core/request/AirbyteClient"; +import { SchemaChange } from "core/api/types/AirbyteClient"; import { convertSnakeToCamel } from "core/utils/strings"; import styles from "./ChangesStatusIcon.module.scss"; diff --git a/airbyte-webapp/src/components/EntityTable/components/FrequencyCell.tsx b/airbyte-webapp/src/components/EntityTable/components/FrequencyCell.tsx index 337581be2c3..df8f9df643b 100644 --- a/airbyte-webapp/src/components/EntityTable/components/FrequencyCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/FrequencyCell.tsx @@ -4,7 +4,7 @@ import { FormattedMessage } from "react-intl"; import { Text } from "components/ui/Text"; -import { ConnectionScheduleData, ConnectionScheduleType } from "core/request/AirbyteClient"; +import { ConnectionScheduleData, ConnectionScheduleType } from "core/api/types/AirbyteClient"; import styles from "./FrequencyCell.module.scss"; diff --git a/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx b/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx index 94bc9710b8b..6b9faf96450 100644 --- a/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/StatusCell.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { SchemaChange, WebBackendConnectionListItem } from "core/request/AirbyteClient"; +import { SchemaChange, WebBackendConnectionListItem } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { ChangesStatusIcon } from "./ChangesStatusIcon"; diff --git a/airbyte-webapp/src/components/EntityTable/types.ts b/airbyte-webapp/src/components/EntityTable/types.ts index 7ac43bf70b3..b87fb03c2df 100644 --- a/airbyte-webapp/src/components/EntityTable/types.ts +++ b/airbyte-webapp/src/components/EntityTable/types.ts @@ -3,7 +3,7 @@ import { ConnectionScheduleType, SchemaChange, WebBackendConnectionListItem, -} from "../../core/request/AirbyteClient"; +} from "../../core/api/types/AirbyteClient"; interface EntityTableDataItem { entityId: string; diff --git a/airbyte-webapp/src/components/EntityTable/utils.tsx b/airbyte-webapp/src/components/EntityTable/utils.tsx index 8d5a72abb9d..d979ec4d2c8 100644 --- a/airbyte-webapp/src/components/EntityTable/utils.tsx +++ b/airbyte-webapp/src/components/EntityTable/utils.tsx @@ -6,7 +6,7 @@ import { SourceRead, SourceSnippetRead, WebBackendConnectionListItem, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { EntityTableDataItem, ConnectionTableDataItem, Status as ConnectionSyncStatus } from "./types"; diff --git a/airbyte-webapp/src/components/JobFailure/JobFailure.tsx b/airbyte-webapp/src/components/JobFailure/JobFailure.tsx index 9ccb3151457..29842b386f0 100644 --- a/airbyte-webapp/src/components/JobFailure/JobFailure.tsx +++ b/airbyte-webapp/src/components/JobFailure/JobFailure.tsx @@ -8,7 +8,7 @@ import { Icon } from "components/ui/Icon"; import { Message } from "components/ui/Message"; import { Text } from "components/ui/Text"; -import { JobConfigType, SynchronousJobRead } from "core/request/AirbyteClient"; +import { JobConfigType, SynchronousJobRead } from "core/api/types/AirbyteClient"; import { downloadFile } from "core/utils/file"; import styles from "./JobFailure.module.scss"; diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx index afccd15ae97..d38caa6a936 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx +++ b/airbyte-webapp/src/components/NotificationSettingsForm/NotificationSettingsForm.tsx @@ -12,7 +12,7 @@ import { Box } from "components/ui/Box"; import { Text } from "components/ui/Text"; import { useCurrentWorkspace, useTryNotificationWebhook } from "core/api"; -import { NotificationReadStatus, NotificationSettings, NotificationTrigger } from "core/request/AirbyteClient"; +import { NotificationReadStatus, NotificationSettings, NotificationTrigger } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { isFulfilled } from "core/utils/promises"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/TestWebhookButton.tsx b/airbyte-webapp/src/components/NotificationSettingsForm/TestWebhookButton.tsx index 9cdc4646cff..9388c7f2ccd 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/TestWebhookButton.tsx +++ b/airbyte-webapp/src/components/NotificationSettingsForm/TestWebhookButton.tsx @@ -4,7 +4,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import { Button } from "components/ui/Button"; import { useTryNotificationWebhook } from "core/api"; -import { NotificationReadStatus, NotificationSettings } from "core/request/AirbyteClient"; +import { NotificationReadStatus, NotificationSettings } from "core/api/types/AirbyteClient"; import { useNotificationService } from "hooks/services/Notification"; import { notificationTriggerMap } from "./NotificationSettingsForm"; diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/formValuesToNotificationSettings.test.ts b/airbyte-webapp/src/components/NotificationSettingsForm/formValuesToNotificationSettings.test.ts index fc7abf751f7..fa860d94c9d 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/formValuesToNotificationSettings.test.ts +++ b/airbyte-webapp/src/components/NotificationSettingsForm/formValuesToNotificationSettings.test.ts @@ -1,4 +1,4 @@ -import { NotificationItem, NotificationSettings } from "core/request/AirbyteClient"; +import { NotificationItem, NotificationSettings } from "core/api/types/AirbyteClient"; import { formValuesToNotificationSettings } from "./formValuesToNotificationSettings"; import { NotificationItemFieldValue, NotificationSettingsFormValues } from "./NotificationSettingsForm"; diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/formValuesToNotificationSettings.ts b/airbyte-webapp/src/components/NotificationSettingsForm/formValuesToNotificationSettings.ts index e5fdaed4f36..a4714fc992d 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/formValuesToNotificationSettings.ts +++ b/airbyte-webapp/src/components/NotificationSettingsForm/formValuesToNotificationSettings.ts @@ -1,4 +1,4 @@ -import { NotificationItem, NotificationSettings } from "core/request/AirbyteClient"; +import { NotificationItem, NotificationSettings } from "core/api/types/AirbyteClient"; import { NotificationSettingsFormValues, notificationKeys } from "./NotificationSettingsForm"; diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts b/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts index 678500667b3..aebb3eacb1a 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts +++ b/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.test.ts @@ -1,4 +1,4 @@ -import { NotificationItem, NotificationSettings } from "core/request/AirbyteClient"; +import { NotificationItem, NotificationSettings } from "core/api/types/AirbyteClient"; import { NotificationItemFieldValue, NotificationSettingsFormValues } from "./NotificationSettingsForm"; import { notificationSettingsToFormValues } from "./notificationSettingsToFormValues"; diff --git a/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.ts b/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.ts index 0d07987ed4a..22c7400f41e 100644 --- a/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.ts +++ b/airbyte-webapp/src/components/NotificationSettingsForm/notificationSettingsToFormValues.ts @@ -1,4 +1,4 @@ -import { NotificationItem, NotificationSettings } from "core/request/AirbyteClient"; +import { NotificationItem, NotificationSettings } from "core/api/types/AirbyteClient"; import { NotificationSettingsFormValues, notificationKeys } from "./NotificationSettingsForm"; diff --git a/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx b/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx index 362f48be9b0..0b19eb0a45b 100644 --- a/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx +++ b/airbyte-webapp/src/components/common/ApiErrorBoundary/ApiErrorBoundary.tsx @@ -5,9 +5,8 @@ import { NavigateFunction, useNavigate } from "react-router-dom"; import { useLocation } from "react-use"; import { LocationSensorState } from "react-use/lib/useLocation"; +import { CommonRequestError, isVersionError } from "core/api"; import { isFormBuildError } from "core/form/FormBuildError"; -import { CommonRequestError } from "core/request/CommonRequestError"; -import { isVersionError } from "core/request/VersionError"; import { TrackErrorFn, useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { ErrorOccurredView } from "views/common/ErrorOccurredView"; import { ResourceNotFoundErrorBoundary } from "views/common/ResourceNotFoundErrorBoundary"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.stories.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.stories.tsx index 92e3b07e901..d613bf9dd9c 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.stories.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.stories.tsx @@ -3,7 +3,7 @@ import { FormattedMessage } from "react-intl"; import { Modal } from "components/ui/Modal"; -import { CatalogDiff } from "core/request/AirbyteClient"; +import { CatalogDiff } from "core/api/types/AirbyteClient"; import { ModalServiceProvider } from "hooks/services/Modal"; import { CatalogDiffModal } from "./CatalogDiffModal"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.test.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.test.tsx index 95fccf67706..3a362c9b06e 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.test.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.test.tsx @@ -8,7 +8,7 @@ import { DestinationSyncMode, StreamTransform, SyncMode, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { ModalServiceProvider } from "hooks/services/Modal"; import { CatalogDiffModal } from "./CatalogDiffModal"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx index 674f6ea6195..f6c29bc3cd5 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/CatalogDiffModal.tsx @@ -4,7 +4,7 @@ import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { ModalBody, ModalFooter } from "components/ui/Modal"; -import { AirbyteCatalog, CatalogDiff } from "core/request/AirbyteClient"; +import { AirbyteCatalog, CatalogDiff } from "core/api/types/AirbyteClient"; import styles from "./CatalogDiffModal.module.scss"; import { DiffSection } from "./DiffSection"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordion.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordion.tsx index 1392090fc7e..3ceab4a7869 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordion.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordion.tsx @@ -1,7 +1,7 @@ import { Disclosure } from "@headlessui/react"; import { useMemo } from "react"; -import { StreamTransform } from "core/request/AirbyteClient"; +import { StreamTransform } from "core/api/types/AirbyteClient"; import styles from "./DiffAccordion.module.scss"; import { DiffAccordionHeader } from "./DiffAccordionHeader"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordionHeader.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordionHeader.tsx index 59e2c7ab204..5676a7ed6f1 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordionHeader.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffAccordionHeader.tsx @@ -3,7 +3,7 @@ import { useIntl } from "react-intl"; import { Icon } from "components/ui/Icon"; -import { StreamDescriptor } from "core/request/AirbyteClient"; +import { StreamDescriptor } from "core/api/types/AirbyteClient"; import styles from "./DiffAccordionHeader.module.scss"; import { DiffIconBlock } from "./DiffIconBlock"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffFieldTable.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffFieldTable.tsx index aab8e0f02af..e66c74dd477 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffFieldTable.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffFieldTable.tsx @@ -1,6 +1,6 @@ import { FormattedMessage } from "react-intl"; -import { FieldTransform } from "core/request/AirbyteClient"; +import { FieldTransform } from "core/api/types/AirbyteClient"; import styles from "./DiffFieldTable.module.scss"; import { DiffHeader } from "./DiffHeader"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffSection.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffSection.tsx index c03cbcc7524..71ae578fbc8 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffSection.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/DiffSection.tsx @@ -1,6 +1,6 @@ import { FormattedMessage } from "react-intl"; -import { AirbyteCatalog, StreamDescriptor, StreamTransform } from "core/request/AirbyteClient"; +import { AirbyteCatalog, StreamDescriptor, StreamTransform } from "core/api/types/AirbyteClient"; import { DiffHeader } from "./DiffHeader"; import styles from "./DiffSection.module.scss"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.tsx index 059591e620d..633644a5ac4 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldRow.tsx @@ -4,7 +4,7 @@ import { FormattedMessage } from "react-intl"; import { Icon } from "components/ui/Icon"; import { Tooltip } from "components/ui/Tooltip"; -import { FieldTransform } from "core/request/AirbyteClient"; +import { FieldTransform } from "core/api/types/AirbyteClient"; import styles from "./FieldRow.module.scss"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldSection.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldSection.tsx index 65e96625a5b..16a5ba35d1c 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldSection.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/FieldSection.tsx @@ -1,6 +1,6 @@ import { FormattedMessage, useIntl } from "react-intl"; -import { StreamTransform } from "core/request/AirbyteClient"; +import { StreamTransform } from "core/api/types/AirbyteClient"; import { DiffAccordion } from "./DiffAccordion"; import { DiffHeader } from "./DiffHeader"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/StreamRow.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/StreamRow.tsx index 482ff97d533..98ca7627961 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/StreamRow.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/StreamRow.tsx @@ -2,7 +2,7 @@ import classnames from "classnames"; import { Icon } from "components/ui/Icon"; -import { StreamTransform } from "core/request/AirbyteClient"; +import { StreamTransform } from "core/api/types/AirbyteClient"; import styles from "./StreamRow.module.scss"; import { DiffVerb } from "./types"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/types.ts b/airbyte-webapp/src/components/connection/CatalogDiffModal/types.ts index 6c1d38aace4..f2694f27fd2 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/types.ts +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/types.ts @@ -1,4 +1,4 @@ -import { FieldTransform, StreamTransform } from "core/request/AirbyteClient"; +import { FieldTransform, StreamTransform } from "core/api/types/AirbyteClient"; export type DiffVerb = "new" | "removed" | "changed"; diff --git a/airbyte-webapp/src/components/connection/CatalogDiffModal/utils.tsx b/airbyte-webapp/src/components/connection/CatalogDiffModal/utils.tsx index 50f280b44aa..a293a4030ab 100644 --- a/airbyte-webapp/src/components/connection/CatalogDiffModal/utils.tsx +++ b/airbyte-webapp/src/components/connection/CatalogDiffModal/utils.tsx @@ -1,4 +1,4 @@ -import { FieldTransform, StreamTransform } from "core/request/AirbyteClient"; +import { FieldTransform, StreamTransform } from "core/api/types/AirbyteClient"; import { SortedDiff } from "./types"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationPreview.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationPreview.tsx index 67e4dc35e86..813e9ffceb0 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationPreview.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ConnectionConfigurationPreview.tsx @@ -5,7 +5,7 @@ import { FormattedMessage } from "react-intl"; import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; -import { ConnectionScheduleType } from "core/request/AirbyteClient"; +import { ConnectionScheduleType } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { useExperiment } from "hooks/services/Experiment"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionFormField.tsx index 5153f59a3d7..92c54a04daa 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/NamespaceDefinitionFormField.tsx @@ -8,7 +8,7 @@ import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; import { TextInputContainer } from "components/ui/TextInputContainer"; -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; import { useModalService } from "hooks/services/Modal"; import { FormConnectionFormValues } from "./formConfig"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceFormField.tsx index f666615f4d0..3e018a0791a 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/NonBreakingChangesPreferenceFormField.tsx @@ -5,7 +5,7 @@ import { FormattedMessage, useIntl } from "react-intl"; import { FormControl } from "components/forms"; import { Message } from "components/ui/Message"; -import { NonBreakingChangesPreference } from "core/request/AirbyteClient"; +import { NonBreakingChangesPreference } from "core/api/types/AirbyteClient"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useExperiment } from "hooks/services/Experiment"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/BasicScheduleFormControl.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/BasicScheduleFormControl.tsx index fdbfbb655c3..0059c30e3f0 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/BasicScheduleFormControl.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/BasicScheduleFormControl.tsx @@ -5,7 +5,7 @@ import { useIntl } from "react-intl"; import { FormLabel } from "components/forms/FormControl"; import { ListBox, Option } from "components/ui/ListBox"; -import { ConnectionScheduleDataBasicSchedule } from "core/request/AirbyteClient"; +import { ConnectionScheduleDataBasicSchedule } from "core/api/types/AirbyteClient"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useBasicFrequencyDropdownData } from "./useBasicFrequencyDropdownData"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/CronScheduleFormControl.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/CronScheduleFormControl.tsx index 5ee14cade1c..a4c8bcb2aa7 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/CronScheduleFormControl.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/CronScheduleFormControl.tsx @@ -10,7 +10,7 @@ import { ExternalLink } from "components/ui/Link"; import { ListBox } from "components/ui/ListBox"; import { Text } from "components/ui/Text"; -import { ConnectionScheduleDataCron } from "core/request/AirbyteClient"; +import { ConnectionScheduleDataCron } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { links } from "core/utils/links"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/ScheduleFormField.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/ScheduleFormField.tsx index 945699778fd..9a61489e74e 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/ScheduleFormField.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/ScheduleFormField.tsx @@ -1,7 +1,7 @@ import React from "react"; import { useWatch } from "react-hook-form"; -import { ConnectionScheduleType } from "core/request/AirbyteClient"; +import { ConnectionScheduleType } from "core/api/types/AirbyteClient"; import { BasicScheduleFormControl } from "./BasicScheduleFormControl"; import { CronScheduleFormControl } from "./CronScheduleFormControl"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/ScheduleTypeFormControl.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/ScheduleTypeFormControl.tsx index ae53e1a6a00..2c9028b7d1b 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/ScheduleTypeFormControl.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/ScheduleTypeFormControl.tsx @@ -5,7 +5,7 @@ import { useIntl } from "react-intl"; import { FormLabel } from "components/forms/FormControl"; import { ListBox, Option } from "components/ui/ListBox"; -import { ConnectionScheduleType } from "core/request/AirbyteClient"; +import { ConnectionScheduleType } from "core/api/types/AirbyteClient"; import { CRON_DEFAULT_VALUE } from "./CronScheduleFormControl"; import { BASIC_FREQUENCY_DEFAULT_VALUE } from "./useBasicFrequencyDropdownData"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useBasicFrequencyDropdownData.test.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useBasicFrequencyDropdownData.test.tsx index 2941433704e..3ffad409097 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useBasicFrequencyDropdownData.test.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useBasicFrequencyDropdownData.test.tsx @@ -2,7 +2,7 @@ import { renderHook } from "@testing-library/react"; import { TestWrapper as wrapper } from "test-utils/testutils"; -import { ConnectionScheduleTimeUnit } from "core/request/AirbyteClient"; +import { ConnectionScheduleTimeUnit } from "core/api/types/AirbyteClient"; import { useBasicFrequencyDropdownData, frequencyConfig } from "./useBasicFrequencyDropdownData"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useBasicFrequencyDropdownData.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useBasicFrequencyDropdownData.tsx index 4b1c2b72b0c..e5a0bb08e96 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useBasicFrequencyDropdownData.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useBasicFrequencyDropdownData.tsx @@ -3,7 +3,7 @@ import { useIntl } from "react-intl"; import { Option } from "components/ui/ListBox"; -import { ConnectionScheduleDataBasicSchedule, WebBackendConnectionRead } from "core/request/AirbyteClient"; +import { ConnectionScheduleDataBasicSchedule, WebBackendConnectionRead } from "core/api/types/AirbyteClient"; export const BASIC_FREQUENCY_DEFAULT_VALUE: ConnectionScheduleDataBasicSchedule = { units: 24, timeUnit: "hours" }; export const frequencyConfig: ConnectionScheduleDataBasicSchedule[] = [ diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useTrackConnectionFrequency.ts b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useTrackConnectionFrequency.ts index 09ec376cef8..9e191e1c164 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useTrackConnectionFrequency.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/ScheduleFormField/useTrackConnectionFrequency.ts @@ -1,6 +1,6 @@ import { useCallback } from "react"; -import { ConnectionScheduleDataBasicSchedule } from "core/request/AirbyteClient"; +import { ConnectionScheduleDataBasicSchedule } from "core/api/types/AirbyteClient"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useConnectionFormService, diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/SchemaChangesBackdrop.test.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/SchemaChangesBackdrop.test.tsx index 63c46692434..adeb3073f76 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/SchemaChangesBackdrop.test.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/SchemaChangesBackdrop.test.tsx @@ -3,7 +3,7 @@ import userEvent from "@testing-library/user-event"; import { mockConnection, TestWrapper } from "test-utils/testutils"; -import { SchemaChange } from "core/request/AirbyteClient"; +import { SchemaChange } from "core/api/types/AirbyteClient"; import { FeatureItem } from "core/services/features"; const mockUseConnectionEditService = jest.fn(); diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts index 8021148809c..d91499e384d 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.test.ts @@ -6,7 +6,7 @@ import { StreamDescriptor, StreamTransformTransformType, SyncMode, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { calculateInitialCatalog } from "./calculateInitialCatalog"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts index 239f0175901..cd457c12199 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/calculateInitialCatalog.ts @@ -8,7 +8,7 @@ import { StreamTransform, AirbyteCatalog, AirbyteStreamAndConfiguration, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; const getDefaultCursorField = (streamNode: AirbyteStreamAndConfiguration): string[] => { if (streamNode.stream?.defaultCursorField?.length) { diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx index d31b14f1dab..10d07134522 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionForm/formConfig.tsx @@ -11,8 +11,6 @@ import { OperationCreate, SchemaChange, SyncMode, -} from "core/api/types/AirbyteClient"; -import { ActorDefinitionVersionRead, ConnectionScheduleData, ConnectionScheduleType, @@ -20,7 +18,7 @@ import { NamespaceDefinitionType, NonBreakingChangesPreference, OperationRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { ConnectionOrPartialConnection, diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/frequencyConfig.ts b/airbyte-webapp/src/components/connection/ConnectionForm/frequencyConfig.ts index c92abb162f6..3c8245bf34f 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/frequencyConfig.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/frequencyConfig.ts @@ -1,4 +1,4 @@ -import { ConnectionScheduleDataBasicSchedule } from "core/request/AirbyteClient"; +import { ConnectionScheduleDataBasicSchedule } from "core/api/types/AirbyteClient"; export const frequencyConfig: ConnectionScheduleDataBasicSchedule[] = [ { diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/schema.ts b/airbyte-webapp/src/components/connection/ConnectionForm/schema.ts index b5e0acb8ba2..5efa531822f 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/schema.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/schema.ts @@ -3,8 +3,8 @@ import { SchemaOf } from "yup"; import { NormalizationType } from "area/connection/types"; import { validateCronExpression, validateCronFrequencyOneHourOrMore } from "area/connection/utils"; -import { AirbyteStreamAndConfiguration } from "core/api/types/AirbyteClient"; import { + AirbyteStreamAndConfiguration, AirbyteStream, AirbyteStreamConfiguration, ConnectionScheduleData, @@ -14,7 +14,7 @@ import { NamespaceDefinitionType, NonBreakingChangesPreference, SyncMode, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { ConnectionFormMode } from "hooks/services/ConnectionForm/ConnectionFormService"; import { dbtOperationReadOrCreateSchema } from "../TransformationForm"; diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/types.ts b/airbyte-webapp/src/components/connection/ConnectionForm/types.ts index dc2be952984..b6605ae8002 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/types.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/types.ts @@ -1,4 +1,4 @@ -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; export const namespaceDefinitionOptions: Record = { [NamespaceDefinitionType.destination]: "destinationFormat", diff --git a/airbyte-webapp/src/components/connection/ConnectionForm/utils.ts b/airbyte-webapp/src/components/connection/ConnectionForm/utils.ts index 9f741285549..872187290a9 100644 --- a/airbyte-webapp/src/components/connection/ConnectionForm/utils.ts +++ b/airbyte-webapp/src/components/connection/ConnectionForm/utils.ts @@ -1,5 +1,5 @@ import { NormalizationType } from "area/connection/types"; -import { AirbyteStreamAndConfiguration, OperationCreate, OperatorType } from "core/request/AirbyteClient"; +import { AirbyteStreamAndConfiguration, OperationCreate, OperatorType } from "core/api/types/AirbyteClient"; import { FormConnectionFormValues } from "./formConfig"; diff --git a/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.tsx b/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.tsx index 7928b7da058..19b6183fdba 100644 --- a/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionOnboarding/ConnectionOnboarding.tsx @@ -11,7 +11,7 @@ import { Tooltip } from "components/ui/Tooltip"; import { ConnectorIds, SvgIcon } from "area/connector/utils"; import { useCurrentWorkspace, useSourceDefinitionList, useDestinationDefinitionList } from "core/api"; -import { DestinationDefinitionRead, SourceDefinitionRead } from "core/request/AirbyteClient"; +import { DestinationDefinitionRead, SourceDefinitionRead } from "core/api/types/AirbyteClient"; import { links } from "core/utils/links"; import { useExperiment } from "hooks/services/Experiment"; import { ConnectionRoutePaths, DestinationPaths, RoutePaths } from "pages/routePaths"; diff --git a/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncButtons.tsx b/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncButtons.tsx index a07cd88def9..4637c4cd0e9 100644 --- a/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncButtons.tsx +++ b/airbyte-webapp/src/components/connection/ConnectionSync/ConnectionSyncButtons.tsx @@ -5,7 +5,7 @@ import { Button, ButtonVariant } from "components/ui/Button"; import { DropdownMenu, DropdownMenuOptionType } from "components/ui/DropdownMenu"; import { Icon } from "components/ui/Icon"; -import { ConnectionStatus } from "core/request/AirbyteClient"; +import { ConnectionStatus } from "core/api/types/AirbyteClient"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; diff --git a/airbyte-webapp/src/components/connection/CreateConnection/SelectExistingConnector.tsx b/airbyte-webapp/src/components/connection/CreateConnection/SelectExistingConnector.tsx index 6ab5f8618ef..c8478210062 100644 --- a/airbyte-webapp/src/components/connection/CreateConnection/SelectExistingConnector.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnection/SelectExistingConnector.tsx @@ -1,7 +1,7 @@ import { Card } from "components/ui/Card"; +import { DestinationRead, SourceRead } from "core/api/types/AirbyteClient"; import { isSource } from "core/domain/connector/source"; -import { DestinationRead, SourceRead } from "core/request/AirbyteClient"; import { ExistingConnectorButton } from "./ExistingConnectorButton"; import styles from "./SelectExistingConnector.module.scss"; diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx index e1950cc2205..5bf68897202 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/CreateConnectionForm.test.tsx @@ -46,6 +46,7 @@ jest.mock("core/api", () => ({ useSourceDefinition: () => mockSourceDefinition, useDestinationDefinition: () => mockDestinationDefinition, useDiscoverSchema: jest.fn(() => mockBaseUseDiscoverSchema), + LogsRequestError: jest.requireActual("core/api/errors").LogsRequestError, })); jest.mock("area/connector/utils", () => ({ diff --git a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx index 3dc70454d07..bc920f50e31 100644 --- a/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx +++ b/airbyte-webapp/src/components/connection/CreateConnectionForm/SchemaError.tsx @@ -6,8 +6,7 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Icon } from "components/ui/Icon"; -import { SchemaError as SchemaErrorType } from "core/api"; -import { LogsRequestError } from "core/request/LogsRequestError"; +import { SchemaError as SchemaErrorType, LogsRequestError } from "core/api"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import styles from "./SchemaError.module.scss"; diff --git a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceDescription.tsx b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceDescription.tsx index 172efb51d9b..e2e52cd1f98 100644 --- a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceDescription.tsx +++ b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceDescription.tsx @@ -5,7 +5,7 @@ import { Box } from "components/ui/Box"; import { ExternalLink } from "components/ui/Link"; import { Text } from "components/ui/Text"; -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; import { links } from "core/utils/links"; import { DestinationNamespaceFormValues } from "./DestinationNamespaceModal"; diff --git a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx index d63e14afcd8..66395a20821 100644 --- a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx +++ b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/DestinationNamespaceModal.tsx @@ -9,7 +9,7 @@ import { Box } from "components/ui/Box"; import { FlexContainer } from "components/ui/Flex"; import { ModalBody, ModalFooter } from "components/ui/Modal"; -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; import { DestinationNamespaceDescription } from "./DestinationNamespaceDescription"; import styles from "./DestinationNamespaceModal.module.scss"; diff --git a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/ExampleSettingsTable.tsx b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/ExampleSettingsTable.tsx index 097b10c19bd..d92a81f8fa4 100644 --- a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/ExampleSettingsTable.tsx +++ b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/ExampleSettingsTable.tsx @@ -2,7 +2,7 @@ import React from "react"; import { Text } from "components/ui/Text"; -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; import styles from "./ExampleSettingsTable.module.scss"; import { useExampleTableData } from "./useExampleSettingsTable"; diff --git a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/useExampleSettingsTable.ts b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/useExampleSettingsTable.ts index 969a94ef39a..4bd8a0bf130 100644 --- a/airbyte-webapp/src/components/connection/DestinationNamespaceModal/useExampleSettingsTable.ts +++ b/airbyte-webapp/src/components/connection/DestinationNamespaceModal/useExampleSettingsTable.ts @@ -1,6 +1,6 @@ import { useIntl } from "react-intl"; -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; export const useExampleTableData = ( namespaceDefinition: NamespaceDefinitionType diff --git a/airbyte-webapp/src/components/connection/EnabledControl/EnabledControl.tsx b/airbyte-webapp/src/components/connection/EnabledControl/EnabledControl.tsx index dd36044624a..752fca9e426 100644 --- a/airbyte-webapp/src/components/connection/EnabledControl/EnabledControl.tsx +++ b/airbyte-webapp/src/components/connection/EnabledControl/EnabledControl.tsx @@ -5,7 +5,7 @@ import { useAsyncFn } from "react-use"; import { Switch } from "components/ui/Switch"; -import { ConnectionStatus } from "core/request/AirbyteClient"; +import { ConnectionStatus } from "core/api/types/AirbyteClient"; import { getFrequencyFromScheduleData, Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; diff --git a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx b/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx index d05a03e937a..ed2fbaf8ee6 100644 --- a/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx +++ b/airbyte-webapp/src/components/connection/JobProgress/JobProgress.tsx @@ -6,7 +6,7 @@ import { Text } from "components/ui/Text"; import { JobWithAttempts } from "area/connection/types/jobs"; import { getJobStatus } from "area/connection/utils/jobs"; -import { AttemptRead, AttemptStatus, SynchronousJobRead } from "core/request/AirbyteClient"; +import { AttemptRead, AttemptStatus, SynchronousJobRead } from "core/api/types/AirbyteClient"; import { formatBytes } from "core/utils/numberHelper"; import styles from "./JobProgress.module.scss"; diff --git a/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.tsx b/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.tsx index b6b62d8cfc6..4a9bd328b9c 100644 --- a/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.tsx +++ b/airbyte-webapp/src/components/connection/JobProgress/StreamProgress.tsx @@ -4,7 +4,7 @@ import { FormattedMessage, FormattedNumber, useIntl } from "react-intl"; import { Text } from "components/ui/Text"; import { Tooltip } from "components/ui/Tooltip"; -import { AttemptStreamStats } from "core/request/AirbyteClient"; +import { AttemptStreamStats } from "core/api/types/AirbyteClient"; import styles from "./StreamProgress.module.scss"; diff --git a/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts b/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts index fd8275847f1..6339f209040 100644 --- a/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts +++ b/airbyte-webapp/src/components/connection/JobProgress/utils.test.ts @@ -1,4 +1,4 @@ -import { AttemptRead, AttemptStats, AttemptStatus, AttemptStreamStats } from "core/request/AirbyteClient"; +import { AttemptRead, AttemptStats, AttemptStatus, AttemptStreamStats } from "core/api/types/AirbyteClient"; import { progressBarCalculations } from "./utils"; diff --git a/airbyte-webapp/src/components/connection/JobProgress/utils.ts b/airbyte-webapp/src/components/connection/JobProgress/utils.ts index 1f3a81e44d0..ce6435307d2 100644 --- a/airbyte-webapp/src/components/connection/JobProgress/utils.ts +++ b/airbyte-webapp/src/components/connection/JobProgress/utils.ts @@ -1,4 +1,4 @@ -import { AttemptRead, AttemptStatus } from "core/request/AirbyteClient"; +import { AttemptRead, AttemptStatus } from "core/api/types/AirbyteClient"; export const progressBarCalculations = (latestAttempt: AttemptRead) => { let numeratorRecords = 0; diff --git a/airbyte-webapp/src/components/connection/StreamStatus/streamStatusUtils.tsx b/airbyte-webapp/src/components/connection/StreamStatus/streamStatusUtils.tsx index e990649403e..34db7238fe6 100644 --- a/airbyte-webapp/src/components/connection/StreamStatus/streamStatusUtils.tsx +++ b/airbyte-webapp/src/components/connection/StreamStatus/streamStatusUtils.tsx @@ -1,5 +1,5 @@ import { AirbyteStreamAndConfigurationWithEnforcedStream, getStreamKey } from "area/connection/utils"; -import { StreamStatusRead } from "core/request/AirbyteClient"; +import { StreamStatusRead } from "core/api/types/AirbyteClient"; import { naturalComparatorBy } from "core/utils/objects"; import { useExperiment } from "hooks/services/Experiment"; diff --git a/airbyte-webapp/src/components/connection/TransformationForm/types.ts b/airbyte-webapp/src/components/connection/TransformationForm/types.ts index d1691f346df..7e5de709913 100644 --- a/airbyte-webapp/src/components/connection/TransformationForm/types.ts +++ b/airbyte-webapp/src/components/connection/TransformationForm/types.ts @@ -1,4 +1,4 @@ -import { OperationId, OperatorDbt } from "core/request/AirbyteClient"; +import { OperationId, OperatorDbt } from "core/api/types/AirbyteClient"; export interface DbtOperationRead { name: string; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamDetailsPanel.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamDetailsPanel.tsx index fd6b41a9335..20211deeb8c 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamDetailsPanel.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamDetailsPanel.tsx @@ -3,7 +3,7 @@ import React from "react"; import { Overlay } from "components/ui/Overlay"; -import { AirbyteStream } from "core/request/AirbyteClient"; +import { AirbyteStream } from "core/api/types/AirbyteClient"; import styles from "./StreamDetailsPanel.module.scss"; import { StreamPanelHeader } from "./StreamPanelHeader"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx index 0728485753f..19ff2d0f770 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamDetailsPanel/StreamPanelHeader.tsx @@ -8,7 +8,7 @@ import { Icon } from "components/ui/Icon"; import { Switch } from "components/ui/Switch"; import { Text } from "components/ui/Text"; -import { AirbyteStream, AirbyteStreamConfiguration } from "core/request/AirbyteClient"; +import { AirbyteStream, AirbyteStreamConfiguration } from "core/api/types/AirbyteClient"; import { useExperiment } from "hooks/services/Experiment"; import styles from "./StreamPanelHeader.module.scss"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.test.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.test.tsx index c5de78fee89..bf3b97a962c 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.test.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.test.tsx @@ -1,6 +1,6 @@ import { mockStreamConfiguration } from "test-utils/mock-data/mockAirbyteStreamConfiguration"; -import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; +import { AirbyteStreamConfiguration } from "core/api/types/AirbyteClient"; import { isChildFieldCursor, isChildFieldPrimaryKey, isCursor, isPrimaryKey } from "./StreamFieldsTable"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.tsx index e9251ec303b..42b1c9f565b 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/StreamFieldsTable.tsx @@ -11,8 +11,8 @@ import { Table } from "components/ui/Table"; import { TextWithOverflowTooltip } from "components/ui/Text"; import { getDataType } from "area/connection/utils"; +import { AirbyteStreamConfiguration } from "core/api/types/AirbyteClient"; import { SyncSchemaField, SyncSchemaFieldObject } from "core/domain/catalog"; -import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useExperiment } from "hooks/services/Experiment"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/SyncFieldCell.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/SyncFieldCell.tsx index 73e1b5d7cc6..13c8660fb21 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/SyncFieldCell.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamFieldsTable/SyncFieldCell.tsx @@ -6,8 +6,8 @@ import { Icon } from "components/ui/Icon"; import { Switch } from "components/ui/Switch"; import { Tooltip } from "components/ui/Tooltip"; +import { SyncMode, DestinationSyncMode } from "core/api/types/AirbyteClient"; import { SyncSchemaField, SyncSchemaFieldObject } from "core/domain/catalog"; -import { SyncMode, DestinationSyncMode } from "core/request/AirbyteClient"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; interface SyncFieldCellProps { diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeader.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeader.tsx index 4e02cae4187..ac1f6d3b7af 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeader.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/StreamsConfigTableHeader.tsx @@ -10,7 +10,7 @@ import { Switch } from "components/ui/Switch"; import { Text } from "components/ui/Text"; import { InfoTooltip, TooltipLearnMoreLink } from "components/ui/Tooltip"; -import { AirbyteStreamAndConfiguration, NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { AirbyteStreamAndConfiguration, NamespaceDefinitionType } from "core/api/types/AirbyteClient"; import { links } from "core/utils/links"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; import { useModalService } from "hooks/services/Modal"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.test.tsx b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.test.tsx index 3d57716b834..c561b2cd3a4 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.test.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/StreamsConfigTable/useStreamsConfigTableRowProps.test.tsx @@ -2,7 +2,7 @@ import { renderHook } from "@testing-library/react"; import { FormConnectionFormValues } from "components/connection/ConnectionForm/formConfig"; -import { AirbyteStreamAndConfiguration } from "core/request/AirbyteClient"; +import { AirbyteStreamAndConfiguration } from "core/api/types/AirbyteClient"; import * as connectionFormService from "hooks/services/ConnectionForm/ConnectionFormService"; import { useStreamsConfigTableRowProps } from "./useStreamsConfigTableRowProps"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRow.tsx b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRow.tsx index c6cea62da93..eb288f99c8b 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRow.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/SyncCatalogRow.tsx @@ -4,9 +4,9 @@ import React, { useCallback, useMemo } from "react"; import { FieldErrors } from "react-hook-form"; import { useToggle } from "react-use"; +import { AirbyteStreamConfiguration, DestinationSyncMode, SyncMode } from "core/api/types/AirbyteClient"; import { SyncSchemaField, SyncSchemaFieldObject } from "core/domain/catalog"; import { traverseSchemaToField } from "core/domain/catalog/traverseSchemaToField"; -import { AirbyteStreamConfiguration, DestinationSyncMode, SyncMode } from "core/request/AirbyteClient"; import { naturalComparatorBy } from "core/utils/objects"; import { useDestinationNamespace } from "hooks/connection/useDestinationNamespace"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/streamConfigHelpers.test.ts b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/streamConfigHelpers.test.ts index f4bbf61f98f..b1112f07725 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/streamConfigHelpers.test.ts +++ b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/streamConfigHelpers.test.ts @@ -1,7 +1,7 @@ import { mockStreamConfiguration } from "test-utils/mock-data/mockAirbyteStreamConfiguration"; +import { AirbyteStreamConfiguration } from "core/api/types/AirbyteClient"; import { SyncSchemaField } from "core/domain/catalog"; -import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; import { mergeFieldPathArrays, diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/streamConfigHelpers.ts b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/streamConfigHelpers.ts index 26d673d3c5d..f1b4ff161fc 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/streamConfigHelpers.ts +++ b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/streamConfigHelpers.ts @@ -1,7 +1,7 @@ import isEqual from "lodash/isEqual"; +import { AirbyteStreamConfiguration, SelectedFieldInfo } from "core/api/types/AirbyteClient"; import { SyncSchemaField } from "core/domain/catalog"; -import { AirbyteStreamConfiguration, SelectedFieldInfo } from "core/request/AirbyteClient"; /** * Merges arrays of SelectedFieldInfo, ensuring there are no duplicates diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/updateStreamSyncMode.ts b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/updateStreamSyncMode.ts index 148f6717ee5..5c588d5c953 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/updateStreamSyncMode.ts +++ b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/updateStreamSyncMode.ts @@ -4,7 +4,7 @@ import { DestinationSyncMode, SelectedFieldInfo, SyncMode, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { mergeFieldPathArrays } from "./streamConfigHelpers"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/useStreamFilters.tsx b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/useStreamFilters.tsx index 043216b512c..30846df6e0f 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/useStreamFilters.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/SyncCatalog/useStreamFilters.tsx @@ -1,6 +1,6 @@ import { useMemo } from "react"; -import { AirbyteStreamAndConfiguration } from "core/request/AirbyteClient"; +import { AirbyteStreamAndConfiguration } from "core/api/types/AirbyteClient"; import { SyncStreamFieldWithId } from "../../ConnectionForm/formConfig"; diff --git a/airbyte-webapp/src/components/connection/syncCatalog/SyncModeSelect/SyncModeSelect.tsx b/airbyte-webapp/src/components/connection/syncCatalog/SyncModeSelect/SyncModeSelect.tsx index 3995abde894..0f054387cc3 100644 --- a/airbyte-webapp/src/components/connection/syncCatalog/SyncModeSelect/SyncModeSelect.tsx +++ b/airbyte-webapp/src/components/connection/syncCatalog/SyncModeSelect/SyncModeSelect.tsx @@ -5,7 +5,7 @@ import { Option } from "components/ui/ListBox"; import { PillButtonVariant } from "components/ui/PillListBox"; import { PillListBox } from "components/ui/PillListBox/PillListBox"; -import { DestinationSyncMode, SyncMode } from "core/request/AirbyteClient"; +import { DestinationSyncMode, SyncMode } from "core/api/types/AirbyteClient"; import styles from "./SyncModeSelect.module.scss"; diff --git a/airbyte-webapp/src/components/connector/ConnectorNavigationTabs.tsx b/airbyte-webapp/src/components/connector/ConnectorNavigationTabs.tsx index 074949162b8..5b64948d8c7 100644 --- a/airbyte-webapp/src/components/connector/ConnectorNavigationTabs.tsx +++ b/airbyte-webapp/src/components/connector/ConnectorNavigationTabs.tsx @@ -4,7 +4,7 @@ import { useParams } from "react-router-dom"; import { Tabs, LinkTab } from "components/ui/Tabs"; -import { DestinationRead, SourceRead } from "core/request/AirbyteClient"; +import { DestinationRead, SourceRead } from "core/api/types/AirbyteClient"; import { RoutePaths } from "pages/routePaths"; enum TabTypes { diff --git a/airbyte-webapp/src/components/connector/ConnectorTitleBlock.tsx b/airbyte-webapp/src/components/connector/ConnectorTitleBlock.tsx index 7603fb2c24f..4694a94ee85 100644 --- a/airbyte-webapp/src/components/connector/ConnectorTitleBlock.tsx +++ b/airbyte-webapp/src/components/connector/ConnectorTitleBlock.tsx @@ -6,8 +6,8 @@ import { Heading } from "components/ui/Heading"; import { SupportLevelBadge } from "components/ui/SupportLevelBadge"; import { Text } from "components/ui/Text"; +import { DestinationRead, SourceRead, ActorDefinitionVersionRead } from "core/api/types/AirbyteClient"; import { shouldDisplayBreakingChangeBanner, ConnectorDefinition } from "core/domain/connector"; -import { DestinationRead, SourceRead, ActorDefinitionVersionRead } from "core/request/AirbyteClient"; import { BreakingChangeBanner } from "./BreakingChangeBanner"; import styles from "./ConnectorTitleBlock.module.scss"; diff --git a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx index 64681c6da9f..45b1a0676b5 100644 --- a/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx +++ b/airbyte-webapp/src/components/connectorBuilder/StreamTestingPanel/StreamTester.tsx @@ -12,8 +12,8 @@ import { ResizablePanels } from "components/ui/ResizablePanels"; import { Spinner } from "components/ui/Spinner"; import { Text } from "components/ui/Text"; +import { CommonRequestError } from "core/api"; import { KnownExceptionInfo } from "core/api/types/ConnectorBuilderClient"; -import { CommonRequestError } from "core/request/CommonRequestError"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { links } from "core/utils/links"; import { useLocalStorage } from "core/utils/useLocalStorage"; diff --git a/airbyte-webapp/src/components/destination/DestinationConnectionTable/DestinationConnectionTable.tsx b/airbyte-webapp/src/components/destination/DestinationConnectionTable/DestinationConnectionTable.tsx index 5d6e11af1ee..86991d9c3d5 100644 --- a/airbyte-webapp/src/components/destination/DestinationConnectionTable/DestinationConnectionTable.tsx +++ b/airbyte-webapp/src/components/destination/DestinationConnectionTable/DestinationConnectionTable.tsx @@ -5,7 +5,7 @@ import { ConnectionTable } from "components/EntityTable"; import { ConnectionTableDataItem } from "components/EntityTable/types"; import { getConnectionTableData } from "components/EntityTable/utils"; -import { WebBackendConnectionListItem } from "core/request/AirbyteClient"; +import { WebBackendConnectionListItem } from "core/api/types/AirbyteClient"; import { RoutePaths } from "pages/routePaths"; import styles from "./DestinationConnectionTable.module.scss"; diff --git a/airbyte-webapp/src/components/destination/DestinationForm/DestinationForm.tsx b/airbyte-webapp/src/components/destination/DestinationForm/DestinationForm.tsx index a12c9cbb94c..6f59fe20b6c 100644 --- a/airbyte-webapp/src/components/destination/DestinationForm/DestinationForm.tsx +++ b/airbyte-webapp/src/components/destination/DestinationForm/DestinationForm.tsx @@ -7,10 +7,9 @@ import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { ConnectionConfiguration } from "area/connector/types"; -import { useGetDestinationDefinitionSpecificationAsync } from "core/api"; +import { useGetDestinationDefinitionSpecificationAsync, LogsRequestError } from "core/api"; import { DestinationDefinitionRead } from "core/api/types/AirbyteClient"; import { Connector } from "core/domain/connector"; -import { LogsRequestError } from "core/request/LogsRequestError"; import { FormError } from "core/utils/errorStatusMessage"; import { ConnectorCard } from "views/Connector/ConnectorCard"; import { ConnectorCardValues } from "views/Connector/ConnectorForm"; diff --git a/airbyte-webapp/src/components/forms/DataResidencyDropdown.tsx b/airbyte-webapp/src/components/forms/DataResidencyDropdown.tsx index f2e1abda89d..2704f7370f1 100644 --- a/airbyte-webapp/src/components/forms/DataResidencyDropdown.tsx +++ b/airbyte-webapp/src/components/forms/DataResidencyDropdown.tsx @@ -4,7 +4,7 @@ import { Path } from "react-hook-form"; import { useIntl } from "react-intl"; import { useAvailableGeographies } from "core/api"; -import { Geography } from "core/request/AirbyteClient"; +import { Geography } from "core/api/types/AirbyteClient"; import styles from "./DataResidencyDropdown.module.scss"; import { FormValues } from "./Form"; diff --git a/airbyte-webapp/src/components/source/SelectConnector/FilterSupportLevel.tsx b/airbyte-webapp/src/components/source/SelectConnector/FilterSupportLevel.tsx index 7133cf5cf2a..72bf961ffe4 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/FilterSupportLevel.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/FilterSupportLevel.tsx @@ -6,7 +6,7 @@ import { FlexContainer } from "components/ui/Flex"; import { SupportLevelBadge } from "components/ui/SupportLevelBadge"; import { Text } from "components/ui/Text"; -import { SupportLevel } from "core/request/AirbyteClient"; +import { SupportLevel } from "core/api/types/AirbyteClient"; import styles from "./FilterSupportLevel.module.scss"; diff --git a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx index 2e532302a5b..e5d31d5da3c 100644 --- a/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx +++ b/airbyte-webapp/src/components/source/SelectConnector/SelectConnector.tsx @@ -10,9 +10,9 @@ import { Text } from "components/ui/Text"; import { SuggestedConnectors } from "area/connector/components"; import { useCurrentWorkspace } from "core/api"; +import { SupportLevel } from "core/api/types/AirbyteClient"; import { ConnectorDefinition } from "core/domain/connector"; import { isSourceDefinition } from "core/domain/connector/source"; -import { SupportLevel } from "core/request/AirbyteClient"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { useLocalStorage } from "core/utils/useLocalStorage"; import { useModalService } from "hooks/services/Modal"; diff --git a/airbyte-webapp/src/components/ui/ConnectorDefinitionBranding/ConnectorDefinitionBranding.tsx b/airbyte-webapp/src/components/ui/ConnectorDefinitionBranding/ConnectorDefinitionBranding.tsx index bb8dbc21130..25297ab31b0 100644 --- a/airbyte-webapp/src/components/ui/ConnectorDefinitionBranding/ConnectorDefinitionBranding.tsx +++ b/airbyte-webapp/src/components/ui/ConnectorDefinitionBranding/ConnectorDefinitionBranding.tsx @@ -2,7 +2,7 @@ import { ConnectorIcon } from "components/common/ConnectorIcon"; import { Text } from "components/ui/Text"; import { useSourceDefinitionList, useDestinationDefinitionList } from "core/api"; -import { DestinationDefinitionId, SourceDefinitionId } from "core/request/AirbyteClient"; +import { DestinationDefinitionId, SourceDefinitionId } from "core/api/types/AirbyteClient"; import styles from "./ConnectorDefinitionBranding.module.scss"; import { FlexContainer } from "../Flex"; diff --git a/airbyte-webapp/src/components/ui/SupportLevelBadge/SupportLevelBadge.tsx b/airbyte-webapp/src/components/ui/SupportLevelBadge/SupportLevelBadge.tsx index 2e4d254604b..58d0d35e97a 100644 --- a/airbyte-webapp/src/components/ui/SupportLevelBadge/SupportLevelBadge.tsx +++ b/airbyte-webapp/src/components/ui/SupportLevelBadge/SupportLevelBadge.tsx @@ -1,6 +1,6 @@ import { FormattedMessage } from "react-intl"; -import { SupportLevel } from "core/request/AirbyteClient"; +import { SupportLevel } from "core/api/types/AirbyteClient"; import { Badge } from "../Badge"; import { Tooltip } from "../Tooltip"; diff --git a/airbyte-webapp/src/components/workspace/WorkspacesPickerList.tsx b/airbyte-webapp/src/components/workspace/WorkspacesPickerList.tsx index b75c7496260..b324bdf4263 100644 --- a/airbyte-webapp/src/components/workspace/WorkspacesPickerList.tsx +++ b/airbyte-webapp/src/components/workspace/WorkspacesPickerList.tsx @@ -11,8 +11,8 @@ import { LoadingSpinner } from "components/ui/LoadingSpinner"; import { SearchInput } from "components/ui/SearchInput"; import { Text } from "components/ui/Text"; +import { WorkspaceRead, WorkspaceReadList } from "core/api/types/AirbyteClient"; import { CloudWorkspaceRead, CloudWorkspaceReadList } from "core/api/types/CloudApi"; -import { WorkspaceRead, WorkspaceReadList } from "core/request/AirbyteClient"; import { RoutePaths } from "pages/routePaths"; import { WORKSPACE_LIST_LENGTH } from "pages/workspaces/WorkspacesPage"; diff --git a/airbyte-webapp/src/core/api/apiCall.ts b/airbyte-webapp/src/core/api/apiCall.ts index 293fda815fa..af1934e795d 100644 --- a/airbyte-webapp/src/core/api/apiCall.ts +++ b/airbyte-webapp/src/core/api/apiCall.ts @@ -1,8 +1,8 @@ import { trackError } from "core/utils/datadog"; import { shortUuid } from "core/utils/uuid"; -import { CommonRequestError } from "../request/CommonRequestError"; -import { VersionError } from "../request/VersionError"; +import { CommonRequestError } from "./errors/CommonRequestError"; +import { VersionError } from "./errors/VersionError"; export interface ApiCallOptions { getAccessToken: () => Promise; diff --git a/airbyte-webapp/src/core/request/CommonRequestError.ts b/airbyte-webapp/src/core/api/errors/CommonRequestError.ts similarity index 100% rename from airbyte-webapp/src/core/request/CommonRequestError.ts rename to airbyte-webapp/src/core/api/errors/CommonRequestError.ts diff --git a/airbyte-webapp/src/core/request/LogsRequestError.ts b/airbyte-webapp/src/core/api/errors/LogsRequestError.ts similarity index 89% rename from airbyte-webapp/src/core/request/LogsRequestError.ts rename to airbyte-webapp/src/core/api/errors/LogsRequestError.ts index b89e3c6fb55..0e6f5f08b34 100644 --- a/airbyte-webapp/src/core/request/LogsRequestError.ts +++ b/airbyte-webapp/src/core/api/errors/LogsRequestError.ts @@ -1,4 +1,5 @@ -import { SynchronousJobRead } from "./AirbyteClient"; +import { SynchronousJobRead } from "core/api/types/AirbyteClient"; + import { CommonRequestError } from "./CommonRequestError"; export class LogsRequestError extends CommonRequestError { diff --git a/airbyte-webapp/src/core/request/ServerError.ts b/airbyte-webapp/src/core/api/errors/ServerError.ts similarity index 100% rename from airbyte-webapp/src/core/request/ServerError.ts rename to airbyte-webapp/src/core/api/errors/ServerError.ts diff --git a/airbyte-webapp/src/core/request/VersionError.ts b/airbyte-webapp/src/core/api/errors/VersionError.ts similarity index 100% rename from airbyte-webapp/src/core/request/VersionError.ts rename to airbyte-webapp/src/core/api/errors/VersionError.ts diff --git a/airbyte-webapp/src/core/api/errors/index.ts b/airbyte-webapp/src/core/api/errors/index.ts new file mode 100644 index 00000000000..36fa2d27bbc --- /dev/null +++ b/airbyte-webapp/src/core/api/errors/index.ts @@ -0,0 +1,4 @@ +export * from "./CommonRequestError"; +export * from "./VersionError"; +export * from "./LogsRequestError"; +export * from "./ServerError"; diff --git a/airbyte-webapp/src/core/api/hooks/actorDefinitionVersions.ts b/airbyte-webapp/src/core/api/hooks/actorDefinitionVersions.ts index 6b4f8cabc10..fc8351534d6 100644 --- a/airbyte-webapp/src/core/api/hooks/actorDefinitionVersions.ts +++ b/airbyte-webapp/src/core/api/hooks/actorDefinitionVersions.ts @@ -1,10 +1,10 @@ import { isDefined } from "core/utils/common"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { getActorDefinitionVersionForDestinationId, getActorDefinitionVersionForSourceId, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; diff --git a/airbyte-webapp/src/core/api/hooks/cloud/cloudWorkspaces.ts b/airbyte-webapp/src/core/api/hooks/cloud/cloudWorkspaces.ts index 81950ea9c75..0defb5af9be 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/cloudWorkspaces.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/cloudWorkspaces.ts @@ -2,7 +2,6 @@ import { useInfiniteQuery, useMutation, useQuery, useQueryClient } from "@tansta import { useCallback } from "react"; import { useCurrentUser } from "core/services/auth"; -import { SCOPE_USER } from "services/Scope"; import { deleteCloudWorkspace, @@ -13,6 +12,7 @@ import { webBackendListWorkspacesByUser, webBackendListWorkspacesByUserPaginated, } from "../../generated/CloudApi"; +import { SCOPE_USER } from "../../scopes"; import { CloudWorkspaceRead, CloudWorkspaceReadList, diff --git a/airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts b/airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts index 06bb1d8a9ff..bfce94831fc 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/dbtCloud.ts @@ -13,7 +13,6 @@ import isEmpty from "lodash/isEmpty"; import { useIntl } from "react-intl"; import { useAsyncFn } from "react-use"; -import { useRequestOptions } from "core/api/useRequestOptions"; import { OperatorType, WebBackendConnectionRead, @@ -22,7 +21,8 @@ import { WebhookConfigRead, WorkspaceRead, WebhookConfigWrite, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; +import { useRequestOptions } from "core/api/useRequestOptions"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/core/api/hooks/cloud/users.ts b/airbyte-webapp/src/core/api/hooks/cloud/users.ts index 71a5450ce8a..6a294315866 100644 --- a/airbyte-webapp/src/core/api/hooks/cloud/users.ts +++ b/airbyte-webapp/src/core/api/hooks/cloud/users.ts @@ -2,6 +2,7 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useCallback, useMemo } from "react"; import { useCurrentWorkspaceId } from "area/workspace/utils"; + import { webBackendRevokeUserFromWorkspace, webBackendResendWithSigninLink, @@ -12,9 +13,8 @@ import { getUserByEmail, updateUser, webBackendRevokeUserSession, -} from "core/api/generated/CloudApi"; -import { SCOPE_WORKSPACE } from "services/Scope"; - +} from "../../generated/CloudApi"; +import { SCOPE_WORKSPACE } from "../../scopes"; import { UserUpdate, UserCreate } from "../../types/CloudApi"; import { useRequestOptions } from "../../useRequestOptions"; import { useSuspenseQuery } from "../../useSuspenseQuery"; diff --git a/airbyte-webapp/src/core/api/hooks/connections.tsx b/airbyte-webapp/src/core/api/hooks/connections.tsx index 580ee94195a..f41d2d18b4b 100644 --- a/airbyte-webapp/src/core/api/hooks/connections.tsx +++ b/airbyte-webapp/src/core/api/hooks/connections.tsx @@ -9,7 +9,6 @@ import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; import { useNotificationService } from "hooks/services/Notification"; import { CloudRoutes } from "packages/cloud/cloudRoutePaths"; import { RoutePaths } from "pages/routePaths"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { useCurrentWorkspace, useInvalidateWorkspaceStateQuery } from "./workspaces"; import { @@ -25,6 +24,7 @@ import { webBackendListConnectionsForWorkspace, webBackendUpdateConnection, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { AirbyteCatalog, ConnectionScheduleData, diff --git a/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts b/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts index 6b441fa073b..f547c50ca54 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorBuilderProject.ts @@ -2,7 +2,6 @@ import { QueryClient, useMutation, useQuery, useQueryClient } from "@tanstack/re import { useCurrentWorkspaceId } from "area/workspace/utils"; import { sourceDefinitionKeys } from "core/api"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { useBuilderResolveManifestQuery } from "./connectorBuilderApi"; import { @@ -16,6 +15,7 @@ import { updateConnectorBuilderProject, updateDeclarativeManifestVersion, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { ConnectorBuilderProjectIdWithWorkspaceId, DeclarativeManifestVersionRead, diff --git a/airbyte-webapp/src/core/api/hooks/connectorCheck.ts b/airbyte-webapp/src/core/api/hooks/connectorCheck.ts index d1f3e964d0c..3c01a335f9e 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorCheck.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorCheck.ts @@ -1,7 +1,7 @@ import { useMutation } from "@tanstack/react-query"; import { ConnectionConfiguration } from "area/connector/types"; -import { LogsRequestError } from "core/request/LogsRequestError"; +import { LogsRequestError } from "core/api"; import { checkConnectionToDestination, diff --git a/airbyte-webapp/src/core/api/hooks/connectorDefinitionSpecification.ts b/airbyte-webapp/src/core/api/hooks/connectorDefinitionSpecification.ts index 2ba94585cd2..72029a3f40f 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorDefinitionSpecification.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorDefinitionSpecification.ts @@ -3,13 +3,13 @@ import { useQuery } from "@tanstack/react-query"; import { useSuspenseQuery, useCurrentWorkspace } from "core/api"; import { isDefined } from "core/utils/common"; -import { SCOPE_WORKSPACE } from "../../../services/Scope"; import { getDestinationDefinitionSpecification, getSourceDefinitionSpecification, getSpecificationForDestinationId, getSpecificationForSourceId, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { DestinationDefinitionSpecificationRead } from "../types/AirbyteClient"; import { useRequestOptions } from "../useRequestOptions"; diff --git a/airbyte-webapp/src/core/api/hooks/connectorUpdates.ts b/airbyte-webapp/src/core/api/hooks/connectorUpdates.ts index 3508c2cf708..bd1fbee05bd 100644 --- a/airbyte-webapp/src/core/api/hooks/connectorUpdates.ts +++ b/airbyte-webapp/src/core/api/hooks/connectorUpdates.ts @@ -1,9 +1,9 @@ import { useQuery } from "@tanstack/react-query"; import { FeatureItem, useFeature } from "core/services/features"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { webBackendCheckUpdates } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { WebBackendCheckUpdatesRead } from "../types/AirbyteClient"; import { useRequestOptions } from "../useRequestOptions"; diff --git a/airbyte-webapp/src/core/api/hooks/destinationDefinitions.ts b/airbyte-webapp/src/core/api/hooks/destinationDefinitions.ts index 3a939888d8c..8597db8de18 100644 --- a/airbyte-webapp/src/core/api/hooks/destinationDefinitions.ts +++ b/airbyte-webapp/src/core/api/hooks/destinationDefinitions.ts @@ -3,7 +3,6 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { isDefined } from "core/utils/common"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { connectorDefinitionKeys } from "./connectorUpdates"; import { @@ -13,6 +12,7 @@ import { listLatestDestinationDefinitions, updateDestinationDefinition, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { DestinationDefinitionCreate, DestinationDefinitionRead, diff --git a/airbyte-webapp/src/core/api/hooks/destinations.tsx b/airbyte-webapp/src/core/api/hooks/destinations.tsx index 22443cb89ae..b0594733cc4 100644 --- a/airbyte-webapp/src/core/api/hooks/destinations.tsx +++ b/airbyte-webapp/src/core/api/hooks/destinations.tsx @@ -4,7 +4,6 @@ import { useCallback } from "react"; import { ConnectionConfiguration } from "area/connector/types"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { isDefined } from "core/utils/common"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { useRemoveConnectionsFromList } from "./connections"; import { useCurrentWorkspace } from "./workspaces"; @@ -15,6 +14,7 @@ import { listDestinationsForWorkspace, updateDestination, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { DestinationRead, WebBackendConnectionListItem } from "../types/AirbyteClient"; import { useRequestErrorHandler } from "../useRequestErrorHandler"; import { useRequestOptions } from "../useRequestOptions"; diff --git a/airbyte-webapp/src/core/api/hooks/geographies.ts b/airbyte-webapp/src/core/api/hooks/geographies.ts index 2749d712dc0..5a9514f896c 100644 --- a/airbyte-webapp/src/core/api/hooks/geographies.ts +++ b/airbyte-webapp/src/core/api/hooks/geographies.ts @@ -1,6 +1,5 @@ -import { SCOPE_USER } from "services/Scope"; - import { webBackendListGeographies } from "../generated/AirbyteClient"; +import { SCOPE_USER } from "../scopes"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; diff --git a/airbyte-webapp/src/core/api/hooks/jobs.ts b/airbyte-webapp/src/core/api/hooks/jobs.ts index ec2cdaf6f44..79b8a1bac56 100644 --- a/airbyte-webapp/src/core/api/hooks/jobs.ts +++ b/airbyte-webapp/src/core/api/hooks/jobs.ts @@ -1,7 +1,6 @@ import { Updater, useIsMutating, useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useExperiment } from "hooks/services/Experiment"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { cancelJob, @@ -10,6 +9,7 @@ import { getJobInfoWithoutLogs, listJobsFor, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { JobListRequestBody, JobReadList, JobStatus } from "../types/AirbyteClient"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; diff --git a/airbyte-webapp/src/core/api/hooks/organizations.ts b/airbyte-webapp/src/core/api/hooks/organizations.ts index 212562d1767..530d2f43889 100644 --- a/airbyte-webapp/src/core/api/hooks/organizations.ts +++ b/airbyte-webapp/src/core/api/hooks/organizations.ts @@ -1,7 +1,6 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useCurrentWorkspaceId } from "area/workspace/utils"; -import { SCOPE_ORGANIZATION, SCOPE_USER } from "services/Scope"; import { useGetWorkspace } from "./workspaces"; import { @@ -11,6 +10,7 @@ import { updateOrganization, } from "../generated/AirbyteClient"; import { OrganizationUpdateRequestBody } from "../generated/AirbyteClient.schemas"; +import { SCOPE_ORGANIZATION, SCOPE_USER } from "../scopes"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; diff --git a/airbyte-webapp/src/core/api/hooks/permissions.ts b/airbyte-webapp/src/core/api/hooks/permissions.ts index ba707b43f9b..5ffa374fc5a 100644 --- a/airbyte-webapp/src/core/api/hooks/permissions.ts +++ b/airbyte-webapp/src/core/api/hooks/permissions.ts @@ -2,7 +2,6 @@ import { useMutation, useQueryClient } from "@tanstack/react-query"; import { useIntl } from "react-intl"; import { useNotificationService } from "hooks/services/Notification"; -import { SCOPE_USER } from "services/Scope"; import { organizationKeys } from "./organizations"; import { workspaceKeys } from "./workspaces"; @@ -13,6 +12,7 @@ import { updatePermission, } from "../generated/AirbyteClient"; import { PermissionCreate, PermissionRead, PermissionUpdate } from "../generated/AirbyteClient.schemas"; +import { SCOPE_USER } from "../scopes"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; diff --git a/airbyte-webapp/src/core/api/hooks/sourceDefinitions.ts b/airbyte-webapp/src/core/api/hooks/sourceDefinitions.ts index 11c57c4114b..26bfb34a7a4 100644 --- a/airbyte-webapp/src/core/api/hooks/sourceDefinitions.ts +++ b/airbyte-webapp/src/core/api/hooks/sourceDefinitions.ts @@ -4,7 +4,6 @@ import { useMemo } from "react"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { isDefined } from "core/utils/common"; import { useAppMonitoringService } from "hooks/services/AppMonitoringService"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { connectorDefinitionKeys } from "./connectorUpdates"; import { @@ -14,6 +13,7 @@ import { listSourceDefinitionsForWorkspace, updateSourceDefinition, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { SourceDefinitionCreate, SourceDefinitionRead, SourceDefinitionReadList } from "../types/AirbyteClient"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; diff --git a/airbyte-webapp/src/core/api/hooks/sources.tsx b/airbyte-webapp/src/core/api/hooks/sources.tsx index 50817f4f91a..be1bd09a8a5 100644 --- a/airbyte-webapp/src/core/api/hooks/sources.tsx +++ b/airbyte-webapp/src/core/api/hooks/sources.tsx @@ -5,7 +5,6 @@ import { flushSync } from "react-dom"; import { ConnectionConfiguration } from "area/connector/types"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { isDefined } from "core/utils/common"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { useRemoveConnectionsFromList } from "./connections"; import { useCurrentWorkspace } from "./workspaces"; @@ -17,6 +16,7 @@ import { listSourcesForWorkspace, updateSource, } from "../generated/AirbyteClient"; +import { SCOPE_WORKSPACE } from "../scopes"; import { AirbyteCatalog, SourceRead, SynchronousJobRead, WebBackendConnectionListItem } from "../types/AirbyteClient"; import { useRequestErrorHandler } from "../useRequestErrorHandler"; import { useRequestOptions } from "../useRequestOptions"; diff --git a/airbyte-webapp/src/core/api/hooks/streams.ts b/airbyte-webapp/src/core/api/hooks/streams.ts index 7656b329d4c..be508f701a4 100644 --- a/airbyte-webapp/src/core/api/hooks/streams.ts +++ b/airbyte-webapp/src/core/api/hooks/streams.ts @@ -1,8 +1,8 @@ import { getStreamKey, sortStreamStatuses } from "area/connection/utils"; -import { SCOPE_WORKSPACE } from "services/Scope"; import { getStreamStatusesByRunState } from "../generated/AirbyteClient"; import { ConnectionIdRequestBody, StreamStatusRead, StreamStatusRunState } from "../generated/AirbyteClient.schemas"; +import { SCOPE_WORKSPACE } from "../scopes"; import { useRequestOptions } from "../useRequestOptions"; import { useSuspenseQuery } from "../useSuspenseQuery"; diff --git a/airbyte-webapp/src/core/api/hooks/workspaces.ts b/airbyte-webapp/src/core/api/hooks/workspaces.ts index b609c7f2496..50810117780 100644 --- a/airbyte-webapp/src/core/api/hooks/workspaces.ts +++ b/airbyte-webapp/src/core/api/hooks/workspaces.ts @@ -3,7 +3,6 @@ import { useCallback, useLayoutEffect } from "react"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useCurrentUser } from "core/services/auth"; -import { SCOPE_USER, SCOPE_WORKSPACE } from "services/Scope"; import { createWorkspace, @@ -16,6 +15,7 @@ import { updateWorkspaceName, webBackendGetWorkspaceState, } from "../generated/AirbyteClient"; +import { SCOPE_USER, SCOPE_WORKSPACE } from "../scopes"; import { WorkspaceCreate, WorkspaceRead, diff --git a/airbyte-webapp/src/core/api/index.ts b/airbyte-webapp/src/core/api/index.ts index ee5103c7e9e..e91fddc9bb3 100644 --- a/airbyte-webapp/src/core/api/index.ts +++ b/airbyte-webapp/src/core/api/index.ts @@ -4,5 +4,7 @@ export { QueryProvider } from "./QueryProvider"; export type { ApiCallOptions } from "./apiCall"; +export * from "./errors"; + // Export all react query hooks to be used everywhere in the product export * from "./hooks"; diff --git a/airbyte-webapp/src/services/Scope.ts b/airbyte-webapp/src/core/api/scopes.ts similarity index 100% rename from airbyte-webapp/src/services/Scope.ts rename to airbyte-webapp/src/core/api/scopes.ts diff --git a/airbyte-webapp/src/core/domain/connector/connector.ts b/airbyte-webapp/src/core/domain/connector/connector.ts index 5993d03b404..b2b0efab001 100644 --- a/airbyte-webapp/src/core/domain/connector/connector.ts +++ b/airbyte-webapp/src/core/domain/connector/connector.ts @@ -4,7 +4,7 @@ import { ActorDefinitionVersionRead, DestinationDefinitionSpecificationRead, SourceDefinitionSpecificationRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { isSource, isSourceDefinition, isSourceDefinitionSpecification } from "./source"; import { ConnectorDefinition, ConnectorT } from "./types"; diff --git a/airbyte-webapp/src/core/domain/connector/source.ts b/airbyte-webapp/src/core/domain/connector/source.ts index 98a822974d4..6f183c8fc5a 100644 --- a/airbyte-webapp/src/core/domain/connector/source.ts +++ b/airbyte-webapp/src/core/domain/connector/source.ts @@ -1,15 +1,16 @@ +import { + DestinationDefinitionSpecificationRead, + SourceDefinitionRead, + SourceDefinitionSpecificationRead, + SourceRead, +} from "core/api/types/AirbyteClient"; + import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorT, SourceDefinitionSpecificationDraft, } from "./types"; -import { - DestinationDefinitionSpecificationRead, - SourceDefinitionRead, - SourceDefinitionSpecificationRead, - SourceRead, -} from "../../request/AirbyteClient"; export function isSource(connector: ConnectorT): connector is SourceRead { return "sourceId" in connector; diff --git a/airbyte-webapp/src/core/domain/connector/types.ts b/airbyte-webapp/src/core/domain/connector/types.ts index 9c2615e8fd0..13313750999 100644 --- a/airbyte-webapp/src/core/domain/connector/types.ts +++ b/airbyte-webapp/src/core/domain/connector/types.ts @@ -5,7 +5,7 @@ import { SourceDefinitionRead, SourceDefinitionSpecificationRead, SourceRead, -} from "../../request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; export type ConnectorDefinition = SourceDefinitionRead | DestinationDefinitionRead; diff --git a/airbyte-webapp/src/core/request/AirbyteClient.ts b/airbyte-webapp/src/core/request/AirbyteClient.ts deleted file mode 100644 index d9f3758b266..00000000000 --- a/airbyte-webapp/src/core/request/AirbyteClient.ts +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Reexport from the existing auto generated file during refactoring and migration - * @deprecated Do not import from here anymore, but use 'core/api' instead. - */ -// Ignore ESLint rule since this is a shim for backwards compatibility -// eslint-disable-next-line import/no-restricted-paths -export * from "../api/generated/AirbyteClient"; -export * from "../api/types/AirbyteClient"; diff --git a/airbyte-webapp/src/core/services/analytics/utils.ts b/airbyte-webapp/src/core/services/analytics/utils.ts index a11cd82bb25..10852c1daf6 100644 --- a/airbyte-webapp/src/core/services/analytics/utils.ts +++ b/airbyte-webapp/src/core/services/analytics/utils.ts @@ -1,4 +1,4 @@ -import { ConnectionScheduleData } from "core/request/AirbyteClient"; +import { ConnectionScheduleData } from "core/api/types/AirbyteClient"; export const getFrequencyFromScheduleData = (scheduleData?: ConnectionScheduleData) => { if (!scheduleData) { diff --git a/airbyte-webapp/src/core/services/auth/EnterpriseAuthService.tsx b/airbyte-webapp/src/core/services/auth/EnterpriseAuthService.tsx index 8d641398299..631eb4889b8 100644 --- a/airbyte-webapp/src/core/services/auth/EnterpriseAuthService.tsx +++ b/airbyte-webapp/src/core/services/auth/EnterpriseAuthService.tsx @@ -8,7 +8,7 @@ import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; import { useGetInstanceConfiguration, useGetOrCreateUser } from "core/api"; -import { UserRead } from "core/request/AirbyteClient"; +import { UserRead } from "core/api/types/AirbyteClient"; import { useNotificationService } from "hooks/services/Notification"; import { createUriWithoutSsoParams } from "packages/cloud/services/auth/KeycloakService"; diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx b/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx index 1384f88cb53..8e3a551863b 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx +++ b/airbyte-webapp/src/core/utils/rbac/rbac.docs.tsx @@ -6,7 +6,7 @@ import { Icon } from "components/ui/Icon"; import { Input } from "components/ui/Input"; import { ListBox } from "components/ui/ListBox"; -import { PermissionType } from "core/request/AirbyteClient"; +import { PermissionType } from "core/api/types/AirbyteClient"; import { RbacQuery, diff --git a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts index 9eb00b06351..8c1420ff261 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbac.test.ts +++ b/airbyte-webapp/src/core/utils/rbac/rbac.test.ts @@ -3,7 +3,7 @@ import { renderHook } from "@testing-library/react"; import { mockUser } from "test-utils/mock-data/mockUser"; import { useListPermissions } from "core/api"; -import { PermissionRead } from "core/request/AirbyteClient"; +import { PermissionRead } from "core/api/types/AirbyteClient"; import { useRbac } from "./rbac"; import { RbacPermission, useRbacPermissionsQuery } from "./rbacPermissionsQuery"; diff --git a/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.ts b/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.ts index 633371365ec..623e4d06e90 100644 --- a/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.ts +++ b/airbyte-webapp/src/core/utils/rbac/rbacPermissionsQuery.ts @@ -1,5 +1,5 @@ import { useGetWorkspace } from "core/api"; -import { PermissionRead } from "core/request/AirbyteClient"; +import { PermissionRead } from "core/api/types/AirbyteClient"; export const RbacResourceHierarchy = ["INSTANCE", "ORGANIZATION", "WORKSPACE"] as const; export const RbacRoleHierarchy = ["ADMIN", "EDITOR", "READER", "MEMBER"] as const; diff --git a/airbyte-webapp/src/core/utils/useLocalStorage.ts b/airbyte-webapp/src/core/utils/useLocalStorage.ts index efa0593225f..d66e8b09464 100644 --- a/airbyte-webapp/src/core/utils/useLocalStorage.ts +++ b/airbyte-webapp/src/core/utils/useLocalStorage.ts @@ -4,7 +4,7 @@ import { useLocalStorage as useLocalStorageWithUndefinedBug } from "react-use"; import { BuilderState } from "components/connectorBuilder/types"; -import { SupportLevel } from "core/request/AirbyteClient"; +import { SupportLevel } from "core/api/types/AirbyteClient"; import { Theme } from "hooks/theme/useAirbyteTheme"; // Represents all the data we store in localStorage across the airbyte app diff --git a/airbyte-webapp/src/hooks/connection/useDestinationNamespace.ts b/airbyte-webapp/src/hooks/connection/useDestinationNamespace.ts index 6200618b45f..7265c644a4f 100644 --- a/airbyte-webapp/src/hooks/connection/useDestinationNamespace.ts +++ b/airbyte-webapp/src/hooks/connection/useDestinationNamespace.ts @@ -1,6 +1,6 @@ import { useIntl } from "react-intl"; -import { NamespaceDefinitionType } from "core/request/AirbyteClient"; +import { NamespaceDefinitionType } from "core/api/types/AirbyteClient"; interface NamespaceOptions { namespaceDefinition: diff --git a/airbyte-webapp/src/hooks/connection/useSchemaChanges.ts b/airbyte-webapp/src/hooks/connection/useSchemaChanges.ts index 6a7d162cd25..a6d30694176 100644 --- a/airbyte-webapp/src/hooks/connection/useSchemaChanges.ts +++ b/airbyte-webapp/src/hooks/connection/useSchemaChanges.ts @@ -1,6 +1,6 @@ import { useMemo } from "react"; -import { SchemaChange } from "core/request/AirbyteClient"; +import { SchemaChange } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; export const useSchemaChanges = (schemaChange: SchemaChange) => { diff --git a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx index 5db4fdda93c..3098498a415 100644 --- a/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx +++ b/airbyte-webapp/src/hooks/services/ConnectionForm/ConnectionFormService.tsx @@ -20,7 +20,7 @@ import { SourceDefinitionRead, SourceDefinitionSpecificationRead, WebBackendConnectionRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { FormError, generateMessageFromError } from "core/utils/errorStatusMessage"; export type ConnectionFormMode = "create" | "edit" | "readonly"; diff --git a/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx b/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx index cb5c04b5fb5..bf1a0070352 100644 --- a/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx +++ b/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx @@ -4,16 +4,15 @@ import { FormattedMessage, useIntl } from "react-intl"; import { useAsyncFn, useEffectOnce, useEvent } from "react-use"; import { v4 as uuid } from "uuid"; -import { useCompleteOAuth, useConsentUrls } from "core/api"; -import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; -import { isSourceDefinitionSpecification } from "core/domain/connector/source"; +import { useCompleteOAuth, useConsentUrls, isCommonRequestError } from "core/api"; import { CompleteOAuthResponse, CompleteOAuthResponseAuthPayload, DestinationOauthConsentRequest, SourceOauthConsentRequest, -} from "core/request/AirbyteClient"; -import { isCommonRequestError } from "core/request/CommonRequestError"; +} from "core/api/types/AirbyteClient"; +import { ConnectorDefinition, ConnectorDefinitionSpecification, ConnectorSpecification } from "core/domain/connector"; +import { isSourceDefinitionSpecification } from "core/domain/connector/source"; import { useAnalyticsTrackFunctions } from "views/Connector/ConnectorForm/components/Sections/auth/useAnalyticsTrackFunctions"; import { useConnectorForm } from "views/Connector/ConnectorForm/connectorFormContext"; diff --git a/airbyte-webapp/src/hooks/services/useWorkspace.tsx b/airbyte-webapp/src/hooks/services/useWorkspace.tsx index 31ea2ae437d..7400141528f 100644 --- a/airbyte-webapp/src/hooks/services/useWorkspace.tsx +++ b/airbyte-webapp/src/hooks/services/useWorkspace.tsx @@ -1,6 +1,6 @@ import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useCurrentWorkspace, useUpdateWorkspace } from "core/api"; -import { NotificationSettings } from "core/request/AirbyteClient"; +import { NotificationSettings } from "core/api/types/AirbyteClient"; export const useUpdateNotificationSettings = () => { const workspaceId = useCurrentWorkspaceId(); diff --git a/airbyte-webapp/src/packages/cloud/services/auth/KeycloakService/KeycloakService.tsx b/airbyte-webapp/src/packages/cloud/services/auth/KeycloakService/KeycloakService.tsx index e765b4b23e7..d3a711e4e55 100644 --- a/airbyte-webapp/src/packages/cloud/services/auth/KeycloakService/KeycloakService.tsx +++ b/airbyte-webapp/src/packages/cloud/services/auth/KeycloakService/KeycloakService.tsx @@ -15,7 +15,7 @@ import { import { config } from "config"; import { useGetOrCreateUser } from "core/api"; -import { UserRead } from "core/request/AirbyteClient"; +import { UserRead } from "core/api/types/AirbyteClient"; const DEFAULT_KEYCLOAK_REALM = "airbyte"; const DEFAULT_KEYCLOAK_CLIENT_ID = "airbyte-webapp"; diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx index c91444487a0..70e2b26a794 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageContext.tsx @@ -5,8 +5,8 @@ import { Option } from "components/ui/ListBox"; import { useCurrentWorkspace } from "core/api"; import { useGetCloudWorkspaceUsage } from "core/api/cloud"; +import { DestinationId, SourceId, SupportLevel } from "core/api/types/AirbyteClient"; import { ConsumptionTimeWindow } from "core/api/types/CloudApi"; -import { DestinationId, SourceId, SupportLevel } from "core/request/AirbyteClient"; import { calculateAvailableSourcesAndDestinations } from "./calculateAvailableSourcesAndDestinations"; import { diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageFilters.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageFilters.tsx index 42e6202ee78..6fdd45d10c7 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageFilters.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/CreditsUsageFilters.tsx @@ -6,8 +6,8 @@ import { Icon } from "components/ui/Icon"; import { ListBox, ListBoxControlButtonProps } from "components/ui/ListBox"; import { Text } from "components/ui/Text"; +import { DestinationId, SourceId } from "core/api/types/AirbyteClient"; import { ConsumptionTimeWindow } from "core/api/types/CloudApi"; -import { DestinationId, SourceId } from "core/request/AirbyteClient"; import { useCreditsContext } from "./CreditsUsageContext"; import styles from "./CreditsUsageFilters.module.scss"; diff --git a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerConnectionTable.tsx b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerConnectionTable.tsx index 139b0d879b0..2531cd465a2 100644 --- a/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerConnectionTable.tsx +++ b/airbyte-webapp/src/packages/cloud/views/billing/BillingPage/components/UsagePerConnectionTable.tsx @@ -13,7 +13,7 @@ import { TextWithOverflowTooltip } from "components/ui/Text"; import { InfoTooltip } from "components/ui/Tooltip"; import { useCurrentWorkspace } from "core/api"; -import { ConnectionScheduleType, ConnectionStatus } from "core/request/AirbyteClient"; +import { ConnectionScheduleType, ConnectionStatus } from "core/api/types/AirbyteClient"; import { RoutePaths } from "pages/routePaths"; import { ConnectionFreeAndPaidUsage } from "./calculateUsageDataObjects"; diff --git a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/InsufficientPermissionsErrorBoundary.tsx b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/InsufficientPermissionsErrorBoundary.tsx index 16d58474e76..b81c25c1577 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/InsufficientPermissionsErrorBoundary.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/CloudMainView/InsufficientPermissionsErrorBoundary.tsx @@ -1,6 +1,6 @@ import React from "react"; -import { CommonRequestError } from "core/request/CommonRequestError"; +import { CommonRequestError } from "core/api"; import { TrackErrorFn } from "hooks/services/AppMonitoringService"; interface BoundaryState { diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/DataResidencyView/DataResidencyView.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/DataResidencyView/DataResidencyView.tsx index c188b743bd5..a107c10b5bc 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/DataResidencyView/DataResidencyView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/DataResidencyView/DataResidencyView.tsx @@ -11,7 +11,7 @@ import { ExternalLink } from "components/ui/Link"; import { Text } from "components/ui/Text"; import { useUpdateWorkspace } from "core/api"; -import { Geography } from "core/request/AirbyteClient"; +import { Geography } from "core/api/types/AirbyteClient"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { trackError } from "core/utils/datadog"; import { links } from "core/utils/links"; diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesCreateControl.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesCreateControl.tsx index 5657951d7a9..6681e161aa7 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesCreateControl.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/CloudWorkspacesCreateControl.tsx @@ -14,7 +14,7 @@ import { Text } from "components/ui/Text"; import { useListWorkspaces } from "core/api"; import { useCreateCloudWorkspace } from "core/api/cloud"; -import { OrganizationRead } from "core/request/AirbyteClient"; +import { OrganizationRead } from "core/api/types/AirbyteClient"; import { trackError } from "core/utils/datadog"; import { useNotificationService } from "hooks/services/Notification"; import { useOrganizationsToCreateWorkspaces } from "pages/workspaces/components/useOrganizationsToCreateWorkspaces"; diff --git a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx index 33fc711339a..5f6272bda70 100644 --- a/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/GeneralOrganizationSettingsPage.tsx @@ -9,7 +9,7 @@ import { Box } from "components/ui/Box"; import { Card } from "components/ui/Card"; import { useCurrentWorkspace, useUpdateOrganization, useOrganization } from "core/api"; -import { OrganizationUpdateRequestBody } from "core/request/AirbyteClient"; +import { OrganizationUpdateRequestBody } from "core/api/types/AirbyteClient"; import { useIntent } from "core/utils/rbac"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx index 55afd598fa2..268acdb3af7 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementCard.tsx @@ -5,7 +5,7 @@ import { Card } from "components/ui/Card"; import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; -import { OrganizationUserRead, WorkspaceUserRead } from "core/request/AirbyteClient"; +import { OrganizationUserRead, WorkspaceUserRead } from "core/api/types/AirbyteClient"; import { AccessManagementTable } from "./AccessManagementTable"; import { AddUserControl } from "./AddUserControl"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx index 76e0d8bd15f..a2be7e3fc31 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AccessManagementTable.tsx @@ -6,7 +6,7 @@ import { Badge } from "components/ui/Badge"; import { FlexContainer } from "components/ui/Flex"; import { Table } from "components/ui/Table"; -import { OrganizationUserRead, WorkspaceUserRead } from "core/request/AirbyteClient"; +import { OrganizationUserRead, WorkspaceUserRead } from "core/api/types/AirbyteClient"; import { useCurrentUser } from "core/services/auth"; import { RoleManagementControl } from "./RoleManagementControl"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AddUserControl.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AddUserControl.tsx index 6b129d042c7..5755f407e7e 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AddUserControl.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/AddUserControl.tsx @@ -12,7 +12,7 @@ import { Text } from "components/ui/Text"; import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useCreatePermission } from "core/api"; -import { OrganizationUserRead, PermissionCreate, PermissionType } from "core/request/AirbyteClient"; +import { OrganizationUserRead, PermissionCreate, PermissionType } from "core/api/types/AirbyteClient"; import styles from "./AddUserControl.module.scss"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx index a02717a50e8..d58304c77c4 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/RoleManagementControl.tsx @@ -9,7 +9,7 @@ import { FlexContainer } from "components/ui/Flex"; import { Text } from "components/ui/Text"; import { useDeletePermissions, useUpdatePermissions } from "core/api"; -import { PermissionRead, PermissionType, PermissionUpdate } from "core/request/AirbyteClient"; +import { PermissionRead, PermissionType, PermissionUpdate } from "core/api/types/AirbyteClient"; import { useCurrentUser } from "core/services/auth"; import { useIntent } from "core/utils/rbac"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx index 6cde75cd27a..b23af7b28f8 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccessManagementPage/components/useGetAccessManagementData.tsx @@ -1,5 +1,5 @@ import { useCurrentWorkspace, useListUsersInOrganization, useListUsersInWorkspace } from "core/api"; -import { OrganizationUserRead, PermissionType, WorkspaceUserRead } from "core/request/AirbyteClient"; +import { OrganizationUserRead, PermissionType, WorkspaceUserRead } from "core/api/types/AirbyteClient"; import { useIntent } from "core/utils/rbac"; export type ResourceType = "workspace" | "organization" | "instance"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx index 229254f7f32..b6183c3f5f6 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/DestinationsPage.tsx @@ -2,7 +2,7 @@ import React, { useCallback, useMemo, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useDestinationDefinitionList, useUpdateDestinationDefinition, useDestinationList } from "core/api"; -import { DestinationDefinitionRead } from "core/request/AirbyteClient"; +import { DestinationDefinitionRead } from "core/api/types/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx index bdc1e4b21aa..ef4c1695d2f 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx @@ -2,7 +2,7 @@ import React, { useCallback, useMemo, useRef, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; import { useListBuilderProjects, useSourceDefinitionList, useUpdateSourceDefinition, useSourceList } from "core/api"; -import { SourceDefinitionRead } from "core/request/AirbyteClient"; +import { SourceDefinitionRead } from "core/api/types/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorCell.tsx index fa336bc1c40..6d134297064 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorCell.tsx @@ -4,7 +4,7 @@ import { FlexContainer } from "components/ui/Flex"; import { SupportLevelBadge } from "components/ui/SupportLevelBadge"; import { SvgIcon } from "area/connector/utils"; -import { SupportLevel } from "core/request/AirbyteClient"; +import { SupportLevel } from "core/api/types/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import styles from "./ConnectorCell.module.scss"; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx index 1343a7c0a46..31dd480e68c 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/ConnectorsView.tsx @@ -10,8 +10,8 @@ import { Table } from "components/ui/Table"; import { InfoTooltip } from "components/ui/Tooltip"; import { BuilderProject } from "core/api"; +import { DestinationDefinitionRead, SourceDefinitionRead } from "core/api/types/AirbyteClient"; import { Connector, ConnectorDefinition } from "core/domain/connector"; -import { DestinationDefinitionRead, SourceDefinitionRead } from "core/request/AirbyteClient"; import { FeatureItem, useFeature } from "core/services/features"; import { useIntent } from "core/utils/rbac"; import { RoutePaths } from "pages/routePaths"; diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx index 72c10fc74c0..887c2d7ac25 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx @@ -17,7 +17,7 @@ import { PageHeader } from "components/ui/PageHeader"; import { Text } from "components/ui/Text"; import { useConnectionList } from "core/api"; -import { WebBackendConnectionListItem } from "core/request/AirbyteClient"; +import { WebBackendConnectionListItem } from "core/api/types/AirbyteClient"; import { useTrackPage, PageTrackingCodes } from "core/services/analytics"; import { naturalComparatorBy } from "core/utils/objects"; import { useExperiment } from "hooks/services/Experiment"; diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsTable.tsx b/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsTable.tsx index 269e7190c83..dd87f40ac14 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsTable.tsx +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/ConnectionsTable.tsx @@ -5,7 +5,7 @@ import { ConnectionTable } from "components/EntityTable"; import { ConnectionTableDataItem } from "components/EntityTable/types"; import { getConnectionTableData } from "components/EntityTable/utils"; -import { WebBackendConnectionListItem } from "core/request/AirbyteClient"; +import { WebBackendConnectionListItem } from "core/api/types/AirbyteClient"; interface ConnectionsTableProps { connections: WebBackendConnectionListItem[]; diff --git a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/JobsList.tsx b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/JobsList.tsx index 0a58ab1db21..3644aa3a2c9 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/JobsList.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionJobHistoryPage/JobsList.tsx @@ -2,7 +2,7 @@ import React, { useMemo } from "react"; import { JobHistoryItem } from "area/connection/components/JobHistoryItem"; import { JobWithAttempts } from "area/connection/types/jobs"; -import { JobWithAttemptsRead } from "core/request/AirbyteClient"; +import { JobWithAttemptsRead } from "core/api/types/AirbyteClient"; interface JobsListProps { jobs: JobWithAttemptsRead[]; diff --git a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionTitleBlock.tsx b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionTitleBlock.tsx index e436d0ae226..65df3f68d58 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionTitleBlock.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionPage/ConnectionTitleBlock.tsx @@ -11,7 +11,7 @@ import { Message } from "components/ui/Message"; import { SupportLevelBadge } from "components/ui/SupportLevelBadge"; import { Text } from "components/ui/Text"; -import { ConnectionStatus, SupportLevel } from "core/request/AirbyteClient"; +import { ConnectionStatus, SupportLevel } from "core/api/types/AirbyteClient"; import { useSchemaChanges } from "hooks/connection/useSchemaChanges"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useConnectionFormService } from "hooks/services/ConnectionForm/ConnectionFormService"; diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx index df069459b5c..f737bb06529 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ConnectionReplicationPage.test.tsx @@ -47,6 +47,7 @@ jest.mock("core/api", () => ({ useGetDestinationDefinitionSpecification: () => mockDestinationDefinitionSpecification, useSourceDefinition: () => mockSourceDefinition, useDestinationDefinition: () => mockDestinationDefinition, + LogsRequestError: jest.requireActual("core/api/errors").LogsRequestError, })); jest.mock("hooks/theme/useAirbyteTheme", () => ({ diff --git a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx index 0e553063e3e..91ee5e7bc6f 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionReplicationPage/ResetWarningModal.tsx @@ -5,7 +5,7 @@ import { LabeledSwitch } from "components"; import { Button } from "components/ui/Button"; import { ModalBody, ModalFooter } from "components/ui/Modal"; -import { ConnectionStateType } from "core/request/AirbyteClient"; +import { ConnectionStateType } from "core/api/types/AirbyteClient"; interface ResetWarningModalProps { onClose: (withReset: boolean) => void; diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx index 153f6b3236e..265c2090860 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/ConnectionSettingsPage.tsx @@ -16,7 +16,7 @@ import { ExternalLink } from "components/ui/Link"; import { Spinner } from "components/ui/Spinner"; import { useDeleteConnection } from "core/api"; -import { Geography, WebBackendConnectionUpdate } from "core/request/AirbyteClient"; +import { Geography, WebBackendConnectionUpdate } from "core/api/types/AirbyteClient"; import { PageTrackingCodes, useTrackPage } from "core/services/analytics"; import { FeatureItem, useFeature } from "core/services/features"; import { links } from "core/utils/links"; diff --git a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx index 3f231391c7e..3734fb03cc3 100644 --- a/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx +++ b/airbyte-webapp/src/pages/connections/ConnectionSettingsPage/StateBlock.tsx @@ -11,7 +11,7 @@ import { Message } from "components/ui/Message"; import { Text } from "components/ui/Text"; import { useCreateOrUpdateState, useGetConnectionState } from "core/api"; -import { AirbyteCatalog, ConnectionState, StreamState } from "core/request/AirbyteClient"; +import { AirbyteCatalog, ConnectionState, StreamState } from "core/api/types/AirbyteClient"; import { haveSameShape } from "core/utils/objects"; import { useConfirmationModalService } from "hooks/services/ConfirmationModal"; diff --git a/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx b/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx index c25a57c0122..727ab46c5ad 100644 --- a/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx +++ b/airbyte-webapp/src/pages/connections/CreateConnectionPage/CreateConnectionTitleBlock.tsx @@ -19,7 +19,7 @@ import { useGetDestination, useGetSource, } from "core/api"; -import { SupportLevel } from "core/request/AirbyteClient"; +import { SupportLevel } from "core/api/types/AirbyteClient"; import { RoutePaths } from "pages/routePaths"; import styles from "./CreateConnectionTitleBlock.module.scss"; diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusMessages.tsx b/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusMessages.tsx index 7018889ee43..da69fad4666 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusMessages.tsx +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusMessages.tsx @@ -9,13 +9,13 @@ import { Message, MessageProps, MessageType, isHigherSeverity, MESSAGE_SEVERITY_ import { useCurrentWorkspaceId } from "area/workspace/utils"; import { useDestinationDefinitionVersion, useSourceDefinitionVersion } from "core/api"; -import { shouldDisplayBreakingChangeBanner, getHumanReadableUpgradeDeadline } from "core/domain/connector"; import { ActorDefinitionVersionRead, FailureOrigin, FailureType, JobWithAttemptsRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; +import { shouldDisplayBreakingChangeBanner, getHumanReadableUpgradeDeadline } from "core/domain/connector"; import { FeatureItem, useFeature } from "core/services/features"; import { useSchemaChanges } from "hooks/connection/useSchemaChanges"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusOverview.tsx b/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusOverview.tsx index afffa4b0694..aa22782a92a 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusOverview.tsx +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/ConnectionStatusOverview.tsx @@ -11,7 +11,7 @@ import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; import { Tooltip } from "components/ui/Tooltip"; -import { JobStatus } from "core/request/AirbyteClient"; +import { JobStatus } from "core/api/types/AirbyteClient"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import styles from "./ConnectionStatusOverview.module.scss"; diff --git a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.tsx b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.tsx index fc9e232313f..836a7fe39c1 100644 --- a/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.tsx +++ b/airbyte-webapp/src/pages/connections/StreamStatusPage/StreamsList.tsx @@ -13,7 +13,7 @@ import { Icon } from "components/ui/Icon"; import { Table } from "components/ui/Table"; import { Text } from "components/ui/Text"; -import { ConnectionStatus } from "core/request/AirbyteClient"; +import { ConnectionStatus } from "core/api/types/AirbyteClient"; import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { StreamActionsMenu } from "./StreamActionsMenu"; diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 025c5520944..d1d2b679d8b 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -20,7 +20,7 @@ import { CompleteOauthRequest } from "views/CompleteOauthRequest"; import MainView from "views/layout/MainView"; import { RoutePaths, DestinationPaths, SourcePaths } from "./routePaths"; -import { WorkspaceRead } from "../core/request/AirbyteClient"; +import { WorkspaceRead } from "../core/api/types/AirbyteClient"; const DefaultView = React.lazy(() => import("./DefaultView")); const ConnectionsRoutes = React.lazy(() => import("./connections/ConnectionsRoutes")); diff --git a/airbyte-webapp/src/pages/source/AllSourcesPage/SourcesTable.tsx b/airbyte-webapp/src/pages/source/AllSourcesPage/SourcesTable.tsx index 32f44ca1ec0..4235f73b82e 100644 --- a/airbyte-webapp/src/pages/source/AllSourcesPage/SourcesTable.tsx +++ b/airbyte-webapp/src/pages/source/AllSourcesPage/SourcesTable.tsx @@ -6,7 +6,7 @@ import { EntityTableDataItem } from "components/EntityTable/types"; import { getEntityTableData } from "components/EntityTable/utils"; import { useConnectionList } from "core/api"; -import { SourceRead } from "core/request/AirbyteClient"; +import { SourceRead } from "core/api/types/AirbyteClient"; interface SourcesTableProps { sources: SourceRead[]; diff --git a/airbyte-webapp/src/pages/source/CreateSourcePage/SourceForm.tsx b/airbyte-webapp/src/pages/source/CreateSourcePage/SourceForm.tsx index 23cbdaf6e69..d06fbf7b829 100644 --- a/airbyte-webapp/src/pages/source/CreateSourcePage/SourceForm.tsx +++ b/airbyte-webapp/src/pages/source/CreateSourcePage/SourceForm.tsx @@ -7,10 +7,9 @@ import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { ConnectionConfiguration } from "area/connector/types"; -import { useGetSourceDefinitionSpecificationAsync } from "core/api"; +import { useGetSourceDefinitionSpecificationAsync, LogsRequestError } from "core/api"; import { SourceDefinitionRead } from "core/api/types/AirbyteClient"; import { Connector } from "core/domain/connector"; -import { LogsRequestError } from "core/request/LogsRequestError"; import { FormError } from "core/utils/errorStatusMessage"; import { ConnectorCard } from "views/Connector/ConnectorCard"; import { ConnectorCardValues } from "views/Connector/ConnectorForm/types"; diff --git a/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionTable.tsx b/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionTable.tsx index 20b6ef4920c..f34d432b3cf 100644 --- a/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionTable.tsx +++ b/airbyte-webapp/src/pages/source/SourceConnectionsPage/SourceConnectionTable.tsx @@ -5,7 +5,7 @@ import { ConnectionTable } from "components/EntityTable"; import { ConnectionTableDataItem } from "components/EntityTable/types"; import { getConnectionTableData } from "components/EntityTable/utils"; -import { WebBackendConnectionListItem } from "core/request/AirbyteClient"; +import { WebBackendConnectionListItem } from "core/api/types/AirbyteClient"; import { RoutePaths } from "pages/routePaths"; import styles from "./SourceConnectionTable.module.scss"; diff --git a/airbyte-webapp/src/pages/workspaces/components/WorkspacesCreateControl.tsx b/airbyte-webapp/src/pages/workspaces/components/WorkspacesCreateControl.tsx index fdfa5a0636c..b65889c8f25 100644 --- a/airbyte-webapp/src/pages/workspaces/components/WorkspacesCreateControl.tsx +++ b/airbyte-webapp/src/pages/workspaces/components/WorkspacesCreateControl.tsx @@ -22,7 +22,7 @@ import { Icon } from "components/ui/Icon"; import { Text } from "components/ui/Text"; import { useListWorkspaces } from "core/api"; -import { OrganizationRead, WorkspaceCreate, WorkspaceRead } from "core/request/AirbyteClient"; +import { OrganizationRead, WorkspaceCreate, WorkspaceRead } from "core/api/types/AirbyteClient"; import { trackError } from "core/utils/datadog"; import { useNotificationService } from "hooks/services/Notification"; diff --git a/airbyte-webapp/src/pages/workspaces/components/WorkspacesList.tsx b/airbyte-webapp/src/pages/workspaces/components/WorkspacesList.tsx index a9a3bde3809..c6466a41a6f 100644 --- a/airbyte-webapp/src/pages/workspaces/components/WorkspacesList.tsx +++ b/airbyte-webapp/src/pages/workspaces/components/WorkspacesList.tsx @@ -7,8 +7,8 @@ import { FlexContainer } from "components/ui/Flex"; import { Heading } from "components/ui/Heading"; import { LoadingSpinner } from "components/ui/LoadingSpinner"; +import { WorkspaceRead } from "core/api/types/AirbyteClient"; import { CloudWorkspaceRead } from "core/api/types/CloudApi"; -import { WorkspaceRead } from "core/request/AirbyteClient"; import { WorkspaceItem } from "./WorkspaceItem"; diff --git a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx index 7af9b83d2aa..ee3c410b626 100644 --- a/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx +++ b/airbyte-webapp/src/services/connectorBuilder/ConnectorBuilderStateService.tsx @@ -34,11 +34,11 @@ import { useBuilderResolvedManifestSuspense, } from "core/api"; import { useIsForeignWorkspace } from "core/api/cloud"; +import { SourceDefinitionIdBody } from "core/api/types/AirbyteClient"; import { ConnectorConfig, KnownExceptionInfo, StreamRead } from "core/api/types/ConnectorBuilderClient"; import { ConnectorManifest, DeclarativeComponentSchema, Spec } from "core/api/types/ConnectorManifest"; import { jsonSchemaToFormBlock } from "core/form/schemaToFormBlock"; import { FormGroupItem } from "core/form/types"; -import { SourceDefinitionIdBody } from "core/request/AirbyteClient"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; import { FeatureItem, useFeature } from "core/services/features"; import { Blocker, useBlocker } from "core/services/navigation"; diff --git a/airbyte-webapp/src/test-utils/mock-data/mockAirbyteStream.ts b/airbyte-webapp/src/test-utils/mock-data/mockAirbyteStream.ts index 7dd7685e47e..58da46b3e20 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockAirbyteStream.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockAirbyteStream.ts @@ -1,4 +1,4 @@ -import { AirbyteStream } from "core/request/AirbyteClient"; +import { AirbyteStream } from "core/api/types/AirbyteClient"; export const mockAirbyteStream: AirbyteStream = { name: "Mock stream", diff --git a/airbyte-webapp/src/test-utils/mock-data/mockAirbyteStreamConfiguration.ts b/airbyte-webapp/src/test-utils/mock-data/mockAirbyteStreamConfiguration.ts index 309932ae5ac..3eebe073c40 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockAirbyteStreamConfiguration.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockAirbyteStreamConfiguration.ts @@ -1,4 +1,4 @@ -import { AirbyteStreamConfiguration } from "core/request/AirbyteClient"; +import { AirbyteStreamConfiguration } from "core/api/types/AirbyteClient"; export const mockStreamConfiguration: AirbyteStreamConfiguration = { fieldSelectionEnabled: false, diff --git a/airbyte-webapp/src/test-utils/mock-data/mockAttempt.ts b/airbyte-webapp/src/test-utils/mock-data/mockAttempt.ts index d965cdbf566..e44db139b11 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockAttempt.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockAttempt.ts @@ -1,4 +1,4 @@ -import { AttemptRead, JobStatus } from "core/request/AirbyteClient"; +import { AttemptRead, JobStatus } from "core/api/types/AirbyteClient"; export const mockAttempt: AttemptRead = { id: 1, diff --git a/airbyte-webapp/src/test-utils/mock-data/mockCatalogDiff.ts b/airbyte-webapp/src/test-utils/mock-data/mockCatalogDiff.ts index a95242228ce..05d6a2ba963 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockCatalogDiff.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockCatalogDiff.ts @@ -1,4 +1,4 @@ -import { CatalogDiff } from "core/request/AirbyteClient"; +import { CatalogDiff } from "core/api/types/AirbyteClient"; export const mockCatalogDiff: CatalogDiff = { transforms: [ diff --git a/airbyte-webapp/src/test-utils/mock-data/mockConnection.ts b/airbyte-webapp/src/test-utils/mock-data/mockConnection.ts index f8b2025af32..b2a8fec6ecb 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockConnection.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockConnection.ts @@ -1,6 +1,6 @@ /* eslint-disable no-template-curly-in-string */ import { ConnectorIds } from "area/connector/utils"; -import { WebBackendConnectionRead } from "core/request/AirbyteClient"; +import { WebBackendConnectionRead } from "core/api/types/AirbyteClient"; export const mockConnection: WebBackendConnectionRead = { connectionId: "a9c8e4b5-349d-4a17-bdff-5ad2f6fbd611", diff --git a/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts b/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts index c524fd9b1ce..93a8337d8a1 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockDestination.ts @@ -4,7 +4,7 @@ import { DestinationDefinitionRead, ActorDefinitionVersionRead, SupportState, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; export const mockDestinationDefinition: DestinationDefinitionRead = { destinationDefinitionId: ConnectorIds.Destinations.Postgres, diff --git a/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinitionSpecification.ts b/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinitionSpecification.ts index 63ade7deda3..fbc61269f81 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinitionSpecification.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockDestinationDefinitionSpecification.ts @@ -1,5 +1,5 @@ import { ConnectorIds } from "area/connector/utils"; -import { DestinationDefinitionSpecificationRead } from "core/request/AirbyteClient"; +import { DestinationDefinitionSpecificationRead } from "core/api/types/AirbyteClient"; export const mockDestinationDefinitionSpecification: DestinationDefinitionSpecificationRead = { destinationDefinitionId: ConnectorIds.Destinations.Postgres, diff --git a/airbyte-webapp/src/test-utils/mock-data/mockInstanceConfig.ts b/airbyte-webapp/src/test-utils/mock-data/mockInstanceConfig.ts index d7c8f47c6fd..165e771b637 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockInstanceConfig.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockInstanceConfig.ts @@ -1,4 +1,4 @@ -import { InstanceConfigurationResponse } from "core/request/AirbyteClient"; +import { InstanceConfigurationResponse } from "core/api/types/AirbyteClient"; export const mockProInstanceConfig: InstanceConfigurationResponse = { auth: { diff --git a/airbyte-webapp/src/test-utils/mock-data/mockJob.ts b/airbyte-webapp/src/test-utils/mock-data/mockJob.ts index 500830a9bc4..7e43aaa6a50 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockJob.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockJob.ts @@ -1,4 +1,4 @@ -import { JobConfigType, JobRead, JobStatus } from "core/request/AirbyteClient"; +import { JobConfigType, JobRead, JobStatus } from "core/api/types/AirbyteClient"; export const mockJob: JobRead = { id: 1, diff --git a/airbyte-webapp/src/test-utils/mock-data/mockSource.ts b/airbyte-webapp/src/test-utils/mock-data/mockSource.ts index 22ebc366126..efc00da8262 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockSource.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockSource.ts @@ -3,7 +3,7 @@ import { SourceDefinitionRead, SourceDefinitionSpecificationRead, SupportState, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { ConnectorIds } from "../../area/connector/utils"; diff --git a/airbyte-webapp/src/test-utils/mock-data/mockSourceDefinition.ts b/airbyte-webapp/src/test-utils/mock-data/mockSourceDefinition.ts index 9e911db36b7..e7218f9d604 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockSourceDefinition.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockSourceDefinition.ts @@ -1,5 +1,5 @@ import { ConnectorIds } from "area/connector/utils"; -import { SourceDefinitionSpecificationRead } from "core/request/AirbyteClient"; +import { SourceDefinitionSpecificationRead } from "core/api/types/AirbyteClient"; export const mockSourceDefinition: SourceDefinitionSpecificationRead = { sourceDefinitionId: ConnectorIds.Sources.PokeApi, diff --git a/airbyte-webapp/src/test-utils/mock-data/mockStreamStatusRead.ts b/airbyte-webapp/src/test-utils/mock-data/mockStreamStatusRead.ts index b11777d9849..fb84da4cdbc 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockStreamStatusRead.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockStreamStatusRead.ts @@ -1,4 +1,4 @@ -import { StreamStatusJobType, StreamStatusRead, StreamStatusRunState } from "core/request/AirbyteClient"; +import { StreamStatusJobType, StreamStatusRead, StreamStatusRunState } from "core/api/types/AirbyteClient"; export const mockStreamStatusRead: StreamStatusRead = { id: "123", diff --git a/airbyte-webapp/src/test-utils/mock-data/mockWorkspace.ts b/airbyte-webapp/src/test-utils/mock-data/mockWorkspace.ts index ba9407f0a62..c3422e13109 100644 --- a/airbyte-webapp/src/test-utils/mock-data/mockWorkspace.ts +++ b/airbyte-webapp/src/test-utils/mock-data/mockWorkspace.ts @@ -1,4 +1,4 @@ -import { WorkspaceRead } from "core/request/AirbyteClient"; +import { WorkspaceRead } from "core/api/types/AirbyteClient"; export const mockWorkspace: WorkspaceRead = { workspaceId: "47c74b9b-9b89-4af1-8331-4865af6c4e4d", diff --git a/airbyte-webapp/src/test-utils/testutils.tsx b/airbyte-webapp/src/test-utils/testutils.tsx index 4e65499a4c9..6f56afcdb67 100644 --- a/airbyte-webapp/src/test-utils/testutils.tsx +++ b/airbyte-webapp/src/test-utils/testutils.tsx @@ -12,7 +12,7 @@ import { NamespaceDefinitionType, SourceRead, WebBackendConnectionRead, -} from "core/request/AirbyteClient"; +} from "core/api/types/AirbyteClient"; import { AnalyticsProvider } from "core/services/analytics"; import { defaultOssFeatures, FeatureItem, FeatureService } from "core/services/features"; import { ConfirmationModalService } from "hooks/services/ConfirmationModal"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx index d18c04ac09c..c4c4a31533a 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/ConnectorCard.tsx @@ -12,6 +12,8 @@ import { Pre } from "components/ui/Pre"; import { Spinner } from "components/ui/Spinner"; import { useAirbyteCloudIps } from "area/connector/utils/useAirbyteCloudIps"; +import { LogsRequestError } from "core/api"; +import { DestinationRead, SourceRead, SupportLevel, SynchronousJobRead } from "core/api/types/AirbyteClient"; import { Connector, ConnectorDefinition, @@ -19,8 +21,6 @@ import { ConnectorSpecification, ConnectorT, } from "core/domain/connector"; -import { DestinationRead, SourceRead, SupportLevel, SynchronousJobRead } from "core/request/AirbyteClient"; -import { LogsRequestError } from "core/request/LogsRequestError"; import { isCloudApp } from "core/utils/app"; import { generateMessageFromError } from "core/utils/errorStatusMessage"; import { links } from "core/utils/links"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/components/Controls.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/components/Controls.tsx index 8c44f102680..d86d75ff31a 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/components/Controls.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/components/Controls.tsx @@ -4,7 +4,7 @@ import { FormattedMessage } from "react-intl"; import { Button } from "components/ui/Button"; import { FlexContainer, FlexItem } from "components/ui/Flex"; -import { SynchronousJobRead } from "core/request/AirbyteClient"; +import { SynchronousJobRead } from "core/api/types/AirbyteClient"; import { TestCard } from "./TestCard"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestCard.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestCard.tsx index 9697e4a2d9a..6b4cda4f972 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestCard.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/components/TestCard.tsx @@ -9,7 +9,7 @@ import { Icon } from "components/ui/Icon"; import { ProgressBar } from "components/ui/ProgressBar"; import { Text } from "components/ui/Text"; -import { SynchronousJobRead } from "core/request/AirbyteClient"; +import { SynchronousJobRead } from "core/api/types/AirbyteClient"; import styles from "./TestCard.module.scss"; import TestingConnectionSuccess from "./TestingConnectionSuccess"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/useAnalyticsTrackFunctions.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/useAnalyticsTrackFunctions.tsx index 1572e6035cf..241e0bb7186 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/useAnalyticsTrackFunctions.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/useAnalyticsTrackFunctions.tsx @@ -1,7 +1,7 @@ import { useCallback } from "react"; +import { SynchronousJobRead } from "core/api/types/AirbyteClient"; import { Connector, ConnectorDefinition } from "core/domain/connector"; -import { SynchronousJobRead } from "core/request/AirbyteClient"; import { Action, Namespace, useAnalyticsService } from "core/services/analytics"; export const useAnalyticsTrackFunctions = (connectorType: "source" | "destination") => { diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/useTestConnector.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/useTestConnector.tsx index d7931604abf..3da620fc25d 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/useTestConnector.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/useTestConnector.tsx @@ -1,9 +1,9 @@ import { useEffect, useRef } from "react"; import { ConnectorCheckParams, useCurrentWorkspace, useCheckConnector } from "core/api"; +import { CheckConnectionRead } from "core/api/types/AirbyteClient"; import { ConnectorHelper } from "core/domain/connector"; import { ConnectorT } from "core/domain/connector/types"; -import { CheckConnectionRead } from "core/request/AirbyteClient"; import { ConnectorCardValues } from "../ConnectorForm"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx index 490b416479f..8dcc9143d9f 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/ConnectorForm.test.tsx @@ -8,9 +8,9 @@ import selectEvent from "react-select-event"; import { render, useMockIntersectionObserver } from "test-utils/testutils"; import { useCompleteOAuth } from "core/api"; +import { DestinationDefinitionSpecificationRead } from "core/api/types/AirbyteClient"; import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; import { AirbyteJSONSchema } from "core/jsonSchema/types"; -import { DestinationDefinitionSpecificationRead } from "core/request/AirbyteClient"; import { FeatureItem } from "core/services/features"; import { OAUTH_BROADCAST_CHANNEL_NAME } from "hooks/services/useConnectorAuth"; import { ConnectorForm } from "views/Connector/ConnectorForm"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthFlowAdapter.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthFlowAdapter.tsx index d2245291cda..8e875b6deab 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthFlowAdapter.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthFlowAdapter.tsx @@ -6,8 +6,8 @@ import { useState } from "react"; import { FieldPath, useFormContext } from "react-hook-form"; import { FormattedMessage } from "react-intl"; +import { AdvancedAuth, CompleteOAuthResponseAuthPayload } from "core/api/types/AirbyteClient"; import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; -import { AdvancedAuth, CompleteOAuthResponseAuthPayload } from "core/request/AirbyteClient"; import { useRunOauthFlow } from "hooks/services/useConnectorAuth"; import { useAuthentication } from "views/Connector/ConnectorForm/useAuthentication"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthRevocationAdapter.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthRevocationAdapter.tsx index e04f1950e01..b2660c3ad2e 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthRevocationAdapter.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/Sections/auth/useOauthRevocationAdapter.tsx @@ -2,8 +2,8 @@ import set from "lodash/set"; import { useFormContext } from "react-hook-form"; import { FormattedMessage } from "react-intl"; +import { AdvancedAuth } from "core/api/types/AirbyteClient"; import { ConnectorDefinition, ConnectorDefinitionSpecification } from "core/domain/connector"; -import { AdvancedAuth } from "core/request/AirbyteClient"; import { useNotificationService } from "../../../../../../hooks/services/Notification"; import { useRunOauthRevocation } from "../../../../../../hooks/services/useConnectorAuthRevocation"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/components/WarningMessage.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/components/WarningMessage.tsx index bdf5832c1b0..e71c56b6a03 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/components/WarningMessage.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/components/WarningMessage.tsx @@ -5,7 +5,7 @@ import { Box } from "components/ui/Box"; import { ExternalLink } from "components/ui/Link"; import { Message } from "components/ui/Message"; -import { SupportLevel } from "core/request/AirbyteClient"; +import { SupportLevel } from "core/api/types/AirbyteClient"; import { links } from "core/utils/links"; export const WarningMessage: React.FC<{ supportLevel?: SupportLevel }> = ({ supportLevel }) => { diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.test.tsx b/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.test.tsx index 0a7bdb83ab8..76ea6631a71 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.test.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/useAuthentication.test.tsx @@ -2,7 +2,7 @@ import { renderHook } from "@testing-library/react"; import get from "lodash/get"; import { FieldError, useFormContext } from "react-hook-form"; -import { SourceDefinitionSpecificationRead } from "core/request/AirbyteClient"; +import { SourceDefinitionSpecificationRead } from "core/api/types/AirbyteClient"; import { FeatureItem, FeatureService } from "core/services/features"; import { useConnectorForm } from "./connectorFormContext"; diff --git a/airbyte-webapp/src/views/Connector/ConnectorForm/utils.ts b/airbyte-webapp/src/views/Connector/ConnectorForm/utils.ts index e434f9a40b3..f3671c54b2a 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorForm/utils.ts +++ b/airbyte-webapp/src/views/Connector/ConnectorForm/utils.ts @@ -1,7 +1,7 @@ import toLower from "lodash/toLower"; +import { AdvancedAuth } from "core/api/types/AirbyteClient"; import { FormBlock } from "core/form/types"; -import { AdvancedAuth } from "core/request/AirbyteClient"; import { naturalComparator } from "core/utils/objects"; import { ConnectorDefinitionSpecification } from "../../../core/domain/connector"; diff --git a/airbyte-webapp/src/views/common/ResourceNotFoundErrorBoundary.tsx b/airbyte-webapp/src/views/common/ResourceNotFoundErrorBoundary.tsx index 16d282ebc83..a0e8409a5c1 100644 --- a/airbyte-webapp/src/views/common/ResourceNotFoundErrorBoundary.tsx +++ b/airbyte-webapp/src/views/common/ResourceNotFoundErrorBoundary.tsx @@ -1,7 +1,7 @@ import React from "react"; import { FormattedMessage } from "react-intl"; -import { CommonRequestError } from "core/request/CommonRequestError"; +import { CommonRequestError } from "core/api"; import { TrackErrorFn } from "hooks/services/AppMonitoringService"; interface BoundaryState { From efa34dccad3dd1694bd742ff0614c6f921634ec1 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Tue, 12 Dec 2023 08:00:13 -0800 Subject: [PATCH 04/27] Java 21 Upgrade Part 1: Move all applications to Java 21 to prep for upgrade. (#10313) Part 1 of upgrading to Java 21: Upgrade the image base to be Java 21. This is fine as Java is backward compatible and should be able to run Java 17. Follow ups: Part 2 will be change Gradle to generate compile and generate Java 21 code. Publish new gradle plugin versions. --- airbyte-api-server/Dockerfile | 2 +- airbyte-base-java-image/Dockerfile | 2 +- airbyte-base-java-python-image/Dockerfile | 2 +- airbyte-bootloader/Dockerfile | 2 +- airbyte-container-orchestrator/Dockerfile | 2 +- airbyte-cron/Dockerfile | 2 +- airbyte-keycloak-setup/Dockerfile | 2 +- airbyte-metrics/reporter/Dockerfile | 2 +- airbyte-server/Dockerfile | 2 +- airbyte-workers/Dockerfile | 2 +- airbyte-workload-api-server/Dockerfile | 2 +- airbyte-workload-launcher/Dockerfile | 2 +- build.gradle | 3 +-- 13 files changed, 13 insertions(+), 14 deletions(-) diff --git a/airbyte-api-server/Dockerfile b/airbyte-api-server/Dockerfile index 61c64f5e499..babb3e8ca0e 100644 --- a/airbyte-api-server/Dockerfile +++ b/airbyte-api-server/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} AS server EXPOSE 8006 5005 ENV APPLICATION airbyte-api-server diff --git a/airbyte-base-java-image/Dockerfile b/airbyte-base-java-image/Dockerfile index 13b1fae880e..c51bbd06982 100644 --- a/airbyte-base-java-image/Dockerfile +++ b/airbyte-base-java-image/Dockerfile @@ -1,4 +1,4 @@ -FROM amazoncorretto:19 +FROM amazoncorretto:21 ARG DOCKER_BUILD_ARCH=amd64 diff --git a/airbyte-base-java-python-image/Dockerfile b/airbyte-base-java-python-image/Dockerfile index ccd36c69a77..166388eca1c 100644 --- a/airbyte-base-java-python-image/Dockerfile +++ b/airbyte-base-java-python-image/Dockerfile @@ -1,4 +1,4 @@ -FROM airbyte/airbyte-base-java-image:2.0.4 +FROM airbyte/airbyte-base-java-image:2.1.0 RUN yum update -y && \ yum groupinstall -y "Development Tools" && \ diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index cbe6c045637..040fe3ead7b 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} WORKDIR /app ADD airbyte-app.tar /app diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index bb0a951372e..07ea6e1d0f5 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} AS orchestrator ARG DOCKER_BUILD_ARCH=amd64 diff --git a/airbyte-cron/Dockerfile b/airbyte-cron/Dockerfile index 7e1b8fb8d5f..3c0abb1bfbe 100644 --- a/airbyte-cron/Dockerfile +++ b/airbyte-cron/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} WORKDIR /app ADD airbyte-app.tar /app diff --git a/airbyte-keycloak-setup/Dockerfile b/airbyte-keycloak-setup/Dockerfile index eada8c9e06d..560ddde30d6 100644 --- a/airbyte-keycloak-setup/Dockerfile +++ b/airbyte-keycloak-setup/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} AS keycloak-setup WORKDIR /app ADD airbyte-app.tar /app diff --git a/airbyte-metrics/reporter/Dockerfile b/airbyte-metrics/reporter/Dockerfile index d6f9f503830..af4b2ccfa83 100644 --- a/airbyte-metrics/reporter/Dockerfile +++ b/airbyte-metrics/reporter/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} WORKDIR /app ADD airbyte-app.tar /app diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index f313be82094..9869c458c4d 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} AS server EXPOSE 8000 5005 diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 9ba5268df94..0384ad5a092 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} AS worker ARG DOCKER_BUILD_ARCH=amd64 diff --git a/airbyte-workload-api-server/Dockerfile b/airbyte-workload-api-server/Dockerfile index 14619bea74c..c1a6d1da0e0 100644 --- a/airbyte-workload-api-server/Dockerfile +++ b/airbyte-workload-api-server/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} AS server EXPOSE 8007 5005 ENV APPLICATION airbyte-workload-api-server diff --git a/airbyte-workload-launcher/Dockerfile b/airbyte-workload-launcher/Dockerfile index b1b979f18f8..a876632dd32 100644 --- a/airbyte-workload-launcher/Dockerfile +++ b/airbyte-workload-launcher/Dockerfile @@ -1,4 +1,4 @@ -ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.0.4 +ARG JDK_IMAGE=airbyte/airbyte-base-java-image:2.1.0 FROM ${JDK_IMAGE} AS airbyte-workload-launcher # Seems to also be here to enable kubectl cmds (namely, cp) diff --git a/build.gradle b/build.gradle index 9812f65a4ec..05d92807584 100644 --- a/build.gradle +++ b/build.gradle @@ -215,7 +215,7 @@ subprojects { sp -> def nginxImage = System.getenv('NGINX_IMAGE') ?: isArm64 ? 'arm64v8/nginx:alpine' : 'amd64/nginx:alpine' // Used by the platform -- Must be an Amazon Corretto-based image for build to work without modification to Dockerfile instructions - def openjdkImage = System.getenv('JDK_IMAGE') ?: 'airbyte/airbyte-base-java-image:2.0.4' + def openjdkImage = System.getenv('JDK_IMAGE') ?: 'airbyte/airbyte-base-java-image:2.1.0' platform = buildPlatform images.add("airbyte/$sp.dockerImageName:$rootProject.ext.image_tag") @@ -250,4 +250,3 @@ tasks.register('generate-docker') { dependsOn(':airbyte-keycloak:assemble') dependsOn(':airbyte-keycloak-setup:assemble') } - From 22f1f12372f43d6e41a831fa95f755f5e3f1c014 Mon Sep 17 00:00:00 2001 From: Catherine Noll Date: Tue, 12 Dec 2023 12:34:03 -0500 Subject: [PATCH 05/27] Add config & state to Sentry events and logged error messages (#10067) Co-authored-by: Ben Church --- .../commons/server/handlers/JobsHandler.java | 30 +++- .../server/handlers/JobsHandlerTest.java | 15 +- .../AttemptConfigReportingContext.java | 18 +++ .../job/errorreporter/JobErrorReporter.java | 24 ++-- .../JobErrorReportingClient.java | 3 +- .../LoggingJobErrorReportingClient.java | 12 +- .../SentryJobErrorReportingClient.java | 48 ++++++- .../errorreporter/JobErrorReporterTest.java | 40 ++++-- .../SentryJobErrorReportingClientTest.java | 133 +++++++++++++++++- .../org.mockito.plugins.MockMaker | 1 + 10 files changed, 284 insertions(+), 40 deletions(-) create mode 100644 airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/AttemptConfigReportingContext.java create mode 100644 airbyte-persistence/job-persistence/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/JobsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/JobsHandler.java index b00fd22d65a..739070ef636 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/JobsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/JobsHandler.java @@ -15,6 +15,7 @@ import io.airbyte.commons.server.errors.BadRequestException; import io.airbyte.commons.server.handlers.helpers.JobCreationAndStatusUpdateHelper; import io.airbyte.config.AttemptFailureSummary; +import io.airbyte.config.AttemptSyncConfig; import io.airbyte.config.JobOutput; import io.airbyte.config.JobResetConnectionConfig; import io.airbyte.config.JobSyncConfig; @@ -23,6 +24,7 @@ import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.errorreporter.AttemptConfigReportingContext; import io.airbyte.persistence.job.errorreporter.JobErrorReporter; import io.airbyte.persistence.job.errorreporter.SyncJobReportingContext; import io.airbyte.persistence.job.models.Attempt; @@ -30,6 +32,7 @@ import jakarta.inject.Singleton; import java.io.IOException; import java.util.List; +import java.util.Optional; import java.util.UUID; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -90,8 +93,7 @@ public InternalOperationResult jobFailure(final JobFailureRequest input) { destinationDefinitionVersionId = jobSyncConfig.getDestinationDefinitionVersionId(); } final SyncJobReportingContext jobContext = new SyncJobReportingContext(jobId, sourceDefinitionVersionId, destinationDefinitionVersionId); - job.getLastFailedAttempt().flatMap(Attempt::getFailureSummary) - .ifPresent(failureSummary -> jobErrorReporter.reportSyncJobFailure(connectionId, failureSummary, jobContext)); + reportIfLastFailedAttempt(job, connectionId, jobContext); jobCreationAndStatusUpdateHelper.trackCompletion(job, JobStatus.FAILED); return new InternalOperationResult().succeeded(true); } catch (final IOException e) { @@ -101,6 +103,30 @@ public InternalOperationResult jobFailure(final JobFailureRequest input) { } } + private void reportIfLastFailedAttempt(Job job, UUID connectionId, SyncJobReportingContext jobContext) { + Optional lastFailedAttempt = job.getLastFailedAttempt(); + if (lastFailedAttempt.isPresent()) { + Attempt attempt = lastFailedAttempt.get(); + Optional failureSummaryOpt = attempt.getFailureSummary(); + + if (failureSummaryOpt.isPresent()) { + AttemptFailureSummary failureSummary = failureSummaryOpt.get(); + AttemptConfigReportingContext attemptConfig = null; + + Optional syncConfigOpt = attempt.getSyncConfig(); + if (syncConfigOpt.isPresent()) { + AttemptSyncConfig syncConfig = syncConfigOpt.get(); + attemptConfig = new AttemptConfigReportingContext( + syncConfig.getSourceConfiguration(), + syncConfig.getDestinationConfiguration(), + syncConfig.getState()); + } + + jobErrorReporter.reportSyncJobFailure(connectionId, failureSummary, jobContext, attemptConfig); + } + } + } + /** * Report a job and a given attempt as successful. */ diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/JobsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/JobsHandlerTest.java index bb7f45d484a..751f714e9cc 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/JobsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/JobsHandlerTest.java @@ -25,6 +25,7 @@ import io.airbyte.commons.server.errors.BadRequestException; import io.airbyte.commons.server.handlers.helpers.JobCreationAndStatusUpdateHelper; import io.airbyte.config.AttemptFailureSummary; +import io.airbyte.config.AttemptSyncConfig; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.JobConfig; @@ -36,6 +37,7 @@ import io.airbyte.config.StandardSyncSummary.ReplicationStatus; import io.airbyte.persistence.job.JobNotifier; import io.airbyte.persistence.job.JobPersistence; +import io.airbyte.persistence.job.errorreporter.AttemptConfigReportingContext; import io.airbyte.persistence.job.errorreporter.JobErrorReporter; import io.airbyte.persistence.job.errorreporter.SyncJobReportingContext; import io.airbyte.persistence.job.models.Attempt; @@ -244,6 +246,9 @@ void setJobFailure() throws IOException { .withSourceDefinitionVersionId(UUID.randomUUID()) .withDestinationDefinitionVersionId(UUID.randomUUID()); + final AttemptSyncConfig mSyncConfig = Mockito.mock(AttemptSyncConfig.class); + when(mAttempt.getSyncConfig()).thenReturn(Optional.of(mSyncConfig)); + final JobConfig mJobConfig = Mockito.mock(JobConfig.class); when(mJobConfig.getConfigType()).thenReturn(SYNC); when(mJobConfig.getSync()).thenReturn(jobSyncConfig); @@ -263,9 +268,15 @@ void setJobFailure() throws IOException { jobSyncConfig.getSourceDefinitionVersionId(), jobSyncConfig.getDestinationDefinitionVersionId()); + final AttemptConfigReportingContext expectedAttemptConfig = + new AttemptConfigReportingContext( + mSyncConfig.getSourceConfiguration(), + mSyncConfig.getDestinationConfiguration(), + mSyncConfig.getState()); + verify(jobPersistence).failJob(JOB_ID); verify(jobNotifier).failJob(eq(failureReason), Mockito.any()); - verify(jobErrorReporter).reportSyncJobFailure(CONNECTION_ID, failureSummary, expectedReportingContext); + verify(jobErrorReporter).reportSyncJobFailure(CONNECTION_ID, failureSummary, expectedReportingContext, expectedAttemptConfig); } @Test @@ -294,7 +305,7 @@ void setJobFailureWithNullJobSyncConfig() throws IOException { verify(jobPersistence).failJob(JOB_ID); verify(jobNotifier).failJob(eq(failureReason), Mockito.any()); - verify(jobErrorReporter).reportSyncJobFailure(eq(CONNECTION_ID), eq(failureSummary), Mockito.any()); + verify(jobErrorReporter).reportSyncJobFailure(eq(CONNECTION_ID), eq(failureSummary), Mockito.any(), Mockito.any()); } } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/AttemptConfigReportingContext.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/AttemptConfigReportingContext.java new file mode 100644 index 00000000000..86d9c465f6e --- /dev/null +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/AttemptConfigReportingContext.java @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.persistence.job.errorreporter; + +import com.fasterxml.jackson.databind.JsonNode; +import edu.umd.cs.findbugs.annotations.Nullable; +import io.airbyte.config.State; + +/** + * Connector Attempt Config Reporting context. + * + * @param sourceConfig source configuration used for the attempt + * @param destinationConfig destination configuration used for the attempt + * @param state state during the attempt + */ +public record AttemptConfigReportingContext(@Nullable JsonNode sourceConfig, @Nullable JsonNode destinationConfig, @Nullable State state) {} diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java index 96dbc857bfd..c2be2d87a31 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReporter.java @@ -84,7 +84,10 @@ public JobErrorReporter(final ConfigRepository configRepository, * @param failureSummary - final attempt failure summary * @param jobContext - sync job reporting context */ - public void reportSyncJobFailure(final UUID connectionId, final AttemptFailureSummary failureSummary, final SyncJobReportingContext jobContext) { + public void reportSyncJobFailure(final UUID connectionId, + final AttemptFailureSummary failureSummary, + final SyncJobReportingContext jobContext, + @Nullable final AttemptConfigReportingContext attemptConfig) { Exceptions.swallow(() -> { final List traceMessageFailures = failureSummary.getFailures().stream() .filter(failure -> failure.getMetadata() != null && failure.getMetadata().getAdditionalProperties().containsKey(FROM_TRACE_MESSAGE)) @@ -105,7 +108,7 @@ public void reportSyncJobFailure(final UUID connectionId, final AttemptFailureSu final Map metadata = MoreMaps.merge(commonMetadata, getSourceMetadata(sourceDefinition, dockerImage, sourceVersion.getReleaseStage())); - reportJobFailureReason(workspace, failureReason, dockerImage, metadata); + reportJobFailureReason(workspace, failureReason, dockerImage, metadata, attemptConfig); } else if (failureOrigin == FailureOrigin.DESTINATION) { final StandardDestinationDefinition destinationDefinition = configRepository.getDestinationDefinitionFromConnection(connectionId); final ActorDefinitionVersion destinationVersion = configRepository.getActorDefinitionVersion(jobContext.destinationVersionId()); @@ -113,7 +116,7 @@ public void reportSyncJobFailure(final UUID connectionId, final AttemptFailureSu final Map metadata = MoreMaps.merge(commonMetadata, getDestinationMetadata(destinationDefinition, dockerImage, destinationVersion.getReleaseStage())); - reportJobFailureReason(workspace, failureReason, dockerImage, metadata); + reportJobFailureReason(workspace, failureReason, dockerImage, metadata, attemptConfig); } else if (failureOrigin == FailureOrigin.NORMALIZATION) { final StandardSourceDefinition sourceDefinition = configRepository.getSourceDefinitionFromConnection(connectionId); final StandardDestinationDefinition destinationDefinition = configRepository.getDestinationDefinitionFromConnection(connectionId); @@ -148,7 +151,7 @@ public void reportSyncJobFailure(final UUID connectionId, final AttemptFailureSu destinationVersion.getNormalizationConfig().getNormalizationRepository() + ":" + destinationVersion.getNormalizationConfig().getNormalizationTag(); - reportJobFailureReason(workspace, failureReason, normalizationDockerImage, metadata); + reportJobFailureReason(workspace, failureReason, normalizationDockerImage, metadata, attemptConfig); } } }); @@ -171,7 +174,7 @@ public void reportSourceCheckJobFailure(final UUID sourceDefinitionId, final Map metadata = MoreMaps.merge( getSourceMetadata(sourceDefinition, jobContext.dockerImage(), jobContext.releaseStage()), Map.of(JOB_ID_KEY, jobContext.jobId().toString())); - reportJobFailureReason(workspace, failureReason.withFailureOrigin(FailureOrigin.SOURCE), jobContext.dockerImage(), metadata); + reportJobFailureReason(workspace, failureReason.withFailureOrigin(FailureOrigin.SOURCE), jobContext.dockerImage(), metadata, null); } /** @@ -192,7 +195,7 @@ public void reportDestinationCheckJobFailure(final UUID destinationDefinitionId, final Map metadata = MoreMaps.merge( getDestinationMetadata(destinationDefinition, jobContext.dockerImage(), jobContext.releaseStage()), Map.of(JOB_ID_KEY, jobContext.jobId().toString())); - reportJobFailureReason(workspace, failureReason.withFailureOrigin(FailureOrigin.DESTINATION), jobContext.dockerImage(), metadata); + reportJobFailureReason(workspace, failureReason.withFailureOrigin(FailureOrigin.DESTINATION), jobContext.dockerImage(), metadata, null); } /** @@ -212,7 +215,7 @@ public void reportDiscoverJobFailure(final UUID sourceDefinitionId, final Map metadata = MoreMaps.merge( getSourceMetadata(sourceDefinition, jobContext.dockerImage(), jobContext.releaseStage()), Map.of(JOB_ID_KEY, jobContext.jobId().toString())); - reportJobFailureReason(workspace, failureReason, jobContext.dockerImage(), metadata); + reportJobFailureReason(workspace, failureReason, jobContext.dockerImage(), metadata, null); } /** @@ -227,7 +230,7 @@ public void reportSpecJobFailure(final FailureReason failureReason, final Connec final Map metadata = Map.of( JOB_ID_KEY, jobContext.jobId().toString(), CONNECTOR_REPOSITORY_META_KEY, connectorRepository); - reportJobFailureReason(null, failureReason, dockerImage, metadata); + reportJobFailureReason(null, failureReason, dockerImage, metadata, null); } private Map getConnectionMetadata(final UUID workspaceId, final UUID connectionId) { @@ -309,7 +312,8 @@ private Map getWorkspaceMetadata(final UUID workspaceId) { private void reportJobFailureReason(@Nullable final StandardWorkspace workspace, final FailureReason failureReason, final String dockerImage, - final Map metadata) { + final Map metadata, + @Nullable final AttemptConfigReportingContext attemptConfig) { // Failure types associated with a config-error or a manual-cancellation should NOT be reported. if (UNSUPPORTED_FAILURETYPES.contains(failureReason.getFailureType())) { return; @@ -329,7 +333,7 @@ private void reportJobFailureReason(@Nullable final StandardWorkspace workspace, metadata); try { - jobErrorReportingClient.reportJobFailureReason(workspace, failureReason, dockerImage, allMetadata); + jobErrorReportingClient.reportJobFailureReason(workspace, failureReason, dockerImage, allMetadata, attemptConfig); } catch (final Exception e) { LOGGER.error("Error when reporting job failure reason: {}", failureReason, e); } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReportingClient.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReportingClient.java index 4ea715a05a1..1d13b2470de 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReportingClient.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/JobErrorReportingClient.java @@ -20,6 +20,7 @@ public interface JobErrorReportingClient { void reportJobFailureReason(@Nullable StandardWorkspace workspace, final FailureReason reason, @Nullable final String dockerImage, - Map metadata); + Map metadata, + @Nullable final AttemptConfigReportingContext attemptConfig); } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/LoggingJobErrorReportingClient.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/LoggingJobErrorReportingClient.java index 090c1605980..2746b7c25b2 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/LoggingJobErrorReportingClient.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/LoggingJobErrorReportingClient.java @@ -14,6 +14,7 @@ /** * Log job error reports. */ +@SuppressWarnings("PMD.AvoidDuplicateLiterals") public class LoggingJobErrorReportingClient implements JobErrorReportingClient { private static final Logger LOGGER = LoggerFactory.getLogger(LoggingJobErrorReportingClient.class); @@ -22,12 +23,17 @@ public class LoggingJobErrorReportingClient implements JobErrorReportingClient { public void reportJobFailureReason(@Nullable final StandardWorkspace workspace, final FailureReason reason, final String dockerImage, - final Map metadata) { - LOGGER.info("Report Job Error -> workspaceId: {}, dockerImage: {}, failureReason: {}, metadata: {}", + final Map metadata, + @Nullable final AttemptConfigReportingContext attemptConfig) { + LOGGER.info( + "Report Job Error -> workspaceId: {}, dockerImage: {}, failureReason: {}, metadata: {}, state: {}, sourceConfig: {}, destinationConfig: {}", workspace != null ? workspace.getWorkspaceId() : "null", dockerImage, reason, - metadata); + metadata, + attemptConfig != null ? attemptConfig.state() : "null", + attemptConfig != null ? attemptConfig.sourceConfig() : "null", + attemptConfig != null ? attemptConfig.destinationConfig() : "null"); } } diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/SentryJobErrorReportingClient.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/SentryJobErrorReportingClient.java index da10a1c2f7b..afdf6ef51f7 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/SentryJobErrorReportingClient.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/errorreporter/SentryJobErrorReportingClient.java @@ -4,6 +4,7 @@ package io.airbyte.persistence.job.errorreporter; +import com.fasterxml.jackson.databind.JsonNode; import edu.umd.cs.findbugs.annotations.Nullable; import io.airbyte.config.FailureReason; import io.airbyte.config.Metadata; @@ -18,6 +19,7 @@ import io.sentry.protocol.SentryException; import io.sentry.protocol.User; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,7 +67,8 @@ static IHub createSentryHubWithDSN(final String sentryDSN) { public void reportJobFailureReason(@Nullable final StandardWorkspace workspace, final FailureReason failureReason, @Nullable final String dockerImage, - final Map metadata) { + final Map metadata, + @Nullable final AttemptConfigReportingContext attemptConfig) { final SentryEvent event = new SentryEvent(); if (dockerImage != null) { @@ -138,10 +141,53 @@ public void reportJobFailureReason(@Nullable final StandardWorkspace workspace, failureReasonContext.put("metadata", failureReasonMeta.toString()); } + if (attemptConfig != null) { + final Map stateContext = new HashMap<>(); + stateContext.put("state", attemptConfig.state() != null ? attemptConfig.state().toString() : "null"); + scope.setContexts("State", stateContext); + scope.setContexts("Source Configuration", getContextFromNode(attemptConfig.sourceConfig())); + scope.setContexts("Destination Configuration", getContextFromNode(attemptConfig.destinationConfig())); + } + scope.setContexts("Failure Reason", failureReasonContext); }); sentryHub.captureEvent(event); } + private static Map getContextFromNode(@Nullable JsonNode node) { + Map flatMap = new HashMap<>(); + if (node != null) { + flattenJsonNode("", node, flatMap); + } + return flatMap; + } + + /** + * This flattens a JsonNode into its related dot paths. + * + * e.g. {"a": { "b": [{"c": 1}]}} -> {"a.b[0].c": 1} + */ + public static void flattenJsonNode(String currentPath, JsonNode node, Map flatMap) { + if (node.isArray()) { + for (int i = 0; i < node.size(); i++) { + JsonNode item = node.get(i); + String newPath = String.format("%s[%d]", currentPath, i); + flattenJsonNode(newPath, item, flatMap); + } + } else if (node.isObject()) { + Iterator> fields = node.fields(); + while (fields.hasNext()) { + Map.Entry field = fields.next(); + String fieldName = field.getKey(); + JsonNode fieldValue = field.getValue(); + + String newPath = currentPath.isEmpty() ? fieldName : currentPath + "." + fieldName; + flattenJsonNode(newPath, fieldValue, flatMap); + } + } else { + flatMap.put(currentPath, node.asText()); + } + } + } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java index b621742c6b2..b87131d344d 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/JobErrorReporterTest.java @@ -6,6 +6,7 @@ import static org.mockito.Mockito.mock; +import com.fasterxml.jackson.databind.ObjectMapper; import io.airbyte.config.ActorDefinitionVersion; import io.airbyte.config.AttemptFailureSummary; import io.airbyte.config.Configs.DeploymentMode; @@ -18,6 +19,7 @@ import io.airbyte.config.StandardDestinationDefinition; import io.airbyte.config.StandardSourceDefinition; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.State; import io.airbyte.config.persistence.ConfigNotFoundException; import io.airbyte.config.persistence.ConfigRepository; import io.airbyte.persistence.job.WebUrlHelper; @@ -134,6 +136,10 @@ void testReportSyncJobFailure() throws ConfigNotFoundException, IOException { SOURCE_DEFINITION_VERSION_ID, DESTINATION_DEFINITION_VERSION_ID); + final ObjectMapper objectMapper = new ObjectMapper(); + final AttemptConfigReportingContext attemptConfig = + new AttemptConfigReportingContext(objectMapper.createObjectNode(), objectMapper.createObjectNode(), new State()); + Mockito.when(configRepository.getSourceDefinitionFromConnection(CONNECTION_ID)) .thenReturn(new StandardSourceDefinition() .withSourceDefinitionId(SOURCE_DEFINITION_ID) @@ -164,7 +170,7 @@ void testReportSyncJobFailure() throws ConfigNotFoundException, IOException { Mockito.when(mWorkspace.getWorkspaceId()).thenReturn(WORKSPACE_ID); Mockito.when(configRepository.getStandardWorkspaceFromConnection(CONNECTION_ID, true)).thenReturn(mWorkspace); - jobErrorReporter.reportSyncJobFailure(CONNECTION_ID, mFailureSummary, jobReportingContext); + jobErrorReporter.reportSyncJobFailure(CONNECTION_ID, mFailureSummary, jobReportingContext, attemptConfig); final Map expectedSourceMetadata = Map.ofEntries( Map.entry(JOB_ID_KEY, String.valueOf(syncJobId)), @@ -218,11 +224,13 @@ void testReportSyncJobFailure() throws ConfigNotFoundException, IOException { Map.entry(CONNECTOR_NAME_KEY, DESTINATION_DEFINITION_NAME), Map.entry(CONNECTOR_RELEASE_STAGE_KEY, DESTINATION_RELEASE_STAGE.toString())); - Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, sourceFailureReason, SOURCE_DOCKER_IMAGE, expectedSourceMetadata); + Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, sourceFailureReason, SOURCE_DOCKER_IMAGE, expectedSourceMetadata, + attemptConfig); Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, destinationFailureReason, DESTINATION_DOCKER_IMAGE, - expectedDestinationMetadata); + expectedDestinationMetadata, attemptConfig); Mockito.verify(jobErrorReportingClient).reportJobFailureReason( - mWorkspace, normalizationFailureReason, String.format("%s:%s", NORMALIZATION_IMAGE, NORMALIZATION_VERSION), expectedNormalizationMetadata); + mWorkspace, normalizationFailureReason, String.format("%s:%s", NORMALIZATION_IMAGE, NORMALIZATION_VERSION), expectedNormalizationMetadata, + attemptConfig); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); } @@ -236,6 +244,10 @@ void testReportSyncJobFailureDoesNotThrow() throws ConfigNotFoundException, IOEx .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR); + final ObjectMapper objectMapper = new ObjectMapper(); + final AttemptConfigReportingContext attemptConfig = + new AttemptConfigReportingContext(objectMapper.createObjectNode(), objectMapper.createObjectNode(), new State()); + Mockito.when(mFailureSummary.getFailures()).thenReturn(List.of(sourceFailureReason)); Mockito.when(configRepository.getSourceDefinitionFromConnection(CONNECTION_ID)) @@ -256,11 +268,11 @@ void testReportSyncJobFailureDoesNotThrow() throws ConfigNotFoundException, IOEx Mockito.doThrow(new RuntimeException("some exception")) .when(jobErrorReportingClient) - .reportJobFailureReason(Mockito.any(), Mockito.eq(sourceFailureReason), Mockito.any(), Mockito.any()); + .reportJobFailureReason(Mockito.any(), Mockito.eq(sourceFailureReason), Mockito.any(), Mockito.any(), Mockito.any()); - Assertions.assertDoesNotThrow(() -> jobErrorReporter.reportSyncJobFailure(CONNECTION_ID, mFailureSummary, jobContext)); + Assertions.assertDoesNotThrow(() -> jobErrorReporter.reportSyncJobFailure(CONNECTION_ID, mFailureSummary, jobContext, attemptConfig)); Mockito.verify(jobErrorReportingClient, Mockito.times(1)) - .reportJobFailureReason(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + .reportJobFailureReason(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); } @Test @@ -299,7 +311,7 @@ void testReportSourceCheckJobFailure() throws JsonValidationException, ConfigNot Map.entry(CONNECTOR_RELEASE_STAGE_KEY, SOURCE_RELEASE_STAGE.toString()), Map.entry(CONNECTOR_COMMAND_KEY, CHECK_COMMAND)); - Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata); + Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata, null); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); } @@ -333,7 +345,7 @@ void testReportSourceCheckJobFailureNullWorkspaceId() throws JsonValidationExcep Map.entry(CONNECTOR_RELEASE_STAGE_KEY, SOURCE_RELEASE_STAGE.toString()), Map.entry(CONNECTOR_COMMAND_KEY, CHECK_COMMAND)); - Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata); + Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata, null); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); } @@ -373,7 +385,7 @@ void testReportDestinationCheckJobFailure() throws JsonValidationException, Conf Map.entry(CONNECTOR_RELEASE_STAGE_KEY, DESTINATION_RELEASE_STAGE.toString()), Map.entry(CONNECTOR_COMMAND_KEY, CHECK_COMMAND)); - Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, DESTINATION_DOCKER_IMAGE, expectedMetadata); + Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, DESTINATION_DOCKER_IMAGE, expectedMetadata, null); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); } @@ -407,7 +419,7 @@ void testReportDestinationCheckJobFailureNullWorkspaceId() throws JsonValidation Map.entry(CONNECTOR_RELEASE_STAGE_KEY, DESTINATION_RELEASE_STAGE.toString()), Map.entry(CONNECTOR_COMMAND_KEY, CHECK_COMMAND)); - Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, DESTINATION_DOCKER_IMAGE, expectedMetadata); + Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, DESTINATION_DOCKER_IMAGE, expectedMetadata, null); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); } @@ -447,7 +459,7 @@ void testReportDiscoverJobFailure() throws JsonValidationException, ConfigNotFou Map.entry(CONNECTOR_RELEASE_STAGE_KEY, SOURCE_RELEASE_STAGE.toString()), Map.entry(CONNECTOR_COMMAND_KEY, DISCOVER_COMMAND)); - Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata); + Mockito.verify(jobErrorReportingClient).reportJobFailureReason(mWorkspace, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata, null); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); } @@ -481,7 +493,7 @@ void testReportDiscoverJobFailureNullWorkspaceId() throws JsonValidationExceptio Map.entry(CONNECTOR_RELEASE_STAGE_KEY, SOURCE_RELEASE_STAGE.toString()), Map.entry(CONNECTOR_COMMAND_KEY, DISCOVER_COMMAND)); - Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata); + Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata, null); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); } @@ -507,7 +519,7 @@ void testReportSpecJobFailure() { Map.entry(CONNECTOR_REPOSITORY_KEY, SOURCE_DOCKER_REPOSITORY), Map.entry(CONNECTOR_COMMAND_KEY, SPEC_COMMAND)); - Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata); + Mockito.verify(jobErrorReportingClient).reportJobFailureReason(null, failureReason, SOURCE_DOCKER_IMAGE, expectedMetadata, null); Mockito.verifyNoMoreInteractions(jobErrorReportingClient); } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/SentryJobErrorReportingClientTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/SentryJobErrorReportingClientTest.java index 0aeb90f38bc..5f7bb2e5d98 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/SentryJobErrorReportingClientTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/errorreporter/SentryJobErrorReportingClientTest.java @@ -10,23 +10,31 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import io.airbyte.config.FailureReason; import io.airbyte.config.FailureReason.FailureOrigin; import io.airbyte.config.FailureReason.FailureType; import io.airbyte.config.StandardWorkspace; +import io.airbyte.config.State; import io.airbyte.persistence.job.errorreporter.SentryExceptionHelper.SentryExceptionPlatform; import io.airbyte.persistence.job.errorreporter.SentryExceptionHelper.SentryParsedException; import io.sentry.IHub; import io.sentry.NoOpHub; +import io.sentry.Scope; +import io.sentry.ScopeCallback; import io.sentry.SentryEvent; import io.sentry.protocol.Message; import io.sentry.protocol.SentryException; import io.sentry.protocol.User; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -34,6 +42,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; class SentryJobErrorReportingClientTest { @@ -42,14 +51,24 @@ class SentryJobErrorReportingClientTest { private static final String DOCKER_IMAGE = "airbyte/source-stripe:1.2.3"; private static final String ERROR_MESSAGE = "RuntimeError: Something went wrong"; + private final ObjectMapper objectMapper = new ObjectMapper(); private final StandardWorkspace workspace = new StandardWorkspace().withWorkspaceId(WORKSPACE_ID).withName(WORKSPACE_NAME); private SentryJobErrorReportingClient sentryErrorReportingClient; private IHub mockSentryHub; + private Scope mockScope; + private SentryExceptionHelper mockSentryExceptionHelper; @BeforeEach void setup() { mockSentryHub = mock(IHub.class); + mockScope = mock(Scope.class); + doAnswer(invocation -> { + final ScopeCallback scopeCallback = invocation.getArgument(0); + scopeCallback.run(mockScope); + return null; // Return null for void methods + }).when(mockSentryHub).configureScope(any(ScopeCallback.class)); + mockSentryExceptionHelper = mock(SentryExceptionHelper.class); sentryErrorReportingClient = new SentryJobErrorReportingClient(mockSentryHub, mockSentryExceptionHelper); } @@ -78,16 +97,24 @@ void testCreateSentryHubWithDSN() { } @Test - void testReportJobFailureReason() { + void testReportJobFailureReason() throws Exception { final ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(SentryEvent.class); final FailureReason failureReason = new FailureReason() .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR) - .withInternalMessage(ERROR_MESSAGE); + .withInternalMessage(ERROR_MESSAGE) + .withTimestamp(System.currentTimeMillis()); final Map metadata = Map.of("some_metadata", "some_metadata_value"); + final ObjectMapper objectMapper = new ObjectMapper(); + final JsonNode sourceConfig = objectMapper.readTree("{\"sourceKey\": \"sourceValue\"}"); + final JsonNode destinationConfig = objectMapper.readTree("{\"destinationKey\": \"destinationValue\"}"); + final State state = new State(); + state.withAdditionalProperty("stateKey", "stateValue"); - sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, metadata); + final AttemptConfigReportingContext attemptConfig = new AttemptConfigReportingContext(sourceConfig, destinationConfig, state); + + sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, metadata, attemptConfig); verify(mockSentryHub).captureEvent(eventCaptor.capture()); final SentryEvent actualEvent = eventCaptor.getValue(); @@ -98,6 +125,12 @@ void testReportJobFailureReason() { assertNull(actualEvent.getTag(STACKTRACE_PARSE_ERROR_TAG_KEY)); assertNull(actualEvent.getExceptions()); + // verify that scope setContexts is called with the correct arguments + verify(mockScope).setContexts(Mockito.eq("Failure Reason"), any(Object.class)); + verify(mockScope).setContexts(Mockito.eq("Source Configuration"), any(Object.class)); + verify(mockScope).setContexts(Mockito.eq("Destination Configuration"), any(Object.class)); + verify(mockScope).setContexts(Mockito.eq("State"), any(Object.class)); + final User sentryUser = actualEvent.getUser(); assertNotNull(sentryUser); assertEquals(WORKSPACE_ID.toString(), sentryUser.getId()); @@ -108,6 +141,22 @@ void testReportJobFailureReason() { assertEquals(ERROR_MESSAGE, message.getFormatted()); } + @Test + void testReportJobNoErrorOnNullAttemptConfig() { + final ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(SentryEvent.class); + + final FailureReason failureReason = new FailureReason() + .withFailureOrigin(FailureOrigin.SOURCE) + .withFailureType(FailureType.SYSTEM_ERROR) + .withInternalMessage(ERROR_MESSAGE) + .withTimestamp(System.currentTimeMillis()); + final Map metadata = Map.of("some_metadata", "some_metadata_value"); + + sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, metadata, null); + + verify(mockSentryHub).captureEvent(eventCaptor.capture()); + } + @Test void testReportJobFailureReasonWithNoWorkspace() { final ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(SentryEvent.class); @@ -115,9 +164,13 @@ void testReportJobFailureReasonWithNoWorkspace() { final FailureReason failureReason = new FailureReason() .withFailureOrigin(FailureOrigin.SOURCE) .withFailureType(FailureType.SYSTEM_ERROR) - .withInternalMessage(ERROR_MESSAGE); + .withInternalMessage(ERROR_MESSAGE) + .withTimestamp(System.currentTimeMillis()); + final ObjectMapper objectMapper = new ObjectMapper(); + final AttemptConfigReportingContext attemptConfig = + new AttemptConfigReportingContext(objectMapper.createObjectNode(), objectMapper.createObjectNode(), new State()); - sentryErrorReportingClient.reportJobFailureReason(null, failureReason, DOCKER_IMAGE, Map.of()); + sentryErrorReportingClient.reportJobFailureReason(null, failureReason, DOCKER_IMAGE, Map.of(), attemptConfig); verify(mockSentryHub).captureEvent(eventCaptor.capture()); final SentryEvent actualEvent = eventCaptor.getValue(); @@ -144,9 +197,13 @@ void testReportJobFailureReasonWithStacktrace() { final FailureReason failureReason = new FailureReason() .withInternalMessage(ERROR_MESSAGE) + .withTimestamp(System.currentTimeMillis()) .withStacktrace("Some valid stacktrace"); + final ObjectMapper objectMapper = new ObjectMapper(); + final AttemptConfigReportingContext attemptConfig = + new AttemptConfigReportingContext(objectMapper.createObjectNode(), objectMapper.createObjectNode(), new State()); - sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, Map.of()); + sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, Map.of(), attemptConfig); verify(mockSentryHub).captureEvent(eventCaptor.capture()); final SentryEvent actualEvent = eventCaptor.getValue(); @@ -165,9 +222,13 @@ void testReportJobFailureReasonWithInvalidStacktrace() { final FailureReason failureReason = new FailureReason() .withInternalMessage("Something went wrong") + .withTimestamp(System.currentTimeMillis()) .withStacktrace(invalidStacktrace); + final ObjectMapper objectMapper = new ObjectMapper(); + final AttemptConfigReportingContext attemptConfig = + new AttemptConfigReportingContext(objectMapper.createObjectNode(), objectMapper.createObjectNode(), new State()); - sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, Map.of()); + sentryErrorReportingClient.reportJobFailureReason(workspace, failureReason, DOCKER_IMAGE, Map.of(), attemptConfig); verify(mockSentryHub).captureEvent(eventCaptor.capture()); final SentryEvent actualEvent = eventCaptor.getValue(); @@ -178,4 +239,62 @@ void testReportJobFailureReasonWithInvalidStacktrace() { assertEquals("Invalid stacktrace, RuntimeError: ", exceptions.get(0).getValue()); } + @Test + void testEmptyJsonNode() { + JsonNode node = objectMapper.createObjectNode(); + Map flatMap = new HashMap<>(); + SentryJobErrorReportingClient.flattenJsonNode("", node, flatMap); + + assertEquals(0, flatMap.size()); + } + + @Test + void testSimpleFlatJson() throws Exception { + JsonNode node = objectMapper.readTree("{\"key1\":\"value1\", \"key2\":\"value2\"}"); + Map flatMap = new HashMap<>(); + SentryJobErrorReportingClient.flattenJsonNode("", node, flatMap); + + assertEquals(2, flatMap.size()); + assertEquals("value1", flatMap.get("key1")); + assertEquals("value2", flatMap.get("key2")); + } + + @Test + void testJsonWithArray() throws Exception { + JsonNode node = objectMapper.readTree("{\"a\": { \"b\": [{\"c\": 1}, {\"c\": 2}]}}"); + Map flatMap = new HashMap<>(); + SentryJobErrorReportingClient.flattenJsonNode("", node, flatMap); + + assertEquals(2, flatMap.size()); + assertEquals("1", flatMap.get("a.b[0].c")); + assertEquals("2", flatMap.get("a.b[1].c")); + } + + @Test + void testJsonWithNestedObject() throws Exception { + JsonNode node = objectMapper.readTree( + "{\"key1\":\"value1\", \"nestedObject\":{\"nestedKey1\":\"nestedValue1\", \"nestedKey2\":\"nestedValue2\"}}"); + Map flatMap = new HashMap<>(); + SentryJobErrorReportingClient.flattenJsonNode("", node, flatMap); + + assertEquals(3, flatMap.size()); + assertEquals("value1", flatMap.get("key1")); + assertEquals("nestedValue1", flatMap.get("nestedObject.nestedKey1")); + assertEquals("nestedValue2", flatMap.get("nestedObject.nestedKey2")); + } + + @Test + void testJsonWithNestedObjectsAndArray() throws Exception { + JsonNode node = objectMapper.readTree( + "{\"key1\":\"value1\", \"nested\":{\"nestedKey1\":\"nestedValue1\", \"array\":[{\"item\":\"value2\"}, {\"item\":\"value3\"}]}}"); + Map flatMap = new HashMap<>(); + SentryJobErrorReportingClient.flattenJsonNode("", node, flatMap); + + assertEquals(4, flatMap.size()); + assertEquals("value1", flatMap.get("key1")); + assertEquals("nestedValue1", flatMap.get("nested.nestedKey1")); + assertEquals("value2", flatMap.get("nested.array[0].item")); + assertEquals("value3", flatMap.get("nested.array[1].item")); + } + } diff --git a/airbyte-persistence/job-persistence/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/airbyte-persistence/job-persistence/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 00000000000..1f0955d450f --- /dev/null +++ b/airbyte-persistence/job-persistence/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline From e9236390f137205274820b38641638179cca9cf9 Mon Sep 17 00:00:00 2001 From: Jon Tan Date: Tue, 12 Dec 2023 11:39:08 -0600 Subject: [PATCH 06/27] Fix Auth Role stuff for AWS secrets manager (#10304) --- .../AwsSecretManagerPersistence.kt | 20 ++++++++++++++++++- .../persistence/RuntimeSecretPersistence.kt | 8 ++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/airbyte-config/config-secrets/src/main/kotlin/secrets/persistence/AwsSecretManagerPersistence.kt b/airbyte-config/config-secrets/src/main/kotlin/secrets/persistence/AwsSecretManagerPersistence.kt index c6170391e52..5dd6ab01721 100644 --- a/airbyte-config/config-secrets/src/main/kotlin/secrets/persistence/AwsSecretManagerPersistence.kt +++ b/airbyte-config/config-secrets/src/main/kotlin/secrets/persistence/AwsSecretManagerPersistence.kt @@ -11,6 +11,7 @@ import com.amazonaws.regions.Regions import com.amazonaws.secretsmanager.caching.SecretCache import com.amazonaws.services.secretsmanager.AWSSecretsManager import com.amazonaws.services.secretsmanager.AWSSecretsManagerClientBuilder +import com.amazonaws.services.secretsmanager.model.AWSSecretsManagerException import com.amazonaws.services.secretsmanager.model.CreateSecretRequest import com.amazonaws.services.secretsmanager.model.DeleteSecretRequest import com.amazonaws.services.secretsmanager.model.ResourceNotFoundException @@ -54,7 +55,24 @@ class AwsSecretManagerPersistence(private val awsClient: AwsClient, private val payload: String, ) { Preconditions.checkArgument(payload.isNotEmpty(), "Payload shouldn't be empty") - if (read(coordinate).isNotEmpty()) { + val existingSecret = + try { + read(coordinate) + } catch (e: AWSSecretsManagerException) { + // We use tags to control access to secrets. + // The AWS SDK doesn't differentiate between role access exceptions and secret not found exceptions to prevent leaking information. + // Because of this we catch the exception and if it's due to the assumed-role not having access, we just ignore it and proceed. + // In theory, the secret should not exist, and we will go straight to attempting to create which is safe because: + // 1. Update and create are distinct actions and we can't create over an already existing secret so we should get an error and no-op + // 2. If the secret does exist, we will get an error and no-op + if (e.localizedMessage.contains("assumed-role")) { + logger.info { "AWS exception caught - Secret ${coordinate.coordinateBase} not found" } + "" + } else { + throw e + } + } + if (existingSecret.isNotEmpty()) { logger.debug { "Secret ${coordinate.coordinateBase} found updating payload." } val request = UpdateSecretRequest() diff --git a/airbyte-config/config-secrets/src/main/kotlin/secrets/persistence/RuntimeSecretPersistence.kt b/airbyte-config/config-secrets/src/main/kotlin/secrets/persistence/RuntimeSecretPersistence.kt index f4abb01c256..bb48f2d3fb0 100644 --- a/airbyte-config/config-secrets/src/main/kotlin/secrets/persistence/RuntimeSecretPersistence.kt +++ b/airbyte-config/config-secrets/src/main/kotlin/secrets/persistence/RuntimeSecretPersistence.kt @@ -12,8 +12,8 @@ import io.github.oshai.kotlinlogging.KotlinLogging import io.micronaut.context.annotation.Property import kotlin.jvm.optionals.getOrElse -private const val AWS_ACCESS_KEY = "AWS_ACCESS_KEY_ID" -private const val AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY" +private const val AWS_ASSUME_ROLE_ACCESS_KEY_ID = "AWS_ASSUME_ROLE_ACCESS_KEY_ID" +private const val AWS_ASSUME_ROLE_SECRET_ACCESS_KEY = "AWS_ASSUME_ROLE_SECRET_ACCESS_KEY" /** * Class representing a RuntimeSecretPersistence to be used for BYO secrets customers. @@ -22,10 +22,10 @@ class RuntimeSecretPersistence(private val secretPersistenceConfig: SecretPersis private val log = KotlinLogging.logger {} @Property(name = "airbyte.secret.store.aws.access-key") - private val awsAccessKey: String? = System.getenv(AWS_ACCESS_KEY) + private val awsAccessKey: String? = System.getenv(AWS_ASSUME_ROLE_ACCESS_KEY_ID) @Property(name = "airbyte.secret.store.aws.secret-key") - private val awsSecretKey: String? = System.getenv(AWS_SECRET_ACCESS_KEY) + private val awsSecretKey: String? = System.getenv(AWS_ASSUME_ROLE_SECRET_ACCESS_KEY) private fun buildSecretPersistence(secretPersistenceConfig: SecretPersistenceConfig): SecretPersistence { return when (secretPersistenceConfig.secretPersistenceType) { From fecceaef771118ece798e44f90d8ebfe564643e0 Mon Sep 17 00:00:00 2001 From: Cole Snodgrass Date: Tue, 12 Dec 2023 10:17:51 -0800 Subject: [PATCH 07/27] remove global.database.[host|port] (#10257) --- charts/airbyte-api-server/README.md | 4 +-- charts/airbyte-api-server/values.yaml | 2 -- charts/airbyte-bootloader/README.md | 4 +-- charts/airbyte-bootloader/values.yaml | 4 --- .../README.md | 4 +-- .../values.yaml | 2 -- charts/airbyte-cron/README.md | 5 ++- charts/airbyte-cron/values.yaml | 2 -- charts/airbyte-keycloak-setup/README.md | 4 +-- charts/airbyte-keycloak-setup/values.yaml | 6 +--- charts/airbyte-keycloak/README.md | 4 +-- charts/airbyte-keycloak/values.yaml | 4 --- charts/airbyte-metrics/README.md | 4 +-- charts/airbyte-metrics/values.yaml | 2 -- charts/airbyte-pod-sweeper/README.md | 4 +-- charts/airbyte-pod-sweeper/values.yaml | 6 ---- charts/airbyte-server/README.md | 4 +-- charts/airbyte-server/values.yaml | 2 -- charts/airbyte-temporal/README.md | 4 +-- .../templates/deployment.yaml | 4 --- charts/airbyte-temporal/values.yaml | 2 -- charts/airbyte-webapp/README.md | 4 +-- charts/airbyte-webapp/values.yaml | 4 +-- charts/airbyte-worker/README.md | 5 ++- charts/airbyte-worker/values.yaml | 2 -- charts/airbyte-workload-api-server/README.md | 5 ++- .../airbyte-workload-api-server/values.yaml | 2 -- charts/airbyte-workload-launcher/README.md | 5 ++- charts/airbyte-workload-launcher/values.yaml | 2 -- charts/airbyte/README.md | 32 +++++++++---------- charts/airbyte/values.yaml | 4 --- charts/test_resources/pro/values.yaml | 4 --- 32 files changed, 35 insertions(+), 111 deletions(-) diff --git a/charts/airbyte-api-server/README.md b/charts/airbyte-api-server/README.md index d6354adb0ad..66e00fbbc2e 100644 --- a/charts/airbyte-api-server/README.md +++ b/charts/airbyte-api-server/README.md @@ -1,6 +1,6 @@ # airbyte-api-server -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-api-server @@ -30,8 +30,6 @@ Helm chart to deploy airbyte-api-server | extraVolumes | list | `[]` | | | global.configMapName | string | `""` | | | global.credVolumeOverride | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | diff --git a/charts/airbyte-api-server/values.yaml b/charts/airbyte-api-server/values.yaml index 40f80771b36..0499c858019 100644 --- a/charts/airbyte-api-server/values.yaml +++ b/charts/airbyte-api-server/values.yaml @@ -13,8 +13,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" logs: ## logs.accessKey.password Logs Access Key ## logs.accessKey.existingSecret diff --git a/charts/airbyte-bootloader/README.md b/charts/airbyte-bootloader/README.md index ee1f034d6d6..431a510dcbd 100644 --- a/charts/airbyte-bootloader/README.md +++ b/charts/airbyte-bootloader/README.md @@ -1,6 +1,6 @@ # airbyte-bootloader -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-bootloader @@ -25,8 +25,6 @@ Helm chart to deploy airbyte-bootloader | extraSelectorLabels | object | `{}` | | | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | diff --git a/charts/airbyte-bootloader/values.yaml b/charts/airbyte-bootloader/values.yaml index 7559cd8a7be..d86e71b833b 100644 --- a/charts/airbyte-bootloader/values.yaml +++ b/charts/airbyte-bootloader/values.yaml @@ -3,8 +3,6 @@ ## secretName Overrides the secrate name with with credentials to S3 Bucket and logging ## database.secretName Name of database secret ## database.secretValue Value of database password key stored in secret -## database.host Hostname of database -## database.port Database port global: serviceAccountName: placeholderServiceAccount deploymentMode: oss @@ -12,8 +10,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" secrets: {} env_vars: {} extraContainers: [] diff --git a/charts/airbyte-connector-builder-server/README.md b/charts/airbyte-connector-builder-server/README.md index 7a7d29b0811..68e47e4cd40 100644 --- a/charts/airbyte-connector-builder-server/README.md +++ b/charts/airbyte-connector-builder-server/README.md @@ -1,6 +1,6 @@ # connector-builder-server -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-connector-builder-server @@ -27,8 +27,6 @@ Helm chart to deploy airbyte-connector-builder-server | extraVolumes | list | `[]` | | | global.configMapName | string | `""` | | | global.credVolumeOverride | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | diff --git a/charts/airbyte-connector-builder-server/values.yaml b/charts/airbyte-connector-builder-server/values.yaml index fc9dcf63166..04c24fc9d39 100644 --- a/charts/airbyte-connector-builder-server/values.yaml +++ b/charts/airbyte-connector-builder-server/values.yaml @@ -13,8 +13,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" enabled: true ## replicaCount Number of server replicas diff --git a/charts/airbyte-cron/README.md b/charts/airbyte-cron/README.md index 5ee01c5e62b..c444eb29e07 100644 --- a/charts/airbyte-cron/README.md +++ b/charts/airbyte-cron/README.md @@ -1,6 +1,6 @@ # cron -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-cron @@ -26,8 +26,6 @@ Helm chart to deploy airbyte-cron | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | | global.configMapName | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | @@ -60,4 +58,5 @@ Helm chart to deploy airbyte-cron | resources.requests | object | `{}` | | | secrets | object | `{}` | | | tolerations | list | `[]` | | +| workloadApi | object | `{}` | | diff --git a/charts/airbyte-cron/values.yaml b/charts/airbyte-cron/values.yaml index 713c065e536..3b2016790d7 100644 --- a/charts/airbyte-cron/values.yaml +++ b/charts/airbyte-cron/values.yaml @@ -11,8 +11,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" secrets: {} enabled: true diff --git a/charts/airbyte-keycloak-setup/README.md b/charts/airbyte-keycloak-setup/README.md index f26afd6a507..ce6642534e2 100644 --- a/charts/airbyte-keycloak-setup/README.md +++ b/charts/airbyte-keycloak-setup/README.md @@ -1,6 +1,6 @@ # keycloak-setup -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-keycloak-setup @@ -25,8 +25,6 @@ Helm chart to deploy airbyte-keycloak-setup | extraSelectorLabels | object | `{}` | | | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | diff --git a/charts/airbyte-keycloak-setup/values.yaml b/charts/airbyte-keycloak-setup/values.yaml index 15818af4bd6..d09235ec58a 100644 --- a/charts/airbyte-keycloak-setup/values.yaml +++ b/charts/airbyte-keycloak-setup/values.yaml @@ -3,8 +3,6 @@ ## secretName Overrides the secrate name with with credentials to S3 Bucket and logging ## database.secretName Name of database secret ## database.secretValue Value of database password key stored in secret -## database.host Hostname of database -## database.port Database port global: serviceAccountName: placeholderServiceAccount deploymentMode: oss @@ -12,8 +10,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" secrets: {} env_vars: {} extraContainers: [] @@ -182,4 +178,4 @@ env_vars: {} ## extraSelectorLables [object] - use to specify own additional selector labels for deployment extraSelectorLabels: {} ## extraLabels [object] - use to specify own additional labels for deployment -extraLabels: {} \ No newline at end of file +extraLabels: {} diff --git a/charts/airbyte-keycloak/README.md b/charts/airbyte-keycloak/README.md index 53573c7182c..2dda5e43035 100644 --- a/charts/airbyte-keycloak/README.md +++ b/charts/airbyte-keycloak/README.md @@ -1,6 +1,6 @@ # keycloak -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-keycloak @@ -25,8 +25,6 @@ Helm chart to deploy airbyte-keycloak | extraSelectorLabels | object | `{}` | | | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | airbyte deployment mode | diff --git a/charts/airbyte-keycloak/values.yaml b/charts/airbyte-keycloak/values.yaml index 3fc6c0945a6..85c7a244313 100644 --- a/charts/airbyte-keycloak/values.yaml +++ b/charts/airbyte-keycloak/values.yaml @@ -3,8 +3,6 @@ ## secretName Overrides the secrate name with with credentials to S3 Bucket and logging ## database.secretName Name of database secret ## database.secretValue Value of database password key stored in secret -## database.host Hostname of database -## database.port Database port global: # serviceAccountName -- Service Account name override serviceAccountName: placeholderServiceAccount @@ -14,8 +12,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" secrets: {} env_vars: {} extraContainers: [] diff --git a/charts/airbyte-metrics/README.md b/charts/airbyte-metrics/README.md index d36f0aa5e47..752690d9ff3 100644 --- a/charts/airbyte-metrics/README.md +++ b/charts/airbyte-metrics/README.md @@ -1,6 +1,6 @@ # metrics -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-metrics @@ -22,8 +22,6 @@ Helm chart to deploy airbyte-metrics | extraEnv | list | `[]` | | | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | diff --git a/charts/airbyte-metrics/values.yaml b/charts/airbyte-metrics/values.yaml index bf953560568..08424a2ac81 100644 --- a/charts/airbyte-metrics/values.yaml +++ b/charts/airbyte-metrics/values.yaml @@ -6,8 +6,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" enabled: true ## metrics.replicaCount Number of metrics-reporter replicas diff --git a/charts/airbyte-pod-sweeper/README.md b/charts/airbyte-pod-sweeper/README.md index 565a84fd10f..83e2dfc9e5b 100644 --- a/charts/airbyte-pod-sweeper/README.md +++ b/charts/airbyte-pod-sweeper/README.md @@ -1,6 +1,6 @@ # pod-sweeper -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-pod-sweeper @@ -19,8 +19,6 @@ Helm chart to deploy airbyte-pod-sweeper | enabled | bool | `true` | | | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.jobs.kube.annotations | object | `{}` | | diff --git a/charts/airbyte-pod-sweeper/values.yaml b/charts/airbyte-pod-sweeper/values.yaml index 687316039b3..c11303f2496 100644 --- a/charts/airbyte-pod-sweeper/values.yaml +++ b/charts/airbyte-pod-sweeper/values.yaml @@ -3,18 +3,12 @@ ## secretName Overrides the secrate name with with credentials to S3 Bucket and logging ## database.secretName Name of database secret ## database.secretValue Value of database password key stored in secret -## database.host Hostname of database -## database.port Database port global: - serviceAccountName: &service-account-name "airbyte-admin" - secretName: "" database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" logs: ## logs.accessKey.password Logs Access Key ## logs.accessKey.existingSecret diff --git a/charts/airbyte-server/README.md b/charts/airbyte-server/README.md index 19829d16c0e..434f73526a3 100644 --- a/charts/airbyte-server/README.md +++ b/charts/airbyte-server/README.md @@ -1,6 +1,6 @@ # server -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-server @@ -30,8 +30,6 @@ Helm chart to deploy airbyte-server | extraVolumes | list | `[]` | | | global.configMapName | string | `""` | | | global.credVolumeOverride | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | diff --git a/charts/airbyte-server/values.yaml b/charts/airbyte-server/values.yaml index 7758f22b1d6..ab4577356c9 100644 --- a/charts/airbyte-server/values.yaml +++ b/charts/airbyte-server/values.yaml @@ -13,8 +13,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" logs: ## logs.accessKey.password Logs Access Key ## logs.accessKey.existingSecret diff --git a/charts/airbyte-temporal/README.md b/charts/airbyte-temporal/README.md index 250e34b92fa..6540aa8e978 100644 --- a/charts/airbyte-temporal/README.md +++ b/charts/airbyte-temporal/README.md @@ -1,6 +1,6 @@ # temporal -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-temporal @@ -26,8 +26,6 @@ Helm chart to deploy airbyte-temporal | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | | global.configMapName | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | diff --git a/charts/airbyte-temporal/templates/deployment.yaml b/charts/airbyte-temporal/templates/deployment.yaml index e48fd3d5412..fdc4a8a7085 100644 --- a/charts/airbyte-temporal/templates/deployment.yaml +++ b/charts/airbyte-temporal/templates/deployment.yaml @@ -56,14 +56,10 @@ spec: - name: DB # The DB engine to use value: "postgresql" - name: DB_PORT - {{- if .Values.global.database.port }} - value: {{ .Values.global.database.port | int | quote }} - {{- else }} valueFrom: configMapKeyRef: name: {{ .Values.global.configMapName | default (printf "%s-airbyte-env" .Release.Name) }} key: DATABASE_PORT - {{- end }} - name: POSTGRES_USER valueFrom: secretKeyRef: diff --git a/charts/airbyte-temporal/values.yaml b/charts/airbyte-temporal/values.yaml index 73f244dd4e4..9a7eb3b46b9 100644 --- a/charts/airbyte-temporal/values.yaml +++ b/charts/airbyte-temporal/values.yaml @@ -11,8 +11,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" enabled: true ## temporal.replicaCount The number of temporal replicas to deploy diff --git a/charts/airbyte-webapp/README.md b/charts/airbyte-webapp/README.md index bfc7f0ef64f..c78323ec611 100644 --- a/charts/airbyte-webapp/README.md +++ b/charts/airbyte-webapp/README.md @@ -1,6 +1,6 @@ # webapp -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-webapp @@ -28,8 +28,6 @@ Helm chart to deploy airbyte-webapp | extraVolumes | list | `[]` | | | fullstory.enabled | bool | `false` | | | global.configMapName | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | diff --git a/charts/airbyte-webapp/values.yaml b/charts/airbyte-webapp/values.yaml index 3e012890855..5b84641e759 100644 --- a/charts/airbyte-webapp/values.yaml +++ b/charts/airbyte-webapp/values.yaml @@ -11,8 +11,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" enabled: true ## webapp.replicaCount Number of webapp replicas @@ -252,4 +250,4 @@ env_vars: {} ## extraSelectorLables [object] - use to specify own additional selector labels for deployment extraSelectorLabels: {} ## extraLabels [object] - use to specify own additional labels for deployment -extraLabels: {} \ No newline at end of file +extraLabels: {} diff --git a/charts/airbyte-worker/README.md b/charts/airbyte-worker/README.md index bc17bf20b16..db9724d0d22 100644 --- a/charts/airbyte-worker/README.md +++ b/charts/airbyte-worker/README.md @@ -1,6 +1,6 @@ # worker -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-worker @@ -28,8 +28,6 @@ Helm chart to deploy airbyte-worker | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | | global.credVolumeOverride | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.extraContainers | list | `[]` | | @@ -86,4 +84,5 @@ Helm chart to deploy airbyte-worker | resources.requests | object | `{}` | | | secrets | object | `{}` | | | tolerations | list | `[]` | | +| workloadApi | object | `{}` | | diff --git a/charts/airbyte-worker/values.yaml b/charts/airbyte-worker/values.yaml index 605ca0145bb..eca8d7e0852 100644 --- a/charts/airbyte-worker/values.yaml +++ b/charts/airbyte-worker/values.yaml @@ -11,8 +11,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" logs: ## logs.accessKey.password Logs Access Key ## logs.accessKey.existingSecret diff --git a/charts/airbyte-workload-api-server/README.md b/charts/airbyte-workload-api-server/README.md index 1074066627a..39aaddd012c 100644 --- a/charts/airbyte-workload-api-server/README.md +++ b/charts/airbyte-workload-api-server/README.md @@ -1,6 +1,6 @@ # workload-api-server -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy the workload-api service @@ -30,8 +30,6 @@ Helm chart to deploy the workload-api service | extraVolumes | list | `[]` | | | global.configMapName | string | `""` | | | global.credVolumeOverride | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.deploymentMode | string | `"oss"` | | @@ -87,4 +85,5 @@ Helm chart to deploy the workload-api service | service.port | int | `8007` | | | service.type | string | `"ClusterIP"` | | | tolerations | list | `[]` | | +| workloadApi | object | `{}` | | diff --git a/charts/airbyte-workload-api-server/values.yaml b/charts/airbyte-workload-api-server/values.yaml index e3ed33ee120..37013d817ce 100644 --- a/charts/airbyte-workload-api-server/values.yaml +++ b/charts/airbyte-workload-api-server/values.yaml @@ -12,8 +12,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" logs: ## logs.accessKey.password Logs Access Key ## logs.accessKey.existingSecret diff --git a/charts/airbyte-workload-launcher/README.md b/charts/airbyte-workload-launcher/README.md index fd5f3021793..571944f07a5 100644 --- a/charts/airbyte-workload-launcher/README.md +++ b/charts/airbyte-workload-launcher/README.md @@ -1,6 +1,6 @@ # workload-launcher -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte-workload-launcher @@ -28,8 +28,6 @@ Helm chart to deploy airbyte-workload-launcher | extraVolumeMounts | list | `[]` | | | extraVolumes | list | `[]` | | | global.credVolumeOverride | string | `""` | | -| global.database.host | string | `"example.com"` | | -| global.database.port | string | `"5432"` | | | global.database.secretName | string | `""` | | | global.database.secretValue | string | `""` | | | global.extraContainers | list | `[]` | | @@ -86,4 +84,5 @@ Helm chart to deploy airbyte-workload-launcher | resources.requests | object | `{}` | | | secrets | object | `{}` | | | tolerations | list | `[]` | | +| workloadApi | object | `{}` | | diff --git a/charts/airbyte-workload-launcher/values.yaml b/charts/airbyte-workload-launcher/values.yaml index d696ed92c15..b11a7cd818f 100644 --- a/charts/airbyte-workload-launcher/values.yaml +++ b/charts/airbyte-workload-launcher/values.yaml @@ -11,8 +11,6 @@ global: database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" logs: ## logs.accessKey.password Logs Access Key ## logs.accessKey.existingSecret diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 1437ecee8b0..5953bcbce8c 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -1,6 +1,6 @@ # airbyte -![Version: 0.50.9](https://img.shields.io/badge/Version-0.50.9-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) +![Version: 0.50.14](https://img.shields.io/badge/Version-0.50.14-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: dev](https://img.shields.io/badge/AppVersion-dev-informational?style=flat-square) Helm chart to deploy airbyte @@ -8,20 +8,20 @@ Helm chart to deploy airbyte | Repository | Name | Version | |------------|------|---------| -| https://airbytehq.github.io/helm-charts/ | airbyte-api-server | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | airbyte-bootloader | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | connector-builder-server | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | cron | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | keycloak | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | keycloak-setup | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | metrics | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | pod-sweeper | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | server | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | temporal | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | webapp | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | worker | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | workload-api-server | 0.50.9 | -| https://airbytehq.github.io/helm-charts/ | workload-launcher | 0.50.9 | +| https://airbytehq.github.io/helm-charts/ | airbyte-api-server | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | airbyte-bootloader | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | connector-builder-server | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | cron | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | keycloak | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | keycloak-setup | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | metrics | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | pod-sweeper | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | server | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | temporal | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | webapp | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | worker | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | workload-api-server | 0.50.14 | +| https://airbytehq.github.io/helm-charts/ | workload-launcher | 0.50.14 | | https://charts.bitnami.com/bitnami | common | 1.x.x | ## Values @@ -133,8 +133,6 @@ Helm chart to deploy airbyte | externalDatabase.port | int | `5432` | Database port number | | externalDatabase.user | string | `"airbyte"` | non-root Username for Airbyte Database | | fullnameOverride | string | `""` | String to fully override airbyte.fullname template with a string | -| global.database.host | string | `"example.com"` | Database host | -| global.database.port | string | `"5432"` | Database port | | global.database.secretName | string | `""` | Secret name where database credentials are stored | | global.database.secretValue | string | `""` | Secret value for database password | | global.deploymentMode | string | `"oss"` | Deployment mode, whether or not render the default env vars and volumes in deployment spec | diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 82f1a176ad2..1216da982a1 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -18,10 +18,6 @@ global: secretName: "" # -- Secret value for database password secretValue: "" - # -- Database host - host: "example.com" - # -- Database port - port: "5432" state: # -- Determines which state storage will be utilized; "MINIO", "S3", or "GCS" storage: diff --git a/charts/test_resources/pro/values.yaml b/charts/test_resources/pro/values.yaml index 70b06976249..baa1b21e7cd 100644 --- a/charts/test_resources/pro/values.yaml +++ b/charts/test_resources/pro/values.yaml @@ -14,13 +14,9 @@ global: ## database [object] -- Object used to overrite database configuration(to use external DB) ## database.secretName -- secret name where DB creds are stored ## database.secretValue -- secret value for database password - ## database.host -- Database host override - ## database.port -- Database port override database: secretName: "" secretValue: "" - host: "example.com" - port: "5432" state: ## state.storage.type Determines which state storage will be utilized. One of "MINIO", "S3" or "GCS" storage: From 893f1162c70eba2ba79416527977ae21dfed6665 Mon Sep 17 00:00:00 2001 From: colesnodgrass Date: Tue, 12 Dec 2023 18:27:16 +0000 Subject: [PATCH 08/27] Bump helm chart version reference to 0.50.15 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte-workload-api-server/Chart.yaml | 2 +- charts/airbyte-workload-launcher/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 32 +++++++++---------- charts/airbyte/Chart.yaml | 30 ++++++++--------- 16 files changed, 45 insertions(+), 45 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 8b03425b359..043f063a688 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index f35db12b5eb..4e437c7db06 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 8ce63f8e87f..ace897ac4a2 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index a022b0c3365..66edc85aa8e 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 4112945d4fe..91fe18641c6 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index cc4fd2a2497..754881965c3 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 3e2c89c23f7..2be81021f20 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 7ecf527aa93..67bd21a329d 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 2e7085ecfe0..fbe621ece10 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 325e9db2bc6..1b149b7210f 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index d54a3034554..87b76dff29d 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index a6761a1254b..a5d1d61f040 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-workload-api-server/Chart.yaml b/charts/airbyte-workload-api-server/Chart.yaml index e7d2755da81..4c0f3391b0b 100644 --- a/charts/airbyte-workload-api-server/Chart.yaml +++ b/charts/airbyte-workload-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-workload-launcher/Chart.yaml b/charts/airbyte-workload-launcher/Chart.yaml index 26eb3b8462c..a15e5c079d2 100644 --- a/charts/airbyte-workload-launcher/Chart.yaml +++ b/charts/airbyte-workload-launcher/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 89b9cc29cc1..9c2900a6bbf 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,45 +4,45 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: workload-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: workload-launcher repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 -digest: sha256:0ea4d8bc5205395771e270357ef3e1625d3916c2c0b1361bd8f7c87fc1145061 -generated: "2023-12-11T14:45:41.395628551Z" + version: 0.50.15 +digest: sha256:464fd8dea63ffa31a8b9c274e96e5a558f8a89edf85acfd6b2055b45f0bcdc73 +generated: "2023-12-12T18:26:55.227730608Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index fd79b6051d7..76843507e50 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.14 +version: 0.50.15 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,56 +32,56 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: workload-api-server.enabled name: workload-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: workload-launcher.enabled name: workload-launcher repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.14 + version: 0.50.15 From 2a2786081eb7a20ecacba30efbcb5ceb01a1e934 Mon Sep 17 00:00:00 2001 From: Jon Tan Date: Tue, 12 Dec 2023 13:48:46 -0600 Subject: [PATCH 09/27] Allow OSS to set region for AWS secret manager (#10326) --- airbyte-api-server/src/main/resources/application.yml | 1 + airbyte-bootloader/src/main/resources/application.yml | 1 + airbyte-server/src/main/resources/application.yml | 1 + airbyte-workers/src/main/resources/application.yml | 1 + airbyte-workload-launcher/src/main/resources/application.yml | 1 + 5 files changed, 5 insertions(+) diff --git a/airbyte-api-server/src/main/resources/application.yml b/airbyte-api-server/src/main/resources/application.yml index d6490fc1219..6ba56a9a4dc 100644 --- a/airbyte-api-server/src/main/resources/application.yml +++ b/airbyte-api-server/src/main/resources/application.yml @@ -47,6 +47,7 @@ airbyte: aws: access-key: ${AWS_ACCESS_KEY:} secret-key: ${AWS_SECRET_ACCESS_KEY:} + region: ${AWS_REGION:} gcp: credentials: ${SECRET_STORE_GCP_CREDENTIALS:} project-id: ${SECRET_STORE_GCP_PROJECT_ID:} diff --git a/airbyte-bootloader/src/main/resources/application.yml b/airbyte-bootloader/src/main/resources/application.yml index c668a934f0f..148d5818662 100644 --- a/airbyte-bootloader/src/main/resources/application.yml +++ b/airbyte-bootloader/src/main/resources/application.yml @@ -37,6 +37,7 @@ airbyte: aws: access-key: ${AWS_ACCESS_KEY:} secret-key: ${AWS_SECRET_ACCESS_KEY:} + region: ${AWS_REGION:} gcp: credentials: ${SECRET_STORE_GCP_CREDENTIALS:} project-id: ${SECRET_STORE_GCP_PROJECT_ID:} diff --git a/airbyte-server/src/main/resources/application.yml b/airbyte-server/src/main/resources/application.yml index 4bd3741507f..f43d3aff1d8 100644 --- a/airbyte-server/src/main/resources/application.yml +++ b/airbyte-server/src/main/resources/application.yml @@ -91,6 +91,7 @@ airbyte: aws: access-key: ${AWS_ACCESS_KEY:} secret-key: ${AWS_SECRET_ACCESS_KEY:} + region: ${AWS_REGION:} gcp: credentials: ${SECRET_STORE_GCP_CREDENTIALS:} project-id: ${SECRET_STORE_GCP_PROJECT_ID:} diff --git a/airbyte-workers/src/main/resources/application.yml b/airbyte-workers/src/main/resources/application.yml index f1728676ff4..c7620281921 100644 --- a/airbyte-workers/src/main/resources/application.yml +++ b/airbyte-workers/src/main/resources/application.yml @@ -228,6 +228,7 @@ airbyte: aws: access-key: ${AWS_ACCESS_KEY:} secret-key: ${AWS_SECRET_ACCESS_KEY:} + region: ${AWS_REGION:} gcp: credentials: ${SECRET_STORE_GCP_CREDENTIALS:} project-id: ${SECRET_STORE_GCP_PROJECT_ID:} diff --git a/airbyte-workload-launcher/src/main/resources/application.yml b/airbyte-workload-launcher/src/main/resources/application.yml index 3e6122072f7..14587fe72fd 100644 --- a/airbyte-workload-launcher/src/main/resources/application.yml +++ b/airbyte-workload-launcher/src/main/resources/application.yml @@ -47,6 +47,7 @@ airbyte: aws: access-key: ${AWS_ACCESS_KEY:} secret-key: ${AWS_SECRET_ACCESS_KEY:} + region: ${AWS_REGION:} gcp: credentials: ${SECRET_STORE_GCP_CREDENTIALS:} project-id: ${SECRET_STORE_GCP_PROJECT_ID:} From 870917ff5807d005d7b3af34574b63737be8fd34 Mon Sep 17 00:00:00 2001 From: terencecho <3916587+terencecho@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:53:19 -0800 Subject: [PATCH 10/27] Forward auth header from airbyte api calls (#10079) --- .../controllers/ConnectionsController.kt | 24 +++++++++---- .../server/controllers/DefaultController.kt | 5 ++- .../controllers/DestinationsController.kt | 12 +++++++ .../api/server/controllers/JobsController.kt | 12 +++++++ .../server/controllers/SourcesController.kt | 15 +++++++- .../server/controllers/StreamsController.kt | 3 ++ .../controllers/WorkspacesController.kt | 16 ++++++--- .../forwardingClient/ConfigApiClient.kt | 36 +++++++++++++++++++ .../api/server/services/ConnectionService.kt | 22 ++++++++---- .../api/server/services/DestinationService.kt | 34 +++++++++++++----- .../airbyte/api/server/services/JobService.kt | 24 +++++++++---- .../api/server/services/OAuthService.kt | 3 ++ .../services/SourceDefinitionService.kt | 4 ++- .../api/server/services/SourceService.kt | 30 ++++++++++++---- .../api/server/services/UserService.kt | 2 +- .../api/server/services/WorkspaceService.kt | 36 +++++++++++++++---- .../jaxrs-spec-api/apiInterface.mustache | 2 +- 17 files changed, 230 insertions(+), 50 deletions(-) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt index 3b07b6e6d09..ae86731f750 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/ConnectionsController.kt @@ -42,6 +42,7 @@ open class ConnectionsController( ) : ConnectionsApi { override fun createConnection( connectionCreateRequest: ConnectionCreateRequest?, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -56,7 +57,7 @@ open class ConnectionsController( // get destination response to retrieve workspace id as well as input for destination sync modes val destinationResponse: DestinationResponse = trackingHelper.callWithTracker( - { destinationService.getDestination(connectionCreateRequest!!.destinationId, validUserInfo) }, + { destinationService.getDestination(connectionCreateRequest!!.destinationId, authorization, validUserInfo) }, CONNECTIONS_PATH, POST, userId, @@ -65,7 +66,7 @@ open class ConnectionsController( // get source schema for catalog id and airbyte catalog val schemaResponse: SourceDiscoverSchemaRead = trackingHelper.callWithTracker( - { sourceService.getSourceSchema(connectionCreateRequest!!.sourceId, false, validUserInfo) }, + { sourceService.getSourceSchema(connectionCreateRequest!!.sourceId, false, authorization, validUserInfo) }, CONNECTIONS_PATH, POST, userId, @@ -106,7 +107,7 @@ open class ConnectionsController( val validDestinationSyncModes = trackingHelper.callWithTracker( - { destinationService.getDestinationSyncModes(destinationResponse, validUserInfo) }, + { destinationService.getDestinationSyncModes(destinationResponse, authorization, validUserInfo) }, CONNECTIONS_PATH, POST, userId, @@ -142,6 +143,7 @@ open class ConnectionsController( catalogId!!, finalConfiguredCatalog!!, destinationResponse.workspaceId, + authorization, validUserInfo, ) }, CONNECTIONS_PATH, POST, userId)!! @@ -159,6 +161,7 @@ open class ConnectionsController( override fun deleteConnection( connectionId: UUID, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -168,6 +171,7 @@ open class ConnectionsController( { connectionService.deleteConnection( connectionId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -188,6 +192,7 @@ open class ConnectionsController( override fun getConnection( connectionId: UUID, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -196,6 +201,7 @@ open class ConnectionsController( trackingHelper.callWithTracker({ connectionService.getConnection( connectionId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, CONNECTIONS_PATH, GET, userId)!! @@ -215,6 +221,7 @@ open class ConnectionsController( includeDeleted: Boolean?, limit: Int?, offset: Int?, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -227,6 +234,7 @@ open class ConnectionsController( limit!!, offset!!, includeDeleted!!, + authorization, getLocalUserInfoIfNull(userInfo), ) }, CONNECTIONS_PATH, GET, userId)!! @@ -245,6 +253,7 @@ open class ConnectionsController( override fun patchConnection( connectionId: UUID, connectionPatchRequest: ConnectionPatchRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -264,7 +273,7 @@ open class ConnectionsController( val currentConnection: ConnectionResponse = trackingHelper.callWithTracker( - { connectionService.getConnection(connectionId, validUserInfo) }, + { connectionService.getConnection(connectionId, authorization, validUserInfo) }, CONNECTIONS_WITH_ID_PATH, PUT, userId, @@ -273,7 +282,7 @@ open class ConnectionsController( // get destination response to retrieve workspace id as well as input for destination sync modes val destinationResponse: DestinationResponse = trackingHelper.callWithTracker( - { destinationService.getDestination(currentConnection.destinationId, validUserInfo) }, + { destinationService.getDestination(currentConnection.destinationId, authorization, validUserInfo) }, CONNECTIONS_WITH_ID_PATH, PUT, userId, @@ -282,7 +291,7 @@ open class ConnectionsController( // get source schema for catalog id and airbyte catalog val schemaResponse = trackingHelper.callWithTracker( - { sourceService.getSourceSchema(currentConnection.sourceId, false, validUserInfo) }, + { sourceService.getSourceSchema(currentConnection.sourceId, false, authorization, validUserInfo) }, CONNECTIONS_PATH, POST, userId, @@ -323,7 +332,7 @@ open class ConnectionsController( val validDestinationSyncModes = trackingHelper.callWithTracker( - { destinationService.getDestinationSyncModes(destinationResponse, validUserInfo) }, + { destinationService.getDestinationSyncModes(destinationResponse, authorization, validUserInfo) }, CONNECTIONS_PATH, POST, userId, @@ -360,6 +369,7 @@ open class ConnectionsController( catalogId!!, finalConfiguredCatalog, destinationResponse.workspaceId, + authorization, validUserInfo, ) }, diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DefaultController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DefaultController.kt index 1251afc5bac..5bd94a12e01 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DefaultController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DefaultController.kt @@ -12,7 +12,10 @@ open class DefaultController() : DefaultApi { @Value("\${airbyte.internal.documentation.host}") var documentationHost: String? = null - override fun getDocumentation(userInfo: String?): Response { + override fun getDocumentation( + authorization: String?, + userInfo: String?, + ): Response { return Response .status(302) .location(URI.create(documentationHost)) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt index 13d8e06358f..70e4bf25f67 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/DestinationsController.kt @@ -38,6 +38,7 @@ open class DestinationsController( ) : DestinationsApi { override fun createDestination( destinationCreateRequest: DestinationCreateRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -68,6 +69,7 @@ open class DestinationsController( destinationService.createDestination( destinationCreateRequest, destinationDefinitionId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -89,6 +91,7 @@ open class DestinationsController( override fun deleteDestination( destinationId: UUID, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -98,6 +101,7 @@ open class DestinationsController( { destinationService.deleteDestination( destinationId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -118,6 +122,7 @@ open class DestinationsController( override fun getDestination( destinationId: UUID, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -127,6 +132,7 @@ open class DestinationsController( { destinationService.getDestination( destinationId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -150,6 +156,7 @@ open class DestinationsController( includeDeleted: Boolean?, limit: Int?, offset: Int?, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -162,6 +169,7 @@ open class DestinationsController( includeDeleted!!, limit!!, offset!!, + authorization, getLocalUserInfoIfNull(userInfo), ) }, DESTINATIONS_PATH, GET, userId) @@ -180,6 +188,7 @@ open class DestinationsController( override fun patchDestination( destinationId: UUID, destinationPatchRequest: DestinationPatchRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -192,6 +201,7 @@ open class DestinationsController( destinationService.partialUpdateDestination( destinationId, destinationPatchRequest, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -214,6 +224,7 @@ open class DestinationsController( override fun putDestination( destinationId: UUID, destinationPutRequest: DestinationPutRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -226,6 +237,7 @@ open class DestinationsController( destinationService.updateDestination( destinationId, destinationPutRequest, + authorization, getLocalUserInfoIfNull(userInfo), ) }, diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt index 12228cc8c74..af1dcfc760f 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/JobsController.kt @@ -12,6 +12,7 @@ import io.airbyte.airbyte_api.model.generated.JobTypeEnum import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody.OrderByFieldEnum import io.airbyte.api.client.model.generated.JobListForWorkspacesRequestBody.OrderByMethodEnum import io.airbyte.api.server.apiTracking.TrackingHelper +import io.airbyte.api.server.constants.AUTH_HEADER import io.airbyte.api.server.constants.DELETE import io.airbyte.api.server.constants.ENDPOINT_API_USER_INFO_HEADER import io.airbyte.api.server.constants.GET @@ -47,6 +48,7 @@ open class JobsController( @Path("/{jobId}") override fun cancelJob( @PathParam("jobId") jobId: Long, + @HeaderParam(AUTH_HEADER) authorization: String?, @HeaderParam(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -56,6 +58,7 @@ open class JobsController( { jobService.cancelJob( jobId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -77,6 +80,7 @@ open class JobsController( override fun createJob( jobCreateRequest: JobCreateRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -86,6 +90,7 @@ open class JobsController( { connectionService.getConnection( jobCreateRequest.connectionId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -101,6 +106,7 @@ open class JobsController( trackingHelper.callWithTracker({ jobService.sync( jobCreateRequest.connectionId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, JOBS_PATH, POST, userId)!! @@ -121,6 +127,7 @@ open class JobsController( trackingHelper.callWithTracker({ jobService.reset( jobCreateRequest.connectionId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, JOBS_PATH, POST, userId)!! @@ -153,6 +160,7 @@ open class JobsController( @Path("/{jobId}") override fun getJob( @PathParam("jobId") jobId: Long, + @HeaderParam(AUTH_HEADER) authorization: String?, @HeaderParam(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -162,6 +170,7 @@ open class JobsController( { jobService.getJobInfoWithoutLogs( jobId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -193,6 +202,7 @@ open class JobsController( updatedAtStart: OffsetDateTime?, updatedAtEnd: OffsetDateTime?, orderBy: String?, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -221,6 +231,7 @@ open class JobsController( filter, orderByField, orderByMethod, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -236,6 +247,7 @@ open class JobsController( filter, orderByField, orderByMethod, + authorization, getLocalUserInfoIfNull(userInfo), ) }, diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt index e187a4b96c8..88184d03909 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/SourcesController.kt @@ -40,6 +40,7 @@ open class SourcesController( ) : SourcesApi { override fun createSource( sourceCreateRequest: SourceCreateRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -70,6 +71,7 @@ open class SourcesController( sourceService.createSource( sourceCreateRequest, sourceDefinitionId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -92,6 +94,7 @@ open class SourcesController( override fun deleteSource( sourceId: UUID, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -101,6 +104,7 @@ open class SourcesController( { sourceService.deleteSource( sourceId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -122,6 +126,7 @@ open class SourcesController( override fun getSource( sourceId: UUID, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -131,6 +136,7 @@ open class SourcesController( { sourceService.getSource( sourceId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -152,9 +158,10 @@ open class SourcesController( override fun initiateOAuth( initiateOauthRequest: InitiateOauthRequest, + authorization: String?, userInfo: String?, ): Response { - return sourceService.controllerInitiateOAuth(initiateOauthRequest, userInfo) + return sourceService.controllerInitiateOAuth(initiateOauthRequest, authorization, userInfo) } override fun listSources( @@ -162,6 +169,7 @@ open class SourcesController( includeDeleted: Boolean?, limit: Int?, offset: Int?, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -174,6 +182,7 @@ open class SourcesController( includeDeleted!!, limit!!, offset!!, + authorization, getLocalUserInfoIfNull(userInfo), ) }, SOURCES_PATH, GET, userId) @@ -193,6 +202,7 @@ open class SourcesController( override fun patchSource( sourceId: UUID, sourcePatchRequest: SourcePatchRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -205,6 +215,7 @@ open class SourcesController( sourceService.partialUpdateSource( sourceId, sourcePatchRequest, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -227,6 +238,7 @@ open class SourcesController( override fun putSource( sourceId: UUID, sourcePutRequest: SourcePutRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -239,6 +251,7 @@ open class SourcesController( sourceService.updateSource( sourceId, sourcePutRequest, + authorization, getLocalUserInfoIfNull(userInfo), ) }, diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt index 7a9d649283e..06d941e5765 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/StreamsController.kt @@ -43,6 +43,7 @@ class StreamsController( sourceId: UUID, destinationId: UUID?, ignoreCache: Boolean?, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -52,6 +53,7 @@ class StreamsController( sourceService.getSourceSchema( sourceId, ignoreCache!!, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -64,6 +66,7 @@ class StreamsController( { destinationService.getDestinationSyncModes( destinationId!!, + authorization, getLocalUserInfoIfNull(userInfo), ) }, diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt index ac7ea6cf5e4..b2eb49aa5b0 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/controllers/WorkspacesController.kt @@ -23,34 +23,39 @@ open class WorkspacesController( override fun createOrUpdateWorkspaceOAuthCredentials( workspaceId: UUID?, workspaceOAuthCredentialsRequest: WorkspaceOAuthCredentialsRequest?, + authorization: String?, userInfo: String?, ): Response { return workspaceService.controllerSetWorkspaceOverrideOAuthParams( workspaceId!!, workspaceOAuthCredentialsRequest, + authorization, userInfo, ) } override fun createWorkspace( workspaceCreateRequest: WorkspaceCreateRequest?, + authorization: String?, userInfo: String?, ): Response { - return workspaceService.controllerCreateWorkspace(workspaceCreateRequest!!, userInfo) + return workspaceService.controllerCreateWorkspace(workspaceCreateRequest!!, authorization, userInfo) } override fun deleteWorkspace( workspaceId: UUID?, + authorization: String?, userInfo: String?, ): Response { - return workspaceService.controllerDeleteWorkspace(workspaceId!!, userInfo) + return workspaceService.controllerDeleteWorkspace(workspaceId!!, authorization, userInfo) } override fun getWorkspace( workspaceId: UUID?, + authorization: String?, userInfo: String?, ): Response { - return workspaceService.controllerGetWorkspace(workspaceId!!, userInfo) + return workspaceService.controllerGetWorkspace(workspaceId!!, authorization, userInfo) } override fun listWorkspaces( @@ -58,6 +63,7 @@ open class WorkspacesController( includeDeleted: Boolean?, limit: Int?, offset: Int?, + authorization: String?, userInfo: String?, ): Response { return workspaceService.controllerListWorkspaces( @@ -65,6 +71,7 @@ open class WorkspacesController( includeDeleted!!, limit!!, offset!!, + authorization, userInfo, ) } @@ -74,8 +81,9 @@ open class WorkspacesController( override fun updateWorkspace( workspaceId: UUID?, workspaceUpdateRequest: WorkspaceUpdateRequest?, + authorization: String?, userInfo: String?, ): Response { - return workspaceService.controllerUpdateWorkspace(workspaceId!!, workspaceUpdateRequest!!, userInfo) + return workspaceService.controllerUpdateWorkspace(workspaceId!!, workspaceUpdateRequest!!, authorization, userInfo) } } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/forwardingClient/ConfigApiClient.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/forwardingClient/ConfigApiClient.kt index fed6f87be14..56735af5aad 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/forwardingClient/ConfigApiClient.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/forwardingClient/ConfigApiClient.kt @@ -46,6 +46,7 @@ import io.airbyte.api.client.model.generated.WorkspaceReadList import io.airbyte.api.client.model.generated.WorkspaceUpdate import io.airbyte.api.server.constants.ANALYTICS_HEADER import io.airbyte.api.server.constants.ANALYTICS_HEADER_VALUE +import io.airbyte.api.server.constants.AUTH_HEADER import io.airbyte.api.server.constants.ENDPOINT_API_USER_INFO_HEADER import io.airbyte.api.server.constants.INTERNAL_API_HOST import io.micronaut.http.HttpHeaders @@ -75,36 +76,42 @@ interface ConfigApiClient { @Post(value = "/api/v1/connections/create", processes = [MediaType.APPLICATION_JSON]) fun createConnection( @Body connectionId: ConnectionCreate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/connections/delete", processes = [MediaType.APPLICATION_JSON]) fun deleteConnection( @Body connectionIdRequestBody: ConnectionIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/connections/sync", processes = [MediaType.APPLICATION_JSON]) fun sync( @Body connectionId: ConnectionIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/connections/reset", processes = [MediaType.APPLICATION_JSON]) fun reset( @Body connectionId: ConnectionIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/connections/get", processes = [MediaType.APPLICATION_JSON]) fun getConnection( @Body connectionIdRequestBody: ConnectionIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/connections/update", processes = [MediaType.APPLICATION_JSON]) fun updateConnection( @Body connectionUpdate: ConnectionUpdate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @@ -112,12 +119,14 @@ interface ConfigApiClient { @Post(value = "/api/v1/source_oauths/get_consent_url", processes = [MediaType.APPLICATION_JSON]) fun getSourceConsentUrl( @Body consentRequest: SourceOauthConsentRequest, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/source_oauths/complete_oauth", processes = [MediaType.APPLICATION_JSON]) fun completeSourceOAuth( @Body completeSourceOauthRequest: CompleteSourceOauthRequest, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @@ -125,36 +134,42 @@ interface ConfigApiClient { @Post(value = "/api/v1/sources/create", processes = [MediaType.APPLICATION_JSON]) fun createSource( @Body sourceCreate: SourceCreate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/sources/delete", processes = [MediaType.APPLICATION_JSON]) fun deleteSource( @Body sourceIdRequestBody: SourceIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/sources/get", processes = [MediaType.APPLICATION_JSON]) fun getSource( @Body sourceIdRequestBody: SourceIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/sources/partial_update", processes = [MediaType.APPLICATION_JSON]) fun partialUpdateSource( @Body partialSourceUpdate: PartialSourceUpdate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/sources/update", processes = [MediaType.APPLICATION_JSON]) fun updateSource( @Body sourceUpdate: SourceUpdate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/sources/discover_schema", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) fun getSourceSchema( @Body sourceId: SourceDiscoverSchemaRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @@ -165,6 +180,7 @@ interface ConfigApiClient { ) fun getSourceDefinitionSpecification( @Body sourceDefinitionIdWithWorkspaceId: SourceDefinitionIdWithWorkspaceId, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @@ -172,36 +188,42 @@ interface ConfigApiClient { @Post(value = "/api/v1/destinations/create", processes = [MediaType.APPLICATION_JSON]) fun createDestination( @Body destinationCreate: DestinationCreate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/destinations/get", processes = [MediaType.APPLICATION_JSON]) fun getDestination( @Body destinationIdRequestBody: DestinationIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/destinations/update", processes = [MediaType.APPLICATION_JSON]) fun updateDestination( @Body destinationUpdate: DestinationUpdate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/destinations/partial_update", processes = [MediaType.APPLICATION_JSON]) fun partialUpdateDestination( @Body partialDestinationUpdate: PartialDestinationUpdate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/destinations/delete", processes = [MediaType.APPLICATION_JSON]) fun deleteDestination( @Body destinationIdRequestBody: DestinationIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/destination_definition_specifications/get", processes = [MediaType.APPLICATION_JSON]) fun getDestinationSpec( @Body destinationDefinitionIdWithWorkspaceId: DestinationDefinitionIdWithWorkspaceId, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @@ -209,24 +231,28 @@ interface ConfigApiClient { @Post(value = "/api/v1/jobs/get_without_logs", processes = [MediaType.APPLICATION_JSON]) fun getJobInfoWithoutLogs( @Body jobId: JobIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/jobs/list", processes = [MediaType.APPLICATION_JSON]) fun getJobList( @Body jobListRequestBody: JobListRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/jobs/list_for_workspaces", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) fun getJobListForWorkspaces( @Body listForWorkspacesRequestBody: JobListForWorkspacesRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) userInfo: String?, ): HttpResponse @Post(value = "/api/v1/jobs/cancel", processes = [MediaType.APPLICATION_JSON]) fun cancelJob( @Body jobIdRequestBody: JobIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @@ -234,53 +260,62 @@ interface ConfigApiClient { @Post(value = "/api/v1/workspaces/get", processes = [MediaType.APPLICATION_JSON]) fun getWorkspace( @Body workspaceIdRequestBody: WorkspaceIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/workspaces/list", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) fun listAllWorkspaces( + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/workspaces/create", processes = [MediaType.APPLICATION_JSON]) fun createWorkspace( @Body workspaceCreate: WorkspaceCreate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/workspaces/update", processes = [MediaType.APPLICATION_JSON]) fun updateWorkspace( @Body workspaceUpdate: WorkspaceUpdate, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/workspaces/delete", processes = [MediaType.APPLICATION_JSON]) fun deleteWorkspace( @Body workspaceIdRequestBody: WorkspaceIdRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/connections/list_paginated", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) fun listConnectionsForWorkspaces( @Body listConnectionsForWorkspacesRequestBody: ListConnectionsForWorkspacesRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/sources/list_paginated", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) fun listSourcesForWorkspaces( @Body listResourcesForWorkspacesRequestBody: ListResourcesForWorkspacesRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/destinations/list_paginated", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) fun listDestinationsForWorkspaces( @Body listResourcesForWorkspacesRequestBody: ListResourcesForWorkspacesRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @Post(value = "/api/v1/workspaces/list_paginated", processes = [MediaType.APPLICATION_JSON], produces = [MediaType.APPLICATION_JSON]) fun listWorkspaces( @Body listResourcesForWorkspacesRequestBody: ListResourcesForWorkspacesRequestBody, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse @@ -291,6 +326,7 @@ interface ConfigApiClient { ) fun setWorkspaceOverrideOAuthParams( @Body workspaceOverrideOauthParamsRequestBody: WorkspaceOverrideOauthParamsRequestBody?, + @Header(AUTH_HEADER) authorization: String?, @Header(ENDPOINT_API_USER_INFO_HEADER) endpointUserInfo: String?, ): HttpResponse<*>? } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/ConnectionService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/ConnectionService.kt index d82f0cb70ab..ac32b956ea4 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/ConnectionService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/ConnectionService.kt @@ -43,16 +43,19 @@ interface ConnectionService { catalogId: UUID, configuredCatalog: AirbyteCatalog, workspaceId: UUID, + authorization: String?, userInfo: String?, ): ConnectionResponse fun deleteConnection( connectionId: UUID, + authorization: String?, userInfo: String?, ) fun getConnection( connectionId: UUID, + authorization: String?, userInfo: String?, ): ConnectionResponse @@ -62,6 +65,7 @@ interface ConnectionService { catalogId: UUID, configuredCatalog: AirbyteCatalog?, workspaceId: UUID, + authorization: String?, userInfo: String?, ): ConnectionResponse @@ -70,6 +74,7 @@ interface ConnectionService { limit: Int = 20, offset: Int = 0, includeDeleted: Boolean = false, + authorization: String?, userInfo: String?, ): ConnectionsResponse } @@ -96,6 +101,7 @@ class ConnectionServiceImpl( catalogId: UUID, configuredCatalog: AirbyteCatalog, workspaceId: UUID, + authorization: String?, userInfo: String?, ): ConnectionResponse { val connectionCreateOss: ConnectionCreate = @@ -103,7 +109,7 @@ class ConnectionServiceImpl( val response = try { - configApiClient.createConnection(connectionCreateOss, userInfo) + configApiClient.createConnection(connectionCreateOss, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for createConnection: ", e) // this is kept as a string to easily parse the error response to determine if a source or a @@ -134,12 +140,13 @@ class ConnectionServiceImpl( */ override fun deleteConnection( connectionId: UUID, + authorization: String?, userInfo: String?, ) { val connectionIdRequestBody = ConnectionIdRequestBody().connectionId(connectionId) val response = try { - configApiClient.deleteConnection(connectionIdRequestBody, userInfo) + configApiClient.deleteConnection(connectionIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for connection delete: ", e) e.response as HttpResponse @@ -153,6 +160,7 @@ class ConnectionServiceImpl( */ override fun getConnection( connectionId: UUID, + authorization: String?, userInfo: String?, ): ConnectionResponse { val connectionIdRequestBody = ConnectionIdRequestBody() @@ -160,7 +168,7 @@ class ConnectionServiceImpl( val response = try { - configApiClient.getConnection(connectionIdRequestBody, userInfo) + configApiClient.getConnection(connectionIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for getConnection: ", e) e.response as HttpResponse @@ -169,7 +177,7 @@ class ConnectionServiceImpl( log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) // get workspace id from source id - val sourceResponse: SourceResponse = sourceService.getSource(response.body()!!.sourceId, userInfo) + val sourceResponse: SourceResponse = sourceService.getSource(response.body()!!.sourceId, authorization, userInfo) return ConnectionReadMapper.from( response.body()!!, @@ -186,6 +194,7 @@ class ConnectionServiceImpl( catalogId: UUID, configuredCatalog: AirbyteCatalog?, workspaceId: UUID, + authorization: String?, userInfo: String?, ): ConnectionResponse { val connectionUpdate: ConnectionUpdate = @@ -200,7 +209,7 @@ class ConnectionServiceImpl( // destination id is invalid val response = try { - configApiClient.updateConnection(connectionUpdate, userInfo) + configApiClient.updateConnection(connectionUpdate, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for updateConnection: ", e) e.response as HttpResponse @@ -232,6 +241,7 @@ class ConnectionServiceImpl( limit: Int, offset: Int, includeDeleted: Boolean, + authorization: String?, userInfo: String?, ): ConnectionsResponse { val pagination: Pagination = Pagination().pageSize(limit).rowOffset(offset) @@ -245,7 +255,7 @@ class ConnectionServiceImpl( val response = try { - configApiClient.listConnectionsForWorkspaces(listConnectionsForWorkspacesRequestBody, userInfo) + configApiClient.listConnectionsForWorkspaces(listConnectionsForWorkspacesRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for listConnectionsForWorkspaces: ", e) e.response as HttpResponse diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/DestinationService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/DestinationService.kt index 582b4909d7a..f472f7dea34 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/DestinationService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/DestinationService.kt @@ -39,28 +39,33 @@ interface DestinationService { fun createDestination( destinationCreateRequest: DestinationCreateRequest, destinationDefinitionId: UUID, + authorization: String?, userInfo: String?, ): DestinationResponse fun getDestination( destinationId: UUID, + authorization: String?, userInfo: String?, ): DestinationResponse fun updateDestination( destinationId: UUID, destinationPutRequest: DestinationPutRequest, + authorization: String?, userInfo: String?, ): DestinationResponse fun partialUpdateDestination( destinationId: UUID, destinationPatchRequest: DestinationPatchRequest, + authorization: String?, userInfo: String?, ): DestinationResponse fun deleteDestination( connectionId: UUID, + authorization: String?, userInfo: String?, ) @@ -69,16 +74,19 @@ interface DestinationService { includeDeleted: Boolean = false, limit: Int = 20, offset: Int = 0, + authorization: String?, userInfo: String?, ): DestinationsResponse? fun getDestinationSyncModes( destinationId: UUID, + authorization: String?, userInfo: String?, ): List fun getDestinationSyncModes( destinationResponse: DestinationResponse, + authorization: String?, userInfo: String?, ): List } @@ -99,6 +107,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva override fun createDestination( destinationCreateRequest: DestinationCreateRequest, destinationDefinitionId: UUID, + authorization: String?, userInfo: String?, ): DestinationResponse { val destinationCreateOss = DestinationCreate() @@ -109,7 +118,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva val response = try { - configApiClient.createDestination(destinationCreateOss, userInfo) + configApiClient.createDestination(destinationCreateOss, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for createDestination: ", e) e.response as HttpResponse @@ -124,6 +133,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva */ override fun getDestination( destinationId: UUID, + authorization: String?, userInfo: String?, ): DestinationResponse { val destinationIdRequestBody = DestinationIdRequestBody() @@ -132,7 +142,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva log.info("getDestination request: $destinationIdRequestBody") val response = try { - configApiClient.getDestination(destinationIdRequestBody, userInfo) + configApiClient.getDestination(destinationIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for getDestination: ", e) e.response as HttpResponse @@ -148,6 +158,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva override fun updateDestination( destinationId: UUID, destinationPutRequest: DestinationPutRequest, + authorization: String?, userInfo: String?, ): DestinationResponse { val destinationUpdate = @@ -158,7 +169,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva val response = try { - configApiClient.updateDestination(destinationUpdate, userInfo) + configApiClient.updateDestination(destinationUpdate, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for updateDestination: ", e) e.response as HttpResponse @@ -174,6 +185,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva override fun partialUpdateDestination( destinationId: UUID, destinationPatchRequest: DestinationPatchRequest, + authorization: String?, userInfo: String?, ): DestinationResponse { val partialDestinationUpdate = @@ -184,7 +196,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva val response = try { - configApiClient.partialUpdateDestination(partialDestinationUpdate, userInfo) + configApiClient.partialUpdateDestination(partialDestinationUpdate, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for partialUpdateDestination: ", e) e.response as HttpResponse @@ -199,12 +211,13 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva */ override fun deleteDestination( connectionId: UUID, + authorization: String?, userInfo: String?, ) { val destinationIdRequestBody = DestinationIdRequestBody().destinationId(connectionId) val response = try { - configApiClient.deleteDestination(destinationIdRequestBody, userInfo) + configApiClient.deleteDestination(destinationIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for destination delete: ", e) e.response as HttpResponse @@ -221,6 +234,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva includeDeleted: Boolean, limit: Int, offset: Int, + authorization: String?, userInfo: String?, ): DestinationsResponse { val pagination: Pagination = Pagination().pageSize(limit).rowOffset(offset) @@ -232,7 +246,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva val response = try { - configApiClient.listDestinationsForWorkspaces(listResourcesForWorkspacesRequestBody, userInfo) + configApiClient.listDestinationsForWorkspaces(listResourcesForWorkspacesRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for listWorkspaces: ", e) e.response as HttpResponse @@ -251,14 +265,16 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva override fun getDestinationSyncModes( destinationId: UUID, + authorization: String?, userInfo: String?, ): List { - val destinationResponse: DestinationResponse = getDestination(destinationId, userInfo) - return getDestinationSyncModes(destinationResponse, userInfo) + val destinationResponse: DestinationResponse = getDestination(destinationId, authorization, userInfo) + return getDestinationSyncModes(destinationResponse, authorization, userInfo) } override fun getDestinationSyncModes( destinationResponse: DestinationResponse, + authorization: String?, userInfo: String?, ): List { val destinationDefinitionId: UUID = @@ -268,7 +284,7 @@ class DestinationServiceImpl(private val configApiClient: ConfigApiClient, priva destinationDefinitionIdWithWorkspaceId.workspaceId = destinationResponse.workspaceId var response: HttpResponse try { - response = configApiClient.getDestinationSpec(destinationDefinitionIdWithWorkspaceId, userInfo) + response = configApiClient.getDestinationSpec(destinationDefinitionIdWithWorkspaceId, authorization, userInfo) log.debug(HTTP_RESPONSE_BODY_DEBUG_MESSAGE + response.body()) } catch (e: HttpClientResponseException) { log.error("Config api response error for getDestinationSpec: ", e) diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt index bda70fdbbfb..1718a77bc27 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/JobService.kt @@ -39,21 +39,25 @@ import java.util.UUID interface JobService { fun sync( connectionId: UUID, + authorization: String?, userInfo: String?, ): JobResponse fun reset( connectionId: UUID, + authorization: String?, userInfo: String?, ): JobResponse fun cancelJob( jobId: Long, + authorization: String?, userInfo: String?, ): JobResponse fun getJobInfoWithoutLogs( jobId: Long, + authorization: String?, userInfo: String?, ): JobResponse @@ -62,6 +66,7 @@ interface JobService { jobsFilter: JobsFilter, orderByField: OrderByFieldEnum = OrderByFieldEnum.CREATEDAT, orderByMethod: OrderByMethodEnum = OrderByMethodEnum.DESC, + authorization: String?, userInfo: String?, ): JobsResponse @@ -70,6 +75,7 @@ interface JobService { jobsFilter: JobsFilter, orderByField: OrderByFieldEnum = OrderByFieldEnum.CREATEDAT, orderByMethod: OrderByMethodEnum = OrderByMethodEnum.DESC, + authorization: String?, userInfo: String?, ): JobsResponse } @@ -89,12 +95,13 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi */ override fun sync( connectionId: UUID, + authorization: String?, userInfo: String?, ): JobResponse { val connectionIdRequestBody = ConnectionIdRequestBody().connectionId(connectionId) val response = try { - configApiClient.sync(connectionIdRequestBody, userInfo) + configApiClient.sync(connectionIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for job sync: ", e) e.response as HttpResponse @@ -115,12 +122,13 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi */ override fun reset( connectionId: UUID, + authorization: String?, userInfo: String?, ): JobResponse { val connectionIdRequestBody = ConnectionIdRequestBody().connectionId(connectionId) val response = try { - configApiClient.reset(connectionIdRequestBody, userInfo) + configApiClient.reset(connectionIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for job reset: ", e) e.response as HttpResponse @@ -135,12 +143,13 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi */ override fun cancelJob( jobId: Long, + authorization: String?, userInfo: String?, ): JobResponse { val jobIdRequestBody = JobIdRequestBody().id(jobId) val response = try { - configApiClient.cancelJob(jobIdRequestBody, userInfo) + configApiClient.cancelJob(jobIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for cancelJob: ", e) e.response as HttpResponse @@ -155,12 +164,13 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi */ override fun getJobInfoWithoutLogs( jobId: Long, + authorization: String?, userInfo: String?, ): JobResponse { val jobIdRequestBody = JobIdRequestBody().id(jobId) val response = try { - configApiClient.getJobInfoWithoutLogs(jobIdRequestBody, userInfo) + configApiClient.getJobInfoWithoutLogs(jobIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error( "Config api response error for getJobInfoWithoutLogs: $jobId", @@ -188,6 +198,7 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi jobsFilter: JobsFilter, orderByField: OrderByFieldEnum, orderByMethod: OrderByMethodEnum, + authorization: String?, userInfo: String?, ): JobsResponse { val configTypes: List = getJobConfigTypes(jobsFilter.jobType) @@ -208,7 +219,7 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi val response = try { - configApiClient.getJobList(jobListRequestBody, userInfo) + configApiClient.getJobList(jobListRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for getJobList: ", e) e.response as HttpResponse @@ -233,6 +244,7 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi jobsFilter: JobsFilter, orderByField: OrderByFieldEnum, orderByMethod: OrderByMethodEnum, + authorization: String?, userInfo: String?, ): JobsResponse { val configTypes = getJobConfigTypes(jobsFilter.jobType) @@ -255,7 +267,7 @@ class JobServiceImpl(private val configApiClient: ConfigApiClient, val userServi val response = try { - configApiClient.getJobListForWorkspaces(requestBody, userInfo) + configApiClient.getJobListForWorkspaces(requestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for getJobList: ", e) e.response as HttpResponse diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/OAuthService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/OAuthService.kt index 2017f678644..e78db541ec3 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/OAuthService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/OAuthService.kt @@ -18,6 +18,7 @@ interface OAuthService { definitionId: @NotBlank UUID, redirectUrl: @NotBlank String, oauthInputConfiguration: JsonNode, + authorization: String?, userInfo: String?, ): OAuthConsentRead @@ -27,6 +28,7 @@ interface OAuthService { redirectUrl: @NotBlank String, queryParameters: @NotBlank Map, oauthInputConfiguration: JsonNode, + authorization: String?, userInfo: String?, ): HttpResponse @@ -35,6 +37,7 @@ interface OAuthService { actorType: ActorTypeEnum, definitionId: UUID, oauthCredentialsConfiguration: JsonNode, + authorization: String?, userInfo: String?, ) } diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceDefinitionService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceDefinitionService.kt index a8575ea066f..3c684c14108 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceDefinitionService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceDefinitionService.kt @@ -22,6 +22,7 @@ interface SourceDefinitionService { fun getSourceDefinitionSpecification( sourceDefinitionId: UUID, workspaceId: UUID, + authorization: String?, userInfo: String?, ): SourceDefinitionSpecificationRead? } @@ -36,13 +37,14 @@ class SourceDefinitionServiceImpl(private val configApiClient: ConfigApiClient) override fun getSourceDefinitionSpecification( sourceDefinitionId: UUID, workspaceId: UUID, + authorization: String?, userInfo: String?, ): SourceDefinitionSpecificationRead? { val sourceDefinitionIdWithWorkspaceId = SourceDefinitionIdWithWorkspaceId().sourceDefinitionId(sourceDefinitionId).workspaceId(workspaceId) var response: HttpResponse try { - response = configApiClient.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId, userInfo!!) + response = configApiClient.getSourceDefinitionSpecification(sourceDefinitionIdWithWorkspaceId, authorization, userInfo!!) } catch (e: HttpClientResponseException) { log.error("Config api response error for cancelJob: ", e) response = e.response as HttpResponse diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceService.kt index beced164a07..b2b777c4505 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/SourceService.kt @@ -41,34 +41,40 @@ interface SourceService { fun createSource( sourceCreateRequest: SourceCreateRequest, sourceDefinitionId: UUID, + authorization: String?, userInfo: String?, ): SourceResponse fun updateSource( sourceId: UUID, sourcePutRequest: SourcePutRequest, + authorization: String?, userInfo: String?, ): SourceResponse fun partialUpdateSource( sourceId: UUID, sourcePatchRequest: SourcePatchRequest, + authorization: String?, userInfo: String?, ): SourceResponse fun deleteSource( sourceId: UUID, + authorization: String?, userInfo: String?, ) fun getSource( sourceId: UUID, + authorization: String?, userInfo: String?, ): SourceResponse fun getSourceSchema( sourceId: UUID, disableCache: Boolean, + authorization: String?, userInfo: String?, ): SourceDiscoverSchemaRead @@ -77,11 +83,13 @@ interface SourceService { includeDeleted: Boolean = false, limit: Int = 20, offset: Int = 0, + authorization: String?, userInfo: String?, ): SourcesResponse fun controllerInitiateOAuth( initiateOauthRequest: InitiateOauthRequest?, + authorization: String?, userInfo: String?, ): Response } @@ -105,6 +113,7 @@ open class SourceServiceImpl( override fun createSource( sourceCreateRequest: SourceCreateRequest, sourceDefinitionId: UUID, + authorization: String?, userInfo: String?, ): SourceResponse { val sourceCreateOss = SourceCreate() @@ -116,7 +125,7 @@ open class SourceServiceImpl( val response = try { - configApiClient.createSource(sourceCreateOss, userInfo) + configApiClient.createSource(sourceCreateOss, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for createSource: ", e) e.response as HttpResponse @@ -133,6 +142,7 @@ open class SourceServiceImpl( override fun updateSource( sourceId: UUID, sourcePutRequest: SourcePutRequest, + authorization: String?, userInfo: String?, ): SourceResponse { val sourceUpdate = @@ -143,7 +153,7 @@ open class SourceServiceImpl( val response = try { - configApiClient.updateSource(sourceUpdate, userInfo) + configApiClient.updateSource(sourceUpdate, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for updateSource: ", e) e.response as HttpResponse @@ -160,6 +170,7 @@ open class SourceServiceImpl( override fun partialUpdateSource( sourceId: UUID, sourcePatchRequest: SourcePatchRequest, + authorization: String?, userInfo: String?, ): SourceResponse { val sourceUpdate = @@ -171,7 +182,7 @@ open class SourceServiceImpl( val response = try { - configApiClient.partialUpdateSource(sourceUpdate, userInfo) + configApiClient.partialUpdateSource(sourceUpdate, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for partialUpdateSource: ", e) e.response as HttpResponse @@ -187,12 +198,13 @@ open class SourceServiceImpl( */ override fun deleteSource( sourceId: UUID, + authorization: String?, userInfo: String?, ) { val sourceIdRequestBody = SourceIdRequestBody().sourceId(sourceId) val response = try { - configApiClient.deleteSource(sourceIdRequestBody, userInfo) + configApiClient.deleteSource(sourceIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for source delete: ", e) e.response as HttpResponse @@ -206,13 +218,14 @@ open class SourceServiceImpl( */ override fun getSource( sourceId: UUID, + authorization: String?, userInfo: String?, ): SourceResponse { val sourceIdRequestBody = SourceIdRequestBody() sourceIdRequestBody.sourceId = sourceId val response = try { - configApiClient.getSource(sourceIdRequestBody, userInfo) + configApiClient.getSource(sourceIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for getSource: ", e) e.response as HttpResponse @@ -228,13 +241,14 @@ open class SourceServiceImpl( override fun getSourceSchema( sourceId: UUID, disableCache: Boolean, + authorization: String?, userInfo: String?, ): SourceDiscoverSchemaRead { val sourceDiscoverSchemaRequestBody = SourceDiscoverSchemaRequestBody().sourceId(sourceId).disableCache(disableCache) val response: HttpResponse = try { - configApiClient.getSourceSchema(sourceDiscoverSchemaRequestBody, userInfo) + configApiClient.getSourceSchema(sourceDiscoverSchemaRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for getSourceSchema: ", e) e.response as HttpResponse @@ -272,6 +286,7 @@ open class SourceServiceImpl( includeDeleted: Boolean, limit: Int, offset: Int, + authorization: String?, userInfo: String?, ): SourcesResponse { val pagination: Pagination = Pagination().pageSize(limit).rowOffset(offset) @@ -283,7 +298,7 @@ open class SourceServiceImpl( val response = try { - configApiClient.listSourcesForWorkspaces(listResourcesForWorkspacesRequestBody, userInfo) + configApiClient.listSourcesForWorkspaces(listResourcesForWorkspacesRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for listWorkspaces: ", e) e.response as HttpResponse @@ -302,6 +317,7 @@ open class SourceServiceImpl( override fun controllerInitiateOAuth( initiateOauthRequest: InitiateOauthRequest?, + authorization: String?, userInfo: String?, ): Response { return Response.status(Response.Status.NOT_IMPLEMENTED).build() diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/UserService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/UserService.kt index 3f0dccd881e..6bc3b7a0bfa 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/UserService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/UserService.kt @@ -35,7 +35,7 @@ open class UserServiceImpl(private val configApiClient: ConfigApiClient) : UserS override fun getAllWorkspaceIdsForUser(userInfo: String?): List { val response = try { - configApiClient.listAllWorkspaces(System.getenv(AIRBYTE_API_AUTH_HEADER_VALUE)) + configApiClient.listAllWorkspaces(System.getenv(AIRBYTE_API_AUTH_HEADER_VALUE), null) } catch (e: HttpClientResponseException) { log.error("Cloud api response error for listWorkspacesByUser: ", e) e.response as HttpResponse diff --git a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/WorkspaceService.kt b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/WorkspaceService.kt index 464908c724a..ba786a380ec 100644 --- a/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/WorkspaceService.kt +++ b/airbyte-api-server/src/main/kotlin/io/airbyte/api/server/services/WorkspaceService.kt @@ -16,7 +16,6 @@ import io.airbyte.api.client.model.generated.WorkspaceIdRequestBody import io.airbyte.api.client.model.generated.WorkspaceRead import io.airbyte.api.client.model.generated.WorkspaceReadList import io.airbyte.api.server.apiTracking.TrackingHelper -import io.airbyte.api.server.constants.AIRBYTE_API_AUTH_HEADER_VALUE import io.airbyte.api.server.constants.DELETE import io.airbyte.api.server.constants.GET import io.airbyte.api.server.constants.HTTP_RESPONSE_BODY_DEBUG_MESSAGE @@ -41,43 +40,51 @@ import javax.ws.rs.core.Response interface WorkspaceService { fun createWorkspace( workspaceCreateRequest: WorkspaceCreateRequest, + authorization: String?, userInfo: String?, ): WorkspaceResponse fun controllerCreateWorkspace( workspaceCreateRequest: WorkspaceCreateRequest, + authorization: String?, userInfo: String?, ): Response fun updateWorkspace( workspaceId: UUID, workspaceUpdateRequest: WorkspaceUpdateRequest, + authorization: String?, userInfo: String?, ): WorkspaceResponse fun controllerUpdateWorkspace( workspaceId: UUID, workspaceUpdateRequest: WorkspaceUpdateRequest, + authorization: String?, userInfo: String?, ): Response fun getWorkspace( workspaceId: UUID, + authorization: String?, userInfo: String?, ): WorkspaceResponse fun controllerGetWorkspace( workspaceId: UUID, + authorization: String?, userInfo: String?, ): Response fun deleteWorkspace( workspaceId: UUID, + authorization: String?, userInfo: String?, ) fun controllerDeleteWorkspace( workspaceId: UUID, + authorization: String?, userInfo: String?, ): Response @@ -86,6 +93,7 @@ interface WorkspaceService { includeDeleted: Boolean = false, limit: Int = 20, offset: Int = 0, + authorization: String?, userInfo: String?, ): WorkspacesResponse @@ -94,12 +102,14 @@ interface WorkspaceService { includeDeleted: Boolean = false, limit: Int = 20, offset: Int = 0, + authorization: String?, userInfo: String?, ): Response fun controllerSetWorkspaceOverrideOAuthParams( workspaceId: UUID?, workspaceOAuthCredentialsRequest: WorkspaceOAuthCredentialsRequest?, + authorization: String?, userInfo: String?, ): Response } @@ -123,12 +133,13 @@ open class WorkspaceServiceImpl( */ override fun createWorkspace( workspaceCreateRequest: WorkspaceCreateRequest, + authorization: String?, userInfo: String?, ): WorkspaceResponse { val workspaceCreate = WorkspaceCreate().name(workspaceCreateRequest.name) val workspaceReadHttpResponse = try { - configApiClient.createWorkspace(workspaceCreate, System.getenv(AIRBYTE_API_AUTH_HEADER_VALUE)) + configApiClient.createWorkspace(workspaceCreate, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for createWorkspace: ", e) e.response as HttpResponse @@ -144,13 +155,14 @@ open class WorkspaceServiceImpl( override fun controllerCreateWorkspace( workspaceCreateRequest: WorkspaceCreateRequest, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) val workspaceResponse: WorkspaceResponse = trackingHelper.callWithTracker( - { createWorkspace(workspaceCreateRequest, userInfo) }, + { createWorkspace(workspaceCreateRequest, authorization, userInfo) }, WORKSPACES_PATH, POST, userId, @@ -173,6 +185,7 @@ open class WorkspaceServiceImpl( override fun updateWorkspace( workspaceId: UUID, workspaceUpdateRequest: WorkspaceUpdateRequest, + authorization: String?, userInfo: String?, ): WorkspaceResponse { // Update workspace in the cloud version of the airbyte API currently only supports name updates, but we don't have name updates in OSS. @@ -182,6 +195,7 @@ open class WorkspaceServiceImpl( override fun controllerUpdateWorkspace( workspaceId: UUID, workspaceUpdateRequest: WorkspaceUpdateRequest, + authorization: String?, userInfo: String?, ): Response { return Response.status(Response.Status.NOT_IMPLEMENTED).build() @@ -192,13 +206,14 @@ open class WorkspaceServiceImpl( */ override fun getWorkspace( workspaceId: UUID, + authorization: String?, userInfo: String?, ): WorkspaceResponse { val workspaceIdRequestBody = WorkspaceIdRequestBody() workspaceIdRequestBody.workspaceId = workspaceId val response = try { - configApiClient.getWorkspace(workspaceIdRequestBody, userInfo) + configApiClient.getWorkspace(workspaceIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for getWorkspace: ", e) e.response as HttpResponse @@ -210,6 +225,7 @@ open class WorkspaceServiceImpl( override fun controllerGetWorkspace( workspaceId: UUID, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -219,6 +235,7 @@ open class WorkspaceServiceImpl( { getWorkspace( workspaceId, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -242,13 +259,14 @@ open class WorkspaceServiceImpl( */ override fun deleteWorkspace( workspaceId: UUID, + authorization: String?, userInfo: String?, ) { val workspaceIdRequestBody = WorkspaceIdRequestBody() workspaceIdRequestBody.workspaceId = workspaceId val response = try { - configApiClient.deleteWorkspace(workspaceIdRequestBody, userInfo) + configApiClient.deleteWorkspace(workspaceIdRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for deleteWorkspace: ", e) e.response as HttpResponse @@ -259,6 +277,7 @@ open class WorkspaceServiceImpl( override fun controllerDeleteWorkspace( workspaceId: UUID, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -268,6 +287,7 @@ open class WorkspaceServiceImpl( { deleteWorkspace( workspaceId!!, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -289,6 +309,7 @@ open class WorkspaceServiceImpl( includeDeleted: Boolean, limit: Int, offset: Int, + authorization: String?, userInfo: String?, ): WorkspacesResponse { val pagination: Pagination = Pagination().pageSize(limit).rowOffset(offset) @@ -301,7 +322,7 @@ open class WorkspaceServiceImpl( listResourcesForWorkspacesRequestBody.workspaceIds = workspaceIdsToQuery val response = try { - configApiClient.listWorkspaces(listResourcesForWorkspacesRequestBody, userInfo) + configApiClient.listWorkspaces(listResourcesForWorkspacesRequestBody, authorization, userInfo) } catch (e: HttpClientResponseException) { log.error("Config api response error for listWorkspaces: ", e) e.response as HttpResponse @@ -323,6 +344,7 @@ open class WorkspaceServiceImpl( includeDeleted: Boolean, limit: Int, offset: Int, + authorization: String?, userInfo: String?, ): Response { val userId: UUID = userService.getUserIdFromUserInfoString(userInfo) @@ -337,6 +359,7 @@ open class WorkspaceServiceImpl( includeDeleted, limit, offset, + authorization, getLocalUserInfoIfNull(userInfo), ) }, @@ -358,6 +381,7 @@ open class WorkspaceServiceImpl( override fun controllerSetWorkspaceOverrideOAuthParams( workspaceId: UUID?, workspaceOAuthCredentialsRequest: WorkspaceOAuthCredentialsRequest?, + authorization: String?, userInfo: String?, ): Response { return Response.status(Response.Status.NOT_IMPLEMENTED).build() diff --git a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/apiInterface.mustache b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/apiInterface.mustache index 2357ebf76fb..0f4a7b1eecc 100644 --- a/airbyte-api/src/main/resources/templates/jaxrs-spec-api/apiInterface.mustache +++ b/airbyte-api/src/main/resources/templates/jaxrs-spec-api/apiInterface.mustache @@ -17,4 +17,4 @@ {{/implicitHeadersParams.0}} @ApiResponses(value = { {{#responses}} @ApiResponse(code = {{{code}}}, message = "{{{message}}}", response = {{{baseType}}}.class{{#returnContainer}}, responseContainer = "{{{.}}}"{{/returnContainer}}){{^-last}},{{/-last}}{{/responses}} }){{/useSwaggerAnnotations}} - {{#supportAsync}}{{>returnAsyncTypeInterface}}{{/supportAsync}}{{^supportAsync}}{{#returnResponse}}Response{{/returnResponse}}{{^returnResponse}}{{>returnTypeInterface}}{{/returnResponse}}{{/supportAsync}} {{nickname}}({{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{>formParams}}, {{/allParams}}@HeaderParam("X-Endpoint-API-UserInfo") final String userInfo); + {{#supportAsync}}{{>returnAsyncTypeInterface}}{{/supportAsync}}{{^supportAsync}}{{#returnResponse}}Response{{/returnResponse}}{{^returnResponse}}{{>returnTypeInterface}}{{/returnResponse}}{{/supportAsync}} {{nickname}}({{#allParams}}{{>queryParams}}{{>pathParams}}{{>headerParams}}{{>bodyParams}}{{>formParams}}, {{/allParams}}@HeaderParam("Authorization") final String authorization, @HeaderParam("X-Endpoint-API-UserInfo") final String userInfo); From 045fe79404dc24794eda66b7bc3cf4817b849659 Mon Sep 17 00:00:00 2001 From: Chandler Prall Date: Tue, 12 Dec 2023 13:28:03 -0700 Subject: [PATCH 11/27] =?UTF-8?q?=F0=9F=AA=9F=C2=A0=F0=9F=8E=89=20uptime?= =?UTF-8?q?=20&=20data=20moved=20connection=20status=20graphs=20(#10253)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../server/handlers/ConnectionsHandler.java | 8 +- .../handlers/ConnectionsHandlerTest.java | 4 +- .../handlers/StreamStatusesHandler.java | 6 +- .../StreamStatusesRepository.java | 12 +- .../handlers/StreamStatusesHandlerTest.java | 14 +- .../StreamStatusesRepositoryTest.java | 80 +++----- .../DataMovedGraph/DataMovedGraph.tsx | 30 ++- .../components/DataMovedGraph/constants.ts | 4 + .../HistoricalOverview/HistoricalOverview.tsx | 30 ++- .../HistoricalOverview/NoDataMessage.tsx | 11 ++ .../UptimeStatusGraph/UptimeStatusGraph.tsx | 183 ++++++++++++++---- .../UptimeStatusGraph/WaffleChart.tsx | 5 +- .../utils/computeStreamStatus.test.ts | 19 +- .../connection/utils/computeStreamStatus.ts | 7 +- .../src/core/api/hooks/connections.tsx | 32 +++ airbyte-webapp/src/locales/en.json | 5 +- 16 files changed, 313 insertions(+), 137 deletions(-) create mode 100644 airbyte-webapp/src/area/connection/components/DataMovedGraph/constants.ts create mode 100644 airbyte-webapp/src/area/connection/components/HistoricalOverview/NoDataMessage.tsx diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java index 4af7b51212b..d9070925d61 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/ConnectionsHandler.java @@ -946,9 +946,11 @@ public List getConnectionStatuses( public List getConnectionDataHistory(final ConnectionDataHistoryRequestBody connectionDataHistoryRequestBody) throws IOException { + final ZoneId requestZone = ZoneId.of(connectionDataHistoryRequestBody.getTimezone()); + // Start time in designated timezone final ZonedDateTime endTimeInUserTimeZone = Instant.now().atZone(ZoneId.of(connectionDataHistoryRequestBody.getTimezone())); - final ZonedDateTime startTimeInUserTimeZone = endTimeInUserTimeZone.minusDays(30); + final ZonedDateTime startTimeInUserTimeZone = endTimeInUserTimeZone.toLocalDate().atStartOfDay(requestZone).minusDays(29); // Convert start time to UTC (since that's what the database uses) final Instant startTimeInUTC = startTimeInUserTimeZone.toInstant(); @@ -963,7 +965,7 @@ public List getConnectionDataHistory(final Connec final LocalDate endDate = endTimeInUserTimeZone.toLocalDate(); for (LocalDate date = startDate; !date.isAfter(endDate); date = date.plusDays(1)) { connectionDataHistoryReadItemsByDate.put(date, new ConnectionDataHistoryReadItem() - .timestamp(Math.toIntExact(date.atStartOfDay(ZoneId.of(connectionDataHistoryRequestBody.getTimezone())).toEpochSecond())) + .timestamp(Math.toIntExact(date.atStartOfDay(requestZone).toEpochSecond())) .bytes(0)); } @@ -973,7 +975,7 @@ public List getConnectionDataHistory(final Connec if (endedAtOptional.isPresent()) { // Convert the endedAt timestamp from the database to the designated timezone final Instant attemptEndedAt = Instant.ofEpochSecond(endedAtOptional.get()); - final LocalDate attemptDateInUserTimeZone = attemptEndedAt.atZone(ZoneId.of(connectionDataHistoryRequestBody.getTimezone())) + final LocalDate attemptDateInUserTimeZone = attemptEndedAt.atZone(requestZone) .toLocalDate(); // Merge it with the bytes synced from the attempt diff --git a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java index 0ffa7949977..c9c9c8aa265 100644 --- a/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java +++ b/airbyte-commons-server/src/test/java/io/airbyte/commons/server/handlers/ConnectionsHandlerTest.java @@ -1487,7 +1487,7 @@ void testDataHistoryWithEmptyResponse() throws IOException { final ConnectionDataHistoryRequestBody requestBody = new ConnectionDataHistoryRequestBody() .connectionId(connectionRead.getConnectionId()) .timezone(TIMEZONE_LOS_ANGELES); - final LocalDate startDate = Instant.now().atZone(ZoneId.of(requestBody.getTimezone())).minusDays(30).toLocalDate(); + final LocalDate startDate = Instant.now().atZone(ZoneId.of(requestBody.getTimezone())).minusDays(29).toLocalDate(); final LocalDate endDate = LocalDate.now(ZoneId.of(requestBody.getTimezone())); final List actual = connectionsHandler.getConnectionDataHistory(requestBody); @@ -1504,7 +1504,7 @@ void testDataHistoryWithEmptyResponse() throws IOException { void testDataHistoryAggregation() throws IOException { final UUID connectionId = UUID.randomUUID(); final Instant endTime = Instant.now(); - final Instant startTime = endTime.minus(30, ChronoUnit.DAYS); + final Instant startTime = endTime.minus(29, ChronoUnit.DAYS); final long attempt1Bytes = 100L; final long attempt2Bytes = 150L; final long attempt3Bytes = 200L; diff --git a/airbyte-server/src/main/java/io/airbyte/server/handlers/StreamStatusesHandler.java b/airbyte-server/src/main/java/io/airbyte/server/handlers/StreamStatusesHandler.java index cd0a81ec688..bf8a966a91c 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/handlers/StreamStatusesHandler.java +++ b/airbyte-server/src/main/java/io/airbyte/server/handlers/StreamStatusesHandler.java @@ -22,7 +22,6 @@ import java.time.LocalDate; import java.time.OffsetDateTime; import java.time.ZoneId; -import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.Comparator; import java.util.List; @@ -100,11 +99,10 @@ public List getConnectionUptimeHistory(final Connectio .minusDays(30) .toLocalDate() .atStartOfDay(ZoneId.of(req.getTimezone())) - .withZoneSameInstant(ZoneOffset.UTC) .toOffsetDateTime(); - final var streamStatuses = repo.findLatestStatusPerRunStateByConnectionIdAndDayAfterTimestamp(req.getConnectionId(), - thirtyDaysAgoInUTC) + final var streamStatuses = repo.findLatestStatusPerStreamByConnectionIdAndDayAfterTimestamp(req.getConnectionId(), + thirtyDaysAgoInUTC, req.getTimezone()) .stream() .map(mapper::map) .toList(); diff --git a/airbyte-server/src/main/java/io/airbyte/server/repositories/StreamStatusesRepository.java b/airbyte-server/src/main/java/io/airbyte/server/repositories/StreamStatusesRepository.java index a6c5bfca6ac..218426bd675 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/repositories/StreamStatusesRepository.java +++ b/airbyte-server/src/main/java/io/airbyte/server/repositories/StreamStatusesRepository.java @@ -79,13 +79,15 @@ default Page findAllFiltered(final FilterParams params) { List findAllPerRunStateByConnectionId(final UUID connectionId); /** - * Returns the latest stream status per run state per day for a connection after a given timestamp. + * Returns the latest stream status per stream per day for a connection after a given timestamp. */ - @Query("SELECT DISTINCT ON (stream_name, stream_namespace, run_state, incomplete_run_cause, job_type, DATE(transitioned_at)) * " - + "FROM stream_statuses WHERE connection_id = :connectionId AND transitioned_at > :timestamp " - + "ORDER BY stream_name, stream_namespace, run_state, incomplete_run_cause, job_type, DATE(transitioned_at), transitioned_at DESC") - List findLatestStatusPerRunStateByConnectionIdAndDayAfterTimestamp(final UUID connectionId, final OffsetDateTime timestamp); + @Query("SELECT DISTINCT ON (stream_name, stream_namespace, 1) DATE_TRUNC('day', transitioned_at, :timezone), * " + + "FROM stream_statuses WHERE connection_id = :connectionId AND transitioned_at >= :timestamp " + + "ORDER BY stream_name, stream_namespace, 1, transitioned_at DESC") + List findLatestStatusPerStreamByConnectionIdAndDayAfterTimestamp(final UUID connectionId, + final OffsetDateTime timestamp, + final String timezone); /** * Pagination params. diff --git a/airbyte-server/src/test/java/io/airbyte/server/handlers/StreamStatusesHandlerTest.java b/airbyte-server/src/test/java/io/airbyte/server/handlers/StreamStatusesHandlerTest.java index a8843ac9780..82360262060 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/handlers/StreamStatusesHandlerTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/handlers/StreamStatusesHandlerTest.java @@ -23,7 +23,6 @@ import java.time.Instant; import java.time.OffsetDateTime; import java.time.ZoneId; -import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.util.List; import java.util.UUID; @@ -119,10 +118,10 @@ void testListPerRunState() { @Test void testGetConnectionUptimeHistory() { final UUID connectionId = UUID.randomUUID(); - final String timezone = "America/Los_Angeles"; + final ZoneId timezone = ZoneId.systemDefault(); final ConnectionUptimeHistoryRequestBody apiReq = new ConnectionUptimeHistoryRequestBody() .connectionId(connectionId) - .timezone(timezone); + .timezone(timezone.getId()); final StreamStatus domainItem = StreamStatus.builder().build(); final StreamStatusRead apiItem = new StreamStatusRead(); @@ -130,17 +129,16 @@ void testGetConnectionUptimeHistory() { apiItem.setRunState(StreamStatusRunState.COMPLETE); final List expected = List.of( - handler.mapStreamStatusToSyncReadResult(apiItem, ZoneId.of(timezone))); + handler.mapStreamStatusToSyncReadResult(apiItem, timezone)); // Calculate 30 days ago in the specified timezone, then convert to UTC OffsetDateTime - final OffsetDateTime thirtyDaysAgoInSpecifiedTZ = ZonedDateTime.now(ZoneId.of(timezone)) + final OffsetDateTime thirtyDaysAgoInSpecifiedTZ = ZonedDateTime.now(timezone) .minusDays(30) .toLocalDate() - .atStartOfDay(ZoneId.of(timezone)) - .withZoneSameInstant(ZoneOffset.UTC) + .atStartOfDay(timezone) .toOffsetDateTime(); - when(repo.findLatestStatusPerRunStateByConnectionIdAndDayAfterTimestamp(connectionId, thirtyDaysAgoInSpecifiedTZ)) + when(repo.findLatestStatusPerStreamByConnectionIdAndDayAfterTimestamp(connectionId, thirtyDaysAgoInSpecifiedTZ, timezone.getId())) .thenReturn(List.of(domainItem)); when(mapper.map(domainItem)) .thenReturn(apiItem); diff --git a/airbyte-server/src/test/java/io/airbyte/server/repositories/StreamStatusesRepositoryTest.java b/airbyte-server/src/test/java/io/airbyte/server/repositories/StreamStatusesRepositoryTest.java index bbaedde2353..af5aad16a60 100644 --- a/airbyte-server/src/test/java/io/airbyte/server/repositories/StreamStatusesRepositoryTest.java +++ b/airbyte-server/src/test/java/io/airbyte/server/repositories/StreamStatusesRepositoryTest.java @@ -94,7 +94,7 @@ static void dbDown() { // Aliasing to cut down on the verbosity significantly private static void assertContainsSameElements(final List expected, final List actual) { - org.assertj.core.api.Assertions.assertThat(expected).containsExactlyInAnyOrderElementsOf(actual); + org.assertj.core.api.Assertions.assertThat(actual).containsExactlyInAnyOrderElementsOf(expected); } @BeforeEach @@ -393,61 +393,41 @@ void testFindAllPerRunStateByConnectionId() { } @Test - void testFindLatestStatusPerRunStateByConnectionIdAndDayAfterTimestamp() { + void testFindLatestStatusPerStreamByConnectionIdAndDayAfterTimestamp() { final long now = Instant.now().toEpochMilli(); final OffsetDateTime time1 = Fixtures.timestamp(now); final OffsetDateTime time2 = Fixtures.timestamp(now + 1); final OffsetDateTime time3 = Fixtures.timestamp(now + 2); - final OffsetDateTime time4 = Fixtures.timestamp(now + 3); - // Two connections with some combination of 4 types of statuses. Each connection should return the - // most recent example of each status PER STREAM. - // connection 1 should return p3, connection 2 will return p4 + // connection 1 final var p1 = Fixtures.pending().transitionedAt(time1).connectionId(Fixtures.connectionId1).build(); - final var p2 = Fixtures.pending().transitionedAt(time2).connectionId(Fixtures.connectionId1).streamName(Fixtures.name2).build(); - final var p3 = Fixtures.pending().transitionedAt(time3).connectionId(Fixtures.connectionId1).build(); - final var p4 = Fixtures.pending().transitionedAt(time4).connectionId(Fixtures.connectionId2).build(); - - // connection 1 should return r2, connection 2 should return r3, r4 - final var r1 = Fixtures.running().transitionedAt(time1).connectionId(Fixtures.connectionId1).build(); - final var r2 = Fixtures.running().transitionedAt(time2).connectionId(Fixtures.connectionId1).build(); - final var r3 = Fixtures.running().transitionedAt(time3).connectionId(Fixtures.connectionId2).streamName(Fixtures.name2).build(); - final var r4 = Fixtures.running().transitionedAt(time4).connectionId(Fixtures.connectionId2).build(); - - // connection 1 should not return any from this set, connection 2 should return c3, c4 - final var c1 = Fixtures.complete().transitionedAt(time1).connectionId(Fixtures.connectionId1).build(); - final var c2 = Fixtures.complete().transitionedAt(time2).connectionId(Fixtures.connectionId2).build(); - final var c3 = Fixtures.complete().transitionedAt(time3).connectionId(Fixtures.connectionId2).streamName(Fixtures.name2).build(); - final var c4 = Fixtures.complete().transitionedAt(time4).connectionId(Fixtures.connectionId2).build(); - - // connection 1 should return if3, if4, connection 2 should return if2 - final var if1 = Fixtures.failed().transitionedAt(time1).connectionId(Fixtures.connectionId2).build(); - final var if2 = Fixtures.failed().transitionedAt(time2).connectionId(Fixtures.connectionId2).build(); - final var if3 = Fixtures.failed().transitionedAt(time3).connectionId(Fixtures.connectionId1).streamNamespace("test2_").build(); - final var if4 = Fixtures.failed().transitionedAt(time4).connectionId(Fixtures.connectionId1).build(); - - // connection 1 should return ic3, ic4, connection 2 will not return any from this set - final var ic1 = Fixtures.canceled().transitionedAt(time1).connectionId(Fixtures.connectionId1).build(); - final var ic2 = Fixtures.canceled().transitionedAt(time2).connectionId(Fixtures.connectionId1).streamName(Fixtures.name2).build(); - final var ic3 = Fixtures.canceled().transitionedAt(time3).connectionId(Fixtures.connectionId1).streamName(Fixtures.name2).build(); - final var ic4 = Fixtures.canceled().transitionedAt(time4).connectionId(Fixtures.connectionId1).build(); - - // connection 1 should only return reset4 from this set, connection 2 will not return any from this - // set - final var reset1 = Fixtures.reset().transitionedAt(time1).connectionId(Fixtures.connectionId1).build(); - final var reset2 = Fixtures.reset().transitionedAt(time2).connectionId(Fixtures.connectionId1).build(); - final var reset3 = Fixtures.reset().transitionedAt(time3).connectionId(Fixtures.connectionId1).build(); - final var reset4 = Fixtures.reset().transitionedAt(time4).connectionId(Fixtures.connectionId1).build(); - - repo.saveAll(List.of(p1, p2, p3, p4, r1, r2, r3, r4, c1, c2, c3, c4, if1, if2, if3, if4, ic1, ic2, ic3, ic4, reset1, reset2, reset3, reset4)); - - final var results1 = repo.findLatestStatusPerRunStateByConnectionIdAndDayAfterTimestamp(Fixtures.connectionId1, - time1); - final var results2 = repo.findLatestStatusPerRunStateByConnectionIdAndDayAfterTimestamp(Fixtures.connectionId2, - time1); - - assertContainsSameElements(List.of(p2, p3, r2, if3, if4, ic3, ic4, reset4), results1); - assertContainsSameElements(List.of(p4, r3, r4, if2, c3, c4), results2); + final var c1 = Fixtures.complete().transitionedAt(time3).connectionId(Fixtures.connectionId1).build(); + + final var c2 = Fixtures.complete().transitionedAt(time2).connectionId(Fixtures.connectionId1).streamName(Fixtures.name2).build(); + final var r1 = Fixtures.reset().transitionedAt(time3).connectionId(Fixtures.connectionId1).streamName(Fixtures.name2).build(); + + final var p2 = Fixtures.pending().transitionedAt(time1).connectionId(Fixtures.connectionId1).streamName(Fixtures.name3).build(); + final var f1 = Fixtures.failed().transitionedAt(time2).connectionId(Fixtures.connectionId1).streamName(Fixtures.name3).build(); + final var r2 = Fixtures.reset().transitionedAt(time3).connectionId(Fixtures.connectionId1).streamName(Fixtures.name3).build(); + + // connection 2 + final var p3 = Fixtures.pending().transitionedAt(time1).connectionId(Fixtures.connectionId2).build(); + + final var r3 = Fixtures.reset().transitionedAt(time2).connectionId(Fixtures.connectionId2).streamName(Fixtures.name2).build(); + final var f2 = Fixtures.failed().transitionedAt(time3).connectionId(Fixtures.connectionId2).streamName(Fixtures.name2).build(); + + final var c3 = Fixtures.complete().transitionedAt(time1).connectionId(Fixtures.connectionId2).streamName(Fixtures.name3).build(); + final var f3 = Fixtures.failed().transitionedAt(time2).connectionId(Fixtures.connectionId2).streamName(Fixtures.name3).build(); + + repo.saveAll(List.of(p1, p2, p3, r1, r2, r3, c1, c2, c3, f1, f2, f3, r1, r2, r3)); + + final var results1 = repo.findLatestStatusPerStreamByConnectionIdAndDayAfterTimestamp(Fixtures.connectionId1, + time1, ZoneId.systemDefault().getId()); + final var results2 = repo.findLatestStatusPerStreamByConnectionIdAndDayAfterTimestamp(Fixtures.connectionId2, + time1, ZoneId.systemDefault().getId()); + + assertContainsSameElements(List.of(c1, r1, r2), results1); + assertContainsSameElements(List.of(p3, f2, f3), results2); } private static class Fixtures { diff --git a/airbyte-webapp/src/area/connection/components/DataMovedGraph/DataMovedGraph.tsx b/airbyte-webapp/src/area/connection/components/DataMovedGraph/DataMovedGraph.tsx index 7116997e0e5..cbc71c9bb79 100644 --- a/airbyte-webapp/src/area/connection/components/DataMovedGraph/DataMovedGraph.tsx +++ b/airbyte-webapp/src/area/connection/components/DataMovedGraph/DataMovedGraph.tsx @@ -1,20 +1,36 @@ import byteSize from "byte-size"; +import { useMemo } from "react"; import { Bar, BarChart, ResponsiveContainer, Tooltip, XAxis } from "recharts"; import { Box } from "components/ui/Box"; +import { useGetConnectionDataHistory } from "core/api"; +import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; + +import { CHART_BASE_HEIGHT, CHART_MAX_HEIGHT, CHART_MIN_HEIGHT, CHART_STREAM_ROW_HEIGHT } from "./constants"; import styles from "./DataMovedGraph.module.scss"; import { tooltipConfig, xAxisConfig } from "../HistoricalOverview/ChartConfig"; - -const sampleData: Array<{ date: number; bytes: number }> = []; -for (let i = 1; i <= 30; i++) { - sampleData.push({ date: Date.UTC(2023, 7, i), bytes: Math.round(Math.random() * 1000 + 200) }); -} +import { NoDataMessage } from "../HistoricalOverview/NoDataMessage"; export const DataMovedGraph: React.FC = () => { + const { connection } = useConnectionEditService(); + const data = useGetConnectionDataHistory(connection.connectionId); + const hasData = data.some(({ bytes }) => bytes > 0); + + const formattedData = useMemo(() => data.map(({ timestamp, bytes }) => ({ date: timestamp * 1000, bytes })), [data]); + + const chartHeight = Math.max( + CHART_MIN_HEIGHT, + Math.min(CHART_MAX_HEIGHT, connection.syncCatalog.streams.length * CHART_STREAM_ROW_HEIGHT + CHART_BASE_HEIGHT) + ); + + if (!hasData) { + return ; + } + return ( - - + + diff --git a/airbyte-webapp/src/area/connection/components/DataMovedGraph/constants.ts b/airbyte-webapp/src/area/connection/components/DataMovedGraph/constants.ts new file mode 100644 index 00000000000..4c5dac5f0c9 --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/DataMovedGraph/constants.ts @@ -0,0 +1,4 @@ +export const CHART_MIN_HEIGHT = 50; +export const CHART_MAX_HEIGHT = 175; +export const CHART_STREAM_ROW_HEIGHT = 20; +export const CHART_BASE_HEIGHT = 40; diff --git a/airbyte-webapp/src/area/connection/components/HistoricalOverview/HistoricalOverview.tsx b/airbyte-webapp/src/area/connection/components/HistoricalOverview/HistoricalOverview.tsx index b39ca694685..33eb1d67793 100644 --- a/airbyte-webapp/src/area/connection/components/HistoricalOverview/HistoricalOverview.tsx +++ b/airbyte-webapp/src/area/connection/components/HistoricalOverview/HistoricalOverview.tsx @@ -1,24 +1,32 @@ -import { useState } from "react"; +import { Suspense, useState } from "react"; import { FormattedMessage } from "react-intl"; +import { LoadingPage } from "components"; import { Box } from "components/ui/Box"; import { Tabs, ButtonTab } from "components/ui/Tabs"; +import { useExperiment } from "hooks/services/Experiment"; + import { DataMovedGraph } from "../DataMovedGraph"; import { UptimeStatusGraph } from "../UptimeStatusGraph"; export const HistoricalOverview: React.FC = () => { - const [selectedTab, setSelectedTab] = useState<"uptimeStatus" | "dataMoved">("uptimeStatus"); + const doUseStreamStatuses = useExperiment("connection.streamCentricUI.v2", false); + const [selectedTab, setSelectedTab] = useState<"uptimeStatus" | "dataMoved">( + doUseStreamStatuses ? "uptimeStatus" : "dataMoved" + ); return ( - } - isActive={selectedTab === "uptimeStatus"} - onSelect={() => setSelectedTab("uptimeStatus")} - /> + {doUseStreamStatuses && ( + } + isActive={selectedTab === "uptimeStatus"} + onSelect={() => setSelectedTab("uptimeStatus")} + /> + )} } @@ -27,8 +35,10 @@ export const HistoricalOverview: React.FC = () => { /> - {selectedTab === "uptimeStatus" && } - {selectedTab === "dataMoved" && } + }> + {selectedTab === "uptimeStatus" && } + {selectedTab === "dataMoved" && } + ); diff --git a/airbyte-webapp/src/area/connection/components/HistoricalOverview/NoDataMessage.tsx b/airbyte-webapp/src/area/connection/components/HistoricalOverview/NoDataMessage.tsx new file mode 100644 index 00000000000..d4b9f28cdaf --- /dev/null +++ b/airbyte-webapp/src/area/connection/components/HistoricalOverview/NoDataMessage.tsx @@ -0,0 +1,11 @@ +import { FormattedMessage } from "react-intl"; + +import { Text } from "components/ui/Text"; + +export const NoDataMessage: React.FC = () => ( +
+ + + +
+); diff --git a/airbyte-webapp/src/area/connection/components/UptimeStatusGraph/UptimeStatusGraph.tsx b/airbyte-webapp/src/area/connection/components/UptimeStatusGraph/UptimeStatusGraph.tsx index e08a5490093..52ac9b5c0e8 100644 --- a/airbyte-webapp/src/area/connection/components/UptimeStatusGraph/UptimeStatusGraph.tsx +++ b/airbyte-webapp/src/area/connection/components/UptimeStatusGraph/UptimeStatusGraph.tsx @@ -1,4 +1,5 @@ -import React, { useEffect, useState } from "react"; +import dayjs from "dayjs"; +import React, { useEffect, useMemo, useState } from "react"; import { ResponsiveContainer, Tooltip, XAxis } from "recharts"; // these are not worth typing // eslint-disable-next-line @typescript-eslint/ban-ts-comment @@ -10,48 +11,23 @@ import { formatAxisMap } from "recharts/es6/util/CartesianUtils"; import { ConnectionStatusIndicatorStatus } from "components/connection/ConnectionStatusIndicator"; +import { getStreamKey } from "area/connection/utils"; +import { useGetConnectionUptimeHistory } from "core/api"; +import { JobStatus } from "core/api/types/AirbyteClient"; +import { useConnectionEditService } from "hooks/services/ConnectionEdit/ConnectionEditService"; import { useAirbyteTheme } from "hooks/theme/useAirbyteTheme"; import styles from "./UptimeStatusGraph.module.scss"; import { UptimeStatusGraphTooltip } from "./UptimeStatusGraphTooltip"; -import { UptimeDayEntry, Waffle } from "./WaffleChart"; +import { ChartStream, UptimeDayEntry, Waffle } from "./WaffleChart"; +import { + CHART_BASE_HEIGHT, + CHART_MAX_HEIGHT, + CHART_MIN_HEIGHT, + CHART_STREAM_ROW_HEIGHT, +} from "../DataMovedGraph/constants"; import { tooltipConfig, xAxisConfig } from "../HistoricalOverview/ChartConfig"; - -// Build placeholder data -const STREAMS_COUNT = 20; -const uptimeData: UptimeDayEntry[] = []; -for (let i = 0; i < 30; i++) { - const date = Date.UTC(2023, 7, i); - const streams: (typeof uptimeData)[number]["streams"] = []; - - for (let j = 0; j < STREAMS_COUNT; j++) { - let status: ConnectionStatusIndicatorStatus; - - if (i > 25 && j >= STREAMS_COUNT - 1) { - // disabled last stream on the last four days - status = ConnectionStatusIndicatorStatus.Disabled; - } else if (j === 2 || j === STREAMS_COUNT - 3 || j === Math.floor(STREAMS_COUNT / 2)) { - // second, middle, and third to last are error - status = ConnectionStatusIndicatorStatus.Error; - } else if (j === 4 && i >= 15) { - // 5th is action required - status = ConnectionStatusIndicatorStatus.ActionRequired; - } else if (j === 5 && i >= 10 && i <= 16) { - // 5th was late for a duration - status = ConnectionStatusIndicatorStatus.Late; - } else { - status = ConnectionStatusIndicatorStatus.OnTime; - } - streams.push({ status }); - } - - uptimeData.push({ date, streams }); -} - -const CHART_MIN_HEIGHT = 50; -const CHART_MAX_HEIGHT = 175; -const CHART_STREAM_ROW_HEIGHT = 15; -const CHART_BASE_HEIGHT = 17; +import { NoDataMessage } from "../HistoricalOverview/NoDataMessage"; const StreamChart = generateCategoricalChart({ chartName: "StreamChart", @@ -60,12 +36,125 @@ const StreamChart = generateCategoricalChart({ formatAxisMap, }); +interface SortableStream { + streamNamespace?: string; + streamName: string; +} +const sortStreams = (a: SortableStream, b: SortableStream) => { + const { streamName: nameA, streamNamespace: namespaceA } = a; + const { streamName: nameB, streamNamespace: namespaceB } = b; + + const namespaceCompare = (namespaceA ?? "")?.localeCompare(namespaceB ?? ""); + if (namespaceCompare !== 0) { + return namespaceCompare; + } + + return nameA.localeCompare(nameB); +}; + +function assertNever(_x: never) {} + +const formatDataForChart = (data: ReturnType) => { + // bucket entries by their timestamp and collect all stream identities so we can fill in gaps + const dateBuckets: Record = {}; + const today = dayjs(); + for (let i = 0; i < 30; i++) { + const date = today.subtract(i, "day").startOf("day").unix(); + dateBuckets[date] = []; + } + + const { bucketedEntries, allStreamIdentities } = data.reduce<{ + bucketedEntries: typeof dateBuckets; + allStreamIdentities: Map; + }>( + (acc, entry) => { + // not destructuring to avoid creating new objects when returning + const bucketedEntries = acc.bucketedEntries; + const allStreamIdentities = acc.allStreamIdentities; + + // add this entry to its bucket + if (bucketedEntries.hasOwnProperty(entry.timestamp)) { + let status: ConnectionStatusIndicatorStatus = ConnectionStatusIndicatorStatus.Pending; + + switch (entry.status) { + case JobStatus.succeeded: + status = ConnectionStatusIndicatorStatus.OnTime; + break; + case JobStatus.failed: + status = ConnectionStatusIndicatorStatus.Error; + break; + case JobStatus.running: + case JobStatus.cancelled: + case JobStatus.incomplete: + case JobStatus.pending: + status = ConnectionStatusIndicatorStatus.Pending; + break; + default: + assertNever(entry.status); + } + bucketedEntries[entry.timestamp].push({ + streamNamespace: entry.streamNamespace, + streamName: entry.streamName, + status, + }); + } + + // add this stream to the map + const streamKey = getStreamKey(entry); + if (allStreamIdentities.has(streamKey) === false) { + allStreamIdentities.set(streamKey, entry); + } + + return acc; + }, + { + bucketedEntries: dateBuckets, + allStreamIdentities: new Map(), + } + ); + + // ensure each date bucket has an entry for each stream and that they align between days + const dateBucketKeys: Array = Object.keys(bucketedEntries); + const streamIdentities = Array.from(allStreamIdentities.values()).sort(sortStreams); + + // entries in the graph's expected format + const uptimeData: UptimeDayEntry[] = []; + + dateBucketKeys.forEach((dateBucketKey) => { + const dateEntries = bucketedEntries[dateBucketKey]; + dateEntries.sort(sortStreams); + + for (let i = 0; i < streamIdentities.length; i++) { + const streamIdentity = streamIdentities[i]; + const dateEntry = dateEntries[i]; + + if ( + !dateEntry || + (streamIdentity.streamNamespace ?? "") !== (dateEntry.streamNamespace ?? "") || + streamIdentity.streamName !== dateEntry.streamName + ) { + dateEntries.splice(i, 0, { + streamNamespace: streamIdentity.streamNamespace, + streamName: streamIdentity.streamName, + status: ConnectionStatusIndicatorStatus.Pending, + }); + } + } + + uptimeData.push({ + date: parseInt(dateBucketKey, 10) * 1000, + streams: dateEntries, + }); + }); + + return { uptimeData, streamIdentities }; +}; + // wrapped in memo to avoid redrawing the chart when the component tree re-renders export const UptimeStatusGraph: React.FC = React.memo(() => { + // read color values from the theme const [colorMap, setColorMap] = useState>({}); - const { colorValues } = useAirbyteTheme(); - useEffect(() => { const colorMap: Record = { green: colorValues[styles.greenVar], @@ -77,12 +166,22 @@ export const UptimeStatusGraph: React.FC = React.memo(() => { setColorMap(colorMap); }, [colorValues]); + const { connection } = useConnectionEditService(); + const data = useGetConnectionUptimeHistory(connection.connectionId); + const hasData = data.length > 0; + + const { uptimeData, streamIdentities } = useMemo(() => formatDataForChart(data), [data]); + + if (!hasData) { + return ; + } + return ( @@ -90,7 +189,7 @@ export const UptimeStatusGraph: React.FC = React.memo(() => { )} diff --git a/airbyte-webapp/src/area/connection/components/UptimeStatusGraph/WaffleChart.tsx b/airbyte-webapp/src/area/connection/components/UptimeStatusGraph/WaffleChart.tsx index 1f18ea1b2d3..36441da8edf 100644 --- a/airbyte-webapp/src/area/connection/components/UptimeStatusGraph/WaffleChart.tsx +++ b/airbyte-webapp/src/area/connection/components/UptimeStatusGraph/WaffleChart.tsx @@ -5,6 +5,8 @@ import { ConnectionStatusIndicatorStatus } from "components/connection/Connectio // Rough idea for the data structure we'll get from API export interface ChartStream { + streamNamespace?: string; + streamName: string; status: ConnectionStatusIndicatorStatus; } export interface UptimeDayEntry { @@ -34,7 +36,8 @@ interface InjectedStreamWaffleChartProps extends StreamWaffleChartProps { isTooltipActive: boolean; } -const getCellColor = (streamStatus: ConnectionStatusIndicatorStatus) => { +type WaffleColor = "green" | "darkBlue" | "red" | "black" | "empty"; +const getCellColor = (streamStatus: ConnectionStatusIndicatorStatus): WaffleColor => { switch (streamStatus) { case ConnectionStatusIndicatorStatus.OnTime: case ConnectionStatusIndicatorStatus.OnTrack: diff --git a/airbyte-webapp/src/area/connection/utils/computeStreamStatus.test.ts b/airbyte-webapp/src/area/connection/utils/computeStreamStatus.test.ts index b830f932440..1ad562f3b85 100644 --- a/airbyte-webapp/src/area/connection/utils/computeStreamStatus.test.ts +++ b/airbyte-webapp/src/area/connection/utils/computeStreamStatus.test.ts @@ -5,8 +5,10 @@ import { mockAirbyteStream } from "test-utils/mock-data/mockAirbyteStream"; import { mockStreamStatusRead } from "test-utils/mock-data/mockStreamStatusRead"; import { + AirbyteStream, ConnectionScheduleData, ConnectionScheduleType, + ConnectionSyncResultRead, StreamStatusIncompleteRunCause, StreamStatusJobType, StreamStatusRead, @@ -31,7 +33,9 @@ describe("getStreamKey", () => { ${"foo"} | ${""} | ${"foo-"} ${"foo"} | ${undefined} | ${"foo-"} `("$name, $namespace", ({ name, namespace, expected }) => - expect(getStreamKey({ ...mockStreamStatusRead, streamName: name, streamNamespace: namespace })).toBe(expected) + expect( + getStreamKey({ ...mockStreamStatusRead, streamName: name, streamNamespace: namespace } as StreamStatusRead) + ).toBe(expected) ); }); @@ -42,7 +46,18 @@ describe("getStreamKey", () => { ${"foo"} | ${""} | ${"foo-"} ${"foo"} | ${undefined} | ${"foo-"} `("$name, $namespace", ({ name, namespace, expected }) => - expect(getStreamKey({ ...mockAirbyteStream, name, namespace })).toBe(expected) + expect(getStreamKey({ ...mockAirbyteStream, name, namespace } as AirbyteStream)).toBe(expected) + ); + }); + + describe("when streamStatus is a ConnectionSyncResultRead", () => { + it.each` + name | namespace | expected + ${"foo"} | ${"bar"} | ${"foo-bar"} + ${"foo"} | ${""} | ${"foo-"} + ${"foo"} | ${undefined} | ${"foo-"} + `("$name, $namespace", ({ name, namespace, expected }) => + expect(getStreamKey({ streamName: name, streamNamespace: namespace } as ConnectionSyncResultRead)).toBe(expected) ); }); }); diff --git a/airbyte-webapp/src/area/connection/utils/computeStreamStatus.ts b/airbyte-webapp/src/area/connection/utils/computeStreamStatus.ts index b1348910fde..43641cc1b9f 100644 --- a/airbyte-webapp/src/area/connection/utils/computeStreamStatus.ts +++ b/airbyte-webapp/src/area/connection/utils/computeStreamStatus.ts @@ -7,6 +7,7 @@ import { AirbyteStreamAndConfiguration, ConnectionScheduleData, ConnectionScheduleType, + ConnectionSyncResultRead, StreamStatusIncompleteRunCause, StreamStatusJobType, StreamStatusRead, @@ -15,7 +16,7 @@ import { export type AirbyteStreamAndConfigurationWithEnforcedStream = AirbyteStreamAndConfiguration & { stream: AirbyteStream }; -export function getStreamKey(streamStatus: StreamStatusRead | AirbyteStream) { +export function getStreamKey(streamStatus: StreamStatusRead | AirbyteStream | ConnectionSyncResultRead) { let name: string; let namespace: string | undefined; @@ -23,6 +24,10 @@ export function getStreamKey(streamStatus: StreamStatusRead | AirbyteStream) { // StreamStatusRead name = streamStatus.streamName; namespace = streamStatus.streamNamespace; + } else if ("streamName" in streamStatus) { + // ConnectionSyncResultRead + name = streamStatus.streamName; + namespace = streamStatus.streamNamespace; } else { // AirbyteStream name = streamStatus.name; diff --git a/airbyte-webapp/src/core/api/hooks/connections.tsx b/airbyte-webapp/src/core/api/hooks/connections.tsx index f41d2d18b4b..ee9154c5ca8 100644 --- a/airbyte-webapp/src/core/api/hooks/connections.tsx +++ b/airbyte-webapp/src/core/api/hooks/connections.tsx @@ -14,6 +14,8 @@ import { useCurrentWorkspace, useInvalidateWorkspaceStateQuery } from "./workspa import { createOrUpdateStateSafe, deleteConnection, + getConnectionDataHistory, + getConnectionUptimeHistory, getState, getStateType, resetConnection, @@ -50,6 +52,8 @@ const connectionsKeys = { all: [SCOPE_WORKSPACE, "connections"] as const, lists: (sourceOrDestinationIds: string[] = []) => [...connectionsKeys.all, "list", ...sourceOrDestinationIds], detail: (connectionId: string) => [...connectionsKeys.all, "details", connectionId] as const, + dataHistory: (connectionId: string) => [...connectionsKeys.all, "dataHistory", connectionId] as const, + uptimeHistory: (connectionId: string) => [...connectionsKeys.all, "uptimeHistory", connectionId] as const, getState: (connectionId: string) => [...connectionsKeys.all, "getState", connectionId] as const, }; @@ -393,3 +397,31 @@ export const useCreateOrUpdateState = () => { } ); }; + +export const useGetConnectionDataHistory = (connectionId: string) => { + const options = useRequestOptions(); + + return useSuspenseQuery(connectionsKeys.dataHistory(connectionId), () => + getConnectionDataHistory( + { + connectionId, + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + options + ) + ); +}; + +export const useGetConnectionUptimeHistory = (connectionId: string) => { + const options = useRequestOptions(); + + return useSuspenseQuery(connectionsKeys.uptimeHistory(connectionId), () => + getConnectionUptimeHistory( + { + connectionId, + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }, + options + ) + ); +}; diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index d5ba0bd706f..41734bcf34a 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -604,8 +604,9 @@ "connection.stream.status.nextSync": "Next sync {sync}", "connection.stream.status.nextTry": "Next try {sync}", - "connection.overview.graph.uptimeStatus": "Uptime status", - "connection.overview.graph.dataMoved": "Data moved", + "connection.overview.graph.noData": "No data from the last 30 days", + "connection.overview.graph.uptimeStatus": "Streams", + "connection.overview.graph.dataMoved": "Records", "form.scheduleType": "Schedule type", "form.scheduleType.message": "Set how often data should sync to the destination", From 51d2ceee6f06ed57bfa872fe5fb4a66c53c3b83c Mon Sep 17 00:00:00 2001 From: Lake Mossman Date: Tue, 12 Dec 2023 12:42:28 -0800 Subject: [PATCH 12/27] =?UTF-8?q?=F0=9F=93=9D=20Document=20the=20session?= =?UTF-8?q?=20storage=20/=20broadcast=20channel=20oauth=20popup=20mechanis?= =?UTF-8?q?m=20(#10330)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/hooks/services/useConnectorAuth.tsx | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx b/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx index bf1a0070352..92d044026f7 100644 --- a/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx +++ b/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx @@ -28,6 +28,25 @@ const OAUTH_REDIRECT_URL = `${window.location.protocol}//${window.location.host} export const OAUTH_BROADCAST_CHANNEL_NAME = "airbyte_oauth_callback"; const OAUTH_POPUP_IDENTIFIER_KEY = "airbyte_oauth_popup_identifier"; +/** + * Since some OAuth providers clear out the window.opener and window.name properties, + * we need to use a different mechanism to relate the popup window back to this tab. + * + * Therefore, this method first opens the window to the /auth_flow page, with the + * consent URL as a query param and a random UUID as the window name. + * + * This /auth_flow page will store the UUID into session storage before redirecting + * to the consent URL. + * + * Once the OAuth consent flow is finished and the window is redirected back to + * /auth_flow, the UUID is retrieved from session storage and attached to the message + * sent to the broadcast channel. + * + * This tab listens for a message on the broadcast channel with the corresponding + * UUID, and completes the OAuth flow when it receives the message. + * + * @param url the OAuth consent URL + */ function openWindow(url: string): void { if (windowObjectReference == null || windowObjectReference.closed) { /* if the pointer to the window object in memory does not exist From d69aa19c5962f065ab556807761d00d9ef72f46e Mon Sep 17 00:00:00 2001 From: Flutra Osmani <129116758+flutra-osmani@users.noreply.github.com> Date: Tue, 12 Dec 2023 14:40:01 -0800 Subject: [PATCH 13/27] update the release version for airbyte protocol (#10274) --- deps.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps.toml b/deps.toml index 028a347e19b..3aafa5c12bf 100644 --- a/deps.toml +++ b/deps.toml @@ -1,5 +1,5 @@ [versions] -airbyte-protocol = "0.4.2" +airbyte-protocol = "0.5.1" bouncycastle = "1.70" commons_io = "2.7" connectors-testcontainers = "1.15.3" From 235bf5c9c9cdb37f3cf1b6af7bb0760e81fdbfd5 Mon Sep 17 00:00:00 2001 From: Benoit Moriceau Date: Tue, 12 Dec 2023 15:10:30 -0800 Subject: [PATCH 14/27] fix helm chart (#10332) --- .../templates/deployment.yaml | 23 +++++++++---------- .../templates/service.yaml | 1 - 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/charts/airbyte-workload-api-server/templates/deployment.yaml b/charts/airbyte-workload-api-server/templates/deployment.yaml index a5e7293fc0d..37035bf906d 100644 --- a/charts/airbyte-workload-api-server/templates/deployment.yaml +++ b/charts/airbyte-workload-api-server/templates/deployment.yaml @@ -1,4 +1,3 @@ ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -58,8 +57,8 @@ spec: {{- if .Values.debug.enabled }} - name: JAVA_TOOL_OPTIONS value: "-Xdebug -agentlib:jdwp=transport=dt_socket,address=0.0.0.0:{{ .Values.debug.remoteDebugPort }},server=y,suspend=n" - {{- end}} - {{- if eq .Values.global.deploymentMode "oss" }} + {{- end }} + {{- if eq .Values.global.deploymentMode "oss" }} - name: AIRBYTE_VERSION valueFrom: configMapKeyRef: @@ -157,7 +156,7 @@ spec: - name: debug containerPort: {{ .Values.debug.remoteDebugPort }} protocol: TCP - {{- end}} + {{- end }} {{- if .Values.resources }} resources: {{- toYaml .Values.resources | nindent 10 }} {{- end }} @@ -166,9 +165,9 @@ spec: {{- end }} volumeMounts: {{- if eq .Values.global.deploymentMode "oss" }} - - name: gcs-log-creds-volume - mountPath: /secrets/gcs-log-creds - readOnly: true + - name: gcs-log-creds-volume + mountPath: /secrets/gcs-log-creds + readOnly: true {{- end }} {{- if .Values.extraVolumeMounts }} {{- toYaml .Values.extraVolumeMounts | nindent 8 }} @@ -184,13 +183,13 @@ spec: {{- end }} volumes: {{- if eq .Values.global.deploymentMode "oss" }} - - name: gcs-log-creds-volume - secret: - secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) (.Values.global.credVolumeOverride) (eq .Values.global.deploymentMode "oss") }} + - name: gcs-log-creds-volume + secret: + secretName: {{ ternary (printf "%s-gcs-log-creds" ( .Release.Name )) (.Values.global.credVolumeOverride) (eq .Values.global.deploymentMode "oss") }} {{- end }} {{- if .Values.extraVolumes }} -{{- toYaml .Values.extraVolumes | nindent 6 }} + {{- toYaml .Values.extraVolumes | nindent 6 }} {{- end }} {{- if .Values.global.extraVolumes }} -{{- toYaml .Values.global.extraVolumes | nindent 6 }} + {{- toYaml .Values.global.extraVolumes | nindent 6 }} {{- end }} diff --git a/charts/airbyte-workload-api-server/templates/service.yaml b/charts/airbyte-workload-api-server/templates/service.yaml index 80f6adab8ba..7319b2e210f 100644 --- a/charts/airbyte-workload-api-server/templates/service.yaml +++ b/charts/airbyte-workload-api-server/templates/service.yaml @@ -1,4 +1,3 @@ ---- apiVersion: v1 kind: Service metadata: From e2bfc032bed3f0f87a9c11387fd6caa10adc29c2 Mon Sep 17 00:00:00 2001 From: benmoriceau Date: Tue, 12 Dec 2023 23:18:53 +0000 Subject: [PATCH 15/27] Bump helm chart version reference to 0.50.16 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte-workload-api-server/Chart.yaml | 2 +- charts/airbyte-workload-launcher/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 32 +++++++++---------- charts/airbyte/Chart.yaml | 30 ++++++++--------- 16 files changed, 45 insertions(+), 45 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index 043f063a688..b4d7e43983c 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 4e437c7db06..53104b2d8f1 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index ace897ac4a2..04307d98c52 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 66edc85aa8e..7c251968d30 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index 91fe18641c6..f4eaef9cbba 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 754881965c3..01907274f95 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index 2be81021f20..a0d7d4c70fe 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 67bd21a329d..6d81d8ef6fd 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index fbe621ece10..72e311c8572 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 1b149b7210f..50ae24a2fce 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 87b76dff29d..625d47fa892 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index a5d1d61f040..7c1b93bdd41 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-workload-api-server/Chart.yaml b/charts/airbyte-workload-api-server/Chart.yaml index 4c0f3391b0b..b05f510c301 100644 --- a/charts/airbyte-workload-api-server/Chart.yaml +++ b/charts/airbyte-workload-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-workload-launcher/Chart.yaml b/charts/airbyte-workload-launcher/Chart.yaml index a15e5c079d2..397e7dbf48f 100644 --- a/charts/airbyte-workload-launcher/Chart.yaml +++ b/charts/airbyte-workload-launcher/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 9c2900a6bbf..0dbd91feb69 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,45 +4,45 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: workload-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: workload-launcher repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 -digest: sha256:464fd8dea63ffa31a8b9c274e96e5a558f8a89edf85acfd6b2055b45f0bcdc73 -generated: "2023-12-12T18:26:55.227730608Z" + version: 0.50.16 +digest: sha256:7f1c46988e74d59f8c438b30faa24e71fa410bd9afb34ed43b99898d948bb6a5 +generated: "2023-12-12T23:18:36.207624617Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 76843507e50..00dd5002e1f 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.15 +version: 0.50.16 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,56 +32,56 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: workload-api-server.enabled name: workload-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: workload-launcher.enabled name: workload-launcher repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.15 + version: 0.50.16 From 73aa3c8903f2273f02fb13774f938d5abaeb2764 Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Tue, 12 Dec 2023 15:26:03 -0800 Subject: [PATCH 16/27] Notifications clean up (#10311) Remove unused parameters and methods --- .../server/handlers/SchedulerHandler.java | 2 -- .../CustomerioNotificationClient.java | 8 -------- .../notification/NotificationClient.java | 6 ------ .../notification/SlackNotificationClient.java | 20 +++---------------- .../SlackNotificationClientTest.java | 7 +++---- 5 files changed, 6 insertions(+), 37 deletions(-) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java index 27ea8fa9712..0f21ede146d 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java @@ -488,7 +488,6 @@ public void notifySchemaPropagated(final NotificationSettings notificationSettin connectionUrl, source.getName(), result.changeDescription(), - item.getSlackConfiguration().getWebhook(), email, isBreakingChange); } @@ -502,7 +501,6 @@ public void notifySchemaPropagated(final NotificationSettings notificationSettin connectionUrl, source.getName(), result.changeDescription(), - null, email, isBreakingChange); } diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java index b53a489d803..6ee5d980208 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java @@ -220,13 +220,6 @@ public boolean notifyFailure(final String message) { throw new NotImplementedException(); } - @Override - public boolean notifySchemaChange(final UUID connectionId, - final boolean isBreaking, - final String url) { - throw new NotImplementedException(); - } - @Override public boolean notifySchemaPropagated(final UUID workspaceId, final String workspaceName, @@ -235,7 +228,6 @@ public boolean notifySchemaPropagated(final UUID workspaceId, final String connectionUrl, final String sourceName, final List changes, - final String url, final String recipient, final boolean isBreaking) throws IOException { diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java index 0fb88087715..cc623fcdc80 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java @@ -70,11 +70,6 @@ public abstract boolean notifyBreakingChangeSyncsDisabled(List receiverE public abstract boolean notifyFailure(String message) throws IOException, InterruptedException; - public abstract boolean notifySchemaChange(final UUID connectionId, - final boolean isBreaking, - final String url) - throws IOException, InterruptedException; - public abstract boolean notifySchemaPropagated(final UUID workspaceId, final String workspaceName, final UUID connectionId, @@ -82,7 +77,6 @@ public abstract boolean notifySchemaPropagated(final UUID workspaceId, final String connectionUrl, final String sourceName, final List changes, - final String url, final String recipient, boolean isBreaking) throws IOException, InterruptedException; diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java index 6ffe63628ca..6e7797ff4e4 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java @@ -146,20 +146,6 @@ public boolean notifyBreakingChangeSyncsDisabled(final List receiverEmai throw new UnsupportedOperationException("Slack notification is not supported for breaking change syncs disabled notification"); } - @Override - public boolean notifySchemaChange(final UUID connectionId, final boolean isBreaking, final String url) - throws IOException, InterruptedException { - final String message = renderTemplate( - isBreaking ? "slack/breaking_schema_change_slack_notification_template.txt" - : "slack/non_breaking_schema_change_slack_notification_template.txt", - connectionId.toString(), url); - final String webhookUrl = config.getWebhook(); - if (!Strings.isEmpty(webhookUrl)) { - return notify(message); - } - return false; - } - @Override public boolean notifySchemaPropagated(final UUID workspaceId, final String workspaceName, @@ -168,7 +154,6 @@ public boolean notifySchemaPropagated(final UUID workspaceId, final String connectionUrl, final String sourceName, final List changes, - final String url, final String recipient, boolean isBreaking) throws IOException, InterruptedException { @@ -178,8 +163,9 @@ public boolean notifySchemaPropagated(final UUID workspaceId, summary.append(change); summary.append("\n"); } - final String message = isBreaking ? renderTemplate("slack/breaking_schema_change_slack_notification_template.txt", connectionId.toString(), url) - : renderTemplate("slack/schema_propagation_slack_notification_template.txt", connectionName, summary.toString(), connectionUrl); + final String message = + isBreaking ? renderTemplate("slack/breaking_schema_change_slack_notification_template.txt", connectionId.toString(), connectionUrl) + : renderTemplate("slack/schema_propagation_slack_notification_template.txt", connectionName, summary.toString(), connectionUrl); final String webhookUrl = config.getWebhook(); if (!Strings.isEmpty(webhookUrl)) { return notify(message); diff --git a/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java b/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java index 4c174dfa9ac..3377fa787f9 100644 --- a/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java +++ b/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java @@ -167,7 +167,6 @@ void testNotifySchemaPropagated() throws IOException, InterruptedException { String connectionName = "PSQL ->> BigQuery"; String sourceName = ""; boolean isBreaking = false; - String url = ""; String connectionUrl = "http://airbyte.io/your_connection"; String recipient = ""; @@ -183,9 +182,9 @@ void testNotifySchemaPropagated() throws IOException, InterruptedException { final SlackNotificationClient client = new SlackNotificationClient(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + TEST_PATH)); - assertTrue(client.notifySchemaPropagated(UUID.randomUUID(), workspaceName, connectionId, connectionName, connectionUrl, sourceName, changes, url, - recipient, - isBreaking)); + assertTrue( + client.notifySchemaPropagated(UUID.randomUUID(), workspaceName, connectionId, connectionName, connectionUrl, sourceName, changes, recipient, + isBreaking)); } From b4005f142b2fcee0c77df8c1e378d9a83f26eb0b Mon Sep 17 00:00:00 2001 From: Flutra Osmani <129116758+flutra-osmani@users.noreply.github.com> Date: Tue, 12 Dec 2023 18:59:55 -0800 Subject: [PATCH 17/27] create a keycloak specific schema for its tables (#10219) --- .../templates/deployment.yaml | 43 +++++++++++++++++-- charts/airbyte/templates/_helpers.tpl | 13 ++++++ charts/airbyte/templates/env-configmap.yaml | 1 + 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/charts/airbyte-keycloak/templates/deployment.yaml b/charts/airbyte-keycloak/templates/deployment.yaml index 5f7f4257c71..5add21462a1 100644 --- a/charts/airbyte-keycloak/templates/deployment.yaml +++ b/charts/airbyte-keycloak/templates/deployment.yaml @@ -49,10 +49,45 @@ spec: {{- if .Values.affinity }} affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} {{- end }} - {{- if .Values.extraInitContainers }} initContainers: - {{- toYaml .Values.extraInitContainers | nindent 6 }} - {{- end }} + {{- if eq .Values.global.deploymentMode "oss" }} + - name: init-db + image: postgres:13-alpine + command: [ "sh", "-c" ] + args: + - > + PGPASSWORD=$DATABASE_PASSWORD psql -h $DATABASE_HOST -p $DATABASE_PORT -U $DATABASE_USER -d $DATABASE_DB -c "CREATE SCHEMA IF NOT EXISTS keycloak"; + env: + - name: DATABASE_HOST + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_HOST + - name: DATABASE_PORT + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_PORT + - name: DATABASE_DB + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-airbyte-env + key: DATABASE_DB + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: {{ .Values.global.secretName | default (printf "%s-airbyte-secrets" .Release.Name) }} + key: DATABASE_USER + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.global.database.secretName | default (printf "%s-airbyte-secrets" .Release.Name ) }} + key: {{ .Values.global.database.secretValue | default "DATABASE_PASSWORD" }} + {{- end }} + + {{- if .Values.extraInitContainers }} + {{- toYaml .Values.extraInitContainers | nindent 8 }} + {{- end }} containers: - name: airbyte-keycloak image: {{ printf "%s:%s" .Values.image.repository (include "keycloak.imageTag" .) }} @@ -73,7 +108,7 @@ spec: valueFrom: configMapKeyRef: name: {{ .Release.Name }}-airbyte-env - key: DATABASE_URL # use the same database as airbyte + key: KEYCLOAK_DATABASE_URL - name: KEYCLOAK_DATABASE_USERNAME valueFrom: secretKeyRef: diff --git a/charts/airbyte/templates/_helpers.tpl b/charts/airbyte/templates/_helpers.tpl index 2a63a3f3d95..e7dde41acd6 100644 --- a/charts/airbyte/templates/_helpers.tpl +++ b/charts/airbyte/templates/_helpers.tpl @@ -187,6 +187,19 @@ Add environment variables to configure database values {{- end -}} {{- end -}} +{{/* +Create JDBC URL specifically for Keycloak with a custom schema +*/}} +{{- define "keycloak.database.url" -}} +{{- if .Values.externalDatabase.jdbcUrl -}} +{{- printf "%s?currentSchema=keycloak" .Values.externalDatabase.jdbcUrl -}} +{{- else -}} +{{- $baseURL := include "airbyte.database.url" . -}} +{{- printf "%s?currentSchema=keycloak" $baseURL -}} +{{- end -}} +{{- end -}} + + {{/* Create a default fully qualified minio name. We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). diff --git a/charts/airbyte/templates/env-configmap.yaml b/charts/airbyte/templates/env-configmap.yaml index c78efcb14af..fa469c28d80 100644 --- a/charts/airbyte/templates/env-configmap.yaml +++ b/charts/airbyte/templates/env-configmap.yaml @@ -22,6 +22,7 @@ data: DATABASE_HOST: {{ include "airbyte.database.host" . }} DATABASE_PORT: {{ include "airbyte.database.port" . | quote }} DATABASE_URL: {{ include "airbyte.database.url" . | quote }} + KEYCLOAK_DATABASE_URL: {{ include "keycloak.database.url" . | quote }} DB_DOCKER_MOUNT: airbyte_db GCS_LOG_BUCKET: {{ .Values.global.logs.gcs.bucket | quote }} GOOGLE_APPLICATION_CREDENTIALS: {{ include "airbyte.gcpLogCredentialsPath" . | quote }} From 0b98392f129d773e25c923a6c5b079fa3c94a864 Mon Sep 17 00:00:00 2001 From: flutra-osmani Date: Wed, 13 Dec 2023 03:09:58 +0000 Subject: [PATCH 18/27] Bump helm chart version reference to 0.50.17 --- charts/airbyte-api-server/Chart.yaml | 2 +- charts/airbyte-bootloader/Chart.yaml | 2 +- .../Chart.yaml | 2 +- charts/airbyte-cron/Chart.yaml | 2 +- charts/airbyte-keycloak-setup/Chart.yaml | 2 +- charts/airbyte-keycloak/Chart.yaml | 2 +- charts/airbyte-metrics/Chart.yaml | 2 +- charts/airbyte-pod-sweeper/Chart.yaml | 2 +- charts/airbyte-server/Chart.yaml | 2 +- charts/airbyte-temporal/Chart.yaml | 2 +- charts/airbyte-webapp/Chart.yaml | 2 +- charts/airbyte-worker/Chart.yaml | 2 +- charts/airbyte-workload-api-server/Chart.yaml | 2 +- charts/airbyte-workload-launcher/Chart.yaml | 2 +- charts/airbyte/Chart.lock | 32 +++++++++---------- charts/airbyte/Chart.yaml | 30 ++++++++--------- 16 files changed, 45 insertions(+), 45 deletions(-) diff --git a/charts/airbyte-api-server/Chart.yaml b/charts/airbyte-api-server/Chart.yaml index b4d7e43983c..c5be0018e0b 100644 --- a/charts/airbyte-api-server/Chart.yaml +++ b/charts/airbyte-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-bootloader/Chart.yaml b/charts/airbyte-bootloader/Chart.yaml index 53104b2d8f1..6007dc96757 100644 --- a/charts/airbyte-bootloader/Chart.yaml +++ b/charts/airbyte-bootloader/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-connector-builder-server/Chart.yaml b/charts/airbyte-connector-builder-server/Chart.yaml index 04307d98c52..ac7a9aa2feb 100644 --- a/charts/airbyte-connector-builder-server/Chart.yaml +++ b/charts/airbyte-connector-builder-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-cron/Chart.yaml b/charts/airbyte-cron/Chart.yaml index 7c251968d30..3cd7d6501a0 100644 --- a/charts/airbyte-cron/Chart.yaml +++ b/charts/airbyte-cron/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-keycloak-setup/Chart.yaml b/charts/airbyte-keycloak-setup/Chart.yaml index f4eaef9cbba..a708167c27c 100644 --- a/charts/airbyte-keycloak-setup/Chart.yaml +++ b/charts/airbyte-keycloak-setup/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-keycloak/Chart.yaml b/charts/airbyte-keycloak/Chart.yaml index 01907274f95..8c9aa7a8cc8 100644 --- a/charts/airbyte-keycloak/Chart.yaml +++ b/charts/airbyte-keycloak/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-metrics/Chart.yaml b/charts/airbyte-metrics/Chart.yaml index a0d7d4c70fe..a43bee4dc5e 100644 --- a/charts/airbyte-metrics/Chart.yaml +++ b/charts/airbyte-metrics/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-pod-sweeper/Chart.yaml b/charts/airbyte-pod-sweeper/Chart.yaml index 6d81d8ef6fd..1e47ff4e29d 100644 --- a/charts/airbyte-pod-sweeper/Chart.yaml +++ b/charts/airbyte-pod-sweeper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-server/Chart.yaml b/charts/airbyte-server/Chart.yaml index 72e311c8572..b6f1d6fc34e 100644 --- a/charts/airbyte-server/Chart.yaml +++ b/charts/airbyte-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-temporal/Chart.yaml b/charts/airbyte-temporal/Chart.yaml index 50ae24a2fce..8df3b17f56b 100644 --- a/charts/airbyte-temporal/Chart.yaml +++ b/charts/airbyte-temporal/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-webapp/Chart.yaml b/charts/airbyte-webapp/Chart.yaml index 625d47fa892..9540ccc4191 100644 --- a/charts/airbyte-webapp/Chart.yaml +++ b/charts/airbyte-webapp/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-worker/Chart.yaml b/charts/airbyte-worker/Chart.yaml index 7c1b93bdd41..692e0680628 100644 --- a/charts/airbyte-worker/Chart.yaml +++ b/charts/airbyte-worker/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be diff --git a/charts/airbyte-workload-api-server/Chart.yaml b/charts/airbyte-workload-api-server/Chart.yaml index b05f510c301..505d6f5d3a1 100644 --- a/charts/airbyte-workload-api-server/Chart.yaml +++ b/charts/airbyte-workload-api-server/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte-workload-launcher/Chart.yaml b/charts/airbyte-workload-launcher/Chart.yaml index 397e7dbf48f..544a9e82b25 100644 --- a/charts/airbyte-workload-launcher/Chart.yaml +++ b/charts/airbyte-workload-launcher/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/airbyte/Chart.lock b/charts/airbyte/Chart.lock index 0dbd91feb69..08603d3cdbb 100644 --- a/charts/airbyte/Chart.lock +++ b/charts/airbyte/Chart.lock @@ -4,45 +4,45 @@ dependencies: version: 1.17.1 - name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: workload-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: workload-launcher repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 -digest: sha256:7f1c46988e74d59f8c438b30faa24e71fa410bd9afb34ed43b99898d948bb6a5 -generated: "2023-12-12T23:18:36.207624617Z" + version: 0.50.17 +digest: sha256:f3c8138658896486c520709f4f8d994cfa104dbd6ebbe10ab9ef61f36d408418 +generated: "2023-12-13T03:09:35.633403548Z" diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 00dd5002e1f..a9e8917def5 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.50.16 +version: 0.50.17 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -32,56 +32,56 @@ dependencies: - condition: airbyte-bootloader.enabled name: airbyte-bootloader repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: temporal.enabled name: temporal repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: webapp.enabled name: webapp repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: server.enabled name: server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: airbyte-api-server.enabled name: airbyte-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: worker.enabled name: worker repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: workload-api-server.enabled name: workload-api-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: workload-launcher.enabled name: workload-launcher repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: pod-sweeper.enabled name: pod-sweeper repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: metrics.enabled name: metrics repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: cron.enabled name: cron repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: connector-builder-server.enabled name: connector-builder-server repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: keycloak.enabled name: keycloak repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 - condition: keycloak-setup.enabled name: keycloak-setup repository: https://airbytehq.github.io/helm-charts/ - version: 0.50.16 + version: 0.50.17 From 453f7ef49f3d801f830dc0242ea0b56eb6359da8 Mon Sep 17 00:00:00 2001 From: Cole Snodgrass Date: Wed, 13 Dec 2023 08:50:18 -0800 Subject: [PATCH 19/27] quick copy of previous local context work (#10312) Co-authored-by: benmoriceau --- airbyte-featureflag/src/main/kotlin/Client.kt | 66 +++++++++++++++---- .../src/test/kotlin/ClientTest.kt | 39 +++++++++++ .../src/test/resources/flags.yml | 28 ++++++++ 3 files changed, 121 insertions(+), 12 deletions(-) diff --git a/airbyte-featureflag/src/main/kotlin/Client.kt b/airbyte-featureflag/src/main/kotlin/Client.kt index 64cd4b739ce..dc551454f8f 100644 --- a/airbyte-featureflag/src/main/kotlin/Client.kt +++ b/airbyte-featureflag/src/main/kotlin/Client.kt @@ -118,12 +118,11 @@ class ConfigFileClient( flag: Flag, context: Context, ): Boolean { - return when (flag) { - is EnvVar -> flag.enabled(context) - else -> - lock.read { - flags[flag.key]?.serve?.let { it as? Boolean } ?: flag.default - } + if (flag is EnvVar) { + return flag.enabled(context) + } + return lock.read { + flags[flag.key]?.serve(context)?.let { it as? Boolean } ?: flag.default } } @@ -131,14 +130,14 @@ class ConfigFileClient( flag: Flag, context: Context, ): String { - return flags[flag.key]?.serve?.let { it as? String } ?: flag.default + return flags[flag.key]?.serve(context)?.let { it as? String } ?: flag.default } override fun intVariation( flag: Flag, context: Context, ): Int { - return flags[flag.key]?.serve?.let { it as? Int } ?: flag.default + return flags[flag.key]?.serve(context)?.let { it as? Int } ?: flag.default } companion object { @@ -249,7 +248,52 @@ private data class ConfigFileFlags(val flags: List) /** * Data wrapper around an individual flag read from the configuration file. */ -private data class ConfigFileFlag(val name: String, val serve: Any) +private data class ConfigFileFlag( + val name: String, + val serve: Any, + val context: List? = null, +) { + /** + * Map of context kind to list of contexts. + * + * Example: + * { + * "workspace": [ + * { "serve": "true", include: ["000000-...", "111111-..."] } + * ] + * } + */ + private val contextsByType: Map> = + context?.groupBy { it.type } ?: mapOf() + + /** + * Serve checks the [ctx] to see if it matches any contexts that may have + * been defined in the flags.yml file. If it does match, the serve value + * from the matching context section will be returned. If it does not + * match, the non-context serve value will be returned. + */ + fun serve(ctx: Context): Any { + if (contextsByType.isEmpty()) { + return serve + } + return when (ctx) { + is Multi -> + ctx.contexts.map { serve(it) } + .find { it != serve } ?: serve + else -> + contextsByType[ctx.kind] + ?.findLast { it.include.contains(ctx.key) } + ?.serve + ?: serve + } + } +} + +private data class ConfigFileFlagContext( + val type: String, + val serve: Any, + val include: List = listOf(), +) /** The yaml mapper is used for reading the feature-flag configuration file. */ private val yamlMapper = ObjectMapper(YAMLFactory()).registerKotlinModule() @@ -260,9 +304,7 @@ private val yamlMapper = ObjectMapper(YAMLFactory()).registerKotlinModule() * @param [path] to yaml config file * @return map of feature-flag name to feature-flag config */ -private fun readConfig(path: Path): Map = - yamlMapper.readValue(path.toFile()).flags - .associateBy { it.name } +private fun readConfig(path: Path): Map = yamlMapper.readValue(path.toFile()).flags.associateBy { it.name } /** * Monitors a [Path] for changes, calling [block] when a change is detected. diff --git a/airbyte-featureflag/src/test/kotlin/ClientTest.kt b/airbyte-featureflag/src/test/kotlin/ClientTest.kt index d66e282d102..6b3d6480259 100644 --- a/airbyte-featureflag/src/test/kotlin/ClientTest.kt +++ b/airbyte-featureflag/src/test/kotlin/ClientTest.kt @@ -201,6 +201,45 @@ class ConfigFileClientTest { assertFalse { boolVariation(evNull, ctx) } } } + + @Test + fun `verify context support`() { + val cfg = Path.of("src", "test", "resources", "flags.yml") + val client: FeatureFlagClient = ConfigFileClient(cfg) + + // included in one context override + val uuidAAAA = UUID.fromString("00000000-aaaa-0000-aaaa-000000000000") + // included in one context override + val uuidBBBB = UUID.fromString("00000000-bbbb-0000-bbbb-000000000000") + // included in no context overrides + val uuidCCCC = UUID.fromString("00000000-cccc-0000-cccc-000000000000") + // included in two context overrides + val uuidDDDD = UUID.fromString("00000000-dddd-0000-dddd-000000000000") + + val flagCtxString = Temporary(key = "test-context-string", default = "default") + val flagCtxBoolean = Temporary(key = "test-context-boolean", default = false) + + val ctxAAAA = Workspace(uuidAAAA) + val ctxBBBB = Workspace(uuidBBBB) + val ctxCCCC = Workspace(uuidCCCC) + val ctxDDDD = Workspace(uuidDDDD) + val multi = Multi(listOf(ctxAAAA, ctxBBBB)) + val multiRandom = Multi(listOf(Workspace(UUID.randomUUID()), Workspace(UUID.randomUUID()))) + val multiFindFirst = Multi(listOf(ctxAAAA, ctxBBBB)) + + with(client) { + assertFalse("aaaa should be false") { boolVariation(flagCtxBoolean, ctxAAAA) } + assertTrue("bbbb should be true") { boolVariation(flagCtxBoolean, ctxBBBB) } + assertTrue("cccc should be true") { boolVariation(flagCtxBoolean, ctxCCCC) } + assertEquals("aaaa", stringVariation(flagCtxString, ctxAAAA), "aaab should be bbbb") + assertEquals("bbbb", stringVariation(flagCtxString, ctxBBBB), "bbbb should be bbbb") + assertEquals("all", stringVariation(flagCtxString, ctxCCCC), "cccc should be all (not included anywhere)") + assertEquals("bbbb", stringVariation(flagCtxString, ctxDDDD), "dddd should be bbbb") + assertEquals("aaaa", stringVariation(flagCtxString, multi), "dddd should be aaaa") + assertEquals("all", stringVariation(flagCtxString, multiRandom), "dddd should be aaaa") + assertEquals("aaaa", stringVariation(flagCtxString, multiFindFirst), "aaab should be bbbb") + } + } } class LaunchDarklyClientTest { diff --git a/airbyte-featureflag/src/test/resources/flags.yml b/airbyte-featureflag/src/test/resources/flags.yml index 6127f53b7c4..cfdc336406a 100644 --- a/airbyte-featureflag/src/test/resources/flags.yml +++ b/airbyte-featureflag/src/test/resources/flags.yml @@ -5,5 +5,33 @@ flags: serve: false - name: test-string serve: "example" + context: + - type: "workspace" + include: + - "00000000-aaaa-0000-aaaa-000000000000" + serve: "context" + + - name: test-context-boolean + serve: true + context: + - type: "workspace" + include: + - "00000000-aaaa-0000-aaaa-000000000000" + serve: false + + - name: test-context-string + serve: "all" + context: + - type: "workspace" + include: + - "00000000-aaaa-0000-aaaa-000000000000" + - "00000000-dddd-0000-dddd-000000000000" + serve: "aaaa" + - type: "workspace" + include: + - "00000000-bbbb-0000-bbbb-000000000000" + - "00000000-dddd-0000-dddd-000000000000" + serve: "bbbb" + - name: test-int serve: 1234 From 83c24ba68a922480106ae3cbfd5b9708ad6c582b Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Wed, 13 Dec 2023 11:27:59 -0800 Subject: [PATCH 20/27] Add helper function to get url of a source or destination in the UI (#10347) --- .../airbyte/persistence/job/WebUrlHelper.java | 22 +++++++++++++++++++ .../persistence/job/WebUrlHelperTest.java | 19 ++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WebUrlHelper.java b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WebUrlHelper.java index 6f0df42df98..6103d2f46c3 100644 --- a/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WebUrlHelper.java +++ b/airbyte-persistence/job-persistence/src/main/java/io/airbyte/persistence/job/WebUrlHelper.java @@ -52,6 +52,28 @@ public String getConnectionUrl(final UUID workspaceId, final UUID connectionId) return String.format("%s/connections/%s", getWorkspaceUrl(workspaceId), connectionId); } + /** + * Get the url for a source. + * + * @param workspaceId workspace id + * @param sourceId source id + * @return url for the source + */ + public String getSourceUrl(final UUID workspaceId, final UUID sourceId) { + return String.format("%s/source/%s", getWorkspaceUrl(workspaceId), sourceId); + } + + /** + * Get the url for a destination. + * + * @param workspaceId workspace id + * @param destinationId destination id + * @return url for the destination + */ + public String getDestinationUrl(final UUID workspaceId, final UUID destinationId) { + return String.format("%s/destination/%s", getWorkspaceUrl(workspaceId), destinationId); + } + public String getConnectionReplicationPageUrl(final UUID workspaceId, final UUID connectionId) { return String.format("%s/connections/%s/replication", getWorkspaceUrl(workspaceId), connectionId); } diff --git a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/WebUrlHelperTest.java b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/WebUrlHelperTest.java index ad1095e41cd..bf92428692f 100644 --- a/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/WebUrlHelperTest.java +++ b/airbyte-persistence/job-persistence/src/test/java/io/airbyte/persistence/job/WebUrlHelperTest.java @@ -12,6 +12,8 @@ class WebUrlHelperTest { private static final UUID WORKSPACE_ID = UUID.randomUUID(); private static final UUID CONNECTION_ID = UUID.randomUUID(); + private static final UUID SOURCE_ID = UUID.randomUUID(); + private static final UUID DESTINATION_ID = UUID.randomUUID(); private static final String LOCALHOST_8000 = "http://localhost:8000"; @Test @@ -42,4 +44,21 @@ void testGetConnectionUrl() { Assertions.assertEquals(expectedUrl, connectionUrl); } + @Test + void testGetSourceUrl() { + final WebUrlHelper webUrlHelper = new WebUrlHelper(LOCALHOST_8000); + final String sourceUrl = webUrlHelper.getSourceUrl(WORKSPACE_ID, SOURCE_ID); + final String expectedUrl = String.format("%s/workspaces/%s/source/%s", LOCALHOST_8000, WORKSPACE_ID, SOURCE_ID); + Assertions.assertEquals(expectedUrl, sourceUrl); + } + + @Test + void testGetDestinationUrl() { + final WebUrlHelper webUrlHelper = new WebUrlHelper(LOCALHOST_8000); + final String destinationUrl = webUrlHelper.getSourceUrl(WORKSPACE_ID, DESTINATION_ID); + final String expectedUrl = String.format("%s/workspaces/%s/source/%s", LOCALHOST_8000, WORKSPACE_ID, DESTINATION_ID); + Assertions.assertEquals(expectedUrl, destinationUrl); + + } + } From 3c1d3d833fa4ad63a1e593e61ee8fd0a0296b8c4 Mon Sep 17 00:00:00 2001 From: Malik Diarra Date: Wed, 13 Dec 2023 11:29:08 -0800 Subject: [PATCH 21/27] Add sourceId to the data dictionary for schema changes notification (#10344) --- .../io/airbyte/commons/server/handlers/SchedulerHandler.java | 2 ++ .../io/airbyte/notification/CustomerioNotificationClient.java | 3 +++ .../main/java/io/airbyte/notification/NotificationClient.java | 1 + .../java/io/airbyte/notification/SlackNotificationClient.java | 1 + .../io/airbyte/notification/SlackNotificationClientTest.java | 4 +++- 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java index 0f21ede146d..36f20b97025 100644 --- a/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java +++ b/airbyte-commons-server/src/main/java/io/airbyte/commons/server/handlers/SchedulerHandler.java @@ -486,6 +486,7 @@ public void notifySchemaPropagated(final NotificationSettings notificationSettin connection.getConnectionId(), connection.getName(), connectionUrl, + source.getSourceId(), source.getName(), result.changeDescription(), email, @@ -499,6 +500,7 @@ public void notifySchemaPropagated(final NotificationSettings notificationSettin connection.getConnectionId(), connection.getName(), connectionUrl, + source.getSourceId(), source.getName(), result.changeDescription(), email, diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java index 6ee5d980208..0b2b298132a 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/CustomerioNotificationClient.java @@ -226,6 +226,7 @@ public boolean notifySchemaPropagated(final UUID workspaceId, final UUID connectionId, final String connectionName, final String connectionUrl, + final UUID sourceId, final String sourceName, final List changes, final String recipient, @@ -249,6 +250,8 @@ public boolean notifySchemaPropagated(final UUID workspaceId, messageDataNode.put("workspace_name", workspaceName); messageDataNode.put("changes_details", String.join("\n", changes)); messageDataNode.put("source", sourceName); + messageDataNode.put("source_name", sourceName); + messageDataNode.put("source_id", sourceId.toString()); node.set("message_data", messageDataNode); diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java index cc623fcdc80..e2838c8f444 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/NotificationClient.java @@ -75,6 +75,7 @@ public abstract boolean notifySchemaPropagated(final UUID workspaceId, final UUID connectionId, final String connectionName, final String connectionUrl, + final UUID sourceId, final String sourceName, final List changes, final String recipient, diff --git a/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java b/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java index 6e7797ff4e4..0392562d450 100644 --- a/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java +++ b/airbyte-notification/src/main/java/io/airbyte/notification/SlackNotificationClient.java @@ -152,6 +152,7 @@ public boolean notifySchemaPropagated(final UUID workspaceId, final UUID connectionId, final String connectionName, final String connectionUrl, + final UUID sourceId, final String sourceName, final List changes, final String recipient, diff --git a/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java b/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java index 3377fa787f9..8cae30296ac 100644 --- a/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java +++ b/airbyte-notification/src/test/java/io/airbyte/notification/SlackNotificationClientTest.java @@ -162,6 +162,7 @@ void testNotifyConnectionDisabledWarning() throws IOException, InterruptedExcept @Test void testNotifySchemaPropagated() throws IOException, InterruptedException { final UUID connectionId = UUID.randomUUID(); + final UUID sourceId = UUID.randomUUID(); final List changes = List.of("Change1", "Some other change"); String workspaceName = ""; String connectionName = "PSQL ->> BigQuery"; @@ -183,7 +184,8 @@ void testNotifySchemaPropagated() throws IOException, InterruptedException { new SlackNotificationClient(new SlackNotificationConfiguration().withWebhook(WEBHOOK_URL + server.getAddress().getPort() + TEST_PATH)); assertTrue( - client.notifySchemaPropagated(UUID.randomUUID(), workspaceName, connectionId, connectionName, connectionUrl, sourceName, changes, recipient, + client.notifySchemaPropagated(UUID.randomUUID(), workspaceName, connectionId, connectionName, connectionUrl, sourceId, sourceName, changes, + recipient, isBreaking)); } From aec1badd6ad8ccc7735ecaf4908c9aa436a73e3b Mon Sep 17 00:00:00 2001 From: Ryan Br Date: Wed, 13 Dec 2023 12:17:56 -0800 Subject: [PATCH 22/27] Rbroughan/build check input (#10314) --- airbyte-commons-worker/build.gradle.kts | 1 + .../SecretPersistenceConfigHelper.java | 2 +- .../workers/models/CheckConnectionInput.java | 26 ++++ .../workers/CheckConnectionInputHydrator.kt | 69 +++++++++++ .../CheckConnectionInputHydratorTest.kt | 112 ++++++++++++++++++ .../connection/CheckConnectionActivity.java | 21 +--- .../CheckConnectionActivityImpl.java | 45 ++----- .../CheckConnectionWorkflowImpl.java | 2 +- .../catalog/DiscoverCatalogActivityImpl.java | 2 +- .../sync/DbtTransformationActivityImpl.java | 2 +- .../sync/NormalizationActivityImpl.java | 2 +- .../sync/WebhookOperationActivityImpl.java | 2 +- .../kotlin/config/ApplicationBeanFactory.kt | 14 +++ .../kotlin/pipeline/stages/BuildInputStage.kt | 33 +++++- .../kotlin/pipeline/stages/LaunchPodStage.kt | 8 +- .../kotlin/pipeline/stages/model/StageIO.kt | 13 +- .../main/kotlin/serde/PayloadDeserializer.kt | 6 +- .../pipeline/stages/BuildInputStageTest.kt | 76 ++++++++++-- .../pipeline/stages/EnforceMutexStageTest.kt | 14 +-- .../pipeline/stages/LaunchPodStageTest.kt | 6 +- 20 files changed, 365 insertions(+), 91 deletions(-) rename {airbyte-workers/src/main/java/io/airbyte/workers/helpers => airbyte-commons-worker/src/main/java/io/airbyte/workers/helper}/SecretPersistenceConfigHelper.java (97%) create mode 100644 airbyte-commons-worker/src/main/java/io/airbyte/workers/models/CheckConnectionInput.java create mode 100644 airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/CheckConnectionInputHydrator.kt create mode 100644 airbyte-commons-worker/src/test/java/io/airbyte/workers/CheckConnectionInputHydratorTest.kt diff --git a/airbyte-commons-worker/build.gradle.kts b/airbyte-commons-worker/build.gradle.kts index d9fae2ce492..713f2f76801 100644 --- a/airbyte-commons-worker/build.gradle.kts +++ b/airbyte-commons-worker/build.gradle.kts @@ -80,6 +80,7 @@ dependencies { testImplementation(libs.docker.java) testImplementation(libs.docker.java.transport.httpclient5) testImplementation(libs.reactor.test) + testImplementation(libs.mockk) testCompileOnly(libs.lombok) testAnnotationProcessor(libs.lombok) diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/helpers/SecretPersistenceConfigHelper.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/SecretPersistenceConfigHelper.java similarity index 97% rename from airbyte-workers/src/main/java/io/airbyte/workers/helpers/SecretPersistenceConfigHelper.java rename to airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/SecretPersistenceConfigHelper.java index d9441b2624b..f12d863a41f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/helpers/SecretPersistenceConfigHelper.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/helper/SecretPersistenceConfigHelper.java @@ -2,7 +2,7 @@ * Copyright (c) 2023 Airbyte, Inc., all rights reserved. */ -package io.airbyte.workers.helpers; +package io.airbyte.workers.helper; import io.airbyte.api.client.model.generated.SecretPersistenceConfig; import io.airbyte.commons.enums.Enums; diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/models/CheckConnectionInput.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/models/CheckConnectionInput.java new file mode 100644 index 00000000000..88d9971b68f --- /dev/null +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/models/CheckConnectionInput.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.workers.models; + +import io.airbyte.config.StandardCheckConnectionInput; +import io.airbyte.persistence.job.models.IntegrationLauncherConfig; +import io.airbyte.persistence.job.models.JobRunConfig; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * CheckConnectionInput. + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class CheckConnectionInput { + + private JobRunConfig jobRunConfig; + private IntegrationLauncherConfig launcherConfig; + private StandardCheckConnectionInput connectionConfiguration; + +} diff --git a/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/CheckConnectionInputHydrator.kt b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/CheckConnectionInputHydrator.kt new file mode 100644 index 00000000000..52973209d0f --- /dev/null +++ b/airbyte-commons-worker/src/main/kotlin/io/airbyte/workers/CheckConnectionInputHydrator.kt @@ -0,0 +1,69 @@ +package io.airbyte.workers + +import com.fasterxml.jackson.databind.JsonNode +import io.airbyte.api.client.generated.SecretsPersistenceConfigApi +import io.airbyte.api.client.invoker.generated.ApiException +import io.airbyte.api.client.model.generated.ScopeType +import io.airbyte.api.client.model.generated.SecretPersistenceConfig +import io.airbyte.api.client.model.generated.SecretPersistenceConfigGetRequestBody +import io.airbyte.config.StandardCheckConnectionInput +import io.airbyte.config.secrets.SecretsRepositoryReader +import io.airbyte.featureflag.FeatureFlagClient +import io.airbyte.featureflag.Organization +import io.airbyte.featureflag.UseRuntimeSecretPersistence +import io.airbyte.workers.helper.SecretPersistenceConfigHelper +import java.lang.RuntimeException +import java.util.UUID + +class CheckConnectionInputHydrator( + private val secretsRepositoryReader: SecretsRepositoryReader, + private val secretsApiClient: SecretsPersistenceConfigApi, + private val featureFlagClient: FeatureFlagClient, +) { + fun getHydratedCheckInput(rawInput: StandardCheckConnectionInput): StandardCheckConnectionInput { + val fullConfig: JsonNode? + val organizationId: UUID? = rawInput.actorContext.organizationId + + fullConfig = + if (useRuntimeHydration(organizationId)) { + hydrateFromRuntimePersistence(rawInput.connectionConfiguration, organizationId!!) + } else { + secretsRepositoryReader.hydrateConfigFromDefaultSecretPersistence(rawInput.connectionConfiguration) + } + + return StandardCheckConnectionInput() + .withActorId(rawInput.actorId) + .withActorType(rawInput.actorType) + .withConnectionConfiguration(fullConfig) + .withActorContext(rawInput.actorContext) + } + + private fun hydrateFromRuntimePersistence( + jsonConfig: JsonNode, + organizationId: UUID, + ): JsonNode? { + val secretPersistenceConfig: SecretPersistenceConfig + try { + secretPersistenceConfig = + secretsApiClient.getSecretsPersistenceConfig( + SecretPersistenceConfigGetRequestBody() + .scopeType(ScopeType.ORGANIZATION) + .scopeId(organizationId), + ) + } catch (e: ApiException) { + throw RuntimeException(e) + } + + val runtimeSecretPersistence = + SecretPersistenceConfigHelper + .fromApiSecretPersistenceConfig(secretPersistenceConfig) + + return secretsRepositoryReader.hydrateConfigFromRuntimeSecretPersistence( + jsonConfig, + runtimeSecretPersistence, + ) + } + + private fun useRuntimeHydration(organizationId: UUID?): Boolean = + organizationId != null && featureFlagClient.boolVariation(UseRuntimeSecretPersistence, Organization(organizationId)) +} diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/CheckConnectionInputHydratorTest.kt b/airbyte-commons-worker/src/test/java/io/airbyte/workers/CheckConnectionInputHydratorTest.kt new file mode 100644 index 00000000000..5711533c940 --- /dev/null +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/CheckConnectionInputHydratorTest.kt @@ -0,0 +1,112 @@ +package io.airbyte.workers + +import com.fasterxml.jackson.databind.node.POJONode +import io.airbyte.api.client.generated.SecretsPersistenceConfigApi +import io.airbyte.api.client.model.generated.ScopeType +import io.airbyte.api.client.model.generated.SecretPersistenceConfig +import io.airbyte.api.client.model.generated.SecretPersistenceType +import io.airbyte.config.ActorContext +import io.airbyte.config.ActorType +import io.airbyte.config.StandardCheckConnectionInput +import io.airbyte.config.secrets.SecretsRepositoryReader +import io.airbyte.config.secrets.persistence.RuntimeSecretPersistence +import io.airbyte.featureflag.FeatureFlagClient +import io.airbyte.featureflag.Organization +import io.airbyte.featureflag.UseRuntimeSecretPersistence +import io.airbyte.workers.helper.SecretPersistenceConfigHelper +import io.mockk.every +import io.mockk.mockk +import io.mockk.mockkStatic +import io.mockk.verify +import org.junit.jupiter.api.Test +import java.util.UUID + +class CheckConnectionInputHydratorTest { + @Test + fun `uses runtime hydration if ff enabled for organization id`() { + val secretsRepositoryReader: SecretsRepositoryReader = mockk() + val secretsApiClient: SecretsPersistenceConfigApi = mockk() + val featureFlagClient: FeatureFlagClient = mockk() + + val hydrator = + CheckConnectionInputHydrator( + secretsRepositoryReader, + secretsApiClient, + featureFlagClient, + ) + + val unhyrdatedConfig = POJONode("un-hydrated") + val hydratedConfig = POJONode("hydrated") + + val orgId = UUID.randomUUID() + val input = + StandardCheckConnectionInput() + .withActorContext(ActorContext().withOrganizationId(orgId)) + .withActorType(ActorType.DESTINATION) + .withActorId(UUID.randomUUID()) + .withConnectionConfiguration(unhyrdatedConfig) + + val secretConfig = + SecretPersistenceConfig() + .scopeId(orgId) + .scopeType(ScopeType.ORGANIZATION) + .secretPersistenceType(SecretPersistenceType.AWS) + + val runtimeSecretPersistence = RuntimeSecretPersistence(mockk()) + + mockkStatic(SecretPersistenceConfigHelper::class) + every { SecretPersistenceConfigHelper.fromApiSecretPersistenceConfig(secretConfig) } returns runtimeSecretPersistence + + every { secretsApiClient.getSecretsPersistenceConfig(any()) } returns secretConfig + every { secretsRepositoryReader.hydrateConfigFromRuntimeSecretPersistence(unhyrdatedConfig, runtimeSecretPersistence) } returns hydratedConfig + + every { featureFlagClient.boolVariation(eq(UseRuntimeSecretPersistence), eq(Organization(orgId))) } returns true + + val result = hydrator.getHydratedCheckInput(input) + + verify { secretsRepositoryReader.hydrateConfigFromRuntimeSecretPersistence(unhyrdatedConfig, runtimeSecretPersistence) } + + assert(input.actorId == result.actorId) + assert(input.actorType == result.actorType) + assert(input.actorContext == result.actorContext) + assert(hydratedConfig == result.connectionConfiguration) + } + + @Test + fun `uses default hydration if ff not enabled for organization id`() { + val secretsRepositoryReader: SecretsRepositoryReader = mockk() + val secretsApiClient: SecretsPersistenceConfigApi = mockk() + val featureFlagClient: FeatureFlagClient = mockk() + + val hydrator = + CheckConnectionInputHydrator( + secretsRepositoryReader, + secretsApiClient, + featureFlagClient, + ) + + val unhyrdatedConfig = POJONode("un-hydrated") + val hydratedConfig = POJONode("hydrated") + + val orgId = UUID.randomUUID() + val input = + StandardCheckConnectionInput() + .withActorContext(ActorContext().withOrganizationId(orgId)) + .withActorType(ActorType.DESTINATION) + .withActorId(UUID.randomUUID()) + .withConnectionConfiguration(unhyrdatedConfig) + + every { secretsRepositoryReader.hydrateConfigFromDefaultSecretPersistence(unhyrdatedConfig) } returns hydratedConfig + + every { featureFlagClient.boolVariation(eq(UseRuntimeSecretPersistence), eq(Organization(orgId))) } returns false + + val result = hydrator.getHydratedCheckInput(input) + + verify { secretsRepositoryReader.hydrateConfigFromDefaultSecretPersistence(unhyrdatedConfig) } + + assert(input.actorId == result.actorId) + assert(input.actorType == result.actorType) + assert(input.actorContext == result.actorContext) + assert(hydratedConfig == result.connectionConfiguration) + } +} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivity.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivity.java index 8390eb49e3b..9e448920df8 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivity.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivity.java @@ -5,15 +5,10 @@ package io.airbyte.workers.temporal.check.connection; import io.airbyte.config.ConnectorJobOutput; -import io.airbyte.config.StandardCheckConnectionInput; import io.airbyte.config.StandardCheckConnectionOutput; -import io.airbyte.persistence.job.models.IntegrationLauncherConfig; -import io.airbyte.persistence.job.models.JobRunConfig; +import io.airbyte.workers.models.CheckConnectionInput; import io.temporal.activity.ActivityInterface; import io.temporal.activity.ActivityMethod; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; /** * Check connection activity temporal interface. @@ -21,20 +16,6 @@ @ActivityInterface public interface CheckConnectionActivity { - /** - * CheckConnectionInput. - */ - @Data - @NoArgsConstructor - @AllArgsConstructor - class CheckConnectionInput { - - private JobRunConfig jobRunConfig; - private IntegrationLauncherConfig launcherConfig; - private StandardCheckConnectionInput connectionConfiguration; - - } - @ActivityMethod ConnectorJobOutput runWithJobOutput(CheckConnectionInput input); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java index 089909d507c..f8e9b578409 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionActivityImpl.java @@ -9,13 +9,8 @@ import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.DOCKER_IMAGE_KEY; import static io.airbyte.metrics.lib.ApmTraceConstants.Tags.JOB_ID_KEY; -import com.fasterxml.jackson.databind.JsonNode; import datadog.trace.api.Trace; import io.airbyte.api.client.AirbyteApiClient; -import io.airbyte.api.client.invoker.generated.ApiException; -import io.airbyte.api.client.model.generated.ScopeType; -import io.airbyte.api.client.model.generated.SecretPersistenceConfig; -import io.airbyte.api.client.model.generated.SecretPersistenceConfigGetRequestBody; import io.airbyte.commons.converters.ConnectorConfigUpdater; import io.airbyte.commons.features.FeatureFlags; import io.airbyte.commons.functional.CheckedSupplier; @@ -35,20 +30,18 @@ import io.airbyte.config.helpers.LogConfigs; import io.airbyte.config.helpers.ResourceRequirementsUtils; import io.airbyte.config.secrets.SecretsRepositoryReader; -import io.airbyte.config.secrets.persistence.RuntimeSecretPersistence; import io.airbyte.featureflag.FeatureFlagClient; -import io.airbyte.featureflag.Organization; -import io.airbyte.featureflag.UseRuntimeSecretPersistence; import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.OssMetricsRegistry; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; +import io.airbyte.workers.CheckConnectionInputHydrator; import io.airbyte.workers.Worker; import io.airbyte.workers.general.DefaultCheckConnectionWorker; import io.airbyte.workers.helper.GsonPksExtractor; -import io.airbyte.workers.helpers.SecretPersistenceConfigHelper; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.VersionedAirbyteStreamFactory; +import io.airbyte.workers.models.CheckConnectionInput; import io.airbyte.workers.process.AirbyteIntegrationLauncher; import io.airbyte.workers.process.IntegrationLauncher; import io.airbyte.workers.process.ProcessFactory; @@ -62,7 +55,6 @@ import java.util.Collections; import java.util.Map; import java.util.Optional; -import java.util.UUID; import java.util.concurrent.atomic.AtomicReference; /** @@ -73,7 +65,6 @@ public class CheckConnectionActivityImpl implements CheckConnectionActivity { private final WorkerConfigsProvider workerConfigsProvider; private final ProcessFactory processFactory; - private final SecretsRepositoryReader secretsRepositoryReader; private final Path workspaceRoot; private final WorkerEnvironment workerEnvironment; private final LogConfigs logConfigs; @@ -82,8 +73,8 @@ public class CheckConnectionActivityImpl implements CheckConnectionActivity { private final AirbyteMessageSerDeProvider serDeProvider; private final AirbyteProtocolVersionedMigratorFactory migratorFactory; private final FeatureFlags featureFlags; - private final FeatureFlagClient featureFlagClient; private final GsonPksExtractor gsonPksExtractor; + private final CheckConnectionInputHydrator inputHydrator; public CheckConnectionActivityImpl(final WorkerConfigsProvider workerConfigsProvider, final ProcessFactory processFactory, @@ -100,7 +91,6 @@ public CheckConnectionActivityImpl(final WorkerConfigsProvider workerConfigsProv final GsonPksExtractor gsonPksExtractor) { this.workerConfigsProvider = workerConfigsProvider; this.processFactory = processFactory; - this.secretsRepositoryReader = secretsRepositoryReader; this.workspaceRoot = workspaceRoot; this.workerEnvironment = workerEnvironment; this.logConfigs = logConfigs; @@ -109,8 +99,11 @@ public CheckConnectionActivityImpl(final WorkerConfigsProvider workerConfigsProv this.serDeProvider = serDeProvider; this.migratorFactory = migratorFactory; this.featureFlags = featureFlags; - this.featureFlagClient = featureFlagClient; this.gsonPksExtractor = gsonPksExtractor; + this.inputHydrator = new CheckConnectionInputHydrator( + secretsRepositoryReader, + airbyteApiClient.getSecretPersistenceConfigApi(), + featureFlagClient); } @Trace(operationName = ACTIVITY_TRACE_OPERATION_NAME) @@ -122,29 +115,7 @@ public ConnectorJobOutput runWithJobOutput(final CheckConnectionInput args) { .addTagsToTrace(Map.of(ATTEMPT_NUMBER_KEY, args.getJobRunConfig().getAttemptId(), JOB_ID_KEY, args.getJobRunConfig().getJobId(), DOCKER_IMAGE_KEY, args.getLauncherConfig().getDockerImage())); final StandardCheckConnectionInput rawInput = args.getConnectionConfiguration(); - final JsonNode fullConfig; - final UUID organizationId = rawInput.getActorContext().getOrganizationId(); - - if (organizationId != null && featureFlagClient.boolVariation(UseRuntimeSecretPersistence.INSTANCE, new Organization(organizationId))) { - try { - final SecretPersistenceConfig secretPersistenceConfig = airbyteApiClient.getSecretPersistenceConfigApi().getSecretsPersistenceConfig( - new SecretPersistenceConfigGetRequestBody().scopeType(ScopeType.ORGANIZATION).scopeId(organizationId)); - final RuntimeSecretPersistence runtimeSecretPersistence = - SecretPersistenceConfigHelper.fromApiSecretPersistenceConfig(secretPersistenceConfig); - fullConfig = - secretsRepositoryReader.hydrateConfigFromRuntimeSecretPersistence(rawInput.getConnectionConfiguration(), runtimeSecretPersistence); - } catch (final ApiException e) { - throw new RuntimeException(e); - } - } else { - fullConfig = secretsRepositoryReader.hydrateConfigFromDefaultSecretPersistence(rawInput.getConnectionConfiguration()); - } - - final StandardCheckConnectionInput input = new StandardCheckConnectionInput() - .withActorId(rawInput.getActorId()) - .withActorType(rawInput.getActorType()) - .withConnectionConfiguration(fullConfig) - .withActorContext(rawInput.getActorContext()); + final StandardCheckConnectionInput input = inputHydrator.getHydratedCheckInput(rawInput); final ActivityExecutionContext context = Activity.getExecutionContext(); final AtomicReference cancellationCallback = new AtomicReference<>(null); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionWorkflowImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionWorkflowImpl.java index d5b3bfaf8c5..ccbdfebfde8 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionWorkflowImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/check/connection/CheckConnectionWorkflowImpl.java @@ -21,7 +21,7 @@ import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.persistence.job.models.IntegrationLauncherConfig; import io.airbyte.persistence.job.models.JobRunConfig; -import io.airbyte.workers.temporal.check.connection.CheckConnectionActivity.CheckConnectionInput; +import io.airbyte.workers.models.CheckConnectionInput; import io.temporal.workflow.Workflow; import java.util.Map; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java index 463072f5325..baa6feb1355 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/discover/catalog/DiscoverCatalogActivityImpl.java @@ -44,7 +44,7 @@ import io.airbyte.workers.Worker; import io.airbyte.workers.general.DefaultDiscoverCatalogWorker; import io.airbyte.workers.helper.GsonPksExtractor; -import io.airbyte.workers.helpers.SecretPersistenceConfigHelper; +import io.airbyte.workers.helper.SecretPersistenceConfigHelper; import io.airbyte.workers.internal.AirbyteStreamFactory; import io.airbyte.workers.internal.VersionedAirbyteStreamFactory; import io.airbyte.workers.process.AirbyteIntegrationLauncher; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java index c7fcdf9abe5..57dd026c3ab 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/DbtTransformationActivityImpl.java @@ -43,7 +43,7 @@ import io.airbyte.workers.Worker; import io.airbyte.workers.general.DbtTransformationRunner; import io.airbyte.workers.general.DbtTransformationWorker; -import io.airbyte.workers.helpers.SecretPersistenceConfigHelper; +import io.airbyte.workers.helper.SecretPersistenceConfigHelper; import io.airbyte.workers.process.ProcessFactory; import io.airbyte.workers.sync.DbtLauncherWorker; import io.airbyte.workers.temporal.TemporalAttemptExecution; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java index e2912f44c04..271c616592f 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/NormalizationActivityImpl.java @@ -55,7 +55,7 @@ import io.airbyte.workers.ContainerOrchestratorConfig; import io.airbyte.workers.Worker; import io.airbyte.workers.general.DefaultNormalizationWorker; -import io.airbyte.workers.helpers.SecretPersistenceConfigHelper; +import io.airbyte.workers.helper.SecretPersistenceConfigHelper; import io.airbyte.workers.internal.NamespacingMapper; import io.airbyte.workers.normalization.DefaultNormalizationRunner; import io.airbyte.workers.process.ProcessFactory; diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/WebhookOperationActivityImpl.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/WebhookOperationActivityImpl.java index 83bfa14ba07..37428800919 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/WebhookOperationActivityImpl.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/sync/WebhookOperationActivityImpl.java @@ -26,7 +26,7 @@ import io.airbyte.metrics.lib.ApmTraceUtils; import io.airbyte.metrics.lib.MetricClientFactory; import io.airbyte.metrics.lib.OssMetricsRegistry; -import io.airbyte.workers.helpers.SecretPersistenceConfigHelper; +import io.airbyte.workers.helper.SecretPersistenceConfigHelper; import jakarta.inject.Singleton; import java.net.URI; import java.net.http.HttpClient; diff --git a/airbyte-workload-launcher/src/main/kotlin/config/ApplicationBeanFactory.kt b/airbyte-workload-launcher/src/main/kotlin/config/ApplicationBeanFactory.kt index d80ca3dd8a2..862b683de79 100644 --- a/airbyte-workload-launcher/src/main/kotlin/config/ApplicationBeanFactory.kt +++ b/airbyte-workload-launcher/src/main/kotlin/config/ApplicationBeanFactory.kt @@ -15,6 +15,7 @@ import io.airbyte.featureflag.FeatureFlagClient import io.airbyte.metrics.lib.MetricClient import io.airbyte.metrics.lib.MetricClientFactory import io.airbyte.metrics.lib.MetricEmittingApps +import io.airbyte.workers.CheckConnectionInputHydrator import io.airbyte.workers.ReplicationInputHydrator import io.micronaut.context.annotation.Factory import jakarta.inject.Singleton @@ -46,4 +47,17 @@ class ApplicationBeanFactory { ): ReplicationInputHydrator { return ReplicationInputHydrator(connectionApi, jobsApi, stateApi, secretsPersistenceConfigApi, secretsRepositoryReader, featureFlagClient) } + + @Singleton + fun checkInputHydrator( + secretsPersistenceConfigApi: SecretsPersistenceConfigApi, + secretsRepositoryReader: SecretsRepositoryReader, + featureFlagClient: FeatureFlagClient, + ): CheckConnectionInputHydrator { + return CheckConnectionInputHydrator( + secretsRepositoryReader, + secretsPersistenceConfigApi, + featureFlagClient, + ) + } } diff --git a/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/BuildInputStage.kt b/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/BuildInputStage.kt index b70d3132aac..35f08008d3d 100644 --- a/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/BuildInputStage.kt +++ b/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/BuildInputStage.kt @@ -1,13 +1,18 @@ package io.airbyte.workload.launcher.pipeline.stages import datadog.trace.api.Trace +import io.airbyte.config.WorkloadType import io.airbyte.persistence.job.models.ReplicationInput +import io.airbyte.workers.CheckConnectionInputHydrator import io.airbyte.workers.ReplicationInputHydrator import io.airbyte.workers.models.ReplicationActivityInput import io.airbyte.workload.launcher.metrics.CustomMetricPublisher import io.airbyte.workload.launcher.metrics.MeterFilterFactory +import io.airbyte.workload.launcher.pipeline.stages.model.CheckPayload import io.airbyte.workload.launcher.pipeline.stages.model.LaunchStage import io.airbyte.workload.launcher.pipeline.stages.model.LaunchStageIO +import io.airbyte.workload.launcher.pipeline.stages.model.SyncPayload +import io.airbyte.workload.launcher.pipeline.stages.model.WorkloadPayload import io.airbyte.workload.launcher.serde.PayloadDeserializer import io.github.oshai.kotlinlogging.KotlinLogging import jakarta.inject.Named @@ -24,6 +29,7 @@ private val logger = KotlinLogging.logger {} @Singleton @Named("build") class BuildInputStage( + private val checkInputHydrator: CheckConnectionInputHydrator, private val replicationInputHydrator: ReplicationInputHydrator, private val deserializer: PayloadDeserializer, metricPublisher: CustomMetricPublisher, @@ -34,15 +40,36 @@ class BuildInputStage( } override fun applyStage(input: LaunchStageIO): LaunchStageIO { - val parsed: ReplicationActivityInput = deserializer.toReplicationActivityInput(input.msg.workloadInput) - val hydrated: ReplicationInput = replicationInputHydrator.getHydratedReplicationInput(parsed) + val built = buildPayload(input.msg.workloadInput, input.msg.workloadType) return input.apply { - replicationInput = hydrated + payload = built } } override fun getStageName(): StageName { return StageName.BUILD } + + private fun buildPayload( + rawPayload: String, + type: WorkloadType, + ): WorkloadPayload = + when (type) { + WorkloadType.CHECK -> { + val parsed = deserializer.toStandardCheckConnectionInput(rawPayload) + val hydrated = checkInputHydrator.getHydratedCheckInput(parsed) + CheckPayload(hydrated) + } + + WorkloadType.SYNC -> { + val parsed: ReplicationActivityInput = deserializer.toReplicationActivityInput(rawPayload) + val hydrated: ReplicationInput = replicationInputHydrator.getHydratedReplicationInput(parsed) + SyncPayload(hydrated) + } + + else -> { + throw NotImplementedError("Unimplemented workload type: $type") + } + } } diff --git a/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/LaunchPodStage.kt b/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/LaunchPodStage.kt index 83b40d728c5..f29a3bb52c6 100644 --- a/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/LaunchPodStage.kt +++ b/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/LaunchPodStage.kt @@ -5,6 +5,7 @@ import io.airbyte.workload.launcher.metrics.CustomMetricPublisher import io.airbyte.workload.launcher.metrics.MeterFilterFactory import io.airbyte.workload.launcher.pipeline.stages.model.LaunchStage import io.airbyte.workload.launcher.pipeline.stages.model.LaunchStageIO +import io.airbyte.workload.launcher.pipeline.stages.model.SyncPayload import io.airbyte.workload.launcher.pods.KubePodClient import io.github.oshai.kotlinlogging.KotlinLogging import jakarta.inject.Named @@ -26,9 +27,12 @@ class LaunchPodStage(private val launcher: KubePodClient, metricPublisher: Custo } override fun applyStage(input: LaunchStageIO): LaunchStageIO { - val replInput = input.replicationInput!! + val payload = input.payload!! - launcher.launchReplication(replInput, input.msg) + when (payload) { + is SyncPayload -> launcher.launchReplication(payload.input!!, input.msg) + else -> throw NotImplementedError("") + } return input } diff --git a/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/model/StageIO.kt b/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/model/StageIO.kt index 621b6cc66b6..fa0177cabfa 100644 --- a/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/model/StageIO.kt +++ b/airbyte-workload-launcher/src/main/kotlin/pipeline/stages/model/StageIO.kt @@ -1,5 +1,6 @@ package io.airbyte.workload.launcher.pipeline.stages.model +import io.airbyte.config.StandardCheckConnectionInput import io.airbyte.persistence.job.models.ReplicationInput import io.airbyte.workload.launcher.pipeline.consumer.LauncherInput @@ -18,5 +19,15 @@ sealed class StageIO { data class LaunchStageIO( override val msg: LauncherInput, override val logCtx: Map = mapOf(), - var replicationInput: ReplicationInput? = null, + var payload: WorkloadPayload? = null, ) : StageIO() + +sealed class WorkloadPayload + +data class SyncPayload( + var input: ReplicationInput? = null, +) : WorkloadPayload() + +data class CheckPayload( + var input: StandardCheckConnectionInput? = null, +) : WorkloadPayload() diff --git a/airbyte-workload-launcher/src/main/kotlin/serde/PayloadDeserializer.kt b/airbyte-workload-launcher/src/main/kotlin/serde/PayloadDeserializer.kt index 66c05c20d31..054cff3c2d6 100644 --- a/airbyte-workload-launcher/src/main/kotlin/serde/PayloadDeserializer.kt +++ b/airbyte-workload-launcher/src/main/kotlin/serde/PayloadDeserializer.kt @@ -1,7 +1,7 @@ package io.airbyte.workload.launcher.serde import io.airbyte.commons.json.Jsons -import io.airbyte.persistence.job.models.ReplicationInput +import io.airbyte.config.StandardCheckConnectionInput import io.airbyte.workers.models.ReplicationActivityInput import jakarta.inject.Singleton @@ -11,7 +11,7 @@ class PayloadDeserializer { return Jsons.deserialize(payload, ReplicationActivityInput::class.java) } - fun toReplicationInput(payload: String): ReplicationInput { - return Jsons.deserialize(payload, ReplicationInput::class.java) + fun toStandardCheckConnectionInput(payload: String): StandardCheckConnectionInput { + return Jsons.deserialize(payload, StandardCheckConnectionInput::class.java) } } diff --git a/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/BuildInputStageTest.kt b/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/BuildInputStageTest.kt index 7b6e59adebb..492922ef913 100644 --- a/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/BuildInputStageTest.kt +++ b/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/BuildInputStageTest.kt @@ -6,48 +6,104 @@ package io.airbyte.workload.launcher.pipeline.stages import com.fasterxml.jackson.databind.node.POJONode import fixtures.RecordFixtures +import io.airbyte.config.ActorType +import io.airbyte.config.StandardCheckConnectionInput +import io.airbyte.config.WorkloadType import io.airbyte.persistence.job.models.ReplicationInput +import io.airbyte.workers.CheckConnectionInputHydrator import io.airbyte.workers.ReplicationInputHydrator import io.airbyte.workers.models.ReplicationActivityInput +import io.airbyte.workload.launcher.pipeline.stages.model.CheckPayload import io.airbyte.workload.launcher.pipeline.stages.model.LaunchStageIO +import io.airbyte.workload.launcher.pipeline.stages.model.SyncPayload import io.airbyte.workload.launcher.serde.PayloadDeserializer import io.mockk.every import io.mockk.mockk import io.mockk.verify import org.junit.jupiter.api.Test +import java.util.UUID class BuildInputStageTest { @Test - fun `parses input and hydrates`() { - val msgStr = "foo" + fun `parses sync input and hydrates`() { + val inputStr = "foo" val sourceConfig = POJONode("bar") val destConfig = POJONode("baz") - val replActivityInput = ReplicationActivityInput() - val replInput = + val unhydrated = ReplicationActivityInput() + val hydrated = ReplicationInput() .withSourceConfiguration(sourceConfig) .withDestinationConfiguration(destConfig) + val checkInputHydrator: CheckConnectionInputHydrator = mockk() val replicationInputHydrator: ReplicationInputHydrator = mockk() val deserializer: PayloadDeserializer = mockk() - every { deserializer.toReplicationActivityInput(msgStr) } returns replActivityInput - every { replicationInputHydrator.getHydratedReplicationInput(replActivityInput) } returns replInput + every { deserializer.toReplicationActivityInput(inputStr) } returns unhydrated + every { replicationInputHydrator.getHydratedReplicationInput(unhydrated) } returns hydrated val stage = BuildInputStage( + checkInputHydrator, replicationInputHydrator, deserializer, mockk(), ) - val io = LaunchStageIO(msg = RecordFixtures.launcherInput("1", msgStr, mapOf("label_key" to "label_value"), "/log/path")) + val io = LaunchStageIO(msg = RecordFixtures.launcherInput(workloadInput = inputStr, workloadType = WorkloadType.SYNC)) val result = stage.applyStage(io) verify { - deserializer.toReplicationActivityInput(msgStr) - replicationInputHydrator.getHydratedReplicationInput(replActivityInput) + deserializer.toReplicationActivityInput(inputStr) + replicationInputHydrator.getHydratedReplicationInput(unhydrated) } - assert(result.replicationInput == replInput) + when (val payload = result.payload) { + is SyncPayload -> assert(hydrated == payload.input) + else -> "Incorrect payload type: ${payload?.javaClass?.name}" + } + } + + @Test + fun `parses check input and hydrates`() { + val inputStr = "foo" + val unhydrated = + StandardCheckConnectionInput() + .withActorId(UUID.randomUUID()) + .withAdditionalProperty("whatever", "random value") + .withActorType(ActorType.DESTINATION) + + val hydrated = + StandardCheckConnectionInput() + .withActorId(UUID.randomUUID()) + .withAdditionalProperty("whatever", "random value") + .withActorType(ActorType.DESTINATION) + .withConnectionConfiguration(POJONode("bar")) + + val checkInputHydrator: CheckConnectionInputHydrator = mockk() + val replicationInputHydrator: ReplicationInputHydrator = mockk() + val deserializer: PayloadDeserializer = mockk() + every { deserializer.toStandardCheckConnectionInput(inputStr) } returns unhydrated + every { checkInputHydrator.getHydratedCheckInput(unhydrated) } returns hydrated + + val stage = + BuildInputStage( + checkInputHydrator, + replicationInputHydrator, + deserializer, + mockk(), + ) + val io = LaunchStageIO(msg = RecordFixtures.launcherInput(workloadInput = inputStr, workloadType = WorkloadType.CHECK)) + + val result = stage.applyStage(io) + + verify { + deserializer.toStandardCheckConnectionInput(inputStr) + checkInputHydrator.getHydratedCheckInput(unhydrated) + } + + when (val payload = result.payload) { + is CheckPayload -> assert(hydrated == payload.input) + else -> "Incorrect payload type: ${payload?.javaClass?.name}" + } } } diff --git a/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/EnforceMutexStageTest.kt b/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/EnforceMutexStageTest.kt index fbb6f95f76b..1e28a4daa8a 100644 --- a/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/EnforceMutexStageTest.kt +++ b/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/EnforceMutexStageTest.kt @@ -5,9 +5,9 @@ package io.airbyte.workload.launcher.pipeline.stages import fixtures.RecordFixtures -import io.airbyte.persistence.job.models.ReplicationInput import io.airbyte.workload.launcher.metrics.CustomMetricPublisher import io.airbyte.workload.launcher.pipeline.stages.model.LaunchStageIO +import io.airbyte.workload.launcher.pipeline.stages.model.SyncPayload import io.airbyte.workload.launcher.pods.KubePodClient import io.mockk.every import io.mockk.mockk @@ -17,7 +17,7 @@ import org.junit.jupiter.api.Test class EnforceMutexStageTest { @Test fun `deletes existing pods for mutex key`() { - val replInput = ReplicationInput() + val payload = SyncPayload() val mutexKey = "a unique key" val launcher: KubePodClient = mockk() @@ -26,7 +26,7 @@ class EnforceMutexStageTest { val stage = EnforceMutexStage(launcher, metricClient) val io = - LaunchStageIO(msg = RecordFixtures.launcherInput(mutexKey = mutexKey), replicationInput = replInput) + LaunchStageIO(msg = RecordFixtures.launcherInput(mutexKey = mutexKey), payload = payload) val result = stage.applyStage(io) @@ -34,12 +34,12 @@ class EnforceMutexStageTest { launcher.deleteMutexPods(mutexKey) } - assert(result.replicationInput == replInput) + assert(result.payload == payload) } @Test fun `noops if mutex key not present`() { - val replInput = ReplicationInput() + val payload = SyncPayload() val launcher: KubePodClient = mockk() val metricClient: CustomMetricPublisher = mockk() @@ -47,7 +47,7 @@ class EnforceMutexStageTest { val stage = EnforceMutexStage(launcher, metricClient) val io = - LaunchStageIO(msg = RecordFixtures.launcherInput(mutexKey = null), replicationInput = replInput) + LaunchStageIO(msg = RecordFixtures.launcherInput(mutexKey = null), payload = payload) val result = stage.applyStage(io) @@ -55,6 +55,6 @@ class EnforceMutexStageTest { launcher.deleteMutexPods(any()) } - assert(result.replicationInput == replInput) + assert(result.payload == payload) } } diff --git a/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/LaunchPodStageTest.kt b/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/LaunchPodStageTest.kt index e3cc0396f28..4f50e517ffa 100644 --- a/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/LaunchPodStageTest.kt +++ b/airbyte-workload-launcher/src/test/kotlin/pipeline/stages/LaunchPodStageTest.kt @@ -7,6 +7,7 @@ package io.airbyte.workload.launcher.pipeline.stages import fixtures.RecordFixtures import io.airbyte.persistence.job.models.ReplicationInput import io.airbyte.workload.launcher.pipeline.stages.model.LaunchStageIO +import io.airbyte.workload.launcher.pipeline.stages.model.SyncPayload import io.airbyte.workload.launcher.pods.KubePodClient import io.mockk.every import io.mockk.mockk @@ -18,6 +19,7 @@ class LaunchPodStageTest { @Test fun `launches replication`() { val replInput = ReplicationInput() + val payload = SyncPayload(replInput) val launcher: KubePodClient = mockk() every { launcher.launchReplication(any(), any()) } returns Unit @@ -25,7 +27,7 @@ class LaunchPodStageTest { val stage = LaunchPodStage(launcher, mockk()) val workloadId = UUID.randomUUID().toString() val msg = RecordFixtures.launcherInput(workloadId) - val io = LaunchStageIO(msg = msg, replicationInput = replInput) + val io = LaunchStageIO(msg = msg, payload = payload) val result = stage.applyStage(io) @@ -33,6 +35,6 @@ class LaunchPodStageTest { launcher.launchReplication(replInput, msg) } - assert(result.replicationInput == replInput) + assert(result.payload == payload) } } From cdec2773e39a256ae56377d6892c952b85422f21 Mon Sep 17 00:00:00 2001 From: Chandler Prall Date: Wed, 13 Dec 2023 14:09:45 -0700 Subject: [PATCH 23/27] =?UTF-8?q?=F0=9F=AA=9F=C2=A0=F0=9F=8E=A8=20update?= =?UTF-8?q?=20connection=20filters=20ui=20(#10318)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/components/ui/ListBox/ListBox.tsx | 8 +- airbyte-webapp/src/locales/en.json | 1 + .../AllConnectionsPage/AllConnectionsPage.tsx | 84 +++++++++++++++---- 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx b/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx index 3c9d9a60e8e..66db5ea52e7 100644 --- a/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx +++ b/airbyte-webapp/src/components/ui/ListBox/ListBox.tsx @@ -3,7 +3,7 @@ import { Listbox } from "@headlessui/react"; import { Float } from "@headlessui-float/react"; import classNames from "classnames"; import isEqual from "lodash/isEqual"; -import React from "react"; +import React, { ComponentPropsWithoutRef } from "react"; import { useIntl } from "react-intl"; import { Text } from "components/ui/Text"; @@ -52,6 +52,7 @@ export interface ListBoxProps { className?: string; optionsMenuClassName?: string; optionClassName?: string; + optionTextAs?: ComponentPropsWithoutRef["as"]; selectedOptionClassName?: string; options: Array>; selectedValue?: T; @@ -87,6 +88,7 @@ export const ListBox = ({ controlButton: ControlButton = DefaultControlButton, optionsMenuClassName, optionClassName, + optionTextAs, selectedOptionClassName, "data-testid": testId, hasError, @@ -152,7 +154,9 @@ export const ListBox = ({ })} > {icon && {icon}} - {label} + + {label} + )} diff --git a/airbyte-webapp/src/locales/en.json b/airbyte-webapp/src/locales/en.json index 41734bcf34a..8fc7cb73135 100644 --- a/airbyte-webapp/src/locales/en.json +++ b/airbyte-webapp/src/locales/en.json @@ -686,6 +686,7 @@ "tables.connections.filters.status.all": "All statuses", "tables.connections.filters.status.healthy": "Healthy", "tables.connections.filters.status.failed": "Failed", + "tables.connections.filters.status.running": "Running", "tables.connections.filters.status.paused": "Paused", "tables.connections.filters.source.all": "All sources", "tables.connections.filters.destination.all": "All destinations", diff --git a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx index 887c2d7ac25..886bff4ced9 100644 --- a/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx +++ b/airbyte-webapp/src/pages/connections/AllConnectionsPage/AllConnectionsPage.tsx @@ -26,11 +26,12 @@ import styles from "./AllConnectionsPage.module.scss"; import ConnectionsTable from "./ConnectionsTable"; import { ConnectionRoutePaths } from "../../routePaths"; -type SummaryKey = "healthy" | "failed" | "paused"; +type SummaryKey = "healthy" | "failed" | "paused" | "running"; const connectionStatColors: Record["color"]> = { healthy: "green600", failed: "red", paused: "grey", + running: "blue", }; const ConnectionsSummary: React.FC> = (props) => { const keys = Object.keys(props) as SummaryKey[]; @@ -76,28 +77,69 @@ const statusFilterOptions: FilterOption[] = [ }, { label: ( - - -   - + + + + + + + + +   + + + ), value: "healthy", }, { label: ( - - -   - + + + + + + + + +   + + + ), value: "failed", }, { label: ( - - -   - + + + + + + + + +   + + + + ), + value: "running", + }, + { + label: ( + + + + + + + + +   + + + ), value: "paused", }, @@ -106,6 +148,11 @@ const statusFilterOptions: FilterOption[] = [ const isConnectionPaused = ( connection: WebBackendConnectionListItem ): connection is WebBackendConnectionListItem & { status: "inactive" } => connection.status === "inactive"; + +const isConnectionRunning = ( + connection: WebBackendConnectionListItem +): connection is WebBackendConnectionListItem & { isSyncing: true } => connection.isSyncing; + const isConnectionFailed = ( connection: WebBackendConnectionListItem ): connection is WebBackendConnectionListItem & { latestSyncJobStatus: "failed" } => @@ -140,10 +187,13 @@ export const AllConnectionsPage: React.FC = () => { return connections.filter((connection) => { if (statusFilter) { const isPaused = isConnectionPaused(connection); + const isRunning = isConnectionRunning(connection); const isFailed = isConnectionFailed(connection); if (statusFilter === "paused" && !isPaused) { return false; - } else if (statusFilter === "failed" && !isFailed) { + } else if (statusFilter === "running" && !isRunning) { + return false; + } else if (statusFilter === "failed" && (!isFailed || isPaused)) { return false; } else if (statusFilter === "healthy" && (isPaused || isFailed)) { return false; @@ -172,6 +222,7 @@ export const AllConnectionsPage: React.FC = () => { // order here governs render order healthy: 0, failed: 0, + running: 0, paused: 0, } ); @@ -226,6 +277,7 @@ export const AllConnectionsPage: React.FC = () => { @@ -238,6 +290,7 @@ export const AllConnectionsPage: React.FC = () => { buttonClassName={styles.filterButton} optionsMenuClassName={styles.filterOptionsMenu} optionClassName={styles.filterOption} + optionTextAs="span" options={availableSourceOptions} selectedValue={sourceFilterSelection.value} onSelect={(value) => @@ -249,6 +302,7 @@ export const AllConnectionsPage: React.FC = () => { @@ -268,7 +322,7 @@ export const AllConnectionsPage: React.FC = () => { setDestinationFilterSelection(availableDestinationOptions[0]); }} > - + From 7b97b711d90b22f2242c8a34ec4f7f6cf9176452 Mon Sep 17 00:00:00 2001 From: Bryce Groff Date: Wed, 13 Dec 2023 14:10:02 -0800 Subject: [PATCH 24/27] Add the OpenAPI Spec for the Applications service. (#10315) --- airbyte-api/src/main/openapi/config.yaml | 131 +++++++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/airbyte-api/src/main/openapi/config.yaml b/airbyte-api/src/main/openapi/config.yaml index b3ec4391299..9ad16d4f3fe 100644 --- a/airbyte-api/src/main/openapi/config.yaml +++ b/airbyte-api/src/main/openapi/config.yaml @@ -83,6 +83,8 @@ tags: description: Interactions with organizations. - name: deployment_metadata description: Metadata about the Airbyte deployment + - name: applications + description: Applications can be used to provide api access for Users paths: /v1/workspaces/create: @@ -4279,6 +4281,84 @@ paths: $ref: "#/components/schemas/DeploymentMetadataRead" "404": $ref: "#/components/responses/NotFoundResponse" + /v1/applications/list: + post: + tags: + - applications + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationReadList" + description: Successful operation + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" + operationId: listApplications + summary: Returns all Applications for a User. + description: List Applications for a User. + /v1/applications/delete: + post: + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationId" + required: true + tags: + - applications + responses: + "200": + description: Successful operation + "404": + $ref: "#/components/responses/NotFoundResponse" + "422": + $ref: "#/components/responses/InvalidInputResponse" + operationId: deleteApplication + summary: Deletes an Application. + description: Deletes an Application. + /v1/applications/create: + post: + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationCreate" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationRead" + description: Successful operation + "422": + $ref: "#/components/responses/InvalidInputResponse" + operationId: createApplication + summary: Creates a new Application. + description: Creates a new Application. + /v1/applications/token: + post: + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/ApplicationTokenRequest" + required: true + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/AccessToken" + description: Successful operation + "422": + $ref: "#/components/responses/InvalidInputResponse" + operationId: applicationTokenRequest + summary: Grant an Access Token for an Application. + description: Takes the client_id and client_secret for an application and returns an Access Token. components: securitySchemes: @@ -8962,6 +9042,57 @@ components: $ref: "#/components/schemas/ScopeId" SecretPersistenceConfigurationJson: type: object + ApplicationCreate: + required: + - name + type: object + properties: + name: + type: string + ApplicationId: + type: string + ApplicationReadList: + required: + - applications + type: object + properties: + applications: + type: array + items: + $ref: "#/components/schemas/ApplicationRead" + ApplicationRead: + required: + - id + - name + - client_id + - client_secret + type: object + properties: + id: + type: string + name: + type: string + client_id: + type: string + client_secret: + type: string + ApplicationTokenRequest: + required: + - client_id + - client_secret + type: object + properties: + client_id: + type: string + client_secret: + type: string + AccessToken: + required: + - access_token + type: object + properties: + access_token: + type: string responses: NotFoundResponse: From 55a1c5b55962739ca5d633df949f83e8c3c4f287 Mon Sep 17 00:00:00 2001 From: Flutra Osmani <129116758+flutra-osmani@users.noreply.github.com> Date: Wed, 13 Dec 2023 14:12:21 -0800 Subject: [PATCH 25/27] drop keycloak tables from config db (#10110) --- .../io/airbyte/bootloader/BootloaderTest.java | 2 +- .../V0_50_33_009__DropKeycloakTables.java | 58 ++++++++++++++ .../V0_50_33_009__DropKeycloakTablesTest.java | 79 +++++++++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_009__DropKeycloakTables.java create mode 100644 airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_33_009__DropKeycloakTablesTest.java diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index 4c935278df3..c1c400ffb28 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -95,7 +95,7 @@ class BootloaderTest { // ⚠️ This line should change with every new migration to show that you meant to make a new // migration to the prod database - private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.33.008"; + private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.33.009"; private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.001"; private static final String CDK_VERSION = "1.2.3"; diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_009__DropKeycloakTables.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_009__DropKeycloakTables.java new file mode 100644 index 00000000000..b1ef9050dbf --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_009__DropKeycloakTables.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import java.util.Arrays; +import java.util.List; +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// Remove all keycloak tables from the public schema. +public class V0_50_33_009__DropKeycloakTables extends BaseJavaMigration { + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_33_009__DropKeycloakTables.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + List tablesToDelete = Arrays.asList( + "admin_event_entity", "associated_policy", "authentication_execution", "authentication_flow", "authenticator_config", + "authenticator_config_entry", "broker_link", "client", "client_attributes", "client_auth_flow_bindings", "client_initial_access", + "client_node_registrations", "client_scope", "client_scope_attributes", "client_scope_client", "client_scope_role_mapping", "client_session", + "client_session_auth_status", "client_session_note", "client_session_prot_mapper", "client_session_role", "client_user_session_note", + "component", "component_config", "composite_role", "credential", "databasechangelog", "databasechangeloglock", "default_client_scope", + "event_entity", "fed_user_attribute", "fed_user_consent", "fed_user_consent_cl_scope", "fed_user_credential", "fed_user_group_membership", + "fed_user_required_action", "fed_user_role_mapping", "federated_identity", "federated_user", "group_attribute", "group_role_mapping", + "identity_provider", "identity_provider_config", "identity_provider_mapper", "idp_mapper_config", "keycloak_group", "keycloak_role", + "migration_model", "offline_client_session", "offline_user_session", "policy_config", "protocol_mapper", "protocol_mapper_config", "realm", + "realm_attribute", "realm_default_groups", "realm_enabled_event_types", "realm_events_listeners", "realm_localizations", + "realm_required_credential", "realm_smtp_config", "realm_supported_locales", "redirect_uris", "required_action_config", + "required_action_provider", "resource_attribute", "resource_policy", "resource_server", "resource_server_perm_ticket", + "resource_server_policy", "resource_server_resource", "resource_server_scope", "resource_uris", "role_attribute", "scope_mapping", + "scope_policy", "user_attribute", "user_consent", "user_consent_client_scope", "user_entity", "user_federation_config", + "user_federation_mapper", "user_federation_mapper_config", "user_federation_provider", "user_group_membership", "user_required_action", + "user_role_mapping", "user_session", "user_session_note", "username_login_failure", "web_origins"); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + dropTables(ctx, tablesToDelete); + + LOGGER.info("Completed migration: {}", this.getClass().getSimpleName()); + } + + public static void dropTables(final DSLContext ctx, final List tablesToDelete) { + tablesToDelete.forEach(tableName -> { + ctx.dropTableIfExists(DSL.table(tableName)).cascade().execute(); + }); + } + +} diff --git a/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_33_009__DropKeycloakTablesTest.java b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_33_009__DropKeycloakTablesTest.java new file mode 100644 index 00000000000..aec82d942bb --- /dev/null +++ b/airbyte-db/db-lib/src/test/java/io/airbyte/db/instance/configs/migrations/V0_50_33_009__DropKeycloakTablesTest.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import io.airbyte.db.instance.configs.AbstractConfigsDatabaseTest; +import java.util.Arrays; +import java.util.List; +import org.jooq.DSLContext; +import org.jooq.exception.IOException; +import org.jooq.impl.DSL; +import org.junit.Test; + +public class V0_50_33_009__DropKeycloakTablesTest extends AbstractConfigsDatabaseTest { + + @Test + void testDropTables() throws IOException { + final DSLContext context = getDslContext(); + + List tablesToDelete = Arrays.asList( + "admin_event_entity", "associated_policy", "authentication_execution", "authentication_flow", "authenticator_config", + "authenticator_config_entry", "broker_link", "client", "client_attributes", "client_auth_flow_bindings", "client_initial_access", + "client_node_registrations", "client_scope", "client_scope_attributes", "client_scope_client", "client_scope_role_mapping", "client_session", + "client_session_auth_status", "client_session_note", "client_session_prot_mapper", "client_session_role", "client_user_session_note", + "component", "component_config", "composite_role", "credential", "databasechangelog", "databasechangeloglock", "default_client_scope", + "event_entity", "fed_user_attribute", "fed_user_consent", "fed_user_consent_cl_scope", "fed_user_credential", "fed_user_group_membership", + "fed_user_required_action", "fed_user_role_mapping", "federated_identity", "federated_user", "group_attribute", "group_role_mapping", + "identity_provider", "identity_provider_config", "identity_provider_mapper", "idp_mapper_config", "keycloak_group", "keycloak_role", + "migration_model", "offline_client_session", "offline_user_session", "policy_config", "protocol_mapper", "protocol_mapper_config", "realm", + "realm_attribute", "realm_default_groups", "realm_enabled_event_types", "realm_events_listeners", "realm_localizations", + "realm_required_credential", "realm_smtp_config", "realm_supported_locales", "redirect_uris", "required_action_config", + "required_action_provider", "resource_attribute", "resource_policy", "resource_server", "resource_server_perm_ticket", + "resource_server_policy", "resource_server_resource", "resource_server_scope", "resource_uris", "role_attribute", "scope_mapping", + "scope_policy", "user_attribute", "user_consent", "user_consent_client_scope", "user_entity", "user_federation_config", + "user_federation_mapper", "user_federation_mapper_config", "user_federation_provider", "user_group_membership", "user_required_action", + "user_role_mapping", "user_session", "user_session_note", "username_login_failure", "web_origins"); + + for (String tableName : tablesToDelete) { + assertTrue("Table " + tableName + " should exist before dropping", tableExists(context, tableName)); + } + + List tablesBeforeMigration = fetchAllTableNames(context); + + // Drop the tables + V0_50_33_009__DropKeycloakTables.dropTables(context, tablesToDelete); + + // Verify that each table does not exist after the drop operation + for (String tableName : tablesToDelete) { + assertFalse("Table " + tableName + " should not exist after dropping", tableExists(context, tableName)); + } + + List tablesAfterMigration = fetchAllTableNames(context); + + tablesBeforeMigration.removeAll(tablesToDelete); + + // Verify that no other tables except Keycloak tables were dropped + assertEquals(tablesBeforeMigration, tablesAfterMigration); + } + + private List fetchAllTableNames(DSLContext ctx) { + return ctx.meta().getTables().stream() + .map(org.jooq.Table::getName) + .collect(java.util.stream.Collectors.toList()); + } + + private boolean tableExists(final DSLContext context, String tableName) { + return context.fetchExists( + org.jooq.impl.DSL.select() + .from("information_schema.tables") + .where(DSL.field("table_name").eq(tableName) + .and(DSL.field("table_schema").eq("public")))); + } + +} From 885c5cf36f7acfbc49fb58ce758b6d33d35afaeb Mon Sep 17 00:00:00 2001 From: Jimmy Ma Date: Wed, 13 Dec 2023 16:15:34 -0800 Subject: [PATCH 26/27] Add termination source tracking to the workload API (#10343) --- .../src/main/openapi/workload-openapi.yaml | 12 + .../io/airbyte/bootloader/BootloaderTest.java | 2 +- airbyte-commons-worker/build.gradle.kts | 1 + .../workers/sync/WorkloadApiWorker.java | 26 +- .../internal/sync/WorkloadApiWorkerTest.kt | 228 ++++++++++++++++++ .../ReplicationJobOrchestrator.java | 30 ++- .../ReplicationJobOrchestratorTest.java | 5 +- ...33_010__AddFailureTrackingToWorkloads.java | 39 +++ .../configs_database/schema_dump.txt | 2 + .../io/airbyte/workload/api/WorkloadApi.kt | 4 +- .../airbyte/workload/api/domain/Workload.kt | 2 + .../api/domain/WorkloadFailureRequest.kt | 2 + .../workload/handler/WorkloadHandler.kt | 12 +- .../workload/handler/WorkloadHandlerImpl.kt | 16 +- .../workload/handler/WorkloadMapper.kt | 2 + .../workload/repository/WorkloadRepository.kt | 7 + .../workload/repository/domain/Workload.kt | 6 + .../airbyte/workload/api/WorkloadApiTest.kt | 12 +- .../handler/WorkloadHandlerImplTest.kt | 28 +-- 19 files changed, 399 insertions(+), 37 deletions(-) create mode 100644 airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/sync/WorkloadApiWorkerTest.kt create mode 100644 airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_010__AddFailureTrackingToWorkloads.java diff --git a/airbyte-api/src/main/openapi/workload-openapi.yaml b/airbyte-api/src/main/openapi/workload-openapi.yaml index fe9f965c179..d72ac1a2da5 100644 --- a/airbyte-api/src/main/openapi/workload-openapi.yaml +++ b/airbyte-api/src/main/openapi/workload-openapi.yaml @@ -329,6 +329,12 @@ components: nullable: true type: $ref: '#/components/schemas/WorkloadType' + terminationSource: + type: string + nullable: true + terminationReason: + type: string + nullable: true WorkloadCancelRequest: required: - reason @@ -386,6 +392,12 @@ components: properties: workloadId: type: string + source: + type: string + nullable: true + reason: + type: string + nullable: true WorkloadHeartbeatRequest: required: - workloadId diff --git a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java index c1c400ffb28..e7563785c41 100644 --- a/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java +++ b/airbyte-bootloader/src/test/java/io/airbyte/bootloader/BootloaderTest.java @@ -95,7 +95,7 @@ class BootloaderTest { // ⚠️ This line should change with every new migration to show that you meant to make a new // migration to the prod database - private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.33.009"; + private static final String CURRENT_CONFIGS_MIGRATION_VERSION = "0.50.33.010"; private static final String CURRENT_JOBS_MIGRATION_VERSION = "0.50.4.001"; private static final String CDK_VERSION = "1.2.3"; diff --git a/airbyte-commons-worker/build.gradle.kts b/airbyte-commons-worker/build.gradle.kts index 713f2f76801..89aacabdc66 100644 --- a/airbyte-commons-worker/build.gradle.kts +++ b/airbyte-commons-worker/build.gradle.kts @@ -72,6 +72,7 @@ dependencies { testImplementation(libs.bundles.micronaut.test) testImplementation(libs.json.path) testImplementation(libs.mockito.inline) + testImplementation(libs.mockk) testImplementation(variantOf(libs.opentracing.util.test) { classifier("tests") }) testImplementation(libs.postgresql) testImplementation(libs.platform.testcontainers.postgresql) diff --git a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/WorkloadApiWorker.java b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/WorkloadApiWorker.java index 6dfab22d106..5ecba00a934 100644 --- a/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/WorkloadApiWorker.java +++ b/airbyte-commons-worker/src/main/java/io/airbyte/workers/sync/WorkloadApiWorker.java @@ -24,6 +24,8 @@ import io.airbyte.persistence.job.models.ReplicationInput; import io.airbyte.workers.Worker; import io.airbyte.workers.exception.WorkerException; +import io.airbyte.workers.internal.exception.DestinationException; +import io.airbyte.workers.internal.exception.SourceException; import io.airbyte.workers.models.ReplicationActivityInput; import io.airbyte.workers.orchestrator.OrchestratorNameGenerator; import io.airbyte.workers.storage.DocumentStoreClient; @@ -54,6 +56,9 @@ */ public class WorkloadApiWorker implements Worker { + private static final String DESTINATION = "destination"; + private static final String SOURCE = "source"; + private static final Logger log = LoggerFactory.getLogger(WorkloadApiWorker.class); private static final Set TERMINAL_STATUSES = Set.of(WorkloadStatus.CANCELLED, WorkloadStatus.FAILURE, WorkloadStatus.SUCCESS); private final DocumentStoreClient documentStoreClient; @@ -112,8 +117,9 @@ public ReplicationOutput run(final ReplicationInput replicationInput, final Path // Wait until workload reaches a terminal status int i = 0; final Duration sleepInterval = Duration.ofSeconds(featureFlagClient.intVariation(WorkloadPollingInterval.INSTANCE, getFeatureFlagContext())); + Workload workload; while (true) { - final Workload workload = getWorkload(workloadId); + workload = getWorkload(workloadId); if (workload.getStatus() != null) { if (TERMINAL_STATUSES.contains(workload.getStatus())) { @@ -130,7 +136,23 @@ public ReplicationOutput run(final ReplicationInput replicationInput, final Path sleep(sleepInterval.toMillis()); } - return getReplicationOutput(workloadId); + if (workload.getStatus() == WorkloadStatus.FAILURE) { + if (SOURCE.equals(workload.getTerminationSource())) { + throw new SourceException(workload.getTerminationReason()); + } else if (DESTINATION.equals(workload.getTerminationSource())) { + throw new DestinationException(workload.getTerminationReason()); + } else { + throw new WorkerException(workload.getTerminationReason()); + } + } else if (workload.getStatus() == WorkloadStatus.CANCELLED) { + throw new WorkerException("Replication cancelled by " + workload.getTerminationSource()); + } + + final ReplicationOutput output = getReplicationOutput(workloadId); + if (output == null) { + throw new WorkerException("Failed to read replication output"); + } + return output; } @Override diff --git a/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/sync/WorkloadApiWorkerTest.kt b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/sync/WorkloadApiWorkerTest.kt new file mode 100644 index 00000000000..72c5016a7a9 --- /dev/null +++ b/airbyte-commons-worker/src/test/java/io/airbyte/workers/internal/sync/WorkloadApiWorkerTest.kt @@ -0,0 +1,228 @@ +package io.airbyte.workers.internal.sync + +import io.airbyte.api.client.AirbyteApiClient +import io.airbyte.api.client.generated.ConnectionApi +import io.airbyte.api.client.model.generated.ConnectionRead +import io.airbyte.api.client.model.generated.Geography +import io.airbyte.commons.json.Jsons +import io.airbyte.config.ReplicationAttemptSummary +import io.airbyte.config.ReplicationOutput +import io.airbyte.config.StandardSyncSummary +import io.airbyte.featureflag.FeatureFlagClient +import io.airbyte.featureflag.TestClient +import io.airbyte.persistence.job.models.JobRunConfig +import io.airbyte.persistence.job.models.ReplicationInput +import io.airbyte.workers.exception.WorkerException +import io.airbyte.workers.internal.exception.DestinationException +import io.airbyte.workers.internal.exception.SourceException +import io.airbyte.workers.models.ReplicationActivityInput +import io.airbyte.workers.orchestrator.OrchestratorNameGenerator +import io.airbyte.workers.storage.DocumentStoreClient +import io.airbyte.workers.sync.WorkloadApiWorker +import io.airbyte.workers.workload.WorkloadIdGenerator +import io.airbyte.workload.api.client.generated.WorkloadApi +import io.airbyte.workload.api.client.model.generated.Workload +import io.airbyte.workload.api.client.model.generated.WorkloadStatus +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertThrows +import java.nio.file.Path +import java.util.Optional +import java.util.UUID + +internal class WorkloadApiWorkerTest { + private var workloadIdGenerator: WorkloadIdGenerator = mockk() + private var documentStoreClient: DocumentStoreClient = mockk() + private var apiClient: AirbyteApiClient = mockk() + private var connectionApi: ConnectionApi = mockk() + private var workloadApi: WorkloadApi = mockk() + private var featureFlagClient: FeatureFlagClient = mockk() + private lateinit var replicationActivityInput: ReplicationActivityInput + private lateinit var replicationInput: ReplicationInput + private lateinit var workloadApiWorker: WorkloadApiWorker + private lateinit var jobRoot: Path + + @BeforeEach + fun beforeEach() { + every { apiClient.connectionApi } returns connectionApi + featureFlagClient = TestClient() + jobRoot = Path.of("test", "path") + replicationActivityInput = ReplicationActivityInput() + replicationInput = ReplicationInput() + workloadApiWorker = + WorkloadApiWorker( + documentStoreClient, + OrchestratorNameGenerator("testNs"), + apiClient, + workloadApi, + workloadIdGenerator, + replicationActivityInput, + featureFlagClient, + ) + } + + @Test + fun testSuccessfulReplication() { + val jobId = 13L + val attemptNumber = 37 + val workloadId = "my-workload" + val expectedDocPrefix = "testNs/orchestrator-repl-job-$jobId-attempt-$attemptNumber" + val expectedOutput = + ReplicationOutput() + .withReplicationAttemptSummary(ReplicationAttemptSummary().withStatus(StandardSyncSummary.ReplicationStatus.COMPLETED)) + initializeReplicationInput(jobId, attemptNumber) + + every { workloadIdGenerator.generateSyncWorkloadId(replicationInput.connectionId, jobId, attemptNumber) } returns workloadId + + every { connectionApi.getConnection(any()) } returns ConnectionRead().geography(Geography.US) + every { workloadApi.workloadCreate(any()) } returns Unit + every { workloadApi.workloadGet(workloadId) } returns mockWorkload(WorkloadStatus.SUCCESS) + + every { documentStoreClient.read("$expectedDocPrefix/SUCCEEDED") } returns Optional.of(Jsons.serialize(expectedOutput)) + + val output = workloadApiWorker.run(replicationInput, jobRoot) + assertEquals(expectedOutput, output) + } + + @Test + fun testReplicationWithMissingOutput() { + val jobId = 42L + val attemptNumber = 1 + val workloadId = "my-failed-workload" + val expectedDocPrefix = "testNs/orchestrator-repl-job-$jobId-attempt-$attemptNumber" + initializeReplicationInput(jobId, attemptNumber) + + every { workloadIdGenerator.generateSyncWorkloadId(replicationInput.connectionId, jobId, attemptNumber) } returns workloadId + + every { connectionApi.getConnection(any()) } returns ConnectionRead().geography(Geography.US) + every { workloadApi.workloadCreate(any()) } returns Unit + every { workloadApi.workloadGet(workloadId) } returns mockWorkload(WorkloadStatus.SUCCESS) + every { documentStoreClient.read("$expectedDocPrefix/SUCCEEDED") } returns Optional.empty() + + assertThrows { workloadApiWorker.run(replicationInput, jobRoot) } + } + + @Test + fun testCancelledReplication() { + val jobId = 42L + val attemptNumber = 1 + val workloadId = "my-failed-workload" + initializeReplicationInput(jobId, attemptNumber) + + every { workloadIdGenerator.generateSyncWorkloadId(replicationInput.connectionId, jobId, attemptNumber) } returns workloadId + + every { connectionApi.getConnection(any()) } returns ConnectionRead().geography(Geography.US) + every { workloadApi.workloadCreate(any()) } returns Unit + every { workloadApi.workloadGet(workloadId) } returns + mockWorkload( + WorkloadStatus.CANCELLED, + terminationSource = "user", + terminationReason = "Oops... user cancelled", + ) + + assertThrows { workloadApiWorker.run(replicationInput, jobRoot) } + } + + @Test + fun testFailedReplicationWithPlatformFailure() { + val jobId = 42L + val attemptNumber = 1 + val workloadId = "my-failed-workload" + initializeReplicationInput(jobId, attemptNumber) + + every { workloadIdGenerator.generateSyncWorkloadId(replicationInput.connectionId, jobId, attemptNumber) } returns workloadId + + every { connectionApi.getConnection(any()) } returns ConnectionRead().geography(Geography.US) + every { workloadApi.workloadCreate(any()) } returns Unit + every { workloadApi.workloadGet(workloadId) } returns + mockWorkload( + WorkloadStatus.FAILURE, + terminationSource = "airbyte_platform", + terminationReason = "Oops... platform died", + ) + + assertThrows { workloadApiWorker.run(replicationInput, jobRoot) } + } + + @Test + fun testFailedReplicationWithSourceFailure() { + val jobId = 43L + val attemptNumber = 1 + val workloadId = "my-failed-workload" + initializeReplicationInput(jobId, attemptNumber) + + every { workloadIdGenerator.generateSyncWorkloadId(replicationInput.connectionId, jobId, attemptNumber) } returns workloadId + + every { connectionApi.getConnection(any()) } returns ConnectionRead().geography(Geography.US) + every { workloadApi.workloadCreate(any()) } returns Unit + every { workloadApi.workloadGet(workloadId) } returns + mockWorkload( + WorkloadStatus.FAILURE, + terminationSource = "source", + terminationReason = "Oops... source died", + ) + + assertThrows { workloadApiWorker.run(replicationInput, jobRoot) } + } + + @Test + fun testFailedReplicationWithDestinationFailure() { + val jobId = 44L + val attemptNumber = 1 + val workloadId = "my-failed-workload" + initializeReplicationInput(jobId, attemptNumber) + + every { workloadIdGenerator.generateSyncWorkloadId(replicationInput.connectionId, jobId, attemptNumber) } returns workloadId + + every { connectionApi.getConnection(any()) } returns ConnectionRead().geography(Geography.US) + every { workloadApi.workloadCreate(any()) } returns Unit + every { workloadApi.workloadGet(workloadId) } returns + mockWorkload( + WorkloadStatus.FAILURE, + terminationSource = "destination", + terminationReason = "Oops... destination died", + ) + + assertThrows { workloadApiWorker.run(replicationInput, jobRoot) } + } + + fun initializeReplicationInput( + jobId: Long, + attemptNumber: Int, + ) { + val workspaceId = UUID.randomUUID() + val connectionId = UUID.randomUUID() + val sourceId = UUID.randomUUID() + val destinationId = UUID.randomUUID() + val jobRunConfig = JobRunConfig().withJobId(jobId.toString()).withAttemptId(attemptNumber.toLong()) + + replicationInput.apply { + this.workspaceId = workspaceId + this.connectionId = connectionId + this.jobRunConfig = jobRunConfig + } + + replicationActivityInput.apply { + this.workspaceId = workspaceId + this.connectionId = connectionId + this.sourceId = sourceId + this.destinationId = destinationId + this.jobRunConfig = jobRunConfig + } + } + + fun mockWorkload( + status: WorkloadStatus, + terminationSource: String? = null, + terminationReason: String? = null, + ): Workload { + val w = mockk() + every { w.status } returns status + every { w.terminationReason } returns terminationReason + every { w.terminationSource } returns terminationSource + return w + } +} diff --git a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java index 00edaf0395b..f84f84e101b 100644 --- a/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java +++ b/airbyte-container-orchestrator/src/main/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestrator.java @@ -14,6 +14,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.temporal.TemporalUtils; import io.airbyte.config.Configs; +import io.airbyte.config.FailureReason; import io.airbyte.config.ReplicationOutput; import io.airbyte.container_orchestrator.AsyncStateManager; import io.airbyte.metrics.lib.ApmTraceUtils; @@ -23,6 +24,9 @@ import io.airbyte.workers.exception.WorkerException; import io.airbyte.workers.general.ReplicationWorker; import io.airbyte.workers.general.ReplicationWorkerFactory; +import io.airbyte.workers.helper.FailureHelper; +import io.airbyte.workers.internal.exception.DestinationException; +import io.airbyte.workers.internal.exception.SourceException; import io.airbyte.workers.process.AsyncKubePodStatus; import io.airbyte.workers.process.KubePodProcess; import io.airbyte.workers.sync.ReplicationLauncherWorker; @@ -121,23 +125,31 @@ public Optional runJob() throws Exception { ReplicationOutput runWithWorkloadEnabled(final ReplicationWorker replicationWorker, final ReplicationInput replicationInput, final Path jobRoot) throws WorkerException, IOException { + final Long jobId = Long.parseLong(jobRunConfig.getJobId()); + final Integer attemptNumber = Math.toIntExact(jobRunConfig.getAttemptId()); final String workloadId = workloadIdGenerator.generateSyncWorkloadId( replicationInput.getConnectionId(), - Long.parseLong(jobRunConfig.getJobId()), - Math.toIntExact(jobRunConfig.getAttemptId())); + jobId, + attemptNumber); try { final ReplicationOutput replicationOutput = replicationWorker.run(replicationInput, jobRoot); switch (replicationOutput.getReplicationAttemptSummary().getStatus()) { - case FAILED -> failWorkload(workloadId); + case FAILED -> failWorkload(workloadId, replicationOutput.getFailures().stream().findFirst()); case CANCELLED -> cancelWorkload(workloadId); case COMPLETED -> succeedWorkload(workloadId); default -> throw new RuntimeException(String.format("Unknown status %s.", replicationOutput.getReplicationAttemptSummary().getStatus())); } return replicationOutput; + } catch (final DestinationException e) { + failWorkload(workloadId, Optional.of(FailureHelper.destinationFailure(e, jobId, attemptNumber))); + throw e; + } catch (final SourceException e) { + failWorkload(workloadId, Optional.of(FailureHelper.sourceFailure(e, jobId, attemptNumber))); + throw e; } catch (final WorkerException e) { - failWorkload(workloadId); + failWorkload(workloadId, Optional.of(FailureHelper.platformFailure(e, jobId, attemptNumber))); throw e; } } @@ -146,8 +158,14 @@ private void cancelWorkload(final String workloadId) throws IOException { workloadApi.workloadCancel(new WorkloadCancelRequest(workloadId, "Replication job has been cancelled", "orchestrator")); } - private void failWorkload(final String workloadId) throws IOException { - workloadApi.workloadFailure(new WorkloadFailureRequest(workloadId)); + private void failWorkload(final String workloadId, final Optional failureReason) throws IOException { + if (failureReason.isPresent()) { + workloadApi.workloadFailure(new WorkloadFailureRequest(workloadId, + failureReason.get().getFailureOrigin().value(), + failureReason.get().getExternalMessage())); + } else { + workloadApi.workloadFailure(new WorkloadFailureRequest(workloadId, null, null)); + } } private void succeedWorkload(final String workloadId) throws IOException { diff --git a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestratorTest.java b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestratorTest.java index 3dc9369269b..afc9f7db4fb 100644 --- a/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestratorTest.java +++ b/airbyte-container-orchestrator/src/test/java/io/airbyte/container_orchestrator/orchestrator/ReplicationJobOrchestratorTest.java @@ -127,7 +127,7 @@ void testRunWithWorkloadEnabledRunFailed() throws Exception { mock(Path.class)); assertEquals(replicationOutput, actualReplicationOutput); - verify(workloadApi).workloadFailure(new WorkloadFailureRequest(WORKLOAD_ID)); + verify(workloadApi).workloadFailure(new WorkloadFailureRequest(WORKLOAD_ID, null, null)); } @Test @@ -152,7 +152,8 @@ void testRunWithWorkloadEnabledRunThrowsException() throws Exception { assertThrows(WorkerException.class, () -> replicationJobOrchestrator.runWithWorkloadEnabled(replicationWorker, new ReplicationInput().withConnectionId(UUID.randomUUID()), mock(Path.class))); - verify(workloadApi).workloadFailure(new WorkloadFailureRequest(WORKLOAD_ID)); + verify(workloadApi) + .workloadFailure(new WorkloadFailureRequest(WORKLOAD_ID, "airbyte_platform", "Something went wrong within the airbyte platform")); } } diff --git a/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_010__AddFailureTrackingToWorkloads.java b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_010__AddFailureTrackingToWorkloads.java new file mode 100644 index 00000000000..2160334dc09 --- /dev/null +++ b/airbyte-db/db-lib/src/main/java/io/airbyte/db/instance/configs/migrations/V0_50_33_010__AddFailureTrackingToWorkloads.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2023 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.db.instance.configs.migrations; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.jooq.DSLContext; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +// TODO: update migration description in the class name +public class V0_50_33_010__AddFailureTrackingToWorkloads extends BaseJavaMigration { + + private static final String WORKLOAD_TABLE = "workload"; + + private static final Logger LOGGER = LoggerFactory.getLogger(V0_50_33_010__AddFailureTrackingToWorkloads.class); + + @Override + public void migrate(final Context context) throws Exception { + LOGGER.info("Running migration: {}", this.getClass().getSimpleName()); + + // Warning: please do not use any jOOQ generated code to write a migration. + // As database schema changes, the generated jOOQ code can be deprecated. So + // old migration may not compile if there is any generated code. + final DSLContext ctx = DSL.using(context.getConnection()); + + ctx.alterTable(WORKLOAD_TABLE) + .addColumnIfNotExists(DSL.field("termination_source", SQLDataType.VARCHAR.nullable(true))) + .execute(); + ctx.alterTable(WORKLOAD_TABLE) + .addColumnIfNotExists(DSL.field("termination_reason", SQLDataType.CLOB.nullable(true))) + .execute(); + } + +} diff --git a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt index b085845a433..2a2264b9607 100644 --- a/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt +++ b/airbyte-db/db-lib/src/main/resources/configs_database/schema_dump.txt @@ -362,6 +362,8 @@ create table "public"."workload" ( "geography" varchar(2147483647) not null default cast('AUTO' as varchar), "mutex_key" varchar(2147483647), "type" "public"."workload_type" not null, + "termination_source" varchar(2147483647), + "termination_reason" text, constraint "workload_pkey" primary key ("id") ); diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/WorkloadApi.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/WorkloadApi.kt index 6b22f179c4f..b97bc718edf 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/WorkloadApi.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/WorkloadApi.kt @@ -139,7 +139,7 @@ open class WorkloadApi( ) workloadFailureRequest: WorkloadFailureRequest, ) { ApmTraceUtils.addTagsToTrace(mutableMapOf(WORKLOAD_ID_TAG to workloadFailureRequest.workloadId) as Map?) - workloadHandler.failWorkload(workloadFailureRequest.workloadId) + workloadHandler.failWorkload(workloadFailureRequest.workloadId, workloadFailureRequest.source, workloadFailureRequest.reason) } @PUT @@ -244,7 +244,7 @@ open class WorkloadApi( WORKLOAD_CANCEL_SOURCE_TAG to workloadCancelRequest.source, ) as Map?, ) - workloadHandler.cancelWorkload(workloadCancelRequest.workloadId) + workloadHandler.cancelWorkload(workloadCancelRequest.workloadId, workloadCancelRequest.source, workloadCancelRequest.reason) } @PUT diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/domain/Workload.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/domain/Workload.kt index bb0060eb1c1..4e23fcb2959 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/domain/Workload.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/domain/Workload.kt @@ -15,4 +15,6 @@ data class Workload( var geography: String = DEFAULT_GEOGRAPHY, var mutexKey: String? = null, var type: WorkloadType = WorkloadType.SYNC, + var terminationSource: String? = null, + var terminationReason: String? = null, ) diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/domain/WorkloadFailureRequest.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/domain/WorkloadFailureRequest.kt index d619433e1a8..c637d5f39a4 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/domain/WorkloadFailureRequest.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/api/domain/WorkloadFailureRequest.kt @@ -5,4 +5,6 @@ import io.swagger.v3.oas.annotations.media.Schema data class WorkloadFailureRequest( @Schema(required = true) var workloadId: String = "", + var source: String? = null, + var reason: String? = null, ) diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandler.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandler.kt index 4d71e72206b..153408a279f 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandler.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandler.kt @@ -35,9 +35,17 @@ interface WorkloadHandler { dataplaneId: String, ): Boolean - fun cancelWorkload(workloadId: String) + fun cancelWorkload( + workloadId: String, + source: String?, + reason: String?, + ) - fun failWorkload(workloadId: String) + fun failWorkload( + workloadId: String, + source: String?, + reason: String?, + ) fun succeedWorkload(workloadId: String) diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandlerImpl.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandlerImpl.kt index 13c16a6ecc2..e1015b79a05 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandlerImpl.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadHandlerImpl.kt @@ -95,7 +95,11 @@ class WorkloadHandlerImpl( return true } - override fun cancelWorkload(workloadId: String) { + override fun cancelWorkload( + workloadId: String, + source: String?, + reason: String?, + ) { val workload = getDomainWorkload(workloadId) when (workload.status) { @@ -103,6 +107,8 @@ class WorkloadHandlerImpl( workloadRepository.update( workloadId, WorkloadStatus.CANCELLED, + source, + reason, ) WorkloadStatus.CANCELLED -> logger.info { "Workload $workloadId is already cancelled. Cancelling an already cancelled workload is a noop" } else -> throw InvalidStatusTransitionException( @@ -111,7 +117,11 @@ class WorkloadHandlerImpl( } } - override fun failWorkload(workloadId: String) { + override fun failWorkload( + workloadId: String, + source: String?, + reason: String?, + ) { val workload = getDomainWorkload(workloadId) when (workload.status) { @@ -119,6 +129,8 @@ class WorkloadHandlerImpl( workloadRepository.update( workloadId, WorkloadStatus.FAILURE, + source, + reason, ) WorkloadStatus.FAILURE -> logger.info { "Workload $workloadId is already marked as failed. Failing an already failed workload is a noop" } else -> throw InvalidStatusTransitionException( diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadMapper.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadMapper.kt index 1f0db977d27..c06828e37b2 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadMapper.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/handler/WorkloadMapper.kt @@ -77,6 +77,8 @@ fun ApiWorkload.toDomain(): DomainWorkload { geography = this.geography, mutexKey = this.mutexKey, type = this.type.toDomain(), + terminationReason = terminationReason, + terminationSource = terminationSource, ) } diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/WorkloadRepository.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/WorkloadRepository.kt index fd5a3718246..03409b2981c 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/WorkloadRepository.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/WorkloadRepository.kt @@ -55,6 +55,13 @@ interface WorkloadRepository : PageableRepository { status: WorkloadStatus, ) + fun update( + @Id id: String, + status: WorkloadStatus, + terminationSource: String?, + terminationReason: String?, + ) + fun update( @Id id: String, status: WorkloadStatus, diff --git a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/domain/Workload.kt b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/domain/Workload.kt index 35f0e412199..f958185380b 100644 --- a/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/domain/Workload.kt +++ b/airbyte-workload-api-server/src/main/kotlin/io/airbyte/workload/repository/domain/Workload.kt @@ -39,6 +39,10 @@ data class Workload( var mutexKey: String?, @field:TypeDef(type = DataType.OBJECT) var type: WorkloadType, + @Nullable + var terminationSource: String? = null, + @Nullable + var terminationReason: String? = null, ) { @JvmOverloads constructor( @@ -64,6 +68,8 @@ data class Workload( lastHeartbeatAt = null, createdAt = null, updatedAt = null, + terminationSource = null, + terminationReason = null, ) } diff --git a/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/api/WorkloadApiTest.kt b/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/api/WorkloadApiTest.kt index a9832fa7cd1..df4e7fe5b60 100644 --- a/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/api/WorkloadApiTest.kt +++ b/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/api/WorkloadApiTest.kt @@ -171,14 +171,14 @@ class WorkloadApiTest( @Test fun `test cancel success`() { - every { workloadHandler.cancelWorkload(any()) } just Runs + every { workloadHandler.cancelWorkload(any(), any(), any()) } just Runs testEndpointStatus(HttpRequest.PUT("/api/v1/workload/cancel", Jsons.serialize(WorkloadCancelRequest())), HttpStatus.NO_CONTENT) } @Test fun `test cancel workload id not found`() { val exceptionMessage = "workload id not found" - every { workloadHandler.cancelWorkload(any()) } throws NotFoundException(exceptionMessage) + every { workloadHandler.cancelWorkload(any(), any(), any()) } throws NotFoundException(exceptionMessage) testErrorEndpointResponse( HttpRequest.PUT("/api/v1/workload/cancel", Jsons.serialize(WorkloadCancelRequest())), HttpStatus.NOT_FOUND, @@ -189,7 +189,7 @@ class WorkloadApiTest( @Test fun `test cancel workload in invalid status`() { val exceptionMessage = "workload in invalid status" - every { workloadHandler.cancelWorkload(any()) } throws InvalidStatusTransitionException(exceptionMessage) + every { workloadHandler.cancelWorkload(any(), any(), any()) } throws InvalidStatusTransitionException(exceptionMessage) testErrorEndpointResponse( HttpRequest.PUT("/api/v1/workload/cancel", Jsons.serialize(WorkloadCancelRequest())), HttpStatus.GONE, @@ -199,14 +199,14 @@ class WorkloadApiTest( @Test fun `test failure success`() { - every { workloadHandler.failWorkload(any()) } just Runs + every { workloadHandler.failWorkload(any(), any(), any()) } just Runs testEndpointStatus(HttpRequest.PUT("/api/v1/workload/failure", Jsons.serialize(WorkloadFailureRequest())), HttpStatus.NO_CONTENT) } @Test fun `test failure workload id not found`() { val exceptionMessage = "workload id not found" - every { workloadHandler.failWorkload(any()) } throws NotFoundException(exceptionMessage) + every { workloadHandler.failWorkload(any(), any(), any()) } throws NotFoundException(exceptionMessage) testErrorEndpointResponse( HttpRequest.PUT("/api/v1/workload/failure", Jsons.serialize(WorkloadFailureRequest())), HttpStatus.NOT_FOUND, @@ -217,7 +217,7 @@ class WorkloadApiTest( @Test fun `test failure workload in invalid status`() { val exceptionMessage = "workload in invalid status" - every { workloadHandler.failWorkload(any()) } throws InvalidStatusTransitionException(exceptionMessage) + every { workloadHandler.failWorkload(any(), any(), any()) } throws InvalidStatusTransitionException(exceptionMessage) testErrorEndpointResponse( HttpRequest.PUT("/api/v1/workload/failure", Jsons.serialize(WorkloadFailureRequest())), HttpStatus.GONE, diff --git a/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/handler/WorkloadHandlerImplTest.kt b/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/handler/WorkloadHandlerImplTest.kt index 3d3586817b2..2ac5ef9bbaa 100644 --- a/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/handler/WorkloadHandlerImplTest.kt +++ b/airbyte-workload-api-server/src/test/kotlin/io/airbyte/workload/handler/WorkloadHandlerImplTest.kt @@ -262,7 +262,7 @@ class WorkloadHandlerImplTest { @Test fun `test workload not found when cancelling workload`() { every { workloadRepository.findById(WORKLOAD_ID) }.returns(Optional.empty()) - assertThrows { workloadHandler.cancelWorkload(WORKLOAD_ID) } + assertThrows { workloadHandler.cancelWorkload(WORKLOAD_ID, "test", "test cancel") } } @ParameterizedTest @@ -277,7 +277,7 @@ class WorkloadHandlerImplTest { ), ) - assertThrows { workloadHandler.cancelWorkload(WORKLOAD_ID) } + assertThrows { workloadHandler.cancelWorkload(WORKLOAD_ID, "test", "invalid cancel") } } @ParameterizedTest @@ -292,10 +292,10 @@ class WorkloadHandlerImplTest { ), ) - every { workloadRepository.update(any(), ofType(WorkloadStatus::class)) } just Runs + every { workloadRepository.update(any(), ofType(WorkloadStatus::class), eq("test"), eq("test cancel")) } just Runs - workloadHandler.cancelWorkload(WORKLOAD_ID) - verify { workloadRepository.update(eq(WORKLOAD_ID), eq(WorkloadStatus.CANCELLED)) } + workloadHandler.cancelWorkload(WORKLOAD_ID, "test", "test cancel") + verify { workloadRepository.update(eq(WORKLOAD_ID), eq(WorkloadStatus.CANCELLED), eq("test"), eq("test cancel")) } } @Test @@ -309,14 +309,14 @@ class WorkloadHandlerImplTest { ), ) - workloadHandler.cancelWorkload(WORKLOAD_ID) - verify(exactly = 0) { workloadRepository.update(eq(WORKLOAD_ID), eq(WorkloadStatus.CANCELLED)) } + workloadHandler.cancelWorkload(WORKLOAD_ID, "test", "test cancel again") + verify(exactly = 0) { workloadRepository.update(eq(WORKLOAD_ID), eq(WorkloadStatus.CANCELLED), "test", "test cancel again") } } @Test fun `test workload not found when failing workload`() { every { workloadRepository.findById(WORKLOAD_ID) }.returns(Optional.empty()) - assertThrows { workloadHandler.failWorkload(WORKLOAD_ID) } + assertThrows { workloadHandler.failWorkload(WORKLOAD_ID, "test", "fail") } } @ParameterizedTest @@ -331,7 +331,7 @@ class WorkloadHandlerImplTest { ), ) - assertThrows { workloadHandler.failWorkload(WORKLOAD_ID) } + assertThrows { workloadHandler.failWorkload(WORKLOAD_ID, "test", "fail") } } @ParameterizedTest @@ -346,10 +346,10 @@ class WorkloadHandlerImplTest { ), ) - every { workloadRepository.update(any(), ofType(WorkloadStatus::class)) } just Runs + every { workloadRepository.update(any(), ofType(WorkloadStatus::class), eq("test"), eq("failing a workload")) } just Runs - workloadHandler.failWorkload(WORKLOAD_ID) - verify { workloadRepository.update(eq(WORKLOAD_ID), eq(WorkloadStatus.FAILURE)) } + workloadHandler.failWorkload(WORKLOAD_ID, "test", "failing a workload") + verify { workloadRepository.update(eq(WORKLOAD_ID), eq(WorkloadStatus.FAILURE), eq("test"), eq("failing a workload")) } } @Test @@ -363,8 +363,8 @@ class WorkloadHandlerImplTest { ), ) - workloadHandler.failWorkload(WORKLOAD_ID) - verify(exactly = 0) { workloadRepository.update(eq(WORKLOAD_ID), eq(WorkloadStatus.FAILURE)) } + workloadHandler.failWorkload(WORKLOAD_ID, "test", "noop") + verify(exactly = 0) { workloadRepository.update(eq(WORKLOAD_ID), eq(WorkloadStatus.FAILURE), eq("test"), eq("noop")) } } @Test From d4ba1b821cf8d08df64d59a2ada76c60ef1509d5 Mon Sep 17 00:00:00 2001 From: Davin Chia Date: Wed, 13 Dec 2023 17:38:15 -0800 Subject: [PATCH 27/27] Java 21 Upgrade: Run Connector Builder on Java 21 Image. (#10360) Do #10313 for Connector Builder Server. --- airbyte-connector-builder-server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-connector-builder-server/Dockerfile b/airbyte-connector-builder-server/Dockerfile index cf62378ddce..10972760bae 100644 --- a/airbyte-connector-builder-server/Dockerfile +++ b/airbyte-connector-builder-server/Dockerfile @@ -1,4 +1,4 @@ -ARG BASE_IMAGE=airbyte/airbyte-base-java-python-image:1.0 +ARG BASE_IMAGE=airbyte/airbyte-base-java-python-image:1.1.0 FROM ${BASE_IMAGE} AS connector-builder-server # Set up CDK requirements