Stream-Centric UI Feature Flag & Status Page Rename (#4790)

Co-authored-by: Krishna (kc) Glick <[email protected]>
Co-authored-by: Davin Chia <[email protected]>

airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx

@@ -7,6 +7,7 @@ import { useNavigate } from "react-router-dom";
 import { SortableTableHeader } from "components/ui/Table";
 
 import { ConnectionScheduleType, SchemaChange } from "core/request/AirbyteClient";
+import { useExperiment } from "hooks/services/Experiment";
 import { FeatureItem, useFeature } from "hooks/services/Feature";
 import { useQuery } from "hooks/useQuery";
 
@@ -16,6 +17,7 @@ import { ConnectorNameCell } from "./components/ConnectorNameCell";
 import { FrequencyCell } from "./components/FrequencyCell";
 import { LastSyncCell } from "./components/LastSyncCell";
 import { StatusCell } from "./components/StatusCell";
+import { StreamsStatusCell } from "./components/StreamStatusCell";
 import styles from "./ConnectionTable.module.scss";
 import { ConnectionTableDataItem, SortOrderEnum } from "./types";
 import { NextTable } from "../ui/NextTable";
@@ -30,6 +32,7 @@ const ConnectionTable: React.FC<ConnectionTableProps> = ({ data, entity, onClick
   const navigate = useNavigate();
   const query = useQuery<{ sortBy?: string; order?: SortOrderEnum }>();
   const allowAutoDetectSchema = useFeature(FeatureItem.AllowAutoDetectSchema);
+  const streamCentricUIEnabled = useExperiment("connection.streamCentricUI.v2", false);
 
   const sortBy = query.sortBy || "entityName";
   const sortOrder = query.order || SortOrderEnum.ASC;
@@ -75,6 +78,11 @@ const ConnectionTable: React.FC<ConnectionTableProps> = ({ data, entity, onClick
 
   const columns = React.useMemo(
     () => [
+      columnHelper.display({
+        id: "stream-status",
+        cell: StreamsStatusCell,
+        size: 170,
+      }),
       columnHelper.accessor("name", {
         header: () => (
           <SortableTableHeader
@@ -190,10 +198,18 @@ const ConnectionTable: React.FC<ConnectionTableProps> = ({ data, entity, onClick
         cell: (props) => <ConnectionSettingsCell id={props.cell.getValue()} />,
       }),
     ],
-    [columnHelper, sortBy, sortOrder, onSortClick, entity, allowAutoDetectSchema]
+    [columnHelper, sortBy, sortOrder, entity, onSortClick, allowAutoDetectSchema]
   );
 
-  return <NextTable columns={columns} data={sortingData} onClickRow={onClickRow} testId="connectionsTable" />;
+  return (
+    <NextTable
+      columns={columns}
+      data={sortingData}
+      onClickRow={onClickRow}
+      testId="connectionsTable"
+      columnVisibility={{ "stream-status": streamCentricUIEnabled, name: !streamCentricUIEnabled }}
+    />
+  );
 };
 
 export default ConnectionTable;

airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.module.scss

@@ -0,0 +1,85 @@
+@use "scss/colors";
+@use "scss/variables";
+
+.bar {
+  width: 100%;
+  max-width: 370px;
+  height: 23px;
+  border-radius: variables.$border-radius-xs;
+  overflow: hidden;
+  position: relative;
+  display: flex;
+  gap: 1px;
+
+  .filling {
+    width: 100%;
+    height: 100%;
+
+    &.onTrack {
+      background: colors.$green;
+    }
+
+    &.behind {
+      background: colors.$blue;
+    }
+
+    &.error {
+      background: colors.$red;
+    }
+
+    &.disabled {
+      background: colors.$grey;
+    }
+  }
+}
+
+$contentPadding: 40px;
+
+.tooltipContainer {
+  width: 248px;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+
+  .bar {
+    padding: 0 $contentPadding;
+    margin-bottom: variables.$spacing-md;
+  }
+
+  .tooltipContent {
+    display: flex;
+    align-items: center;
+    padding: 0 $contentPadding;
+    width: 100%;
+
+    .streamsDetail {
+      display: flex;
+      align-items: center;
+      gap: variables.$spacing-sm;
+      flex: 1;
+    }
+
+    .syncContainer {
+      display: flex;
+      align-items: center;
+    }
+  }
+}
+
+// TODO: This could be added to the upcoming Icon component!
+@keyframes spin {
+  0% {
+    transform: rotate(0deg);
+  }
+
+  100% {
+    transform: rotate(359deg);
+  }
+}
+
+.syncing {
+  animation-name: spin;
+  animation-duration: 2000ms;
+  animation-iteration-count: infinite;
+  animation-timing-function: linear;
+}

airbyte-webapp/src/components/EntityTable/components/StreamStatusCell.tsx

@@ -0,0 +1,193 @@
+import { CellContext } from "@tanstack/react-table";
+import classNames from "classnames";
+import dayjs from "dayjs";
+import { FormattedMessage } from "react-intl";
+
+import { Checkmark } from "components/icons/Checkmark";
+import { Error } from "components/icons/Error";
+import { Inactive } from "components/icons/Inactive";
+import { Late } from "components/icons/Late";
+import { Syncing } from "components/icons/Syncing";
+import { Tooltip } from "components/ui/Tooltip";
+
+import {
+  WebBackendConnectionRead,
+  AirbyteStreamConfiguration,
+  ConnectionScheduleData,
+  ConnectionScheduleType,
+  JobStatus,
+  ConnectionStatus,
+} from "core/request/AirbyteClient";
+import { AirbyteStreamAndConfiguration } from "core/request/AirbyteClient";
+import { useGetConnection } from "hooks/services/useConnectionHook";
+
+import styles from "./StreamStatusCell.module.scss";
+import { ConnectionTableDataItem } from "../types";
+
+const enum StreamStatusType {
+  // TODO: When we have Actionable Errors, uncomment
+  /* "actionRequired" = "actionRequired", */
+  onTrack = "onTrack",
+  disabled = "disabled",
+  error = "error",
+  behind = "behind",
+}
+
+const statusMap: Readonly<Record<StreamStatusType, string>> = {
+  [StreamStatusType.onTrack]: styles.onTrack,
+  [StreamStatusType.disabled]: styles.disabled,
+  [StreamStatusType.error]: styles.error,
+  [StreamStatusType.behind]: styles.behind,
+};
+
+const iconMap: Readonly<Record<StreamStatusType, React.ReactNode>> = {
+  [StreamStatusType.onTrack]: <Checkmark />,
+  [StreamStatusType.disabled]: <Inactive />,
+  [StreamStatusType.error]: <Error />,
+  [StreamStatusType.behind]: <Late />,
+};
+
+interface FakeStreamConfigWithStatus extends AirbyteStreamConfiguration {
+  status: ConnectionStatus;
+  latestSyncJobStatus?: JobStatus;
+  scheduleType?: ConnectionScheduleType;
+  latestSyncJobCreatedAt?: number;
+  scheduleData?: ConnectionScheduleData;
+  isSyncing: boolean;
+}
+
+interface AirbyteStreamWithStatusAndConfiguration extends AirbyteStreamAndConfiguration {
+  config?: FakeStreamConfigWithStatus;
+}
+
+function filterStreamsWithTypecheck(
+  v: AirbyteStreamWithStatusAndConfiguration | null
+): v is AirbyteStreamWithStatusAndConfiguration {
+  return Boolean(v);
+}
+
+const generateFakeStreamsWithStatus = (
+  connection: WebBackendConnectionRead
+): AirbyteStreamWithStatusAndConfiguration[] => {
+  return connection.syncCatalog.streams
+    .map<AirbyteStreamWithStatusAndConfiguration | null>(({ stream, config }) => {
+      if (stream && config) {
+        return {
+          stream,
+          config: {
+            ...config,
+            status: connection.status,
+            latestSyncJobStatus: connection.latestSyncJobStatus,
+            scheduleType: connection.scheduleType,
+            latestSyncJobCreatedAt: connection.latestSyncJobCreatedAt,
+            scheduleData: connection.scheduleData,
+            isSyncing: connection.isSyncing || true,
+          },
+        };
+      }
+      return null;
+    })
+    .filter(filterStreamsWithTypecheck);
+};
+
+const isStreamBehind = (stream: AirbyteStreamWithStatusAndConfiguration) => {
+  return (
+    // This can be undefined due to historical data, but should always be present
+    stream.config?.scheduleType &&
+    !["cron", "manual"].includes(stream.config.scheduleType) &&
+    stream.config.latestSyncJobCreatedAt &&
+    stream.config.scheduleData?.basicSchedule?.units &&
+    stream.config.latestSyncJobCreatedAt * 1000 < // x1000 for a JS datetime
+      dayjs()
+        // Subtract 2x the scheduled interval and compare it to last sync time
+        .subtract(stream.config.scheduleData.basicSchedule.units, stream.config.scheduleData.basicSchedule.timeUnit)
+        .valueOf()
+  );
+};
+
+const getStatusForStream = (stream: AirbyteStreamWithStatusAndConfiguration): StreamStatusType => {
+  if (stream.config && stream.config.selected) {
+    if (stream.config.status === "active" && stream.config.latestSyncJobStatus !== "failed") {
+      if (isStreamBehind(stream)) {
+        return StreamStatusType.behind;
+      }
+      return StreamStatusType.onTrack;
+    } else if (stream.config.latestSyncJobStatus === "failed") {
+      return StreamStatusType.error;
+    }
+  }
+  return StreamStatusType.disabled;
+};
+
+const sortStreams = (streams: AirbyteStreamWithStatusAndConfiguration[]): Record<StreamStatusType, number> =>
+  streams.reduce(
+    (sortedStreams, stream) => {
+      sortedStreams[getStatusForStream(stream)]++;
+      return sortedStreams;
+    },
+    // This is the intended display order thanks to Javascript object insertion order!
+    {
+      /* [StatusType.actionRequired]: 0, */ [StreamStatusType.error]: 0,
+      [StreamStatusType.behind]: 0,
+      [StreamStatusType.onTrack]: 0,
+      [StreamStatusType.disabled]: 0,
+    }
+  );
+
+const StreamsBar: React.FC<{ streams: AirbyteStreamWithStatusAndConfiguration[] }> = ({ streams }) => {
+  const sortedStreams = sortStreams(streams);
+  const nonEmptyStreams = Object.entries(sortedStreams).filter(([, count]) => !!count);
+  return (
+    <div className={styles.bar}>
+      {nonEmptyStreams.map(([statusType, count]) => (
+        <div
+          style={{ width: `${Number(count / streams.length) * 100}%` }}
+          className={classNames(styles.filling, statusMap[statusType as unknown as StreamStatusType])}
+          key={statusType}
+        />
+      ))}
+    </div>
+  );
+};
+
+const StreamsPerStatus: React.FC<{ streams: AirbyteStreamWithStatusAndConfiguration[] }> = ({ streams }) => {
+  const sortedStreams = sortStreams(streams);
+  const nonEmptyStreams = Object.entries(sortedStreams).filter(([, count]) => !!count);
+  return (
+    <>
+      {nonEmptyStreams.map(([statusType, count], index) => (
+        <div className={styles.tooltipContent} key={statusType}>
+          <div className={styles.streamsDetail}>
+            {iconMap[statusType as unknown as StreamStatusType]} <b>{count}</b>{" "}
+            <FormattedMessage id={`connection.stream.status.${statusType}`} />
+          </div>
+          {streams[index].config?.isSyncing ? (
+            <div className={styles.syncContainer}>
+              {count} <Syncing className={styles.syncing} />
+            </div>
+          ) : null}
+        </div>
+      ))}
+    </>
+  );
+};
+
+const StreamStatusPopover = ({ streams }: { streams: AirbyteStreamWithStatusAndConfiguration[] }) => {
+  return (
+    <div className={styles.tooltipContainer}>
+      <StreamsBar streams={streams} />
+      <StreamsPerStatus streams={streams} />
+    </div>
+  );
+};
+
+export const StreamsStatusCell: React.FC<CellContext<ConnectionTableDataItem, unknown>> = ({ row }) => {
+  const connection = useGetConnection(row.original.connectionId);
+  const fakeStreamsWithStatus = generateFakeStreamsWithStatus(connection);
+
+  return (
+    <Tooltip theme="light" control={<StreamsBar streams={fakeStreamsWithStatus} />}>
+      <StreamStatusPopover streams={fakeStreamsWithStatus} />
+    </Tooltip>
+  );
+};

airbyte-webapp/src/components/icons/Checkmark.tsx

@@ -0,0 +1,9 @@
+export const Checkmark = () => (
+  <svg width="30" height="30" viewBox="0 0 30 30" fill="none">
+    <rect width="30" height="30" rx="5" fill="#EFF0F5" />
+    <path
+      d="M15 21.25C11.5481 21.25 8.75 18.4519 8.75 15C8.75 11.5481 11.5481 8.75 15 8.75C18.4519 8.75 21.25 11.5481 21.25 15C21.25 18.4519 18.4519 21.25 15 21.25ZM14.3769 17.5L18.7956 13.0806L17.9119 12.1969L14.3769 15.7325L12.6087 13.9644L11.725 14.8481L14.3769 17.5Z"
+      fill="#67DAE1"
+    />
+  </svg>
+);

airbyte-webapp/src/components/icons/Error.tsx

@@ -0,0 +1,11 @@
+export const Error = () => (
+  <svg width="30" height="30" viewBox="0 0 30 30" fill="none">
+    <rect width="30" height="30" rx="5" fill="#FFEFF2" />
+    <path
+      fillRule="evenodd"
+      clipRule="evenodd"
+      d="M8.75 15C8.75 18.4519 11.5481 21.25 15 21.25C18.4519 21.25 21.25 18.4519 21.25 15C21.25 11.5481 18.4519 8.75 15 8.75C11.5481 8.75 8.75 11.5481 8.75 15ZM14.1667 16.6667V10.8333H15.8333V16.6667H14.1667ZM14.1667 19.1667V17.5H15.8333V19.1667H14.1667Z"
+      fill="#FF5E7B"
+    />
+  </svg>
+);

airbyte-webapp/src/components/icons/Inactive.tsx

@@ -0,0 +1,9 @@
+export const Inactive = () => (
+  <svg width="30" height="30" viewBox="0 0 30 30" fill="none">
+    <rect width="30" height="30" rx="5" fill="#F7F8FC" fillOpacity="0.5" />
+    <path
+      d="M15.25 21.5C11.7981 21.5 9 18.7019 9 15.25C9 11.7981 11.7981 9 15.25 9C18.7019 9 21.5 11.7981 21.5 15.25C21.5 18.7019 18.7019 21.5 15.25 21.5Z"
+      fill="#E6E7EF"
+    />
+  </svg>
+);

airbyte-webapp/src/components/icons/Late.tsx

@@ -0,0 +1,9 @@
+export const Late = () => (
+  <svg width="30" height="30" viewBox="0 0 30 30" fill="none">
+    <rect width="30" height="30" rx="5" fill="#EFF0F5" />
+    <path
+      d="M15.25 21.5C11.7981 21.5 9 18.7019 9 15.25C9 11.7981 11.7981 9 15.25 9C18.7019 9 21.5 11.7981 21.5 15.25C21.5 18.7019 18.7019 21.5 15.25 21.5ZM15.875 15.25V12.125H14.625V16.5H18.375V15.25H15.875Z"
+      fill="#565C94"
+    />
+  </svg>
+);

airbyte-webapp/src/components/icons/Syncing.tsx

@@ -0,0 +1,8 @@
+export const Syncing: React.FC<{ className?: string }> = ({ className }) => (
+  <svg width="20" height="20" viewBox="0 0 20 20" fill="none" className={className}>
+    <path
+      d="M10 16.25C6.54813 16.25 3.75 13.4519 3.75 10C3.75 6.54813 6.54813 3.75 10 3.75C13.4519 3.75 16.25 6.54813 16.25 10C16.25 13.4519 13.4519 16.25 10 16.25ZM13.0125 13.1725C13.7669 12.4574 14.2421 11.4971 14.3529 10.4635C14.4637 9.42994 14.2029 8.39069 13.6171 7.53194C13.0314 6.67319 12.1591 6.05106 11.1563 5.77699C10.1536 5.50292 9.08611 5.59483 8.145 6.03625L8.75437 7.13313C9.22995 6.92643 9.74949 6.84133 10.2662 6.8855C10.7828 6.92966 11.2804 7.10171 11.714 7.38612C12.1476 7.67054 12.5036 8.05838 12.7499 8.51469C12.9962 8.97101 13.1251 9.48145 13.125 10H11.25L13.0125 13.1725ZM11.855 13.9637L11.2456 12.8669C10.77 13.0736 10.2505 13.1587 9.73384 13.1145C9.21717 13.0703 8.71961 12.8983 8.28601 12.6139C7.85241 12.3295 7.49641 11.9416 7.25008 11.4853C7.00376 11.029 6.87486 10.5186 6.875 10H8.75L6.9875 6.8275C6.23306 7.54258 5.75787 8.50293 5.64708 9.5365C5.53628 10.5701 5.79712 11.6093 6.38285 12.4681C6.96859 13.3268 7.84095 13.9489 8.84366 14.223C9.84636 14.4971 10.9139 14.4052 11.855 13.9637V13.9637Z"
+      fill="#1A194D"
+    />
+  </svg>
+);