From 5a0272c272e412e133c2031d237d05cf12a783ef Mon Sep 17 00:00:00 2001 From: Brent Bovenzi Date: Thu, 21 Nov 2024 14:07:09 -0500 Subject: [PATCH] Create dag graph with nested groups and join_ids (#44199) * Create dag graph with nested groups and join_ids * Add edge labels, setup/teardown tasks, and mapped tasks * move all graph formatting utils inside of useGraphLayout hook * move opengroup logic to a local context provider * Add vite plugin to fix graph css * Remove commented out code --- airflow/ui/package.json | 8 +- airflow/ui/pnpm-lock.yaml | 455 ++++++++++++++++++ .../src/components/ui/Dialog/CloseTrigger.tsx | 37 +- .../ui/src/context/colorMode/useColorMode.tsx | 2 +- .../context/openGroups/OpenGroupsProvider.tsx | 69 +++ airflow/ui/src/context/openGroups/index.ts | 21 + .../src/context/openGroups/useOpenGroups.ts | 34 ++ airflow/ui/src/pages/DagsList/Dag/Dag.tsx | 38 +- .../ui/src/pages/DagsList/Dag/DagVizModal.tsx | 48 ++ .../ui/src/pages/DagsList/Dag/Graph/Edge.tsx | 81 ++++ .../ui/src/pages/DagsList/Dag/Graph/Graph.tsx | 69 +++ .../src/pages/DagsList/Dag/Graph/JoinNode.tsx | 36 ++ .../pages/DagsList/Dag/Graph/NodeWrapper.tsx | 36 ++ .../src/pages/DagsList/Dag/Graph/TaskName.tsx | 63 +++ .../src/pages/DagsList/Dag/Graph/TaskNode.tsx | 120 +++++ .../ui/src/pages/DagsList/Dag/Graph/data.ts | 216 +++++++++ .../ui/src/pages/DagsList/Dag/Graph/index.ts | 20 + .../DagsList/Dag/Graph/reactflowUtils.ts | 140 ++++++ .../DagsList/Dag/Graph/useGraphLayout.ts | 288 +++++++++++ airflow/ui/src/pages/DagsList/Dag/Tabs.tsx | 90 ++++ airflow/ui/vite.config.ts | 2 + 21 files changed, 1823 insertions(+), 50 deletions(-) create mode 100644 airflow/ui/src/context/openGroups/OpenGroupsProvider.tsx create mode 100644 airflow/ui/src/context/openGroups/index.ts create mode 100644 airflow/ui/src/context/openGroups/useOpenGroups.ts create mode 100644 airflow/ui/src/pages/DagsList/Dag/DagVizModal.tsx create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/Edge.tsx create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/Graph.tsx create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/JoinNode.tsx create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/NodeWrapper.tsx create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/TaskName.tsx create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/TaskNode.tsx create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/data.ts create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/index.ts create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/reactflowUtils.ts create mode 100644 airflow/ui/src/pages/DagsList/Dag/Graph/useGraphLayout.ts create mode 100644 airflow/ui/src/pages/DagsList/Dag/Tabs.tsx diff --git a/airflow/ui/package.json b/airflow/ui/package.json index b17e35b31409a..e2aeada97dd57 100644 --- a/airflow/ui/package.json +++ b/airflow/ui/package.json @@ -24,10 +24,14 @@ "@tanstack/react-table": "^8.20.1", "@uiw/codemirror-themes-all": "^4.23.5", "@uiw/react-codemirror": "^4.23.5", + "@visx/group": "^3.12.0", + "@visx/shape": "^3.12.0", + "@xyflow/react": "^12.3.5", "axios": "^1.7.7", "chakra-react-select": "6.0.0-next.2", "chart.js": "^4.4.6", "dayjs": "^1.11.13", + "elkjs": "^0.9.3", "next-themes": "^0.3.0", "react": "^18.3.1", "react-chartjs-2": "^5.2.0", @@ -69,6 +73,8 @@ "typescript": "~5.5.4", "typescript-eslint": "^8.5.0", "vite": "^5.4.6", - "vitest": "^2.1.1" + "vite-plugin-css-injected-by-js": "^3.5.2", + "vitest": "^2.1.1", + "web-worker": "^1.3.0" } } diff --git a/airflow/ui/pnpm-lock.yaml b/airflow/ui/pnpm-lock.yaml index db2f836e5d861..425192db8879e 100644 --- a/airflow/ui/pnpm-lock.yaml +++ b/airflow/ui/pnpm-lock.yaml @@ -32,6 +32,15 @@ importers: '@uiw/react-codemirror': specifier: ^4.23.5 version: 4.23.5(@babel/runtime@7.25.6)(@codemirror/autocomplete@6.18.2(@codemirror/language@6.10.3)(@codemirror/state@6.4.1)(@codemirror/view@6.34.1)(@lezer/common@1.2.3))(@codemirror/language@6.10.3)(@codemirror/lint@6.8.2)(@codemirror/search@6.5.6)(@codemirror/state@6.4.1)(@codemirror/theme-one-dark@6.1.2)(@codemirror/view@6.34.1)(codemirror@6.0.1(@lezer/common@1.2.3))(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@visx/group': + specifier: ^3.12.0 + version: 3.12.0(react@18.3.1) + '@visx/shape': + specifier: ^3.12.0 + version: 3.12.0(react@18.3.1) + '@xyflow/react': + specifier: ^12.3.5 + version: 12.3.5(@types/react@18.3.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) axios: specifier: ^1.7.7 version: 1.7.7 @@ -44,6 +53,9 @@ importers: dayjs: specifier: ^1.11.13 version: 1.11.13 + elkjs: + specifier: ^0.9.3 + version: 0.9.3 next-themes: specifier: ^0.3.0 version: 0.3.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -162,9 +174,15 @@ importers: vite: specifier: ^5.4.6 version: 5.4.6(@types/node@22.5.4) + vite-plugin-css-injected-by-js: + specifier: ^3.5.2 + version: 3.5.2(vite@5.4.6(@types/node@22.5.4)) vitest: specifier: ^2.1.1 version: 2.1.1(@types/node@22.5.4)(happy-dom@15.10.2) + web-worker: + specifier: ^1.3.0 + version: 1.3.0 packages: @@ -886,15 +904,72 @@ packages: '@types/aria-query@5.0.4': resolution: {integrity: sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==} + '@types/d3-array@3.0.3': + resolution: {integrity: sha512-Reoy+pKnvsksN0lQUlcH6dOGjRZ/3WRwXR//m+/8lt1BXeI4xyaUZoqULNjyXXRuh0Mj4LNpkCvhUpQlY3X5xQ==} + + '@types/d3-color@3.1.0': + resolution: {integrity: sha512-HKuicPHJuvPgCD+np6Se9MQvS6OCbJmOjGvylzMJRlDwUXjKTTXs6Pwgk79O09Vj/ho3u1ofXnhFOaEWWPrlwA==} + + '@types/d3-color@3.1.3': + resolution: {integrity: sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==} + + '@types/d3-delaunay@6.0.1': + resolution: {integrity: sha512-tLxQ2sfT0p6sxdG75c6f/ekqxjyYR0+LwPrsO1mbC9YDBzPJhs2HbJJRrn8Ez1DBoHRo2yx7YEATI+8V1nGMnQ==} + + '@types/d3-drag@3.0.7': + resolution: {integrity: sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ==} + + '@types/d3-format@3.0.1': + resolution: {integrity: sha512-5KY70ifCCzorkLuIkDe0Z9YTf9RR2CjBX1iaJG+rgM/cPP+sO+q9YdQ9WdhQcgPj1EQiJ2/0+yUkkziTG6Lubg==} + + '@types/d3-geo@3.1.0': + resolution: {integrity: sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ==} + + '@types/d3-interpolate@3.0.1': + resolution: {integrity: sha512-jx5leotSeac3jr0RePOH1KdR9rISG91QIE4Q2PYTu4OymLTZfA3SrnURSLzKH48HmXVUru50b8nje4E79oQSQw==} + + '@types/d3-interpolate@3.0.4': + resolution: {integrity: sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==} + + '@types/d3-path@1.0.11': + resolution: {integrity: sha512-4pQMp8ldf7UaB/gR8Fvvy69psNHkTpD/pVw3vmEi8iZAB9EPMBruB1JvHO4BIq9QkUUd2lV1F5YXpMNj7JPBpw==} + + '@types/d3-scale@4.0.2': + resolution: {integrity: sha512-Yk4htunhPAwN0XGlIwArRomOjdoBFXC3+kCxK2Ubg7I9shQlVSJy/pG/Ht5ASN+gdMIalpk8TJ5xV74jFsetLA==} + + '@types/d3-selection@3.0.11': + resolution: {integrity: sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w==} + + '@types/d3-shape@1.3.12': + resolution: {integrity: sha512-8oMzcd4+poSLGgV0R1Q1rOlx/xdmozS4Xab7np0eamFFUYq71AU9pOCJEFnkXW2aI/oXdVYJzw6pssbSut7Z9Q==} + + '@types/d3-time-format@2.1.0': + resolution: {integrity: sha512-/myT3I7EwlukNOX2xVdMzb8FRgNzRMpsZddwst9Ld/VFe6LyJyRp0s32l/V9XoUzk+Gqu56F/oGk6507+8BxrA==} + + '@types/d3-time@3.0.0': + resolution: {integrity: sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==} + + '@types/d3-transition@3.0.9': + resolution: {integrity: sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg==} + + '@types/d3-zoom@3.0.8': + resolution: {integrity: sha512-iqMC4/YlFCSlO8+2Ii1GGGliCAY4XdeG748w5vQUbevlbDu0zSjH/+jojorQVBK/se0j6DUFNPBGSqD3YWYnDw==} + '@types/estree@1.0.6': resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==} + '@types/geojson@7946.0.14': + resolution: {integrity: sha512-WCfD5Ht3ZesJUsONdhvm84dmzWOiOzOAqOncN0++w0lBw1o8OuDNJF2McvvCef/yBqb/HYRahp1BYtODFQ8bRg==} + '@types/hast@2.3.10': resolution: {integrity: sha512-McWspRw8xx8J9HurkVBfYj0xKoE25tOFlHGdx4MJ5xORQrMGZNqJhVQWaIbm6Oyla5kYOXtDiopzKRJzEOkwJw==} '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} + '@types/lodash@4.17.13': + resolution: {integrity: sha512-lfx+dftrEZcdBPczf9d0Qv0x+j/rfNCMuC6OcfXmO8gkfeNAY88PgKUbvG56whcN23gc27yenwF6oJZXGFpYxg==} + '@types/node@22.5.4': resolution: {integrity: sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg==} @@ -1143,6 +1218,25 @@ packages: react: '>=16.8.0' react-dom: '>=16.8.0' + '@visx/curve@3.12.0': + resolution: {integrity: sha512-Ng1mefXIzoIoAivw7dJ+ZZYYUbfuwXgZCgQynShr6ZIVw7P4q4HeQfJP3W24ON+1uCSrzoycHSXRelhR9SBPcw==} + + '@visx/group@3.12.0': + resolution: {integrity: sha512-Dye8iS1alVXPv7nj/7M37gJe6sSKqJLH7x6sEWAsRQ9clI0kFvjbKcKgF+U3aAVQr0NCohheFV+DtR8trfK/Ag==} + peerDependencies: + react: ^16.0.0-0 || ^17.0.0-0 || ^18.0.0-0 + + '@visx/scale@3.12.0': + resolution: {integrity: sha512-+ubijrZ2AwWCsNey0HGLJ0YKNeC/XImEFsr9rM+Uef1CM3PNM43NDdNTrdBejSlzRq0lcfQPWYMYQFSlkLcPOg==} + + '@visx/shape@3.12.0': + resolution: {integrity: sha512-/1l0lrpX9tPic6SJEalryBKWjP/ilDRnQA+BGJTI1tj7i23mJ/J0t4nJHyA1GrL4QA/bM/qTJ35eyz5dEhJc4g==} + peerDependencies: + react: ^16.3.0-0 || ^17.0.0-0 || ^18.0.0-0 + + '@visx/vendor@3.12.0': + resolution: {integrity: sha512-SVO+G0xtnL9dsNpGDcjCgoiCnlB3iLSM9KLz1sLbSrV7RaVXwY3/BTm2X9OWN1jH2a9M+eHt6DJ6sE6CXm4cUg==} + '@vitejs/plugin-react-swc@3.7.0': resolution: {integrity: sha512-yrknSb3Dci6svCd/qhHqhFPDSw0QtjumcqdKMoNNzmOl5lMXTTiqzjWtG4Qask2HdvvzaNgSunbQGet8/GrKdA==} peerDependencies: @@ -1187,6 +1281,15 @@ packages: '@vitest/utils@2.1.1': resolution: {integrity: sha512-Y6Q9TsI+qJ2CC0ZKj6VBb+T8UPz593N113nnUykqwANqhgf3QkZeHFlusgKLTqrnVHbj/XDKZcDHol+dxVT+rQ==} + '@xyflow/react@12.3.5': + resolution: {integrity: sha512-wAYqpicdrVo1rxCu0X3M9s3YIF45Agqfabw0IBryTGqjWvr2NyfciI8gIP4MB+NKpWWN5kxZ9tiZ9u8lwC7iAg==} + peerDependencies: + react: '>=17' + react-dom: '>=17' + + '@xyflow/system@0.0.46': + resolution: {integrity: sha512-bmFXvboVdiydIFZmDCjrbBCYgB0d5pYdkcZPWbAxGmhMRUZ+kW3CksYgYxWabrw51rwpWitLEadvLrivG0mVfA==} + '@zag-js/accordion@0.74.2': resolution: {integrity: sha512-0E6LpQgmcbDe12akh2sKYVvk+fwxVUwjVdclj8ntzlkAYy8PNTTbd9kfNB6rX9+lJUXk/Iqb5+Qgy9RjWplnNw==} @@ -1618,6 +1721,12 @@ packages: citty@0.1.6: resolution: {integrity: sha512-tskPPKEs8D2KPafUypv2gxwJP8h/OaJmC82QQGGDQcHvXX43xF2VDACcJVmZ0EuSxkpO9Kc4MlrA3q0+FG58AQ==} + classcat@5.0.5: + resolution: {integrity: sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w==} + + classnames@2.5.1: + resolution: {integrity: sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==} + clean-regexp@1.0.0: resolution: {integrity: sha512-GfisEZEJvzKrmGWkvfhgzcz/BllN1USeqD2V6tg14OAOgaCD2Z/PUEuxnAZ/nPvmaHRG7a8y77p1T/IRQ4D1Hw==} engines: {node: '>=4'} @@ -1685,6 +1794,78 @@ packages: csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + d3-array@3.2.1: + resolution: {integrity: sha512-gUY/qeHq/yNqqoCKNq4vtpFLdoCdvyNpWoC/KNjhGbhDuQpAM9sIQQKkXSNpXa9h5KySs/gzm7R88WkUutgwWQ==} + engines: {node: '>=12'} + + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-delaunay@6.0.2: + resolution: {integrity: sha512-IMLNldruDQScrcfT+MWnazhHbDJhcRJyOEBAJfwQnHle1RPh6WDuLvxNArUju2VSMSUuKlY5BGHRJ2cYyoFLQQ==} + engines: {node: '>=12'} + + d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + + d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-format@3.1.0: + resolution: {integrity: sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA==} + engines: {node: '>=12'} + + d3-geo@3.1.0: + resolution: {integrity: sha512-JEo5HxXDdDYXCaWdwLRt79y7giK8SbhZJbFWXqbRTolCHFI5jRqteLzCsq51NKbUoX0PjBVSohxrx+NoOUujYA==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-path@1.0.9: + resolution: {integrity: sha512-VLaYcn81dtHVTjEHd8B+pbe9yHWpXKZUC87PzoFmsFrJqgFwDe/qxfp5MlfsfM1V5E/iVt0MmEbWQ7FVIXh/bg==} + + d3-scale@4.0.2: + resolution: {integrity: sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==} + engines: {node: '>=12'} + + d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + + d3-shape@1.3.7: + resolution: {integrity: sha512-EUkvKjqPFUAZyOlhY5gzCxCeI0Aep04LwIRpsZ/mLFelJiUfnK56jo5JMDSE7yyP2kLSb6LtF+S5chMk7uqPqw==} + + d3-time-format@4.1.0: + resolution: {integrity: sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==} + engines: {node: '>=12'} + + d3-time@3.1.0: + resolution: {integrity: sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + + d3-transition@3.0.1: + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + + d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + damerau-levenshtein@1.0.8: resolution: {integrity: sha512-sdQSFB7+llfUcQHUQO3+B8ERRj0Oa4w9POWMI/puGtuf7gFywGmkaLCElnudfTiKZV+NvHqL0ifzdrI8Ro7ESA==} @@ -1743,6 +1924,9 @@ packages: defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} + delaunator@5.0.1: + resolution: {integrity: sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw==} + delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} @@ -1781,6 +1965,9 @@ packages: electron-to-chromium@1.5.19: resolution: {integrity: sha512-kpLJJi3zxTR1U828P+LIUDZ5ohixyo68/IcYOHLqnbTPr/wdgn4i1ECvmALN9E16JPA6cvCG5UG79gVwVdEK5w==} + elkjs@0.9.3: + resolution: {integrity: sha512-f/ZeWvW/BCXbhGEf1Ujp29EASo/lk1FDnETgNKwJrsVvGZhUWCZyg3xLJjAsxfOmt8KjswHmI5EwCQcPMpOYhQ==} + emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} @@ -2202,6 +2389,10 @@ packages: resolution: {integrity: sha512-NGnrKwXzSms2qUUih/ILZ5JBqNTSa1+ZmP6flaIp6KmSElgE9qdndzS3cqjrDovwFdmwsGsLdeFgB6suw+1e9g==} engines: {node: '>= 0.4'} + internmap@2.0.3: + resolution: {integrity: sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==} + engines: {node: '>=12'} + is-alphabetical@1.0.4: resolution: {integrity: sha512-DwzsA04LQ10FHTZuL0/grVDk4rFoVH1pjAToYwBrHSxcrBIGQuXrQMtD5U1b0U2XVgKZCTLLP8u2Qxqhy3l2Vg==} @@ -2922,6 +3113,9 @@ packages: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + robust-predicates@3.0.2: + resolution: {integrity: sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg==} + rollup@4.24.0: resolution: {integrity: sha512-DOmrlGSXNk1DM0ljiQA+i+o0rSLhtii1je5wgk60j49d1jHT5YYttBv1iWOnYSTG+fZZESUOSNiAl89SIet+Cg==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -3228,6 +3422,11 @@ packages: '@types/react': optional: true + use-sync-external-store@1.2.2: + resolution: {integrity: sha512-PElTlVMwpblvbNqQ82d2n6RjStvdSoNe9FG28kNfz3WiXilJm4DdNkEzRhCZuIDwY8U08WVihhGR5iRqAwfDiw==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 + usehooks-ts@3.1.0: resolution: {integrity: sha512-bBIa7yUyPhE1BCc0GmR96VU/15l/9gP1Ch5mYdLcFBaFGQsdmXkvjV0TtOqW1yUd6VjIwDunm+flSciCQXujiw==} engines: {node: '>=16.15.0'} @@ -3242,6 +3441,11 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true + vite-plugin-css-injected-by-js@3.5.2: + resolution: {integrity: sha512-2MpU/Y+SCZyWUB6ua3HbJCrgnF0KACAsmzOQt1UvRVJCGF6S8xdA3ZUhWcWdM9ivG4I5az8PnQmwwrkC2CAQrQ==} + peerDependencies: + vite: '>2.0.0-0' + vite@5.4.6: resolution: {integrity: sha512-IeL5f8OO5nylsgzd9tq4qD2QqI0k2CQLGrWD0rCN0EQJZpBK5vJAx0I+GDkMOXxQX/OfFHMuLIx6ddAxGX/k+Q==} engines: {node: ^18.0.0 || >=20.0.0} @@ -3301,6 +3505,9 @@ packages: w3c-keyname@2.2.8: resolution: {integrity: sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==} + web-worker@1.3.0: + resolution: {integrity: sha512-BSR9wyRsy/KOValMgd5kMyr3JzpdeoR9KVId8u5GVlTTAtNChlsE4yTxeY7zMdNSyOmoKBv8NH2qeRY9Tg+IaA==} + webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} @@ -3364,6 +3571,21 @@ packages: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} + zustand@4.5.5: + resolution: {integrity: sha512-+0PALYNJNgK6hldkgDq2vLrw5f6g/jCInz52n9RTpropGgeAf/ioFUCdtsjCqu4gNhW9D01rUQBROoRjdzyn2Q==} + engines: {node: '>=12.7.0'} + peerDependencies: + '@types/react': '>=16.8' + immer: '>=9.0.6' + react: '>=16.8' + peerDependenciesMeta: + '@types/react': + optional: true + immer: + optional: true + react: + optional: true + snapshots: '@7nohe/openapi-react-query-codegen@1.6.0(commander@12.1.0)(glob@11.0.0)(magicast@0.3.5)(ts-morph@23.0.0)(typescript@5.5.4)': @@ -4127,14 +4349,69 @@ snapshots: '@types/aria-query@5.0.4': {} + '@types/d3-array@3.0.3': {} + + '@types/d3-color@3.1.0': {} + + '@types/d3-color@3.1.3': {} + + '@types/d3-delaunay@6.0.1': {} + + '@types/d3-drag@3.0.7': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-format@3.0.1': {} + + '@types/d3-geo@3.1.0': + dependencies: + '@types/geojson': 7946.0.14 + + '@types/d3-interpolate@3.0.1': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-interpolate@3.0.4': + dependencies: + '@types/d3-color': 3.1.3 + + '@types/d3-path@1.0.11': {} + + '@types/d3-scale@4.0.2': + dependencies: + '@types/d3-time': 3.0.0 + + '@types/d3-selection@3.0.11': {} + + '@types/d3-shape@1.3.12': + dependencies: + '@types/d3-path': 1.0.11 + + '@types/d3-time-format@2.1.0': {} + + '@types/d3-time@3.0.0': {} + + '@types/d3-transition@3.0.9': + dependencies: + '@types/d3-selection': 3.0.11 + + '@types/d3-zoom@3.0.8': + dependencies: + '@types/d3-interpolate': 3.0.4 + '@types/d3-selection': 3.0.11 + '@types/estree@1.0.6': {} + '@types/geojson@7946.0.14': {} + '@types/hast@2.3.10': dependencies: '@types/unist': 2.0.11 '@types/json-schema@7.0.15': {} + '@types/lodash@4.17.13': {} + '@types/node@22.5.4': dependencies: undici-types: 6.19.8 @@ -4639,6 +4916,60 @@ snapshots: - '@codemirror/lint' - '@codemirror/search' + '@visx/curve@3.12.0': + dependencies: + '@types/d3-shape': 1.3.12 + d3-shape: 1.3.7 + + '@visx/group@3.12.0(react@18.3.1)': + dependencies: + '@types/react': 18.3.5 + classnames: 2.5.1 + prop-types: 15.8.1 + react: 18.3.1 + + '@visx/scale@3.12.0': + dependencies: + '@visx/vendor': 3.12.0 + + '@visx/shape@3.12.0(react@18.3.1)': + dependencies: + '@types/d3-path': 1.0.11 + '@types/d3-shape': 1.3.12 + '@types/lodash': 4.17.13 + '@types/react': 18.3.5 + '@visx/curve': 3.12.0 + '@visx/group': 3.12.0(react@18.3.1) + '@visx/scale': 3.12.0 + classnames: 2.5.1 + d3-path: 1.0.9 + d3-shape: 1.3.7 + lodash: 4.17.21 + prop-types: 15.8.1 + react: 18.3.1 + + '@visx/vendor@3.12.0': + dependencies: + '@types/d3-array': 3.0.3 + '@types/d3-color': 3.1.0 + '@types/d3-delaunay': 6.0.1 + '@types/d3-format': 3.0.1 + '@types/d3-geo': 3.1.0 + '@types/d3-interpolate': 3.0.1 + '@types/d3-scale': 4.0.2 + '@types/d3-time': 3.0.0 + '@types/d3-time-format': 2.1.0 + d3-array: 3.2.1 + d3-color: 3.1.0 + d3-delaunay: 6.0.2 + d3-format: 3.1.0 + d3-geo: 3.1.0 + d3-interpolate: 3.0.1 + d3-scale: 4.0.2 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + internmap: 2.0.3 + '@vitejs/plugin-react-swc@3.7.0(@swc/helpers@0.5.13)(vite@5.4.6(@types/node@22.5.4))': dependencies: '@swc/core': 1.7.14(@swc/helpers@0.5.13) @@ -4704,6 +5035,27 @@ snapshots: loupe: 3.1.1 tinyrainbow: 1.2.0 + '@xyflow/react@12.3.5(@types/react@18.3.5)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': + dependencies: + '@xyflow/system': 0.0.46 + classcat: 5.0.5 + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + zustand: 4.5.5(@types/react@18.3.5)(react@18.3.1) + transitivePeerDependencies: + - '@types/react' + - immer + + '@xyflow/system@0.0.46': + dependencies: + '@types/d3-drag': 3.0.7 + '@types/d3-selection': 3.0.11 + '@types/d3-transition': 3.0.9 + '@types/d3-zoom': 3.0.8 + d3-drag: 3.0.0 + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + '@zag-js/accordion@0.74.2': dependencies: '@zag-js/anatomy': 0.74.2 @@ -5457,6 +5809,10 @@ snapshots: dependencies: consola: 3.2.3 + classcat@5.0.5: {} + + classnames@2.5.1: {} + clean-regexp@1.0.0: dependencies: escape-string-regexp: 1.0.5 @@ -5527,6 +5883,78 @@ snapshots: csstype@3.1.3: {} + d3-array@3.2.1: + dependencies: + internmap: 2.0.3 + + d3-color@3.1.0: {} + + d3-delaunay@6.0.2: + dependencies: + delaunator: 5.0.1 + + d3-dispatch@3.0.1: {} + + d3-drag@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + + d3-ease@3.0.1: {} + + d3-format@3.1.0: {} + + d3-geo@3.1.0: + dependencies: + d3-array: 3.2.1 + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-path@1.0.9: {} + + d3-scale@4.0.2: + dependencies: + d3-array: 3.2.1 + d3-format: 3.1.0 + d3-interpolate: 3.0.1 + d3-time: 3.1.0 + d3-time-format: 4.1.0 + + d3-selection@3.0.0: {} + + d3-shape@1.3.7: + dependencies: + d3-path: 1.0.9 + + d3-time-format@4.1.0: + dependencies: + d3-time: 3.1.0 + + d3-time@3.1.0: + dependencies: + d3-array: 3.2.1 + + d3-timer@3.0.1: {} + + d3-transition@3.0.1(d3-selection@3.0.0): + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + + d3-zoom@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + damerau-levenshtein@1.0.8: {} data-view-buffer@1.0.1: @@ -5596,6 +6024,10 @@ snapshots: defu@6.1.4: {} + delaunator@5.0.1: + dependencies: + robust-predicates: 3.0.2 + delayed-stream@1.0.0: {} dequal@2.0.3: {} @@ -5625,6 +6057,8 @@ snapshots: electron-to-chromium@1.5.19: {} + elkjs@0.9.3: {} + emoji-regex@8.0.0: {} emoji-regex@9.2.2: {} @@ -6200,6 +6634,8 @@ snapshots: hasown: 2.0.2 side-channel: 1.0.6 + internmap@2.0.3: {} + is-alphabetical@1.0.4: {} is-alphanumerical@1.0.4: @@ -6901,6 +7337,8 @@ snapshots: reusify@1.0.4: {} + robust-predicates@3.0.2: {} + rollup@4.24.0: dependencies: '@types/estree': 1.0.6 @@ -7241,6 +7679,10 @@ snapshots: optionalDependencies: '@types/react': 18.3.5 + use-sync-external-store@1.2.2(react@18.3.1): + dependencies: + react: 18.3.1 + usehooks-ts@3.1.0(react@18.3.1): dependencies: lodash.debounce: 4.0.8 @@ -7268,6 +7710,10 @@ snapshots: - supports-color - terser + vite-plugin-css-injected-by-js@3.5.2(vite@5.4.6(@types/node@22.5.4)): + dependencies: + vite: 5.4.6(@types/node@22.5.4) + vite@5.4.6(@types/node@22.5.4): dependencies: esbuild: 0.21.5 @@ -7314,6 +7760,8 @@ snapshots: w3c-keyname@2.2.8: {} + web-worker@1.3.0: {} + webidl-conversions@7.0.0: {} whatwg-mimetype@3.0.0: {} @@ -7388,3 +7836,10 @@ snapshots: yaml@1.10.2: {} yocto-queue@0.1.0: {} + + zustand@4.5.5(@types/react@18.3.5)(react@18.3.1): + dependencies: + use-sync-external-store: 1.2.2(react@18.3.1) + optionalDependencies: + '@types/react': 18.3.5 + react: 18.3.1 diff --git a/airflow/ui/src/components/ui/Dialog/CloseTrigger.tsx b/airflow/ui/src/components/ui/Dialog/CloseTrigger.tsx index 0ea9beba0b9cf..2267df43f7b97 100644 --- a/airflow/ui/src/components/ui/Dialog/CloseTrigger.tsx +++ b/airflow/ui/src/components/ui/Dialog/CloseTrigger.tsx @@ -19,21 +19,24 @@ import { Dialog as ChakraDialog } from "@chakra-ui/react"; import { forwardRef } from "react"; -import { CloseButton } from "../CloseButton"; +import { CloseButton, type CloseButtonProps } from "../CloseButton"; -export const CloseTrigger = forwardRef< - HTMLButtonElement, - ChakraDialog.CloseTriggerProps ->((props, ref) => ( - - - {props.children} - - -)); +type Props = { + closeButtonProps?: CloseButtonProps; +} & ChakraDialog.CloseTriggerProps; + +export const CloseTrigger = forwardRef( + ({ children, closeButtonProps, ...rest }, ref) => ( + + + {children} + + + ), +); diff --git a/airflow/ui/src/context/colorMode/useColorMode.tsx b/airflow/ui/src/context/colorMode/useColorMode.tsx index 5c9ea1076e9a1..f1ccf76833af6 100644 --- a/airflow/ui/src/context/colorMode/useColorMode.tsx +++ b/airflow/ui/src/context/colorMode/useColorMode.tsx @@ -25,7 +25,7 @@ export const useColorMode = () => { }; return { - colorMode: resolvedTheme, + colorMode: resolvedTheme as "dark" | "light" | undefined, setColorMode: setTheme, toggleColorMode, }; diff --git a/airflow/ui/src/context/openGroups/OpenGroupsProvider.tsx b/airflow/ui/src/context/openGroups/OpenGroupsProvider.tsx new file mode 100644 index 0000000000000..1fdf3b32e9eeb --- /dev/null +++ b/airflow/ui/src/context/openGroups/OpenGroupsProvider.tsx @@ -0,0 +1,69 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { + createContext, + useCallback, + useMemo, + type PropsWithChildren, +} from "react"; +import { useLocalStorage } from "usehooks-ts"; + +export type OpenGroupsContextType = { + openGroupIds: Array; + setOpenGroupIds: (groupIds: Array) => void; + toggleGroupId: (groupId: string) => void; +}; + +export const OpenGroupsContext = createContext< + OpenGroupsContextType | undefined +>(undefined); + +type Props = { + readonly dagId: string; +} & PropsWithChildren; + +export const OpenGroupsProvider = ({ children, dagId }: Props) => { + const openGroupsKey = `${dagId}/open-groups`; + const [openGroupIds, setOpenGroupIds] = useLocalStorage>( + openGroupsKey, + [], + ); + + const toggleGroupId = useCallback( + (groupId: string) => { + if (openGroupIds.includes(groupId)) { + setOpenGroupIds(openGroupIds.filter((id) => id !== groupId)); + } else { + setOpenGroupIds([...openGroupIds, groupId]); + } + }, + [openGroupIds, setOpenGroupIds], + ); + + const value = useMemo( + () => ({ openGroupIds, setOpenGroupIds, toggleGroupId }), + [openGroupIds, setOpenGroupIds, toggleGroupId], + ); + + return ( + + {children} + + ); +}; diff --git a/airflow/ui/src/context/openGroups/index.ts b/airflow/ui/src/context/openGroups/index.ts new file mode 100644 index 0000000000000..4ee355fdb8a54 --- /dev/null +++ b/airflow/ui/src/context/openGroups/index.ts @@ -0,0 +1,21 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from "./OpenGroupsProvider"; +export * from "./useOpenGroups"; diff --git a/airflow/ui/src/context/openGroups/useOpenGroups.ts b/airflow/ui/src/context/openGroups/useOpenGroups.ts new file mode 100644 index 0000000000000..85fc86d88e580 --- /dev/null +++ b/airflow/ui/src/context/openGroups/useOpenGroups.ts @@ -0,0 +1,34 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useContext } from "react"; + +import { + OpenGroupsContext, + type OpenGroupsContextType, +} from "./OpenGroupsProvider"; + +export const useOpenGroups = (): OpenGroupsContextType => { + const context = useContext(OpenGroupsContext); + + if (context === undefined) { + throw new Error("useOpenGroup must be used within a OpenGroupsProvider"); + } + + return context; +}; diff --git a/airflow/ui/src/pages/DagsList/Dag/Dag.tsx b/airflow/ui/src/pages/DagsList/Dag/Dag.tsx index 82b4320911f15..ac149d10b46e2 100644 --- a/airflow/ui/src/pages/DagsList/Dag/Dag.tsx +++ b/airflow/ui/src/pages/DagsList/Dag/Dag.tsx @@ -16,14 +16,9 @@ * specific language governing permissions and limitations * under the License. */ -import { Box, Button, Tabs } from "@chakra-ui/react"; +import { Box, Button } from "@chakra-ui/react"; import { FiChevronsLeft } from "react-icons/fi"; -import { - Outlet, - Link as RouterLink, - useLocation, - useParams, -} from "react-router-dom"; +import { Outlet, Link as RouterLink, useParams } from "react-router-dom"; import { useDagServiceGetDagDetails, @@ -31,11 +26,10 @@ import { } from "openapi/queries"; import { ErrorAlert } from "src/components/ErrorAlert"; import { ProgressBar } from "src/components/ui"; -import { capitalize } from "src/utils"; +import { OpenGroupsProvider } from "src/context/openGroups"; import { Header } from "./Header"; - -const tabs = ["runs", "tasks", "events", "code"]; +import { DagTabs } from "./Tabs"; export const Dag = () => { const { dagId } = useParams(); @@ -57,17 +51,12 @@ export const Dag = () => { enabled: Boolean(dagId), }); - const { pathname } = useLocation(); - const runs = runsData?.dags.find((dagWithRuns) => dagWithRuns.dag_id === dagId) ?.latest_dag_runs ?? []; - const activeTab = - tabs.find((tab) => pathname.endsWith(`/${tab}`)) ?? "overview"; - return ( - <> + + ) : undefined} + + + {Boolean(isMapped) || Boolean(isGroup && !isOpen) ? ( + <> + + + + ) : undefined} + + + ); +}; diff --git a/airflow/ui/src/pages/DagsList/Dag/Graph/data.ts b/airflow/ui/src/pages/DagsList/Dag/Graph/data.ts new file mode 100644 index 0000000000000..68759fc4ebf73 --- /dev/null +++ b/airflow/ui/src/pages/DagsList/Dag/Graph/data.ts @@ -0,0 +1,216 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export type Edge = { + is_setup_teardown?: boolean; + label?: string; + source_id: string; + target_id: string; +}; + +export type Node = { + children?: Array; + id: string; + is_mapped?: boolean; + label: string; + setup_teardown_type?: "setup" | "teardown"; + tooltip?: string; + type: + | "asset_alias" + | "asset_condition" + | "asset" + | "dag" + | "join" + | "sensor" + | "task" + | "trigger"; +}; + +export type GraphData = { + arrange: "BT" | "LR" | "RL" | "TB"; + edges: Array; + nodes: Array; +}; + +export const graphData: GraphData = { + arrange: "LR", + edges: [ + { + source_id: "section_1.upstream_join_id", + target_id: "section_1.taskgroup_setup", + }, + { + source_id: "section_1.downstream_join_id", + target_id: "section_2.upstream_join_id", + }, + { + source_id: "section_1.normal", + target_id: "section_1.taskgroup_teardown", + }, + { + is_setup_teardown: true, + label: "setup and teardown", + source_id: "section_1.taskgroup_setup", + target_id: "section_1.taskgroup_teardown", + }, + { + label: "test", + source_id: "section_1.taskgroup_teardown", + target_id: "section_1.downstream_join_id", + }, + { + source_id: "section_1.taskgroup_setup", + target_id: "section_1.normal", + }, + { + source_id: "section_2.downstream_join_id", + target_id: "end", + }, + { + source_id: "section_2.inner_section_2.task_2", + target_id: "section_2.inner_section_2.task_4", + }, + { + source_id: "section_2.inner_section_2.task_3", + target_id: "section_2.inner_section_2.task_4", + }, + { + source_id: "section_2.inner_section_2.task_4", + target_id: "section_2.downstream_join_id", + }, + { + source_id: "section_2.task_1", + target_id: "section_2.downstream_join_id", + }, + { + source_id: "section_2.upstream_join_id", + target_id: "section_2.inner_section_2.task_2", + }, + { + source_id: "section_2.upstream_join_id", + target_id: "section_2.inner_section_2.task_3", + }, + { + source_id: "section_2.upstream_join_id", + target_id: "section_2.task_1", + }, + { + label: "I am a realllllllllllllllllly long label", + source_id: "start", + target_id: "section_1.upstream_join_id", + }, + ], + nodes: [ + { + id: "end", + label: "end", + type: "task", + }, + { + children: [ + { + id: "section_1.normal", + label: "normal", + type: "task", + }, + { + id: "section_1.taskgroup_setup", + label: "taskgroup_setup", + setup_teardown_type: "setup", + type: "task", + }, + { + id: "section_1.taskgroup_teardown", + label: "taskgroup_teardown", + setup_teardown_type: "teardown", + type: "task", + }, + { + id: "section_1.upstream_join_id", + label: "", + type: "join", + }, + { + id: "section_1.downstream_join_id", + label: "", + type: "join", + }, + ], + id: "section_1", + is_mapped: false, + label: "section_1", + tooltip: "Tasks for section_1", + type: "task", + }, + { + children: [ + { + children: [ + { + id: "section_2.inner_section_2.task_2", + label: "task_2", + type: "task", + }, + { + id: "section_2.inner_section_2.task_3", + is_mapped: true, + label: "task_3", + type: "task", + }, + { + id: "section_2.inner_section_2.task_4", + label: "task_4", + type: "task", + }, + ], + id: "section_2.inner_section_2", + label: "inner_section_2", + tooltip: "Tasks for inner_section2", + type: "task", + }, + { + id: "section_2.task_1", + is_mapped: true, + label: "task_1", + type: "task", + }, + { + id: "section_2.upstream_join_id", + label: "", + type: "join", + }, + { + id: "section_2.downstream_join_id", + label: "", + type: "join", + }, + ], + id: "section_2", + is_mapped: false, + label: "section_2", + tooltip: "Tasks for section_2", + type: "task", + }, + { + id: "start", + label: "start", + type: "task", + }, + ], +}; diff --git a/airflow/ui/src/pages/DagsList/Dag/Graph/index.ts b/airflow/ui/src/pages/DagsList/Dag/Graph/index.ts new file mode 100644 index 0000000000000..132a9d58992cf --- /dev/null +++ b/airflow/ui/src/pages/DagsList/Dag/Graph/index.ts @@ -0,0 +1,20 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export * from "./Graph"; diff --git a/airflow/ui/src/pages/DagsList/Dag/Graph/reactflowUtils.ts b/airflow/ui/src/pages/DagsList/Dag/Graph/reactflowUtils.ts new file mode 100644 index 0000000000000..76eb4c7489f36 --- /dev/null +++ b/airflow/ui/src/pages/DagsList/Dag/Graph/reactflowUtils.ts @@ -0,0 +1,140 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import type { Node as FlowNodeType, Edge as FlowEdgeType } from "@xyflow/react"; +import type { ElkExtendedEdge } from "elkjs"; + +import type { Node } from "./data"; +import type { LayoutNode } from "./useGraphLayout"; + +export type CustomNodeProps = { + childCount?: number; + height?: number; + isActive?: boolean; + isGroup?: boolean; + isMapped?: boolean; + isOpen?: boolean; + label: string; + setupTeardownType?: Node["setup_teardown_type"]; + width?: number; +}; + +type NodeType = FlowNodeType; + +type FlattenNodesProps = { + children?: Array; + parent?: NodeType; +}; + +// Generate a flattened list of nodes for react-flow to render +export const flattenGraph = ({ + children, + parent, +}: FlattenNodesProps): { + edges: Array; + nodes: Array; +} => { + let nodes: Array = []; + let edges: Array = []; + + if (!children) { + return { edges, nodes }; + } + const parentNode = parent ? { parentNode: parent.id } : undefined; + + children.forEach((node) => { + const x = (parent?.position.x ?? 0) + (node.x ?? 0); + const y = (parent?.position.y ?? 0) + (node.y ?? 0); + const newNode = { + data: node, + id: node.id, + position: { + x, + y, + }, + type: node.type, + ...parentNode, + } satisfies NodeType; + + edges = [ + ...edges, + ...(node.edges ?? []).map((edge) => ({ + ...edge, + labels: edge.labels?.map((label) => ({ + ...label, + x: (label.x ?? 0) + x, + y: (label.y ?? 0) + y, + })), + sections: edge.sections?.map((section) => ({ + ...section, + // eslint-disable-next-line max-nested-callbacks + bendPoints: section.bendPoints?.map((bp) => ({ + x: bp.x + x, + y: bp.y + y, + })), + endPoint: { + x: section.endPoint.x + x, + y: section.endPoint.y + y, + }, + startPoint: { + x: section.startPoint.x + x, + y: section.startPoint.y + y, + }, + })), + })), + ]; + + nodes.push(newNode); + + if (node.children) { + const { edges: childEdges, nodes: childNodes } = flattenGraph({ + children: node.children, + parent: newNode, + }); + + nodes = [...nodes, ...childNodes]; + edges = [...edges, ...childEdges]; + } + }); + + return { + edges, + nodes, + }; +}; + +type Edge = { + parentNode?: string; +} & ElkExtendedEdge; + +export type EdgeData = { + rest: { isSetupTeardown?: boolean } & ElkExtendedEdge; +}; + +export const formatFlowEdges = ({ + edges, +}: { + edges: Array; +}): Array> => + edges.map((edge) => ({ + data: { rest: edge }, + id: edge.id, + source: edge.sources[0] ?? "", + target: edge.targets[0] ?? "", + type: "custom", + })); diff --git a/airflow/ui/src/pages/DagsList/Dag/Graph/useGraphLayout.ts b/airflow/ui/src/pages/DagsList/Dag/Graph/useGraphLayout.ts new file mode 100644 index 0000000000000..cd0d1089078ca --- /dev/null +++ b/airflow/ui/src/pages/DagsList/Dag/Graph/useGraphLayout.ts @@ -0,0 +1,288 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { useQuery } from "@tanstack/react-query"; +import ELK, { type ElkNode, type ElkExtendedEdge, type ElkShape } from "elkjs"; + +import type { Edge, Node } from "./data"; +import { flattenGraph, formatFlowEdges } from "./reactflowUtils"; + +type EdgeLabel = { + height: number; + id: string; + text: string; + width: number; +}; + +type FormattedNode = { + childCount?: number; + edges?: Array; + isGroup: boolean; + isMapped?: boolean; + isOpen?: boolean; + setupTeardownType?: Node["setup_teardown_type"]; +} & ElkShape & + Node; + +type FormattedEdge = { + id: string; + isSetupTeardown?: boolean; + labels?: Array; + parentNode?: string; +} & ElkExtendedEdge; + +export type LayoutNode = ElkNode & Node; + +// Take text and font to calculate how long each node should be +const getTextWidth = (text: string, font: string) => { + const context = document.createElement("canvas").getContext("2d"); + + if (context) { + context.font = font; + const metrics = context.measureText(text); + + return metrics.width; + } + + return text.length * 9; +}; + +const getDirection = (arrange: string) => { + switch (arrange) { + case "BT": + return "UP"; + case "RL": + return "LEFT"; + case "TB": + return "DOWN"; + default: + return "RIGHT"; + } +}; + +const formatElkEdge = ( + edge: Edge, + font: string, + node?: Node, +): FormattedEdge => ({ + id: `${edge.source_id}-${edge.target_id}`, + isSetupTeardown: edge.is_setup_teardown, + // isSourceAsset: e.isSourceAsset, + labels: + edge.label === undefined + ? [] + : [ + { + height: 16, + id: edge.label, + text: edge.label, + width: getTextWidth(edge.label, font), + }, + ], + parentNode: node?.id, + sources: [edge.source_id], + targets: [edge.target_id], +}); + +const getNestedChildIds = (children: Array) => { + let childIds: Array = []; + + children.forEach((child) => { + childIds.push(child.id); + if (child.children) { + const nestedChildIds = getNestedChildIds(child.children); + + childIds = [...childIds, ...nestedChildIds]; + } + }); + + return childIds; +}; + +type GenerateElkProps = { + arrange: string; + edges: Array; + font: string; + nodes: Array; + openGroupIds?: Array; +}; + +const generateElkGraph = ({ + arrange, + edges: unformattedEdges, + font, + nodes, + openGroupIds, +}: GenerateElkProps): ElkNode => { + const closedGroupIds: Array = []; + let filteredEdges = unformattedEdges; + + const formatChildNode = (node: Node): FormattedNode => { + const isOpen = openGroupIds?.includes(node.id); + + const childCount = + node.children?.filter((child) => child.type !== "join").length ?? 0; + const childIds = + node.children === undefined ? [] : getNestedChildIds(node.children); + + if (isOpen && node.children !== undefined) { + return { + ...node, + childCount, + children: node.children.map(formatChildNode), + edges: filteredEdges + .filter((edge) => { + if ( + childIds.includes(edge.source_id) && + childIds.includes(edge.target_id) + ) { + // Remove edge from array when we add it here + filteredEdges = filteredEdges.filter( + (fe) => + !( + fe.source_id === edge.source_id && + fe.target_id === edge.target_id + ), + ); + + return true; + } + + return false; + }) + .map((edge) => formatElkEdge(edge, font, node)), + id: node.id, + isGroup: true, + isOpen, + label: node.label, + layoutOptions: { + "elk.padding": "[top=80,left=15,bottom=15,right=15]", + }, + }; + } + + if (!Boolean(isOpen) && node.children !== undefined) { + filteredEdges = filteredEdges + // Filter out internal group edges + .filter( + (fe) => + !( + childIds.includes(fe.source_id) && childIds.includes(fe.target_id) + ), + ) + // For external group edges, point to the group itself instead of a child node + .map((fe) => ({ + ...fe, + source_id: childIds.includes(fe.source_id) ? node.id : fe.source_id, + target_id: childIds.includes(fe.target_id) ? node.id : fe.target_id, + })); + closedGroupIds.push(node.id); + } + + const label = node.is_mapped ? `${node.label} [100]` : node.label; + const labelLength = getTextWidth(label, font); + let width = labelLength > 200 ? labelLength : 200; + let height = 80; + + if (node.type === "join") { + width = 10; + height = 10; + } else if (node.type === "asset_condition") { + width = 30; + height = 30; + } + + return { + childCount, + height, + id: node.id, + isGroup: Boolean(node.children), + isMapped: node.is_mapped, + label: node.label, + setupTeardownType: node.setup_teardown_type, + type: node.type, + width, + }; + }; + + const children = nodes.map(formatChildNode); + + const edges = filteredEdges.map((fe) => formatElkEdge(fe, font)); + + return { + children, + edges, + id: "root", + layoutOptions: { + "elk.core.options.EdgeLabelPlacement": "CENTER", + "elk.direction": getDirection(arrange), + hierarchyHandling: "INCLUDE_CHILDREN", + "spacing.edgeLabel": "10.0", + }, + }; +}; + +type LayoutProps = { + arrange?: string; + edges: Array; + nodes: Array; + openGroupIds: Array; +}; + +export const useGraphLayout = ({ + arrange = "LR", + edges, + nodes, + openGroupIds = [], +}: LayoutProps) => + useQuery({ + queryFn: async () => { + const font = `bold 16px ${ + globalThis.getComputedStyle(document.body).fontFamily + }`; + const elk = new ELK(); + + // 1. Format graph data to pass for elk to process + const graph = generateElkGraph({ + arrange, + edges, + font, + nodes, + openGroupIds, + }); + + // 2. use elk to generate the size and position of nodes and edges + const data = (await elk.layout(graph)) as LayoutNode; + + // 3. Flatten the nodes and edges for xyflow to actually render the graph + const flattenedData = flattenGraph({ + children: data.children, + }); + + // merge & dedupe edges + const flatEdges = [...(data.edges ?? []), ...flattenedData.edges].filter( + (value, index, self) => + index === self.findIndex((edge) => edge.id === value.id), + ); + + const formattedEdges = formatFlowEdges({ edges: flatEdges }); + + return { edges: formattedEdges, nodes: flattenedData.nodes }; + }, + queryKey: ["graphLayout", nodes.length, openGroupIds, arrange], + }); diff --git a/airflow/ui/src/pages/DagsList/Dag/Tabs.tsx b/airflow/ui/src/pages/DagsList/Dag/Tabs.tsx new file mode 100644 index 0000000000000..c2d0c4d2c40c4 --- /dev/null +++ b/airflow/ui/src/pages/DagsList/Dag/Tabs.tsx @@ -0,0 +1,90 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { Button, Flex, Tabs } from "@chakra-ui/react"; +import { + Link as RouterLink, + useLocation, + useParams, + useSearchParams, +} from "react-router-dom"; + +import type { DAGResponse } from "openapi/requests/types.gen"; +import { DagIcon } from "src/assets/DagIcon"; +import { capitalize } from "src/utils"; + +import { DagVizModal } from "./DagVizModal"; + +const tabs = ["runs", "tasks", "events", "code"]; + +const MODAL = "modal"; + +export const DagTabs = ({ dag }: { readonly dag?: DAGResponse }) => { + const { dagId } = useParams(); + const [searchParams, setSearchParams] = useSearchParams(); + + const modal = searchParams.get(MODAL); + + const isGraphOpen = modal === "graph"; + const onClose = () => { + searchParams.delete(MODAL); + setSearchParams(searchParams); + }; + + const onOpen = () => { + searchParams.set(MODAL, "graph"); + setSearchParams(searchParams); + }; + + const { pathname } = useLocation(); + + const activeTab = + tabs.find((tab) => pathname.endsWith(`/${tab}`)) ?? "overview"; + + return ( + <> + + + + + Overview + + {tabs.map((tab) => ( + + + {capitalize(tab)} + + + ))} + + + + + + + + + ); +}; diff --git a/airflow/ui/vite.config.ts b/airflow/ui/vite.config.ts index 7bc48d640418a..10bda68590345 100644 --- a/airflow/ui/vite.config.ts +++ b/airflow/ui/vite.config.ts @@ -17,6 +17,7 @@ * under the License. */ import react from "@vitejs/plugin-react-swc"; +import cssInjectedByJsPlugin from "vite-plugin-css-injected-by-js"; import { defineConfig } from "vitest/config"; // https://vitejs.dev/config/ @@ -32,6 +33,7 @@ export default defineConfig({ .replace(`src="/assets/`, `src="/static/assets/`) .replace(`href="/`, `href="/webapp/`), }, + cssInjectedByJsPlugin(), ], resolve: { alias: { openapi: "/openapi-gen", src: "/src" } }, test: {