Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

♻️ Extract workbench from projects to projects_nodes table 🗃️ #7010

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
72 commits
Select commit Hold shift + click to select a range
5807f81
extract first columns
giancarloromeo Jan 7, 2025
4573217
add migration script
giancarloromeo Jan 7, 2025
05f619b
update thumbnail
giancarloromeo Jan 7, 2025
4c566ad
update script
giancarloromeo Jan 7, 2025
9494ab0
add more data
giancarloromeo Jan 7, 2025
dbc6366
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 7, 2025
bc54acc
fix test
giancarloromeo Jan 7, 2025
e97a1cf
fix node create
giancarloromeo Jan 7, 2025
bd9c666
fix node creation mock
giancarloromeo Jan 8, 2025
11455b2
fix fixture
giancarloromeo Jan 8, 2025
031d947
fix fixture
giancarloromeo Jan 8, 2025
d0f989e
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 8, 2025
49415a5
add columns
giancarloromeo Jan 8, 2025
0b701a7
fix sql
giancarloromeo Jan 8, 2025
4b91286
fix column name
giancarloromeo Jan 8, 2025
ebd80bb
add fields
giancarloromeo Jan 8, 2025
99a9eab
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 9, 2025
3bbd502
fix imports
giancarloromeo Jan 9, 2025
a325b15
Merge remote-tracking branch 'upstream/master' into is5646/extract-pr…
giancarloromeo Jan 9, 2025
a0fbe08
Merge remote-tracking branch 'upstream/master' into is5646/extract-pr…
giancarloromeo Jan 10, 2025
c7845fc
add new repo
giancarloromeo Jan 10, 2025
95bce7e
Merge remote-tracking branch 'upstream/master' into is5646/extract-pr…
giancarloromeo Jan 10, 2025
7d00d5d
fix import
giancarloromeo Jan 13, 2025
48145b4
fix mapping
giancarloromeo Jan 13, 2025
f71c1c6
fix model
giancarloromeo Jan 13, 2025
56099a8
fix model
giancarloromeo Jan 13, 2025
9136b34
fix field name
giancarloromeo Jan 13, 2025
279e538
add output update
giancarloromeo Jan 13, 2025
d45aace
rename service
giancarloromeo Jan 13, 2025
9572145
add missing column
giancarloromeo Jan 13, 2025
851c5be
fix tests
giancarloromeo Jan 13, 2025
cee9f8a
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 13, 2025
d2fd817
store only set values
giancarloromeo Jan 14, 2025
0ac9501
Merge branch 'is5646/extract-projects-workbench-column' of github.com…
giancarloromeo Jan 14, 2025
55b60bc
fix update node state
giancarloromeo Jan 14, 2025
d73fed1
fix packages
giancarloromeo Jan 14, 2025
46cfc4b
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 14, 2025
ac8c9a6
update sql
giancarloromeo Jan 14, 2025
6a865f3
fix name
giancarloromeo Jan 14, 2025
459d567
fix pylint
giancarloromeo Jan 14, 2025
0db6bc5
Merge remote-tracking branch 'upstream/master' into is5646/extract-pr…
giancarloromeo Jan 15, 2025
be92b63
revert _base.py
giancarloromeo Jan 15, 2025
be56df7
add migration SQL cmd
giancarloromeo Jan 15, 2025
a79bc7e
update script
giancarloromeo Jan 15, 2025
c48529d
fix pylint
giancarloromeo Jan 15, 2025
5d92523
fix model
giancarloromeo Jan 15, 2025
733055a
blank
giancarloromeo Jan 15, 2025
e93491f
add get
giancarloromeo Jan 15, 2025
31c4f50
fix column
giancarloromeo Jan 15, 2025
8f44d30
add more columns
giancarloromeo Jan 15, 2025
2d45d5c
blank
giancarloromeo Jan 15, 2025
623f926
add issue reference
giancarloromeo Jan 16, 2025
0e46459
Merge remote-tracking branch 'upstream/master' into is5646/extract-pr…
giancarloromeo Jan 20, 2025
2abb1e2
fixes
giancarloromeo Jan 20, 2025
61c13e9
requild script
giancarloromeo Jan 20, 2025
9d80edd
remove validation
giancarloromeo Jan 20, 2025
931bbec
rename method
giancarloromeo Jan 20, 2025
cdd0e95
pylint
giancarloromeo Jan 20, 2025
904d6e9
update comment
giancarloromeo Jan 20, 2025
1b232d8
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 20, 2025
3f7f275
remove deprecated
giancarloromeo Jan 20, 2025
893e9ed
remove unused model
giancarloromeo Jan 21, 2025
d2536cf
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 21, 2025
b227c16
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 21, 2025
febe935
update comment
giancarloromeo Jan 21, 2025
24f0155
Merge branch 'is5646/extract-projects-workbench-column' of github.com…
giancarloromeo Jan 21, 2025
cb26397
Merge remote-tracking branch 'upstream/master' into is5646/extract-pr…
giancarloromeo Jan 21, 2025
3a168e9
update script
giancarloromeo Jan 21, 2025
58c1aed
fix sonar
giancarloromeo Jan 21, 2025
65b43a1
fix names
giancarloromeo Jan 21, 2025
fc408e1
fix import
giancarloromeo Jan 21, 2025
5ece129
Merge branch 'master' into is5646/extract-projects-workbench-column
giancarloromeo Jan 21, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

from ..api_schemas_directorv2.dynamic_services import RetrieveDataOut
from ..basic_types import PortInt
from ..projects_nodes import InputID, InputsDict
from ..projects_nodes import InputID, InputsDict, PartialNode
from ..projects_nodes_io import NodeID
from ..services import ServiceKey, ServicePortKey, ServiceVersion
from ..services_enums import ServiceState
Expand All @@ -26,14 +26,26 @@ class NodeCreate(InputSchemaWithoutCamelCase):


class NodePatch(InputSchemaWithoutCamelCase):
service_key: ServiceKey | None = Field(default=None, alias="key")
service_version: ServiceVersion | None = Field(default=None, alias="version")
label: str | None = Field(default=None)
service_key: Annotated[
ServiceKey | None,
Field(alias="key"),
] = None
service_version: Annotated[
ServiceVersion | None,
Field(alias="version"),
] = None
label: str | None = None
inputs: Annotated[
InputsDict, Field(default_factory=dict, json_schema_extra={"default": {}})
]
inputs_required: list[InputID] | None = Field(default=None, alias="inputsRequired")
input_nodes: list[NodeID] | None = Field(default=None, alias="inputNodes")
inputs_required: Annotated[
list[InputID] | None,
Field(alias="inputsRequired"),
] = None
input_nodes: Annotated[
list[NodeID] | None,
Field(alias="inputNodes"),
] = None
progress: Annotated[
float | None,
Field(
Expand All @@ -47,6 +59,13 @@ class NodePatch(InputSchemaWithoutCamelCase):
str, Any
] | None = None # NOTE: it is used by frontend for File Picker

def to_domain_model(self) -> PartialNode:
data = self.model_dump(
exclude_unset=True,
by_alias=True,
)
return PartialNode.model_construct(**data)


class NodeCreated(OutputSchema):
node_id: NodeID
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -274,3 +274,9 @@ def _convert_from_enum(cls, v):
extra="forbid",
populate_by_name=True,
)


class PartialNode(Node):
giancarloromeo marked this conversation as resolved.
Show resolved Hide resolved
key: Annotated[ServiceKey, Field(default=None)]
version: Annotated[ServiceVersion, Field(default=None)]
label: Annotated[str, Field(default=None)]
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
"""extract workbench column

Revision ID: ecd4eadaa781
Revises: a3a58471b0f1
Create Date: 2025-01-21 13:13:18.256109+00:00

"""
import sqlalchemy as sa
from alembic import op
from sqlalchemy.dialects import postgresql

# revision identifiers, used by Alembic.
revision = "ecd4eadaa781"
down_revision = "a3a58471b0f1"
branch_labels = None
depends_on = None


def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column(
"projects_nodes",
sa.Column(
"key",
sa.String(),
nullable=False,
comment="Distinctive name (based on the Docker registry path)",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"version", sa.String(), nullable=False, comment="Semantic version number"
),
)
op.add_column(
"projects_nodes",
sa.Column(
"label", sa.String(), nullable=False, comment="Short name used for display"
),
)
op.add_column(
"projects_nodes",
sa.Column(
"progress", sa.Numeric(), nullable=True, comment="Progress value (0-100)"
),
)
op.add_column(
"projects_nodes",
sa.Column(
"thumbnail",
sa.String(),
nullable=True,
comment="Url of the latest screenshot",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"input_access",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="Map with key - access level pairs",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"input_nodes",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="IDs of the nodes where is connected to",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"inputs",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="Input properties values",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"inputs_required",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="Required input IDs",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"inputs_units",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="Input units",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"output_nodes",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="Node IDs of those connected to the output",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"outputs",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="Output properties values",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"run_hash",
sa.String(),
nullable=True,
comment="HEX digest of the resolved inputs + outputs hash at the time when the last outputs were generated",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"state",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="State",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"parent",
sa.String(),
nullable=True,
comment="Parent's (group-nodes) node ID",
),
)
op.add_column(
"projects_nodes",
sa.Column(
"boot_options",
postgresql.JSONB(astext_type=sa.Text()),
nullable=True,
comment="Some services provide alternative parameters to be injected at boot time.The user selection should be stored here, and it will overwrite the services's defaults",
),
)
# ### end Alembic commands ###

op.execute(
"""
UPDATE projects_nodes
SET key = subquery.key,
version = subquery.version,
label = subquery.label,
progress = subquery.progress::numeric,
thumbnail = subquery.thumbnail,
input_access = subquery.input_access::jsonb,
input_nodes = subquery.input_nodes::jsonb,
inputs = subquery.inputs::jsonb,
inputs_required = subquery.inputs_required::jsonb,
inputs_units = subquery.inputs_units::jsonb,
output_nodes = subquery.output_nodes::jsonb,
outputs = subquery.outputs::jsonb,
run_hash = subquery.run_hash,
state = subquery.state::jsonb,
parent = subquery.parent,
boot_options = subquery.boot_options::jsonb
FROM (
SELECT
projects.uuid AS project_id,
js.key AS node_id,
js.value::jsonb ->> 'key' AS key,
js.value::jsonb ->> 'label' AS label,
js.value::jsonb ->> 'version' AS version,
(js.value::jsonb ->> 'progress')::numeric AS progress,
js.value::jsonb ->> 'thumbnail' AS thumbnail,
js.value::jsonb ->> 'inputAccess' AS input_access,
js.value::jsonb ->> 'inputNodes' AS input_nodes,
js.value::jsonb ->> 'inputs' AS inputs,
js.value::jsonb ->> 'inputsRequired' AS inputs_required,
js.value::jsonb ->> 'inputsUnits' AS inputs_units,
js.value::jsonb ->> 'outputNodes' AS output_nodes,
js.value::jsonb ->> 'outputs' AS outputs,
js.value::jsonb ->> 'runHash' AS run_hash,
js.value::jsonb ->> 'state' AS state,
js.value::jsonb ->> 'parent' AS parent,
js.value::jsonb ->> 'bootOptions' AS boot_options
FROM projects,
json_each(projects.workbench) AS js
) AS subquery
WHERE projects_nodes.project_uuid = subquery.project_id
AND projects_nodes.node_id = subquery.node_id;
"""
)
op.alter_column("projects_nodes", "key", nullable=False)
op.alter_column("projects_nodes", "version", nullable=False)
op.alter_column("projects_nodes", "label", nullable=False)


def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("projects_nodes", "boot_options")
op.drop_column("projects_nodes", "parent")
op.drop_column("projects_nodes", "state")
op.drop_column("projects_nodes", "run_hash")
op.drop_column("projects_nodes", "outputs")
op.drop_column("projects_nodes", "output_nodes")
op.drop_column("projects_nodes", "inputs_units")
op.drop_column("projects_nodes", "inputs_required")
op.drop_column("projects_nodes", "inputs")
op.drop_column("projects_nodes", "input_nodes")
op.drop_column("projects_nodes", "input_access")
op.drop_column("projects_nodes", "thumbnail")
op.drop_column("projects_nodes", "progress")
op.drop_column("projects_nodes", "label")
op.drop_column("projects_nodes", "version")
op.drop_column("projects_nodes", "key")
# ### end Alembic commands ###
Loading
Loading