Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-48169: Add new components_json field to the jira_fields table. #28

Merged
merged 5 commits into from
Jan 10, 2025
Merged

DM-48169: Add new components_json field to the jira_fields table. #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0def901
Add missing exclude parameters to test_find_messages.py.
sebastian-aranda Dec 20, 2024
a5c2333
Add new components_json field to the jira_fields table.
sebastian-aranda Dec 16, 2024
8823c65
Extend tests to support new components_json field.
sebastian-aranda Dec 18, 2024
36ad769
Add get_version endpoint to identify REST API current version
sebastian-aranda Dec 20, 2024
f31a7c9
Refactor components_json field structure to represent hierarchy among…
sebastian-aranda Jan 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions alembic/versions/49ef39173f83_add_components_json_field_to_jira_.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
"""add components_json field to jira_fields table

Revision ID: 49ef39173f83
Revises: 54f755dbdb6f
Create Date: 2024-12-18 17:01:39.676895

"""
import logging

import sqlalchemy as sa

from alembic import op

# revision identifiers, used by Alembic.
revision = "49ef39173f83"
down_revision = "54f755dbdb6f"
branch_labels = None
depends_on = None


JIRA_FIELDS_TABLE_NAME = "jira_fields"


def upgrade(log: logging.Logger, table_names: set[str]) -> None:
if JIRA_FIELDS_TABLE_NAME not in table_names:
log.info(f"No {JIRA_FIELDS_TABLE_NAME} table; nothing to do")
return
log.info("Add 'components_json'")

op.add_column(
JIRA_FIELDS_TABLE_NAME,
sa.Column("components_json", sa.JSON(), nullable=True),
)


def downgrade(log: logging.Logger, table_names: set[str]) -> None:
if JIRA_FIELDS_TABLE_NAME not in table_names:
log.info(f"No {JIRA_FIELDS_TABLE_NAME} table; nothing to do")
return

log.info("Drop 'components_json'")
op.drop_column(JIRA_FIELDS_TABLE_NAME, "components_json")
16 changes: 15 additions & 1 deletion src/narrativelog/create_tables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import sqlalchemy as sa
import sqlalchemy.types as saty
from sqlalchemy.dialects.postgresql import UUID
from sqlalchemy.dialects.postgresql import JSONB, UUID

# Length of the site_id field.
SITE_ID_LEN = 16
Expand Down Expand Up @@ -60,8 +60,14 @@ def create_message_table(metadata: sa.MetaData) -> sa.Table:
sa.Column("date_invalidated", saty.DateTime(), nullable=True),
sa.Column("parent_id", UUID(as_uuid=True), nullable=True),
# Added 2022-07-19
# 'systems' field is deprecated and will be removed in v1.0.0.
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't know any foolproof way to do deprecation for an API unless the API is always accessed via a paired client. Pairing client and API would be important if there were lots of non-Rubin users using the API. I assume that is NOT the case (if my assumption is wrong, maybe we should talk about what I've done in NOIRLab). But, it would be good to add an API endpiont that gives a semantic version. API documentation could warn that when the major component of the version increments, there have backwards incompatible changes. That would at least let API user know when an API change put their (API using) code in a dangerous place.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it 👍 I'll work on adding a /version endpoint.

# Please use 'components_json' instead
sa.Column("systems", saty.ARRAY(sa.Text), nullable=True),
# 'subsystems' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
sa.Column("subsystems", saty.ARRAY(sa.Text), nullable=True),
# 'cscs' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
sa.Column("cscs", saty.ARRAY(sa.Text), nullable=True),
# Added 2022-07-37
sa.Column("date_end", saty.DateTime(), nullable=True),
Expand Down Expand Up @@ -110,10 +116,18 @@ def create_jira_fields_table(metadata: sa.MetaData) -> sa.Table:
sa.Column(
"id", UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
),
# Added 2024-12-16
sa.Column("components_json", JSONB, nullable=True),
# 'components' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
sa.Column("components", saty.ARRAY(sa.Text), nullable=True),
# 'primary_software_components' field is deprecated
# and will be removed in v1.0.0. Please use 'components_json' instead
sa.Column(
"primary_software_components", saty.ARRAY(sa.Text), nullable=True
),
# 'primary_hardware_components' field is deprecated
# and will be removed in v1.0.0. Please use 'components_json' instead
sa.Column(
"primary_hardware_components", saty.ARRAY(sa.Text), nullable=True
),
Expand Down
2 changes: 2 additions & 0 deletions src/narrativelog/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
find_messages,
get_configuration,
get_message,
get_version,
)

app = fastapi.FastAPI()
Expand All @@ -27,6 +28,7 @@
subapp.include_router(find_messages.router)
subapp.include_router(get_configuration.router)
subapp.include_router(get_message.router)
subapp.include_router(get_version.router)


@subapp.get("/", response_class=fastapi.responses.HTMLResponse)
Expand Down
59 changes: 53 additions & 6 deletions src/narrativelog/message.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

from pydantic import BaseModel, Field

from .utils import JIRA_OBS_SYSTEMS_HIERARCHY_MARKDOWN_LINK


class Message(BaseModel):
id: uuid.UUID = Field(title="Message ID: a UUID that is the primary key.")
Expand Down Expand Up @@ -46,13 +48,21 @@ class Message(BaseModel):
)
# Added 2022-07-19
systems: None | list[str] = Field(
title="Zero or more system names.",
title="Zero or more system names. "
"This field is deprecated and will be removed in v1.0.0. "
"Please use 'components_json' instead.",
)
subsystems: None | list[str] = Field(
title="Zero or more subsystem names. "
"This field is deprecated and will be removed in v1.0.0. "
"Please use 'components_json' instead.",
)
subsystems: None | list[str] = Field(title="Zero or more subsystem names.")
cscs: None | list[str] = Field(
title="Zero or more CSCs names. "
"Each entry should be in the form 'name' or 'name:index', "
"where 'name' is the SAL component name and 'index' is the SAL index."
"where 'name' is the SAL component name and 'index' is the SAL index. "
"This field is deprecated and will be removed in v1.0.0. "
"Please use 'components_json' instead.",
)
# Added 2022-07-27
date_end: None | datetime.datetime = Field(
Expand All @@ -61,15 +71,21 @@ class Message(BaseModel):
# Added 2023-08-10
components: None | list[str] = Field(
title="Zero or more component names. "
"Each entry should be a valid component name entry on the OBS jira project.",
"Each entry should be a valid component name entry on the OBS jira project. "
"This field is deprecated and will be removed in v1.0.0. "
"Please use 'components_json' instead.",
)
primary_software_components: None | list[str] = Field(
title="Zero or more primary software component names. "
"Each entry should be a valid component name entry on the OBS jira project.",
"Each entry should be a valid component name entry on the OBS jira project. "
"This field is deprecated and will be removed in v1.0.0. "
"Please use 'components_json' instead.",
)
primary_hardware_components: None | list[str] = Field(
title="Zero or more primary hardware component names. "
"Each entry should be a valid component name entry on the OBS jira project.",
"Each entry should be a valid component name entry on the OBS jira project. "
"This field is deprecated and will be removed in v1.0.0. "
"Please use 'components_json' instead.",
)
# Added 2023-10-24
category: None | str = Field(
Expand All @@ -78,16 +94,47 @@ class Message(BaseModel):
time_lost_type: None | str = Field(
title="Type of time lost.",
)
# Added 2024-12-16
components_json: None | dict = Field(
default_factory=dict,
title="""
JSON representation of systems, subsystems and components hierarchy
on the OBS jira project. An example of a valid payload is:

{
"name": "AuxTel",
"children": [
{
"name": "Dome",
"children": [
{
"name": "AT Azimuth Drives"
}
]
}
]
}

For a full list of valid systems, subsystems and components please refer to: """
f"""{JIRA_OBS_SYSTEMS_HIERARCHY_MARKDOWN_LINK}.""",
)

class Config:
orm_mode = True
from_attributes = True


JIRA_FIELDS = (
# 'components' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
"components",
# 'primary_software_components' field is deprecated
# and will be removed in v1.0.0. Please use 'components_json' instead
"primary_software_components",
# 'primary_hardware_components' field is deprecated
# and will be removed in v1.0.0. Please use 'components_json' instead
"primary_hardware_components",
"components_json",
)
MESSAGE_FIELDS = tuple(
set(Message.schema()["properties"].keys()) - set(JIRA_FIELDS)
Expand Down
71 changes: 65 additions & 6 deletions src/narrativelog/routers/add_message.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

from ..message import Message
from ..shared_state import SharedState, get_shared_state
from ..utils import JIRA_OBS_SYSTEMS_HIERARCHY_MARKDOWN_LINK
from .normalize_tags import TAG_DESCRIPTION, normalize_tags

router = fastapi.APIRouter()
Expand Down Expand Up @@ -41,37 +42,73 @@ async def add_message(
systems: None
| list[str] = fastapi.Body(
default=None,
description="Zero or more systems to which the message applies.",
description="Zero or more systems to which the message applies. "
"**This field is deprecated and will be removed in v1.0.0**. "
"Please use 'components_json' instead.",
),
subsystems: None
| list[str] = fastapi.Body(
default=None,
description="Zero or more subsystems to which the message applies",
description="Zero or more subsystems to which the message applies. "
"**This field is deprecated and will be removed in v1.0.0**. "
"Please use 'components_json' instead.",
),
cscs: None
| list[str] = fastapi.Body(
default=None,
description="Zero or more CSCs to which the message applies. "
"Each entry should be in the form 'name' or 'name:index', "
"where 'name' is the SAL component name and 'index' is the SAL index.",
"where 'name' is the SAL component name and 'index' is the SAL index. "
"**This field is deprecated and will be removed in v1.0.0**. "
"Please use 'components_json' instead.",
),
components: None
| list[str] = fastapi.Body(
default=None,
description="Zero or more components to which the message applies. "
"Each entry should be a valid component name entry on the OBS jira project.",
"Each entry should be a valid component name entry on the OBS jira project. "
"**This field is deprecated and will be removed in v1.0.0**. "
"Please use 'components_json' instead.",
),
primary_software_components: None
| list[str] = fastapi.Body(
default=None,
description="Primary software components to which the message applies. "
"Each entry should be a valid component name entry on the OBS jira project.",
"Each entry should be a valid component name entry on the OBS jira project. "
"**This field is deprecated and will be removed in v1.0.0**. "
"Please use 'components_json' instead.",
),
primary_hardware_components: None
| list[str] = fastapi.Body(
default=None,
description="Primary hardware components to which the message applies. "
"Each entry should be a valid component name entry on the OBS jira project.",
"Each entry should be a valid component name entry on the OBS jira project. "
"**This field is deprecated and will be removed in v1.0.0**. "
"Please use 'components_json' instead.",
),
components_json: None
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe the API documentation is in another place. If this documentation that will show on swagger, there should be more detail here. Its not clear to me if you are allowing for search against the parts of this JSONB file. If you are allowing search against parts, it will take a special kind of search which means special kind of description for how to do it. When I've done something similar, I was able to make searches against the keys in the jsonb field to look like searches in the schema columns. To do so, I confined the jsonb field to one level (keys within in are like column names in the schema, and the values associated with they keys are column values. ) I did that because end users (anyone in the world) might access the API. There are more restrictions on your narrativelog API. Or maybe you are handling it somewhere else and I just haven't gotten to that part yet.

| dict = fastapi.Body(
default=None,
description="""
JSON representation of systems, subsystems and components hierarchy
on the OBS jira project. An example of a valid payload is:

{
"name": "AuxTel",
"children": [
{
"name": "Dome",
"children": [
{
"name": "AT Azimuth Drives"
}
]
}
]
}

For a full list of valid systems, subsystems and components please refer to: """
f"""{JIRA_OBS_SYSTEMS_HIERARCHY_MARKDOWN_LINK}.""",
),
urls: list[str] = fastapi.Body(
default=[],
Expand Down Expand Up @@ -147,8 +184,14 @@ async def add_message(
user_agent=user_agent,
is_human=is_human,
date_added=curr_tai.tai.datetime,
# 'systems' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
systems=systems,
# 'subsystems' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
subsystems=subsystems,
# 'cscs' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
cscs=cscs,
category=category,
time_lost_type=time_lost_type,
Expand All @@ -166,17 +209,33 @@ async def add_message(
if any(
field is not None
for field in (
# 'components' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
components,
# 'primary_software_components' field is deprecated
# and will be removed in v1.0.0. Please use 'components_json' instead
primary_software_components,
# 'primary_hardware_components' field is deprecated
# and will be removed in v1.0.0. Please use 'components_json' instead
primary_hardware_components,
components_json,
)
):
result_jira_fields = await connection.execute(
jira_fields_table.insert()
.values(
# 'components' field is deprecated and will be removed in v1.0.0.
# Please use 'components_json' instead
components=components,
# 'primary_software_components' field is deprecated
# and will be removed in v1.0.0.
# Please use 'components_json' instead
primary_software_components=primary_software_components,
# 'primary_hardware_components' field is deprecated
# and will be removed in v1.0.0.
# Please use 'components_json' instead
primary_hardware_components=primary_hardware_components,
components_json=components_json,
message_id=row_message.id,
)
.returning(sa.literal_column("*"))
Expand Down
Loading
Loading