Merge pull request #19 from lsst-sqre/tickets/DM-40369

DM-40369: create jira_fields table to include new parameters

.pre-commit-config.yaml

@@ -6,18 +6,19 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/pycqa/isort
-    rev: 5.5.2
+    rev: 5.12.0
     hooks:
       - id: isort
+        name: isort (python)
         additional_dependencies:
           - toml
 
   - repo: https://github.com/psf/black
-    rev: 22.3.0
+    rev: 23.3.0
     hooks:
       - id: black
 
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 4.0.1
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
     hooks:
       - id: flake8

CHANGELOG.rst

@@ -2,6 +2,13 @@
 Change Log
 ==========
 
+0.5.0
+-----
+
+* Add jira_fields table used to store specific jira metadata associated to messages: ``components``, ``primary_software_components`` and ``primary_hardware_components``.
+* Update pre-commit hooks.
+* Update some requirements versions to avoid conflicts.
+
 0.4.0
 -----
 

requirements/dev.in

@@ -6,11 +6,11 @@
 #     make update-deps
 
 click~=8.0
-black==22.3.0
+black==23.3.0
 coverage[toml]~=6.3
 flake8~=4.0
 httpx~=0.22
-mypy==0.950
+mypy==1.4.0
 pre-commit~=2.19
 pytest~=7.1
 testing.postgresql~=1.3

requirements/main.in

@@ -5,13 +5,13 @@
 # After editing, update requirements/main.txt by running:
 #     make update-deps
 
-# prevent a resolution conflict by preventing h11 0.13.0
-# uvicorn (many verisons) gets 0.13.0, but httpcore 0.14.5 demands h11 0.12.0
-h11==0.12.0
+# fastapi set to 0.79 to keep compatibility
+# with pydantic v1
+h11~=0.14.0
 alembic~=1.7
-astropy~=5.0
+astropy~=5.0.0
 asyncpg~=0.25
-fastapi<1
+fastapi~=0.79.0
 importlib_metadata~=4.11
 sqlalchemy~=1.4, <1.5
 structlog~=21.5

src/narrativelog/create_message_table.py → src/narrativelog/create_tables.py

@@ -1,4 +1,8 @@
-__all__ = ["SITE_ID_LEN", "create_message_table"]
+__all__ = [
+    "SITE_ID_LEN",
+    "create_message_table",
+    "create_jira_fields_table",
+]
 
 import uuid
 
@@ -10,11 +14,22 @@
 SITE_ID_LEN = 16
 
 
-def create_message_table() -> sa.Table:
-    """Make a model of the narrativelog message table."""
+def create_message_table(metadata: sa.MetaData) -> sa.Table:
+    """Make a model of the narrativelog message table.
+
+    Parameters
+    ----------
+    metadata: sa.MetaData
+        SQLAlchemy metadata object.
+
+    Returns
+    -------
+    table: sa.Table
+        SQLAlchemy table object for message.
+    """
     table = sa.Table(
         "message",
-        sa.MetaData(),
+        metadata,
         # See https://stackoverflow.com/a/49398042 for UUID:
         sa.Column(
             "id", UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
@@ -39,9 +54,9 @@ def create_message_table() -> sa.Table:
         sa.Column("date_invalidated", saty.DateTime(), nullable=True),
         sa.Column("parent_id", UUID(as_uuid=True), nullable=True),
         # Added 2022-07-19
-        sa.Column("systems", saty.ARRAY(sa.Text), nullable=False),
-        sa.Column("subsystems", saty.ARRAY(sa.Text), nullable=False),
-        sa.Column("cscs", saty.ARRAY(sa.Text), nullable=False),
+        sa.Column("systems", saty.ARRAY(sa.Text), nullable=True),
+        sa.Column("subsystems", saty.ARRAY(sa.Text), nullable=True),
+        sa.Column("cscs", saty.ARRAY(sa.Text), nullable=True),
         # Added 2022-07-37
         sa.Column("date_end", saty.DateTime(), nullable=True),
         # Constraints
@@ -59,3 +74,37 @@ def create_message_table() -> sa.Table:
         sa.Index(f"idx_{name}", table.columns[name])
 
     return table
+
+
+def create_jira_fields_table(metadata: sa.MetaData) -> sa.Table:
+    """Make a model of the narrativelog jira fields table.
+
+    Parameters
+    ----------
+    metadata: sa.MetaData
+        SQLAlchemy metadata object.
+
+    Returns
+    -------
+    table: sa.Table
+        SQLAlchemy table object for jira_fields.
+    """
+    table = sa.Table(
+        "jira_fields",
+        metadata,
+        sa.Column(
+            "id", UUID(as_uuid=True), primary_key=True, default=uuid.uuid4
+        ),
+        sa.Column("components", saty.ARRAY(sa.Text), nullable=True),
+        sa.Column(
+            "primary_software_components", saty.ARRAY(sa.Text), nullable=True
+        ),
+        sa.Column(
+            "primary_hardware_components", saty.ARRAY(sa.Text), nullable=True
+        ),
+        sa.Column("message_id", UUID(as_uuid=True), nullable=False),
+        # Constraints
+        sa.ForeignKeyConstraint(["message_id"], ["message.id"]),
+    )
+
+    return table

src/narrativelog/log_message_database.py

@@ -11,27 +11,32 @@
 
 
 class LogMessageDatabase:
-    """Connection to the narrative log database and message table.
+    """Connection to the narrative log database and tables creation.
 
-    Create the table if it does not exist.
+    Creates message and jira_fields table if they do not exist.
 
     Parameters
     ----------
-    message_table
+    message_table: sa.Table
         Message table.
-    url
+    jira_fields_table: sa.Table
+        Jira fields table.
+    url: str
         URL of narrative log database server in the form:
         postgresql://[user[:password]@][netloc][:port][/dbname]
     """
 
-    def __init__(self, message_table: sa.Table, url: str):
+    def __init__(
+        self, message_table: sa.Table, jira_fields_table: sa.Table, url: str
+    ):
         self._closed = False
         self.url = url
         self.logger = structlog.get_logger("LogMessageDatabase")
         sa_url = sqlalchemy.engine.make_url(url)
         sa_url = sa_url.set(drivername="postgresql+asyncpg")
         self.engine = create_async_engine(sa_url, future=True)
         self.message_table = message_table
+        self.jira_fields_table = jira_fields_table
         self.start_task = asyncio.create_task(self.start())
 
     async def start(self) -> None:

src/narrativelog/message.py

@@ -3,75 +3,88 @@
 import datetime
 import uuid
 
-import pydantic
+from pydantic import BaseModel, Field
 
 
-class Message(pydantic.BaseModel):
-    id: uuid.UUID = pydantic.Field(
-        title="Message ID: a UUID that is the primary key."
-    )
-    site_id: str = pydantic.Field(
-        title="Site at which the message was created."
-    )
-    message_text: str = pydantic.Field(title="Message.")
-    level: int = pydantic.Field(
+class Message(BaseModel):
+    id: uuid.UUID = Field(title="Message ID: a UUID that is the primary key.")
+    site_id: str = Field(title="Site at which the message was created.")
+    message_text: str = Field(title="Message.")
+    level: int = Field(
         title="Message level. A python logging level: "
         "info=20, warning=30, error=40."
     )
-    tags: list[str] = pydantic.Field(
+    tags: list[str] = Field(
         title="Zero or more space-separated keywords relevant to this message."
     )
-    urls: list[str] = pydantic.Field(
+    urls: list[str] = Field(
         title="Zero or more space-separated URLS to JIRA tickets, screen shots, etc."
     )
-    time_lost: datetime.timedelta = pydantic.Field(
+    time_lost: datetime.timedelta = Field(
         title="Estimate of lost on-sky time."
     )
-    date_begin: None | datetime.datetime = pydantic.Field(
+    date_begin: None | datetime.datetime = Field(
         title="Approximate initial TAI date at which the message is relevant "
         "(if different than the time at which the message was specified)"
     )
-    user_id: str = pydantic.Field(title="User ID.")
-    user_agent: str = pydantic.Field(
+    user_id: str = Field(title="User ID.")
+    user_agent: str = Field(
         title="User agent: the application that created the message."
     )
-    is_human: bool = pydantic.Field(
-        title="Was it a human who created the message?"
-    )
-    is_valid: bool = pydantic.Field(
+    is_human: bool = Field(title="Was it a human who created the message?")
+    is_valid: bool = Field(
         title="Is this message still valid (false if deleted or edited)."
     )
-    date_added: datetime.datetime = pydantic.Field(
+    date_added: datetime.datetime = Field(
         title="TAI date at which the message was added."
     )
-    date_invalidated: None | datetime.datetime = pydantic.Field(
+    date_invalidated: None | datetime.datetime = Field(
         title="TAI date at which is_valid was last set true."
     )
-    parent_id: None | uuid.UUID = pydantic.Field(
+    parent_id: None | uuid.UUID = Field(
         title="Message ID of message this is an edited version of."
     )
     # Added 2022-07-19
-    systems: list[str] = pydantic.Field(
+    systems: None | list[str] = Field(
         title="Zero or more system names.",
     )
-    subsystems: list[str] = pydantic.Field(
-        title="Zero or more subsystem names."
-    )
-    cscs: list[str] = pydantic.Field(
+    subsystems: None | list[str] = Field(title="Zero or more subsystem names.")
+    cscs: None | list[str] = Field(
         title="Zero or more CSCs names. "
         "Each entry should be in the form 'name' or 'name:index', "
         "where 'name' is the SAL component name and 'index' is the SAL index."
     )
     # Added 2022-07-27
-    date_end: None | datetime.datetime = pydantic.Field(
+    date_end: None | datetime.datetime = Field(
         title="Approximate final TAI date at which the message is relevant"
     )
+    # Added 2023-08-10
+    components: None | list[str] = Field(
+        title="Zero or more component names. "
+        "Each entry should be a valid component name entry on the OBS jira project.",
+    )
+    primary_software_components: None | list[str] = Field(
+        title="Zero or more primary software component names. "
+        "Each entry should be a valid component name entry on the OBS jira project.",
+    )
+    primary_hardware_components: None | list[str] = Field(
+        title="Zero or more primary hardware component names. "
+        "Each entry should be a valid component name entry on the OBS jira project.",
+    )
 
     class Config:
         orm_mode = True
+        from_attributes = True
 
 
-MESSAGE_FIELDS = tuple(Message.schema()["properties"].keys())
+JIRA_FIELDS = (
+    "components",
+    "primary_software_components",
+    "primary_hardware_components",
+)
+MESSAGE_FIELDS = tuple(
+    set(Message.schema()["properties"].keys()) - set(JIRA_FIELDS)
+)
 
 
 def _make_message_order_by_values() -> tuple[str, ...]: