From e611c91472ebf36dd569197f8ea41a712e60de40 Mon Sep 17 00:00:00 2001 From: Joelle Maslak Date: Wed, 25 Oct 2023 08:39:59 -0700 Subject: [PATCH] Handle broken EUNetworks cancellation messages (#243) * Handle broken EUNetworks cancellation messages * Code formatting should fit black's constraints * Rework to allow empty circuit lists in cancelled notifications without a circuit list --- README.md | 2 +- circuit_maintenance_parser/output.py | 15 ++--- circuit_maintenance_parser/parser.py | 8 ++- .../data/eunetworks/eunetworks_cancel.eml | 63 +++++++++++++++++++ .../data/eunetworks/eunetworks_cancel.json | 16 +++++ tests/unit/data/ical/ical7 | 18 ++++++ tests/unit/data/ical/ical7_result.json | 16 +++++ tests/unit/test_e2e.py | 4 +- tests/unit/test_parsers.py | 5 ++ 9 files changed, 135 insertions(+), 12 deletions(-) create mode 100644 tests/unit/data/eunetworks/eunetworks_cancel.eml create mode 100644 tests/unit/data/eunetworks/eunetworks_cancel.json create mode 100644 tests/unit/data/ical/ical7 create mode 100644 tests/unit/data/ical/ical7_result.json diff --git a/README.md b/README.md index 96011919..b460da0a 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ You can leverage this library in your automation framework to process circuit ma - **provider**: identifies the provider of the service that is the subject of the maintenance notification. - **account**: identifies an account associated with the service that is the subject of the maintenance notification. - **maintenance_id**: contains text that uniquely identifies (at least within the context of a specific provider) the maintenance that is the subject of the notification. -- **circuits**: list of circuits affected by the maintenance notification and their specific impact. +- **circuits**: list of circuits affected by the maintenance notification and their specific impact. Note that in a maintenance cancelled notification, some providers omit the circuit list, so this may be blank for maintenance notifications with a status of CANCELLED. - **start**: timestamp that defines the starting date/time of the maintenance in GMT. - **end**: timestamp that defines the ending date/time of the maintenance in GMT. - **stamp**: timestamp that defines the update date/time of the maintenance in GMT. diff --git a/circuit_maintenance_parser/output.py b/circuit_maintenance_parser/output.py index c45e31b3..5effd8b4 100644 --- a/circuit_maintenance_parser/output.py +++ b/circuit_maintenance_parser/output.py @@ -98,14 +98,15 @@ class Maintenance(BaseModel, extra=Extra.forbid): provider: identifies the provider of the service that is the subject of the maintenance notification account: identifies an account associated with the service that is the subject of the maintenance notification maintenance_id: contains text that uniquely identifies the maintenance that is the subject of the notification - circuits: list of circuits affected by the maintenance notification and their specific impact + circuits: list of circuits affected by the maintenance notification and their specific impact. Note this can be + an empty list for notifications with a CANCELLED status if the provider does not populate the circuit list. + status: defines the overall status or confirmation for the maintenance start: timestamp that defines the start date of the maintenance in GMT end: timestamp that defines the end date of the maintenance in GMT stamp: timestamp that defines the update date of the maintenance in GMT organizer: defines the contact information included in the original notification Optional attributes: - status: defines the overall status or confirmation for the maintenance summary: description of the maintenace notification uid: specific unique identifier for each notification sequence: sequence number - initially zero - to serialize updates in case they are received or processed out of @@ -126,18 +127,18 @@ class Maintenance(BaseModel, extra=Extra.forbid): ... summary="This is a maintenance notification", ... uid="1111", ... ) - Maintenance(provider='A random NSP', account='12345000', maintenance_id='VNOC-1-99999999999', circuits=[CircuitImpact(circuit_id='123', impact=), CircuitImpact(circuit_id='456', impact=)], start=1533704400, end=1533712380, stamp=1533595768, organizer='myemail@example.com', status=, uid='1111', sequence=1, summary='This is a maintenance notification') + Maintenance(provider='A random NSP', account='12345000', maintenance_id='VNOC-1-99999999999', status=, circuits=[CircuitImpact(circuit_id='123', impact=), CircuitImpact(circuit_id='456', impact=)], start=1533704400, end=1533712380, stamp=1533595768, organizer='myemail@example.com', uid='1111', sequence=1, summary='This is a maintenance notification') """ provider: StrictStr account: StrictStr maintenance_id: StrictStr + status: Status circuits: List[CircuitImpact] start: StrictInt end: StrictInt stamp: StrictInt organizer: StrictStr - status: Status # Non mandatory attributes uid: StrictStr = "0" @@ -160,9 +161,9 @@ def validate_empty_strings(cls, value): return value @validator("circuits") - def validate_empty_circuits(cls, value): - """Validate emptry strings.""" - if len(value) < 1: + def validate_empty_circuits(cls, value, values): + """Validate non-cancel notifications have a populated circuit list.""" + if len(value) < 1 and values["status"] != "CANCELLED": raise ValueError("At least one circuit has to be included in the maintenance") return value diff --git a/circuit_maintenance_parser/parser.py b/circuit_maintenance_parser/parser.py index 170af547..abd21ea4 100644 --- a/circuit_maintenance_parser/parser.py +++ b/circuit_maintenance_parser/parser.py @@ -127,7 +127,11 @@ def parse_ical(gcal: Calendar) -> List[Dict]: data = {key: value for key, value in data.items() if value != "None"} # In a VEVENT sometimes there are mutliple object ID with custom impacts - circuits = component.get("X-MAINTNOTE-OBJECT-ID") + # In addition, while circuits should always be populated according to the BCOP, sometimes + # they are not in the real world, at least in maintenances with a CANCELLED status. Thus + # we allow empty circuit lists, but will validate elsewhere that they are only empty in a + # maintenance object with a CANCELLED status. + circuits = component.get("X-MAINTNOTE-OBJECT-ID", []) if isinstance(circuits, list): data["circuits"] = [ CircuitImpact( @@ -136,7 +140,7 @@ def parse_ical(gcal: Calendar) -> List[Dict]: object.params.get("X-MAINTNOTE-OBJECT-IMPACT", component.get("X-MAINTNOTE-IMPACT")) ), ) - for object in component.get("X-MAINTNOTE-OBJECT-ID") + for object in component.get("X-MAINTNOTE-OBJECT-ID", []) ] else: data["circuits"] = [ diff --git a/tests/unit/data/eunetworks/eunetworks_cancel.eml b/tests/unit/data/eunetworks/eunetworks_cancel.eml new file mode 100644 index 00000000..31fa47cc --- /dev/null +++ b/tests/unit/data/eunetworks/eunetworks_cancel.eml @@ -0,0 +1,63 @@ +Delivered-To: example@example.com +Return-Path: +Date: Fri, 6 Oct 2023 09:57:00 +0000 (GMT) +From: Maintenance +To: user@example.com" +Message-ID: +Subject: CANCELLED euNetworks Emergency Work: 02345678 +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----=_Part_9999_1445545000.1696586220000" +X-Priority: 3 +X-Sender: postmaster@salesforce.com +X-mail_abuse_inquiries: http://www.salesforce.com/company/abuse.jsp +Reply-To: Maintenance + +------=_Part_9999_1445545000.1696586220000 +Content-Type: multipart/alternative; + boundary="----=_Part_9998_1422331000.1696586220000" + +------=_Part_9998_1422331000.1696586220000 +Content-Type: text/plain; charset=ISO-8859-1 +Content-Transfer-Encoding: quoted-printable + +Maintenance Announcement +Dear Customer, + +Please be advised that the previously announced works have been cancelled. = +If an alternative date should be required, this will be announced in time. + +TRIMMED STUFF HERE. + + +ref:00D200000000A05.500N1000002uvs4:ref +------=_Part_9998_1422331000.1696586220000 +Content-Type: text/html; charset=ISO-8859-1 +Content-Transfer-Encoding: quoted-printable + +TRIMMED STUFF HERE +------=_Part_9998_1422331000.1696586220000-- + +------=_Part_9999_1445545000.1696586220000 +Content-Type: text/calendar; name="X-02345678.ics" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; filename="X-02345678.ics" + +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Microsoft Corporation//Outlook 11.0 MIMEDIR//EN +BEGIN:VEVENT +SUMMARY: Maintenance note +DTSTART;VALUE=DATE-TIME:20231006T230000Z +DTEND;VALUE=DATE-TIME:20231007T170000Z +DTSTAMP;VALUE=DATE-TIME:20231006T095600Z +UID:02345678@euNetworks.com +SEQUENCE:2 +X-MAINTNOTE-PROVIDER:euNetworks.com +X-MAINTNOTE-ACCOUNT:A Company +X-MAINTNOTE-MAINTENANCE-ID:02345678 +X-MAINTNOTE-STATUS:CANCELLED +ORGANIZER;CN=noc@eunetworks.com:mailto:noc@eunetworks.com +END:VEVENT +END:VCALENDAR +------=_Part_9999_1445545000.1696586220000-- diff --git a/tests/unit/data/eunetworks/eunetworks_cancel.json b/tests/unit/data/eunetworks/eunetworks_cancel.json new file mode 100644 index 00000000..312b45a6 --- /dev/null +++ b/tests/unit/data/eunetworks/eunetworks_cancel.json @@ -0,0 +1,16 @@ +[ + { + "account": "A Company", + "circuits": [], + "end": 1696698000, + "maintenance_id": "02345678", + "organizer": "mailto:noc@eunetworks.com", + "provider": "euNetworks.com", + "sequence": 2, + "stamp": 1696586160, + "start": 1696633200, + "status": "CANCELLED", + "summary": " Maintenance note ", + "uid": "02345678@euNetworks.com" + } +] diff --git a/tests/unit/data/ical/ical7 b/tests/unit/data/ical/ical7 new file mode 100644 index 00000000..e05ee6cd --- /dev/null +++ b/tests/unit/data/ical/ical7 @@ -0,0 +1,18 @@ +BEGIN:VCALENDAR +VERSION:2.0 +PRODID:-//Microsoft Corporation//Outlook 11.0 MIMEDIR//EN +BEGIN:VEVENT +SUMMARY:Maintenance note +DTSTART;VALUE=DATE-TIME:20231006T230000Z +DTEND;VALUE=DATE-TIME:20231007T170000Z +DTSTAMP;VALUE=DATE-TIME:20231006T095600Z +UID:02345678@euNetworks.com +SEQUENCE:2 +X-MAINTNOTE-PROVIDER:euNetworks.com +X-MAINTNOTE-ACCOUNT:Bogus Corp +X-MAINTNOTE-MAINTENANCE-ID:02345678 +X-MAINTNOTE-IMPACT:OUTAGE +X-MAINTNOTE-STATUS:CANCELLED +ORGANIZER;CN=noc@eunetworks.com:mailto:noc@eunetworks.com +END:VEVENT +END:VCALENDAR diff --git a/tests/unit/data/ical/ical7_result.json b/tests/unit/data/ical/ical7_result.json new file mode 100644 index 00000000..b95d78f7 --- /dev/null +++ b/tests/unit/data/ical/ical7_result.json @@ -0,0 +1,16 @@ +[ + { + "account": "Bogus Corp", + "circuits": [], + "end": 1696698000, + "maintenance_id": "02345678", + "organizer": "mailto:noc@eunetworks.com", + "provider": "euNetworks.com", + "sequence": 2, + "stamp": 1696586160, + "start": 1696633200, + "status": "CANCELLED", + "summary": "Maintenance note", + "uid": "02345678@euNetworks.com" + } +] diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index 61f1d24a..0a19a759 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -274,10 +274,10 @@ ( EUNetworks, [ - ("ical", GENERIC_ICAL_DATA_PATH), + ("email", Path(dir_path, "data", "eunetworks", "eunetworks_cancel.eml")), ], [ - GENERIC_ICAL_RESULT_PATH, + Path(dir_path, "data", "eunetworks", "eunetworks_cancel.json"), ], ), # EXA / GTT diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py index b2a2a900..f0f706fb 100644 --- a/tests/unit/test_parsers.py +++ b/tests/unit/test_parsers.py @@ -68,6 +68,11 @@ Path(dir_path, "data", "ical", "ical6"), Path(dir_path, "data", "ical", "ical6_result.json"), ), + ( + ICal, + Path(dir_path, "data", "ical", "ical7"), + Path(dir_path, "data", "ical", "ical7_result.json"), + ), # AquaComms ( HtmlParserAquaComms1,