Handle broken EUNetworks cancellation messages (#243)

* Handle broken EUNetworks cancellation messages * Code formatting should fit black's constraints * Rework to allow empty circuit lists in cancelled notifications without a circuit list
networktocode · Oct 25, 2023 · e611c91 · e611c91
1 parent a5dee28
commit e611c91
Show file tree

Hide file tree

Showing 9 changed files with 135 additions and 12 deletions.
diff --git a/README.md b/README.md
@@ -24,7 +24,7 @@ You can leverage this library in your automation framework to process circuit ma
 - **provider**: identifies the provider of the service that is the subject of the maintenance notification.
 - **account**: identifies an account associated with the service that is the subject of the maintenance notification.
 - **maintenance_id**: contains text that uniquely identifies (at least within the context of a specific provider) the maintenance that is the subject of the notification.
-- **circuits**: list of circuits affected by the maintenance notification and their specific impact.
+- **circuits**: list of circuits affected by the maintenance notification and their specific impact.  Note that in a maintenance cancelled notification, some providers omit the circuit list, so this may be blank for maintenance notifications with a status of CANCELLED.
 - **start**: timestamp that defines the starting date/time of the maintenance in GMT.
 - **end**: timestamp that defines the ending date/time of the maintenance in GMT.
 - **stamp**: timestamp that defines the update date/time of the maintenance in GMT.

diff --git a/circuit_maintenance_parser/output.py b/circuit_maintenance_parser/output.py
@@ -98,14 +98,15 @@ class Maintenance(BaseModel, extra=Extra.forbid):
         provider: identifies the provider of the service that is the subject of the maintenance notification
         account:  identifies an account associated with the service that is the subject of the maintenance notification
         maintenance_id:  contains text that uniquely identifies the maintenance that is the subject of the notification
-        circuits: list of circuits affected by the maintenance notification and their specific impact
+        circuits: list of circuits affected by the maintenance notification and their specific impact. Note this can be
+            an empty list for notifications with a CANCELLED status if the provider does not populate the circuit list.
+        status: defines the overall status or confirmation for the maintenance
         start: timestamp that defines the start date of the maintenance in GMT
         end: timestamp that defines the end date of the maintenance in GMT
         stamp: timestamp that defines the update date of the maintenance in GMT
         organizer: defines the contact information included in the original notification
 
     Optional attributes:
-        status: defines the overall status or confirmation for the maintenance
         summary: description of the maintenace notification
         uid: specific unique identifier for each notification
         sequence: sequence number - initially zero - to serialize updates in case they are received or processed out of
@@ -126,18 +127,18 @@ class Maintenance(BaseModel, extra=Extra.forbid):
         ...     summary="This is a maintenance notification",
         ...     uid="1111",
         ... )
-        Maintenance(provider='A random NSP', account='12345000', maintenance_id='VNOC-1-99999999999', circuits=[CircuitImpact(circuit_id='123', impact=<Impact.NO_IMPACT: 'NO-IMPACT'>), CircuitImpact(circuit_id='456', impact=<Impact.OUTAGE: 'OUTAGE'>)], start=1533704400, end=1533712380, stamp=1533595768, organizer='[email protected]', status=<Status.COMPLETED: 'COMPLETED'>, uid='1111', sequence=1, summary='This is a maintenance notification')
+        Maintenance(provider='A random NSP', account='12345000', maintenance_id='VNOC-1-99999999999', status=<Status.COMPLETED: 'COMPLETED'>, circuits=[CircuitImpact(circuit_id='123', impact=<Impact.NO_IMPACT: 'NO-IMPACT'>), CircuitImpact(circuit_id='456', impact=<Impact.OUTAGE: 'OUTAGE'>)], start=1533704400, end=1533712380, stamp=1533595768, organizer='[email protected]', uid='1111', sequence=1, summary='This is a maintenance notification')
     """
 
     provider: StrictStr
     account: StrictStr
     maintenance_id: StrictStr
+    status: Status
     circuits: List[CircuitImpact]
     start: StrictInt
     end: StrictInt
     stamp: StrictInt
     organizer: StrictStr
-    status: Status
 
     # Non mandatory attributes
     uid: StrictStr = "0"
@@ -160,9 +161,9 @@ def validate_empty_strings(cls, value):
         return value
 
     @validator("circuits")
-    def validate_empty_circuits(cls, value):
-        """Validate emptry strings."""
-        if len(value) < 1:
+    def validate_empty_circuits(cls, value, values):
+        """Validate non-cancel notifications have a populated circuit list."""
+        if len(value) < 1 and values["status"] != "CANCELLED":
             raise ValueError("At least one circuit has to be included in the maintenance")
         return value
 

diff --git a/circuit_maintenance_parser/parser.py b/circuit_maintenance_parser/parser.py
@@ -127,7 +127,11 @@ def parse_ical(gcal: Calendar) -> List[Dict]:
                 data = {key: value for key, value in data.items() if value != "None"}
 
                 # In a VEVENT sometimes there are mutliple object ID with custom impacts
-                circuits = component.get("X-MAINTNOTE-OBJECT-ID")
+                # In addition, while circuits should always be populated according to the BCOP, sometimes
+                # they are not in the real world, at least in maintenances with a CANCELLED status.  Thus
+                # we allow empty circuit lists, but will validate elsewhere that they are only empty in a
+                # maintenance object with a CANCELLED status.
+                circuits = component.get("X-MAINTNOTE-OBJECT-ID", [])
                 if isinstance(circuits, list):
                     data["circuits"] = [
                         CircuitImpact(
@@ -136,7 +140,7 @@ def parse_ical(gcal: Calendar) -> List[Dict]:
                                 object.params.get("X-MAINTNOTE-OBJECT-IMPACT", component.get("X-MAINTNOTE-IMPACT"))
                             ),
                         )
-                        for object in component.get("X-MAINTNOTE-OBJECT-ID")
+                        for object in component.get("X-MAINTNOTE-OBJECT-ID", [])
                     ]
                 else:
                     data["circuits"] = [

diff --git a/tests/unit/data/eunetworks/eunetworks_cancel.eml b/tests/unit/data/eunetworks/eunetworks_cancel.eml
@@ -0,0 +1,63 @@
+Delivered-To: [email protected]
+Return-Path: <[email protected]>
+Date: Fri, 6 Oct 2023 09:57:00 +0000 (GMT)
+From: Maintenance <[email protected]>
+To: [email protected]"
+Message-ID: <[email protected]>
+Subject: CANCELLED euNetworks Emergency Work: 02345678
+MIME-Version: 1.0
+Content-Type: multipart/mixed; 
+	boundary="----=_Part_9999_1445545000.1696586220000"
+X-Priority: 3
+X-Sender: [email protected]
+X-mail_abuse_inquiries: http://www.salesforce.com/company/abuse.jsp
+Reply-To: Maintenance <[email protected]>
+
+------=_Part_9999_1445545000.1696586220000
+Content-Type: multipart/alternative; 
+	boundary="----=_Part_9998_1422331000.1696586220000"
+
+------=_Part_9998_1422331000.1696586220000
+Content-Type: text/plain; charset=ISO-8859-1
+Content-Transfer-Encoding: quoted-printable
+
+Maintenance Announcement
+Dear Customer,
+
+Please be advised that the previously announced works have been cancelled. =
+If an alternative date should be required, this will be announced in time.
+
+TRIMMED STUFF HERE.
+
+
+ref:00D200000000A05.500N1000002uvs4:ref
+------=_Part_9998_1422331000.1696586220000
+Content-Type: text/html; charset=ISO-8859-1
+Content-Transfer-Encoding: quoted-printable
+
+TRIMMED STUFF HERE
+------=_Part_9998_1422331000.1696586220000--
+
+------=_Part_9999_1445545000.1696586220000
+Content-Type: text/calendar; name="X-02345678.ics"
+Content-Transfer-Encoding: 7bit
+Content-Disposition: attachment; filename="X-02345678.ics"
+
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Microsoft Corporation//Outlook 11.0 MIMEDIR//EN
+BEGIN:VEVENT
+SUMMARY: Maintenance note 
+DTSTART;VALUE=DATE-TIME:20231006T230000Z
+DTEND;VALUE=DATE-TIME:20231007T170000Z
+DTSTAMP;VALUE=DATE-TIME:20231006T095600Z
+UID:[email protected]
+SEQUENCE:2
+X-MAINTNOTE-PROVIDER:euNetworks.com
+X-MAINTNOTE-ACCOUNT:A Company
+X-MAINTNOTE-MAINTENANCE-ID:02345678
+X-MAINTNOTE-STATUS:CANCELLED
+ORGANIZER;[email protected]:mailto:[email protected]
+END:VEVENT
+END:VCALENDAR
+------=_Part_9999_1445545000.1696586220000--
diff --git a/tests/unit/data/eunetworks/eunetworks_cancel.json b/tests/unit/data/eunetworks/eunetworks_cancel.json
@@ -0,0 +1,16 @@
+[
+    {
+        "account": "A Company",
+        "circuits": [],
+        "end": 1696698000,
+        "maintenance_id": "02345678",
+        "organizer": "mailto:[email protected]",
+        "provider": "euNetworks.com",
+        "sequence": 2,
+        "stamp": 1696586160,
+        "start": 1696633200,
+        "status": "CANCELLED",
+        "summary": " Maintenance note ",
+        "uid": "[email protected]"
+    }
+]
diff --git a/tests/unit/data/ical/ical7 b/tests/unit/data/ical/ical7
@@ -0,0 +1,18 @@
+BEGIN:VCALENDAR
+VERSION:2.0
+PRODID:-//Microsoft Corporation//Outlook 11.0 MIMEDIR//EN
+BEGIN:VEVENT
+SUMMARY:Maintenance note
+DTSTART;VALUE=DATE-TIME:20231006T230000Z
+DTEND;VALUE=DATE-TIME:20231007T170000Z
+DTSTAMP;VALUE=DATE-TIME:20231006T095600Z
+UID:[email protected]
+SEQUENCE:2
+X-MAINTNOTE-PROVIDER:euNetworks.com
+X-MAINTNOTE-ACCOUNT:Bogus Corp
+X-MAINTNOTE-MAINTENANCE-ID:02345678
+X-MAINTNOTE-IMPACT:OUTAGE
+X-MAINTNOTE-STATUS:CANCELLED
+ORGANIZER;[email protected]:mailto:[email protected]
+END:VEVENT
+END:VCALENDAR
diff --git a/tests/unit/data/ical/ical7_result.json b/tests/unit/data/ical/ical7_result.json
@@ -0,0 +1,16 @@
+[
+    {
+        "account": "Bogus Corp",
+        "circuits": [],
+        "end": 1696698000,
+        "maintenance_id": "02345678",
+        "organizer": "mailto:[email protected]",
+        "provider": "euNetworks.com",
+        "sequence": 2,
+        "stamp": 1696586160,
+        "start": 1696633200,
+        "status": "CANCELLED",
+        "summary": "Maintenance note",
+        "uid": "[email protected]"
+    }
+]
diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py
@@ -274,10 +274,10 @@
         (
             EUNetworks,
             [
-                ("ical", GENERIC_ICAL_DATA_PATH),
+                ("email", Path(dir_path, "data", "eunetworks", "eunetworks_cancel.eml")),
             ],
             [
-                GENERIC_ICAL_RESULT_PATH,
+                Path(dir_path, "data", "eunetworks", "eunetworks_cancel.json"),
             ],
         ),
         # EXA / GTT

diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py
@@ -68,6 +68,11 @@
             Path(dir_path, "data", "ical", "ical6"),
             Path(dir_path, "data", "ical", "ical6_result.json"),
         ),
+        (
+            ICal,
+            Path(dir_path, "data", "ical", "ical7"),
+            Path(dir_path, "data", "ical", "ical7_result.json"),
+        ),
         # AquaComms
         (
             HtmlParserAquaComms1,