From 9586d47afe950ad67e402df0c98595c850c22246 Mon Sep 17 00:00:00 2001 From: Mark Kekez <92973272+aliex-13@users.noreply.github.com> Date: Mon, 20 Mar 2023 23:12:13 +1030 Subject: [PATCH] Allow Lumen maintenance multiple windows to be parsed (#216) * Add multiple window checks for Lumen * NRE-605 fix comments * Add unit tests with multiple windows Lumen * Fix linter issues from PR * Fix another linter typo * Fix linter pydocstyle * readme changes for more instructions on local testing * Added some more instructions and examples for readme * Update test_e2e tests for Lumen * remove date tests in e2e file for Lumen8 --------- Co-authored-by: mkekez --- README.md | 22 +++ circuit_maintenance_parser/parsers/lumen.py | 25 ++- tests/unit/data/lumen/lumen8.html | 164 ++++++++++++++++++++ tests/unit/data/lumen/lumen8_result.json | 40 +++++ tests/unit/test_e2e.py | 48 ++++++ tests/unit/test_parsers.py | 5 + 6 files changed, 300 insertions(+), 4 deletions(-) create mode 100644 tests/unit/data/lumen/lumen8.html create mode 100644 tests/unit/data/lumen/lumen8_result.json diff --git a/README.md b/README.md index 8a429291..96011919 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,28 @@ The project is following Network to Code software development guidelines and is - The `Provider` also supports the definition of a `_include_filter` and a `_exclude_filter` to limit the notifications that are actually processed, avoiding false positive errors for notification that are not relevant. 4. Update the `unit/test_e2e.py` with the new provider, providing some data to test and validate the final `Maintenances` created. 5. **Expose the new `Provider` class** updating the map `SUPPORTED_PROVIDERS` in `circuit_maintenance_parser/__init__.py` to officially expose the `Provider`. +6. You can run some tests here to verify that your new unit tests do not cause issues with existing tests, and in general they work as expected. You can do this by running `pytest --log-cli-level=DEBUG --capture=tee-sys`. You can narrow down the tests that you want to execute with the `-k` flag. If successful, your results should look similar to the following: + +``` +-> % pytest --log-cli-level=DEBUG --capture=tee-sys -k test_parsers +...omitted debug logs... +====================================================== 99 passed, 174 deselected, 17 warnings in 10.35s ====================================================== +``` +7. Run some final CI tests locally to ensure that there is no linting/formatting issues with your changes. You should look to get a code score of 10/10. See the example below: `invoke tests --local` + +``` +-> % invoke tests --local +LOCAL - Running command black --check --diff . +All done! ✨ 🍰 ✨ +41 files would be left unchanged. +LOCAL - Running command flake8 . +LOCAL - Running command find . -name "*.py" | xargs pylint +************* Module tasks +tasks.py:4:0: W0402: Uses of a deprecated module 'distutils.util' (deprecated-module) + +-------------------------------------------------------------------- +Your code has been rated at 10.00/10 (previous run: 10.00/10, +0.00) +``` ### How to debug circuit-maintenance-parser library locally diff --git a/circuit_maintenance_parser/parsers/lumen.py b/circuit_maintenance_parser/parsers/lumen.py index eb0ca468..2b835d94 100644 --- a/circuit_maintenance_parser/parsers/lumen.py +++ b/circuit_maintenance_parser/parsers/lumen.py @@ -2,6 +2,7 @@ import logging from typing import Dict +from copy import deepcopy from dateutil import parser import bs4 # type: ignore from bs4.element import ResultSet # type: ignore @@ -19,10 +20,22 @@ class HtmlParserLumen1(Html): def parse_html(self, soup): """Execute parsing.""" + maintenances = [] data = {} self.parse_spans(soup.find_all("span"), data) self.parse_tables(soup.find_all("table"), data) - return [data] + + # Iterates over multiple windows and duplicates other maintenance info to a new dictionary while also updating start and end times for the specific window. + for window in data["windows"]: + maintenance = deepcopy(data) + maintenance["start"], maintenance["end"] = window + del maintenance["windows"] + maintenances.append(maintenance) + + # Deleting the key after we are finished checking for multiple windows and duplicating data. + del data["windows"] + + return maintenances def parse_spans(self, spans: ResultSet, data: Dict): """Parse Span tag.""" @@ -56,8 +69,11 @@ def parse_spans(self, spans: ResultSet, data: Dict): data["stamp"] = self.dt2ts(stamp) break - def parse_tables(self, tables: ResultSet, data: Dict): + def parse_tables(self, tables: ResultSet, data: Dict): # pylint: disable=too-many-locals """Parse Table tag.""" + # Initialise multiple windows list that will be used in parse_html + data["windows"] = [] + circuits = [] for table in tables: cells = table.find_all("td") @@ -68,9 +84,10 @@ def parse_tables(self, tables: ResultSet, data: Dict): for idx in range(num_columns, len(cells), num_columns): if "GMT" in cells[idx].string and "GMT" in cells[idx + 1].string: start = parser.parse(cells[idx].string.split(" GMT")[0]) - data["start"] = self.dt2ts(start) + start_ts = self.dt2ts(start) end = parser.parse(cells[idx + 1].string.split(" GMT")[0]) - data["end"] = self.dt2ts(end) + end_ts = self.dt2ts(end) + data["windows"].append((start_ts, end_ts)) break elif cells[0].string == "Customer Name": diff --git a/tests/unit/data/lumen/lumen8.html b/tests/unit/data/lumen/lumen8.html new file mode 100644 index 00000000..a4ed0f5c --- /dev/null +++ b/tests/unit/data/lumen/lumen8.html @@ -0,0 +1,164 @@ + + BODY { font-family: Arial; font-size:10pt;} + + + + + + + + + + + + + + + + + + + + + + + +

+         3D"Lumen" +

+
+ +Scheduled Maintenance #: 12345678 +

+Summary: +

+Lumen intends to carry out internal maintenance within its network. This h= +as been designated as ESSENTIAL. The nature of this work is to repair fiber= + and is required in order to avoid unplanned outages from damages related t= +o natural causes. +
+
The estimated GPS location of work is:=01 +
LAT: 2 LONG: -1 +
+
An alternate/backup maintenance window has been scheduled in the event = +work is unable to be completed on the primary night. Currently no services= + are scheduled to be impacted on the alternate night. Should the alternate= + night be deemed necessary, you will receive an updated notification advisi= +ng which services will be impacted on that night. Please see the date and = +times of the alternate night below. +
+
Lumen sincerely apologizes for any inconvenience caused by this main= +tenance. +

+Updates: +

+ +2023-03-09 16:23:23 GMT - This maintenance is scheduled.
+
+ +Customer Impact: +

+

12345678-1
= +
StartEnd
2023-03-22 04:00 GMT (G= +reenwich Mean Time)2023-03-22 10:00 GMT (Greenwich Mean Time)
2023-03-22 00:00 EDT (Eastern Daylight Time)2023-03-2= +2 06:00 EDT (Eastern Daylight Time)


Maintenance Location(s): RANDOM LOCATION= +;ANOTHER RANDOM ADDRESS SOMEWHERE

= +<= +td>Outage
Customer NameCircuit IDAlt= + Circuit IDBandwidthA LocationZ LocationImpact TypeMaximum DurationOrder Number
Customer123456789N/A100 GIG122 LATELY ST PONARM MI USA992 A PRAM= +KA ST CHICAGO IL USAOutage4 hours  
CUSTOMER = + 12345678910GLAN-1234567810GIG-E LAN= +SOMETHING SOMETHING SOMETHING RANDOM LOCATION IN THE USA 4 hours  

Alternate Night:
12345678-2
StartEnd
2023= +-03-23 04:00 GMT (Greenwich Mean Time)2023-03-23 10:00 GMT (Greenw= +ich Mean Time)
2023-03-23 00:00 EDT (Eastern Daylight Time= +)2023-03-23 06:00 EDT (Eastern Daylight Time)
+

+ + +Click here to open a case for assistance on this scheduled maintenance = +via Email. + +

+Click here for immediate information on scheduled maintenances via the = +Lumen Customer Portal. + +

+Click here to manage your notification subscriptions via the Lumen Port= +al. + +

+Click here to open a case for assistance on this scheduled maintenance = +via the Lumen Customer Portal. +


+Network Change Management Team +
+Lumen +
+1-855-244-6468 option 1 +

+The information in this communication is confidential and may not be disclo= +sed to=20 +third parties or shared further without the express permission of Lumen.= \ No newline at end of file diff --git a/tests/unit/data/lumen/lumen8_result.json b/tests/unit/data/lumen/lumen8_result.json new file mode 100644 index 00000000..17b0f886 --- /dev/null +++ b/tests/unit/data/lumen/lumen8_result.json @@ -0,0 +1,40 @@ +[ + { + "account": "Customer", + "circuits": [ + { + "circuit_id": "123456789", + "impact": "OUTAGE" + }, + { + "circuit_id": "123456789", + "impact": "OUTAGE" + } + ], + "end": 1679479200, + "maintenance_id": "12345678", + "stamp": 1678379003, + "start": 1679457600, + "status": "IN-PROCESS", + "summary": "Lumen intends to carry out internal maintenance within its network. This has been designated as ESSENTIAL. The nature of this work is to repair fiber and is required in order to avoid unplanned outages from damages related to natural causes." + }, + { + "account": "Customer", + "circuits": [ + { + "circuit_id": "123456789", + "impact": "OUTAGE" + }, + { + "circuit_id": "123456789", + "impact": "OUTAGE" + } + ], + "end": 1679565600, + "maintenance_id": "12345678", + "stamp": 1678379003, + "start": 1679544000, + "status": "IN-PROCESS", + "summary": "Lumen intends to carry out internal maintenance within its network. This has been designated as ESSENTIAL. The nature of this work is to repair fiber and is required in order to avoid unplanned outages from damages related to natural causes." + } +] \ No newline at end of file diff --git a/tests/unit/test_e2e.py b/tests/unit/test_e2e.py index 9522420e..bb1e5993 100644 --- a/tests/unit/test_e2e.py +++ b/tests/unit/test_e2e.py @@ -423,6 +423,54 @@ Path(dir_path, "data", "date", "email_date_1_result.json"), ], ), + ( + Lumen, + [ + ("html", Path(dir_path, "data", "lumen", "lumen5.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), + ], + [ + Path(dir_path, "data", "lumen", "lumen5_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), + ( + Lumen, + [ + ("html", Path(dir_path, "data", "lumen", "lumen6.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), + ], + [ + Path(dir_path, "data", "lumen", "lumen6_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), + ( + Lumen, + [ + ("html", Path(dir_path, "data", "lumen", "lumen7.html")), + (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), + ], + [ + Path(dir_path, "data", "lumen", "lumen7_result.json"), + Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), + ( + Lumen, + [ + ("html", Path(dir_path, "data", "lumen", "lumen8.html")), + # (EMAIL_HEADER_DATE, Path(dir_path, "data", "date", "email_date_1")), + (EMAIL_HEADER_SUBJECT, Path(dir_path, "data", "lumen", "subject_work_planned")), + ], + [ + Path(dir_path, "data", "lumen", "lumen8_result.json"), + # Path(dir_path, "data", "date", "email_date_1_result.json"), + ], + ), # Megaport ( Megaport, diff --git a/tests/unit/test_parsers.py b/tests/unit/test_parsers.py index 1be4978b..f0963aaf 100644 --- a/tests/unit/test_parsers.py +++ b/tests/unit/test_parsers.py @@ -317,6 +317,11 @@ Path(dir_path, "data", "lumen", "lumen7.html"), Path(dir_path, "data", "lumen", "lumen7_result.json"), ), + ( + HtmlParserLumen1, + Path(dir_path, "data", "lumen", "lumen8.html"), + Path(dir_path, "data", "lumen", "lumen8_result.json"), + ), # Megaport ( HtmlParserMegaport1,