diff --git a/CHANGELOG.md b/CHANGELOG.md index 323f6484e0..955a0d4ce8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## 0.16.11-dev1 +### Fixes + +- Fix ipv4 regex to correctly include up to three digit octets. + ### Enhancements - **Enhance quote standardization tests** with additional Unicode scenarios diff --git a/test_unstructured/cleaners/test_extract.py b/test_unstructured/cleaners/test_extract.py index 6ca059883c..86ac0e8487 100644 --- a/test_unstructured/cleaners/test_extract.py +++ b/test_unstructured/cleaners/test_extract.py @@ -5,7 +5,7 @@ from unstructured.cleaners import extract EMAIL_META_DATA_INPUT = """from ABC.DEF.local ([ba23::58b5:2236:45g2:88h2]) by - \n ABC.DEF.local ([ba23::58b5:2236:45g2:88h2%25]) with mapi id\ + \n ABC.DEF.local ([68.183.71.12]) with mapi id\ n 32.88.5467.123; Fri, 26 Mar 2021 11:04:09 +1200""" @@ -37,7 +37,7 @@ def test_extract_email_address(): def test_extract_ip_address(): assert extract.extract_ip_address(EMAIL_META_DATA_INPUT) == [ "ba23::58b5:2236:45g2:88h2", - "ba23::58b5:2236:45g2:88h2%25", + "68.183.71.12", ] diff --git a/unstructured/nlp/patterns.py b/unstructured/nlp/patterns.py index e18f067c4a..b3a77f77a8 100644 --- a/unstructured/nlp/patterns.py +++ b/unstructured/nlp/patterns.py @@ -92,9 +92,9 @@ ONE_LINE_BREAK_PARAGRAPH_PATTERN = r"^(?:(?!\.\s*$).)*$" ONE_LINE_BREAK_PARAGRAPH_PATTERN_RE = re.compile(ONE_LINE_BREAK_PARAGRAPH_PATTERN) -# IP Address examples: ba23::58b5:2236:45g2:88h2 or 10.0.2.01 +# IP Address examples: ba23::58b5:2236:45g2:88h2, 10.0.2.01 or 68.183.71.12 IP_ADDRESS_PATTERN = ( - r"[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}", + r"(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)){3}", "[a-z0-9]{4}::[a-z0-9]{4}:[a-z0-9]{4}:[a-z0-9]{4}:[a-z0-9]{4}%?[0-9]*", ) IP_ADDRESS_PATTERN_RE = re.compile(f"({'|'.join(IP_ADDRESS_PATTERN)})")