fix dns extraction

bbot/core/helpers/regexes.py

@@ -38,10 +38,13 @@
 )
 ip_range_regexes = [re.compile(r, re.I) for r in _ip_range_regexes]
 
-# dns names with periods
+# all dns names including IP addresses and bare hostnames (e.g. "localhost")
 _dns_name_regex = r"(?:\w(?:[\w-]{0,100}\w)?\.?)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
-dns_name_extraction_regex = re.compile(_dns_name_regex, re.I)
-dns_name_validation_regex = re.compile(r"^" + _dns_name_regex + r"$", re.I)
+# dns names with periods (e.g. "www.example.com")
+_dns_name_regex_with_period = r"(?:\w(?:[\w-]{0,100}\w)?\.)+(?:[xX][nN]--)?[^\W_]{1,63}\.?"
+
+dns_name_extraction_regex = re.compile(_dns_name_regex_with_period, re.I)
+dns_name_validation_regex = re.compile(r"^" + _dns_name_regex_with_period + r"$", re.I)
 
 _email_regex = r"(?:[^\W_][\w\-\.\+']{,100})@" + _dns_name_regex
 email_regex = re.compile(_email_regex, re.I)

bbot/test/test_step_1/test_dns.py

@@ -776,6 +776,37 @@ async def test_dns_graph_structure(bbot_scanner):
     assert str(events_by_data["evilcorp.com"].module) == "host"
 
 
+@pytest.mark.asyncio
+async def test_hostname_extraction(bbot_scanner):
+    scan = bbot_scanner("evilcorp.com", config={"dns": {"minimal": False}})
+    await scan.helpers.dns._mock_dns(
+        {
+            "evilcorp.com": {
+                "A": ["127.0.0.1"],
+                "TXT": ["v=spf1 include:spf-a.evilcorp.com include:spf-b.evilcorp.com include:icpbounce.com include:shops.shopify.com include:_spf.qemailserver.com include:spf.mandrillapp.com include:spf.protection.office365.us include:spf-003ea501.gpphosted.com 127.0.0.1 -all"]
+            }
+        }
+    )
+    events = [e async for e in scan.async_start()]
+    dns_name_events = [e for e in events if e.type == "DNS_NAME"]
+    main_dns_event = [e for e in dns_name_events if e.data == "evilcorp.com"]
+    assert len(main_dns_event) == 1
+    main_dns_event = main_dns_event[0]
+    dns_children = main_dns_event.dns_children
+    assert dns_children["A"] == {"127.0.0.1"}
+    assert dns_children["TXT"] == {
+        "spf-a.evilcorp.com",
+        "spf-b.evilcorp.com",
+        "icpbounce.com",
+        "shops.shopify.com",
+        "_spf.qemailserver.com",
+        "spf.mandrillapp.com",
+        "spf.protection.office365.us",
+        "spf-003ea501.gpphosted.com",
+        "127.0.0.1",
+    }
+
+
 @pytest.mark.asyncio
 async def test_dns_helpers(bbot_scanner):
     assert service_record("") is False