From dd26772b5622277eb751997a04d257cdaa290ad2 Mon Sep 17 00:00:00 2001 From: FahadAnyit Date: Tue, 16 Jan 2024 18:12:44 +0500 Subject: [PATCH 1/4] added whoisfreaks module --- modules/sfp_whoisfreaks.py | 174 ++++++++++++++++++++++ test/unit/modules/test_sfp_whoisfreaks.py | 0 2 files changed, 174 insertions(+) create mode 100644 modules/sfp_whoisfreaks.py create mode 100644 test/unit/modules/test_sfp_whoisfreaks.py diff --git a/modules/sfp_whoisfreaks.py b/modules/sfp_whoisfreaks.py new file mode 100644 index 0000000000..951b356cdf --- /dev/null +++ b/modules/sfp_whoisfreaks.py @@ -0,0 +1,174 @@ +# -*- coding: utf-8 -*- +# ------------------------------------------------------------------------------- +# Name: sfp_whoisfreaks +# Purpose: Query whoisfreaks.com using their API. +# +# Author: Mian Fahad +# +# Created: 11/10/2023 +# Copyright: (c) Mian Fahad 2024 +# Licence: MIT +# ------------------------------------------------------------------------------- + +import json + +from spiderfoot import SpiderFootEvent, SpiderFootPlugin + + +class sfp_whoisfreaks(SpiderFootPlugin): + meta = { + 'name': "WhoisFreaks", + 'summary': "Reverse Whois Lookup by owner email or name or company name", + 'flags': ["slow", "apiKey"], + 'useCases': ["Investigate", "Passive", "Footprint"], + 'categories': ["Search Engines"], + 'dataSource': { + 'website': "https://whoisfreaks.com/", + 'model': "FREE_AUTH_LIMITED", + 'references': [ + "https://whoisfreaks.com/products/whois-api.html" + ], + 'apiKeyInstructions': [ + "Visit https://whoisfreaks.com/signup.html", + "Register an account.", + "The API key will be available on billing dashboard after signup", + "500 Free credits upon signup", + "select a plan to request beyond this limit", + "Visit https://whoisfreaks.com/pricing/api-plans.html", + + ], + 'favIcon': "https://whoisfreaks.com/images/icons/favicon.ico", + 'logo': "https://whoisfreaks.com/images/logo.webp", + 'description': "Search domain names by owner email or name or company name" + " through our Reverse WHOIS lookup API" + } + } + + # Default options + opts = { + "api_key": "" + } + + # Option descriptions + optdescs = { + "api_key": "WhoisFreaks account registered API key.", + } + + # Be sure to completely clear any class variables in setup() + # or you run the risk of data persisting between scan runs. + + results = None + errorState = False + + def setup(self, sfc, userOpts=dict()): + self.sf = sfc + self.results = self.tempStorage() + + for opt in list(userOpts.keys()): + self.opts[opt] = userOpts[opt] + + # What events is this module interested in for input + def watchedEvents(self): + return [ + "COMPANY_NAME", + "HUMAN_NAME", + "EMAILADDR", + "EMAILADDR_GENERIC", + ] + + # What events this module produces + def producedEvents(self): + return [ + 'AFFILIATE_DOMAIN_NAME' + ] + + # Search WhoisFreaks + def query(self, qry, querytype, page=1, accum=None): + url = "https://api.whoisfreaks.com/v1.0/whois?whois=reverse&mode=mini&apiKey=" + self.opts['api_key'] + url += "&" + querytype + "=" + qry + "&page=" + str(page) + + res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], + useragent="SpiderFoot") + + if res['code'] in ["401", "429", "413", "412"]: + self.error("WhoisFreaks API key seems to have been rejected or you have exceeded usage limits.") + self.errorState = True + return None + + if res['code'] in ["404", "400"]: + self.error("Incorrect paramter or record not found.") + self.errorState = True + return None + + if res['code'] in ["500", "503", "504"]: + self.error("request timed out or server error") + self.errorState = True + return None + + if res['content'] is None: + self.info("No WhoisFreaks info found for " + qry) + return None + + try: + info = json.loads(res['content']) + + if info.get("total_pages", 1) > 1: + if info.get("current_page") < info.get("total_pages"): + if accum: + accum.extend(info.get('whois_domains_historical')) + else: + accum = info.get('whois_domains_historical') + return self.query(qry, querytype, page + 1, accum) + + # We are at the last page + accum.extend(info.get('whois_domains_historical', [])) + return accum + + return info.get('whois_domains_historical', []) + except Exception as e: + self.error("Error processing JSON response from WhoisFreaks: " + str(e)) + return None + + # Handle events sent to this module + def handleEvent(self, event): + eventName = event.eventType + srcModuleName = event.module + eventData = event.data + + if self.errorState: + return + + self.debug(f"Received event, {eventName}, from {srcModuleName}") + + if self.opts['api_key'] == "": + self.error("You enabled whoisfreaks but did not set an API key!") + self.errorState = True + return + + if eventData in self.results: + self.debug(f"Skipping {eventData}, already checked.") + return + + self.results[eventData] = True + + query_type = None + if eventName in "COMPANY_NAME": + query_type = "company" + elif eventName in "HUMAN_NAME": + query_type = "owner" + elif eventName in ["EMAILADDR", "EMAILADDR_GENERIC"]: + query_type = "email" + + records = self.query(eventData, query_type) + if records is not None: + for record in records: + domain_name = record.get('domain_name') + if domain_name: + evt = SpiderFootEvent("AFFILIATE_INTERNET_NAME", domain_name, self.__name__, event) + self.notifyListeners(evt) + + if self.sf.isDomain(domain_name, self.opts['_internettlds']): + evt = SpiderFootEvent('AFFILIATE_DOMAIN_NAME', domain_name, self.__name__, event) + self.notifyListeners(evt) + + # End of sfp_whoisfreaks class \ No newline at end of file diff --git a/test/unit/modules/test_sfp_whoisfreaks.py b/test/unit/modules/test_sfp_whoisfreaks.py new file mode 100644 index 0000000000..e69de29bb2 From c278b482ea451af7d49f7352c7d60f779254cd17 Mon Sep 17 00:00:00 2001 From: FahadAnyit Date: Tue, 16 Jan 2024 18:14:13 +0500 Subject: [PATCH 2/4] added whoisfreaks moded unit test --- test/unit/modules/test_sfp_whoisfreaks.py | 27 +++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/unit/modules/test_sfp_whoisfreaks.py b/test/unit/modules/test_sfp_whoisfreaks.py index e69de29bb2..02dfc29996 100644 --- a/test/unit/modules/test_sfp_whoisfreaks.py +++ b/test/unit/modules/test_sfp_whoisfreaks.py @@ -0,0 +1,27 @@ +import unittest + +import pytest + +from modules.sfp_whoisfreaks import sfp_whoisfreaks +from sflib import SpiderFoot + + +@pytest.mark.usefixtures +class TestModuleWhoisfreaks(unittest.TestCase): + + def test_opts(self): + module = sfp_whoisfreaks() + self.assertEqual(len(module.opts), len(module.optdescs)) + + def test_setup(self): + sf = SpiderFoot(self.default_options) + module = sfp_whoisfreaks() + module.setup(sf, dict()) + + def test_watchedEvents_should_return_list(self): + module = sfp_whoisfreaks() + self.assertIsInstance(module.watchedEvents(), list) + + def test_producedEvents_should_return_list(self): + module = sfp_whoisfreaks() + self.assertIsInstance(module.producedEvents(), list) \ No newline at end of file From 01595b3a3d55566a5293afba8f220d7625231076 Mon Sep 17 00:00:00 2001 From: FahadAnyit Date: Tue, 16 Jan 2024 18:23:02 +0500 Subject: [PATCH 3/4] updated produced events of whoisfreaks module --- modules/sfp_whoisfreaks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/sfp_whoisfreaks.py b/modules/sfp_whoisfreaks.py index 951b356cdf..1792a74d09 100644 --- a/modules/sfp_whoisfreaks.py +++ b/modules/sfp_whoisfreaks.py @@ -79,6 +79,7 @@ def watchedEvents(self): # What events this module produces def producedEvents(self): return [ + 'AFFILIATE_INTERNET_NAME', 'AFFILIATE_DOMAIN_NAME' ] From d5b5900dabbf2da3aa962f46e35f050aa577b617 Mon Sep 17 00:00:00 2001 From: FahadAnyit Date: Tue, 16 Jan 2024 19:11:47 +0500 Subject: [PATCH 4/4] updated README.md file for whoisfreaks module --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 1f140137f7..9dc9356c25 100644 --- a/README.md +++ b/README.md @@ -349,6 +349,7 @@ Web Spider|Spidering of web-pages to extract content for searching.|Internal [WhatCMS](https://whatcms.org/)|Check web technology using WhatCMS.org API.|Tiered API [Whoisology](https://whoisology.com/)|Reverse Whois lookups using Whoisology.com.|Commercial API Whois|Perform a WHOIS look-up on domain names and owned netblocks.|Internal +[WhoisFreaks](https://whoisfreaks.com/)|Reverse Whois lookup by owner name, email or company name|Tiered API [Whoxy](https://www.whoxy.com/)|Reverse Whois lookups using Whoxy.com.|Commercial API [WiGLE](https://wigle.net/)|Query WiGLE to identify nearby WiFi access points.|Free API [Wikileaks](https://wikileaks.org/)|Search Wikileaks for mentions of domain names and e-mail addresses.|Free API