From a62b65f81f8c29cb50d081cf07ae24f5c9f974c5 Mon Sep 17 00:00:00 2001 From: s0i37 Date: Sun, 15 Nov 2020 19:08:36 +0500 Subject: [PATCH 1/5] added custom Headers/Cookies --- dirhunt/crawler.py | 4 ++-- dirhunt/management.py | 16 +++++++++++++--- dirhunt/sessions.py | 12 +++++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/dirhunt/crawler.py b/dirhunt/crawler.py index 218b613..e4e8e29 100644 --- a/dirhunt/crawler.py +++ b/dirhunt/crawler.py @@ -33,7 +33,7 @@ class Crawler(ThreadPoolExecutor): def __init__(self, max_workers=None, interesting_extensions=None, interesting_files=None, std=None, progress_enabled=True, timeout=10, depth=3, not_follow_subdomains=False, exclude_sources=(), - not_allow_redirects=False, proxies=None, delay=0, limit=1000, to_file=None, user_agent=None): + not_allow_redirects=False, proxies=None, delay=0, limit=1000, to_file=None, user_agent=None, cookies={}, headers={}): if not max_workers and not delay: max_workers = (multiprocessing.cpu_count() or 1) * 5 elif not max_workers and delay: @@ -44,7 +44,7 @@ def __init__(self, max_workers=None, interesting_extensions=None, interesting_fi self.index_of_processors = [] self.proxies = proxies self.delay = delay - self.sessions = Sessions(proxies, delay, user_agent) + self.sessions = Sessions(proxies, delay, user_agent, cookies, headers) self.processing = {} self.processed = {} self.add_lock = Lock() diff --git a/dirhunt/management.py b/dirhunt/management.py index ed2d9cf..d55f87e 100644 --- a/dirhunt/management.py +++ b/dirhunt/management.py @@ -56,6 +56,12 @@ def comma_separated_files(ctx, param, value): items.append(value) return items +def key_value(ctx, param, values): + items = {} + for value in values: + key,val = map(lambda x:x.strip(), value.split(':')) + items[key] = val + return items def status_code_range(start, end): return list(filter(lambda x: start <= x <= end, STATUS_CODES)) @@ -126,12 +132,16 @@ def flags_range(flags): @click.option('--not-allow-redirects', is_flag=True, help='Redirectors will not be followed') @click.option('--limit', type=int, default=1000, help='Max number of pages processed to search for directories.') @click.option('--to-file', type=Path(writable=True), default=None, help='Create a report file in JSON.') -@click.option('--user-agent', type=str, default=None, help='User agent to use. By default a random browser.') +@click.option('-U', '--user-agent', type=str, default=None, help='User agent to use. By default a random browser.') +@click.option('-C', '--cookies', callback=key_value, multiple=True, + help='Custom cookies') +@click.option('-H', '--headers', callback=key_value, multiple=True, + help='Custom headers') @click.option('--version', is_flag=True, callback=print_version, expose_value=False, is_eager=True) def hunt(urls, threads, exclude_flags, include_flags, interesting_extensions, interesting_files, stdout_flags, progress_enabled, timeout, max_depth, not_follow_subdomains, exclude_sources, proxies, delay, - not_allow_redirects, limit, to_file, user_agent): + not_allow_redirects, limit, to_file, user_agent, cookies, headers): """Find web directories without bruteforce """ if exclude_flags and include_flags: @@ -150,7 +160,7 @@ def hunt(urls, threads, exclude_flags, include_flags, interesting_extensions, in progress_enabled=progress_enabled, timeout=timeout, depth=max_depth, not_follow_subdomains=not_follow_subdomains, exclude_sources=exclude_sources, not_allow_redirects=not_allow_redirects, proxies=proxies, delay=delay, limit=limit, - to_file=to_file, user_agent=user_agent) + to_file=to_file, user_agent=user_agent, cookies=cookies, headers=headers) if os.path.exists(crawler.get_resume_file()): click.echo('Resuming the previous program execution...') try: diff --git a/dirhunt/sessions.py b/dirhunt/sessions.py index bf1ea7d..e8ada01 100644 --- a/dirhunt/sessions.py +++ b/dirhunt/sessions.py @@ -78,7 +78,7 @@ def __getitem__(self, item): class Session(object): - def __init__(self, sessions, proxy, user_agent=None): + def __init__(self, sessions, proxy, user_agent=None, cookies={}, headers={}): self.sessions = sessions self.proxy_name = proxy self.proxy = normalize_proxy(self.proxy_name, sessions) @@ -86,6 +86,10 @@ def __init__(self, sessions, proxy, user_agent=None): self.session.headers = { 'User-Agent': user_agent or get_random_user_agent(), } + if cookies: + self.session.cookies.update(cookies) + if headers: + self.session.headers.update(headers) adapter = HTTPAdapter(pool_connections=POOL_CONNECTIONS, pool_maxsize=POOL_CONNECTIONS) self.session.mount('http://', adapter) self.session.mount('https://', adapter) @@ -124,11 +128,13 @@ def get(self, url, **kwargs): class Sessions(object): - def __init__(self, proxies=None, delay=0, user_agent=None): + def __init__(self, proxies=None, delay=0, user_agent=None, cookies={}, headers={}): self.availables = Queue() self.proxies_lists = RandomProxies() self.delay = delay self.user_agent = user_agent + self.cookies = cookies + self.headers = headers self.sessions = self.create_sessions(proxies or [None]) for session in self.sessions: self.availables.put(session) @@ -140,7 +146,7 @@ def add_available(self, session): self.availables.put(session) def create_sessions(self, proxies): - return [Session(self, proxy, self.user_agent) for proxy in proxies] + return [Session(self, proxy, self.user_agent, self.cookies, self.headers) for proxy in proxies] def get_random_session(self): return random.choice(self.sessions) From 0357813f71ad5207a713df100f257e41e5e24909 Mon Sep 17 00:00:00 2001 From: s0i37 Date: Mon, 16 Nov 2020 13:12:42 +0500 Subject: [PATCH 2/5] fixed error with mutable arguments --- dirhunt/crawler.py | 2 +- dirhunt/sessions.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dirhunt/crawler.py b/dirhunt/crawler.py index e4e8e29..55b27ac 100644 --- a/dirhunt/crawler.py +++ b/dirhunt/crawler.py @@ -33,7 +33,7 @@ class Crawler(ThreadPoolExecutor): def __init__(self, max_workers=None, interesting_extensions=None, interesting_files=None, std=None, progress_enabled=True, timeout=10, depth=3, not_follow_subdomains=False, exclude_sources=(), - not_allow_redirects=False, proxies=None, delay=0, limit=1000, to_file=None, user_agent=None, cookies={}, headers={}): + not_allow_redirects=False, proxies=None, delay=0, limit=1000, to_file=None, user_agent=None, cookies=None, headers=None): if not max_workers and not delay: max_workers = (multiprocessing.cpu_count() or 1) * 5 elif not max_workers and delay: diff --git a/dirhunt/sessions.py b/dirhunt/sessions.py index e8ada01..f41e48c 100644 --- a/dirhunt/sessions.py +++ b/dirhunt/sessions.py @@ -78,7 +78,7 @@ def __getitem__(self, item): class Session(object): - def __init__(self, sessions, proxy, user_agent=None, cookies={}, headers={}): + def __init__(self, sessions, proxy, user_agent=None, cookies=None, headers=None): self.sessions = sessions self.proxy_name = proxy self.proxy = normalize_proxy(self.proxy_name, sessions) @@ -128,7 +128,7 @@ def get(self, url, **kwargs): class Sessions(object): - def __init__(self, proxies=None, delay=0, user_agent=None, cookies={}, headers={}): + def __init__(self, proxies=None, delay=0, user_agent=None, cookies=None, headers=None): self.availables = Queue() self.proxies_lists = RandomProxies() self.delay = delay From 708f1a0b35c6d2bb7ab1d7298a72e86b4a05c105 Mon Sep 17 00:00:00 2001 From: Nekmo Date: Wed, 18 Nov 2020 20:54:02 +0100 Subject: [PATCH 3/5] Pull request #85: added custom Headers/Cookies (ensure that cookies/headers is defined). --- dirhunt/crawler.py | 3 ++- dirhunt/sessions.py | 10 ++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dirhunt/crawler.py b/dirhunt/crawler.py index 55b27ac..a76210e 100644 --- a/dirhunt/crawler.py +++ b/dirhunt/crawler.py @@ -33,7 +33,8 @@ class Crawler(ThreadPoolExecutor): def __init__(self, max_workers=None, interesting_extensions=None, interesting_files=None, std=None, progress_enabled=True, timeout=10, depth=3, not_follow_subdomains=False, exclude_sources=(), - not_allow_redirects=False, proxies=None, delay=0, limit=1000, to_file=None, user_agent=None, cookies=None, headers=None): + not_allow_redirects=False, proxies=None, delay=0, limit=1000, to_file=None, user_agent=None, + cookies=None, headers=None): if not max_workers and not delay: max_workers = (multiprocessing.cpu_count() or 1) * 5 elif not max_workers and delay: diff --git a/dirhunt/sessions.py b/dirhunt/sessions.py index f41e48c..dd01a86 100644 --- a/dirhunt/sessions.py +++ b/dirhunt/sessions.py @@ -86,10 +86,8 @@ def __init__(self, sessions, proxy, user_agent=None, cookies=None, headers=None) self.session.headers = { 'User-Agent': user_agent or get_random_user_agent(), } - if cookies: - self.session.cookies.update(cookies) - if headers: - self.session.headers.update(headers) + self.session.cookies.update(cookies or {}) + self.session.headers.update(headers or {}) adapter = HTTPAdapter(pool_connections=POOL_CONNECTIONS, pool_maxsize=POOL_CONNECTIONS) self.session.mount('http://', adapter) self.session.mount('https://', adapter) @@ -133,8 +131,8 @@ def __init__(self, proxies=None, delay=0, user_agent=None, cookies=None, headers self.proxies_lists = RandomProxies() self.delay = delay self.user_agent = user_agent - self.cookies = cookies - self.headers = headers + self.cookies = cookies or {} + self.headers = headers or {} self.sessions = self.create_sessions(proxies or [None]) for session in self.sessions: self.availables.put(session) From 2d4157eda3dcce57cdd259528608a81ea9d60e41 Mon Sep 17 00:00:00 2001 From: Nekmo Date: Wed, 18 Nov 2020 21:18:24 +0100 Subject: [PATCH 4/5] Pull request #85: added custom Headers/Cookies (throw an error if the format for key:value is invalid). --- dirhunt/management.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/dirhunt/management.py b/dirhunt/management.py index d55f87e..3fdd41b 100644 --- a/dirhunt/management.py +++ b/dirhunt/management.py @@ -8,7 +8,7 @@ import sys -from click import BadOptionUsage, Path +from click import BadOptionUsage, Path, BadParameter from dirhunt.crawler import Crawler from dirhunt.exceptions import DirHuntError, catch, IncompatibleVersionError @@ -20,8 +20,8 @@ init(autoreset=True) -STATUS_CODES = lrange(100, 102+1) + lrange(200, 208+1) + [226] + lrange(300, 308+1) + lrange(400, 418+1) + \ - lrange(421, 426+1) + [428, 429, 431, 451] + lrange(500, 511+1) +STATUS_CODES = lrange(100, 102 + 1) + lrange(200, 208 + 1) + [226] + lrange(300, 308 + 1) + lrange(400, 418 + 1) + \ + lrange(421, 426 + 1) + [428, 429, 431, 451] + lrange(500, 511 + 1) INTERESTING_EXTS = ['php', 'zip', 'sh', 'asp', 'csv', 'log'] INTERESTING_FILES = ['access_log', 'error_log', 'error', 'logs', 'dump'] STDOUT_FLAGS = ['blank', 'not_found.fake', 'html'] @@ -56,12 +56,13 @@ def comma_separated_files(ctx, param, value): items.append(value) return items + def key_value(ctx, param, values): - items = {} - for value in values: - key,val = map(lambda x:x.strip(), value.split(':')) - items[key] = val - return items + items = [item.split(':', 1) for item in values] + if any(filter(lambda x: len(x) < 2, items)): + raise BadParameter('Expect a value with format key:bar', ctx, param) + return {x[0].strip(): x[1].strip() for x in items} + def status_code_range(start, end): return list(filter(lambda x: start <= x <= end, STATUS_CODES)) @@ -132,10 +133,10 @@ def flags_range(flags): @click.option('--not-allow-redirects', is_flag=True, help='Redirectors will not be followed') @click.option('--limit', type=int, default=1000, help='Max number of pages processed to search for directories.') @click.option('--to-file', type=Path(writable=True), default=None, help='Create a report file in JSON.') -@click.option('-U', '--user-agent', type=str, default=None, help='User agent to use. By default a random browser.') -@click.option('-C', '--cookies', callback=key_value, multiple=True, +@click.option('-u', '--user-agent', type=str, default=None, help='User agent to use. By default a random browser.') +@click.option('-c', '--cookies', callback=key_value, multiple=True, help='Custom cookies') -@click.option('-H', '--headers', callback=key_value, multiple=True, +@click.option('-h', '--headers', callback=key_value, multiple=True, help='Custom headers') @click.option('--version', is_flag=True, callback=print_version, expose_value=False, is_eager=True) @@ -169,7 +170,7 @@ def hunt(urls, threads, exclude_flags, include_flags, interesting_extensions, in click.echo(e) crawler.add_init_urls(*urls) while True: - choice = catch_keyboard_interrupt_choices(crawler.print_results, ['abort', 'continue', 'results'], 'a')\ + choice = catch_keyboard_interrupt_choices(crawler.print_results, ['abort', 'continue', 'results'], 'a') \ (set(exclude_flags), set(include_flags)) if choice == 'a': crawler.close(True) From 7e9618e0ac36567d106bd1ae180147dca3063f67 Mon Sep 17 00:00:00 2001 From: Nekmo Date: Wed, 18 Nov 2020 21:34:17 +0100 Subject: [PATCH 5/5] Pull request #85: added custom Headers/Cookies (docs). --- dirhunt/management.py | 8 ++++---- docs/usage.rst | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/dirhunt/management.py b/dirhunt/management.py index 3fdd41b..3ba5911 100644 --- a/dirhunt/management.py +++ b/dirhunt/management.py @@ -134,10 +134,10 @@ def flags_range(flags): @click.option('--limit', type=int, default=1000, help='Max number of pages processed to search for directories.') @click.option('--to-file', type=Path(writable=True), default=None, help='Create a report file in JSON.') @click.option('-u', '--user-agent', type=str, default=None, help='User agent to use. By default a random browser.') -@click.option('-c', '--cookies', callback=key_value, multiple=True, - help='Custom cookies') -@click.option('-h', '--headers', callback=key_value, multiple=True, - help='Custom headers') +@click.option('-c', '--cookie', 'cookies', callback=key_value, multiple=True, + help='Add a cookie to requests in the cookie_name:value format.') +@click.option('-h', '--header', 'headers', callback=key_value, multiple=True, + help='Add a header to requests in the header:value format.') @click.option('--version', is_flag=True, callback=print_version, expose_value=False, is_eager=True) def hunt(urls, threads, exclude_flags, include_flags, interesting_extensions, interesting_files, stdout_flags, diff --git a/docs/usage.rst b/docs/usage.rst index 3b0ab31..0d67831 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -434,6 +434,31 @@ Example for **interesting files** (``-f``):: It is necessary to put the complete path to the file, or the relative using ``./``. Each value of the files must be separated by newlines. +Custom headers +-------------- +To add custom HTTP headers to requests you can use the ``--header`` parameter. + +.. code:: + + $ dirhunt --header : + +This parameter can be used more than once, for example:: + + $ dirhunt http://domain1/blog/ --header "Authorization: token foo" --header "X-Server: prod" + + +Custom cookies +-------------- +To add custom cookies to requests you can use the ``--cookie`` parameter. + +.. code:: + + $ dirhunt --cookie : + +This parameter can be used more than once, for example:: + + $ dirhunt http://domain1/blog/ --cookie "session: secret" --cookie "user: 123" + Progress bar ------------