From 2bfe16c0de16a2fdf43fd710ec3c8363d81e5442 Mon Sep 17 00:00:00 2001 From: Dhrumil Mistry <56185972+dmdhrumilmistry@users.noreply.github.com> Date: Wed, 2 Oct 2024 19:41:30 +0530 Subject: [PATCH 1/6] add ansible playbook for native-setup --- setup/ansible/README.md | 44 +++++ setup/ansible/native-setup.yml | 6 + setup/ansible/roles/native-setup/README.md | 41 ++++ .../roles/native-setup/defaults/main.yml | 2 + .../roles/native-setup/handlers/main.yml | 2 + .../ansible/roles/native-setup/meta/main.yml | 34 ++++ .../native-setup/tasks/install-tools.yml | 186 ++++++++++++++++++ .../ansible/roles/native-setup/tasks/main.yml | 65 ++++++ .../roles/native-setup/tests/inventory | 2 + .../ansible/roles/native-setup/tests/test.yml | 5 + .../ansible/roles/native-setup/vars/main.yml | 2 + 11 files changed, 389 insertions(+) create mode 100644 setup/ansible/README.md create mode 100644 setup/ansible/native-setup.yml create mode 100644 setup/ansible/roles/native-setup/README.md create mode 100644 setup/ansible/roles/native-setup/defaults/main.yml create mode 100644 setup/ansible/roles/native-setup/handlers/main.yml create mode 100644 setup/ansible/roles/native-setup/meta/main.yml create mode 100644 setup/ansible/roles/native-setup/tasks/install-tools.yml create mode 100644 setup/ansible/roles/native-setup/tasks/main.yml create mode 100644 setup/ansible/roles/native-setup/tests/inventory create mode 100644 setup/ansible/roles/native-setup/tests/test.yml create mode 100644 setup/ansible/roles/native-setup/vars/main.yml diff --git a/setup/ansible/README.md b/setup/ansible/README.md new file mode 100644 index 0000000..8b34907 --- /dev/null +++ b/setup/ansible/README.md @@ -0,0 +1,44 @@ +# Ansible Setup + +Configures mantis dependencies on ubuntu x86_64 arch + +## Configuration + +### Ansible hosts/nodes + +* They must have python3 installed + + ```bash + sudo apt install python3 -y + ``` + +### Ansible Controller + +* Python deps + + ```bash + python3 -m pip install ansible ansible-core + ``` + +* Generate host file `hosts` + + ```txt + [mantisnodes] + mantis-node-1 ansible_host=192.168.0.105 ansible_user=root + mantis-node-2 ansible_host=192.168.0.107 ansible_user=root + mantis-node-3 ansible_host=192.168.0.109 ansible_user=root + ``` + + > Please ensure that ansible controller is able to ssh into the `mantisnodes` as `root` user using its private key + +* Test connectivity with hosts + + ```bash + ansible all -m ping + ``` + +* Run playbook + + ```bash + ansible-playbook -i hosts native-setup.yml + ``` diff --git a/setup/ansible/native-setup.yml b/setup/ansible/native-setup.yml new file mode 100644 index 0000000..f9185b0 --- /dev/null +++ b/setup/ansible/native-setup.yml @@ -0,0 +1,6 @@ +--- + +- hosts: mantisnodes + become: true + roles: + - { role : "native-setup" } diff --git a/setup/ansible/roles/native-setup/README.md b/setup/ansible/roles/native-setup/README.md new file mode 100644 index 0000000..567e30d --- /dev/null +++ b/setup/ansible/roles/native-setup/README.md @@ -0,0 +1,41 @@ +native-setup +========= + +Role `native-setup` configures + +Requirements +------------ + +* ubuntu machine (x86_64 arch) +* python3 + + + +Example Playbook +---------------- + +Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: + + ```yaml + - hosts: mantisnodes + roles: + - { role: native-setup } + ``` + +License +------- + +[Apache 2.0](/LICENSE) + +Author Information +------------------ + +* [dmdhrumilmistry](https://github.com/dmdhrumilmistry) diff --git a/setup/ansible/roles/native-setup/defaults/main.yml b/setup/ansible/roles/native-setup/defaults/main.yml new file mode 100644 index 0000000..a9c7d90 --- /dev/null +++ b/setup/ansible/roles/native-setup/defaults/main.yml @@ -0,0 +1,2 @@ +--- +# defaults file for native-setup diff --git a/setup/ansible/roles/native-setup/handlers/main.yml b/setup/ansible/roles/native-setup/handlers/main.yml new file mode 100644 index 0000000..e8a5b02 --- /dev/null +++ b/setup/ansible/roles/native-setup/handlers/main.yml @@ -0,0 +1,2 @@ +--- +# handlers file for native-setup diff --git a/setup/ansible/roles/native-setup/meta/main.yml b/setup/ansible/roles/native-setup/meta/main.yml new file mode 100644 index 0000000..6999a76 --- /dev/null +++ b/setup/ansible/roles/native-setup/meta/main.yml @@ -0,0 +1,34 @@ +galaxy_info: + author: dmdhrumilmistry + description: Configures requirements for running mantis on ubuntu machines + # company: your company (optional) + + # If the issue tracker for your role is not on github, uncomment the + # next line and provide a value + # issue_tracker_url: http://example.com/issue/tracker + + # Choose a valid license ID from https://spdx.org - some suggested licenses: + # - BSD-3-Clause (default) + # - MIT + # - GPL-2.0-or-later + # - GPL-3.0-only + # - Apache-2.0 + # - CC-BY-4.0 + license: Apache-2.0 + + min_ansible_version: "2.1" + + # If this a Container Enabled role, provide the minimum Ansible Container version. + # min_ansible_container_version: + + galaxy_tags: [] + # List tags for your role here, one per line. A tag is a keyword that describes + # and categorizes the role. Users find roles by searching for tags. Be sure to + # remove the '[]' above, if you add tags to this list. + # + # NOTE: A tag is limited to a single word comprised of alphanumeric characters. + # Maximum 20 tags per role. + +dependencies: [] + # List your role dependencies here, one per line. Be sure to remove the '[]' above, + # if you add dependencies to this list. diff --git a/setup/ansible/roles/native-setup/tasks/install-tools.yml b/setup/ansible/roles/native-setup/tasks/install-tools.yml new file mode 100644 index 0000000..a6c412b --- /dev/null +++ b/setup/ansible/roles/native-setup/tasks/install-tools.yml @@ -0,0 +1,186 @@ +--- +- name: Install Amass + block: + - name: Download Amass + get_url: + url: https://github.com/owasp-amass/amass/releases/download/v4.1.0/amass_Linux_amd64.zip + dest: /tmp/amass.zip + + - name: Unzip Amass + unarchive: + src: /tmp/amass.zip + dest: /tmp/ + remote_src: yes + extra_opts: ["-j"] + + - name: Move Amass to /usr/bin + command: mv /tmp/amass /usr/bin/amass + + - name: Ensure /usr/bin/amass is executable + file: + path: /usr/bin/amass + mode: "0755" + state: file + +- name: Install Subfinder + block: + - name: Download Subfinder + get_url: + url: https://github.com/projectdiscovery/subfinder/releases/download/v2.6.3/subfinder_2.6.3_linux_amd64.zip + dest: /tmp/subfinder.zip + + - name: Unzip Subfinder + unarchive: + src: /tmp/subfinder.zip + dest: /tmp/ + remote_src: yes + extra_opts: ["-j"] + + - name: Move Subfinder to /usr/bin + command: mv /tmp/subfinder /usr/bin/subfinder + + - name: Ensure /usr/bin/subfinder is executable + file: + path: /usr/bin/subfinder + mode: "0755" + state: file + +- name: Install Httpx + block: + - name: Download Httpx + get_url: + url: https://github.com/projectdiscovery/httpx/releases/download/v1.3.7/httpx_1.3.7_linux_amd64.zip + dest: /tmp/httpx.zip + + - name: Unzip Httpx + unarchive: + src: /tmp/httpx.zip + dest: /tmp/ + remote_src: yes + extra_opts: ["-j"] + + - name: Move Httpx to /usr/bin + command: mv /tmp/httpx /usr/bin/httpx + + - name: Ensure /usr/bin/httpx is executable + file: + path: /usr/bin/httpx + mode: "0755" + state: file + +- name: Install Naabu + block: + - name: Download Naabu + get_url: + url: https://github.com/projectdiscovery/naabu/releases/download/v2.1.9/naabu_2.1.9_linux_amd64.zip + dest: /tmp/naabu.zip + + - name: Unzip Naabu + unarchive: + src: /tmp/naabu.zip + dest: /tmp/ + remote_src: yes + extra_opts: ["-j"] + + - name: Move Naabu to /usr/bin + command: mv /tmp/naabu /usr/bin/naabu + + - name: Ensure /usr/bin/naabu is executable + file: + path: /usr/bin/naabu + mode: "0755" + state: file + +- name: Install Nuclei + block: + - name: Download Nuclei + get_url: + url: https://github.com/projectdiscovery/nuclei/releases/download/v3.0.4/nuclei_3.0.4_linux_amd64.zip + dest: /tmp/nuclei.zip + + - name: Unzip Nuclei + unarchive: + src: /tmp/nuclei.zip + dest: /tmp/ + remote_src: yes + extra_opts: ["-j"] + + - name: Move Nuclei to /usr/bin + command: mv /tmp/nuclei /usr/bin/nuclei + + - name: Ensure /usr/bin/nuclei is executable + file: + path: /usr/bin/nuclei + mode: "0755" + state: file + +- name: Install Gitleaks + block: + - name: Download Gitleaks + get_url: + url: https://github.com/gitleaks/gitleaks/releases/download/v8.18.1/gitleaks_8.18.1_linux_x64.tar.gz + dest: /tmp/gitleaks.tar.gz + + - name: Extract Gitleaks + unarchive: + src: /tmp/gitleaks.tar.gz + dest: /tmp/ + remote_src: yes + + - name: Move Gitleaks to /usr/bin + command: mv /tmp/gitleaks /usr/bin/gitleaks + + - name: Ensure /usr/bin/gitleaks is executable + file: + path: /usr/bin/gitleaks + mode: "0755" + state: file + +- name: Install gau + block: + - name: Download gau tarball + get_url: + url: https://github.com/lc/gau/releases/download/v2.2.1/gau_2.2.1_linux_amd64.tar.gz + dest: /tmp/gau.tar.gz + + - name: Extract gau tarball + unarchive: + src: /tmp/gau.tar.gz + dest: /tmp/ + remote_src: yes + + - name: Move gau to /usr/bin + command: mv /tmp/gau /usr/bin/gau + + - name: Ensure /usr/bin/gau is executable + file: + path: /usr/bin/gau + mode: "0755" + state: file + +- name: Install Corsy + block: + - name: Download Corsy zip + get_url: + url: https://github.com/s0md3v/Corsy/archive/refs/tags/1.0-rc.zip + dest: /tmp/1.0-rc.zip + + - name: Unzip Corsy + unarchive: + src: /tmp/1.0-rc.zip + dest: /tmp + remote_src: yes + + - name: Move Corsy to /usr/bin + command: mv /tmp/Corsy-1.0-rc /usr/bin/Corsy + +- name: Install IPinfo + block: + - name: Download IPinfo deb package + get_url: + url: https://github.com/ipinfo/cli/releases/download/ipinfo-3.2.0/ipinfo_3.2.0.deb + dest: /tmp/ipinfo_3.2.0.deb + + - name: Install IPinfo deb package + apt: + deb: /tmp/ipinfo_3.2.0.deb \ No newline at end of file diff --git a/setup/ansible/roles/native-setup/tasks/main.yml b/setup/ansible/roles/native-setup/tasks/main.yml new file mode 100644 index 0000000..4706925 --- /dev/null +++ b/setup/ansible/roles/native-setup/tasks/main.yml @@ -0,0 +1,65 @@ +--- +# tasks file for native-setup +- name: Install aptitude + apt: + name: aptitude + state: latest + update_cache: true + +- name: Add deadsnakes PPA for Python 3.9 + apt_repository: + repo: ppa:deadsnakes/ppa + +- name: Install required system packages + apt: + pkg: + - apt-transport-https + - ca-certificates + - curl + - gcc + - git + - gnupg + - libpcap-dev + - dnsutils + - python3.9 + - python3.9-dev + - python3-pip + - python3-setuptools + - python3-requests + - unzip + - software-properties-common + - wget + state: latest + update_cache: true + +- name: Add ~/.local/bin to PATH in ~/.bashrc + lineinfile: + path: ~/.bashrc + line: "export PATH='$HOME/.local/bin:$PATH'" + create: yes + state: present + +- name: Reload ~/.bashrc + shell: source ~/.bashrc + args: + executable: /bin/bash + +- include_tasks: install-tools.yml + +- name: Install pip dependencies + pip: + name: + - "dnspython<=2.0.0" + - "dnstwist[full]" + - ipwhois + - wafw00f + - "git+https://github.com/cisagov/findcdn.git" + +- name: Download requirements + get_url: + url: https://raw.githubusercontent.com/PhonePe/mantis/refs/heads/main/requirements.txt + dest: /tmp/mantis-requirements.txt + +- name: Install Python dependencies + pip: + requirements: /tmp/mantis-requirements.txt diff --git a/setup/ansible/roles/native-setup/tests/inventory b/setup/ansible/roles/native-setup/tests/inventory new file mode 100644 index 0000000..878877b --- /dev/null +++ b/setup/ansible/roles/native-setup/tests/inventory @@ -0,0 +1,2 @@ +localhost + diff --git a/setup/ansible/roles/native-setup/tests/test.yml b/setup/ansible/roles/native-setup/tests/test.yml new file mode 100644 index 0000000..ed20304 --- /dev/null +++ b/setup/ansible/roles/native-setup/tests/test.yml @@ -0,0 +1,5 @@ +--- +- hosts: localhost + remote_user: root + roles: + - native-setup diff --git a/setup/ansible/roles/native-setup/vars/main.yml b/setup/ansible/roles/native-setup/vars/main.yml new file mode 100644 index 0000000..314654d --- /dev/null +++ b/setup/ansible/roles/native-setup/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for native-setup From ddf8252116991dbc38452ea5f64c994c2110d375 Mon Sep 17 00:00:00 2001 From: Dhrumil Mistry <56185972+dmdhrumilmistry@users.noreply.github.com> Date: Sat, 5 Oct 2024 17:45:17 +0530 Subject: [PATCH 2/6] add listing feature with filters --- .gitignore | 3 +- mantis/models/args_model.py | 3 ++ mantis/utils/args_parse.py | 29 +++++++++++++++-- mantis/utils/list_subcommand_utils.py | 47 +++++++++++++++++++++------ mantis/workflows/list_workflow.py | 23 +++++++++---- 5 files changed, 84 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index 1d82482..04542d9 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ logs/ book src stacks -docker/configs \ No newline at end of file +docker/configs +.venv \ No newline at end of file diff --git a/mantis/models/args_model.py b/mantis/models/args_model.py index a843f94..392e369 100644 --- a/mantis/models/args_model.py +++ b/mantis/models/args_model.py @@ -21,4 +21,7 @@ class ArgsModel(BaseModel): subdomain: str = Field(None) list_: bool = False list_orgs: bool = False + list_domains: bool = False + orgs_list: list[str] = False + asset_types_list: list[str] = False \ No newline at end of file diff --git a/mantis/utils/args_parse.py b/mantis/utils/args_parse.py index 43916c2..fc4821b 100644 --- a/mantis/utils/args_parse.py +++ b/mantis/utils/args_parse.py @@ -233,9 +233,17 @@ def args_parse() -> ArgsModel: list_parser = subparser.add_parser("list", help="List entities present in db", usage=ArgsParse.list_msg()) - list_sub_parser = list_parser.add_subparsers(title="List Subcommands", dest="list_sub_command") + list_parser.add_argument("-l","--list-orgs", help="list all orgs from database", dest="list_sub_command_ls_orgs", action="store_true") + + list_parser.add_argument("-d","--list-domains", help="list domains (tlds/subdomains) for selected orgs", dest="list_sub_command_ls_domains", action="store_true") + list_parser.add_argument("-o","--org", help="select org by name", dest="list_sub_command_orgs_list", type=list[str], action="append") + list_parser.add_argument("-t", "--tlds", help="list tlds for selected orgs", action="store_true", dest="list_sub_command_ls_subs_tlds") + list_parser.add_argument("-s", "--subs", help="list subdomains for selected orgs", action="store_true", dest="list_sub_command_ls_subs_domains") + - list_sub_parser.add_parser("orgs", help="List orgs present in DB") + # list_sub_parser = list_parser.add_subparsers(title="List Subcommands", dest="list_sub_command") + # list_org_sub_parser = list_sub_parser.add_parser("orgs", help="List orgs present in DB") + # list_org_sub_parser.add_argument() # display help, if no arguments are passed args = parser.parse_args(args=None if argv[1:] else ['--help']) @@ -302,12 +310,27 @@ def args_parse() -> ArgsModel: if args.subdomain: parsed_args["subdomain"] = args.host + # TODO: update command if args.subcommand == "list": parsed_args["list_"] = True - if args.list_sub_command == "orgs": + # python launch.py list -l + if args.list_sub_command_ls_orgs: parsed_args["list_orgs"] = True + # python launch.py list -d -s -t -o + if args.list_sub_command_ls_domains: + asset_types = [] + if args.list_sub_command_ls_subs_tlds: + asset_types.append("TLD") + if args.list_sub_command_ls_subs_domains: + asset_types.append("subdomain") + + parsed_args["asset_types_list"] = asset_types + parsed_args["orgs_list"] = [ ''.join(org) for org in args.list_sub_command_orgs_list] + parsed_args["list_domains"] = True + + args_pydantic_obj = ArgsModel.parse_obj(parsed_args) logging.info(f'parsed args - {args_pydantic_obj}') logging.info(f"Parsing Arguements - Completed") diff --git a/mantis/utils/list_subcommand_utils.py b/mantis/utils/list_subcommand_utils.py index 243e953..369ffcd 100644 --- a/mantis/utils/list_subcommand_utils.py +++ b/mantis/utils/list_subcommand_utils.py @@ -1,21 +1,48 @@ +import logging from mantis.db.crud_assets import read_assets + async def get_orgs(): pipeline = [] - pipeline.extend([ - {"$group": { - "_id" : None, - "orgs":{ - "$addToSet": "$org" - } - } - } - ]) + pipeline.extend([{"$group": {"_id": None, "orgs": {"$addToSet": "$org"}}}]) orgs = await read_assets(pipeline) if orgs: return orgs[0]["orgs"] else: - return None \ No newline at end of file + return None + + +async def get_domains(orgs:list[str], asset_types:list[str]): + if len(orgs) == 0: + logging.warning('No orgs selected') + return [] + + if len(asset_types) == 0: + logging.warning('no asset type was selected') + return [] + + pipeline = [ + { + "$match": { + "org": {"$in": orgs}, + "asset_type": {"$in": asset_types}, + } + }, + { + "$group": { + "_id": None, + "asset": {"$addToSet": "$asset"}, + }, + }, + ] + + result = await read_assets(pipeline) + domains = [] + if result and len(result) > 0: + domains = result[0].get('asset', []) + else: + logging.warning('No match found') + return domains diff --git a/mantis/workflows/list_workflow.py b/mantis/workflows/list_workflow.py index df0c157..6ec305f 100644 --- a/mantis/workflows/list_workflow.py +++ b/mantis/workflows/list_workflow.py @@ -1,17 +1,26 @@ import logging from mantis.models.args_model import ArgsModel -from mantis.utils.list_subcommand_utils import get_orgs +from mantis.utils.list_subcommand_utils import get_orgs, get_domains + class ListWorkflow: @staticmethod async def executor(args: ArgsModel): - + if args.list_orgs: logging.info("Getting orgs from database") orgs = await get_orgs() - print() - print(f'''Total Number of orgs onboarded: {len(orgs)}''') - print() - for org in orgs: - print(org) \ No newline at end of file + if orgs: + print(f"\nTotal Number of orgs onboarded: {len(orgs)}\n") + print("\n".join(orgs)) + else: + logging.info("No orgs found in database") + + if args.list_domains: + logging.info("Getting subdomains from database") + domains = await get_domains(args.orgs_list, args.asset_types_list) + if domains: + print(f"\nOrgs Filter: {','.join(args.orgs_list)}") + print(f"Total Domains Found based on provided filters: {len(domains)}\n") + print('\n'.join(domains)) From 80c3e225cb92f69f659c965e7ed2ac3d8b9245ad Mon Sep 17 00:00:00 2001 From: Dhrumil Mistry <56185972+dmdhrumilmistry@users.noreply.github.com> Date: Sat, 5 Oct 2024 18:28:25 +0530 Subject: [PATCH 3/6] implement after and before filter --- mantis/models/args_model.py | 3 +++ mantis/utils/args_parse.py | 11 ++++++++--- mantis/utils/list_subcommand_utils.py | 20 +++++++++++++++----- mantis/workflows/list_workflow.py | 4 +++- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/mantis/models/args_model.py b/mantis/models/args_model.py index 392e369..6bed852 100644 --- a/mantis/models/args_model.py +++ b/mantis/models/args_model.py @@ -24,4 +24,7 @@ class ArgsModel(BaseModel): list_domains: bool = False orgs_list: list[str] = False asset_types_list: list[str] = False + after_datetime_filter: str = None + before_datetime_filter: str = None + \ No newline at end of file diff --git a/mantis/utils/args_parse.py b/mantis/utils/args_parse.py index fc4821b..1fe2131 100644 --- a/mantis/utils/args_parse.py +++ b/mantis/utils/args_parse.py @@ -239,7 +239,8 @@ def args_parse() -> ArgsModel: list_parser.add_argument("-o","--org", help="select org by name", dest="list_sub_command_orgs_list", type=list[str], action="append") list_parser.add_argument("-t", "--tlds", help="list tlds for selected orgs", action="store_true", dest="list_sub_command_ls_subs_tlds") list_parser.add_argument("-s", "--subs", help="list subdomains for selected orgs", action="store_true", dest="list_sub_command_ls_subs_domains") - + list_parser.add_argument("-a","--after", type=str, help="Start date in YYYY-MM-DD format", dest="list_sub_command_ls_subs_after_filter") + list_parser.add_argument("-b","--before", type=str, help="End date in YYYY-MM-DD format", dest="list_sub_command_ls_subs_before_filter") # list_sub_parser = list_parser.add_subparsers(title="List Subcommands", dest="list_sub_command") # list_org_sub_parser = list_sub_parser.add_parser("orgs", help="List orgs present in DB") @@ -310,7 +311,6 @@ def args_parse() -> ArgsModel: if args.subdomain: parsed_args["subdomain"] = args.host - # TODO: update command if args.subcommand == "list": parsed_args["list_"] = True @@ -318,7 +318,7 @@ def args_parse() -> ArgsModel: if args.list_sub_command_ls_orgs: parsed_args["list_orgs"] = True - # python launch.py list -d -s -t -o + # python launch.py list -d -s -t -o -a 2024-10-04 -b 2024-10-05 if args.list_sub_command_ls_domains: asset_types = [] if args.list_sub_command_ls_subs_tlds: @@ -330,6 +330,11 @@ def args_parse() -> ArgsModel: parsed_args["orgs_list"] = [ ''.join(org) for org in args.list_sub_command_orgs_list] parsed_args["list_domains"] = True + if args.list_sub_command_ls_subs_after_filter: + parsed_args["after_datetime_filter"] = f"{args.list_sub_command_ls_subs_after_filter}T00:00:00Z" + + if args.list_sub_command_ls_subs_before_filter: + parsed_args["before_datetime_filter"] = f"{args.list_sub_command_ls_subs_before_filter}T23:59:59Z" args_pydantic_obj = ArgsModel.parse_obj(parsed_args) logging.info(f'parsed args - {args_pydantic_obj}') diff --git a/mantis/utils/list_subcommand_utils.py b/mantis/utils/list_subcommand_utils.py index 369ffcd..e0790cb 100644 --- a/mantis/utils/list_subcommand_utils.py +++ b/mantis/utils/list_subcommand_utils.py @@ -15,7 +15,7 @@ async def get_orgs(): return None -async def get_domains(orgs:list[str], asset_types:list[str]): +async def get_domains(orgs:list[str], asset_types:list[str], after_filter:str, before_filter:str): if len(orgs) == 0: logging.warning('No orgs selected') return [] @@ -23,13 +23,23 @@ async def get_domains(orgs:list[str], asset_types:list[str]): if len(asset_types) == 0: logging.warning('no asset type was selected') return [] + + match_filter = { + "org": {"$in": orgs}, + "asset_type": {"$in": asset_types}, + } + + if after_filter: + match_filter.setdefault("created_timestamp", {}) + match_filter["created_timestamp"]["$gte"] = after_filter + + if before_filter: + match_filter.setdefault("created_timestamp", {}) + match_filter["created_timestamp"]["$lte"] = before_filter pipeline = [ { - "$match": { - "org": {"$in": orgs}, - "asset_type": {"$in": asset_types}, - } + "$match": match_filter }, { "$group": { diff --git a/mantis/workflows/list_workflow.py b/mantis/workflows/list_workflow.py index 6ec305f..ad90c19 100644 --- a/mantis/workflows/list_workflow.py +++ b/mantis/workflows/list_workflow.py @@ -19,8 +19,10 @@ async def executor(args: ArgsModel): if args.list_domains: logging.info("Getting subdomains from database") - domains = await get_domains(args.orgs_list, args.asset_types_list) + domains = await get_domains(args.orgs_list, args.asset_types_list, args.after_datetime_filter, args.before_datetime_filter) if domains: print(f"\nOrgs Filter: {','.join(args.orgs_list)}") print(f"Total Domains Found based on provided filters: {len(domains)}\n") print('\n'.join(domains)) + else: + logging.info("No domains found") From 3ad1957097f724a824bb97d8378d502d79eca4a1 Mon Sep 17 00:00:00 2001 From: Dhrumil Mistry <56185972+dmdhrumilmistry@users.noreply.github.com> Date: Sat, 19 Oct 2024 20:55:21 +0530 Subject: [PATCH 4/6] remove ansible commits --- configs/local.yml | 2 +- setup/ansible/README.md | 44 ----- setup/ansible/native-setup.yml | 6 - setup/ansible/roles/native-setup/README.md | 41 ---- .../roles/native-setup/defaults/main.yml | 2 - .../roles/native-setup/handlers/main.yml | 2 - .../ansible/roles/native-setup/meta/main.yml | 34 ---- .../native-setup/tasks/install-tools.yml | 186 ------------------ .../ansible/roles/native-setup/tasks/main.yml | 65 ------ .../roles/native-setup/tests/inventory | 2 - .../ansible/roles/native-setup/tests/test.yml | 5 - .../ansible/roles/native-setup/vars/main.yml | 2 - 12 files changed, 1 insertion(+), 390 deletions(-) delete mode 100644 setup/ansible/README.md delete mode 100644 setup/ansible/native-setup.yml delete mode 100644 setup/ansible/roles/native-setup/README.md delete mode 100644 setup/ansible/roles/native-setup/defaults/main.yml delete mode 100644 setup/ansible/roles/native-setup/handlers/main.yml delete mode 100644 setup/ansible/roles/native-setup/meta/main.yml delete mode 100644 setup/ansible/roles/native-setup/tasks/install-tools.yml delete mode 100644 setup/ansible/roles/native-setup/tasks/main.yml delete mode 100644 setup/ansible/roles/native-setup/tests/inventory delete mode 100644 setup/ansible/roles/native-setup/tests/test.yml delete mode 100644 setup/ansible/roles/native-setup/vars/main.yml diff --git a/configs/local.yml b/configs/local.yml index d9f2c00..3257ef2 100644 --- a/configs/local.yml +++ b/configs/local.yml @@ -111,7 +111,7 @@ notify: dbConfig: - mongoConnectionString: 'mongodb://mantis.db:27017' + mongoConnectionString: 'mongodb://localhost:27000' logging: version: 1 diff --git a/setup/ansible/README.md b/setup/ansible/README.md deleted file mode 100644 index 8b34907..0000000 --- a/setup/ansible/README.md +++ /dev/null @@ -1,44 +0,0 @@ -# Ansible Setup - -Configures mantis dependencies on ubuntu x86_64 arch - -## Configuration - -### Ansible hosts/nodes - -* They must have python3 installed - - ```bash - sudo apt install python3 -y - ``` - -### Ansible Controller - -* Python deps - - ```bash - python3 -m pip install ansible ansible-core - ``` - -* Generate host file `hosts` - - ```txt - [mantisnodes] - mantis-node-1 ansible_host=192.168.0.105 ansible_user=root - mantis-node-2 ansible_host=192.168.0.107 ansible_user=root - mantis-node-3 ansible_host=192.168.0.109 ansible_user=root - ``` - - > Please ensure that ansible controller is able to ssh into the `mantisnodes` as `root` user using its private key - -* Test connectivity with hosts - - ```bash - ansible all -m ping - ``` - -* Run playbook - - ```bash - ansible-playbook -i hosts native-setup.yml - ``` diff --git a/setup/ansible/native-setup.yml b/setup/ansible/native-setup.yml deleted file mode 100644 index f9185b0..0000000 --- a/setup/ansible/native-setup.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- - -- hosts: mantisnodes - become: true - roles: - - { role : "native-setup" } diff --git a/setup/ansible/roles/native-setup/README.md b/setup/ansible/roles/native-setup/README.md deleted file mode 100644 index 567e30d..0000000 --- a/setup/ansible/roles/native-setup/README.md +++ /dev/null @@ -1,41 +0,0 @@ -native-setup -========= - -Role `native-setup` configures - -Requirements ------------- - -* ubuntu machine (x86_64 arch) -* python3 - - - -Example Playbook ----------------- - -Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: - - ```yaml - - hosts: mantisnodes - roles: - - { role: native-setup } - ``` - -License -------- - -[Apache 2.0](/LICENSE) - -Author Information ------------------- - -* [dmdhrumilmistry](https://github.com/dmdhrumilmistry) diff --git a/setup/ansible/roles/native-setup/defaults/main.yml b/setup/ansible/roles/native-setup/defaults/main.yml deleted file mode 100644 index a9c7d90..0000000 --- a/setup/ansible/roles/native-setup/defaults/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# defaults file for native-setup diff --git a/setup/ansible/roles/native-setup/handlers/main.yml b/setup/ansible/roles/native-setup/handlers/main.yml deleted file mode 100644 index e8a5b02..0000000 --- a/setup/ansible/roles/native-setup/handlers/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# handlers file for native-setup diff --git a/setup/ansible/roles/native-setup/meta/main.yml b/setup/ansible/roles/native-setup/meta/main.yml deleted file mode 100644 index 6999a76..0000000 --- a/setup/ansible/roles/native-setup/meta/main.yml +++ /dev/null @@ -1,34 +0,0 @@ -galaxy_info: - author: dmdhrumilmistry - description: Configures requirements for running mantis on ubuntu machines - # company: your company (optional) - - # If the issue tracker for your role is not on github, uncomment the - # next line and provide a value - # issue_tracker_url: http://example.com/issue/tracker - - # Choose a valid license ID from https://spdx.org - some suggested licenses: - # - BSD-3-Clause (default) - # - MIT - # - GPL-2.0-or-later - # - GPL-3.0-only - # - Apache-2.0 - # - CC-BY-4.0 - license: Apache-2.0 - - min_ansible_version: "2.1" - - # If this a Container Enabled role, provide the minimum Ansible Container version. - # min_ansible_container_version: - - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes - # and categorizes the role. Users find roles by searching for tags. Be sure to - # remove the '[]' above, if you add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. - -dependencies: [] - # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. diff --git a/setup/ansible/roles/native-setup/tasks/install-tools.yml b/setup/ansible/roles/native-setup/tasks/install-tools.yml deleted file mode 100644 index a6c412b..0000000 --- a/setup/ansible/roles/native-setup/tasks/install-tools.yml +++ /dev/null @@ -1,186 +0,0 @@ ---- -- name: Install Amass - block: - - name: Download Amass - get_url: - url: https://github.com/owasp-amass/amass/releases/download/v4.1.0/amass_Linux_amd64.zip - dest: /tmp/amass.zip - - - name: Unzip Amass - unarchive: - src: /tmp/amass.zip - dest: /tmp/ - remote_src: yes - extra_opts: ["-j"] - - - name: Move Amass to /usr/bin - command: mv /tmp/amass /usr/bin/amass - - - name: Ensure /usr/bin/amass is executable - file: - path: /usr/bin/amass - mode: "0755" - state: file - -- name: Install Subfinder - block: - - name: Download Subfinder - get_url: - url: https://github.com/projectdiscovery/subfinder/releases/download/v2.6.3/subfinder_2.6.3_linux_amd64.zip - dest: /tmp/subfinder.zip - - - name: Unzip Subfinder - unarchive: - src: /tmp/subfinder.zip - dest: /tmp/ - remote_src: yes - extra_opts: ["-j"] - - - name: Move Subfinder to /usr/bin - command: mv /tmp/subfinder /usr/bin/subfinder - - - name: Ensure /usr/bin/subfinder is executable - file: - path: /usr/bin/subfinder - mode: "0755" - state: file - -- name: Install Httpx - block: - - name: Download Httpx - get_url: - url: https://github.com/projectdiscovery/httpx/releases/download/v1.3.7/httpx_1.3.7_linux_amd64.zip - dest: /tmp/httpx.zip - - - name: Unzip Httpx - unarchive: - src: /tmp/httpx.zip - dest: /tmp/ - remote_src: yes - extra_opts: ["-j"] - - - name: Move Httpx to /usr/bin - command: mv /tmp/httpx /usr/bin/httpx - - - name: Ensure /usr/bin/httpx is executable - file: - path: /usr/bin/httpx - mode: "0755" - state: file - -- name: Install Naabu - block: - - name: Download Naabu - get_url: - url: https://github.com/projectdiscovery/naabu/releases/download/v2.1.9/naabu_2.1.9_linux_amd64.zip - dest: /tmp/naabu.zip - - - name: Unzip Naabu - unarchive: - src: /tmp/naabu.zip - dest: /tmp/ - remote_src: yes - extra_opts: ["-j"] - - - name: Move Naabu to /usr/bin - command: mv /tmp/naabu /usr/bin/naabu - - - name: Ensure /usr/bin/naabu is executable - file: - path: /usr/bin/naabu - mode: "0755" - state: file - -- name: Install Nuclei - block: - - name: Download Nuclei - get_url: - url: https://github.com/projectdiscovery/nuclei/releases/download/v3.0.4/nuclei_3.0.4_linux_amd64.zip - dest: /tmp/nuclei.zip - - - name: Unzip Nuclei - unarchive: - src: /tmp/nuclei.zip - dest: /tmp/ - remote_src: yes - extra_opts: ["-j"] - - - name: Move Nuclei to /usr/bin - command: mv /tmp/nuclei /usr/bin/nuclei - - - name: Ensure /usr/bin/nuclei is executable - file: - path: /usr/bin/nuclei - mode: "0755" - state: file - -- name: Install Gitleaks - block: - - name: Download Gitleaks - get_url: - url: https://github.com/gitleaks/gitleaks/releases/download/v8.18.1/gitleaks_8.18.1_linux_x64.tar.gz - dest: /tmp/gitleaks.tar.gz - - - name: Extract Gitleaks - unarchive: - src: /tmp/gitleaks.tar.gz - dest: /tmp/ - remote_src: yes - - - name: Move Gitleaks to /usr/bin - command: mv /tmp/gitleaks /usr/bin/gitleaks - - - name: Ensure /usr/bin/gitleaks is executable - file: - path: /usr/bin/gitleaks - mode: "0755" - state: file - -- name: Install gau - block: - - name: Download gau tarball - get_url: - url: https://github.com/lc/gau/releases/download/v2.2.1/gau_2.2.1_linux_amd64.tar.gz - dest: /tmp/gau.tar.gz - - - name: Extract gau tarball - unarchive: - src: /tmp/gau.tar.gz - dest: /tmp/ - remote_src: yes - - - name: Move gau to /usr/bin - command: mv /tmp/gau /usr/bin/gau - - - name: Ensure /usr/bin/gau is executable - file: - path: /usr/bin/gau - mode: "0755" - state: file - -- name: Install Corsy - block: - - name: Download Corsy zip - get_url: - url: https://github.com/s0md3v/Corsy/archive/refs/tags/1.0-rc.zip - dest: /tmp/1.0-rc.zip - - - name: Unzip Corsy - unarchive: - src: /tmp/1.0-rc.zip - dest: /tmp - remote_src: yes - - - name: Move Corsy to /usr/bin - command: mv /tmp/Corsy-1.0-rc /usr/bin/Corsy - -- name: Install IPinfo - block: - - name: Download IPinfo deb package - get_url: - url: https://github.com/ipinfo/cli/releases/download/ipinfo-3.2.0/ipinfo_3.2.0.deb - dest: /tmp/ipinfo_3.2.0.deb - - - name: Install IPinfo deb package - apt: - deb: /tmp/ipinfo_3.2.0.deb \ No newline at end of file diff --git a/setup/ansible/roles/native-setup/tasks/main.yml b/setup/ansible/roles/native-setup/tasks/main.yml deleted file mode 100644 index 4706925..0000000 --- a/setup/ansible/roles/native-setup/tasks/main.yml +++ /dev/null @@ -1,65 +0,0 @@ ---- -# tasks file for native-setup -- name: Install aptitude - apt: - name: aptitude - state: latest - update_cache: true - -- name: Add deadsnakes PPA for Python 3.9 - apt_repository: - repo: ppa:deadsnakes/ppa - -- name: Install required system packages - apt: - pkg: - - apt-transport-https - - ca-certificates - - curl - - gcc - - git - - gnupg - - libpcap-dev - - dnsutils - - python3.9 - - python3.9-dev - - python3-pip - - python3-setuptools - - python3-requests - - unzip - - software-properties-common - - wget - state: latest - update_cache: true - -- name: Add ~/.local/bin to PATH in ~/.bashrc - lineinfile: - path: ~/.bashrc - line: "export PATH='$HOME/.local/bin:$PATH'" - create: yes - state: present - -- name: Reload ~/.bashrc - shell: source ~/.bashrc - args: - executable: /bin/bash - -- include_tasks: install-tools.yml - -- name: Install pip dependencies - pip: - name: - - "dnspython<=2.0.0" - - "dnstwist[full]" - - ipwhois - - wafw00f - - "git+https://github.com/cisagov/findcdn.git" - -- name: Download requirements - get_url: - url: https://raw.githubusercontent.com/PhonePe/mantis/refs/heads/main/requirements.txt - dest: /tmp/mantis-requirements.txt - -- name: Install Python dependencies - pip: - requirements: /tmp/mantis-requirements.txt diff --git a/setup/ansible/roles/native-setup/tests/inventory b/setup/ansible/roles/native-setup/tests/inventory deleted file mode 100644 index 878877b..0000000 --- a/setup/ansible/roles/native-setup/tests/inventory +++ /dev/null @@ -1,2 +0,0 @@ -localhost - diff --git a/setup/ansible/roles/native-setup/tests/test.yml b/setup/ansible/roles/native-setup/tests/test.yml deleted file mode 100644 index ed20304..0000000 --- a/setup/ansible/roles/native-setup/tests/test.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -- hosts: localhost - remote_user: root - roles: - - native-setup diff --git a/setup/ansible/roles/native-setup/vars/main.yml b/setup/ansible/roles/native-setup/vars/main.yml deleted file mode 100644 index 314654d..0000000 --- a/setup/ansible/roles/native-setup/vars/main.yml +++ /dev/null @@ -1,2 +0,0 @@ ---- -# vars file for native-setup From a23f73bea50d7da3ad43fe412ea51ad4fdeb24e4 Mon Sep 17 00:00:00 2001 From: Dhrumil Mistry <56185972+dmdhrumilmistry@users.noreply.github.com> Date: Sat, 19 Oct 2024 20:57:19 +0530 Subject: [PATCH 5/6] revert local.yml file values --- configs/local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/local.yml b/configs/local.yml index ace2446..4bb05be 100644 --- a/configs/local.yml +++ b/configs/local.yml @@ -111,7 +111,7 @@ notify: dbConfig: - mongoConnectionString: 'mongodb://localhost:27000' + mongoConnectionString: 'mongodb://mantis.db:27017' logging: version: 1 From 36738eaa205fabc9cd82db3392c4a2a42b68f0a1 Mon Sep 17 00:00:00 2001 From: Dhrumil Mistry <56185972+dmdhrumilmistry@users.noreply.github.com> Date: Sat, 19 Oct 2024 21:10:49 +0530 Subject: [PATCH 6/6] remove comments --- mantis/utils/args_parse.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mantis/utils/args_parse.py b/mantis/utils/args_parse.py index 0c9b00e..bf7a4cb 100644 --- a/mantis/utils/args_parse.py +++ b/mantis/utils/args_parse.py @@ -248,10 +248,6 @@ def args_parse() -> ArgsModel: list_parser.add_argument("-a","--after", type=str, help="Start date in YYYY-MM-DD format", dest="list_sub_command_ls_subs_after_filter") list_parser.add_argument("-b","--before", type=str, help="End date in YYYY-MM-DD format", dest="list_sub_command_ls_subs_before_filter") - # list_sub_parser = list_parser.add_subparsers(title="List Subcommands", dest="list_sub_command") - # list_org_sub_parser = list_sub_parser.add_parser("orgs", help="List orgs present in DB") - # list_org_sub_parser.add_argument() - # display help, if no arguments are passed args = parser.parse_args(args=None if argv[1:] else ['--help']) logging.info(f"Arguments Passed - {args}")