diff --git a/.envs_template/.local/.django b/.envs_template/.local/.django index 7c40d3a4c..60a284a5a 100644 --- a/.envs_template/.local/.django +++ b/.envs_template/.local/.django @@ -9,38 +9,3 @@ DJANGO_ACCOUNT_EMAIL_VERIFICATION=none # ------------------------------------------------------------------------------ QCLUSTER_NAME=soar # QCLUSTER_CONNECTION= - -# Virustotal -# ------------------------------------------------------------------------------ -VIRUSTOTAL_API_KEY=None - -# Slack -# ------------------------------------------------------------------------------ -SLACK_ENABLE=True -SLACK_EMOJI=:ghost: -SLACK_CHANNEL=#ghostwriter -SLACK_ALERT_TARGET= -SLACK_USERNAME=ghostwriter -SLACK_URL=https://hooks.slack.com/services/ - -# Company Info -# ------------------------------------------------------------------------------ -COMPANY_NAME=Ghostwriter -COMPANY_TWITTER=@ghostwriter -COMPANY_EMAIL=info@ghostwriter.local - -# Namecheap -# ------------------------------------------------------------------------------ -NAMECHEAP_ENABLE=False -NAMECHEAP_API_KEY= -NAMECHEAP_USERNAME= -NAMECHEAP_API_USERNAME= -CLIENT_IP= -NAMECHEAP_PAGE_SIZE=100 - -# Cloud Services -# ------------------------------------------------------------------------------ -ENABLE_CLOUD_MONITOR=False -AWS_KEY= -AWS_SECRET= -DO_API_KEY= \ No newline at end of file diff --git a/.envs_template/.production/.django b/.envs_template/.production/.django index 2444d7ba6..3842fa37f 100644 --- a/.envs_template/.production/.django +++ b/.envs_template/.production/.django @@ -38,38 +38,3 @@ REDIS_URL=redis://redis:6379/0 # ------------------------------------------------------------------------------ QCLUSTER_NAME=soar # QCLUSTER_CONNECTION= - -# Virustotal -# ------------------------------------------------------------------------------ -VIRUSTOTAL_API_KEY=None - -# Slack -# ------------------------------------------------------------------------------ -SLACK_ENABLE=True -SLACK_EMOJI=:ghost: -SLACK_CHANNEL=#ghostwriter -SLACK_ALERT_TARGET= -SLACK_USERNAME=ghostwriter -SLACK_URL=https://hooks.slack.com/services/ - -# Company Info -# ------------------------------------------------------------------------------ -COMPANY_NAME=Ghostwriter -COMPANY_TWITTER=@ghostwriter -COMPANY_EMAIL=info@ghostwriter.local - -# Namecheap -# ------------------------------------------------------------------------------ -NAMECHEAP_ENABLE=False -NAMECHEAP_API_KEY= -NAMECHEAP_USERNAME= -NAMECHEAP_API_USERNAME= -CLIENT_IP= -NAMECHEAP_PAGE_SIZE=100 - -# Cloud Services -# ------------------------------------------------------------------------------ -ENABLE_CLOUD_MONITOR=False -AWS_KEY= -AWS_SECRET= -DO_API_KEY= \ No newline at end of file diff --git a/DOCS/CHANGELOG.RST b/DOCS/CHANGELOG.RST index ba738fe84..d5caa45b6 100644 --- a/DOCS/CHANGELOG.RST +++ b/DOCS/CHANGELOG.RST @@ -1,6 +1,59 @@ Changelog ========= +20 November 2020 +---------------- +* Tagged release v2.0 + * More details: https://posts.specterops.io/ghostwriter-v2-0-release-638cef16deb7 +* Upgraded to Django 3 and updated all dependencies +* Initial commit of CommandCenter application and related configuration options + * VirusTotal Configuration + * Global Report Configuration + * Slack Configuration + * Company information + * Namecheap Configuration +* Initial support for adding users to groups for Role-Based Access Controls +* Automated Acitivty Logging (Oplog application) moved out of beta +* Implemented initial "overwatch" notifications + * Domain check-out: alert if domain will expire soon and is not set to auto-renew + * Domain check-out: alert if domain is marked as burned + * Domain check-out: alert if domain has been previously used with selected client +* Updated user interface elements + * New tabbed dashboards for clients, projects, and domains + * New inline forms for creating and managing clients and projects and related items + * New sidebar menu to improve legibility + * Migrated buttons and background tasks to WebSockets and AJAX for a more seamless experience +* Initial release of refactored reporting engine + * New drag-and-drop report management interface + * Added many more options to the WYSIWYG editor's formatting menus + * Initial support for rich text objects for Word documents + * Added new `filter_severity` filter for Word templates + * Closes #89 +* Initial support for report template and management + * Upload report tempalte files for Word and PowerPoint + * New template linter to check and verify templates + * Closes #28 + * Closes #90 +* Security updates and fixes + * Resolved potential stored cross-site scripting in operational logs + * Resolved unvalidated evidence file uploads and new note creation + * Associated user account is now set server-side + * Resolved issues with WebSocket authentication + * Locked-down evidence uploads to close potential loopholes + * Evidence form now only allows specific filetypes: md, txt, log, jpg, jpeg, png + * Requesting an evidence file requires an active user session +* Removed web scraping from domain health checks + * Checks now use VirusTotal and link to the results + * Closes #50 + * Closes #84 +* Numerous bug fixes and enhancements to address reported issues + * Closes #54 + * Closes #55 + * Closes #69 + * Closes #92 + * Closes #93 + * Closes #98 + 25 August 2020 -------------- * Cleaned and refactored each application to improve UI/UX and performance diff --git a/DOCS/examples/post_activity_logs.py b/DOCS/examples/post_activity_logs.py new file mode 100644 index 000000000..d3a9e8249 --- /dev/null +++ b/DOCS/examples/post_activity_logs.py @@ -0,0 +1,39 @@ +import requests +import json + +# Replace with a URL, API, and ID key for your instance +url = "http://127.0.0.1:8000/oplog/api/entries/" +api_key = "API_KEY" +oplog_id = 1 + +headers = { + "user-agent": "Python", + "Content-Type": "application/json", + "Authorization": f"Api-Key {api_key}", +} + +data = { + "start_date": None, + "end_date": None, + "source_ip": "WIN10VM (10.20.10.10)", + "dest_ip": "127.0.0.1", + "tool": "Beacon", + "user_context": "ADMIN", + "command": "execute_assembly /tmp/Seatbelt.exe logonevents", + "description": "", + "output": "", + "comments": "", + "operator_name": "Benny", + "oplog_id": "1", +} + +print("[+] Sending request to Ghostwriter...") + +resp = requests.post(url, headers=headers, data=json.dumps(data)) + +if resp.status_code == 201: + print(f"[+] Received code 201, so log was created: {resp.text}") +else: + print( + f"[!] Received status code {resp.status_code}, so something went wrong: {resp.text}" + ) diff --git a/DOCS/images/logo.png b/DOCS/images/logo.png index ad70f73f6..10d0c6b6a 100644 Binary files a/DOCS/images/logo.png and b/DOCS/images/logo.png differ diff --git a/DOCS/sample_reports/so-con_template.docx b/DOCS/sample_reports/so-con_template.docx new file mode 100644 index 000000000..26d6206b1 Binary files /dev/null and b/DOCS/sample_reports/so-con_template.docx differ diff --git a/DOCS/sample_reports/template.docx b/DOCS/sample_reports/template.docx new file mode 100644 index 000000000..65f5a20ac Binary files /dev/null and b/DOCS/sample_reports/template.docx differ diff --git a/DOCS/sample_reports/template.pptx b/DOCS/sample_reports/template.pptx new file mode 100644 index 000000000..d1bb702a5 Binary files /dev/null and b/DOCS/sample_reports/template.pptx differ diff --git a/README.md b/README.md index 6546776ad..60b187972 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,54 @@ # Ghostwriter -[![Python Version](https://img.shields.io/badge/Python-3.7-brightgreen.svg)](.) [![License](https://img.shields.io/badge/License-BSD3-darkred.svg)](.) [![Black Hat Arsenal 2019](https://img.shields.io/badge/2019-Black%20Hat%20Arsenal-lightgrey.svg)](.) +[![Python Version](https://img.shields.io/badge/Python-3.8-brightgreen.svg)](.) [![License](https://img.shields.io/badge/License-BSD3-darkred.svg)](.) [![Black Hat Arsenal 2019](https://img.shields.io/badge/2019-Black%20Hat%20Arsenal-lightgrey.svg)](https://www.blackhat.com/us-19/arsenal/schedule/index.html#ghostwriter-15475) -![ghostwriter](https://github.com/GhostManager/Ghostwriter/raw/master/DOCS/images/logo.png) +![GitHub release (latest by date)](https://img.shields.io/github/v/release/GhostManager/Ghostwriter) +![GitHub Release Date](https://img.shields.io/github/release-date/ghostmanager/ghostwriter) -Ghostwriter is a Django project written in Python 3.7 and is designed to be used by a team of operators. The platform is made up of several Django apps that own different roles but work together. See the Wiki for more information. +![ghostwriter](DOCS/images/logo.png) +Ghostwriter is a Django project written in Python 3.8 and is designed to be used by a team of operators. The platform is made up of several Django apps that own different roles but work together. + +## Details + +Check-out the introductory blogpost: [Introducing Ghostwriter](https://posts.specterops.io/introducing-ghostwriter-part-1-61e7bd014aff) + +This blogpost discusses the design and intent behind Ghostwriter: [Introducing Ghostwriter: Part 2](https://posts.specterops.io/introducing-ghostwriter-part-2-f2d8368a1ed6) + +## Documentation + +The Ghostwriter Wiki contains everything you need to know to use or customize Ghostwriter: + +[Ghostwriter Wiki](https://ghostwriter.wiki/) + +The wiki covers everything from installation and setup information for first time users to database schemas, the project's code style guide, and how to expand or customie parts of the project to fit your needs. + +## Getting Help + +[![Slack Status](https://img.shields.io/badge/Slack-%23ghostwriter-blueviolet)](https://bloodhoundgang.herokuapp.com) + +The quickest way to get help is Slack. The BloodHound Slack Team has a #ghostwriter channel for discussing this project and requesting assistance. There is also a #reporting channel for discussing various topics related to writing and managing reports and findings. + +You can submit an issue. If you do, please use the issue template and provide as much information as possible. + +Before submitting an issue, review the [Ghostwriter Wiki](https://ghostwriter.wiki/). Many of the common issues new users encounter stem from missing a step (like loading the seed data to prep the database) or an issue with Docker on their host system. + +## Contributing to the Project + +Please open issues or submit pull requests! The project team welcomes feedback, new ideas, and external contributions. Before submitting a PR, please check open and closed issues for any previous related discussion. Also, the submitted code must follow the [Code Style Guide](https://ghostwriter.wiki/coding-style-guide/style-guide) to be accepted. + +We only ask that you limit PR submissions to those that fix a bug, enhance an existing feature, or add something new. + +## Contributions + +The following people have contributed much to this project: + +* [@covertgeek](https://github.com/covertgeek) +* [@hotnops](https://github.com/hotnops) +* [@andrewchiles](https://github.com/andrewchiles) + +These folks kindly submitted feedback and PRs to fix bugs and enhance existing features. Thank you! + +* [@fastlorenzo](https://github.com/fastlorenzo) +* [@mattreduce](https://github.com/mattreduce) +* [@dbuentello](https://github.com/dbuentello) diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index 33170fa2e..5c3c15aaf 100644 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -1,21 +1,21 @@ -FROM python:3.7-alpine +FROM python:3.8-alpine ENV PYTHONUNBUFFERED 1 RUN apk update \ - # psycopg2 dependencies - && apk add --virtual build-deps gcc python3-dev musl-dev \ - && apk add postgresql-dev \ - # Pillow dependencies - && apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ - # CFFI dependencies - && apk add libffi-dev py-cffi \ - # XLSX dependencies - && apk add libxml2-dev libxslt-dev \ - # Translations dependencies - && apk add gettext \ - # https://docs.djangoproject.com/en/dev/ref/django-admin/#dbshell - && apk add postgresql-client + # psycopg2 dependencies + && apk add --virtual build-deps gcc python3-dev musl-dev \ + && apk add postgresql-dev \ + # Pillow dependencies + && apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ + # CFFI dependencies + && apk add libffi-dev py-cffi \ + # XLSX dependencies + && apk add libxml2-dev libxslt-dev \ + # Translations dependencies + && apk add gettext \ + # https://docs.djangoproject.com/en/dev/ref/django-admin/#dbshell + && apk add postgresql-client # Requirements are installed here to ensure they will be cached. COPY ./requirements /requirements diff --git a/compose/local/django/start b/compose/local/django/start index fae5352ab..88c7b089a 100644 --- a/compose/local/django/start +++ b/compose/local/django/start @@ -6,9 +6,17 @@ set -o nounset AVATAR_DIR=/app/ghostwriter/media/user_avatars EVIDENCE_DIR=/app/ghostwriter/media/evidence +TEMPLATE_DIR=/app/ghostwriter/media/templates + +TEMPLATE_PATH_DOCX=/app/ghostwriter/reporting/templates/reports/template.docx +TEMPLATE_PATH_PPTX=/app/ghostwriter/reporting/templates/reports/template.pptx [[ ! -d "$EVIDENCE_DIR" ]] && mkdir -p "$EVIDENCE_DIR" [[ ! -d "$AVATAR_DIR" ]] && mkdir -p "$AVATAR_DIR" +[[ ! -d "$TEMPLATE_DIR" ]] && mkdir -p "$TEMPLATE_DIR" + +cp -u -p "$TEMPLATE_PATH_DOCX" "$TEMPLATE_DIR" +cp -u -p "$TEMPLATE_PATH_PPTX" "$TEMPLATE_DIR" python manage.py makemigrations python manage.py migrate diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index bde9e9ab6..5f6a5c618 100644 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -1,27 +1,27 @@ -FROM python:3.7-alpine +FROM python:3.8-alpine ENV PYTHONUNBUFFERED 1 ENV PYTHONPATH="$PYTHONPATH:/app/config" RUN apk update \ - # psycopg2 dependencies - && apk add --virtual build-deps gcc python3-dev musl-dev \ - && apk add postgresql-dev \ - # Pillow dependencies - && apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ - # CFFI dependencies - && apk add libffi-dev py-cffi \ - # XLSX dependencies - && apk add libxml2-dev libxslt-dev + # psycopg2 dependencies + && apk add --virtual build-deps gcc python3-dev musl-dev \ + && apk add postgresql-dev \ + # Pillow dependencies + && apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ + # CFFI dependencies + && apk add libffi-dev py-cffi \ + # XLSX dependencies + && apk add libxml2-dev libxslt-dev RUN addgroup -S django \ - && adduser -S -G django django + && adduser -S -G django django # Requirements are installed here to ensure they will be cached. COPY ./requirements /requirements RUN pip install --no-cache-dir -r /requirements/production.txt \ - && rm -rf /requirements + && rm -rf /requirements COPY ./compose/production/django/entrypoint /entrypoint RUN sed -i 's/\r$//g' /entrypoint diff --git a/compose/production/django/start b/compose/production/django/start index 9f506921e..8fe660053 100644 --- a/compose/production/django/start +++ b/compose/production/django/start @@ -6,9 +6,17 @@ set -o nounset AVATAR_DIR=/app/ghostwriter/media/user_avatars EVIDENCE_DIR=/app/ghostwriter/media/evidence +TEMPLATE_DIR=/app/ghostwriter/media/templates + +TEMPLATE_PATH_DOCX=/app/ghostwriter/reporting/templates/reports/template.docx +TEMPLATE_PATH_PPTX=/app/ghostwriter/reporting/templates/reports/template.pptx [[ ! -d "$EVIDENCE_DIR" ]] && mkdir -p "$EVIDENCE_DIR" && chown -R django "$EVIDENCE_DIR" -[[ ! -d "$AVATAR_DIR" ]] && mkdir -p "$AVATAR_DIR" && chown -R django "$AVATAR_DIR" +[[ ! -d "$AVATAR_DIR" ]] && mkdir -p "$AVATAR_DIR" && chown -R django "$AVATAR_DIR" +[[ ! -d "$TEMPLATE_DIR" ]] && mkdir -p "$TEMPLATE_DIR" && chown -R django "$TEMPLATE_DIR" + +cp -u -p "$TEMPLATE_PATH_DOCX" "$TEMPLATE_DIR" +cp -u -p "$TEMPLATE_PATH_PPTX" "$TEMPLATE_DIR" python /app/manage.py collectstatic --noinput python /app/manage.py migrate diff --git a/config/routing.py b/config/routing.py new file mode 100644 index 000000000..85f00dc64 --- /dev/null +++ b/config/routing.py @@ -0,0 +1,17 @@ +"""This contains all of the WebSocket routes used by the Ghostwriter application.""" + +# Django & Other 3rd Party Libraries +from django.urls import path +from channels.routing import ProtocolTypeRouter, URLRouter +from channels.auth import AuthMiddlewareStack + +# Ghostwriter Libraries +from ghostwriter.oplog.consumers import OplogEntryConsumer + +application = ProtocolTypeRouter( + { + "websocket": AuthMiddlewareStack( + URLRouter([path("ws/oplog//entries", OplogEntryConsumer)]) + ) + } +) diff --git a/config/settings/base.py b/config/settings/base.py index 4381671ab..36dad1343 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -54,7 +54,7 @@ ROOT_URLCONF = "config.urls" # https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application -#WSGI_APPLICATION = "config.wsgi.application" +# WSGI_APPLICATION = "config.wsgi.application" ASGI_APPLICATION = "config.routing.application" # APPS @@ -93,6 +93,8 @@ "ghostwriter.shepherd.apps.ShepherdConfig", "ghostwriter.reporting.apps.ReportingConfig", "ghostwriter.oplog.apps.OplogConfig", + "ghostwriter.commandcenter.apps.CommandCenterConfig", + "ghostwriter.singleton.apps.SingletonConfig", ] # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS @@ -105,7 +107,9 @@ CHANNEL_LAYERS = { "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", - "CONFIG": {"hosts": [("redis", 6379)],}, + "CONFIG": { + "hosts": [("redis", 6379)], + }, }, } @@ -245,7 +249,7 @@ # ADMIN # ------------------------------------------------------------------------------ -# Django Admin URL. +# Django Admin URL ADMIN_URL = "admin/" # https://docs.djangoproject.com/en/dev/ref/settings/#admins ADMINS = [] @@ -330,54 +334,20 @@ ), } -# DOMAIN HEALTH CHECKS -# ------------------------------------------------------------------------------ -# Enter a VirusTotal API key (free or paid) -DOMAINCHECK_CONFIG = { - "virustotal_api_key": env("VIRUSTOTAL_API_KEY", default=None), - "sleep_time": 20, -} - -# SLACK -# ------------------------------------------------------------------------------ -SLACK_CONFIG = { - "enable_slack": env("SLACK_ENABLE", default=False), - "slack_emoji": env("SLACK_EMOJI", default=":ghost:"), - "slack_channel": env("SLACK_CHANNEL", default="#ghostwriter"), - "slack_alert_target": env("SLACK_ALERT_TARGET", default="<@ghostwriter>"), - "slack_username": env("SLACK_USERNAME", default="Ghostwriter"), - "slack_webhook_url": env("SLACK_URL", default=""), -} - -# GLOBAL COMPANY SETTINGS +# SETTINGS # ------------------------------------------------------------------------------ -COMPANY_NAME = env("COMPANY_NAME", default="Ghostwriter") -COMPANY_TWITTER = env("COMPANY_TWITTER", default="@ghostwriter") -COMPANY_EMAIL = env("COMPANY_EMAIL", default="info@ghostwriter.local") +# All settings are stored in singleton models in the CommandCenter app +# Settings can be cached to avoid repeated database queries -TEMPLATE_LOC = env( - "TEMPLATE_LOC", default=str(APPS_DIR("reporting", "templates", "reports")) -) - -# NAMECHEAP -# ------------------------------------------------------------------------------ -NAMECHEAP_CONFIG = { - "enable_namecheap": env("NAMECHEAP_ENABLE", default=False), - "namecheap_api_key": env("NAMECHEAP_API_KEY", default=None), - "namecheap_username": env("NAMECHEAP_USERNAME", default=None), - "namecheap_api_username": env("NAMECHEAP_API_USERNAME", default=None), - "client_ip": env("CLIENT_IP", default=None), - "namecheap_page_size": env("NAMECHEAP_PAGE_SIZE", default="100"), -} +# The cache that should be used, e.g. 'default' +# Set to ``None`` to disable caching +# Ghostwriter does not use a cache by default +SOLO_CACHE = None +SOLO_CACHE_TIMEOUT = 60 * 5 +SOLO_CACHE_PREFIX = "solo" -# CLOUD SERVICES -# ------------------------------------------------------------------------------ -CLOUD_SERVICE_CONFIG = { - "enable_cloud_monitor": env("ENABLE_CLOUD_MONITOR", default=False), - "aws_key": env("AWS_KEY", default=None), - "aws_secret": env("AWS_SECRET", default=None), - "do_api_key": env("DO_API_KEY", default=None), -} +# Default location for report templates +TEMPLATE_LOC = env("TEMPLATE_LOC", default=str(APPS_DIR("media", "templates"))) # BLEACH # ------------------------------------------------------------------------------ @@ -396,6 +366,15 @@ "b", "i", "pre", + "sub", + "sup", + "del", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", ] # Which HTML attributes are allowed BLEACH_ALLOWED_ATTRIBUTES = ["href", "title", "style", "class", "src"] @@ -417,21 +396,19 @@ # Django REST Configuration # ------------------------------------------------------------------------------ REST_FRAMEWORK = { - 'DEFAULT_AUTHENTICATION_CLASSES': [ - #'rest_framework.authentication.TokenAuthentication', - 'rest_framework.authentication.SessionAuthentication', + "DEFAULT_AUTHENTICATION_CLASSES": [ + # 'rest_framework.authentication.TokenAuthentication', + "rest_framework.authentication.SessionAuthentication", ], - 'DEFAULT_PERMISSION_CLASSES': [ - 'rest_framework.permissions.IsAuthenticated', - #'rest_framework_api_key.permissions.HasAPIKey', + "DEFAULT_PERMISSION_CLASSES": [ + "rest_framework.permissions.IsAuthenticated", + # 'rest_framework_api_key.permissions.HasAPIKey', ], } CHANNEL_LAYERS = { "default": { "BACKEND": "channels_redis.core.RedisChannelLayer", - "CONFIG": { - "hosts": [("redis", "6379")] - } + "CONFIG": {"hosts": [("redis", "6379")]}, } } diff --git a/config/settings/production.py b/config/settings/production.py index ed6699c57..794bd8a38 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -6,7 +6,9 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#secret-key SECRET_KEY = env("DJANGO_SECRET_KEY") # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts -ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["ghostwriter.local", "localhost"]) +ALLOWED_HOSTS = env.list( + "DJANGO_ALLOWED_HOSTS", default=["ghostwriter.local", "localhost"] +) # DATABASES # ------------------------------------------------------------------------------ @@ -79,9 +81,7 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#server-email SERVER_EMAIL = env("DJANGO_SERVER_EMAIL", default=DEFAULT_FROM_EMAIL) # https://docs.djangoproject.com/en/dev/ref/settings/#email-subject-prefix -EMAIL_SUBJECT_PREFIX = env( - "DJANGO_EMAIL_SUBJECT_PREFIX", default="[Ghostwriter]" -) +EMAIL_SUBJECT_PREFIX = env("DJANGO_EMAIL_SUBJECT_PREFIX", default="[Ghostwriter]") # ADMIN # ------------------------------------------------------------------------------ @@ -113,6 +113,7 @@ # https://github.com/antonagestam/collectfast#installation INSTALLED_APPS = ["collectfast"] + INSTALLED_APPS # noqa F405 AWS_PRELOAD_METADATA = True +COLLECTFAST_STRATEGY = "collectfast.strategies.filesystem.FileSystemStrategy" # LOGGING # ------------------------------------------------------------------------------ diff --git a/config/urls.py b/config/urls.py index 93eb6c4a6..a108fa40a 100644 --- a/config/urls.py +++ b/config/urls.py @@ -2,13 +2,15 @@ # Django & Other 3rd Party Libraries from django.conf import settings -from django.conf.urls.static import static from django.contrib import admin from django.urls import include, path, re_path from django.views import defaults as default_views from django.views.generic import RedirectView # Ghostwriter Libraries +from ghostwriter.home.views import ( + protected_serve, +) from ghostwriter.users.views import ( account_change_password, account_reset_password_from_key, @@ -38,9 +40,14 @@ path("reporting/", include("ghostwriter.reporting.urls", namespace="reporting")), path("", RedirectView.as_view(pattern_name="home:dashboard"), name="home"), path("oplog/", include("ghostwriter.oplog.urls", namespace="oplog")), + re_path( + r"^%s(?P.*)$" % settings.MEDIA_URL[1:], + protected_serve, + {"document_root": settings.MEDIA_ROOT}, + ) # Add additional custom paths below this line... # Your stuff: custom urls includes go here -] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) +] if settings.DEBUG: # This allows the error pages to be debugged during development, just visit diff --git a/ghostwriter/commandcenter/__init__.py b/ghostwriter/commandcenter/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ghostwriter/commandcenter/admin.py b/ghostwriter/commandcenter/admin.py new file mode 100644 index 000000000..5cf2abebe --- /dev/null +++ b/ghostwriter/commandcenter/admin.py @@ -0,0 +1,30 @@ +"""This contains customizations for displaying the CommandCenter application models in the admin panel.""" + +# Django & Other 3rd Party Libraries +from django.contrib import admin + +# Ghostwriter Libraries +from ghostwriter.singleton.admin import SingletonModelAdmin +from .forms import ReportConfigurationForm + +from .models import ( + CloudServicesConfiguration, + CompanyInformation, + NamecheapConfiguration, + ReportConfiguration, + SlackConfiguration, + VirusTotalConfiguration, +) + +admin.site.register(CloudServicesConfiguration, SingletonModelAdmin) +admin.site.register(CompanyInformation, SingletonModelAdmin) +admin.site.register(NamecheapConfiguration, SingletonModelAdmin) +admin.site.register(SlackConfiguration, SingletonModelAdmin) +admin.site.register(VirusTotalConfiguration, SingletonModelAdmin) + + +class ReportConfigurationAdmin(SingletonModelAdmin): + form = ReportConfigurationForm + + +admin.site.register(ReportConfiguration, ReportConfigurationAdmin) diff --git a/ghostwriter/commandcenter/apps.py b/ghostwriter/commandcenter/apps.py new file mode 100644 index 000000000..3df6eb597 --- /dev/null +++ b/ghostwriter/commandcenter/apps.py @@ -0,0 +1,14 @@ +"""This contains the configuration of the CommandCenter application.""" + +# Django & Other 3rd Party Libraries +from django.apps import AppConfig + + +class CommandCenterConfig(AppConfig): + name = "ghostwriter.commandcenter" + + def ready(self): + try: + import ghostwriter.commandcenter.signals # noqa F401 + except ImportError: + pass diff --git a/ghostwriter/commandcenter/forms.py b/ghostwriter/commandcenter/forms.py new file mode 100644 index 000000000..df9f38571 --- /dev/null +++ b/ghostwriter/commandcenter/forms.py @@ -0,0 +1,45 @@ +"""This contains all of the forms used by the CommandCenter application.""" + +# Django & Other 3rd Party Libraries +from django import forms +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + +# Ghostwriter Libraries +from ghostwriter.commandcenter.models import ReportConfiguration + + +class ReportConfigurationForm(forms.ModelForm): + """ + Save an individual :model:`commandcenter.ReportConfiguration`. + """ + + class Meta: + model = ReportConfiguration + fields = "__all__" + + def clean_default_docx_template(self, *args, **kwargs): + docx_template = self.cleaned_data["default_docx_template"] + if docx_template: + docx_template_status = docx_template.get_status() + if docx_template_status == "error" or docx_template_status == "failed": + raise ValidationError( + _( + "Your selected Word template failed linting and cannot be used as a default template" + ), + "invalid", + ) + return docx_template + + def clean_default_pptx_template(self, *args, **kwargs): + pptx_template = self.cleaned_data["default_pptx_template"] + if pptx_template: + pptx_template_status = pptx_template.get_status() + if pptx_template_status == "error" or pptx_template_status == "failed": + raise ValidationError( + _( + "Your selected PowerPoint template failed linting and cannot be used as a default template" + ), + "invalid", + ) + return pptx_template diff --git a/ghostwriter/commandcenter/migrations/0001_initial.py b/ghostwriter/commandcenter/migrations/0001_initial.py new file mode 100644 index 000000000..e6cc7a2ca --- /dev/null +++ b/ghostwriter/commandcenter/migrations/0001_initial.py @@ -0,0 +1,95 @@ +# Generated by Django 2.2.3 on 2020-09-25 17:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='CloudServicesConfiguration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enable', models.BooleanField(default=False, help_text='Enable to allow the cloud monitoring task to run')), + ('aws_key', models.CharField(default='Your AWS Access Key', max_length=255, verbose_name='AWS Access Key')), + ('aws_secret', models.CharField(default='Your AWS Secret Key', max_length=255, verbose_name='AWS Secret Key')), + ('do_api_key', models.CharField(default='Digital Ocean API Key', max_length=255, verbose_name='Digital Ocean API Key')), + ('ignore_tag', models.CharField(default='gw_ignore', help_text='Ghostwriter will ignore cloud assets with one of these tags (comma-seperated list)', max_length=255, verbose_name='Ignore Tags')), + ], + options={ + 'verbose_name': 'Cloud Services Configuration', + }, + ), + migrations.CreateModel( + name='CompanyInformation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('company_name', models.CharField(default='SpecterOps', help_text='Company name handle to reference in reports', max_length=255)), + ('company_twitter', models.CharField(default='@specterops', help_text='Twitter handle to reference in reports', max_length=255, verbose_name='Twitter Handle')), + ('company_email', models.CharField(default='info@specterops.io', help_text='Email address to reference in reports', max_length=255)), + ], + options={ + 'verbose_name': 'Company Information', + }, + ), + migrations.CreateModel( + name='NamecheapConfiguration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enable', models.BooleanField(default=False)), + ('api_key', models.CharField(default='Namecheap API Key', max_length=255)), + ('username', models.CharField(default='Account Username', max_length=255)), + ('api_username', models.CharField(default='API Username', max_length=255)), + ('client_ip', models.CharField(default='Whitelisted IP Address', max_length=255)), + ('page_size', models.IntegerField(default=100)), + ], + options={ + 'verbose_name': 'Namecheap Configuration', + }, + ), + migrations.CreateModel( + name='ReportConfiguration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('border_weight', models.IntegerField(default=12700, help_text='Weight in EMUs – 12700 is equal to the 1pt weight in Word')), + ('border_color', models.CharField(default='2D2B6B', max_length=6, verbose_name='Picture Border Color')), + ('prefix_figure', models.CharField(default='–', help_text='Unicode character to place between `Figure` and your caption in Word reports', max_length=255, verbose_name='Character Before Figure Captions')), + ('prefix_table', models.CharField(default='–', help_text='Unicode character to place between `Table` and your table name in Word reports', max_length=255, verbose_name='Character Before Table Titles')), + ], + options={ + 'verbose_name': 'Global Report Configuration', + }, + ), + migrations.CreateModel( + name='SlackConfiguration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enable', models.BooleanField(default=False)), + ('webhook_url', models.CharField(default='https://hooks.slack.com/services/', max_length=255)), + ('slack_emoji', models.CharField(default=':ghost:', help_text='Emoji used for the avatar wrapped in colons', max_length=255)), + ('slack_channel', models.CharField(default='#ghostwriter', help_text='Default channel for Slack notifications', max_length=255)), + ('slack_username', models.CharField(default='Ghostwriter', help_text='Display name for the Slack bot', max_length=255)), + ('slack_alert_target', models.CharField(default='', help_text='Alert target for the notifications – blank for no target', max_length=255)), + ], + options={ + 'verbose_name': 'Slack Configuration', + }, + ), + migrations.CreateModel( + name='VirusTotalConfiguration', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('enable', models.BooleanField(default=False, help_text='Enable to allow domain health checks with VirusTotal')), + ('api_key', models.CharField(default='VirusTotal API Key', max_length=255)), + ('sleep_time', models.IntegerField(default=20, help_text='Sleep time in seconds – free API keys can only make 4 requests per minute')), + ], + options={ + 'verbose_name': 'VirusTotal Configuration', + }, + ), + ] diff --git a/ghostwriter/commandcenter/migrations/0002_auto_20201009_1918.py b/ghostwriter/commandcenter/migrations/0002_auto_20201009_1918.py new file mode 100644 index 000000000..6bbf738be --- /dev/null +++ b/ghostwriter/commandcenter/migrations/0002_auto_20201009_1918.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.3 on 2020-10-09 19:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('commandcenter', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='cloudservicesconfiguration', + name='ignore_tag', + field=models.CharField(default='gw_ignore', help_text='Ghostwriter will ignore cloud assets with one of these tags (comma-separated list)', max_length=255, verbose_name='Ignore Tags'), + ), + ] diff --git a/ghostwriter/commandcenter/migrations/0003_auto_20201027_1914.py b/ghostwriter/commandcenter/migrations/0003_auto_20201027_1914.py new file mode 100644 index 000000000..e8851b564 --- /dev/null +++ b/ghostwriter/commandcenter/migrations/0003_auto_20201027_1914.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.10 on 2020-10-27 19:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0018_auto_20201027_1914'), + ('commandcenter', '0002_auto_20201009_1918'), + ] + + operations = [ + migrations.AddField( + model_name='reportconfiguration', + name='default_docx_template', + field=models.ForeignKey(help_text='Select a default Word template', limit_choices_to={'doc_type__doc_type__iexact': 'docx'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='reportconfiguration_docx_set', to='reporting.ReportTemplate'), + ), + migrations.AddField( + model_name='reportconfiguration', + name='default_pptx_template', + field=models.ForeignKey(help_text='Select a default Word template', limit_choices_to={'doc_type__doc_type__iexact': 'pptx'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='reportconfiguration_pptx_set', to='reporting.ReportTemplate'), + ), + ] diff --git a/ghostwriter/commandcenter/migrations/0004_auto_20201028_1633.py b/ghostwriter/commandcenter/migrations/0004_auto_20201028_1633.py new file mode 100644 index 000000000..cee6e8135 --- /dev/null +++ b/ghostwriter/commandcenter/migrations/0004_auto_20201028_1633.py @@ -0,0 +1,30 @@ +# Generated by Django 3.0.10 on 2020-10-28 16:33 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0018_auto_20201027_1914'), + ('commandcenter', '0003_auto_20201027_1914'), + ] + + operations = [ + migrations.AddField( + model_name='reportconfiguration', + name='enable_borders', + field=models.BooleanField(default=False, help_text='Enable borders around images in Word documents'), + ), + migrations.AlterField( + model_name='reportconfiguration', + name='default_docx_template', + field=models.ForeignKey(blank=True, help_text='Select a default Word template', limit_choices_to={'doc_type__doc_type__iexact': 'docx'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='reportconfiguration_docx_set', to='reporting.ReportTemplate'), + ), + migrations.AlterField( + model_name='reportconfiguration', + name='default_pptx_template', + field=models.ForeignKey(blank=True, help_text='Select a default PowerPoint template', limit_choices_to={'doc_type__doc_type__iexact': 'pptx'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='reportconfiguration_pptx_set', to='reporting.ReportTemplate'), + ), + ] diff --git a/ghostwriter/commandcenter/migrations/0005_auto_20201102_2207.py b/ghostwriter/commandcenter/migrations/0005_auto_20201102_2207.py new file mode 100644 index 000000000..6b9cff890 --- /dev/null +++ b/ghostwriter/commandcenter/migrations/0005_auto_20201102_2207.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.10 on 2020-11-02 22:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('commandcenter', '0004_auto_20201028_1633'), + ] + + operations = [ + migrations.AddField( + model_name='reportconfiguration', + name='label_figure', + field=models.CharField(default='Figure', help_text='The label that comes before the figure number and caption in Word reports', max_length=255, verbose_name='Label Used for Figures'), + ), + migrations.AddField( + model_name='reportconfiguration', + name='label_table', + field=models.CharField(default='Table', help_text='The label that comes before the table number and caption in Word reports', max_length=255, verbose_name='Label Used for Tables'), + ), + migrations.AlterField( + model_name='reportconfiguration', + name='prefix_figure', + field=models.CharField(default='–', help_text='Unicode character to place between the label and your figure caption in Word reports', max_length=255, verbose_name='Character Before Figure Captions'), + ), + migrations.AlterField( + model_name='reportconfiguration', + name='prefix_table', + field=models.CharField(default='–', help_text='Unicode character to place between the label and your table caption in Word reports', max_length=255, verbose_name='Character Before Table Titles'), + ), + ] diff --git a/ghostwriter/commandcenter/migrations/__init__.py b/ghostwriter/commandcenter/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ghostwriter/commandcenter/models.py b/ghostwriter/commandcenter/models.py new file mode 100644 index 000000000..1e4ddf871 --- /dev/null +++ b/ghostwriter/commandcenter/models.py @@ -0,0 +1,263 @@ +"""This contains all of the database models for the CommandCenter application.""" + +# Django & Other 3rd Party Libraries +from django.db import models + +# Ghostwriter Libraries +from ghostwriter.singleton.models import SingletonModel + + +def sanitize(sensitive_thing): + """ + Sanitize the provided input and return for display in a template. + """ + sanitized_string = sensitive_thing + length = len(sensitive_thing) + if sensitive_thing: + if "http" in sensitive_thing: + # Split the URL – expecting a Slack (or other) webhook + sensitive_thing = sensitive_thing.split("/") + # Get just the last part for sanitization + webhook_tail = "".join(sensitive_thing[-1:]) + length = len(webhook_tail) + # Construct a sanitized string + sanitized_string = ( + "/".join(sensitive_thing[:-1]) + + "/" + + webhook_tail[0:4] + + "\u2717" * (length - 8) + + webhook_tail[length - 5 : length - 1] + ) + # Handle anything else that's long enough to be a key + elif length > 15: + sanitized_string = ( + sensitive_thing[0:4] + + "\u2717" * (length - 8) + + sensitive_thing[length - 5 : length - 1] + ) + return sanitized_string + + +class NamecheapConfiguration(SingletonModel): + enable = models.BooleanField(default=False) + api_key = models.CharField(max_length=255, default="Namecheap API Key") + username = models.CharField(max_length=255, default="Account Username") + api_username = models.CharField(max_length=255, default="API Username") + client_ip = models.CharField(max_length=255, default="Whitelisted IP Address") + page_size = models.IntegerField(default=100) + + def __str__(self): + return "Namecheap Configuration" + + class Meta: + verbose_name = "Namecheap Configuration" + + @property + def sanitized_api_key(self): + return sanitize(self.api_key) + + +class ReportConfiguration(SingletonModel): + enable_borders = models.BooleanField( + default=False, help_text="Enable borders around images in Word documents" + ) + border_weight = models.IntegerField( + default=12700, + help_text="Weight in EMUs – 12700 is equal to the 1pt weight in Word", + ) + border_color = models.CharField( + "Picture Border Color", max_length=6, default="2D2B6B" + ) + + prefix_figure = models.CharField( + "Character Before Figure Captions", + max_length=255, + default=u"\u2013", + help_text="Unicode character to place between the label and your figure caption in Word reports", + ) + label_figure = models.CharField( + "Label Used for Figures", + max_length=255, + default="Figure", + help_text="The label that comes before the figure number and caption in Word reports", + ) + prefix_table = models.CharField( + "Character Before Table Titles", + max_length=255, + default=u"\u2013", + help_text="Unicode character to place between the label and your table caption in Word reports", + ) + label_table = models.CharField( + "Label Used for Tables", + max_length=255, + default="Table", + help_text="The label that comes before the table number and caption in Word reports", + ) + # Foreign Keys + default_docx_template = models.ForeignKey( + "reporting.reporttemplate", + related_name="reportconfiguration_docx_set", + on_delete=models.SET_NULL, + limit_choices_to={ + "doc_type__doc_type__iexact": "docx", + }, + null=True, + blank=True, + help_text="Select a default Word template", + ) + default_pptx_template = models.ForeignKey( + "reporting.reporttemplate", + related_name="reportconfiguration_pptx_set", + on_delete=models.SET_NULL, + limit_choices_to={ + "doc_type__doc_type__iexact": "pptx", + }, + null=True, + blank=True, + help_text="Select a default PowerPoint template", + ) + + def __str__(self): + return "Global Report Configuration" + + class Meta: + verbose_name = "Global Report Configuration" + + @property + def border_color_rgb(self): + """ + Return the border color code as a list of RGB values. + """ + return tuple(int(self.color[i : i + 2], 16) for i in (0, 2, 4)) + + @property + def border_color_hex(self): + """ + Return the border color code as a list of hexadecimal. + """ + n = 2 + return tuple( + hex(int(self.color[i : i + n], 16)) for i in range(0, len(self.color), n) + ) + + +class SlackConfiguration(SingletonModel): + enable = models.BooleanField(default=False) + webhook_url = models.CharField( + max_length=255, default="https://hooks.slack.com/services/" + ) + slack_emoji = models.CharField( + max_length=255, + default=":ghost:", + help_text="Emoji used for the avatar wrapped in colons", + ) + slack_channel = models.CharField( + max_length=255, + default="#ghostwriter", + help_text="Default channel for Slack notifications", + ) + slack_username = models.CharField( + max_length=255, + default="Ghostwriter", + help_text="Display name for the Slack bot", + ) + slack_alert_target = models.CharField( + max_length=255, + default="", + help_text="Alert target for the notifications – blank for no target", + ) + + def __str__(self): + return "Slack Configuration" + + class Meta: + verbose_name = "Slack Configuration" + + @property + def sanitized_webhook(self): + return sanitize(self.webhook_url) + + +class CompanyInformation(SingletonModel): + company_name = models.CharField( + max_length=255, + default="SpecterOps", + help_text="Company name handle to reference in reports", + ) + company_twitter = models.CharField( + "Twitter Handle", + max_length=255, + default="@specterops", + help_text="Twitter handle to reference in reports", + ) + company_email = models.CharField( + max_length=255, + default="info@specterops.io", + help_text="Email address to reference in reports", + ) + + def __str__(self): + return "Company Information" + + class Meta: + verbose_name = "Company Information" + + +class CloudServicesConfiguration(SingletonModel): + enable = models.BooleanField( + default=False, help_text="Enable to allow the cloud monitoring task to run" + ) + aws_key = models.CharField( + "AWS Access Key", max_length=255, default="Your AWS Access Key" + ) + aws_secret = models.CharField( + "AWS Secret Key", max_length=255, default="Your AWS Secret Key" + ) + do_api_key = models.CharField( + "Digital Ocean API Key", max_length=255, default="Digital Ocean API Key" + ) + ignore_tag = models.CharField( + "Ignore Tags", + max_length=255, + default="gw_ignore", + help_text="Ghostwriter will ignore cloud assets with one of these tags (comma-separated list)", + ) + + def __str__(self): + return "Cloud Services Configuration" + + class Meta: + verbose_name = "Cloud Services Configuration" + + @property + def sanitized_aws_key(self): + return sanitize(self.aws_key) + + @property + def sanitized_aws_secret(self): + return sanitize(self.aws_secret) + + @property + def sanitized_do_api_key(self): + return sanitize(self.do_api_key) + + +class VirusTotalConfiguration(SingletonModel): + enable = models.BooleanField( + default=False, help_text="Enable to allow domain health checks with VirusTotal" + ) + api_key = models.CharField(max_length=255, default="VirusTotal API Key") + sleep_time = models.IntegerField( + default=20, + help_text="Sleep time in seconds – free API keys can only make 4 requests per minute", + ) + + def __str__(self): + return "VirusTotal Configuration" + + class Meta: + verbose_name = "VirusTotal Configuration" + + @property + def sanitized_api_key(self): + return sanitize(self.api_key) diff --git a/ghostwriter/home/consumers.py b/ghostwriter/home/consumers.py index f28084aab..3617077f5 100644 --- a/ghostwriter/home/consumers.py +++ b/ghostwriter/home/consumers.py @@ -4,8 +4,7 @@ import json # Django & Other 3rd Party Libraries -from channels.generic.websocket import AsyncWebsocketConsumer, WebsocketConsumer -from django.contrib import messages +from channels.generic.websocket import AsyncWebsocketConsumer class UserConsumer(AsyncWebsocketConsumer): @@ -14,15 +13,22 @@ class UserConsumer(AsyncWebsocketConsumer): """ async def connect(self): - self.username = self.scope["url_route"]["kwargs"]["username"] - self.user_group_name = "notify_%s" % self.username - # Join user group - await self.channel_layer.group_add(self.user_group_name, self.channel_name) - await self.accept() + self.user = self.scope["user"] + if self.user.is_active: + self.username = self.scope["url_route"]["kwargs"]["username"] + self.user_group_name = "notify_%s" % self.username + # Join user group + await self.channel_layer.group_add(self.user_group_name, self.channel_name) + await self.accept() async def disconnect(self, close_code): - # Leave user group - await self.channel_layer.group_discard(self.user_group_name, self.channel_name) + if self.user.is_active: + # Leave user group + await self.channel_layer.group_discard( + self.user_group_name, self.channel_name + ) + else: + pass # Receive message from WebSocket async def receive(self, text_data): @@ -46,7 +52,12 @@ async def task(self, event): assignments = event["assignments"] # Send message to WebSocket await self.send( - text_data=json.dumps({"message": message, "assignments": assignments,}) + text_data=json.dumps( + { + "message": message, + "assignments": assignments, + } + ) ) @@ -56,11 +67,15 @@ class ProjectConsumer(AsyncWebsocketConsumer): """ async def connect(self): - self.project_id = self.scope["url_route"]["kwargs"]["project_id"] - self.project_group_name = "notify_%s" % self.project_id - # Join project group - await self.channel_layer.group_add(self.project_group_name, self.channel_name) - await self.accept() + user = self.scope["user"] + if user.is_active: + self.project_id = self.scope["url_route"]["kwargs"]["project_id"] + self.project_group_name = "notify_%s" % self.project_id + # Join project group + await self.channel_layer.group_add( + self.project_group_name, self.channel_name + ) + await self.accept() async def disconnect(self, close_code): # Leave project group diff --git a/ghostwriter/home/forms.py b/ghostwriter/home/forms.py index 08c6dd546..157128e48 100644 --- a/ghostwriter/home/forms.py +++ b/ghostwriter/home/forms.py @@ -25,7 +25,6 @@ class Meta: model = UserProfile exclude = ("user",) widgets = { - "user": forms.HiddenInput(), "avatar": forms.ClearableFileInput(), } diff --git a/ghostwriter/home/routing.py b/ghostwriter/home/routing.py index f6fed7324..5c7681776 100644 --- a/ghostwriter/home/routing.py +++ b/ghostwriter/home/routing.py @@ -3,6 +3,7 @@ # Django & Other 3rd Party Libraries from django.urls import re_path +# Ghostwriter Libraries from . import consumers websocket_urlpatterns = [ diff --git a/ghostwriter/home/templates/home/management.html b/ghostwriter/home/templates/home/management.html index ea28a333b..c7a1498a4 100644 --- a/ghostwriter/home/templates/home/management.html +++ b/ghostwriter/home/templates/home/management.html @@ -1,8 +1,27 @@ {% extends "base_generic.html" %} +{% load settings_tags %} + {% block pagetitle %}Manage API Access{% endblock %} +{% block breadcrumbs %} + +{% endblock %} + {% block content %} + {% comment %} Get the current configuration from the singleton models {% endcomment %} + {% get_solo "commandcenter.CompanyInformation" as company_config %} + {% get_solo "commandcenter.CloudServicesConfiguration" as cloud_config %} + {% get_solo "commandcenter.NamecheapConfiguration" as namecheap_config %} + {% get_solo "commandcenter.ReportConfiguration" as report_config %} + {% get_solo "commandcenter.SlackConfiguration" as slack_config %} + {% get_solo "commandcenter.VirusTotalConfiguration" as vt_config %} +

API & Notification Configurations

-

These settings are managed in your .django environment variables file and config/settings/base.py. Changing them will require restarting the application.

- +

These settings are managed in the Django administration panel under the Command Center application.

+

To change the Timezone, edit the application's settings file and restart the server. +

@@ -27,15 +47,15 @@

API & Notification Configurations

- + - + - + @@ -50,15 +70,15 @@

API & Notification Configurations

- {% if virustotal_api_key and virustotal_api_key != "None" %} - + {% if vt_config.enable %} + {% else %} - + {% endif %} - + @@ -71,35 +91,35 @@

API & Notification Configurations

- {% if enable_namecheap %} + {% if namecheap_config.enable %} - + - + - + - + - + - + {% else %} - + {% endif %} @@ -108,32 +128,32 @@

API & Notification Configurations

- + - {% if enable_cloud_monitor %} + {% if cloud_config.enable %} - + - + - + - + {% else %} - + {% endif %} @@ -147,47 +167,89 @@

API & Notification Configurations

- {% if enable_slack %} + {% if slack_config.enable %} - + - + - + - + - + - + {% else %} - + {% endif %}
General Settings
Company Name{{ company_name }}{{ company_config.company_name }}
Twitter{{ company_twitter }}{{ company_config.company_twitter }}
Email{{ company_email }}{{ company_config.company_email }}
VirusTotal API Key{{ virustotal_api_key }}{{ vt_config.sanitized_api_key }}Not ConfiguredDisabled
Sleep Time{{ sleep_time }} seconds{{ vt_config.sleep_time }} seconds
Domain Registrar API
Namecheap API Enabled{{ enable_namecheap }}{{ namecheap_config.enable }}
Namecheap Whitelisted IP{{ namecheap_client_ip }}{{ namecheap_config.client_ip }}
Namecheap API Key{{ namecheap_api_key }}{{ namecheap_config.sanitized_api_key }}
Namecheap Username{{ namecheap_username }}{{ namecheap_config.username }}
Namecheap API Username{{ namecheap_api_username }}{{ namecheap_config.api_username }}
Namecheap Page Size{{ namecheap_page_size }}{{ namecheap_config.page_size }}
Namecheap API Enabled{{ enable_namecheap }}Disabled
 
Cloud Services
Cloud Monitoring Enabled{{ enable_cloud_monitor }}{{ cloud_config.enable }}
AWS Access Key{{ aws_key }}{{ cloud_config.sanitized_aws_key }}
> AWS Access Key Secret{{ aws_secret }}{{ cloud_config.sanitized_aws_secret }}
Digital Ocean API Key{{ do_api_key }}{{ cloud_config.sanitized_do_api_key }}
Cloud Monitoring Enabled{{ enable_cloud_monitor }}Disabled
Notifications
Slack Enabled{{ enable_slack }}{{ slack_config.enable }}
Slack WebHook{{ slack_webhook_url }}{{ slack_config.sanitized_webhook }}
Slack Bot Name{{ slack_username }}{{ slack_config.slack_username }}
Slack Bot Avatar{{ slack_emoji }}{{ slack_config.slack_emoji }}
Global Slack Channel{{ slack_channel }}{{ slack_config.slack_channel }}
Slack Target{{ slack_alert_target }}{{ slack_config.slack_alert_target }}
Slack Enabled{{ enable_slack }}Disabled
- {% if enable_slack %} - Test Configuration -
+
Test Configurations
+
+ +
+ +
+{% endblock %} -
- {% csrf_token %} - - -
- {% endif %} +{% block morescripts %} + + {% endblock %} diff --git a/ghostwriter/home/templates/home/profile.html b/ghostwriter/home/templates/home/profile.html index 5473bfd58..94987cb94 100644 --- a/ghostwriter/home/templates/home/profile.html +++ b/ghostwriter/home/templates/home/profile.html @@ -1,5 +1,6 @@ {% extends "base_generic.html" %} {% load crispy_forms_tags %} +{% load custom_tags %} {% block pagetitle %}User Profile{% endblock %} @@ -23,7 +24,7 @@ - + @@ -33,7 +34,7 @@ - + + + + +
Username Username {{ user.username }}
Full Name{{ user.email }}
Access Access {% if user.is_superuser %} Superuser, Staff @@ -44,6 +45,12 @@ {% endif %}
Groups + {{ user|get_groups }} +
diff --git a/ghostwriter/home/templatetags/custom_tags.py b/ghostwriter/home/templatetags/custom_tags.py index cf27c3050..1faae13f5 100644 --- a/ghostwriter/home/templatetags/custom_tags.py +++ b/ghostwriter/home/templatetags/custom_tags.py @@ -23,6 +23,19 @@ def has_group(user, group_name): return True if group in user.groups.all() else False +@register.filter(name="get_groups") +def get_groups(user): + """ + Collect a list of all memberships in :model:`django.contrib.auth.Group` for + an individual :model:`users.User`. + """ + groups = Group.objects.filter(user=user) + group_list = [] + for group in groups: + group_list.append(group.name) + return ", ".join(group_list) + + @register.simple_tag def count_assignments(request): """ diff --git a/ghostwriter/home/urls.py b/ghostwriter/home/urls.py index b1343ace1..03a9918d1 100644 --- a/ghostwriter/home/urls.py +++ b/ghostwriter/home/urls.py @@ -4,21 +4,43 @@ from django.urls import path # Ghostwriter Libraries -from ghostwriter.home.views import ( - dashboard, - management, - profile, - send_slack_test_msg, - upload_avatar, -) +import ghostwriter.home.views as views app_name = "home" # URLs for the basic views urlpatterns = [ - path("", dashboard, name="dashboard"), - path("profile/", profile, name="profile"), - path("management/", management, name="management"), - path("management/test_slack", send_slack_test_msg, name="send_slack_test_msg"), - path("profile/avatar", upload_avatar, name="upload_avatar"), + path("", views.dashboard, name="dashboard"), + path("profile/", views.profile, name="profile"), + path("management/", views.Management.as_view(), name="management"), + path("profile/avatar", views.upload_avatar, name="upload_avatar"), +] + +# URLs for AJAX test functions +urlpatterns += [ + path( + "ajax/management/test/aws", + views.TestAWSConnection.as_view(), + name="ajax_test_aws", + ), + path( + "ajax/management/test/do", + views.TestDOConnection.as_view(), + name="ajax_test_do", + ), + path( + "ajax/management/test/namecheap", + views.TestNamecheapConnection.as_view(), + name="ajax_test_namecheap", + ), + path( + "ajax/management/test/slack", + views.TestSlackConnection.as_view(), + name="ajax_test_slack", + ), + path( + "ajax/management/test/virustotal", + views.TestVirusTotalConnection.as_view(), + name="ajax_test_virustotal", + ), ] diff --git a/ghostwriter/home/views.py b/ghostwriter/home/views.py index f1ff906f3..b9996ad21 100644 --- a/ghostwriter/home/views.py +++ b/ghostwriter/home/views.py @@ -7,13 +7,15 @@ # Django & Other 3rd Party Libraries from django.conf import settings from django.contrib import messages -from django.contrib.admin.views.decorators import staff_member_required from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required +from django.contrib.auth.mixins import LoginRequiredMixin, UserPassesTestMixin from django.db.models import Q -from django.http import HttpResponseRedirect +from django.http import JsonResponse from django.shortcuts import redirect, render from django.urls import reverse +from django.views.static import serve +from django.views.generic.edit import View from django_q.models import Task from django_q.tasks import async_task @@ -34,6 +36,14 @@ ################## +@login_required +def protected_serve(request, path, document_root=None, show_indexes=False): + """ + Serve static files from ``MEDIA_ROOT`` for authenticated requests. + """ + return serve(request, path, document_root, show_indexes) + + @login_required def dashboard(request): """ @@ -42,11 +52,11 @@ def dashboard(request): **Context** ``user_projects`` - Active :model:`reporting.ProjectAssignment` for current :model:`users.User`. + Active :model:`reporting.ProjectAssignment` for current :model:`users.User` ``upcoming_projects`` - Future :model:`reporting.ProjectAssignment` for current :model:`users.User`. + Future :model:`reporting.ProjectAssignment` for current :model:`users.User` ``recent_tasks`` - Five most recent :model:`django_q.Task` entries. + Five most recent :model:`django_q.Task` entries ``user_tasks`` Incomplete :model:`reporting.ReportFindingLink` for current :model:`users.User` @@ -67,7 +77,11 @@ def dashboard(request): # Get active :model:`reporting.ProjectAssignment` for current :model:`users.User` user_projects = ProjectAssignment.objects.select_related( "project", "project__client", "role" - ).filter(Q(operator=request.user) & Q(start_date__lte=datetime.datetime.now())) + ).filter( + Q(operator=request.user) + & Q(start_date__lte=datetime.datetime.now()) + & Q(end_date__gte=datetime.datetime.now()) + ) # Get future :model:`reporting.ProjectAssignment` for current :model:`users.User` upcoming_project = ProjectAssignment.objects.select_related( "project", "project__client", "role" @@ -127,159 +141,204 @@ def upload_avatar(request): ) -@login_required -@staff_member_required -def management(request): +class Management(LoginRequiredMixin, UserPassesTestMixin, View): """ Display the current Ghostwriter settings. **Context** - ``company_name `` - The current value of ``settings.COMPANY_NAME``. - ``company_twitter`` - The current value of ``settings.COMPANY_TWITTER``. - ``company_email`` - The current value of ``settings.COMPANY_EMAIL``. ``timezone`` - The current value of ``settings.TIME_ZONE``. - ``sleep_time`` - The associated value from ``settings.DOMAINCHECK_CONFIG``. - ``virustotal_api_key`` - The associated value from ``settings.DOMAINCHECK_CONFIG``. - ``slack_emoji`` - The associated value from ``settings.SLACK_CONFIG``. - ``enable_slack`` - The associated value from ``settings.SLACK_CONFIG``. - ``slack_channel`` - The associated value from ``settings.SLACK_CONFIG``. - ``slack_username`` - The associated value from ``settings.SLACK_CONFIG``. - ``slack_webhook_url`` - The associated value from ``settings.SLACK_CONFIG``. - ``slack_alert_target`` - The associated value from ``settings.SLACK_CONFIG``. - ``namecheap_client_ip`` - The associated value from ``settings.NAMECHEAP_CONFIG``. - ``enable_namecheap`` - The associated value from ``settings.NAMECHEAP_CONFIG``. - ``namecheap_api_key`` - The associated value from ``settings.NAMECHEAP_CONFIG``. - ``namecheap_username`` - The associated value from ``settings.NAMECHEAP_CONFIG``. - ``namecheap_page_size`` - The associated value from ``settings.NAMECHEAP_CONFIG``. - ``namecheap_api_username`` - The associated value from ``settings.NAMECHEAP_CONFIG``. - ``enable_cloud_monitor`` - The associated value from ``settings.CLOUD_SERVICE_CONFIG``. - ``aws_key`` - The associated value from ``settings.CLOUD_SERVICE_CONFIG``. - ``aws_secret`` - The associated value from ``settings.CLOUD_SERVICE_CONFIG``. - ``do_api_key`` - The associated value from ``settings.CLOUD_SERVICE_CONFIG``. + The current value of ``settings.TIME_ZONE`` **Template** :template:`home/management.html` """ - """View function to display the current settings configured for - Ghostwriter. + def test_func(self): + return self.request.user.is_staff + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that") + return redirect("home:dashboard") + + def get(self, request, *args, **kwargs): + context = { + "timezone": settings.TIME_ZONE, + } + return render(request, "home/management.html", context=context) + + +class TestAWSConnection(LoginRequiredMixin, UserPassesTestMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``AWS Test`` with + :task:`shepherd.tasks.test_aws_keys` to test AWS keys in + :model:`commandcenter.CloudServicesConfiguration`. """ - # Get the *_CONFIG dictionaries from settings.py - config = {} - config.update(settings.SLACK_CONFIG) - config.update(settings.NAMECHEAP_CONFIG) - config.update(settings.DOMAINCHECK_CONFIG) - config.update(settings.CLOUD_SERVICE_CONFIG) - - def sanitize(sensitive_thing): - """ - Sanitize the provided input and return for display in the template. - """ - sanitized_string = sensitive_thing - length = len(sensitive_thing) - if sensitive_thing: - if "http" in sensitive_thing: - # Split the URL – expecting a Slack (or other) webhook - sensitive_thing = sensitive_thing.split("/") - # Get just the last part for sanitization - webhook_tail = "".join(sensitive_thing[-1:]) - length = len(webhook_tail) - # Construct a sanitized string - sanitized_string = ( - "/".join(sensitive_thing[:-1]) - + "/" - + webhook_tail[0:4] - + "\u2717" * (length - 8) - + webhook_tail[length - 5 : length - 1] - ) - # Handle anything else that's long enough to be a key - elif length > 15: - sanitized_string = ( - sensitive_thing[0:4] - + "\u2717" * (length - 8) - + sensitive_thing[length - 5 : length - 1] - ) - return sanitized_string - - # Pass the relevant settings to management.html - context = { - "company_name": settings.COMPANY_NAME, - "company_twitter": settings.COMPANY_TWITTER, - "company_email": settings.COMPANY_EMAIL, - "timezone": settings.TIME_ZONE, - "sleep_time": config["sleep_time"], - "slack_emoji": config["slack_emoji"], - "enable_slack": config["enable_slack"], - "slack_channel": config["slack_channel"], - "slack_username": config["slack_username"], - "slack_webhook_url": sanitize(config["slack_webhook_url"]), - "virustotal_api_key": sanitize(config["virustotal_api_key"]), - "slack_alert_target": config["slack_alert_target"], - "namecheap_client_ip": config["client_ip"], - "enable_namecheap": config["enable_namecheap"], - "namecheap_api_key": sanitize(config["namecheap_api_key"]), - "namecheap_username": config["namecheap_username"], - "namecheap_page_size": config["namecheap_page_size"], - "namecheap_api_username": config["namecheap_api_username"], - "enable_cloud_monitor": config["sleep_time"], - "aws_key": sanitize(config["aws_key"]), - "aws_secret": sanitize(config["aws_secret"]), - "do_api_key": sanitize(config["do_api_key"]), - } - return render(request, "home/management.html", context=context) + def test_func(self): + return self.request.user.is_staff -@login_required -@staff_member_required -def send_slack_test_msg(request): + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``AWS Test`` + result = "success" + try: + task_id = async_task( + "ghostwriter.shepherd.tasks.test_aws_keys", + self.request.user, + group="AWS Test", + ) + message = "AWS access key test has been successfully queued" + except Exception: + result = "error" + message = "AWS access key test could not be queued" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + + +class TestDOConnection(LoginRequiredMixin, UserPassesTestMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``Digital Ocean Test`` with + :task:`shepherd.tasks.test_digital_ocean` to test the Digital Ocean API key stored in + :model:`commandcenter.CloudServicesConfiguration`. """ - Create an individual :model:`django_q.Task` to test sending Slack messages. - **Template** + def test_func(self): + return self.request.user.is_staff - :template:`home/management.html` + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Digital Ocean Test`` + result = "success" + try: + task_id = async_task( + "ghostwriter.shepherd.tasks.test_digital_ocean", + self.request.user, + group="Digital Ocean Test", + ) + message = "Digital Ocean API key test has been successfully queued" + except Exception: + result = "error" + message = "Digital Ocean API key test could not be queued" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + + +class TestNamecheapConnection(LoginRequiredMixin, UserPassesTestMixin, View): """ - # Check if the request is a POST and proceed with the task - if request.method == "POST": - # Add an async task grouped as `Test Slack Message` + Create an individual :model:`django_q.Task` under group ``Namecheap Test`` with + :task:`shepherd.tasks.test_namecheap` to test the Namecheap API configuration stored + in :model:`commandcenter.NamecheapConfiguration`. + """ + + def test_func(self): + return self.request.user.is_staff + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Namecheap Test`` + result = "success" try: task_id = async_task( - "ghostwriter.shepherd.tasks.send_slack_test_msg", - group="Test Slack Message", + "ghostwriter.shepherd.tasks.test_namecheap", + self.request.user, + group="Namecheap Test", ) - messages.success( - request, - "Test Slack message has been successfully queued.", - extra_tags="alert-success", + message = "Namecheap API test has been successfully queued" + except Exception: + result = "error" + message = "Namecheap API test could not be queued" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + + +class TestSlackConnection(LoginRequiredMixin, UserPassesTestMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``Slack Test`` with + :task:`shepherd.tasks.test_slack_webhook` to test the Slack Webhook configuration + stored in :model:`commandcenter.SlackConfiguration`. + """ + + def test_func(self): + return self.request.user.is_staff + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``Slack Test`` + result = "success" + try: + task_id = async_task( + "ghostwriter.shepherd.tasks.test_slack_webhook", + self.request.user, + group="Slack Test", ) + message = "Slack Webhook test has been successfully queued" except Exception: - messages.error( - request, - "Test Slack message task could not be queued. Is the AMQP server running?", - extra_tags="alert-danger", + result = "error" + message = "Slack Webhook test could not be queued" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) + + +class TestVirusTotalConnection(LoginRequiredMixin, UserPassesTestMixin, View): + """ + Create an individual :model:`django_q.Task` under group ``VirusTotal Test`` with + :task:`shepherd.tasks.test_virustotal` to test the VirusTotal API key stored in + :model:`commandcenter.SlackConfiguration`. + """ + + def test_func(self): + return self.request.user.is_staff + + def handle_no_permission(self): + messages.error(self.request, "You do not have permission to access that") + return redirect("home:dashboard") + + def post(self, request, *args, **kwargs): + # Add an async task grouped as ``VirusTotal Test`` + result = "success" + try: + task_id = async_task( + "ghostwriter.shepherd.tasks.test_virustotal", + self.request.user, + group="Slack Test", ) - return HttpResponseRedirect(reverse("home:management")) + message = "VirusTotal API test has been successfully queued" + except Exception: + result = "error" + message = "VirusTotal API test could not be queued" + + data = { + "result": result, + "message": message, + } + return JsonResponse(data) diff --git a/ghostwriter/modules/dns.py b/ghostwriter/modules/dns.py deleted file mode 100644 index 2ed43b15e..000000000 --- a/ghostwriter/modules/dns.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -"""This module contains the tools required for collecting and parsing -DNS records. -""" - -import dns.resolver - - -class DNSCollector(object): - """Class to retrieve DNS records and perform some basic analysis.""" - # Setup a DNS resolver so a timeout can be set - # No timeout means a very, very long wait if a domain has no records - resolver = dns.resolver.Resolver() - resolver.timeout = 1 - resolver.lifetime = 1 - - def __init__(self): - """Everything that should be initiated with a new object goes here.""" - pass - - def get_dns_record(self, domain, record_type): - """Collect the specified DNS record type for the target domain. - - Parameters: - domain The domain to be used for DNS record collection - record_type The DNS record type to collect - """ - answer = self.resolver.query(domain, record_type) - return answer - - def parse_dns_answer(self, dns_record): - """Parse the provided DNS record and return a list containing each item. - - Parameters: - dns_record The DNS record to be parsed - """ - temp = [] - for rdata in dns_record.response.answer: - for item in rdata.items: - temp.append(item.to_text()) - return ", ".join(temp) - - def return_dns_record_list(self, domain, record_type): - """Collect and parse a DNS record for the given domain and DNS record - type and then return a list. - - Parameters: - domain The domain to be used for DNS record collection - record_type The DNS record type to collect - """ - record = self.get_dns_record(domain, record_type) - return self.parse_dns_answer(record) diff --git a/ghostwriter/modules/dns_toolkit.py b/ghostwriter/modules/dns_toolkit.py new file mode 100644 index 000000000..e7f9f8f6e --- /dev/null +++ b/ghostwriter/modules/dns_toolkit.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +This module contains the tools required for collecting and parsing DNS records. +""" + +# Standard Libraries +import asyncio +import logging +from asyncio import Semaphore +from typing import Union + +# Django & Other 3rd Party Libraries +from dns import asyncresolver +from dns.resolver import NXDOMAIN, Answer, NoAnswer + +# Using __name__ resolves to ghostwriter.modules.dns_toolkit +logger = logging.getLogger(__name__) + + +class DNSCollector(object): + """ + Retrieve and parse DNS records asynchronously. + + **Parameters** + + ``concurrent_limit`` + Set limit on number of concurrent DNS requests to avoid hitting system limits + """ + + # Configure the DNS resolver to be asynchronous and use specific nameservers + resolver = asyncresolver.Resolver() + resolver.lifetime = 1 + resolver.nameservers = ["8.8.8.8", "8.8.4.4", "1.1.1.1"] + + def __init__(self, concurrent_limit=50): + # Limit used for Semaphore to avoid hitting system limits on open requests + self.semaphore = Semaphore(value=concurrent_limit) + + async def _query( + self, domain: str, record_type: str + ) -> Union[Answer, NXDOMAIN, NoAnswer]: + """ + Execute a DNS query for the target domain and record type. + + **Parameters** + + ``domain`` + Domain to be used for DNS record collection + ``record_type`` + DNS record type to collect + """ + try: + # Wait to acquire the semaphore to avoid too many concurrent DNS requests + await self.semaphore.acquire() + answer = await self.resolver.resolve(domain, record_type) + except Exception as e: + answer = e + # Release semaphore to allow next request + self.semaphore.release() + return answer + + async def _parse_answer(self, dns_record: Answer) -> list: + """ + Parse the provided instance of ``dns.resolver.Answer``. + + **Parameters** + + ``dns_record`` + Instance of ``dns.resolve.Answer`` + """ + record_list = [] + for rdata in dns_record.response.answer: + for item in rdata.items: + record_list.append(item.to_text()) + return record_list + + async def _fetch_record(self, domain: str, record_type: str) -> dict: + """ + Fetch a DNS record for the given domain and record type. + + **Parameters** + + ``domain`` + Domain to be used for DNS record collection + ``record_type`` + DNS record type to collect (A, NS, SOA, TXT, MX, CNAME, DMARC) + """ + logger.debug("Fetching %s records for %s", record_type, domain) + # Prepare the results dictionary + result = {} + result[domain] = {} + result[domain]["domain"] = domain + + # Handle DMARC as a special record type + if record_type.lower() == "dmarc": + record_type = "A" + query_type = "dmarc_record" + query_domain = "_dmarc." + domain + else: + query_type = record_type.lower() + "_record" + query_domain = domain + + # Execute query and await completion + response = await self._query(query_domain, record_type) + + # Only parse result if it's an ``Answer`` + if isinstance(response, Answer): + record = await self._parse_answer(response) + result[domain][query_type] = record + else: + # Return the type of exception (e.g., NXDOMAIN) + result[domain][query_type] = type(response).__name__ + return result + + async def _prepare_async_dns(self, domains: list, record_types: list) -> list: + """ + Prepare asynchronous DNS queries for a list of domain names. + + **Parameters** + + ``domains`` + Queryset of :model:`shepherd.Domain` entries + ``record_types`` + List of record types represented as strings (e.g., ["A", "TXT"]) + """ + tasks = [] + # For each domain, create a task for each DNS record of interest + for domain in domains: + for record_type in record_types: + tasks.append( + self._fetch_record(domain=domain.name, record_type=record_type) + ) + # Gather all tasks for execution + all_tasks = await asyncio.gather(*tasks) + return all_tasks + + def run_async_dns(self, domains: list, record_types: list) -> dict: + """ + Execute asynchronous DNS queries for a list of domain names. + + **Parameters** + + ``domains`` + List of domain names + ``record_types`` + List of record types represented as strings (e.g., ["A", "TXT"]) + """ + # Setup an event loop + event_loop = asyncio.get_event_loop() + # Use an event loop (instead of ``asyncio.run()``) to easily get list of results + results = event_loop.run_until_complete( + self._prepare_async_dns(domains=domains, record_types=record_types) + ) + # Result is a list of dicts – seven for each domain name + combined = {} + # Combine all dicts with the same domain name + for res in results: + for key, value in res.items(): + if key in combined: + combined[key].update(value) + else: + combined[key] = {} + combined[key].update(value) + return combined diff --git a/ghostwriter/modules/exceptions.py b/ghostwriter/modules/exceptions.py new file mode 100644 index 000000000..9088b92a5 --- /dev/null +++ b/ghostwriter/modules/exceptions.py @@ -0,0 +1,5 @@ +"""This contains all of the custom exceptions for the Ghostwriter application.""" + + +class MissingTemplate(Exception): + pass diff --git a/ghostwriter/modules/reportwriter.py b/ghostwriter/modules/reportwriter.py index 4cf58dbc4..6946c7f87 100644 --- a/ghostwriter/modules/reportwriter.py +++ b/ghostwriter/modules/reportwriter.py @@ -12,31 +12,62 @@ import json import logging import os +import random import re +from datetime import datetime # Django & Other 3rd Party Libraries import docx +import jinja2 import pptx from bs4 import BeautifulSoup, NavigableString from django.conf import settings from django.core.serializers.json import DjangoJSONEncoder +from docxtpl import DocxTemplate, RichText from docx.enum.dml import MSO_THEME_COLOR_INDEX -from docx.enum.text import WD_ALIGN_PARAGRAPH +from docx.enum.text import WD_ALIGN_PARAGRAPH, WD_COLOR_INDEX +from docx.opc.exceptions import PackageNotFoundError as DocxPackageNotFoundError from docx.oxml.shared import OxmlElement, qn +from docx.enum.style import WD_STYLE_TYPE from docx.shared import Inches, Pt, RGBColor -from docxtpl import DocxTemplate +from jinja2.exceptions import TemplateSyntaxError from pptx import Presentation -from pptx.enum.text import MSO_ANCHOR, PP_ALIGN +from pptx.dml.color import RGBColor as PptxRGBColor +from pptx.enum.text import MSO_ANCHOR, MSO_AUTO_SIZE, PP_ALIGN +from pptx.exc import PackageNotFoundError as PptxPackageNotFoundError from xlsxwriter.workbook import Workbook +# Ghostwriter Libraries +from ghostwriter.commandcenter.models import CompanyInformation, ReportConfiguration + # Using __name__ resolves to ghostwriter.modules.reporting logger = logging.getLogger(__name__) +class ReportConstants: + """Constant values used for report generation.""" + + DEFAULT_STYLE_VALUES = { + "bold": False, + "underline": False, + "italic": False, + "inline_code": False, + "strikethrough": False, + "font_family": None, + "font_size": None, + "font_color": None, + "highlight": None, + "superscript": False, + "subscript": False, + "hyperlink": False, + "hyperlink_url": None, + } + + class Reportwriter: """Generate report documents in Microsoft Office formats.""" - # allowlisted HTML tags expected to come from the WYSIWYG + # Allowlist for HTML tags expected to come from the WYSIWYG tag_allowlist = [ "code", "span", @@ -50,39 +81,12 @@ class Reportwriter: "u", "b", "pre", + "sub", + "sup", + "del", ] - # Track report type for different Office XML - report_type = None - - # Color codes used for finding severity in all reports - - # Blue - informational_color = "8eaadb" - informational_color_hex = [0x83, 0xAA, 0xDB] - # Green - low_color = "a8d08d" - low_color_hex = [0xA8, 0xD0, 0x8D] - # Orange - medium_color = "f4b083" - medium_color_hex = [0xF4, 0xB0, 0x83] - # Red - high_color = "ff7e79" - high_color_hex = [0xFF, 0x7E, 0x79] - # Purple - critical_color = "966FD6" - critical_color_hex = [0x96, 0x6F, 0xD6] - - # Picture border settings for Word - - # Picture border color - border_color = "2d2b6b" - border_color_hex = [0x45, 0x43, 0x107] - - # Picture border weight – 12700 is equal to the 1pt weight in Word - border_weight = "12700" - - # Extensions allowed for evidence + # Allowlist for evidence file extensions / filetypes image_extensions = ["png", "jpeg", "jpg"] text_extensions = ["txt", "ps1", "py", "md", "log"] @@ -92,6 +96,31 @@ def __init__(self, report_queryset, output_path, evidence_path, template_loc=Non self.evidence_path = evidence_path self.report_queryset = report_queryset + # Get the global report configuration + global_report_config = ReportConfiguration.get_solo() + self.company_config = CompanyInformation.get_solo() + + # Track report type for different Office XML + self.report_type = None + + # Picture border settings for Word + self.enable_borders = global_report_config.enable_borders + self.border_color = global_report_config.border_color + self.border_weight = global_report_config.border_weight + + # Caption options + prefix_figure = global_report_config.prefix_figure.strip() + self.prefix_figure = f" {prefix_figure} " + label_figure = global_report_config.label_figure.strip() + self.label_figure = f"{label_figure} " + label_table = global_report_config.label_table.strip() + self.label_table = f"{label_table} " + + # Setup Jinja2 rendering environment + custom filters + self.jinja_env = jinja2.Environment(extensions=["jinja2.ext.debug"]) + self.jinja_env.filters["filter_severity"] = self.filter_severity + self.jinja_env.filters["strip_html"] = self.strip_html + logger.info( "Generating a report for %s using the template at %s and referencing the evidence in %s", self.report_queryset, @@ -99,6 +128,43 @@ def __init__(self, report_queryset, output_path, evidence_path, template_loc=Non self.evidence_path, ) + def filter_severity(self, findings, allowlist): + """ + Filter list of findings to return only those with a severity in the allowlist. + + **Parameters** + + ``findings`` + List of dictionary objects (JSON) for findings + ``allowlist`` + List of strings matching severity categories to allow through filter + """ + filtered_values = [] + allowlist = [severity.lower() for severity in allowlist] + for finding in findings: + if finding["severity"].lower() in allowlist: + filtered_values.append(finding) + return filtered_values + + def strip_html(self, s): + """ + Strip HTML tags from the provided HTML while preserving newlines created by + ``
`` and ``

`` tags and spaces. + + **Parameters** + + ``s`` + String of HTML text to strip down + """ + html = BeautifulSoup(s, "lxml") + output = "" + for tag in html.descendants: + if isinstance(tag, str): + output += tag + elif tag.name == "br" or tag.name == "p": + output += "\n" + return output + def valid_xml_char_ordinal(self, c): """ Clean string to make all characters XML compatible for Word documents. @@ -108,7 +174,7 @@ def valid_xml_char_ordinal(self, c): **Parameters** - ``c`` + ``c`` : string String of characters to validate """ codepoint = ord(c) @@ -148,19 +214,61 @@ def generate_json(self): report_dict["project"] = {} report_dict["project"]["id"] = self.report_queryset.project.id report_dict["project"]["name"] = project_name - report_dict["project"]["start_date"] = self.report_queryset.project.start_date - report_dict["project"]["end_date"] = self.report_queryset.project.end_date report_dict["project"]["codename"] = self.report_queryset.project.codename report_dict["project"][ "project_type" ] = self.report_queryset.project.project_type.project_type report_dict["project"]["note"] = self.report_queryset.project.note + # Project dates + start_datetime = self.report_queryset.project.start_date + end_datetime = self.report_queryset.project.end_date + start_month = start_datetime.strftime("%B") + start_day = start_datetime.day + start_year = start_datetime.year + end_month = end_datetime.strftime("%B") + end_day = end_datetime.day + end_year = end_datetime.year + if start_month == end_month: + if start_year == end_year: + execution_window = f"{start_month} {start_day}-{end_day}, {start_year}" + execution_window_uk = ( + f"{start_day}-{end_day} {start_month} {start_year}" + ) + else: + execution_window = f"{start_month} {start_day}, {start_year} - {end_month} {end_day}, {end_year}" + execution_window_uk = f"{start_day} {start_month} {start_year} - {end_day} {end_month} {end_year}" + else: + if start_year == end_year: + execution_window = ( + f"{start_month} {start_day} - {end_month} {end_day}, {end_year}" + ) + execution_window_uk = ( + f"{start_day} {start_month} - {end_day} {end_month} {end_year}" + ) + else: + execution_window = f"{start_month} {start_day}, {start_year} - {end_month} {end_day}, {end_year}" + execution_window_uk = f"{start_day} {start_month} {start_year} - {end_day} {end_month} {end_year}" + report_dict["project"]["start_date"] = start_datetime.strftime("%B %d, %Y") + report_dict["project"]["start_date_uk"] = start_datetime.strftime("%d %B %Y") + report_dict["project"]["end_date"] = end_datetime.strftime("%B %d, %Y") + report_dict["project"]["end_date_uk"] = end_datetime.strftime("%d %B %Y") + report_dict["project"]["execution_window"] = execution_window + report_dict["project"]["execution_window_uk"] = execution_window_uk # Finding data report_dict["findings"] = {} for finding in self.report_queryset.reportfindinglink_set.all(): report_dict["findings"][finding.id] = {} report_dict["findings"][finding.id]["title"] = finding.title report_dict["findings"][finding.id]["severity"] = finding.severity.severity + report_dict["findings"][finding.id][ + "severity_color" + ] = finding.severity.color + report_dict["findings"][finding.id][ + "severity_color_rgb" + ] = finding.severity.color_rgb + report_dict["findings"][finding.id][ + "severity_color_hex" + ] = finding.severity.color_hex if finding.affected_entities: report_dict["findings"][finding.id][ "affected_entities" @@ -339,67 +447,130 @@ def generate_json(self): report_dict["team"][operator.operator.id]["note"] = operator.note return json.dumps(report_dict, indent=2, cls=DjangoJSONEncoder) - def create_newline(self): - """ - Create the appropriate ```` element to add a blank line that can act as a - separator between Word document elements. + def make_figure(self, par, ref=None): """ - # A paragraph must be added and followed by a run - p = self.spenny_doc.add_paragraph() - run = p.add_run() - # Add break to run to create the ```` needed for the doc XML - run.add_break() + Append a text run configured as an auto-incrementing figure to the provided + paragraph. The label and number are wrapped in ``w:bookmarkStart`` and + ``w:bookmarkEnd``. - def set_contextual_spacing(self, par): - """ - Apply ``contextualSpacing`` to the specified paragraph. + Source: + https://github.com/python-openxml/python-docx/issues/359 **Parameters** ``par`` : docx.paragraph.Paragraph Paragraph to alter + ``ref`` : string + String to use as the ``w:name`` value for the bookmark """ - styling = par.style._element.xpath("//w:pPr")[0] - # Applies the "Don't add spaces between paragraphs of the same style" option - contextual_spacing = OxmlElement("w:contextualSpacing") - styling.append(contextual_spacing) - return par - def make_figure(self, par): - """ - Make the specified paragraph an auto-incrementing Figure in the Word document. + def generate_ref(): + """Generate a random eight character reference ID.""" + return random.randint(10000000, 99999999) - Source: - https://github.com/python-openxml/python-docx/issues/359 + if ref: + ref = f"_Ref{ref}" + else: + ref = f"_Ref{generate_ref()}" + # Start a bookmark run with the figure label + p = par._p + bookmark_start = OxmlElement("w:bookmarkStart") + bookmark_start.set(qn("w:name"), ref) + bookmark_start.set(qn("w:id"), "0") + p.append(bookmark_start) + + # Add the figure label + run = par.add_run(self.label_figure) + + # Append XML for a new field character run + run = par.add_run() + r = run._r + fldChar = OxmlElement("w:fldChar") + fldChar.set(qn("w:fldCharType"), "begin") + r.append(fldChar) + + # Add field code instructions with ``instrText`` + run = par.add_run() + r = run._r + instrText = OxmlElement("w:instrText") + # Sequential figure with arabic numbers + instrText.text = " SEQ Figure \\* ARABIC" + r.append(instrText) + + # An optional ``separate`` value to enforce a space between label and number + run = par.add_run() + r = run._r + fldChar = OxmlElement("w:fldChar") + fldChar.set(qn("w:fldCharType"), "separate") + r.append(fldChar) + + # Include ``#`` as a placeholder for the number when Word updates fields + run = par.add_run("#") + r = run._r + # Close the field character run + fldChar = OxmlElement("w:fldChar") + fldChar.set(qn("w:fldCharType"), "end") + r.append(fldChar) + + # End the bookmark after the number + p = par._p + bookmark_end = OxmlElement("w:bookmarkEnd") + bookmark_end.set(qn("w:id"), "0") + p.append(bookmark_end) + + def make_cross_ref(self, par, ref): + """ + Append a text run configured as a cross-reference to the provided paragraph. **Parameters** ``par`` : docx.paragraph.Paragraph Paragraph to alter + ``ref`` : string + The ``w:name`` value of the target bookmark """ - run = run = par.add_run() - # Get the XML within the run + # Start the field character run for the label and number + run = par.add_run() r = run._r - # Assemble the proper Open XML and append it fldChar = OxmlElement("w:fldChar") fldChar.set(qn("w:fldCharType"), "begin") r.append(fldChar) + + # Add field code instructions with ``instrText`` that points to the target bookmark + run = par.add_run() + r = run._r instrText = OxmlElement("w:instrText") - instrText.text = " SEQ Figure \\* ARABIC" + instrText.text = f" REF _Ref{ref} \\h " r.append(instrText) + + # An optional ``separate`` value to enforce a space between label and number + run = par.add_run() + r = run._r + fldChar = OxmlElement("w:fldChar") + fldChar.set(qn("w:fldCharType"), "separate") + r.append(fldChar) + + # Add runs for the figure label and number + run = par.add_run(self.label_figure) + # This ``#`` is a placeholder Word will replace with the figure's number + run = par.add_run("#") + + # Close the field character run + run = par.add_run() + r = run._r fldChar = OxmlElement("w:fldChar") fldChar.set(qn("w:fldCharType"), "end") r.append(fldChar) + return par + def list_number(self, par, prev=None, level=None, num=True): """ - Makes the specified paragraph a list item with a specific level and - optional restart. + Makes the specified paragraph a list item with a specific level and optional restart. - An attempt will be made to retrieve an abstract numbering style that - corresponds to the style of the paragraph. If that is not possible, - the default numbering or bullet style will be used based on the - ``num`` parameter. + An attempt will be made to retrieve an abstract numbering style that corresponds + to the style of the paragraph. If that is not possible, the default numbering or + bullet style will be used based on the ``num`` parameter. Source: https://github.com/python-openxml/python-docx/issues/25#issuecomment-400787031 @@ -407,23 +578,20 @@ def list_number(self, par, prev=None, level=None, num=True): **Parameters** ``par`` : docx.paragraph.Paragraph - The paragraph to turn into a list item. + The docx paragraph to turn into a list item. ``prev`` : docx.paragraph.Paragraph or None - The previous paragraph in the list. If specified, the numbering - and styles will be taken as a continuation of this paragraph. - If omitted, a new numbering scheme will be started. + The previous paragraph in the list. If specified, the numbering and styles will + be taken as a continuation of this paragraph. If omitted, a new numbering scheme + will be started. ``level`` : int or None - The level of the paragraph within the outline. If ``prev`` is - set, defaults to the same level as in ``prev``. Otherwise, - defaults to zero. + The level of the paragraph within the outline. If ``prev`` is set, defaults + to the same level as in ``prev``. Otherwise, defaults to zero. ``num`` : bool - If ``prev`` is :py:obj:`None` and the style of the paragraph - does not correspond to an existing numbering style, this will - determine wether or not the list will be numbered or bulleted. - The result is not guaranteed, but is fairly safe for most Word - templates. + If ``prev`` is :py:obj:`None` and the style of the paragraph does not correspond + to an existing numbering style, this will determine wether or not the list will + be numbered or bulleted. The result is not guaranteed, but is fairly safe for + most Word templates. """ - # Open XML options used below xpath_options = { True: {"single": "count(w:lvl)=1 and ", "level": 0}, False: {"single": "", "level": level}, @@ -452,12 +620,13 @@ def type_xpath(prefer_single=True): ).format(type=type, **xpath_options[prefer_single]) def get_abstract_id(): - """Select as follows: - 1. Match single-level by style (get min ID) - 2. Match exact style and level (get min ID) - 3. Match single-level decimal/bullet types (get min ID) - 4. Match decimal/bullet in requested level (get min ID) - 3. 0 + """ + Select as follows: + + 1. Match single-level by style (get min ID) + 2. Match exact style and level (get min ID) + 3. Match single-level decimal/bullet types (get min ID) + 4. Match decimal/bullet in requested level (get min ID) """ for fn in (style_xpath, type_xpath): for prefer_single in (True, False): @@ -476,9 +645,9 @@ def get_abstract_id(): if level is None: level = 0 numbering = ( - self.spenny_doc.part.numbering_part.numbering_definitions._numbering + self.sacrificial_doc.part.numbering_part.numbering_definitions._numbering ) - # Compute the abstract ID first by style, then by num + # Compute the abstract ID first by style, then by ``num`` abstract = get_abstract_id() # Set the concrete numbering based on the abstract numbering ID num = numbering.add_num(abstract) @@ -494,21 +663,72 @@ def get_abstract_id(): par._p.get_or_add_pPr().get_or_add_numPr().get_or_add_numId().val = num par._p.get_or_add_pPr().get_or_add_numPr().get_or_add_ilvl().val = level - def process_evidence(self, finding, keyword, file_path, extension, p): + return par + + def get_styles(self, tag): + """ + Get styles from an BS4 ``Tag`` object's ``styles`` attribute and convert + the string to a dictionary. + + **Parameters** + + ``tag`` : Tag + BS4 ``Tag`` witth a ``styles`` attribute + """ + tag_styles = {} + style_str = tag.attrs["style"] + # Filter any blanks from the split + style_list = list(filter(None, style_str.split(";"))) + for style in style_list: + temp = style.split(":") + key = temp[0].strip() + value = temp[1].strip() + try: + if key == "font-size": + # Remove the "pt" from the font size and convert it to a ``Pt`` object + value = value.replace("pt", "") + value = float(value) + value = Pt(value) + if key == "font-family": + # Some fonts include a list of values – get just the first one + font_list = value.split(",") + priority_font = ( + font_list[0].replace("'", "").replace('"', "").strip() + ) + value = priority_font + if key == "color": + # Convert the color hex value to an ``RGCBolor`` object + value = value.replace("#", "") + n = 2 + hex_color = [ + hex(int(value[i : i + n], 16)) for i in range(0, len(value), n) + ] + if self.report_type == "pptx": + value = PptxRGBColor(*map(lambda v: int(v, 16), hex_color)) + else: + value = RGBColor(*map(lambda v: int(v, 16), hex_color)) + tag_styles[key] = value + except Exception: + logger.exception( + "Failed to convert one of the inline styles for a text run" + ) + return tag_styles + + def process_evidence(self, finding, keyword, file_path, extension, par): """ Process the specified evidence file for the named finding to add it to the Word document. **Parameters** - ``finding`` + ``finding`` : dict Finding currently being processed - ``keyword`` + ``keyword`` : string Evidence keyword corresponding to the evidence attached to the finding - ``file_path`` + ``file_path`` : string File path for the evidence file - ``extension`` + ``extension`` : string Evidence file's extension - ``par`` + ``par`` : Paragraph Paragraph meant to hold the evidence """ if extension in self.text_extensions: @@ -516,9 +736,9 @@ def process_evidence(self, finding, keyword, file_path, extension, p): # Read in evidence text evidence_text = evidence_contents.read() if self.report_type == "pptx": - # Place new textbox to the mid-right + self.delete_paragraph(par) top = Inches(1.65) - left = Inches(6) + left = Inches(8) width = Inches(4.5) height = Inches(3) # Create new textbox, textframe, paragraph, and run @@ -534,65 +754,77 @@ def process_evidence(self, finding, keyword, file_path, extension, p): font.size = Pt(11) font.name = "Courier New" else: - # Drop in text evidence using the Code Block style - p.text = evidence_text - p.style = "CodeBlock" - p.alignment = WD_ALIGN_PARAGRAPH.LEFT - p = self.spenny_doc.add_paragraph("Figure ", style="Caption") - self.make_figure(p) + # Drop in text evidence using the ``CodeBlock`` style + par.text = evidence_text + par.style = "CodeBlock" + par.alignment = WD_ALIGN_PARAGRAPH.LEFT + # Add a caption paragraph below the evidence + p = self.sacrificial_doc.add_paragraph(style="Caption") + ref_name = re.sub( + "[^A-Za-z0-9]+", + "", + finding["evidence"][keyword]["friendly_name"], + ) + self.make_figure(p, ref_name) run = p.add_run( - u" \u2013 " + finding["evidence"][keyword]["caption"] + self.prefix_figure + finding["evidence"][keyword]["caption"] ) elif extension in self.image_extensions: # Drop in the image at the full 6.5" width and add the caption if self.report_type == "pptx": + self.delete_paragraph(par) + # Place new textbox to the mid-right top = Inches(1.65) left = Inches(8) width = Inches(4.5) - image = self.finding_slide.shapes.add_picture( - file_path, left, top, width=width - ) + self.finding_slide.shapes.add_picture(file_path, left, top, width=width) else: - p.alignment = WD_ALIGN_PARAGRAPH.CENTER - run = p.add_run() + par.alignment = WD_ALIGN_PARAGRAPH.CENTER + run = par.add_run() # Add the picture to the document and then add a border - inline_shape = run.add_picture(file_path, width=Inches(6.5)) - - # Add the border – see Ghostwriter Wiki for documentation - inline_class = run._r.xpath("//wp:inline")[-1] - inline_class.attrib["distT"] = "0" - inline_class.attrib["distB"] = "0" - inline_class.attrib["distL"] = "0" - inline_class.attrib["distR"] = "0" - - # Set the shape's "effect extent" attributes to the border weight - effect_extent = OxmlElement("wp:effectExtent") - effect_extent.set("l", self.border_weight) - effect_extent.set("t", self.border_weight) - effect_extent.set("r", self.border_weight) - effect_extent.set("b", self.border_weight) - # Insert just below ```` or it will not work - inline_class.insert(1, effect_extent) - - # Find inline shape properties – ``pic:spPr`` - pic_data = run._r.xpath("//pic:spPr")[-1] - # Assemble OXML for a solid border - ln_xml = OxmlElement("a:ln") - ln_xml.set("w", self.border_weight) - solidfill_xml = OxmlElement("a:solidFill") - color_xml = OxmlElement("a:srgbClr") - color_xml.set("val", self.border_color) - solidfill_xml.append(color_xml) - ln_xml.append(solidfill_xml) - pic_data.append(ln_xml) + run.add_picture(file_path, width=Inches(6.5)) + + if self.enable_borders: + # Add the border – see Ghostwriter Wiki for documentation + inline_class = run._r.xpath("//wp:inline")[-1] + inline_class.attrib["distT"] = "0" + inline_class.attrib["distB"] = "0" + inline_class.attrib["distL"] = "0" + inline_class.attrib["distR"] = "0" + + # Set the shape's "effect extent" attributes to the border weight + effect_extent = OxmlElement("wp:effectExtent") + effect_extent.set("l", str(self.border_weight)) + effect_extent.set("t", str(self.border_weight)) + effect_extent.set("r", str(self.border_weight)) + effect_extent.set("b", str(self.border_weight)) + # Insert just below ```` or it will not work + inline_class.insert(1, effect_extent) + + # Find inline shape properties – ``pic:spPr`` + pic_data = run._r.xpath("//pic:spPr")[-1] + # Assemble OXML for a solid border + ln_xml = OxmlElement("a:ln") + ln_xml.set("w", str(self.border_weight)) + solidfill_xml = OxmlElement("a:solidFill") + color_xml = OxmlElement("a:srgbClr") + color_xml.set("val", self.border_color) + solidfill_xml.append(color_xml) + ln_xml.append(solidfill_xml) + pic_data.append(ln_xml) # Create the caption for the image - p = self.spenny_doc.add_paragraph("Figure ", style="Caption") - self.make_figure(p) - run = p.add_run(u" \u2013 " + finding["evidence"][keyword]["caption"]) + p = self.sacrificial_doc.add_paragraph(style="Caption") + ref_name = re.sub( + "[^A-Za-z0-9]+", "", finding["evidence"][keyword]["friendly_name"] + ) + self.make_figure(p, ref_name) + run = p.add_run( + self.prefix_figure + finding["evidence"][keyword]["caption"] + ) # Skip unapproved files else: - p = None + par = None pass def delete_paragraph(self, par): @@ -608,47 +840,127 @@ def delete_paragraph(self, par): parent_element = p.getparent() parent_element.remove(p) + def write_xml(self, text, par, styles): + """ + Write the provided text to Office XML. + + **Parameters** + + ``text`` : string + Text to check for keywords + ``par`` : Paragraph + Paragraph for the processed text + ``styles`` : dict + Copy of ``ReportConstants.DEFAULT_STYLE_VALUES`` with styles for the text + """ + # Handle hyperlinks based on Office report type + # Easy with ``python-pptx`` API, but custom work required for ``python-docx`` + if styles["hyperlink"] and styles["hyperlink_url"]: + if self.report_type == "pptx": + run = par.add_run() + run.text = text + run.hyperlink.address = styles["hyperlink_url"] + else: + logger.info("Text is: %s, styles: %s", text, styles) + # For Word, this code is modified from this issue: + # https://github.com/python-openxml/python-docx/issues/384 + # Get an ID from the ``document.xml.rels`` file + part = par.part + r_id = part.relate_to( + styles["hyperlink_url"], + docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, + is_external=True, + ) + # Create the ``w:hyperlink`` tag and add needed values + hyperlink = docx.oxml.shared.OxmlElement("w:hyperlink") + hyperlink.set( + docx.oxml.shared.qn("r:id"), + r_id, + ) + # Create the ``w:r`` and ``w:rPr`` elements + new_run = docx.oxml.shared.OxmlElement("w:r") + rPr = docx.oxml.shared.OxmlElement("w:rPr") + new_run.append(rPr) + new_run.text = text + hyperlink.append(new_run) + # Create a new Run object and add the hyperlink into it + run = par.add_run() + run._r.append(hyperlink) + # A workaround for the lack of a hyperlink style + if "Hyperlink" in self.sacrificial_doc.styles: + run.style = "Hyperlink" + else: + run.font.color.theme_color = MSO_THEME_COLOR_INDEX.HYPERLINK + run.font.underline = True + else: + run = par.add_run() + run.text = text + + # Apply font-based styles that work for both APIs + font = run.font + font.bold = styles["bold"] + font.italic = styles["italic"] + font.underline = styles["underline"] + if styles["font_color"]: + font.color.rgb = styles["font_color"] + font.name = styles["font_family"] + font.size = styles["font_size"] + if styles["inline_code"]: + if self.report_type == "pptx": + font.name = "Courier New" + else: + run.style = "CodeInline" + font.no_proof = True + + # These styles require extra work due to limitations of the ``python-pptx`` API + if styles["highlight"]: + if self.report_type == "pptx": + rPr = run._r.get_or_add_rPr() + highlight = OxmlElement("a:highlight") + srgbClr = OxmlElement("a:srgbClr") + srgbClr.set("val", "FFFF00") + highlight.append(srgbClr) + rPr.append(highlight) + else: + font.highlight_color = WD_COLOR_INDEX.YELLOW + if styles["strikethrough"]: + if self.report_type == "pptx": + font._element.set("strike", "sngStrike") + else: + font.strike = styles["strikethrough"] + if styles["subscript"]: + if self.report_type == "pptx": + font._element.set("baseline", "-25000") + else: + font.subscript = styles["subscript"] + if styles["superscript"]: + if self.report_type == "pptx": + font._element.set("baseline", "30000") + else: + font.superscript = styles["superscript"] + def replace_and_write( - self, - text, - p, - finding, - italic=False, - underline=False, - bold=False, - inline_code=False, - link_run=False, - link_url=None, + self, text, par, finding, styles=ReportConstants.DEFAULT_STYLE_VALUES.copy() ): """ Find and replace template keywords in the provided text. **Parameters** - ``text`` + ``text`` : string Text to check for keywords - ``p`` + ``par`` : Paragraph Paragraph for the processed text - ``italic`` - Boolean to enable italic style - ``bold`` - Boolean to enable bold style - ``inline_code`` - Boolean to enable inline code block style - ``code`` - Boolean to enable code block style - ``link_run`` - Run containing the hyperlink – used if the text is a hyperlink - ``link_url`` - URL for the hyperlink – used if the text is a hyperlink + ``styles`` : dict + Copy of ``ReportConstants.DEFAULT_STYLE_VALUES`` with styles for the text """ text = text.replace("\r\n", "") - # Regex for searching for bracketed template placeholders, e.g. {{.client}} + # Regex for searching for bracketed template placeholders, e.g. ``{{.client}}`` keyword_regex = r"\{\{\.(.*?)\}\}" - # Search for {{. }} keywords + # Search for ``{{. }}`` keywords match = re.search(keyword_regex, text) if match: - # Get just the first match, set it as the "keyword," and remove it from the line + # Get just the first match, set it as the ``keyword``, and remove it from the line # There should never be - or need to be - multiple matches match = match[0] keyword = match.replace("}}", "").replace("{{.", "").strip() @@ -664,19 +976,72 @@ def replace_and_write( text = text.replace( "{{.client}}", self.report_json["client"]["full_name"] ) - + # Perform replacement of project-related placeholders + if "{{.project_start}}" in text: + text = text.replace( + "{{.project_start}}", self.report_json["project"]["start_date"] + ) + if "{{.project_end}}" in text: + text = text.replace( + "{{.project_end}}", self.report_json["project"]["end_date"] + ) + if "{{.project_start_uk}}" in text: + text = text.replace( + "{{.project_start_uk}}", self.report_json["project"]["start_date_uk"] + ) + if "{{.project_end_uk}}" in text: + text = text.replace( + "{{.project_end_uk}}", self.report_json["project"]["end_date_uk"] + ) + if "{{.project_type}}" in text: + text = text.replace( + "{{.project_type}}", self.report_json["project"]["project_type"].lower() + ) # Transform caption placeholders into figures - if "{{.caption}}" in text: - text = text.replace("{{.caption}}", "") + if keyword.startswith("caption"): + logger.info("GOT A CAPTION: %s", repr(keyword)) + ref_name = keyword.lstrip("caption ") + ref_name = re.sub("[^A-Za-z0-9]+", "", ref_name) + logger.info(text) + text = text.replace("{{.%s}}" % keyword, "") + logger.info(text) if self.report_type == "pptx": - # Only option would be to make the caption a slide bullet and that would be weird - so just pass - pass + if ref_name: + run = par.add_run() + run.text = f"See {ref_name}" + font = run.font + font.italic = True else: - p.style = "Caption" - p.text = "Figure " - self.make_figure(p) - run = p.add_run(u" \u2013 " + text) - return + par.style = "Caption" + if ref_name: + self.make_figure(par, ref_name) + else: + self.make_figure(par) + par.add_run(self.prefix_figure + text) + return par + + # Transform references into bookmarks + if keyword and keyword.startswith("ref "): + ref_keyword = keyword + keyword = keyword.lstrip("ref ") + ref_name = re.sub("[^A-Za-z0-9]+", "", keyword) + ref_placeholder = "{{.%s}}" % ref_keyword + exploded_text = re.split(f"({ref_placeholder})", text) + for text in exploded_text: + if text == ref_placeholder: + if self.report_type == "pptx": + run = par.add_run() + run.text = f"See {ref_name}" + font = run.font + font.italic = True + else: + self.make_cross_ref( + par, + ref_name, + ) + else: + self.write_xml(text, par, styles) + return par # Handle evidence keywords if "evidence" in finding: @@ -688,138 +1053,83 @@ def replace_and_write( ) extension = finding["evidence"][keyword]["url"].split(".")[-1] if os.path.exists(file_path): - self.process_evidence(finding, keyword, file_path, extension, p) - return + self.process_evidence(finding, keyword, file_path, extension, par) + return par else: raise FileNotFoundError(file_path) - # Add a new run to the paragraph - if self.report_type == "pptx": - run = p.add_run() - run.text = text - # For pptx, formatting is applied via the font instead of on the run object - font = run.font - if inline_code: - font.name = "Courier New" - # font.size = Pt(11) - font.bold = bold - font.italic = italic - font.underline = underline - else: - # Make this section a hyperlink - # Modified from this issue: - # https://github.com/python-openxml/python-docx/issues/384 - if link_run and link_url: - # This gets access to the document.xml.rels file and gets a new relation id value - part = p.part - r_id = part.relate_to( - link_url, - docx.opc.constants.RELATIONSHIP_TYPE.HYPERLINK, - is_external=True, - ) - # Create the w:hyperlink tag and add needed values - hyperlink = docx.oxml.shared.OxmlElement("w:hyperlink") - hyperlink.set( - docx.oxml.shared.qn("r:id"), r_id, - ) - # Create a w:r element and a new w:rPr element - new_run = docx.oxml.shared.OxmlElement("w:r") - rPr = docx.oxml.shared.OxmlElement("w:rPr") - # Join all the xml elements together add add the required text to the w:r element - new_run.append(rPr) - new_run.text = text - hyperlink.append(new_run) - # Create a new Run object and add the hyperlink into it - run = p.add_run() - run._r.append(hyperlink) - # A workaround for the lack of a hyperlink style (doesn't go purple after using the link) - # Delete this if using a template that has the hyperlink style in it - if "Hyperlink" in self.spenny_doc.styles: - run.style = "Hyperlink" - else: - run.font.color.theme_color = MSO_THEME_COLOR_INDEX.HYPERLINK - run.font.underline = True - else: - run = p.add_run() - run.text = text - if inline_code: - run.style = "Code (inline)" - run.bold = bold - run.italic = italic - run.underline = underline + # If nothing above triggers, write the text + self.write_xml(text, par, styles) + return par - def process_nested_tags(self, contents, p, finding, prev_p=None, num=True): + def process_nested_tags(self, contents, par, finding): """ - Process BeautifulSoup 4 Tag objects containing nested HTML tags. + Process BeautifulSoup4 ``Tag`` objects containing nested HTML tags. **Parameters** - ``contents`` - The contents of a BS4 Tag - ``p`` - A docx paragraph object - ``finding`` + ``contents`` : Tag.contents + Contents of a BS4 ``Tag`` + ``par`` : Paragraph + Word docx ``Paragraph`` object + ``finding`` : dict The current finding (JSON) being processed - ``prev_p`` - Previous paragraph object for continuing lists (Defaults to None) - ``num`` - Boolean value to determine if a list Tag isordered/numbered (Defaults to True) """ - # Iterate over the provided list for part in contents: - # Track temporary styles for text runs - link_url = None - link_run = False - bold_font = False - underline = False - italic_font = False - inline_code = False - # Track "global" styles for the whole object - nested_link_run = False - nested_bold_font = False - nested_underline = False - nested_italic_font = False - nested_inline_code = False - # Get each part's name to check if it's a Tag object + styles = ReportConstants.DEFAULT_STYLE_VALUES.copy() + nested_styles = ReportConstants.DEFAULT_STYLE_VALUES.copy() + # Get each part's name to check if it's a ``Tag`` object # A plain string will return ``None`` - no tag part_name = part.name if part_name: # Split part into list of plain text and tag objects part_contents = part.contents - # Get all of the nested tags - all_nested_tags = part.find_all() - # Append the first tag to the start of the list - all_nested_tags = [part] + all_nested_tags - # Count all of the tags for the next step - tag_count = len(all_nested_tags) - # A list length > 1 means nested formatting # Get the parent object's style info first as it applies to all future runs - # A code tag here is inline code inside of a p tag + # A ``code`` tag here is inline code inside of a ``p`` tag if part_name == "code": - nested_inline_code = True + nested_styles["inline_code"] = True # An ``em`` tag designates italics and appears rarely elif part_name == "em": - nested_italic_font = True + nested_styles["italic_font"] = True # A ``strong`` or ``b`` tag designates bold/strong font style and appears rarely elif part_name == "strong" or part_name == "b": - nested_bold_font = True + nested_styles["bold"] = True # A ``u`` tag desingates underlined text elif part_name == "u": - nested_underline = True + nested_styles["underline"] = True + elif part_name == "sub": + nested_styles["subscript"] = True + elif part_name == "sup": + nested_styles["superscript"] = True + elif part_name == "del": + nested_styles["strikethrough"] = True # A span tag will contain one or more classes for formatting elif part_name == "span": if "class" in part.attrs: part_attrs = part.attrs["class"] # Check existence of supported classes if "italic" in part_attrs: - nested_italic_font = True + nested_styles["italic"] = True if "bold" in part_attrs: - nested_bold_font = True + nested_styles["bold"] = True if "underline" in part_attrs: - nested_underline = True + nested_styles["underline"] = True + if "highlight" in part_attrs: + nested_styles["highlight"] = True + if "style" in part.attrs: + part_style = self.get_styles(part) + # Check existence of supported styles + if "font-size" in part_style: + nested_styles["font_size"] = part_style["font-size"] + if "font-family" in part_style: + nested_styles["font_family"] = part_style["font-family"] + if "color" in part_style: + nested_styles["font_color"] = part_style["color"] + if "background-color" in part_style: + nested_styles["highlight"] = part_style["background-color"] elif part_name == "a": - nested_link_run = True - link_url = part["href"] + nested_styles["hyperlink"] = True + nested_styles["hyperlink_url"] = part["href"] # Any other tags are unexpected and ignored else: if part_name not in self.tag_allowlist: @@ -832,30 +1142,45 @@ def process_nested_tags(self, contents, p, finding, prev_p=None, num=True): tag_name = tag.name if tag_name and tag_name in self.tag_allowlist: tag_contents = tag.contents + content_text = "" # Check for an additional tags in the contents # This happens when a hyperlink is formatted with a font style if tag_contents[0].name: if tag_contents[0].name == "a": - link_run = True - link_url = tag_contents[0]["href"] + styles["hyperlink"] = True + styles["hyperlink_url"] = tag_contents[0]["href"] content_text = tag_contents[0].text else: content_text = " ".join(tag_contents) if tag_name == "code": - inline_code = True + styles["inline_code"] = True elif tag_name == "em": - italic_font = True + styles["italic"] = True elif tag_name == "span": - tag_attrs = tag.attrs["class"] - if "italic" in tag_attrs: - italic_font = True - if "bold" in tag_attrs: - bold_font = True - if "underline" in tag_attrs: - underline = True + if "class" in tag.attrs: + tag_attrs = tag.attrs["class"] + if "italic" in tag_attrs: + styles["italic"] = True + if "bold" in tag_attrs: + styles["bold"] = True + if "underline" in tag_attrs: + styles["underline"] = True + if "highlight" in tag_attrs: + styles["highlight"] = True + if "style" in part.attrs: + tag_style = self.get_styles(tag) + # Check existence of supported styles + if "font-size" in tag_style: + styles["font_size"] = tag_style["font-size"] + if "font-family" in tag_style: + styles["font_family"] = tag_style["font-family"] + if "color" in tag_style: + styles["font_color"] = tag_style["color"] + if "background-color" in tag_style: + styles["highlight"] = tag_style["background-color"] elif tag_name == "a": - link_run = True - link_url = tag["href"] + styles["hyperlink"] = True + styles["hyperlink_url"] = tag["href"] else: logger.warning( "Encountered an unexpected nested HTML tag: %s", @@ -869,42 +1194,55 @@ def process_nested_tags(self, contents, p, finding, prev_p=None, num=True): continue else: content_text = tag - # Conditionally apply text styles - if inline_code or nested_inline_code: - inline_code = True - if underline or nested_underline: - underline = True - if bold_font or nested_bold_font: - bold_font = True - if italic_font or nested_italic_font: - italic_font = True - if link_run or nested_link_run: - link_run = True + + # Conditionally set text styles + if styles["inline_code"] or nested_styles["inline_code"]: + styles["inline_code"] = True + if styles["underline"] or nested_styles["underline"]: + styles["underline"] = True + if styles["bold"] or nested_styles["bold"]: + styles["bold"] = True + if styles["italic"] or nested_styles["italic"]: + styles["italic"] = True + if styles["strikethrough"] or nested_styles["strikethrough"]: + styles["strikethrough"] = True + if styles["superscript"] or nested_styles["superscript"]: + styles["superscript"] = True + if styles["subscript"] or nested_styles["subscript"]: + styles["subscript"] = True + if styles["highlight"] or nested_styles["highlight"]: + styles["highlight"] = True + # These styles can be deeply nested, so we favor the run's style + # Take the nested style if the current run's style is ``None`` + if styles["hyperlink"] or nested_styles["hyperlink"]: + styles["hyperlink"] = True + if not styles["hyperlink_url"]: + styles["hyperlink_url"] = nested_styles["hyperlink_url"] + if styles["font_size"] or nested_styles["font_size"]: + if not styles["font_size"]: + styles["font_size"] = nested_styles["font_size"] + if styles["font_family"] or nested_styles["font_family"]: + if not styles["font_family"]: + styles["font_family"] = nested_styles["font_family"] + if styles["font_color"] or nested_styles["font_color"]: + if not styles["font_color"]: + styles["font_color"] = nested_styles["font_color"] + # Write the text for this run - self.replace_and_write( - content_text, - p, - finding, - italic_font, - underline, - bold_font, - inline_code, - link_run, - link_url, - ) + par = self.replace_and_write(content_text, par, finding, styles) # Reset temporary run styles - bold_font = False - underline = False - italic_font = False - inline_code = False + styles = ReportConstants.DEFAULT_STYLE_VALUES.copy() # There are no tags to process, so write the string else: if isinstance(part, NavigableString): - self.replace_and_write(part, p, finding) + par = self.replace_and_write(part, par, finding) else: - self.replace_and_write(part.text, p, finding) + par = self.replace_and_write(part.text, par, finding) + return par - def create_list_paragraph(self, prev_p, level, num=False): + def create_list_paragraph( + self, prev_p, level, num=False, alignment=WD_ALIGN_PARAGRAPH.LEFT + ): """ Create a new paragraph in the document for a list. @@ -915,19 +1253,24 @@ def create_list_paragraph(self, prev_p, level, num=False): ``level`` Indentation level for the list item ``num`` - Boolean to determine if the line item will be numbered (Default to False) + Boolean to determine if the line item will be numbered (Default: False) """ if self.report_type == "pptx": # Move to new paragraph/line and indent bullets based on level p = self.finding_body_shape.text_frame.add_paragraph() p.level = level else: - if num: - p = self.spenny_doc.add_paragraph(style="Number List") - else: - p = self.spenny_doc.add_paragraph(style="Bullet List") - self.list_number(p, prev=prev_p, level=level, num=num) - p.alignment = WD_ALIGN_PARAGRAPH.LEFT + styles = self.sacrificial_doc.styles + try: + if num: + list_style = styles["Number List"] + else: + list_style = styles["Bullet List"] + except Exception: + list_style = styles["List Paragraph"] + p = self.sacrificial_doc.add_paragraph(style=list_style) + p = self.list_number(p, prev=prev_p, level=level, num=num) + p.alignment = alignment return p def parse_nested_html_lists(self, tag, prev_p, num, finding, level=0): @@ -961,7 +1304,7 @@ def parse_nested_html_lists(self, tag, prev_p, num, finding, level=0): # Create the paragraph for this list item p = self.create_list_paragraph(prev_p, level, num) if li_contents[0].name: - self.process_nested_tags(li_contents, p, finding, prev_p, num) + self.process_nested_tags(li_contents, p, finding) else: self.replace_and_write(part.text, p, finding) # Bigger lists mean more tags, so process nested tags @@ -985,14 +1328,12 @@ def parse_nested_html_lists(self, tag, prev_p, num, finding, level=0): p = self.create_list_paragraph(prev_p, level, num) if len(temp) == 1: if temp[0].name: - self.process_nested_tags( - temp, p, finding, prev_p, num - ) + self.process_nested_tags(temp, p, finding) else: self.replace_and_write(temp[0], p, finding) # Bigger lists mean more tags, so process nested tags else: - self.process_nested_tags(temp, p, finding, prev_p, num) + self.process_nested_tags(temp, p, finding) # Recursively process this list and any other nested lists inside of it if nested_list: # Increment the list level counter for this nested list @@ -1002,7 +1343,7 @@ def parse_nested_html_lists(self, tag, prev_p, num, finding, level=0): ) else: p = self.create_list_paragraph(prev_p, level, num) - self.process_nested_tags(part.contents, p, finding, prev_p, num) + self.process_nested_tags(part.contents, p, finding) prev_p = p # If ol tag encountered, increment level and switch to numbered list elif part.name == "ol": @@ -1015,7 +1356,7 @@ def parse_nested_html_lists(self, tag, prev_p, num, finding, level=0): # No change in list type, so proceed with writing the line elif part.name: p = self.create_list_paragraph(prev_p, level, num) - self.process_nested_tags(part, p, finding, prev_p, num) + self.process_nested_tags(part, p, finding) else: if not isinstance(part, NavigableString): logger.warning( @@ -1025,8 +1366,7 @@ def parse_nested_html_lists(self, tag, prev_p, num, finding, level=0): if part.strip() != "": p = self.create_list_paragraph(prev_p, level, num) self.replace_and_write(part.strip(), p, finding) - # Track the paragraph used for this list item - # prev_p = p + # Return last paragraph created return p @@ -1040,36 +1380,61 @@ def process_text_xml(self, text, finding): ``text`` Text to convert to Office XML ``finding`` - Current report fidning being processed + Current report finding being processed """ prev_p = None - # Setup the first text frame for the PowerPoint slide - if self.report_type == "pptx": - if self.finding_body_shape.has_text_frame: - self.finding_body_shape.text_frame.clear() - self.delete_paragraph(self.finding_body_shape.text_frame.paragraphs[0]) + # Clean text to make it XML compatible for Office XML text = "".join(c for c in text if self.valid_xml_char_ordinal(c)) - # Parse the HTML into a BS4 soup object + if text: + # Parse the HTML into a BS4 soup object soup = BeautifulSoup(text, "lxml") # Each WYSIWYG field begins with `` so get the contents of body body = soup.find("body") contents_list = body.contents - # Loop over all bs4.element.Tag objects in the body + # Loop over all ``bs4.element.Tag`` objects in the body for tag in contents_list: - # If it came from Ghostwriter, tag names will be p, pre, ul, or ol - # Anything else is logged and ignored b/c all other tags should appear within these tags tag_name = tag.name + # Hn – Headings + if tag_name in ["h1", "h2", "h3", "h4", "h5", "h6"]: + if self.report_type == "pptx": + # No headings in PPTX, so add a new line and bold it as a pseudo-heading + p = self.finding_body_shape.text_frame.add_paragraph() + run = p.add_run() + run.text = tag.text + font = run.font + font.bold = True + else: + heading_num = int(tag_name[1]) + # Add the heading to the document + # This discards any inline formatting, but that should be managed + # by editing the style in the template + p = self.sacrificial_doc.add_heading(tag.text, heading_num) + # P – Paragraphs - if tag_name == "p": + elif tag_name == "p": # Get the tag's contents to check for additional formatting contents = tag.contents if self.report_type == "pptx": p = self.finding_body_shape.text_frame.add_paragraph() + ALIGNMENT = PP_ALIGN else: - p = self.spenny_doc.add_paragraph() + p = self.sacrificial_doc.add_paragraph() + ALIGNMENT = WD_ALIGN_PARAGRAPH + # Check for alignment classes + if "class" in tag.attrs: + tag_attrs = tag.attrs["class"] + if "left" in tag_attrs: + p.alignment = ALIGNMENT.LEFT + if "center" in tag_attrs: + p.alignment = ALIGNMENT.CENTER + if "right" in tag_attrs: + p.alignment = ALIGNMENT.RIGHT + if "justify" in tag_attrs: + p.alignment = ALIGNMENT.JUSTIFY self.process_nested_tags(contents, p, finding) + # PRE – Code Blocks elif tag_name == "pre": # The WYSIWYG editor doesn't allow users to format text inside of a code block @@ -1079,7 +1444,7 @@ def process_text_xml(self, text, finding): # Place new textbox to the mid-right if contents: top = Inches(1.65) - left = Inches(6) + left = Inches(8) width = Inches(4.5) height = Inches(3) # Create new textbox, textframe, paragraph, and run @@ -1095,9 +1460,8 @@ def process_text_xml(self, text, finding): parts = code.split("\r\n") # Iterate over the list of code lines to make paragraphs for code_line in parts: - # Create paragraph and apply 'CodeBlock' style - # Style is managed in the docx template p = text_frame.add_paragraph() + p.alignment = PP_ALIGN.LEFT run = p.add_run() # Insert code block and apply formatting run.text = code_line @@ -1116,9 +1480,12 @@ def process_text_xml(self, text, finding): for code_line in parts: # Create paragraph and apply 'CodeBlock' style # Style is managed in the docx template - p = self.spenny_doc.add_paragraph(code_line) + p = self.sacrificial_doc.add_paragraph( + code_line + ) p.style = "CodeBlock" p.alignment = WD_ALIGN_PARAGRAPH.LEFT + # OL & UL – Ordered/Numbered & Unordered Lists elif tag_name == "ol" or tag_name == "ul": # Ordered/numbered lists need numbers and linked paragraphs @@ -1128,6 +1495,7 @@ def process_text_xml(self, text, finding): num = True else: num = False + # Get the list tag's contents and check each item contents = tag.contents for part in contents: @@ -1142,11 +1510,11 @@ def process_text_xml(self, text, finding): # Create the paragraph for this list item p = self.create_list_paragraph(prev_p, level, num) if li_contents[0].name: - self.process_nested_tags( - li_contents, p, finding, prev_p, num + p = self.process_nested_tags( + li_contents, p, finding ) else: - self.replace_and_write(part.text, p, finding) + p = self.replace_and_write(part.text, p, finding) # Bigger lists mean more tags, so process nested tags else: # Identify a part with a nested list @@ -1176,17 +1544,17 @@ def process_text_xml(self, text, finding): ) if len(temp) == 1: if temp[0].name: - self.process_nested_tags( - temp, p, finding, prev_p, num + p = self.process_nested_tags( + temp, p, finding ) else: - self.replace_and_write( + p = self.replace_and_write( temp[0], p, finding ) # Bigger lists mean more tags, so process nested tags else: - self.process_nested_tags( - temp, p, finding, prev_p, num + p = self.process_nested_tags( + temp, p, finding ) # Recursively process this list and any other nested lists inside of it if nested_list: @@ -1198,8 +1566,8 @@ def process_text_xml(self, text, finding): # No nested list, proceed as normal else: p = self.create_list_paragraph(prev_p, level, num) - self.process_nested_tags( - part.contents, p, finding, prev_p, num + p = self.process_nested_tags( + part.contents, p, finding ) # Track the paragraph used for this list item to link subsequent paragraphs prev_p = p @@ -1223,15 +1591,52 @@ def generate_word_docx(self): # Generate the JSON for the report self.report_json = json.loads(self.generate_json()) # Create Word document writer using the specified template file - if self.template_loc: - try: - self.main_spenny_doc = DocxTemplate(self.template_loc) - except Exception: - raise - else: - raise + try: + self.word_doc = DocxTemplate(self.template_loc) + except DocxPackageNotFoundError: + logger.exception( + "Failed to load the provided template document because file could not be found: %s", + self.template_loc, + ) + raise DocxPackageNotFoundError + except Exception: + logger.exception( + "Failed to load the provided template document: %s", self.template_loc + ) + + # Check for styles + styles = self.word_doc.styles + if "CodeBlock" not in styles: + codeblock_style = styles.add_style("CodeBlock", WD_STYLE_TYPE.PARAGRAPH) + codeblock_style.base_style = styles["Normal"] + codeblock_style.hidden = False + codeblock_style.quick_style = True + codeblock_style.priority = 2 + # Set font and size + codeblock_font = codeblock_style.font + codeblock_font.name = "Courier New" + codeblock_font.size = Pt(11) + # Set alignment + codeblock_par = codeblock_style.paragraph_format + codeblock_par.alignment = WD_ALIGN_PARAGRAPH.LEFT + codeblock_par.line_spacing = 1 + codeblock_par.left_indent = Inches(0.2) + codeblock_par.right_indent = Inches(0.2) + + if "CodeInline" not in styles: + codeinline_style = styles.add_style("CodeInline", WD_STYLE_TYPE.PARAGRAPH) + codeinline_style.hidden = False + codeinline_style.quick_style = True + codeinline_style.priority = 3 + # Set font and size + codeblock_font = codeinline_style.font + codeblock_font.name = "Courier New" + codeblock_font.size = Pt(11) + # Prepare the ``context`` dict for the Word template rendering context = {} + context["report_date"] = datetime.now().strftime("%B %d, %Y") + context["report_date_uk"] = datetime.now().strftime("%d %B %Y") # Client information context["client"] = self.report_json["client"]["full_name"] @@ -1240,10 +1645,22 @@ def generate_word_docx(self): # Assessment information context["assessment_name"] = self.report_json["project"]["name"] - context["project_type"] = self.report_json["project"]["project_type"] - context["company"] = settings.COMPANY_NAME + context["assessment_type"] = self.report_json["project"]["project_type"] + context["project_type"] = context["assessment_type"] + context["company"] = self.company_config.company_name context["company_pocs"] = self.report_json["team"].values() + # Project dates + context["project_start_date"] = self.report_json["project"]["start_date"] + context["project_start_date_uk"] = self.report_json["project"]["start_date_uk"] + context["project_end_date"] = self.report_json["project"]["end_date"] + context["project_end_date_uk"] = self.report_json["project"]["end_date_uk"] + + context["execution_window"] = self.report_json["project"]["execution_window"] + context["execution_window_uk"] = self.report_json["project"][ + "execution_window_uk" + ] + # Infrastructure information context["domains"] = self.report_json["infrastructure"]["domains"].values() context["static_servers"] = self.report_json["infrastructure"]["servers"][ @@ -1258,120 +1675,64 @@ def generate_word_docx(self): # Findings information context["findings"] = self.report_json["findings"].values() - for finding in context["findings"]: - finding_color = self.informational_color - if finding["severity"].lower() == "informational": - finding_color = self.informational_color - elif finding["severity"].lower() == "low": - finding_color = self.low_color - elif finding["severity"].lower() == "medium": - finding_color = self.medium_color - elif finding["severity"].lower() == "high": - finding_color = self.high_color - elif finding["severity"].lower() == "critical": - finding_color = self.critical_color - finding["color"] = finding_color - - # Generate the subdocument for findings - self.spenny_doc = self.main_spenny_doc.new_subdoc() - self.generate_finding_subdoc() - context["findings_subdoc"] = self.spenny_doc + context["total_findings"] = len(self.report_json["findings"].values()) + + # Generate the XML for the styled findings + context = self.process_findings(context) # Render the Word document + auto-escape any unsafe XML/HTML - self.main_spenny_doc.render(context, autoescape=True) + self.word_doc.render(context, self.jinja_env, autoescape=True) # Return the final rendered document - return self.main_spenny_doc + return self.word_doc - def generate_finding_subdoc(self): + def process_findings(self, context: dict) -> dict: """ - Generate a Word sub-document for the current report's findings. + Update the document context with ``RichText`` and ``Subdocument`` objects for + each finding. This + + **Parameters** + + ``context`` + Pre-defined template context """ - counter = 0 - total_findings = len(self.report_json["findings"].values()) - for finding in self.report_json["findings"].values(): - # There's a special Heading 3 for the finding title so we don't use ``add_heading()`` here - p = self.spenny_doc.add_paragraph(finding["title"]) - p.style = "Heading 3 - Finding" - # This is Heading 4 but we want to make severity a run to color it so we don't use ``add_heading()`` here - p = self.spenny_doc.add_paragraph() - p.style = "Heading 4" - run = p.add_run("Severity – ") - run = p.add_run("{}".format(finding["severity"])) - font = run.font - if finding["severity"].lower() == "informational": - font.color.rgb = RGBColor( - self.informational_color_hex[0], - self.informational_color_hex[1], - self.informational_color_hex[2], - ) - elif finding["severity"].lower() == "low": - font.color.rgb = RGBColor( - self.low_color_hex[0], self.low_color_hex[1], self.low_color_hex[2] - ) - elif finding["severity"].lower() == "medium": - font.color.rgb = RGBColor( - self.medium_color_hex[0], - self.medium_color_hex[1], - self.medium_color_hex[2], - ) - elif finding["severity"].lower() == "high": - font.color.rgb = RGBColor( - self.high_color_hex[0], - self.high_color_hex[1], - self.high_color_hex[2], - ) - else: - font.color.rgb = RGBColor( - self.critical_color_hex[0], - self.critical_color_hex[1], - self.critical_color_hex[2], - ) - # Add an Affected Entities section - self.spenny_doc.add_heading("Affected Entities", 4) - self.process_text_xml(finding["affected_entities"], finding) - - # Add a Description section that may also include evidence figures - self.spenny_doc.add_heading("Description", 4) - self.process_text_xml(finding["description"], finding) - - # Create Impact section - self.spenny_doc.add_heading("Impact", 4) - self.process_text_xml(finding["impact"], finding) - - # Create Recommendations section - self.spenny_doc.add_heading("Recommendation", 4) - self.process_text_xml(finding["recommendation"], finding) - - # Create Replication section - self.spenny_doc.add_heading("Replication Steps", 4) - self.process_text_xml(finding["replication_steps"], finding) - - # Check if techniques are provided before creating a host detection section - if finding["host_detection_techniques"]: - # \u2013 is an em-dash - self.spenny_doc.add_heading( - u"Adversary Detection Techniques \u2013 Host", 4 - ) - self.process_text_xml(finding["host_detection_techniques"], finding) - # Check if techniques are provided before creating a network detection section - if finding["network_detection_techniques"]: - # \u2013 is an em-dash - self.spenny_doc.add_heading( - u"Adversary Detection Techniques \u2013 Network", 4 - ) - self.process_text_xml(finding["network_detection_techniques"], finding) + def render_subdocument(section, finding): + self.sacrificial_doc = self.word_doc.new_subdoc() + self.process_text_xml(section, finding) + return self.sacrificial_doc - # Create References section - if finding["references"]: - self.spenny_doc.add_heading("References", 4) - self.process_text_xml(finding["references"], finding) - counter += 1 + for finding in context["findings"]: + logger.info("Processing %s", finding["title"]) + # Create ``RichText()`` object for a colored severity category + finding["severity_rt"] = RichText( + finding["severity"], color=finding["severity_color"] + ) + # Create subdocuments for each finding section + finding["affected_entities_rt"] = render_subdocument( + finding["affected_entities"], finding + ) + finding["description_rt"] = render_subdocument( + finding["description"], finding + ) + finding["impact_rt"] = render_subdocument(finding["impact"], finding) + finding["recommendation_rt"] = render_subdocument( + finding["recommendation"], finding + ) + finding["replication_steps_rt"] = render_subdocument( + finding["replication_steps"], finding + ) + finding["host_detection_techniques_rt"] = render_subdocument( + finding["host_detection_techniques"], finding + ) + finding["network_detection_techniques_rt"] = render_subdocument( + finding["network_detection_techniques"], finding + ) + finding["references_rt"] = render_subdocument( + finding["references"], finding + ) - # Check if this is the last finding to avoid an extra blank page - if not counter == total_findings: - self.spenny_doc.add_page_break() + return context def process_text_xlsx(self, html, text_format, finding): """ @@ -1402,7 +1763,27 @@ def process_text_xlsx(self, html, text_format, finding): text = text.replace( "{{.client}}", self.report_json["client"]["full_name"] ) - text = text.replace("{{.caption}}", u"Caption \u2013 ") + if "{{.project_start}}" in text: + text = text.replace( + "{{.project_start}}", self.report_json["project"]["start_date"] + ) + if "{{.project_end}}" in text: + text = text.replace( + "{{.project_end}}", self.report_json["project"]["end_date"] + ) + if "{{.project_start_uk}}" in text: + text = text.replace( + "{{.project_start_uk}}", self.report_json["project"]["start_date_uk"] + ) + if "{{.project_end_uk}}" in text: + text = text.replace( + "{{.project_end_uk}}", self.report_json["project"]["end_date_uk"] + ) + if "{{.project_type}}" in text: + text = text.replace( + "{{.project_type}}", self.report_json["project"]["project_type"].lower() + ) + text = text.replace("{{.caption}}", "Caption \u2013 ") # Find/replace evidence keywords because they're ugly and don't make sense when read match = re.findall(keyword_regex, text) if match: @@ -1411,7 +1792,7 @@ def process_text_xlsx(self, html, text_format, finding): # \u2013 is an em-dash text = text.replace( "{{." + keyword + "}}", - u"\n\nCaption \u2013 {}".format( + "\n\nCaption \u2013 {}".format( finding["evidence"][keyword]["friendly_name"], finding["evidence"][keyword]["caption"], ), @@ -1431,21 +1812,21 @@ def generate_excel_xlsx(self, memory_object): self.report_json = json.loads(self.generate_json()) # Create xlsxwriter - spenny_doc = memory_object - self.worksheet = spenny_doc.add_worksheet("Findings") + word_doc = memory_object + self.worksheet = word_doc.add_worksheet("Findings") # Create some basic formats # Header format - bold_format = spenny_doc.add_format({"bold": True}) + bold_format = word_doc.add_format({"bold": True}) bold_format.set_text_wrap() bold_format.set_align("vcenter") # Affected assets format - asset_format = spenny_doc.add_format() + asset_format = word_doc.add_format() asset_format.set_text_wrap() asset_format.set_align("vcenter") asset_format.set_align("center") # Remaining cells - wrap_format = spenny_doc.add_format() + wrap_format = word_doc.add_format() wrap_format.set_text_wrap() wrap_format.set_align("vcenter") # Create header row for findings @@ -1478,21 +1859,12 @@ def generate_excel_xlsx(self, memory_object): self.worksheet.write(self.row, self.col, finding["title"], wrap_format) self.col += 1 # Severity - severity_format = spenny_doc.add_format({"bold": True}) + severity_format = word_doc.add_format({"bold": True}) severity_format.set_align("vcenter") severity_format.set_align("center") severity_format.set_font_color("black") # Color the cell based on corresponding severity color - if finding["severity"].lower() == "informational": - severity_format.set_bg_color(self.informational_color) - elif finding["severity"].lower() == "low": - severity_format.set_bg_color(self.low_color) - elif finding["severity"].lower() == "medium": - severity_format.set_bg_color(self.medium_color) - elif finding["severity"].lower() == "high": - severity_format.set_bg_color(self.high_color) - elif finding["severity"].lower() == "critical": - severity_format.set_bg_color(self.critical_color) + severity_format.set_bg_color(finding["severity_color"]) self.worksheet.write(self.row, 1, finding["severity"], severity_format) self.col += 1 @@ -1562,43 +1934,36 @@ def generate_excel_xlsx(self, memory_object): ) # Finalize document - spenny_doc.close() - return spenny_doc + word_doc.close() + return word_doc def generate_powerpoint_pptx(self): """ Generate a complete PowerPoint slide deck for the current report. """ self.report_type = "pptx" - # Generate the JSON for the report self.report_json = json.loads(self.generate_json()) # Create document writer using the specified template - if self.template_loc: - try: - self.spenny_ppt = Presentation(self.template_loc) - except Exception: - raise - else: - raise - self.ppt_color_info = pptx.dml.color.RGBColor( - self.informational_color_hex[0], - self.informational_color_hex[1], - self.informational_color_hex[2], - ) - self.ppt_color_low = pptx.dml.color.RGBColor( - self.low_color_hex[0], self.low_color_hex[1], self.low_color_hex[2] - ) - self.ppt_color_medium = pptx.dml.color.RGBColor( - self.medium_color_hex[0], self.medium_color_hex[1], self.medium_color_hex[2] - ) - self.ppt_color_high = pptx.dml.color.RGBColor( - self.high_color_hex[0], self.high_color_hex[1], self.high_color_hex[2] - ) - self.ppt_color_critical = pptx.dml.color.RGBColor( - self.critical_color_hex[0], - self.critical_color_hex[1], - self.critical_color_hex[2], - ) + try: + self.ppt_presentation = Presentation(self.template_loc) + except ValueError: + logger.exception( + "Failed to load the provided template document because it is not a PowerPoint file: %s", + self.template_loc, + ) + raise ValueError + except PptxPackageNotFoundError: + logger.exception( + "Failed to load the provided template document because file could not be found: %s", + self.template_loc, + ) + raise PptxPackageNotFoundError + except Exception: + logger.exception( + "Failed to load the provided template document for unknown reason: %s", + self.template_loc, + ) + # Loop through the dict of findings to create slides based on findings # Initialize findings stats dict findings_stats = { @@ -1608,6 +1973,14 @@ def generate_powerpoint_pptx(self): "Low": 0, "Informational": 0, } + + def get_textframe(shape): + text_frame = shape.text_frame + # Fits content to text frame, but only triggers on update after opening file + text_frame.auto_size = MSO_AUTO_SIZE.TEXT_TO_FIT_SHAPE + + return text_frame + # Calculate finding stats for finding in self.report_json["findings"].values(): findings_stats[finding["severity"]] += 1 @@ -1617,15 +1990,14 @@ def generate_powerpoint_pptx(self): SLD_LAYOUT_FINAL = 12 # Add a title slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title body_shape = shapes.placeholders[1] - title_shape.text = settings.COMPANY_NAME - text_frame = body_shape.text_frame - # Use text_frame.text for first line/paragraph or - # text_frame.paragraphs[0] + title_shape.text = self.company_config.company_name + text_frame = get_textframe(body_shape) + # Use ``text_frame.text`` for first line/paragraph or ``text_frame.paragraphs[0]`` text_frame.text = "{} Debrief".format( self.report_json["project"]["project_type"] ) @@ -1633,49 +2005,49 @@ def generate_powerpoint_pptx(self): p.text = self.report_json["client"]["full_name"] # Add Agenda slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title title_shape.text = "Agenda" body_shape = shapes.placeholders[1] - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) # Add Introduction slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title title_shape.text = "Introduction" body_shape = shapes.placeholders[1] - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) # Add Methodology slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title title_shape.text = "Methodology" body_shape = shapes.placeholders[1] - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) # Add Attack Path Overview slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title title_shape.text = "Attack Path Overview" body_shape = shapes.placeholders[1] - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) # Add Findings Overview Slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title body_shape = shapes.placeholders[1] title_shape.text = "Findings Overview" - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) for stat in findings_stats: p = text_frame.add_paragraph() p.text = "{} Findings".format(stat) @@ -1721,16 +2093,10 @@ def generate_powerpoint_pptx(self): # Set cell color fill type to solid risk_cell.fill.solid() # Color the risk cell based on corresponding severity color - if finding["severity"].lower() == "informational": - risk_cell.fill.fore_color.rgb = self.ppt_color_info - elif finding["severity"].lower() == "low": - risk_cell.fill.fore_color.rgb = self.ppt_color_low - elif finding["severity"].lower() == "medium": - risk_cell.fill.fore_color.rgb = self.ppt_color_medium - elif finding["severity"].lower() == "high": - risk_cell.fill.fore_color.rgb = self.ppt_color_high - elif finding["severity"].lower() == "critical": - risk_cell.fill.fore_color.rgb = self.ppt_color_critical + cell_color = pptx.dml.color.RGBColor( + *map(lambda v: int(v, 16), finding["severity_color_hex"]) + ) + risk_cell.fill.fore_color.rgb = cell_color row_iter += 1 # Set all cells alignment to center and vertical center for cell in table.iter_cells(): @@ -1743,16 +2109,26 @@ def generate_powerpoint_pptx(self): # Create slide for each finding for finding in self.report_json["findings"].values(): - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - self.finding_slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[ + SLD_LAYOUT_TITLE_AND_CONTENT + ] + self.finding_slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = self.finding_slide.shapes title_shape = shapes.title + + # Prepare text frame self.finding_body_shape = shapes.placeholders[1] + if self.finding_body_shape.has_text_frame: + text_frame = get_textframe(self.finding_body_shape) + text_frame.clear() + self.delete_paragraph(text_frame.paragraphs[0]) + title_shape.text = "{} [{}]".format(finding["title"], finding["severity"]) if finding["description"]: self.process_text_xml(finding["description"], finding) else: self.process_text_xml("

No description provided

", finding) + # Strip all HTML tags and replace any \x0D characters for pptx entities = BeautifulSoup(finding["affected_entities"], "lxml").text.replace( "\x0D", "" @@ -1781,51 +2157,51 @@ def generate_powerpoint_pptx(self): ) # Add Observations slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title body_shape = shapes.placeholders[1] title_shape.text = "Positive Observations" - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) # Add Recommendations slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title body_shape = shapes.placeholders[1] title_shape.text = "Recommendations" - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) # Add Conclusion slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_TITLE_AND_CONTENT] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes title_shape = shapes.title body_shape = shapes.placeholders[1] title_shape.text = "Positive Observations" - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) # Add final slide - slide_layout = self.spenny_ppt.slide_layouts[SLD_LAYOUT_FINAL] - slide = self.spenny_ppt.slides.add_slide(slide_layout) + slide_layout = self.ppt_presentation.slide_layouts[SLD_LAYOUT_FINAL] + slide = self.ppt_presentation.slides.add_slide(slide_layout) shapes = slide.shapes body_shape = shapes.placeholders[1] - text_frame = body_shape.text_frame + text_frame = get_textframe(body_shape) text_frame.clear() p = text_frame.paragraphs[0] p.line_spacing = 0.7 - p.text = settings.COMPANY_NAME + p.text = self.company_config.company_name p = text_frame.add_paragraph() - p.text = settings.COMPANY_TWITTER + p.text = self.company_config.company_twitter p.line_spacing = 0.7 p = text_frame.add_paragraph() - p.text = settings.COMPANY_EMAIL + p.text = self.company_config.company_email p.line_spacing = 0.7 # Finalize document and return it for an HTTP response - return self.spenny_ppt + return self.ppt_presentation def generate_all_reports(self, docx_template, pptx_template): """ @@ -1858,3 +2234,177 @@ def generate_all_reports(self, docx_template, pptx_template): raise # Return each memory object return self.report_json, word_stream, excel_stream, ppt_stream + + +class TemplateLinter: + """Lint template files to catch undefined variables and syntax errors.""" + + def __init__(self, template_loc): + self.template_loc = template_loc + self.jinja_template_env = jinja2.Environment( + undefined=jinja2.DebugUndefined, extensions=["jinja2.ext.debug"] + ) + self.jinja_template_env.filters["filter_severity"] = self.dummy_filter_severity + self.jinja_template_env.filters["strip_html"] = self.dummy_strip_html + + def dummy_filter_severity(self, value, allowlist): + return [] + + def dummy_strip_html(self, value): + return [] + + def lint_docx(self): + """ + Lint the provided Word docx file from :model:`reporting.ReportTemplate`. + """ + results = {"result": "success", "warnings": [], "errors": []} + if self.template_loc: + if os.path.exists(self.template_loc): + logger.info("Found template file at %s", self.template_loc) + try: + # Dummy context data to identify undefined custom variables in template + context = {} + + context["report_date"] = datetime.now().strftime("%B %d, %Y") + context["report_date_uk"] = datetime.now().strftime("%d %B %Y") + + # Client information + context["client"] = "Kabletown" + context["client_short"] = "Kabletown" + context["client_pocs"] = "" + + # Assessment information + context["assessment_name"] = "" + context["assessment_type"] = "" + context["project_type"] = "" + context["company"] = "" + context["company_pocs"] = "" + + # Project dates + context["project_start_date"] = "" + context["project_start_date_uk"] = "" + context["project_end_date"] = "" + context["project_end_date_uk"] = "" + + context["execution_window"] = "" + context["execution_window_uk"] = "" + + # Infrastructure information + context["domains"] = "" + context["static_servers"] = "" + context["cloud_servers"] = "" + context["domains_and_servers"] = "" + + # Findings information + context["findings"] = "" + context["findings_subdoc"] = "" + + # Step 1: Load the document as a template + template_document = DocxTemplate(self.template_loc) + logger.info("Template loaded for linting") + + # Step 2: Check document's styles + document_styles = template_document.styles + if "Bullet List" not in document_styles: + results["warnings"].append( + "Template is missing a recommended style (see documentation): Bullet List" + ) + if "Number List" not in document_styles: + results["warnings"].append( + "Template is missing a recommended style (see documentation): Number List" + ) + if "CodeBlock" not in document_styles: + results["warnings"].append( + "Template is missing a recommended style (see documentation): CodeBlock" + ) + if "CodeInline" not in document_styles: + results["warnings"].append( + "Template is missing a recommended style (see documentation): CodeInline" + ) + logger.info("Completed Word style checks") + + # Step 3: Test rendering the document + try: + template_document.render( + context, self.jinja_template_env, autoescape=True + ) + undefined_vars = template_document.undeclared_template_variables + if undefined_vars: + for variable in undefined_vars: + results["warnings"].append( + f"Undefined variable: {variable}" + ) + if results["warnings"]: + results["result"] = "warning" + logger.info("Completed document rendering test") + except TemplateSyntaxError as error: + logger.error("Template syntax error: %s", error) + results = { + "result": "failed", + "errors": [ + f"Jinja2 template syntax error: {error.message}" + ], + } + except Exception: + logger.exception("Template failed rendering") + results = { + "result": "failed", + "errors": ["Template rendering failed unexpectedly"], + } + else: + logger.error("Template file path did not exist: %s", self.template_loc) + results = { + "result": "failed", + "errors": ["Template file does not exist – upload it again"], + } + else: + logger.error("Received a `None` value for template location") + + logger.info("Template linting completed") + return json.dumps(results) + + def lint_pptx(self): + """ + Lint the provided PowerPoint pptx file from :model:`reporting.ReportTemplate`. + """ + results = {"result": "success", "warnings": [], "errors": []} + if self.template_loc: + if os.path.exists(self.template_loc): + logger.info("Found template file at %s", self.template_loc) + try: + # Test 1: Check if the document is a PPTX file + template_document = Presentation(self.template_loc) + + # Test 2: Check for existing slides + slide_count = len(template_document.slides) + logger.warning("Slide count was %s", slide_count) + if slide_count > 0: + results["warnings"].append( + "Template can be used, but it has slides when it should be empty (see documentation)" + ) + except ValueError: + logger.exception( + "Failed to load the provided template document because it is not a PowerPoint file: %s", + self.template_loc, + ) + results = { + "result": "failed", + "errors": ["Template file is not a PowerPoint presentation"], + } + except Exception: + logger.exception("Template failed rendering") + results = { + "result": "failed", + "errors": ["Template rendering failed unexpectedly"], + } + else: + logger.error("Template file path did not exist: %s", self.template_loc) + results = { + "result": "failed", + "errors": ["Template file does not exist – upload it again"], + } + else: + logger.error("Received a `None` value for template location") + + logger.info("Template linting completed") + return json.dumps(results) diff --git a/ghostwriter/modules/review.py b/ghostwriter/modules/review.py index 8f79d96c5..50a6b43f0 100644 --- a/ghostwriter/modules/review.py +++ b/ghostwriter/modules/review.py @@ -5,42 +5,48 @@ This module contains the ``DomainReview`` class. The class checks if a domain name is properly categorized, has not been flagged in VirusTotal, or tagged with a bad category. -``DomainReview`` checks VirusTotal, Cisco Talos, Bluecoat, -IBM X-Force, Fortiguard, TrendMicro, OpeDNS, and MXToolbox. Domains will also -be checked against malwaredomains.com's list of reported domains. +``DomainReview`` checks VirusTotal and malwaredomains.com's list of reported domains. """ # Standard Libraries -import base64 -import json -import os -import re -import shutil from time import sleep +import traceback +import logging # Django & Other 3rd Party Libraries import requests -from bs4 import BeautifulSoup -from django.conf import settings -from lxml import etree + +# Ghostwriter Libraries +from ghostwriter.commandcenter.models import VirusTotalConfiguration # Disable requests warnings for things like disabling certificate checking requests.packages.urllib3.disable_warnings() +# Using __name__ resolves to ghostwriter.modules.review +logger = logging.getLogger(__name__) + class DomainReview(object): - """Class to pull a list of registered domains belonging to a Namecheap - account and then check the web reputation of each domain. """ + Pull a list of domain names and check their web reputation. + """ + + # Get API configuration + virustotal_config = VirusTotalConfiguration.get_solo() + if virustotal_config.enable is False: + logger.error( + "Tried to run a domain review without VirusTotal configured and enabled" + ) + exit() + sleep_time = virustotal_config.sleep_time # API endpoints malwaredomains_url = "http://mirror1.malwaredomains.com/files/justdomains" - virustotal_domain_report_uri = ( - "https://www.virustotal.com/vtapi/v2/domain/report?apikey={}&domain={}" - ) + virustotal_domain_report_uri = "https://www.virustotal.com/vtapi/v2/domain/report?apikey={api_key}&domain={domain}" + # Categories we don't want to see # These are lowercase to avoid inconsistencies with how each service might return the categories - blacklisted = [ + blocklist = [ "phishing", "web ads/analytics", "suspicious", @@ -55,489 +61,202 @@ class DomainReview(object): useragent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36" session = requests.Session() - def __init__(self, domain_queryset): - """Everything that needs to be setup when a new DomainReview object is - created goes here. - """ - # Domain query results from the Django models + def __init__(self, domain_queryset, sleep_time_override): self.domain_queryset = domain_queryset - # Try to get the sleep time configured in settings - try: - self.request_delay = settings.DOMAINCHECK_CONFIG["sleep_time"] - except Exception: - self.request_delay = 20 - try: - self.virustotal_api_key = settings.DOMAINCHECK_CONFIG["virustotal_api_key"] - except Exception: - self.virustotal_api_key = None - print( - "[!] A VirusTotal API key could not be pulled from " - "settings.py. Review settings to perform VirusTotal checks." - ) - exit() + + # Override globally configured sleep time + if sleep_time_override: + self.sleep_time = sleep_time_override def check_virustotal(self, domain, ignore_case=False): - """Check the provided domain name with VirusTotal. VirusTotal's API is - case sensitive, so the domain will be converted to lowercase by - default. This can be disabled using the ignore_case parameter. + """ + Look-up the provided domain name in VirusTotal. - This uses the VirusTotal /domain/report endpoint: + **Parameters** - https://developers.virustotal.com/v2.0/reference#domain-report + ``domain`` + Domain name to search + ``ignore_case`` + Do not convert domain name to lowercase (Default: False) """ - if self.virustotal_api_key: + results = {} + results["result"] = "success" + if self.virustotal_config.enable: + # The VT API is case sensitive, so domains should always be lowercase if not ignore_case: domain = domain.lower() try: req = self.session.get( self.virustotal_domain_report_uri.format( - self.virustotal_api_key, domain + api_key=self.virustotal_config.api_key, domain=domain ) ) - vt_data = req.json() + if req.ok: + vt_data = req.json() + results["data"] = vt_data + else: + results["result"] = "error" + results["error"] = "VirusTotal rejected the API key in settings" except Exception: - vt_data = None - return vt_data + trace = traceback.format_exc() + logger.exception("Failed to contact VirusTotal") + results["result"] = "error" + results["error"] = "{exception}".format(exception=trace) else: - return None - - def check_talos(self, domain): - """Check the provided domain's category as determined by - Cisco Talos. - """ - categories = [] - cisco_talos_uri = "https://talosintelligence.com/sb_api/query_lookup?query=%2Fapi%2Fv2%2Fdetails%2Fdomain%2F&query_entry={}&offset=0&order=ip+asc" - headers = { - "User-Agent": self.useragent, - "Referer": "https://www.talosintelligence.com/reputation_center/lookup?search=" - + domain, - } - try: - req = self.session.get(cisco_talos_uri.format(domain), headers=headers) - if req.ok: - json_data = req.json() - print(req.text) - if "category" in json_data: - categories.append(json_data["category"]["description"]) - else: - categories.append("Uncategorized") - else: - print( - "[!] Cisco Talos check request failed. Talos did not " - "return a 200 response." - ) - print('L.. Request returned status "{}"'.format(req.status_code)) - except Exception as error: - print("[!] Cisco Talos request failed: {}".format(error)) - return categories - - def check_ibm_xforce(self, domain): - """Check the provided domain's category as determined by - IBM X-Force. - """ - categories = [] - xforce_uri = "https://exchange.xforce.ibmcloud.com/url/{}".format(domain) - headers = { - "User-Agent": self.useragent, - "Accept": "application/json, text/plain, */*", - "x-ui": "XFE", - "Origin": xforce_uri, - "Referer": xforce_uri, - } - xforce_api_uri = "https://api.xforce.ibmcloud.com/url/{}".format(domain) - try: - req = self.session.get(xforce_api_uri, headers=headers, verify=False) - if req.ok: - response = req.json() - if not response["result"]["cats"]: - categories.append("Uncategorized") - else: - # Parse all dictionary keys and append to single string to - # get Category names - for key in response["result"]["cats"]: - categories.append(key) - # IBM X-Force returns a 404 with {"error":"Not found."} if the - # domain is unknown - elif req.status_code == 404: - categories.append("Unknown") - else: - print( - "[!] IBM X-Force check request failed. X-Force did not " - "return a 200 response." - ) - print('L.. Request returned status "{}"'.format(req.status_code)) - except Exception as error: - print("[!] IBM X-Force request failed: {}".format(error)) - return categories - - def check_fortiguard(self, domain): - """Check the provided domain's category as determined by - Fortiguard Webfilter. - """ - categories = [] - fortiguard_uri = "https://fortiguard.com/webfilter?q=" + domain - headers = { - "User-Agent": self.useragent, - "Origin": "https://fortiguard.com", - "Referer": "https://fortiguard.com/webfilter", - } - try: - req = self.session.get(fortiguard_uri, headers=headers) - if req.ok: - """ - Example HTML result: -
-
-
-

Category: Education

- """ - # TODO: Might be best to BS4 for this rather than regex - cat = re.findall('Category: (.*?)" />', req.text, re.DOTALL) - categories.append(cat[0]) - else: - print( - "[!] Fortiguard check request failed. Fortiguard did " - "not return a 200 response." - ) - print('L.. Request returned status "{}"'.format(req.status_code)) - except Exception as error: - print("[!] Fortiguard request failed: {}".format(error)) - return categories - - def check_bluecoat(self, domain, ocr=True): - """Check the provided domain's category as determined by - Symantec Bluecoat. - """ - categories = [] - bluecoat_uri = "https://sitereview.bluecoat.com/#/" - bluecoat_captcha = "https://sitereview.bluecoat.com/resource/captcha-request" - bluecoat_lookup = "https://sitereview.bluecoat.com/resource/lookup" - post_data_1 = {"check": "captcha"} - post_data_2 = {"url": domain, "captcha": ""} - headers = {"User-Agent": self.useragent} - try: - response = self.session.get(bluecoat_uri, headers=headers, verify=False) - response = self.session.post( - bluecoat_captcha, headers=headers, json=post_data_1, verify=False - ) - # Update headers with CSRF token - headers = { - "User-Agent": self.useragent, - "Content-Type": "application/json; charset=UTF-8", - "Referer": "https://sitereview.bluecoat.com/lookup", - "X-XSRF-TOKEN": self.session.cookies["XSRF-TOKEN"], - } - response = self.session.post( - bluecoat_lookup, headers=headers, json=post_data_2, verify=False - ) - root = etree.fromstring(response.text) - for node in root.xpath( - "//CategorizationResult//categorization//categorization//name" - ): - categories.append(node.text) - except Exception as error: - print("[!] Bluecoat request failed: {0}".format(error)) - return categories + results["result"] = "error" + results["error"] = "VirusTotal is disabled in settings" - def check_mxtoolbox(self, domain): - """Check if the provided domain is blacklisted as spam as determined - by MX Toolkit. - """ - issues = [] - mxtoolbox_url = "https://mxtoolbox.com/Public/Tools/BrandReputation.aspx" - headers = { - "User-Agent": self.useragent, - "Origin": mxtoolbox_url, - "Referer": mxtoolbox_url, - } - try: - response = self.session.get(url=mxtoolbox_url, headers=headers) - soup = BeautifulSoup(response.content, "lxml") - viewstate = soup.select("input[name=__VIEWSTATE]")[0]["value"] - viewstategenerator = soup.select("input[name=__VIEWSTATEGENERATOR]")[0][ - "value" - ] - eventvalidation = soup.select("input[name=__EVENTVALIDATION]")[0]["value"] - data = { - "__EVENTTARGET": "", - "__EVENTARGUMENT": "", - "__VIEWSTATE": viewstate, - "__VIEWSTATEGENERATOR": viewstategenerator, - "__EVENTVALIDATION": eventvalidation, - "ctl00$ContentPlaceHolder1$brandReputationUrl": domain, - "ctl00$ContentPlaceHolder1$brandReputationDoLookup": "Brand Reputation Lookup", - "ctl00$ucSignIn$hfRegCode": "missing", - "ctl00$ucSignIn$hfRedirectSignUp": "/Public/Tools/BrandReputation.aspx", - "ctl00$ucSignIn$hfRedirectLogin": "", - "ctl00$ucSignIn$txtEmailAddress": "", - "ctl00$ucSignIn$cbNewAccount": "cbNewAccount", - "ctl00$ucSignIn$txtFullName": "", - "ctl00$ucSignIn$txtModalNewPassword": "", - "ctl00$ucSignIn$txtPhone": "", - "ctl00$ucSignIn$txtCompanyName": "", - "ctl00$ucSignIn$drpTitle": "", - "ctl00$ucSignIn$txtTitleName": "", - "ctl00$ucSignIn$txtModalPassword": "", - } - response = self.session.post(url=mxtoolbox_url, headers=headers, data=data) - soup = BeautifulSoup(response.content, "lxml") - if soup.select("div[id=ctl00_ContentPlaceHolder1_noIssuesFound]"): - issues.append("No issues found") - else: - if soup.select( - "div[id=ctl00_ContentPlaceHolder1_" "googleSafeBrowsingIssuesFound]" - ): - issues.append("Google SafeBrowsing Issues Found.") - if soup.select( - "div[id=ctl00_ContentPlaceHolder1_" "phishTankIssuesFound]" - ): - issues.append("PhishTank Issues Found") - except Exception: - print( - "[!] Error retrieving Google SafeBrowsing and PhishTank " "reputation!" - ) - return issues + return results - def check_opendns(self, domain): - """Check the provided domain's category as determined by the - OpenDNS community. + def download_malware_domains(self): """ - categories = [] - opendns_uri = "https://domain.opendns.com/{}" - headers = {"User-Agent": self.useragent} - try: - response = self.session.get( - opendns_uri.format(domain), headers=headers, verify=False - ) - soup = BeautifulSoup(response.content, "lxml") - tags = soup.find("span", {"class": "normal"}) - if tags: - categories = tags.text.strip().split(", ") - else: - categories.append("No Tags") - except Exception as error: - print("[!] OpenDNS request failed: {0}".format(error)) - return categories - - def check_trendmicro(self, domain): - """Check the provided domain's category as determined by - Trend Micro. + Download the malwaredomains.com list of malicious domains. """ - categories = [] - trendmicro_uri = "https://global.sitesafety.trendmicro.com/" - trendmicro_stage_1_uri = "https://global.sitesafety.trendmicro.com/lib/idn.php" - trendmicro_stage_2_uri = "https://global.sitesafety.trendmicro.com/result.php" - headers = {"User-Agent": self.useragent} - headers_stage_1 = { - "Host": "global.sitesafety.trendmicro.com", - "Accept": "*/*", - "Origin": "https://global.sitesafety.trendmicro.com", - "X-Requested-With": "XMLHttpRequest", - "User-Agent": self.useragent, - "Content-Type": "application/x-www-form-urlencoded", - "Referer": "https://global.sitesafety.trendmicro." "com/index.php", - "Accept-Encoding": "gzip, deflate", - "Accept-Language": "en-US, en;q=0.9", - } - headers_stage_2 = { - "Origin": "https://global.sitesafety.trendmicro.com", - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": self.useragent, - "Accept": "text/html, application/xhtml+xml, " - "application/xml;q=0.9, image/webp, image/apng, " - "*/*;q=0.8", - "Referer": "https://global.sitesafety.trendmicro." "com/index.php", - "Accept-Encoding": "gzip, deflate", - "Accept-Language": "en-US, en;q=0.9", - } - data_stage_1 = {"url": domain} - data_stage_2 = {"urlname": domain, "getinfo": "Check Now"} - try: - response = self.session.get(trendmicro_uri, headers=headers) - response = self.session.post( - trendmicro_stage_1_uri, headers=headers_stage_1, data=data_stage_1 - ) - response = self.session.post( - trendmicro_stage_2_uri, headers=headers_stage_2, data=data_stage_2 - ) - # Check if session was redirected to /captcha.php - if "captcha" in response.url: - print( - "[!] TrendMicro responded with a reCAPTCHA, so cannot " - "proceed with TrendMicro." - ) - print( - "L.. You can try solving it yourself: " - "https://global.sitesafety.trendmicro.com/captcha.php" - ) - else: - soup = BeautifulSoup(response.content, "lxml") - tags = soup.find("div", {"class": "labeltitlesmallresult"}) - if tags: - categories = tags.text.strip().split(", ") - else: - categories.append("Uncategorized") - except Exception as error: - print("[!] Trend Micro request failed: {0}".format(error)) - return categories - - def download_malware_domains(self): - """Downloads the malwaredomains.com list of malicious domains.""" + results = {} + results["result"] = "success" headers = {"User-Agent": self.useragent} response = self.session.get( url=self.malwaredomains_url, headers=headers, verify=False ) malware_domains = response.text if response.status_code == 200: - return malware_domains + results["malware_domains"] = malware_domains + logger.info( + "Successfully collected list of %s domains from malwaredomains.com", + len(results["malware_domains"]), + ) + return results else: - print( - "[!] Error reaching: {}, Status: {}".format( - self.malwaredomains_url, response.status_code - ) + results["result"] = "error" + results["error"] = "Received status code {status_code}".format( + status_code=response.status_code + ) + logger.error( + "Failed to fetch the malwaredomains.com list from %s with status code: %s", + self.malwaredomains_url, + response.status_code, ) - return None + return results def check_domain_status(self): - """Check the status of each domain in the provided list collected from - the Domain model. Each domain will be checked to ensure the domain is - not flagged/blacklisted. A domain will be considered burned if - VirusTotal returns detections for the domain or one of the domain's - categories appears in the list of bad categories. - - VirusTotal allows 4 requests every 1 minute. A minimum of 20 seconds - is recommended to allow for some consideration on the service. - + """ + Check the status of each domain name in the provided :model:`shepherd.Domain` + queryset. Mark the domain as burned if a vendor has flagged it for malware or + phishing or assigned it an undesirable category. """ lab_results = {} malware_domains = self.download_malware_domains() for domain in self.domain_queryset: - print("[+] Starting update of {}".format(domain.name)) - burned_dns = False - domain_categories = [] - # Sort the domain information from queryset - domain_name = domain.name - health = domain.health_status - print("[+] Domain is currently flagged as {}".format(health)) - # Check if domain is known to be burned and skip it if so - # This just saves time and operators can edit a domain and set - # status to `Healthy` as needed - # The domain will be included in the next update after the edit - if health != "Healthy": - burned = False - else: - burned = True - if not burned: + burned = False + if domain.is_expired() is False: + domain_categories = [] burned_explanations = [] - if domain.burned_explanation: - burned_explanations.append(domain.burned_explanation) + lab_results[domain] = {} + logger.info("Starting domain category update for %s", domain.name) + + # Sort the domain information from queryset + domain_name = domain.name + health = domain.health_status + logger.info("Domain is currently considered to be %s", health) + # Check if domain is flagged for malware - if malware_domains: - if domain_name in malware_domains: - print( - "[!] {}: Identified as a known malware domain " - "(malwaredomains.com)!".format(domain_name) + if malware_domains["result"] == "success": + if domain_name in malware_domains["malware_domains"]: + logger.warning( + "The domain %s is listed as a known malicious domain", + domain_name, ) burned = True - burned_explanations.append("Flagged by malwaredomains.com") + burned_explanations.append( + f"

Flagged for malware by malwaredomains.com. See {self.malwaredomains_url} for the list.

" + ) + # Check domain name with VirusTotal vt_results = self.check_virustotal(domain_name) - if vt_results: - if "categories" in vt_results: - domain_categories = vt_results["categories"] - # Check if VirusTotal has any detections for URLs or - # malware samples - if "detected_downloaded_samples" in vt_results: - if len(vt_results["detected_downloaded_samples"]) > 0: - print( - "[!] {}: Identified as having a downloaded " - "sample on VirusTotal!".format(domain_name) + if vt_results["result"] == "success": + logger.info("Received results for %s from VirusTotal", domain_name) + lab_results[domain]["vt_results"] = vt_results["data"] + + # Locate category data + if "categories" in vt_results["data"]: + domain_categories = vt_results["data"]["categories"] + search_key = "category" + more_categories = [ + val + for key, val in vt_results["data"].items() + if search_key in key + ] + domain_categories.extend(more_categories) + + # Make categories unique + domain_categories = list(set(domain_categories)) + + # Check if VirusTotal has any detections for URLs or malware samples + if "detected_downloaded_samples" in vt_results["data"]: + if len(vt_results["data"]["detected_downloaded_samples"]) > 0: + total_detections = len(vt_results["data"]["detected_downloaded_samples"]) + logger.warning( + "Domain %s is tied to {total_detections} VirusTotal malware sample(s)", + domain_name, ) burned = True burned_explanations.append( - "Tied to a VirusTotal " "detected malware" "sample" + f"

Tied to {total_detections} VirusTotal malware sample(s):

" ) - if "detected_urls" in vt_results: - if len(vt_results["detected_urls"]) > 0: - print( - "[!] {}: Identified as having a URL " - "detection on VirusTotal!".format(domain_name) + for detection in vt_results["data"]["detected_downloaded_samples"]: + burned_explanations.append( + f"

SHA256 hash {detection['sha256']} flagged by {detection['positives']} vendors on {detection['date']}

" + ) + if "detected_urls" in vt_results["data"]: + if len(vt_results["data"]["detected_urls"]) > 0: + total_detections = len(vt_results["data"]["detected_urls"]) + logger.warning( + "Domain %s has a positive malware detection on VirusTotal.", + domain_name, ) burned = True burned_explanations.append( - "Tied to a VirusTotal " "detected URL" - ) - # Get passive DNS results from VirusTotal JSON - ip_addresses = [] - if "resolutions" in vt_results: - for address in vt_results["resolutions"]: - ip_addresses.append( - { - "address": address["ip_address"], - "timestamp": address["last_resolved"].split(" ")[0], - } + f"

Domain has {total_detections} malware detections on VirusTotal:

" ) - bad_addresses = [] - if burned_dns: - print( - "[*] {}: Identified as pointing to suspect IP " - "addresses (VirusTotal passive DNS).".format(domain_name) - ) - health_dns = "Flagged DNS ({})".format(", ".join(bad_addresses)) + for detection in vt_results["data"]["detected_urls"]: + burned_explanations.append( + f"

{detection['url']} flagged by {detection['positives']} vendors on {detection['scan_date']}

" + ) else: - health_dns = "Healthy" - # Collect categories from the other sources - xforce_results = self.check_ibm_xforce(domain_name) - domain_categories.extend(xforce_results) - talos_results = self.check_talos(domain_name) - domain_categories.extend(talos_results) - bluecoat_results = self.check_bluecoat(domain_name) - domain_categories.extend(bluecoat_results) - fortiguard_results = self.check_fortiguard(domain_name) - domain_categories.extend(fortiguard_results) - opendns_results = self.check_opendns(domain_name) - domain_categories.extend(opendns_results) - trendmicro_results = self.check_trendmicro(domain_name) - domain_categories.extend(trendmicro_results) - mxtoolbox_results = self.check_mxtoolbox(domain_name) - domain_categories.extend(domain_categories) - # Make categories unique - domain_categories = list(set(domain_categories)) - # Check if any categopries are suspect + lab_results[domain]["vt_results"] = "none" + logger.warning( + "Did not receive results for %s from VirusTotal", domain_name + ) + + # Check if any categories are suspect bad_categories = [] for category in domain_categories: - if category.lower() in self.blacklisted: - bad_categories.append(category.capitalize()) + if category.lower() in self.blocklist: + bad_categories.append(category) if bad_categories: + logger.warning( + "Domain %s is now burned because of undesirable categories: %s", + domain_name, + bad_categories, + ) burned = True - burned_explanations.append("Tagged with a bad category") + burned_explanations.append( + "

Tagged with one or more undesirable categories: {bad_cat}

".format( + bad_cat=", ".join(bad_categories) + ) + ) + # Assemble the dictionary to return for this domain - lab_results[domain] = {} - lab_results[domain]["categories"] = {} lab_results[domain]["burned"] = burned - lab_results[domain]["health_dns"] = health_dns - lab_results[domain]["burned_explanation"] = ", ".join( - burned_explanations - ) - lab_results[domain]["categories"]["all"] = ", ".join(bad_categories) - lab_results[domain]["categories"]["bad"] = ", ".join(domain_categories) - lab_results[domain]["categories"]["talos"] = ", ".join(talos_results) - lab_results[domain]["categories"]["xforce"] = ", ".join(xforce_results) - lab_results[domain]["categories"]["opendns"] = ", ".join( - opendns_results - ) - lab_results[domain]["categories"]["bluecoat"] = ", ".join( - bluecoat_results - ) - lab_results[domain]["categories"]["mxtoolbox"] = ", ".join( - mxtoolbox_results - ) - lab_results[domain]["categories"]["fortiguard"] = ", ".join( - fortiguard_results - ) - lab_results[domain]["categories"]["trendmicro"] = ", ".join( - trendmicro_results - ) + lab_results[domain]["categories"] = domain_categories + if burned: + lab_results[domain]["burned_explanation"] = burned_explanations + # Sleep for a while for VirusTotal's API - sleep(self.request_delay) + sleep(self.sleep_time) + else: + logger.warning( + "Domain %s is expired, so skipped it", + domain.name, + ) return lab_results diff --git a/ghostwriter/oplog/admin.py b/ghostwriter/oplog/admin.py index f947dfb7c..714dda96e 100644 --- a/ghostwriter/oplog/admin.py +++ b/ghostwriter/oplog/admin.py @@ -1,36 +1,13 @@ """This contains customizations for displaying the Oplog application models in the admin panel.""" -# Standard Libraries -from datetime import datetime as dt - # Django & Other 3rd Party Libraries from django.contrib import admin from import_export import resources from import_export.admin import ImportExportModelAdmin -# Register your models here. +# Ghostwriter Libraries from .models import Oplog, OplogEntry - - -class OplogEntryResource(resources.ModelResource): - def before_import_row(self, row, **kwargs): - if "start_date" in row.keys(): - try: - timestamp = int(row["start_date"]) - dt_object = dt.fromtimestamp(timestamp / 1000) - row["start_date"] = str(dt_object) - except ValueError: - pass - if "end_date" in row.keys(): - try: - timestamp = int(row["end_date"]) - dt_object = dt.fromtimestamp(timestamp / 1000) - row["end_date"] = str(dt_object) - except ValueError: - pass - - class Meta: - model = OplogEntry +from .resources import OplogEntryResource class OplogResource(resources.ModelResource): diff --git a/ghostwriter/oplog/consumers.py b/ghostwriter/oplog/consumers.py index 408f4210a..bc3ca3b24 100644 --- a/ghostwriter/oplog/consumers.py +++ b/ghostwriter/oplog/consumers.py @@ -2,18 +2,18 @@ # Standard Libraries import json +import logging # Django & Other 3rd Party Libraries from channels.db import database_sync_to_async from channels.generic.websocket import AsyncWebsocketConsumer from django.core.serializers import serialize +# Ghostwriter Libraries from .models import OplogEntry - -@database_sync_to_async -def getAllLogEntries(oplogId): - return OplogEntry.objects.filter(oplog_id=oplogId).order_by("start_date") +# Using __name__ resolves to ghostwriter.oplog.consumers +logger = logging.getLogger(__name__) @database_sync_to_async @@ -25,10 +25,8 @@ def createOplogEntry(oplog_id): @database_sync_to_async def deleteOplogEntry(oplogEntryId): - try: - OplogEntry.objects.get(pk=oplogEntryId).delete() - except OplogEntry.DoesNotExist: - pass + OplogEntry.objects.get(pk=oplogEntryId).delete() + @database_sync_to_async def copyOplogEntry(oplogEntryId): @@ -49,24 +47,31 @@ def editOplogEntry(oplogEntryId, modifiedRow): class OplogEntryConsumer(AsyncWebsocketConsumer): + @database_sync_to_async + def getAllLogEntries(self, oplogId): + entries = OplogEntry.objects.filter(oplog_id=oplogId).order_by("start_date") + serialized_entries = json.loads(serialize("json", entries)) + return serialized_entries + async def send_oplog_entry(self, event): await self.send(text_data=event["text"]) async def connect(self): - oplog_id = self.scope["url_route"]["kwargs"]["pk"] - await self.channel_layer.group_add(str(oplog_id), self.channel_name) - await self.accept() + user = self.scope["user"] + if user.is_active: + oplog_id = self.scope["url_route"]["kwargs"]["pk"] + await self.channel_layer.group_add(str(oplog_id), self.channel_name) + await self.accept() - entries = await getAllLogEntries(oplog_id) - serialized_entries = json.loads(serialize("json", entries)) - message = json.dumps({"action": "sync", "data": serialized_entries}) + serialized_entries = await self.getAllLogEntries(oplog_id) + message = json.dumps({"action": "sync", "data": serialized_entries}) - await self.channel_layer.group_send( - str(oplog_id), {"type": "send_oplog_entry", "text": message} - ) + await self.channel_layer.group_send( + str(oplog_id), {"type": "send_oplog_entry", "text": message} + ) async def disconnect(self, close_code): - print(f"[*] Disconnected: {close_code}") + logger.info("WebSocket disconnected with close code: %s", close_code) async def receive(self, text_data=None, bytes_data=None): json_data = json.loads(text_data) diff --git a/ghostwriter/oplog/forms.py b/ghostwriter/oplog/forms.py index d44b7a05b..723f2b435 100644 --- a/ghostwriter/oplog/forms.py +++ b/ghostwriter/oplog/forms.py @@ -1,53 +1,64 @@ """This contains all of the forms used by the Oplog application.""" # Django & Other 3rd Party Libraries -from crispy_forms.bootstrap import Alert, TabHolder from crispy_forms.helper import FormHelper +from crispy_forms.layout import ( + HTML, + ButtonHolder, + Layout, + Submit, +) from django import forms # Ghostwriter Libraries -from ghostwriter.rolodex.models import Client, Project +from ghostwriter.rolodex.models import Project from .models import Oplog, OplogEntry -class ShortNameModelChoiceField(forms.ModelChoiceField): - def label_from_instance(self, obj): - return obj.short_name - - -class OplogCreateForm(forms.ModelForm): +class OplogForm(forms.ModelForm): """ - Form used with the OplogCreate for creating new oplog entries. + Save an individual :model:`oplog.Oplog`. """ - client_list = ShortNameModelChoiceField(queryset=Client.objects.all().order_by('name')) class Meta: model = Oplog - fields = ['client_list', 'project', 'name'] + fields = "__all__" - def __init__(self, *args, **kwargs): - super(OplogCreateForm, self).__init__(*args, **kwargs) + def __init__(self, project=None, *args, **kwargs): + super(OplogForm, self).__init__(*args, **kwargs) + self.project_instance = project + # Limit the list to just projects not marked as complete + active_projects = Project.objects.filter(complete=False) + if active_projects: + self.fields["project"].empty_label = "-- Select an Active Project --" + else: + self.fields["project"].empty_label = "-- No Active Projects --" + self.fields["project"].queryset = active_projects + for field in self.fields: + self.fields[field].widget.attrs["autocomplete"] = "off" + # Design form layout with Crispy FormHelper self.helper = FormHelper() - self.helper.form_class = "form-inline" + self.helper.form_show_labels = True self.helper.form_method = "post" - self.helper.field_class = "newitem" - self.fields['project'].queryset = Project.objects.none() - - if 'client_list' in self.data: - try: - client_id = int(self.data.get('client_list')) - self.fields['project'].queryset = Project.objects.filter(client=client_id).order_by("name") - except (ValueError, TypeError): - pass - - elif self.instance.pk: - self.fields['project'].queryset = self.instance.client.project_set.order_by("name") + self.helper.form_class = "newitem" + self.helper.layout = Layout( + "name", + "project", + ButtonHolder( + Submit("submit_btn", "Submit", css_class="btn btn-primary col-md-4"), + HTML( + """ + + """ + ), + ), + ) -class OplogCreateEntryForm(forms.ModelForm): +class OplogEntryForm(forms.ModelForm): """ - Form used for creating entries in the oplog + Save an individual :model:`oplog.OplogEntry`. """ class Meta: @@ -55,8 +66,10 @@ class Meta: fields = "__all__" def __init__(self, *args, **kwargs): - super(OplogCreateEntryForm, self).__init__(*args, **kwargs) + super(OplogEntryForm, self).__init__(*args, **kwargs) # self.oplog_id = pk + for field in self.fields: + self.fields[field].widget.attrs["autocomplete"] = "off" self.helper = FormHelper() self.helper.form_class = "form-inline" self.helper.form_method = "post" diff --git a/ghostwriter/oplog/models.py b/ghostwriter/oplog/models.py index 5b14f58b2..c8d2737db 100644 --- a/ghostwriter/oplog/models.py +++ b/ghostwriter/oplog/models.py @@ -14,6 +14,10 @@ class Oplog(models.Model): + """ + Stores an individual operation log. + """ + name = models.CharField(max_length=50) project = models.ForeignKey( "rolodex.Project", @@ -32,8 +36,7 @@ def __str__(self): # Create your models here. class OplogEntry(models.Model): """ - A model representing a single entry in the operational log. This - represents a single action taken by an operator in a target network. + Stores an individual log entry, related to :model:`oplog.Oplog`. """ oplog_id = models.ForeignKey( @@ -97,6 +100,10 @@ class OplogEntry(models.Model): @receiver(pre_save, sender=OplogEntry) def oplog_pre_save(sender, instance, **kwargs): + """ + Set any missing ``start_date`` and ``end_date`` values for an entry for + :model:`oplog.OplogEntry`. + """ if not instance.start_date: instance.start_date = datetime.utcnow() if not instance.end_date: @@ -105,6 +112,10 @@ def oplog_pre_save(sender, instance, **kwargs): @receiver(post_save, sender=OplogEntry) def signal_oplog_entry(sender, instance, **kwargs): + """ + Send a WebSockets message to update a user's log entry list with the + new or updated instance of :model:`oplog.OplogEntry`. + """ channel_layer = get_channel_layer() oplog_id = instance.oplog_id.id serialized_entry = serialize("json", [instance,]) @@ -118,6 +129,10 @@ def signal_oplog_entry(sender, instance, **kwargs): @receiver(post_delete, sender=OplogEntry) def delete_oplog_entry(sender, instance, **kwargs): + """ + Send a WebSockets message to update a user's log entry list and remove + the deleted instance of :model:`oplog.OplogEntry`. + """ channel_layer = get_channel_layer() oplog_id = instance.oplog_id.id entry_id = instance.id diff --git a/ghostwriter/oplog/resources.py b/ghostwriter/oplog/resources.py new file mode 100644 index 000000000..f92b5970b --- /dev/null +++ b/ghostwriter/oplog/resources.py @@ -0,0 +1,47 @@ +"""This contains all of the ``import_export`` model resources used by the Oplog application.""" + +# Standard Libraries +from datetime import datetime as dt + +# Django & Other 3rd Party Libraries +from import_export import resources + +# Ghostwriter Libraries +from .models import OplogEntry + + +class OplogEntryResource(resources.ModelResource): + def before_import_row(self, row, **kwargs): + if "start_date" in row.keys(): + try: + timestamp = int(row["start_date"]) + dt_object = dt.fromtimestamp(timestamp / 1000) + row["start_date"] = str(dt_object) + except ValueError: + pass + if "end_date" in row.keys(): + try: + timestamp = int(row["end_date"]) + dt_object = dt.fromtimestamp(timestamp / 1000) + row["end_date"] = str(dt_object) + except ValueError: + pass + + class Meta: + model = OplogEntry + skip_unchanged = True + exclude = ("id",) + import_id_fields = ( + "oplog_id", + "start_date", + "end_date", + "source_ip", + "dest_ip", + "tool", + "user_context", + "command", + "description", + "output", + "comments", + "operator_name", + ) diff --git a/ghostwriter/oplog/routing.py b/ghostwriter/oplog/routing.py index 9124ccf21..fe4e71641 100644 --- a/ghostwriter/oplog/routing.py +++ b/ghostwriter/oplog/routing.py @@ -1,4 +1,4 @@ -"""This contains all of the WebSocket routes used by the Home application.""" +"""This contains all of the WebSocket routes used by the Oplog application.""" # Django & Other 3rd Party Libraries from django.urls import path diff --git a/ghostwriter/oplog/serializers.py b/ghostwriter/oplog/serializers.py index 7f07c3305..014ecdd4d 100644 --- a/ghostwriter/oplog/serializers.py +++ b/ghostwriter/oplog/serializers.py @@ -1,6 +1,8 @@ """This contains all of the serializers used by the Oplog application's REST API.""" +# Django & Other 3rd Party Libraries from rest_framework import serializers + from .models import Oplog, OplogEntry @@ -14,4 +16,3 @@ class OplogEntrySerializer(serializers.ModelSerializer): class Meta: model = OplogEntry fields = "__all__" - diff --git a/ghostwriter/oplog/templates/oplog/entries_list.html b/ghostwriter/oplog/templates/oplog/entries_list.html index e3007e1a9..6db4db1d2 100644 --- a/ghostwriter/oplog/templates/oplog/entries_list.html +++ b/ghostwriter/oplog/templates/oplog/entries_list.html @@ -4,19 +4,19 @@ {% block pagetitle %}{{ name }} Entries{% endblock %} {% block breadcrumbs %} - + {% endblock %} {% block content %}
-
Disconnected
+
Disconnected
@@ -25,8 +25,10 @@
-
@@ -92,6 +94,14 @@ $("#oplogTableDiv").width($width); } + function jsEscape(s){ + if (typeof s !== 'undefined') { + return s.toString().replace(/&/g, '&').replace(/ { entry = element['fields'] id = element['pk'] var newRow = ` - ${entry["start_date"].replace(/\.\d+/, "").replace("Z", "").replace("T", " ")} - ${entry["end_date"].replace(/\.\d+/, "").replace("Z", "").replace("T", " ")} - ${id} - ${entry["source_ip"]} - ${entry["dest_ip"]} - ${entry["tool"]} - ${entry["user_context"]} -
${entry["command"]}
-
${entry["description"]}
-
${entry["output"]}
-
${entry["comments"]}
- ${entry["operator_name"]} + ${jsEscape(entry["start_date"].replace(/\.\d+/, "").replace("Z", "").replace("T", " "))} + ${jsEscape(entry["end_date"].replace(/\.\d+/, "").replace("Z", "").replace("T", " "))} + ${jsEscape(id)} + ${jsEscape(entry["source_ip"])} + ${jsEscape(entry["dest_ip"])} + ${jsEscape(entry["tool"])} + ${jsEscape(entry["user_context"])} +
${jsEscape(entry["command"])}
+
${jsEscape(entry["description"])}
+
${jsEscape(entry["output"])}
+
${jsEscape(entry["comments"])}
+ ${jsEscape(entry["operator_name"])} ` @@ -221,18 +233,18 @@ entry = element['fields'] id = element['pk'] var newRow = ` - ${entry["start_date"].replace(/\.\d+/, "").replace("Z", "").replace("T", " ")} - ${entry["end_date"].replace(/\.\d+/, "").replace("Z", "").replace("T", " ")} - ${id} - ${entry["source_ip"]} - ${entry["dest_ip"]} - ${entry["tool"]} - ${entry["user_context"]} -
${entry["command"]}
-
${entry["description"]}
-
${entry["output"]}
-
${entry["comments"]}
- ${entry["operator_name"]} + ${jsEscape(entry["start_date"].replace(/\.\d+/, "").replace("Z", "").replace("T", " "))} + ${jsEscape(entry["end_date"].replace(/\.\d+/, "").replace("Z", "").replace("T", " "))} + ${jsEscape(id)} + ${jsEscape(entry["source_ip"])} + ${jsEscape(entry["dest_ip"])} + ${jsEscape(entry["tool"])} + ${jsEscape(entry["user_context"])} +
${jsEscape(entry["command"])}
+
${jsEscape(entry["description"])}
+
${jsEscape(entry["output"])}
+
${jsEscape(entry["comments"])}
+ ${jsEscape(entry["operator_name"])} ` @@ -272,7 +284,7 @@ } else { - $('#oplogTableNoEntries').hide() + $('#oplogTableNoEntries').hide() } } @@ -478,7 +490,27 @@ 'action': 'create', 'oplog_id': {{ pk }} })) - }) + }) + + $('#importNewEntries').click(function () { + window.open('{% url "oplog:oplog_import" %}', '_self'); + }) + + function download(url, filename) { + fetch(url).then(function (t) { + return t.blob().then((b) => { + var a = document.createElement("a"); + a.href = URL.createObjectURL(b); + a.setAttribute("download", filename); + a.click(); + }); + }); + } + + $('#exportEntries').click(function () { + filename = generateDownloadName('{{ name }}-log-export-{{ id }}.csv'); + download(`/oplog/api/entries?export=csv&&oplog_id={{ pk }}`, filename); + }) }); diff --git a/ghostwriter/oplog/templates/oplog/oplog_form.html b/ghostwriter/oplog/templates/oplog/oplog_form.html index bfc341aee..e03c965a7 100644 --- a/ghostwriter/oplog/templates/oplog/oplog_form.html +++ b/ghostwriter/oplog/templates/oplog/oplog_form.html @@ -2,16 +2,22 @@ {% load crispy_forms_tags %} {% block pagetitle %}Oplog Creation{% endblock %} + {% block breadcrumbs %} {% endblock %} {% block content %} + +

Provide a meaningful name for your log and select the project to which it should associated:

+ + {% if form.errors %} {% endif %} -
- {% csrf_token %} - {{ form|crispy }} - - -
-{% endblock %} - -{% block morescripts %} - + + {% crispy form form.helper %} {% endblock %} diff --git a/ghostwriter/oplog/templates/oplog/oplog_import.html b/ghostwriter/oplog/templates/oplog/oplog_import.html index c05d4100b..e1f561e3c 100644 --- a/ghostwriter/oplog/templates/oplog/oplog_import.html +++ b/ghostwriter/oplog/templates/oplog/oplog_import.html @@ -3,19 +3,19 @@ {% block pagetitle %}Domain Entry{% endblock %} {% block breadcrumbs %} - + {% endblock %} {% block content %} -

Upload Oplog Entries CSV

-

Upload a csv file containing oplog entries to be imported:

+

Upload Operation Log Entries CSV

+

Upload a csv file containing log entries to be imported:

{% csrf_token %}
@@ -38,20 +38,23 @@

Upload Oplog Entries CSV

Instructions

-

Your csv file must have these headers:

-
-

+

+
-

- To avoid "KeyError" errors, ensure the csv file was saved without a Byte Order Marker (BOM).
- Download a template with the headers and an example entry: -

+
+ +
{% endblock %} diff --git a/ghostwriter/oplog/templates/oplog/oplog_list.html b/ghostwriter/oplog/templates/oplog/oplog_list.html index b554d57d2..6b5cffeb6 100644 --- a/ghostwriter/oplog/templates/oplog/oplog_list.html +++ b/ghostwriter/oplog/templates/oplog/oplog_list.html @@ -1,63 +1,89 @@ {% extends "base_generic.html" %} {% load crispy_forms_tags %} +{% load bleach_tags %} -{% block pagetitle %}Report List{% endblock %} +{% block pagetitle %}Operation Logs{% endblock %} {% block breadcrumbs %} - + {% endblock %} {% block content %} - - - - - - - - - - - {% for log in op_logs %} - - - - - - + {% comment %} Conditional display of API key for newly created oplogs {% endcomment %} + {% for message in messages %} + {% if "api-key" in message.tags %} + + {% endif %} {% endfor %} -
NameProjectIDExport to CSV
{{ log.name }}{{ log.project }}{{ log.id }}Export
-

Create new oplog

+ + +

+ Start New Operation Log + OR + Import Oplog +

+ {% if op_logs %} + + + + + + + + + + {% for log in op_logs %} + + + + + + + {% endfor %} +
IDNameProjectExport CSV
{{ log.id }}{{ log.name }}{{ log.project.client }} {{ log.project.project_type }} ({{ log.project.start_date }})Export
+ {% else %} +

There are no logs to see here, yet.

+ {% endif %} {% endblock %} {% block morescripts %} - - + + + -{% endblock %} \ No newline at end of file + $(".js-export-oplog").on("click", function () { + id = $(this).attr('oplog-id') + name = $(this).attr('oplog-name') + base_name = `${name}-log-export-${id}.csv` + filename = generateDownloadName(base_name); + download(`/oplog/api/entries?export=csv&&oplog_id=${id}`, filename); + }); + +{% endblock %} diff --git a/ghostwriter/oplog/templates/oplog/oplogentry_form.html b/ghostwriter/oplog/templates/oplog/oplogentry_form.html index 09b3716e2..70f32485b 100644 --- a/ghostwriter/oplog/templates/oplog/oplogentry_form.html +++ b/ghostwriter/oplog/templates/oplog/oplogentry_form.html @@ -1,17 +1,17 @@ {% extends "base_generic.html" %} {% load crispy_forms_tags %} -{% block pagetitle %}New Oplog Entry{% endblock %} +{% block pagetitle %}Operation Log Entry{% endblock %} {% block content %} -

Enter log details

+

Enter log details:

{% csrf_token %} - Oplog Entry Details + Operation Log Entry Details
{{ form.oplog_id|as_crispy_field }} {{ form.start_date|as_crispy_field }} diff --git a/ghostwriter/oplog/urls.py b/ghostwriter/oplog/urls.py index 9d0c8bd38..b5510d636 100644 --- a/ghostwriter/oplog/urls.py +++ b/ghostwriter/oplog/urls.py @@ -1,42 +1,37 @@ """This contains all of the URL mappings used by the Oplog application.""" - + # Django & Other 3rd Party Libraries from django.urls import include, path from rest_framework import routers -from .views import ( - OplogCreateWithoutProject, - OplogEntriesImport, - OplogEntryCreate, - OplogEntryDelete, - OplogEntryUpdate, - OplogEntryViewSet, - OplogListEntries, - OplogViewSet, - index, - load_projects -) +from . import views app_name = "ghostwriter.oplog" router = routers.DefaultRouter() -router.register("entries", OplogEntryViewSet) -router.register("oplogs", OplogViewSet) +router.register("entries", views.OplogEntryViewSet) +router.register("oplogs", views.OplogViewSet) urlpatterns = [ - path("", index, name="index"), + path("", views.index, name="index"), path("api/", include(router.urls)), - path("create/", OplogCreateWithoutProject.as_view(), name="oplog_create"), - path("load-projects/", load_projects, name="load_projects"), + path("create/", views.OplogCreate.as_view(), name="oplog_create"), + path("create/", views.OplogCreate.as_view(), name="oplog_create_no_project"), path( - "/entries/create", OplogEntryCreate.as_view(), name="oplog_entry_create" + "/entries/create", + views.OplogEntryCreate.as_view(), + name="oplog_entry_create", ), path( - "/entries/update", OplogEntryUpdate.as_view(), name="oplog_entry_update" + "/entries/update", + views.OplogEntryUpdate.as_view(), + name="oplog_entry_update", ), path( - "/entries/delete", OplogEntryDelete.as_view(), name="oplog_entry_delete" + "/entries/delete", + views.OplogEntryDelete.as_view(), + name="oplog_entry_delete", ), - path("/entries", OplogListEntries, name="oplog_entries"), - path("import", OplogEntriesImport, name="oplog_import"), + path("/entries", views.OplogListEntries, name="oplog_entries"), + path("import", views.OplogEntriesImport, name="oplog_import"), ] diff --git a/ghostwriter/oplog/views.py b/ghostwriter/oplog/views.py index 383ddcaa2..25987e0b9 100644 --- a/ghostwriter/oplog/views.py +++ b/ghostwriter/oplog/views.py @@ -1,29 +1,49 @@ """This contains all of the views used by the Oplog application.""" +# Standard Libraries +import logging + # Django & Other 3rd Party Libraries +from django.contrib import messages from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin from django.http import HttpResponse, HttpResponseRedirect -from django.shortcuts import render +from django.shortcuts import get_object_or_404, render from django.urls import reverse from django.views.generic.edit import CreateView, DeleteView, UpdateView from rest_framework import viewsets from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from rest_framework_api_key.models import APIKey from rest_framework_api_key.permissions import HasAPIKey from tablib import Dataset +# Ghostwriter Libraries +from ghostwriter.rolodex.models import Project + from .admin import OplogEntryResource -from .forms import OplogCreateEntryForm, OplogCreateForm +from .forms import OplogEntryForm, OplogForm from .models import Oplog, OplogEntry from .serializers import OplogEntrySerializer, OplogSerializer -from ghostwriter.rolodex.models import Project +# Using __name__ resolves to ghostwriter.oplog.views +logger = logging.getLogger(__name__) + + +################## +# View Functions # +################## -# Create your views here. @login_required def index(request): + """ + Display a list of all :model:`oplog.Oplog`. + + **Template** + + :template:`oplog/oplog_list.html` + """ op_logs = Oplog.objects.all() context = {"op_logs": op_logs} return render(request, "oplog/oplog_list.html", context=context) @@ -31,6 +51,14 @@ def index(request): @login_required def OplogEntriesImport(request): + """ + Import a collection of :model:`oplog.OplogEntry` entries for an individual + :model:`oplog.Oplog`. + + **Template** + + :template:`oplog/oplog_import.html` + """ if request.method == "POST": oplog_entry_resource = OplogEntryResource() @@ -40,49 +68,179 @@ def OplogEntriesImport(request): imported_data = dataset.load(new_entries, format="csv") result = oplog_entry_resource.import_data(imported_data, dry_run=True) - if not result.has_errors(): + if result.has_errors(): + row_errors = result.row_errors() + for exc in row_errors: + messages.error( + request, + f"There was an error in row {exc[0]}: {exc[1][0].error}", + extra_tags="alert-danger", + ) + return HttpResponseRedirect(reverse("oplog:oplog_import")) + else: oplog_entry_resource.import_data(imported_data, format="csv", dry_run=False) - return HttpResponseRedirect(reverse("oplog:index")) + # Get the first ``oplog_id`` value to use for a redirect + oplog_id = imported_data["oplog_id"][0] + messages.success( + request, + "Successfully imported log data", + extra_tags="alert-success", + ) + return HttpResponseRedirect( + reverse("oplog:oplog_entries", kwargs={"pk": oplog_id}) + ) return render(request, "oplog/oplog_import.html") @login_required def OplogListEntries(request, pk): + """ + Display all :model:`oplog.OplogEntry` associated with an individual + :model:`oplog.Oplog`. + + **Template** + + :template:`oplog/entries_list.html` + """ entries = OplogEntry.objects.filter(oplog_id=pk).order_by("-start_date") name = Oplog.objects.get(pk=pk).name context = {"entries": entries, "pk": pk, "name": name} return render(request, "oplog/entries_list.html", context=context) -@login_required -def load_projects(request): - client = request.GET.get('client') - try: - projects = Project.objects.filter(client=client) - except: - projects = Project.objects.none() - return render(request, 'project_dropdown_list.html', {'projects': projects}) +################ +# View Classes # +################ + + +class OplogCreate(LoginRequiredMixin, CreateView): + """ + Create an individual instance of :model:`oplog.Oplog`. + + **Context** + + ``project`` + Instance of :model:`rolodex.Project` associated with this log + ``cancel_link`` + Link for the form's Cancel button to return to oplog list or details page + + **Template** + + :template:`oplog/oplog_form.html` + """ -class OplogCreateWithoutProject(LoginRequiredMixin, CreateView): model = Oplog - form_class = OplogCreateForm + form_class = OplogForm + + def setup(self, request, *args, **kwargs): + super().setup(request, *args, **kwargs) + # Check if this request is for a specific project or not + self.project = "" + # Determine if ``pk`` is in the kwargs + if "pk" in self.kwargs: + pk = self.kwargs.get("pk") + # Try to get the project from :model:`rolodex.Project` + if pk: + try: + self.project = get_object_or_404(Project, pk=self.kwargs.get("pk")) + except Project.DoesNotExist: + logger.info( + "Received report create request for Project ID %s, but that Project does not exist", + pk, + ) + + def get_form_kwargs(self): + kwargs = super(OplogCreate, self).get_form_kwargs() + kwargs.update({"project": self.project}) + return kwargs + + def get_context_data(self, **kwargs): + ctx = super(OplogCreate, self).get_context_data(**kwargs) + ctx["project"] = self.project + if self.project: + ctx["cancel_link"] = reverse( + "rolodex:project_detail", kwargs={"pk": self.project.pk} + ) + else: + ctx["cancel_link"] = reverse("oplog:index") + return ctx + + def get_form(self, form_class=None): + form = super(OplogCreate, self).get_form(form_class) + if not form.fields["project"].queryset: + messages.error( + self.request, + "There are no active projects for a new operation log", + extra_tags="alert-error", + ) + return form + + def form_valid(self, form): + # Save the new :model:`oplog.Oplog` instance + form.save() + messages.success( + self.request, + "New operation log was successfully created", + extra_tags="alert-success", + ) + # Create new API key for this oplog + try: + oplog_name = form.instance.name + api_key_name = oplog_name + api_key, key = APIKey.objects.create_key(name=api_key_name) + # Pass the API key via the messages framework + messages.info( + self.request, + f"The API key for your log is { api_key }: { key }\r\nPlease store it somewhere safe: you will not be able to see it again.", + extra_tags="api-key no-toast", + ) + except Exception: + logger.exception("Failed to create new API key") + messages.error( + self.request, + "Could not generate an API key for your new operation log – contact your admin!", + extra_tags="alert-danger", + ) + return super().form_valid(form) + + def get_initial(self): + if self.project: + name = f"{self.project.client} {self.project.project_type} Log" + return {"name": name, "project": self.project.id} def get_success_url(self): + messages.success( + self.request, + "Successfully created new operation log", + extra_tags="alert-success", + ) return reverse("oplog:index") class OplogEntryCreate(LoginRequiredMixin, CreateView): + """ + Create an individual :model:`oplog.OplogEntry`. + + **Template** + + :template:`oplog/oplogentry_form.html` + """ + model = OplogEntry - form_class = OplogCreateEntryForm + form_class = OplogEntryForm def get_success_url(self): return reverse("oplog:oplog_entries", args=(self.object.oplog_id.id,)) class OplogEntryUpdate(LoginRequiredMixin, UpdateView): - """View for updating existing oplog entries. This view defaults to the - oplogentry_form.html template. + """ + Update an individual :model:`oplog.OplogEntry`. + + **Template** + + :template:`oplog/oplogentry_form.html` """ model = OplogEntry @@ -94,15 +252,14 @@ def get_success_url(self): class OplogEntryDelete(LoginRequiredMixin, DeleteView): - """View for deleting existing oplog entries. This view defaults to the - oplogentry_form.html template. + """ + Delete an individual :model:`oplog.OplogEntry`. """ model = OplogEntry fields = "__all__" def get_success_url(self): - """Override the function to return to the new record after creation.""" return reverse("oplog:oplog_entries", args=(self.object.oplog_id.id,)) diff --git a/ghostwriter/reporting/admin.py b/ghostwriter/reporting/admin.py index c7c59ef1b..925ffb252 100644 --- a/ghostwriter/reporting/admin.py +++ b/ghostwriter/reporting/admin.py @@ -4,8 +4,10 @@ from django.contrib import admin from import_export.admin import ImportExportModelAdmin +# Ghostwriter Libraries from .models import ( Archive, + DocType, Evidence, Finding, FindingNote, @@ -13,6 +15,7 @@ LocalFindingNote, Report, ReportFindingLink, + ReportTemplate, Severity, ) from .resources import FindingResource @@ -23,6 +26,11 @@ class ArchiveAdmin(admin.ModelAdmin): pass +@admin.register(DocType) +class DocTypeAdmin(admin.ModelAdmin): + pass + + @admin.register(Evidence) class EvidenceAdmin(admin.ModelAdmin): list_display = ("document", "upload_date", "uploaded_by") @@ -33,7 +41,15 @@ class EvidenceAdmin(admin.ModelAdmin): "Evidence Document", {"fields": ("friendly_name", "caption", "description", "document")}, ), - ("Report Information", {"fields": ("finding", "uploaded_by",)},), + ( + "Report Information", + { + "fields": ( + "finding", + "uploaded_by", + ) + }, + ), ) @@ -130,3 +146,42 @@ class ReportFindingLinkAdmin(admin.ModelAdmin): @admin.register(Severity) class SeverityAdmin(admin.ModelAdmin): pass + + +@admin.register(ReportTemplate) +class ReportTemplateAdmin(admin.ModelAdmin): + list_display = ( + "get_status", + "name", + "client", + "last_update", + ) + readonly_fields = ("get_status",) + list_filter = ("client",) + list_display_links = ("name",) + fieldsets = ( + ( + "Report Template", + { + "fields": ( + "name", + "document", + "description", + "client", + ) + }, + ), + ( + "Template Linting", + { + "fields": ( + "get_status", + "lint_result", + ) + }, + ), + ( + "Admin Settings", + {"fields": ("protected",)}, + ), + ) diff --git a/ghostwriter/reporting/filters.py b/ghostwriter/reporting/filters.py index a5f72490e..294505c0b 100644 --- a/ghostwriter/reporting/filters.py +++ b/ghostwriter/reporting/filters.py @@ -8,6 +8,7 @@ from django import forms from django.forms.widgets import TextInput +# Ghostwriter Libraries from .models import Archive, Finding, FindingType, Report, Severity @@ -65,7 +66,8 @@ def __init__(self, *args, **kwargs): ), Row( Column( - InlineCheckboxes("severity"), css_class="form-group col-md-12", + InlineCheckboxes("severity"), + css_class="form-group col-md-12", ), css_class="form-row", ), @@ -136,7 +138,10 @@ def __init__(self, *args, **kwargs): PrependedText("title", ''), css_class="form-group col-md-6", ), - Column("complete", css_class="form-group col-md-6",), + Column( + "complete", + css_class="form-group col-md-6", + ), css_class="form-row", ), ButtonHolder( diff --git a/ghostwriter/reporting/fixtures/initial.json b/ghostwriter/reporting/fixtures/initial.json index 414fd9e0e..795985203 100644 --- a/ghostwriter/reporting/fixtures/initial.json +++ b/ghostwriter/reporting/fixtures/initial.json @@ -1,91 +1,135 @@ -[ - { - "model": "reporting.severity", - "pk": 1, - "fields": { - "severity": "Informational", - "weight": 5 - } - }, - { - "model": "reporting.severity", - "pk": 2, - "fields": { - "severity": "Low", - "weight": 4 - } - }, - { - "model": "reporting.severity", - "pk": 3, - "fields": { - "severity": "Medium", - "weight": 3 - } - }, - { - "model": "reporting.severity", - "pk": 4, - "fields": { - "severity": "High", - "weight": 2 - } - }, - { - "model": "reporting.severity", - "pk": 5, - "fields": { - "severity": "Critical", - "weight": 1 - } - }, - { - "model": "reporting.findingtype", - "pk": 1, - "fields": { - "finding_type": "Network" - } - }, - { - "model": "reporting.findingtype", - "pk": 2, - "fields": { - "finding_type": "Physical" - } - }, - { - "model": "reporting.findingtype", - "pk": 3, - "fields": { - "finding_type": "Wireless" - } - }, - { - "model": "reporting.findingtype", - "pk": 4, - "fields": { - "finding_type": "Web" - } - }, - { - "model": "reporting.findingtype", - "pk": 5, - "fields": { - "finding_type": "Mobile" - } - }, - { - "model": "reporting.findingtype", - "pk": 6, - "fields": { - "finding_type": "Cloud" - } - }, - { - "model": "reporting.findingtype", - "pk": 7, - "fields": { - "finding_type": "Host" - } - } - ] \ No newline at end of file +[{ + "model": "reporting.severity", + "pk": 1, + "fields": { + "severity": "Informational", + "weight": 5, + "color": "8EAADB" + } + }, + { + "model": "reporting.severity", + "pk": 2, + "fields": { + "severity": "Low", + "weight": 4, + "color": "A8D08D" + } + }, + { + "model": "reporting.severity", + "pk": 3, + "fields": { + "severity": "Medium", + "weight": 3, + "color": "F4B083" + } + }, + { + "model": "reporting.severity", + "pk": 4, + "fields": { + "severity": "High", + "weight": 2, + "color": "FF7E79" + } + }, + { + "model": "reporting.severity", + "pk": 5, + "fields": { + "severity": "Critical", + "weight": 1, + "color": "966FD6" + } + }, + { + "model": "reporting.findingtype", + "pk": 1, + "fields": { + "finding_type": "Network" + } + }, + { + "model": "reporting.findingtype", + "pk": 2, + "fields": { + "finding_type": "Physical" + } + }, + { + "model": "reporting.findingtype", + "pk": 3, + "fields": { + "finding_type": "Wireless" + } + }, + { + "model": "reporting.findingtype", + "pk": 4, + "fields": { + "finding_type": "Web" + } + }, + { + "model": "reporting.findingtype", + "pk": 5, + "fields": { + "finding_type": "Mobile" + } + }, + { + "model": "reporting.findingtype", + "pk": 6, + "fields": { + "finding_type": "Cloud" + } + }, + { + "model": "reporting.findingtype", + "pk": 7, + "fields": { + "finding_type": "Host" + } + }, + { + "model": "reporting.doctype", + "pk": 1, + "fields": { + "doc_type": "docx" + } + }, + { + "model": "reporting.doctype", + "pk": 2, + "fields": { + "doc_type": "pptx" + } + }, + { + "model": "reporting.reporttemplate", + "pk": 1, + "fields": { + "name": "Default Word Template", + "document": "/app/ghostwriter/media/templates/template.docx", + "description": "A sample Word template provided by Ghostwriter.", + "upload_date": "2020-11-12", + "last_update": "2020-11-12", + "doc_type": 1, + "lint_result": "{\"result\": \"success\", \"warnings\": [], \"errors\": []}" + } + }, + { + "model": "reporting.reporttemplate", + "pk": 2, + "fields": { + "name": "Default PowerPoint Template", + "document": "/app/ghostwriter/media/templates/template.pptx", + "description": "A sample PowerPoint presentation template provided by Ghostwriter.", + "upload_date": "2020-11-12", + "last_update": "2020-11-12", + "doc_type": 2, + "lint_result": "{\"result\": \"success\", \"warnings\": [], \"errors\": []}" + } + } +] diff --git a/ghostwriter/reporting/forms.py b/ghostwriter/reporting/forms.py index a22494d5e..437b47f8d 100644 --- a/ghostwriter/reporting/forms.py +++ b/ghostwriter/reporting/forms.py @@ -16,7 +16,7 @@ from django import forms from django.core.exceptions import ValidationError from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # Ghostwriter Libraries from ghostwriter.modules.custom_layout_object import CustomTab @@ -29,6 +29,7 @@ LocalFindingNote, Report, ReportFindingLink, + ReportTemplate, ) @@ -110,7 +111,7 @@ def __init__(self, *args, **kwargs): css_class="nav-justified", ), ButtonHolder( - Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), + Submit("submit_btn", "Submit", css_class="btn btn-primary col-md-4"), HTML( """ @@ -122,7 +123,7 @@ def __init__(self, *args, **kwargs): class ReportForm(forms.ModelForm): """ - Create an individual :model:`reporting.Report` associated with an indivudal + Save an individual :model:`reporting.Report` associated with an indivudal :model:`rolodex.Project`. """ @@ -134,12 +135,19 @@ def __init__(self, project=None, *args, **kwargs): super(ReportForm, self).__init__(*args, **kwargs) self.project_instance = project # Limit the list to just projects not marked as complete - active_projects = Project.objects.filter(complete=False) + active_projects = Project.objects.filter(complete=False).order_by( + "start_date", "client", "project_type" + ) if active_projects: self.fields["project"].empty_label = "-- Select an Active Project --" else: self.fields["project"].empty_label = "-- No Active Projects --" self.fields["project"].queryset = active_projects + self.fields[ + "project" + ].label_from_instance = ( + lambda obj: f"{obj.start_date} {obj.client.name} {obj.project_type} ({obj.codename})" + ) # Design form layout with Crispy FormHelper self.helper = FormHelper() self.helper.form_show_labels = True @@ -148,6 +156,17 @@ def __init__(self, project=None, *args, **kwargs): self.helper.layout = Layout( "title", "project", + HTML( + """ +
Assign Templates
+
+ """ + ), + Row( + Column("docx_template", css_class="form-group col-md-6 mb-0"), + Column("pptx_template", css_class="form-group col-md-6 mb-0"), + css_class="form-row", + ), ButtonHolder( Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), HTML( @@ -172,7 +191,8 @@ class Meta: def __init__(self, *args, **kwargs): super(ReportFindingLinkUpdateForm, self).__init__(*args, **kwargs) evidence_upload_url = reverse( - "reporting:upload_evidence_modal", kwargs={"pk": self.instance.id}, + "reporting:upload_evidence_modal", + kwargs={"pk": self.instance.id, "modal": "modal"}, ) self.fields["affected_entities"].widget.attrs[ "placeholder" @@ -241,7 +261,10 @@ def __init__(self, *args, **kwargs): Field("mitigation", css_class="enable-evidence-upload"), Field("replication_steps", css_class="enable-evidence-upload"), Field("host_detection_techniques", css_class="enable-evidence-upload"), - Field("network_detection_techniques", css_class="enable-evidence-upload",), + Field( + "network_detection_techniques", + css_class="enable-evidence-upload", + ), HTML( """
References
@@ -250,7 +273,7 @@ def __init__(self, *args, **kwargs): ), "references", ButtonHolder( - Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), + Submit("submit_btn", "Submit", css_class="btn btn-primary col-md-4"), HTML( """ @@ -273,17 +296,14 @@ class Meta: "document", "description", "caption", - "uploaded_by", - "finding", ) widgets = { - "uploaded_by": forms.HiddenInput(), - "finding": forms.HiddenInput(), "document": forms.FileInput(attrs={"class": "form-control"}), } def __init__(self, *args, **kwargs): self.is_modal = kwargs.pop("is_modal", None) + self.evidence_queryset = kwargs.pop("evidence_queryset", None) super(EvidenceForm, self).__init__(*args, **kwargs) self.fields["caption"].required = True self.fields["caption"].widget.attrs["autocomplete"] = "off" @@ -317,7 +337,6 @@ def __init__(self, *args, **kwargs): self.helper.form_show_labels = True self.helper.form_method = "post" self.helper.form_class = "newitem" - # Set a special form attribute to provide a URL for the evidence upload modal self.helper.attrs = {"enctype": "multipart/form-data"} self.helper.form_id = "evidence-upload-form" self.helper.layout = Layout( @@ -338,11 +357,14 @@ def __init__(self, *args, **kwargs): """ Upload a File
-

Attach text evidence (*.txt, *.log, *.md, *.ps1, or *.py) or image evidence (*.png, *.jpg, or *.jpeg).

+

Attach text evidence (*.txt, *.log, or *.md) or image evidence (*.png, *.jpg, or *.jpeg).

""" ), Div( - "document", + Field( + "document", + id="id_document", + ), HTML( """ @@ -350,19 +372,14 @@ def __init__(self, *args, **kwargs): ), css_class="custom-file", ), - "uploaded_by", - "finding", ButtonHolder(submit, cancel_button), ) def clean(self): cleaned_data = super(EvidenceForm, self).clean() friendly_name = cleaned_data.get("friendly_name") - finding = cleaned_data.get("finding") # Check if provided name has already been used for another file for this report - report_queryset = Evidence.objects.filter(finding=finding.id).values_list( - "id", "friendly_name" - ) + report_queryset = self.evidence_queryset.values_list("id", "friendly_name") for evidence in report_queryset: if friendly_name == evidence[1] and not self.instance.id == evidence[0]: raise ValidationError( @@ -382,12 +399,7 @@ class FindingNoteForm(forms.ModelForm): class Meta: model = FindingNote - fields = "__all__" - widgets = { - "timestamp": forms.HiddenInput(), - "operator": forms.HiddenInput(), - "finding": forms.HiddenInput(), - } + fields = ("note",) def __init__(self, *args, **kwargs): super(FindingNoteForm, self).__init__(*args, **kwargs) @@ -396,7 +408,7 @@ def __init__(self, *args, **kwargs): self.helper.form_class = "newitem" self.helper.form_show_labels = False self.helper.layout = Layout( - Div("note", "operator", "finding"), + Div("note"), ButtonHolder( Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), HTML( @@ -412,7 +424,8 @@ def clean_note(self): # Check if note is empty if not note: raise ValidationError( - _("You must provide some content for the note"), code="required", + _("You must provide some content for the note"), + code="required", ) return note @@ -425,11 +438,7 @@ class LocalFindingNoteForm(forms.ModelForm): class Meta: model = LocalFindingNote - fields = "__all__" - widgets = { - "operator": forms.HiddenInput(), - "finding": forms.HiddenInput(), - } + fields = ("note",) def __init__(self, *args, **kwargs): super(LocalFindingNoteForm, self).__init__(*args, **kwargs) @@ -438,7 +447,7 @@ def __init__(self, *args, **kwargs): self.helper.form_class = "newitem" self.helper.form_show_labels = False self.helper.layout = Layout( - Div("note", "operator", "finding"), + Div("note"), ButtonHolder( Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), HTML( @@ -454,6 +463,106 @@ def clean_note(self): # Check if note is empty if not note: raise ValidationError( - _("You must provide some content for the note"), code="required", + _("You must provide some content for the note"), + code="required", ) return note + + +class ReportTemplateForm(forms.ModelForm): + """ + Save an individual :model:`reporting.ReportTemplate`. + """ + + class Meta: + model = ReportTemplate + exclude = ("upload_date", "last_update", "lint_result") + widgets = { + "document": forms.FileInput(attrs={"class": "form-control"}), + "uploaded_by": forms.HiddenInput(), + } + + def __init__(self, *args, **kwargs): + super(ReportTemplateForm, self).__init__(*args, **kwargs) + self.fields["document"].label = "" + self.fields["document"].widget.attrs["class"] = "custom-file-input" + # Design form layout with Crispy FormHelper + self.helper = FormHelper() + self.helper.form_show_labels = True + self.helper.form_method = "post" + self.helper.form_class = "newitem" + self.helper.attrs = {"enctype": "multipart/form-data"} + self.helper.layout = Layout( + HTML( + """ + Template Information +
+

The name appears in the template dropdown menus in reports.

+ """ + ), + Row( + Column("name", css_class="form-group col-md-8 mb-0"), + Column("doc_type", css_class="form-group col-md-4 mb-0"), + css_class="form-row", + ), + "description", + HTML( + """ + Upload a File +
+

Attach a document that matches your selected filetype to use as a report template

+ """ + ), + Div( + "document", + HTML( + """ + + """ + ), + css_class="custom-file", + ), + "changelog", + "client", + "protected", + "uploaded_by", + ButtonHolder( + Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), + HTML( + """ + + """ + ), + ), + ) + + +class SelectReportTemplateForm(forms.ModelForm): + """ + Modify the ``docx_template`` and ``pptx_template`` values of an individual + :model:`reporting.Report`. + """ + + class Meta: + model = Report + fields = ("docx_template", "pptx_template") + + def __init__(self, *args, **kwargs): + super(SelectReportTemplateForm, self).__init__(*args, **kwargs) + self.fields["docx_template"].help_text = None + self.fields["pptx_template"].help_text = None + self.fields["docx_template"].empty_label = "-- Select a Word Template --" + self.fields["pptx_template"].empty_label = "-- Select a PPT Template --" + # Design form layout with Crispy FormHelper + self.helper = FormHelper() + self.helper.form_show_labels = False + self.helper.form_method = "post" + self.helper.form_id = "report-template-swap-form" + self.helper.form_tag = True + self.helper.form_action = reverse( + "reporting:ajax_swap_report_template", kwargs={"pk": self.instance.id} + ) + self.helper.layout = Layout( + Field("docx_template", css_class="col-md-4 offset-md-4"), + Field("pptx_template", css_class="col-md-4 offset-md-4"), + ) diff --git a/ghostwriter/reporting/migrations/0009_auto_20200915_0011.py b/ghostwriter/reporting/migrations/0009_auto_20200915_0011.py new file mode 100644 index 000000000..796974482 --- /dev/null +++ b/ghostwriter/reporting/migrations/0009_auto_20200915_0011.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.3 on 2020-09-15 00:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0008_auto_20200825_1947'), + ] + + operations = [ + migrations.AddField( + model_name='severity', + name='color', + field=models.CharField(default='7A7A7A', help_text='Six character hex color code associated with this severity for reports (e.g., FF7E79)', max_length=6, verbose_name='Severity Color'), + ), + migrations.AlterField( + model_name='severity', + name='severity', + field=models.CharField(help_text='Name for this severity rating (e.g. High, Low)', max_length=255, unique=True, verbose_name='Severity'), + ), + migrations.AlterField( + model_name='severity', + name='weight', + field=models.IntegerField(default=1, help_text='Weight for sorting severity categories in reports (lower numbers are more severe)', verbose_name='Severity Weight'), + ), + ] diff --git a/ghostwriter/reporting/migrations/0010_reporttemplate.py b/ghostwriter/reporting/migrations/0010_reporttemplate.py new file mode 100644 index 000000000..19c72fc4c --- /dev/null +++ b/ghostwriter/reporting/migrations/0010_reporttemplate.py @@ -0,0 +1,38 @@ +# Generated by Django 2.2.3 on 2020-09-18 23:23 + +from django.conf import settings +import django.core.files.storage +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('rolodex', '0006_auto_20200825_1947'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('reporting', '0009_auto_20200915_0011'), + ] + + operations = [ + migrations.CreateModel( + name='ReportTemplate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('document', models.FileField(blank=True, storage=django.core.files.storage.FileSystemStorage(base_url='/templates', location='/app/ghostwriter/reporting/templates/reports'), upload_to='')), + ('name', models.CharField(help_text='Provide a name to be used when selecting this template', max_length=255, null=True, verbose_name='Template Name')), + ('upload_date', models.DateField(auto_now=True, help_text='Date and time the template was first uploaded', verbose_name='Upload Date')), + ('last_update', models.DateField(auto_now=True, help_text='Date and time the report was last modified', verbose_name='Last Modified')), + ('description', models.TextField(blank=True, help_text='Provide a description of this template', verbose_name='Description')), + ('protected', models.BooleanField(default=False, help_text='Only administrators can edit this template', verbose_name='Protected')), + ('default', models.BooleanField(default=False, help_text='Make this the default template for all new reports or just for the selected client', verbose_name='Default')), + ('client', models.ForeignKey(blank=True, help_text='Template will only be displayed for this client', null=True, on_delete=django.db.models.deletion.CASCADE, to='rolodex.Client')), + ('uploaded_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'verbose_name': 'Report Template', + 'verbose_name_plural': 'Report Templates', + 'ordering': ['name', 'document', 'client'], + }, + ), + ] diff --git a/ghostwriter/reporting/migrations/0011_report_template.py b/ghostwriter/reporting/migrations/0011_report_template.py new file mode 100644 index 000000000..db6817c41 --- /dev/null +++ b/ghostwriter/reporting/migrations/0011_report_template.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.3 on 2020-09-18 23:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0010_reporttemplate'), + ] + + operations = [ + migrations.AddField( + model_name='report', + name='template', + field=models.ForeignKey(help_text='Select the report template to use for ths report', null=True, on_delete=django.db.models.deletion.SET_NULL, to='reporting.ReportTemplate'), + ), + ] diff --git a/ghostwriter/reporting/migrations/0012_auto_20200923_2228.py b/ghostwriter/reporting/migrations/0012_auto_20200923_2228.py new file mode 100644 index 000000000..96f706d9c --- /dev/null +++ b/ghostwriter/reporting/migrations/0012_auto_20200923_2228.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.3 on 2020-09-23 22:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0011_report_template'), + ] + + operations = [ + migrations.AlterModelOptions( + name='findingnote', + options={'ordering': ['finding', '-timestamp'], 'verbose_name': 'Finding note', 'verbose_name_plural': 'Finding notes'}, + ), + migrations.AlterModelOptions( + name='reporttemplate', + options={'ordering': ['-default', 'client', 'name'], 'verbose_name': 'Report template', 'verbose_name_plural': 'Report templates'}, + ), + ] diff --git a/ghostwriter/reporting/migrations/0013_reporttemplate_lint_result.py b/ghostwriter/reporting/migrations/0013_reporttemplate_lint_result.py new file mode 100644 index 000000000..43d0a73d4 --- /dev/null +++ b/ghostwriter/reporting/migrations/0013_reporttemplate_lint_result.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.3 on 2020-09-24 17:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0012_auto_20200923_2228'), + ] + + operations = [ + migrations.AddField( + model_name='reporttemplate', + name='lint_result', + field=models.CharField(blank=True, help_text='Results returned by the linter for this template', max_length=255, null=True, verbose_name='Template Linter Results'), + ), + ] diff --git a/ghostwriter/reporting/migrations/0014_auto_20200924_1822.py b/ghostwriter/reporting/migrations/0014_auto_20200924_1822.py new file mode 100644 index 000000000..f0f920a40 --- /dev/null +++ b/ghostwriter/reporting/migrations/0014_auto_20200924_1822.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.3 on 2020-09-24 18:22 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0013_reporttemplate_lint_result'), + ] + + operations = [ + migrations.AlterField( + model_name='reporttemplate', + name='lint_result', + field=models.TextField(blank=True, help_text='Results returned by the linter for this template', null=True, verbose_name='Template Linter Results'), + ), + ] diff --git a/ghostwriter/reporting/migrations/0015_auto_20201016_1756.py b/ghostwriter/reporting/migrations/0015_auto_20201016_1756.py new file mode 100644 index 000000000..72b90f706 --- /dev/null +++ b/ghostwriter/reporting/migrations/0015_auto_20201016_1756.py @@ -0,0 +1,40 @@ +# Generated by Django 3.0.10 on 2020-10-16 17:56 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0014_auto_20200924_1822'), + ] + + operations = [ + migrations.CreateModel( + name='DocType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('doc_type', models.CharField(help_text='Enter a file extension for a report template filetype', max_length=5, unique=True, verbose_name='Document Type')), + ], + options={ + 'verbose_name': 'Document type', + 'verbose_name_plural': 'Document types', + 'ordering': ['doc_type'], + }, + ), + migrations.AlterModelOptions( + name='reporttemplate', + options={'ordering': ['-default', 'doc_type', 'client', 'name'], 'verbose_name': 'Report template', 'verbose_name_plural': 'Report templates'}, + ), + migrations.AddField( + model_name='reporttemplate', + name='changelog', + field=models.TextField(blank=True, help_text='Add a line explaining any file changes', null=True, verbose_name='Template Change Log'), + ), + migrations.AddField( + model_name='reporttemplate', + name='doc_type', + field=models.ForeignKey(blank=True, help_text='Select the filetype for this template', null=True, on_delete=django.db.models.deletion.SET_NULL, to='reporting.DocType'), + ), + ] diff --git a/ghostwriter/reporting/migrations/0016_auto_20201017_0014.py b/ghostwriter/reporting/migrations/0016_auto_20201017_0014.py new file mode 100644 index 000000000..0fccd1e56 --- /dev/null +++ b/ghostwriter/reporting/migrations/0016_auto_20201017_0014.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.10 on 2020-10-17 00:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0015_auto_20201016_1756'), + ] + + operations = [ + migrations.RemoveField( + model_name='report', + name='template', + ), + migrations.AddField( + model_name='report', + name='docx_template', + field=models.ForeignKey(help_text='Select the Word template to use for this report', limit_choices_to={'doc_type__iexact': 'docx'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='reporttemplate_docx_set', to='reporting.ReportTemplate'), + ), + migrations.AddField( + model_name='report', + name='pptx_template', + field=models.ForeignKey(help_text='Select the PowerPoint template to use for this report', limit_choices_to={'doc_type__iexact': 'pptx'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='reporttemplate_pptx_set', to='reporting.ReportTemplate'), + ), + migrations.AlterField( + model_name='finding', + name='finding_guidance', + field=models.TextField(blank=True, help_text='Provide notes for your team that describes how the finding is intended to be used or edited during editing', null=True, verbose_name='Finding Guidance'), + ), + ] diff --git a/ghostwriter/reporting/migrations/0017_auto_20201019_2318.py b/ghostwriter/reporting/migrations/0017_auto_20201019_2318.py new file mode 100644 index 000000000..159343efd --- /dev/null +++ b/ghostwriter/reporting/migrations/0017_auto_20201019_2318.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.10 on 2020-10-19 23:18 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0016_auto_20201017_0014'), + ] + + operations = [ + migrations.AlterField( + model_name='report', + name='docx_template', + field=models.ForeignKey(help_text='Select the Word template to use for this report', limit_choices_to={'doc_type__doc_type__iexact': 'docx'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='reporttemplate_docx_set', to='reporting.ReportTemplate'), + ), + migrations.AlterField( + model_name='report', + name='pptx_template', + field=models.ForeignKey(help_text='Select the PowerPoint template to use for this report', limit_choices_to={'doc_type__doc_type__iexact': 'pptx'}, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='reporttemplate_pptx_set', to='reporting.ReportTemplate'), + ), + ] diff --git a/ghostwriter/reporting/migrations/0018_auto_20201027_1914.py b/ghostwriter/reporting/migrations/0018_auto_20201027_1914.py new file mode 100644 index 000000000..a899b320a --- /dev/null +++ b/ghostwriter/reporting/migrations/0018_auto_20201027_1914.py @@ -0,0 +1,21 @@ +# Generated by Django 3.0.10 on 2020-10-27 19:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0017_auto_20201019_2318'), + ] + + operations = [ + migrations.AlterModelOptions( + name='reporttemplate', + options={'ordering': ['doc_type', 'client', 'name'], 'verbose_name': 'Report template', 'verbose_name_plural': 'Report templates'}, + ), + migrations.RemoveField( + model_name='reporttemplate', + name='default', + ), + ] diff --git a/ghostwriter/reporting/migrations/0019_auto_20201105_0609.py b/ghostwriter/reporting/migrations/0019_auto_20201105_0609.py new file mode 100644 index 000000000..eab867671 --- /dev/null +++ b/ghostwriter/reporting/migrations/0019_auto_20201105_0609.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.10 on 2020-11-05 06:09 + +import django.core.files.storage +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0018_auto_20201027_1914'), + ] + + operations = [ + migrations.AlterField( + model_name='reporttemplate', + name='document', + field=models.FileField(blank=True, storage=django.core.files.storage.FileSystemStorage(base_url='/templates', location='/app/ghostwriter/media/templates'), upload_to=''), + ), + ] diff --git a/ghostwriter/reporting/migrations/0020_auto_20201105_0641.py b/ghostwriter/reporting/migrations/0020_auto_20201105_0641.py new file mode 100644 index 000000000..c230eae84 --- /dev/null +++ b/ghostwriter/reporting/migrations/0020_auto_20201105_0641.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.10 on 2020-11-05 06:41 + +import django.core.files.storage +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0019_auto_20201105_0609'), + ] + + operations = [ + migrations.AlterField( + model_name='reporttemplate', + name='document', + field=models.FileField(blank=True, storage=django.core.files.storage.FileSystemStorage(location='/app/ghostwriter/media/templates'), upload_to=''), + ), + ] diff --git a/ghostwriter/reporting/migrations/0021_auto_20201119_2343.py b/ghostwriter/reporting/migrations/0021_auto_20201119_2343.py new file mode 100644 index 000000000..99022c073 --- /dev/null +++ b/ghostwriter/reporting/migrations/0021_auto_20201119_2343.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.10 on 2020-11-19 23:43 + +from django.db import migrations, models +import ghostwriter.reporting.models +import ghostwriter.reporting.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('reporting', '0020_auto_20201105_0641'), + ] + + operations = [ + migrations.AlterField( + model_name='evidence', + name='document', + field=models.FileField(blank=True, upload_to=ghostwriter.reporting.models.Evidence.set_upload_destination, validators=[ghostwriter.reporting.validators.validate_evidence_extension]), + ), + ] diff --git a/ghostwriter/reporting/models.py b/ghostwriter/reporting/models.py index d3f78be63..9a2b5cea7 100644 --- a/ghostwriter/reporting/models.py +++ b/ghostwriter/reporting/models.py @@ -1,14 +1,23 @@ """This contains all of the database models used by the Reporting application.""" # Standard Libraries +import json +import logging import os # Django & Other 3rd Party Libraries from django.conf import settings from django.core.exceptions import ValidationError +from django.core.files.storage import FileSystemStorage from django.db import models from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ + +# Ghostwriter Libraries +from .validators import validate_evidence_extension + +# Using __name__ resolves to ghostwriter.reporting.models +logger = logging.getLogger(__name__) class Severity(models.Model): @@ -20,13 +29,18 @@ class Severity(models.Model): "Severity", max_length=255, unique=True, - help_text="Severity rating (e.g. High, Low)", + help_text="Name for this severity rating (e.g. High, Low)", ) weight = models.IntegerField( "Severity Weight", default=1, - help_text="Used for custom sorting in reports. Lower numbers are " - "more severe.", + help_text="Weight for sorting severity categories in reports (lower numbers are more severe)", + ) + color = models.CharField( + "Severity Color", + max_length=6, + default="7A7A7A", + help_text="Six character hex color code associated with this severity for reports (e.g., FF7E79)", ) def count_findings(self): @@ -35,6 +49,23 @@ def count_findings(self): """ return Finding.objects.filter(severity=self).count() + @property + def color_rgb(self): + """ + Return the severity color code as a list of RGB values. + """ + return tuple(int(self.color[i : i + 2], 16) for i in (0, 2, 4)) + + @property + def color_hex(self): + """ + Return the severity color code as a list of hexadecimal. + """ + n = 2 + return tuple( + hex(int(self.color[i : i + n], 16)) for i in range(0, len(self.color), n) + ) + count = property(count_findings) class Meta: @@ -130,8 +161,7 @@ class Finding(models.Model): "Finding Guidance", null=True, blank=True, - help_text="Provide notes for your team that describes how the finding is intended to be used or edited" - "during editing", + help_text="Provide notes for your team that describes how the finding is intended to be used or edited during editing", ) # Foreign Keys severity = models.ForeignKey( @@ -159,6 +189,134 @@ def __str__(self): return f"[{self.severity}] {self.title}" +class DocType(models.Model): + """ + Stores an individual document type, related to :model:`reporting.ReportTemplate`. + """ + + doc_type = models.CharField( + "Document Type", + max_length=5, + unique=True, + help_text="Enter a file extension for a report template filetype", + ) + + class Meta: + ordering = [ + "doc_type", + ] + verbose_name = "Document type" + verbose_name_plural = "Document types" + + def __str__(self): + return f"{self.doc_type}" + + +class ReportTemplate(models.Model): + """ + Stores an individual report template file, related to :model:`reporting.Report`. + """ + + # Direct template uploads to ``TEMPLATE_LOC`` instead of ``MEDIA`` + template_storage = FileSystemStorage(location=settings.TEMPLATE_LOC) + + document = models.FileField(storage=template_storage, blank=True) + name = models.CharField( + "Template Name", + null=True, + max_length=255, + help_text="Provide a name to be used when selecting this template", + ) + upload_date = models.DateField( + "Upload Date", + auto_now=True, + help_text="Date and time the template was first uploaded", + ) + last_update = models.DateField( + "Last Modified", + auto_now=True, + help_text="Date and time the report was last modified", + ) + description = models.TextField( + "Description", + blank=True, + help_text="Provide a description of this template", + ) + protected = models.BooleanField( + "Protected", + default=False, + help_text="Only administrators can edit this template", + ) + lint_result = models.TextField( + "Template Linter Results", + null=True, + blank=True, + help_text="Results returned by the linter for this template", + ) + changelog = models.TextField( + "Template Change Log", + null=True, + blank=True, + help_text="Add a line explaining any file changes", + ) + # Foreign Keys + uploaded_by = models.ForeignKey( + settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, blank=True + ) + client = models.ForeignKey( + "rolodex.Client", + on_delete=models.CASCADE, + null=True, + blank=True, + help_text="Template will only be displayed for this client", + ) + doc_type = models.ForeignKey( + "reporting.DocType", + on_delete=models.SET_NULL, + null=True, + blank=True, + help_text="Select the filetype for this template", + ) + + class Meta: + ordering = ["doc_type", "client", "name"] + verbose_name = "Report template" + verbose_name_plural = "Report templates" + + def get_absolute_url(self): + return reverse("reporting:template_file", args=[str(self.id)]) + + def __str__(self): + return f"{self.name}" + + def clean(self, *args, **kwargs): + super(ReportTemplate, self).clean(*args, **kwargs) + # Validate here in the model so there is always a file asociated with the entry + if not self.document: + raise ValidationError(_("You must provide a template file"), "incomplete") + + @property + def filename(self): + return os.path.basename(self.document.name) + + def get_status(self): + result_code = "unknown" + if self.lint_result: + try: + lint_result = json.loads(self.lint_result) + result_code = lint_result["result"] + except json.decoder.JSONDecodeError: + logger.exception( + "Could not decode data in model as JSON: %s", self.lint_result + ) + except Exception: + logger.exception( + "Encountered an exceptio while trying to decode this as JSON: %s", + self.lint_result, + ) + return result_code + + class Report(models.Model): """ Stores an individual report, related to :model:`rolodex.Project` and :model:`users.User`. @@ -189,6 +347,24 @@ class Report(models.Model): null=True, help_text="Select the project tied to this report", ) + docx_template = models.ForeignKey( + "ReportTemplate", + related_name="reporttemplate_docx_set", + on_delete=models.SET_NULL, + limit_choices_to={ + "doc_type__doc_type__iexact": "docx", + }, + null=True, + help_text="Select the Word template to use for this report", + ) + pptx_template = models.ForeignKey( + "ReportTemplate", + related_name="reporttemplate_pptx_set", + on_delete=models.SET_NULL, + limit_choices_to={"doc_type__doc_type__iexact": "pptx"}, + null=True, + help_text="Select the PowerPoint template to use for this report", + ) created_by = models.ForeignKey( settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, blank=True ) @@ -219,7 +395,10 @@ class ReportFindingLink(models.Model): max_length=255, help_text="Enter a title for this finding that will appear in the reports", ) - position = models.IntegerField("Report Position", default=1,) + position = models.IntegerField( + "Report Position", + default=1, + ) affected_entities = models.TextField( "Affected Entities", null=True, @@ -336,7 +515,11 @@ def set_upload_destination(instance, filename): """ return os.path.join("evidence", str(instance.finding.report.id), filename) - document = models.FileField(upload_to=set_upload_destination, blank=True) + document = models.FileField( + upload_to=set_upload_destination, + validators=[validate_evidence_extension], + blank=True, + ) friendly_name = models.CharField( "Friendly Name", null=True, @@ -355,7 +538,9 @@ def set_upload_destination(instance, filename): help_text="Provide a one line caption to be used in the report - keep it brief", ) description = models.TextField( - "Description", blank=True, help_text="Describe this evidence to your team", + "Description", + blank=True, + help_text="Describe this evidence to your team", ) # Foreign Keys finding = models.ForeignKey("ReportFindingLink", on_delete=models.CASCADE) @@ -380,6 +565,10 @@ def clean(self, *args, **kwargs): if not self.document: raise ValidationError(_("You must provide an evidence file"), "incomplete") + @property + def filename(self): + return os.path.basename(self.document.name) + class Archive(models.Model): """ @@ -424,8 +613,8 @@ class FindingNote(models.Model): class Meta: ordering = ["finding", "-timestamp"] - verbose_name = "Local finding note" - verbose_name_plural = "Local finding notes" + verbose_name = "Finding note" + verbose_name_plural = "Finding notes" def __str__(self): return f"{self.finding} {self.timestamp}: {self.note}" diff --git a/ghostwriter/reporting/resources.py b/ghostwriter/reporting/resources.py index 2a96ac8d8..255715d32 100644 --- a/ghostwriter/reporting/resources.py +++ b/ghostwriter/reporting/resources.py @@ -5,6 +5,7 @@ from import_export.fields import Field from import_export.widgets import ForeignKeyWidget +# Ghostwriter Libraries from .models import Finding, FindingType, Severity diff --git a/ghostwriter/reporting/signals.py b/ghostwriter/reporting/signals.py new file mode 100644 index 000000000..8226ad91a --- /dev/null +++ b/ghostwriter/reporting/signals.py @@ -0,0 +1,127 @@ +"""This contains all of the model Signals used by the Reporting application.""" + +# Standard Libraries +import logging +import os + +# Django & Other 3rd Party Libraries +from django.db.models.signals import post_init, post_save +from django.dispatch import receiver + +# Ghostwriter Libraries +from ghostwriter.modules.reportwriter import TemplateLinter +from ghostwriter.reporting.models import Evidence, ReportTemplate + +# Using __name__ resolves to ghostwriter.reporting.signals +logger = logging.getLogger(__name__) + + +@receiver(post_init, sender=Evidence) +def backup_evidence_path(sender, instance, **kwargs): + """ + Backup the file path of the old evidence file in the :model:`reporting.Evidence` + instance when a new file is uploaded. + """ + instance._current_evidence = instance.document + + +@receiver(post_save, sender=Evidence) +def delete_old_evidence(sender, instance, **kwargs): + """ + Delete the old evidence file in the :model:`reporting.Evidence` instance when a + new file is uploaded. + """ + if hasattr(instance, "_current_evidence"): + if instance._current_evidence: + if instance._current_evidence.path not in instance.document.path: + try: + os.remove(instance._current_evidence.path) + logger.info( + "Deleted old evidence file %s", instance._current_evidence.path + ) + except Exception: + logger.exception( + "Failed deleting old evidence file: %s", + instance._current_evidence.path, + ) + + +@receiver(post_init, sender=ReportTemplate) +def backup_template_attr(sender, instance, **kwargs): + """ + Backup the file path and document type of the old template file in the + :model:`reporting.ReportTemplate` instance when a new file is uploaded. + """ + instance._current_template = instance.document + instance._current_type = instance.doc_type + + +@receiver(post_save, sender=ReportTemplate) +def clean_template(sender, instance, created, **kwargs): + """ + Delete the old template file and lint the replacement file for an instance of + :model:`reporting.ReportTemplate`. + """ + lint_template = False + if hasattr(instance, "_current_template"): + if instance._current_template: + if instance._current_template.path not in instance.document.path: + lint_template = True + try: + if os.path.exists(instance._current_template.path): + try: + os.remove(instance._current_template.path) + logger.info( + "Deleted old template file %s", + instance._current_template.path, + ) + except Exception: + logger.exception( + "Failed to delete old tempalte file: %s", + instance._current_template.path, + ) + else: + logger.warning( + "Old template file could not be found at %s", + instance._current_template.path, + ) + except Exception: + logger.exception( + "Failed deleting old template file: %s", + instance._current_template.path, + ) + else: + logger.info( + "Template file paths match, so will not re-run the linter or delete any files" + ) + + if hasattr(instance, "_current_type"): + if instance._current_type != instance.doc_type: + lint_template = True + + if created or lint_template: + logger.info("Template file change detected, so starting linter") + logger.info( + "Linting newly uploaded template: %s", + instance.document.path, + ) + try: + template_loc = instance.document.path + linter = TemplateLinter(template_loc=template_loc) + if instance.doc_type.doc_type == "docx": + results = linter.lint_docx() + elif instance.doc_type.doc_type == "pptx": + results = linter.lint_pptx() + else: + logger.warning( + "Template had an unknown filetype not supported by the linter: %s", + instance.doc_type, + ) + results = {} + instance.lint_result = results + # Disconnect signal to save model and avoid infinite loop + post_save.disconnect(clean_template, sender=ReportTemplate) + instance.save() + post_save.connect(clean_template, sender=ReportTemplate) + except Exception: + logger.exception("Failed to update new template with linting results") diff --git a/ghostwriter/reporting/tasks.py b/ghostwriter/reporting/tasks.py index 1d404c469..9168125e3 100644 --- a/ghostwriter/reporting/tasks.py +++ b/ghostwriter/reporting/tasks.py @@ -23,13 +23,15 @@ def zip_directory(path, zip_handler): - """Zip the target directory and all of its contents, for archiving purposes. - - Parameters: + """ + Zip the target directory and all of its contents to create a project archive. - path The file path to archive. + **Parameters** - zip_handler A `zipfile.ZipFile()` object. + ``path`` + File path to archive + ``zip_handler`` + A ``zipfile.ZipFile()`` object to create the archive """ # Walk the target directory abs_src = os.path.abspath(path) @@ -42,9 +44,10 @@ def zip_directory(path, zip_handler): def archive_projects(): - """Collect all completed projects that have not yet been archived and - archive the associated reports. The archived reports are deleted and - the new archive file is logged in the `Archive` model. + """ + Collect all completed :model:`rolodex.Project` that have not yet been archived and + archive the associated reports. The archived reports are deleted and the new archive + file is logged in the :model:`rolodex.Archive`. """ # Get the non-archived reports for all projects marked as complete report_queryset = Report.objects.select_related("project").filter( diff --git a/ghostwriter/reporting/templates/reporting/evidence_detail.html b/ghostwriter/reporting/templates/reporting/evidence_detail.html index a57f5c49a..1d5b556f4 100644 --- a/ghostwriter/reporting/templates/reporting/evidence_detail.html +++ b/ghostwriter/reporting/templates/reporting/evidence_detail.html @@ -28,10 +28,13 @@

- {{ evidence.document }} + + + + @@ -41,12 +44,24 @@

- - + + - - + +
Original Filename{{ evidence.filename }}
Uploaded by {{ evidence.uploaded_by }}{{ evidence.upload_date }}
Description{{ evidence.description|bleach }}Report Caption + {% if evidence.caption %} + {{ evidence.caption }} + {% else %} + --- + {% endif %} +
Report Caption{{ evidence.caption }}Description + {% if evidence.description %} + {{ evidence.description|bleach }} + {% else %} + --- + {% endif %} +
diff --git a/ghostwriter/reporting/templates/reporting/evidence_form_modal.html b/ghostwriter/reporting/templates/reporting/evidence_form_modal.html index bac8b3077..4250e2280 100644 --- a/ghostwriter/reporting/templates/reporting/evidence_form_modal.html +++ b/ghostwriter/reporting/templates/reporting/evidence_form_modal.html @@ -13,22 +13,25 @@ if (event.data.mceAction ==='evidence_upload'){ var used_friendly_names = []; {% for name in used_friendly_names %} - used_friendly_names.push('{{ name|escapejs }}') + used_friendly_names.push('{{ name|escapejs }}') {% endfor %} - var value = { - friendly_name: document.getElementById('id_friendly_name').value, - evidence_file: document.getElementById('id_document').value, - caption: document.getElementById('id_caption').value, - used_friendly_names: used_friendly_names - }; + try{ + var value = { + friendly_name: document.getElementById('id_friendly_name').value, + evidence_file: document.getElementById('id_document').value, + caption: document.getElementById('id_caption').value, + used_friendly_names: used_friendly_names + }; + document.getElementById('evidence-upload-form').submit(); - document.getElementById('evidence-upload-form').submit(); - - window.parent.postMessage({ - mceAction: 'execCommand', - cmd: 'upload_and_insert', - value - }, origin); + window.parent.postMessage({ + mceAction: 'execCommand', + cmd: 'upload_and_insert', + value + }, origin); + } catch(err) { + displayToastTop({type:'error', string:'You must provide a friendly name, caption, and evidence file', context:'form'}); + } } }); diff --git a/ghostwriter/reporting/templates/reporting/finding_form.html b/ghostwriter/reporting/templates/reporting/finding_form.html index c455e7a50..9335d275d 100644 --- a/ghostwriter/reporting/templates/reporting/finding_form.html +++ b/ghostwriter/reporting/templates/reporting/finding_form.html @@ -26,18 +26,38 @@

Ghostwriter supports several template keywords you may utilize to format text and insert various pieces of information:

{% verbatim %} - +
- - + + - - + + + + + + + + + + + + + + + + + + + + + + {% endverbatim %}
Keyword Usage
{{.client}}This keyword will be replaced with the client's short name. The full name will be used if a short name has not been set for the client.{{.client}}This keyword will be replaced with the client's short name. The full name will be used if a short name has not been set for the client.
{{.caption}}Start a line of text with this keyword to make it a caption. This is intended to follow a code block.{{.project_type}}This keyword will be replaced with the project type in lowercase (e.g., penetration test).
{{.project_start}}This keyword will be replaced with the project's start date in M D, YYYY format (e.g., October 31, 2020).
{{.project_start_uk}}This keyword will be replaced with the project's start date in D M YYYY format (e.g., 31 October 2020).
{{.project_end}}This keyword will be replaced with the project's end date in M D, YYYY format (e.g., October 31, 2020).
{{.project_end_uk}}This keyword will be replaced with the project's end date in D M YYYY format (e.g., 31 October 2020).
{{.caption}}Start a line of text with this keyword to make it a caption. This is intended to follow a code block.
diff --git a/ghostwriter/reporting/templates/reporting/local_edit.html b/ghostwriter/reporting/templates/reporting/local_edit.html index 7136786ee..ad360e079 100644 --- a/ghostwriter/reporting/templates/reporting/local_edit.html +++ b/ghostwriter/reporting/templates/reporting/local_edit.html @@ -23,20 +23,20 @@

Ghostwriter supports several template keywords you may utilize to format text and insert various pieces of information. Begin typing @ to open the autocomplete dialog for keywords.

- +
- {% verbatim %}{% endverbatim %} - {% endverbatim %} + + + {% verbatim %}{% endverbatim %} + + + + {% verbatim %}{% endverbatim %} + + + + {% verbatim %}{% endverbatim %} + + + + {% verbatim %}{% endverbatim %} + + + + {% verbatim %}{% endverbatim %} + + {% verbatim %} - - + + {% endverbatim %} {% if reportfindinglink.evidence_set.all %} {% for finding in reportfindinglink.evidence_set.all %} - - + + + + + {% endfor %} {% endif %}
Keyword Usage
{{.client}} + {% verbatim %}{{.client}} {% if reportfindinglink.report.project.client.short_name %} This keyword will be replaced with the client's short name, "{{ reportfindinglink.report.project.client.short_name }}." {% else %} @@ -44,27 +44,51 @@ {% endif %}
{{.project_type}}This keyword will be replaced with the project type in lowercase, {{ reportfindinglink.report.project.project_type|lower }}.
{{.project_start}}This keyword will be replaced with the project's start date in M D, YYYY format: {{ reportfindinglink.report.project.start_date|date:"M d, Y" }}
{{.project_start_uk}}This keyword will be replaced with the project's start date in D M YYYY format: {{ reportfindinglink.report.project.start_date|date:"d M Y" }}
{{.project_end}}This keyword will be replaced with the project's end date in M D, YYYY format: {{ reportfindinglink.report.project.end_date|date:"M d, Y" }}
{{.project_end_uk}}This keyword will be replaced with the project's end date in D M YYYY format: {{ reportfindinglink.report.project.end_date|date:"d M Y" }}
{{.caption}}Start a line of text with this keyword to make it a caption. This is intended to follow a code block.{{.caption}}Start a line of text with this keyword to make it a caption. This is intended to follow a code block.
+ {% templatetag openvariable %}.{{ finding.friendly_name }}{% templatetag closevariable %} On a new line, reference this evidence file, {{ finding.document.name }}, to insert it into the finding.On a new line, reference this evidence file, {{ finding.document.name }}, to insert it into the finding.
+ {% templatetag openvariable %}.ref {{ finding.friendly_name }}{% templatetag closevariable %} + Add a cross-reference to the caption of thr above evidence file.
-

Insert evidence by using the above keywords. Image evidence may be inserted and previewed inline by clicking the WYSIWIG editor's image button and entering the evidence image's Ghostwriter URL (e.g., https://ghosttwriter.local/media/evidence/2/ghostwriter.png).

-

Inserting an external image's URL will work but these images will not be carried over to the report outputs.

-

For additional formatting, utilize the WYSIWIG HTML formatting to apply bold, italic, code, and inline code text styles.

-

These styles will carry over to the Word report output as bold, italic, "CodeBlock," and "Code (Inline)" styles respectively.

+

Insert evidence by using the above keywords. For additional formatting, utilize the WYSIWIG HTML formatting to apply bold, italic, code, inline code, and other text styles.

+

These styles will carry over to the Word and PowerPoint reports. See the documentaiton for more details.

@@ -161,6 +185,11 @@ var evidenceFiles = [ { text: '\{\{.client\}\}', value: '\{\{.client\}\}' }, { text: '\{\{.caption\}\}', value: '\{\{.caption\}\}' }, + { text: '\{\{.project_start\}\}', value: '\{\{.project_start\}\}' }, + { text: '\{\{.project_end\}\}', value: '\{\{.project_end\}\}' }, + { text: '\{\{.project_start_uk\}\}', value: '\{\{.project_start_uk\}\}' }, + { text: '\{\{.project_end_uk\}\}', value: '\{\{.project_end_uk\}\}' }, + { text: '\{\{.project_type\}\}', value: '\{\{.project_type\}\}' }, {% if reportfindinglink.evidence_set.all %} {% for finding in reportfindinglink.evidence_set.all %} { text: '\{\{.{{ finding.friendly_name|escapejs }}\}\}', value: '\{\{.{{ finding.friendly_name|escapejs }}\}\}' }, diff --git a/ghostwriter/reporting/templates/reporting/report_detail.html b/ghostwriter/reporting/templates/reporting/report_detail.html index 179e0d07d..3402bae5c 100644 --- a/ghostwriter/reporting/templates/reporting/report_detail.html +++ b/ghostwriter/reporting/templates/reporting/report_detail.html @@ -1,4 +1,5 @@ {% extends "base_generic.html" %} +{% load crispy_forms_tags %} {% load report_tags %} {% block pagetitle %}{{ report.title }}{% endblock %} @@ -179,7 +180,7 @@

Attached Findings

{% endfor %} {% else %} - Add a {{ group }} finding or drag-and-drop a finding here to update its severity. + Add {{ group }} findings or drag-and-drop a finding here to update its severity. {% endif %} {% endfor %} @@ -193,12 +194,17 @@

Attached Findings

Generate Reports

+
+ The report engine will use the selected templates: + {% crispy form form.helper %} +
+
-
docx +
docx
xlsx
pptx
JSON -
All +
All
{% endblock %} @@ -208,6 +214,27 @@

Generate Reports

{% endblock %} {% block morescripts %} + + + + + - - - + + + {% comment %} Include the reusable delete confirmation modal and related scripts {% endcomment %} {% include "confirm_delete_modal.html" %} {% endblock %} diff --git a/ghostwriter/reporting/templates/reporting/report_form.html b/ghostwriter/reporting/templates/reporting/report_form.html index 13d1e9e53..d59741e24 100644 --- a/ghostwriter/reporting/templates/reporting/report_form.html +++ b/ghostwriter/reporting/templates/reporting/report_form.html @@ -14,6 +14,7 @@
  • {{ project }}
  • Project Report
    • {% else %} +
  • Reports
  • New Report Form
    • {% endif %} @@ -22,7 +23,7 @@ {% block content %} -

    Provide a name for this report below:

    +

    Provide a meaningful name for this report, select the project to which it should be associated, and optionally select report templates to use:

    {% if form.errors %} diff --git a/ghostwriter/reporting/templates/reporting/report_template_detail.html b/ghostwriter/reporting/templates/reporting/report_template_detail.html new file mode 100644 index 000000000..5f1688486 --- /dev/null +++ b/ghostwriter/reporting/templates/reporting/report_template_detail.html @@ -0,0 +1,154 @@ +{% extends 'base_generic.html' %} + +{% load report_tags %} + +{% load bleach_tags %} + +{% block pagetitle %}Report Template Detail{% endblock %} + +{% block breadcrumbs %} + +{% endblock %} + +{% block content %} +

    + {{ reporttemplate.name }} +
    + +
    + Download + Lint + {% if reporttemplate.protected and request.user.is_staff or not reporttemplate.protected %} + Edit + Delete + {% endif %} +
    +
    +

    + + {% if reporttemplate.protected %} + + {% endif %} + + +
    + + + + + + + + + + + + + + + + + + + + + +
    Template Type{{ reporttemplate.doc_type }}
    Uploaded by + {% if reporttemplate.uploaded_by %} + {{ reporttemplate.uploaded_by }} + {% else %} + Unknown + {% endif %} +
    Upload Date{{ reporttemplate.upload_date }}
    Last Update{{ reporttemplate.last_update }}
    Client + {% if reporttemplate.client %} + {{ reporttemplate.client }} + {% else %} + N/A + {% endif %} +
    +
    + +
    + {% if reporttemplate.description %} + {{ reporttemplate.description|bleach }} + {% endif %} +
    + + +
    + {% include "snippets/template_lint_results.html" %} +
    + + +

    Template CHANGELOG

    +
    +
    + {% if reporttemplate.changelog %} + {{ reporttemplate.changelog|bleach }} + {% else %} +

    No CHANGELOG data

    + {% endif %} +
    +{% endblock %} + +{% block morescripts %} + + +{% endblock %} diff --git a/ghostwriter/reporting/templates/reporting/report_template_form.html b/ghostwriter/reporting/templates/reporting/report_template_form.html new file mode 100644 index 000000000..2d9949b1a --- /dev/null +++ b/ghostwriter/reporting/templates/reporting/report_template_form.html @@ -0,0 +1,47 @@ +{% extends "base_generic.html" %} +{% load crispy_forms_tags %} + +{% block pagetitle %}Report Creation{% endblock %} + +{% block breadcrumbs %} + +{% endblock %} + +{% block content %} + +

    Provide informatin for this report template:

    + + + {% if form.errors %} + + {% endif %} + + + {% crispy form form.helper %} +{% endblock %} + +{% block morescripts %} + + +{% endblock %} diff --git a/ghostwriter/reporting/templates/reporting/report_templates_list.html b/ghostwriter/reporting/templates/reporting/report_templates_list.html new file mode 100644 index 000000000..8afc651fc --- /dev/null +++ b/ghostwriter/reporting/templates/reporting/report_templates_list.html @@ -0,0 +1,86 @@ +{% extends "base_generic.html" %} +{% load crispy_forms_tags %} + +{% block pagetitle %}Report Template List{% endblock %} + +{% block breadcrumbs %} + +{% endblock %} + +{% block content %} +

    + Upload a Report Template +

    + + + {% if reporttemplate_list %} +

    Note: Only admins may edit protected report templates.

    + +

    This page lists all report templates and their current statuses. To investigate a status, view the template's details page.

    + + + + + + + + + + + + + {% for template in reporttemplate_list %} + + + + + + + + {% endfor %} +
    StatusDoc TypeNameClientOptions
    + {% with template.get_status as status %} + Ready + {% else %} + {% if status == "warning" %} + badge-warning + {% elif status == "unknown" %} + badge-secondary + {% else %} + badge-danger + {% endif %} + ">{{ status|capfirst }} + {% endif %} + {% endwith %} + {{ template.doc_type }}{{ template.name }} + {% if template.client %} + {{ template.client }} + {% else%} + -- + {% endif %} + +
    + +
    + Download + {% if template.protected and request.user.is_staff or not template.protected %} + Edit + Delete + {% else %} + Editing Locked + {% endif %} +
    +
    +
    + {% else %} +

    You have not uploaded any templates yet!

    + {% endif %} +{% endblock %} diff --git a/ghostwriter/reporting/templates/reports/blank_template.docx b/ghostwriter/reporting/templates/reports/blank_template.docx deleted file mode 100644 index d69dae6fc..000000000 Binary files a/ghostwriter/reporting/templates/reports/blank_template.docx and /dev/null differ diff --git a/ghostwriter/reporting/templates/reports/template.docx b/ghostwriter/reporting/templates/reports/template.docx index 412da09bf..ee688cc3c 100644 Binary files a/ghostwriter/reporting/templates/reports/template.docx and b/ghostwriter/reporting/templates/reports/template.docx differ diff --git a/ghostwriter/reporting/templates/reports/template.pptx b/ghostwriter/reporting/templates/reports/template.pptx index 8b1f13523..95231fd19 100644 Binary files a/ghostwriter/reporting/templates/reports/template.pptx and b/ghostwriter/reporting/templates/reports/template.pptx differ diff --git a/ghostwriter/reporting/templates/snippets/template_lint_results.html b/ghostwriter/reporting/templates/snippets/template_lint_results.html new file mode 100644 index 000000000..2e69a76af --- /dev/null +++ b/ghostwriter/reporting/templates/snippets/template_lint_results.html @@ -0,0 +1,59 @@ +{% load report_tags %} + +

    Linting Results

    +
    +{% if reporttemplate.lint_result %} + {% with reporttemplate.lint_result|load_json as lint_result %} + + + {% if lint_result.warnings %} + + {% for warning in lint_result.warnings %} + + + + + {% endfor %} +
    + Warning + + {{ warning }} +
    + {% endif %} + + {% if lint_result.errors %} + + {% for error in lint_result.errors %} + + + + + {% endfor %} +
    + Error + + {{ error }} +
    + {% endif %} + {% endwith %} +{% else %} + +{% endif %} \ No newline at end of file diff --git a/ghostwriter/reporting/templatetags/report_tags.py b/ghostwriter/reporting/templatetags/report_tags.py index a3cba8f25..7c4d3b26f 100644 --- a/ghostwriter/reporting/templatetags/report_tags.py +++ b/ghostwriter/reporting/templatetags/report_tags.py @@ -1,22 +1,33 @@ """Custom template tags for the reporting app.""" -# Import Django libraries +# Standard Libraries +import json +import logging +from collections import defaultdict + +# Django & Other 3rd Party Libraries from django import template -from django.db.models import Q -# Import custom models -from ghostwriter.reporting.models import ReportFindingLink, Severity - -# Other Python imports -from collections import defaultdict +# Ghostwriter Libraries +from ghostwriter.reporting.models import Severity register = template.Library() +# Using __name__ resolves to ghostwriter.reporting.template_tags.report_tags +logger = logging.getLogger(__name__) + @register.filter def get_item(dictionary, key): - """Accepts a dictionary and key and returns the contents. Used for - referencing dictionary keys with variables. + """ + Return a key value from a dictionary object. + + **Parameters** + + ``dictonary`` + Python dictionary object to parse + ``key`` + Key name tor etrieve from the dictionary """ # Use `get` to return `None` if not found return dictionary.get(key) @@ -24,9 +35,13 @@ def get_item(dictionary, key): @register.simple_tag def group_by_severity(queryset): - """Accepts a queryset and returns a dictionary with the queryset - grouped by the `Severity` field. Works with the `Finding` and - `ReportFindingLink` models. + """ + Group a queryset by the ``Severity`` field. + + **Parameters** + + ``queryset`` + Instance of :model:`reporting.Report` or :model:`reporting.Finding` """ all_severity = Severity.objects.all().order_by("weight") severity_dict = defaultdict(list) @@ -36,3 +51,21 @@ def group_by_severity(queryset): severity_dict[str(finding.severity)].append(finding) # Return a basic dict because templates can't handle defaultdict return dict(severity_dict) + + +@register.filter +def load_json(data): + """ + Parse a string as JSON and return JSON suitable for iterating. + + **Parameters** + + ``data`` + String to parse as JSON + """ + try: + return json.loads(data) + except json.decoder.JSONDecodeError: + logger.exception("Could not decode the string in the string: %s", data) + except Exception: + logger.exception("Encountered an error while trying to decode data as JSON") diff --git a/ghostwriter/reporting/urls.py b/ghostwriter/reporting/urls.py index 10c105a04..471bfcec5 100644 --- a/ghostwriter/reporting/urls.py +++ b/ghostwriter/reporting/urls.py @@ -1,8 +1,10 @@ """This contains all of the URL mappings used by the Reporting application.""" # Django & Other 3rd Party Libraries +from ghostwriter.reporting.views import EvidenceCreate from django.urls import path +# Ghostwriter Libraries from . import views app_name = "reporting" @@ -12,6 +14,7 @@ path("", views.index, name="index"), path("findings/", views.findings_list, name="findings"), path("reports/", views.reports_list, name="reports"), + path("templates/", views.ReportTemplateListView.as_view(), name="templates"), path("reports/archive", views.archive_list, name="archived_reports"), path( "reports/archive/download//", @@ -63,10 +66,25 @@ name="ajax_assign_finding", ), path( - "report/finding/delete/", + "ajax/finding/delete/", views.ReportFindingLinkDelete.as_view(), name="ajax_delete_local_finding", ), + path( + "ajax/report/template/swap/", + views.ReportTemplateSwap.as_view(), + name="ajax_swap_report_template", + ), + path( + "ajax/report/template/lint/", + views.ReportTemplateLint.as_view(), + name="ajax_lint_report_template", + ), + path( + "ajax/report/template/lint/results/", + views.ajax_update_template_lint_results, + name="ajax_update_template_lint_results", + ), ] # URLs for creating, updating, and deleting findings @@ -109,6 +127,31 @@ views.assign_blank_finding, name="assign_blank_finding", ), + path( + "reports/template/", + views.ReportTemplateDetailView.as_view(), + name="template_detail", + ), + path( + "reports/template/create", + views.ReportTemplateCreate.as_view(), + name="template_create", + ), + path( + "reports/template/update/", + views.ReportTemplateUpdate.as_view(), + name="template_update", + ), + path( + "reports/template/delete/", + views.ReportTemplateDelete.as_view(), + name="template_delete", + ), + path( + "reports/template/download/", + views.ReportTemplateDownload.as_view(), + name="template_download", + ), ] # URLs for local edits @@ -120,12 +163,12 @@ ), path( "reports/evidence/upload/", - views.upload_evidence, + views.EvidenceCreate.as_view(), name="upload_evidence", ), path( - "reports/evidence/modal/", - views.upload_evidence_modal, + "reports/evidence/upload//", + views.EvidenceCreate.as_view(), name="upload_evidence_modal", ), path( @@ -133,7 +176,11 @@ views.upload_evidence_modal_success, name="upload_evidence_modal_success", ), - path("reports/evidence/", views.view_evidence, name="evidence_detail"), + path( + "reports/evidence/", + views.EvidenceDetailView.as_view(), + name="evidence_detail", + ), path( "reports/evidence/update/", views.EvidenceUpdate.as_view(), diff --git a/ghostwriter/reporting/validators.py b/ghostwriter/reporting/validators.py new file mode 100644 index 000000000..8e90156c9 --- /dev/null +++ b/ghostwriter/reporting/validators.py @@ -0,0 +1,13 @@ +"""This contains custom validators for the Reporting application.""" + +# Django & Other 3rd Party Libraries +from django.core.validators import FileExtensionValidator + + +def validate_evidence_extension(value): + """ + Enforce a limited allowlist for filetypes. Allowed filetypes are limited to + text and image files that will work for report documents. + """ + allowlist = ["txt", "md", "log", "jpg", "jpeg", "png"] + return FileExtensionValidator(allowed_extensions=allowlist)(value) diff --git a/ghostwriter/reporting/views.py b/ghostwriter/reporting/views.py index 5c1dfd60c..6d710fd5e 100644 --- a/ghostwriter/reporting/views.py +++ b/ghostwriter/reporting/views.py @@ -2,6 +2,7 @@ # Standard Libraries # Import Python libraries for various things +from ghostwriter.commandcenter.models import ReportConfiguration import io import json import logging @@ -17,22 +18,30 @@ from django.contrib import messages from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required -from django.contrib.auth.mixins import LoginRequiredMixin +from django.contrib.auth.mixins import LoginRequiredMixin, PermissionRequiredMixin from django.core.files import File from django.db.models import Q -from django.db.models.signals import post_init, post_save -from django.dispatch import receiver -from django.http import Http404, HttpResponse, HttpResponseRedirect, JsonResponse +from django.http import ( + FileResponse, + Http404, + HttpResponse, + HttpResponseRedirect, + JsonResponse, +) from django.shortcuts import get_object_or_404, render +from django.template.loader import render_to_string from django.urls import reverse, reverse_lazy +from django.views import generic from django.views.generic.detail import DetailView, SingleObjectMixin from django.views.generic.edit import CreateView, DeleteView, UpdateView, View -from docx.opc.exceptions import PackageNotFoundError +from docx.opc.exceptions import PackageNotFoundError as DocxPackageNotFoundError +from pptx.exc import PackageNotFoundError as PptxPackageNotFoundError from xlsxwriter.workbook import Workbook # Ghostwriter Libraries from ghostwriter.modules import reportwriter from ghostwriter.rolodex.models import Project, ProjectAssignment +from ghostwriter.modules.exceptions import MissingTemplate from .filters import ArchiveFilter, FindingFilter, ReportFilter from .forms import ( @@ -42,6 +51,8 @@ LocalFindingNoteForm, ReportFindingLinkUpdateForm, ReportForm, + ReportTemplateForm, + SelectReportTemplateForm, ) from .models import ( Archive, @@ -52,6 +63,7 @@ LocalFindingNote, Report, ReportFindingLink, + ReportTemplate, Severity, ) from .resources import FindingResource @@ -64,41 +76,27 @@ logger = logging.getLogger(__name__) -##################### -# Signals Functions # -##################### +################## +# AJAX Functions # +################## -@receiver(post_init, sender=Evidence) -def backup_evidence_path(sender, instance, **kwargs): - """ - Backup the file path of the old evidence file in the :model:`reporting.Evidence` instance - when a new file is uploaded. +@login_required +def ajax_update_template_lint_results(request, pk): """ - instance._current_evidence = instance.document + Return an updated version of the template following a request to update linter results + for an individual :model:`reporting.ReportTemplate`. + **Template** -@receiver(post_save, sender=Evidence) -def delete_old_evidence(sender, instance, **kwargs): - """ - Delete the old evidence file in the :model:`reporting.Evidence` instance when a new file - is uploaded. + :template:`snippets/template_lint_results.html` """ - if hasattr(instance, "_current_evidence"): - if instance._current_evidence: - if instance._current_evidence.path not in instance.document.path: - try: - os.remove(instance._current_evidence.path) - logger.info( - "Deleted old evidence file %s", instance._current_evidence.path - ) - except Exception: - pass - - -################## -# AJAX Functions # -################## + template_instance = get_object_or_404(ReportTemplate, pk=pk) + html = render_to_string( + "snippets/template_lint_results.html", + {"reporttemplate": template_instance}, + ) + return HttpResponse(html) @login_required @@ -460,8 +458,11 @@ def post(self, *args, **kwargs): display_status = "Ready" classes = "healthy" else: - message = "Could not update the finding's status to: {}".format(status) result = "error" + message = "Could not update the finding's status to: {}".format(status) + display_status = "Error" + classes = "burned" + self.object.save() # Prepare the JSON response data data = { "result": result, @@ -486,6 +487,190 @@ def post(self, *args, **kwargs): return JsonResponse(data) +class ReportTemplateSwap(LoginRequiredMixin, SingleObjectMixin, View): + """ + Update the ``template`` value for an individual :model:`reporting.Report`. + """ + + model = Report + + def post(self, *args, **kwargs): + self.object = self.get_object() + docx_template_id = self.request.POST.get("docx_template", None) + pptx_template_id = self.request.POST.get("pptx_template", None) + if docx_template_id and pptx_template_id: + docx_template_query = None + pptx_template_query = None + try: + docx_template_id = int(docx_template_id) + pptx_template_id = int(pptx_template_id) + + if docx_template_id == -1: + pass + else: + docx_template_query = ReportTemplate.objects.get( + pk=docx_template_id + ) + self.object.docx_template = docx_template_query + + if pptx_template_id == -1: + pass + else: + pptx_template_query = ReportTemplate.objects.get( + pk=pptx_template_id + ) + self.object.pptx_template = pptx_template_query + + self.object.save() + + data = {"result": "success", "message": "Template successfully swapped"} + # Check template for linting issues + try: + if docx_template_query: + template_status = docx_template_query.get_status() + data["lint_result"] = template_status + if template_status != "success": + if template_status == "warning": + data[ + "docx_lint_message" + ] = "Selected Word template has warnings from linter. Check the template before generating a report." + elif template_status == "error": + data[ + "docx_lint_message" + ] = "Selected Word template has linting errors and cannot be used to generate a report." + elif template_status == "failed": + data[ + "docx_lint_message" + ] = "Selected Word template failed basic linter checks and can't be used to generate a report." + else: + data[ + "docx_lint_message" + ] = "Selected Word template has an unknown linter status. Check and lint the template before generating a report." + except Exception: + logger.exception("Failed to get the template status") + data[ + "docx_lint_message" + ] = "Could not retrieve the Word template's linter status. Check and lint the template before generating a report." + try: + if pptx_template_query: + template_status = pptx_template_query.get_status() + data["lint_result"] = template_status + if template_status != "success": + if template_status == "warning": + data[ + "pptx_lint_message" + ] = "Selected PowerPoint template has warnings from linter. Check the template before generating a report." + elif template_status == "error": + data[ + "pptx_lint_message" + ] = "Selected PowerPoint template has linting errors and cannot be used to generate a report." + elif template_status == "failed": + data[ + "pptx_lint_message" + ] = "Selected PowerPoint template failed basic linter checks and can't be used to generate a report." + else: + data[ + "pptx_lint_message" + ] = "Selected PowerPoint template has an unknown linter status. Check and lint the template before generating a report." + except Exception: + logger.exception("Failed to get the template status") + data[ + "pptx_lint_message" + ] = "Could not retrieve the PowerPoint template's linter status. Check and lint the template before generating a report." + logger.info( + "Swapped template for %s %s by request of %s", + self.object.__class__.__name__, + self.object.id, + self.request.user, + ) + except ValueError: + data = { + "result": "error", + "message": "Submitted template ID was not an integer", + } + logger.error( + "Received one or two invalid (non-integer) template IDs (%s & %s) from a request submitted by %s", + docx_template_id, + pptx_template_id, + self.request.user, + ) + except ReportTemplate.DoesNotExist: + data = { + "result": "error", + "message": "Submitted template ID does not exist", + } + logger.error( + "Received one or two invalid (non-existent) template IDs (%s & %s) from a request submitted by %s", + docx_template_id, + pptx_template_id, + self.request.user, + ) + except Exception: + data = { + "result": "error", + "message": "An exception prevented the template change", + } + logger.exception( + "Encountered an error trying to update %s %s with template IDs %s & %s from a request submitted by %s", + self.object.__class__.__name__, + self.object.id, + docx_template_id, + pptx_template_id, + self.request.user, + ) + else: + data = {"result": "error", "message": "Submitted request was incomplete"} + logger.warning( + "Received bad template IDs (%s & %s) from a request submitted by %s", + docx_template_id, + pptx_template_id, + self.request.user, + ) + return JsonResponse(data) + + +class ReportTemplateLint(LoginRequiredMixin, SingleObjectMixin, View): + """ + Check an individual :model:`reporting.ReportTemplate` for Jinja2 syntax errors + and undefined variables. + """ + + model = ReportTemplate + + def post(self, *args, **kwargs): + self.object = self.get_object() + template_loc = self.object.document.path + linter = reportwriter.TemplateLinter(template_loc=template_loc) + if self.object.doc_type.doc_type == "docx": + results = linter.lint_docx() + elif self.object.doc_type.doc_type == "pptx": + results = linter.lint_pptx() + else: + logger.warning( + "Template had an unknown filetype not supported by the linter: %s", + self.object.doc_type, + ) + results = {} + self.object.lint_result = results + self.object.save() + + data = json.loads(results) + if data["result"] == "success": + data[ + "message" + ] = "Template linter returned results with no errors or warnings" + elif not data["result"]: + data[ + "message" + ] = f"Template had an unknown filetype not supported by the linter: {self.object.doc_type}" + else: + data[ + "message" + ] = "Template linter returned results with issues that require attention" + + return JsonResponse(data) + + ################## # View Functions # ################## @@ -611,7 +796,7 @@ def get_position(report_pk): except Exception: messages.error( request, - "A valid report could not be found for this blank finding.", + "A valid report could not be found for this blank finding", extra_tags="alert-danger", ) return HttpResponseRedirect(reverse("reporting:reports")) @@ -633,106 +818,12 @@ def get_position(report_pk): report_link.save() messages.success( request, - "A blank finding has been successfully added to " "report.", + "Added a blank finding to the report", extra_tags="alert-success", ) return HttpResponseRedirect(reverse("reporting:report_detail", args=(report.id,))) -@login_required -def upload_evidence(request, pk): - """ - Create an individual :model:`reporting.Evidence` entry linked to an individual - :model:`reporting.ReportFindingLink`. - - **Template** - - :template:`reporting/evidence_form.html` - """ - finding_instance = get_object_or_404(ReportFindingLink, pk=pk) - cancel_link = reverse( - "reporting:report_detail", kwargs={"pk": finding_instance.report.pk} - ) - if request.method == "POST": - form = EvidenceForm(request.POST, request.FILES) - if form.is_valid(): - new_evidence = form.save() - if os.path.isfile(new_evidence.document.path): - messages.success( - request, - "Evidence uploaded successfully", - extra_tags="alert-success", - ) - return HttpResponseRedirect( - reverse( - "reporting:report_detail", - args=(new_evidence.finding.report.id,), - ) - ) - else: - messages.error( - request, "Evidence file failed to upload", extra_tags="alert-danger" - ) - return HttpResponseRedirect( - reverse( - "reporting:report_detail", - args=(new_evidence.finding.report.id,), - ) - ) - else: - form = EvidenceForm(initial={"finding": pk, "uploaded_by": request.user}) - return render( - request, - "reporting/evidence_form.html", - {"form": form, "cancel_link": cancel_link}, - ) - - -@login_required -def upload_evidence_modal(request, pk): - """ - Create an individual :model:`reporting.Evidence` entry linked to an individual - :model:`reporting.ReportFindingLink` using a TinyMCE URLDialog. - - **Template** - - :template:`reporting/evidence_form_modal.html` - """ - # Get a list of previously used friendly names for this finding - report_queryset = Evidence.objects.filter(finding=pk).values_list( - "friendly_name", flat=True - ) - used_friendly_names = [] - # Convert the queryset into a list to pass to JavaScript later - for name in report_queryset: - used_friendly_names.append(name) - # If request is a POST, validate the form and move to success page - if request.method == "POST": - form = EvidenceForm(request.POST, request.FILES) - if form.is_valid(): - new_evidence = form.save() - if os.path.isfile(new_evidence.document.path): - messages.success( - request, - "Evidence uploaded successfully", - extra_tags="alert-success", - ) - else: - messages.error( - request, "Evidence file failed to upload", extra_tags="alert-danger" - ) - return HttpResponseRedirect( - reverse("reporting:upload_evidence_modal_success") - ) - else: - # This is for the modal pop-up, so set the ``is_modal`` parameter to hide the usual form buttons - form = EvidenceForm( - initial={"finding": pk, "uploaded_by": request.user}, is_modal=True - ) - context = {"form": form, "used_friendly_names": used_friendly_names} - return render(request, "reporting/evidence_form_modal.html", context=context) - - @login_required def upload_evidence_modal_success(request): """ @@ -746,52 +837,16 @@ def upload_evidence_modal_success(request): return render(request, "reporting/evidence_modal_success.html") -@login_required -def view_evidence(request, pk): +def generate_report_name(report_instance): """ - Display an individual :model:`reporting.Evidence`. - - **Template** - - :template:`reporting/evidence_detail.html` + Generate a filename for a report based on the current time and attributes of an + individual :model:`reporting.Report`. """ - evidence_instance = Evidence.objects.get(pk=pk) - file_content = None - if os.path.isfile(evidence_instance.document.path): - if ( - evidence_instance.document.name.endswith(".txt") - or evidence_instance.document.name.endswith(".log") - or evidence_instance.document.name.endswith(".ps1") - or evidence_instance.document.name.endswith(".py") - or evidence_instance.document.name.endswith(".md") - ): - filetype = "text" - file_content = [] - temp = evidence_instance.document.read().splitlines() - for line in temp: - try: - file_content.append(line.decode()) - except Exception: - file_content.append(line) - - elif ( - evidence_instance.document.name.endswith(".jpg") - or evidence_instance.document.name.endswith(".png") - or evidence_instance.document.name.endswith(".jpeg") - ): - filetype = "image" - else: - filetype = "unknown" - else: - filetype = "text" - file_content = [] - file_content.append("FILE NOT FOUND") - context = { - "filetype": filetype, - "evidence": evidence_instance, - "file_content": file_content, - } - return render(request, "reporting/evidence_detail.html", context=context) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + client_name = report_instance.project.client + assessment_type = report_instance.project.project_type + report_name = f"{timestamp}_{client_name}_{assessment_type}" + return report_name @login_required @@ -799,34 +854,80 @@ def generate_docx(request, pk): """ Generate a Word document report for an individual :model:`reporting.Report`. """ - report_instance = Report.objects.get(pk=pk) - # Ask Spenny to make us a report with these findings - output_path = os.path.join(settings.MEDIA_ROOT, report_instance.title) - evidence_path = os.path.join(settings.MEDIA_ROOT) - template_loc = os.path.join(settings.TEMPLATE_LOC, "template.docx") - spenny = reportwriter.Reportwriter( - report_instance, output_path, evidence_path, template_loc - ) try: - docx = spenny.generate_word_docx() + report_instance = Report.objects.get(pk=pk) + output_path = os.path.join(settings.MEDIA_ROOT, report_instance.title) + evidence_path = os.path.join(settings.MEDIA_ROOT) + report_name = generate_report_name(report_instance) + + # Get the template for this report + if report_instance.docx_template: + report_template = report_instance.docx_template + else: + report_config = ReportConfiguration.get_solo() + report_template = report_config.default_docx_template + if not report_template: + raise MissingTemplate + template_loc = report_template.document.path + + # Check template's linting status + template_status = report_template.get_status() + if template_status == "error" or template_status == "failed": + messages.error( + request, + "The selected report template has linting errors and cannot be used to render a Word document", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": report_instance.pk}) + ) + + # Template available and passes linting checks, so proceed with generation + engine = reportwriter.Reportwriter( + report_instance, output_path, evidence_path, template_loc + ) + + docx = engine.generate_word_docx() response = HttpResponse( content_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document" ) - response["Content-Disposition"] = "attachment; filename=report.docx" + response["Content-Disposition"] = f"attachment; filename={report_name}.docx" docx.save(response) return response - except PackageNotFoundError: + except MissingTemplate: messages.error( request, - "The specified Word docx template could not be found: {}".format( - template_loc - ), + "You do not have a Word template selected and have not configured a default template", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) + except Report.DoesNotExist: + messages.error( + request, + "The target report does not exist", + extra_tags="alert-danger", + ) + except ReportTemplate.DoesNotExist: + messages.error( + request, + "You do not have a Word template selected and have not configured a default template", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) + except DocxPackageNotFoundError: + messages.error( + request, + "Your selected Word template could not be found on the server – try uploading it again", extra_tags="alert-danger", ) except FileNotFoundError as error: messages.error( request, - "Halt document generation because an evidence file is missing: {}".format( + "Halted document generation because an evidence file is missing: {}".format( error ), extra_tags="alert-danger", @@ -834,12 +935,12 @@ def generate_docx(request, pk): except Exception as error: messages.error( request, - "Encountered an error generating the document: {}".format(error), + "Encountered an error generating the document: {}".format(error) + .replace('"', "") + .replace("'", "`"), extra_tags="alert-danger", ) - return HttpResponseRedirect( - reverse("reporting:report_detail", kwargs={"pk": report_instance.pk}) - ) + return HttpResponseRedirect(reverse("reporting:report_detail", kwargs={"pk": pk})) @login_required @@ -849,33 +950,37 @@ def generate_xlsx(request, pk): """ try: report_instance = Report.objects.get(pk=pk) - # Ask Spenny to make us a report with these findings output_path = os.path.join(settings.MEDIA_ROOT, report_instance.title) evidence_path = os.path.join(settings.MEDIA_ROOT) - template_loc = None - spenny = reportwriter.Reportwriter( - report_instance, output_path, evidence_path, template_loc + report_name = generate_report_name(report_instance) + + engine = reportwriter.Reportwriter( + report_instance, output_path, evidence_path, template_loc=None ) output = io.BytesIO() workbook = Workbook(output, {"in_memory": True}) - spenny.generate_excel_xlsx(workbook) + engine.generate_excel_xlsx(workbook) output.seek(0) response = HttpResponse( output.read(), content_type="application/application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", ) - response["Content-Disposition"] = "attachment; filename=report.xlsx" + response["Content-Disposition"] = f"attachment; filename={report_name}.xlsx" output.close() return response + except Report.DoesNotExist: + messages.error( + request, + "The target report does not exist", + extra_tags="alert-danger", + ) except Exception as error: messages.error( request, - "Encountered an error generating the document: {}".format(error), + "Encountered an error generating the spreadsheet: {}".format(error), extra_tags="alert-danger", ) - return HttpResponseRedirect( - reverse("reporting:report_detail", kwargs={"pk": report_instance.pk}) - ) + return HttpResponseRedirect(reverse("reporting:report_detail", kwargs={"pk": pk})) @login_required @@ -885,29 +990,76 @@ def generate_pptx(request, pk): """ try: report_instance = Report.objects.get(pk=pk) - # Ask Spenny to make us a report with these findings output_path = os.path.join(settings.MEDIA_ROOT, report_instance.title) evidence_path = os.path.join(settings.MEDIA_ROOT) - template_loc = os.path.join(settings.TEMPLATE_LOC, "template.pptx") - spenny = reportwriter.Reportwriter( + report_name = generate_report_name(report_instance) + + # Get the template for this report + if report_instance.docx_template: + report_template = report_instance.pptx_template + else: + report_config = ReportConfiguration.get_solo() + report_template = report_config.default_pptx_template + if not report_template: + raise MissingTemplate + template_loc = report_template.document.path + + engine = reportwriter.Reportwriter( report_instance, output_path, evidence_path, template_loc ) - pptx = spenny.generate_powerpoint_pptx() + pptx = engine.generate_powerpoint_pptx() response = HttpResponse( content_type="application/application/vnd.openxmlformats-officedocument.presentationml.presentation" ) - response["Content-Disposition"] = "attachment; filename=report.pptx" + response["Content-Disposition"] = f"attachment; filename={report_name}.pptx" pptx.save(response) return response + except MissingTemplate: + messages.error( + request, + "You do not have a PowerPoint template selected and have not configured a default template", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) + except ValueError as exception: + messages.error( + request, + f"Your selected template could not be loaded as a PowerPoint template: {exception}", + extra_tags="alert-danger", + ) + except Report.DoesNotExist: + messages.error( + request, + "The target report does not exist", + extra_tags="alert-danger", + ) + except ReportTemplate.DoesNotExist: + messages.error( + request, + "You do not have a PowerPoint template selected and have not configured a default template", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) + except PptxPackageNotFoundError: + messages.error( + request, + "Your selected PowerPoint template could not be found on the server – try uploading it again", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) except Exception as error: messages.error( request, "Encountered an error generating the document: {}".format(error), extra_tags="alert-danger", ) - return HttpResponseRedirect( - reverse("reporting:report_detail", kwargs={"pk": report_instance.pk}) - ) + return HttpResponseRedirect(reverse("reporting:report_detail", kwargs={"pk": pk})) @login_required @@ -915,16 +1067,14 @@ def generate_json(request, pk): """ Generate a JSON report for an individual :model:`reporting.Report`. """ - report_instance = Report.objects.get(pk=pk) - # Ask Spenny to make us a report with these findings + report_instance = get_object_or_404(Report, pk=pk) output_path = os.path.join(settings.MEDIA_ROOT, report_instance.title) evidence_path = os.path.join(settings.MEDIA_ROOT) - template_loc = None - spenny = reportwriter.Reportwriter( - report_instance, output_path, evidence_path, template_loc + engine = reportwriter.Reportwriter( + report_instance, output_path, evidence_path, template_loc=None ) - json = spenny.generate_json() - return HttpResponse(json, "application/json") + json_report = engine.generate_json() + return HttpResponse(json_report, "application/json") @login_required @@ -934,20 +1084,39 @@ def generate_all(request, pk): """ try: report_instance = Report.objects.get(pk=pk) - docx_template_loc = os.path.join(settings.TEMPLATE_LOC, "template.docx") - pptx_template_loc = os.path.join(settings.TEMPLATE_LOC, "template.pptx") - # Ask Spenny to make us reports with these findings output_path = os.path.join(settings.MEDIA_ROOT, report_instance.title) evidence_path = os.path.join(settings.MEDIA_ROOT) - template_loc = os.path.join(settings.MEDIA_ROOT, "templates", "template.docx") - spenny = reportwriter.Reportwriter( - report_instance, output_path, evidence_path, template_loc + report_name = generate_report_name(report_instance) + + # Get the templates for Word and PowerPoint + if report_instance.docx_template: + docx_template = report_instance.docx_template + else: + report_config = ReportConfiguration.get_solo() + docx_template = report_config.default_docx_template + if not docx_template: + raise MissingTemplate + docx_template = docx_template.document.path + + if report_instance.docx_template: + pptx_template = report_instance.pptx_template + else: + report_config = ReportConfiguration.get_solo() + pptx_template = report_config.default_pptx_template + if not pptx_template: + raise MissingTemplate + pptx_template = pptx_template.document.path + + engine = reportwriter.Reportwriter( + report_instance, output_path, evidence_path, template_loc=None ) - json_doc, word_doc, excel_doc, ppt_doc = spenny.generate_all_reports( - docx_template_loc, pptx_template_loc + json_doc, word_doc, excel_doc, ppt_doc = engine.generate_all_reports( + docx_template, pptx_template ) + # Convert the dict to pretty JSON output for the file pretty_json = json.dumps(json_doc, indent=4) + # Create a zip file in memory and add the reports to it zip_buffer = io.BytesIO() zf = zipfile.ZipFile(zip_buffer, "a") @@ -957,20 +1126,64 @@ def generate_all(request, pk): zf.writestr("report.pptx", ppt_doc.getvalue()) zf.close() zip_buffer.seek(0) + # Return the buffer in the HTTP response response = HttpResponse(content_type="application/x-zip-compressed") - response["Content-Disposition"] = "attachment; filename=reports.zip" + response["Content-Disposition"] = f"attachment; filename={report_name}.zip" response.write(zip_buffer.read()) return response - except Exception: + except MissingTemplate: messages.error( request, - "Failed to generate one or more documents for the archive", + "You do not have a PowerPoint template selected and have not configured a default template", extra_tags="alert-danger", ) - return HttpResponseRedirect( - reverse("reporting:report_detail", kwargs={"pk": report_instance.pk}) - ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) + except ValueError as exception: + messages.error( + request, + f"Your selected template could not be loaded as a PowerPoint template: {exception}", + extra_tags="alert-danger", + ) + except Report.DoesNotExist: + messages.error( + request, + "The target report does not exist", + extra_tags="alert-danger", + ) + except ReportTemplate.DoesNotExist: + messages.error( + request, + "You do not have a PowerPoint template selected and have not configured a default template", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) + except DocxPackageNotFoundError: + messages.error( + request, + "Your selected Word template could not be found on the server – try uploading it again", + extra_tags="alert-danger", + ) + except PptxPackageNotFoundError: + messages.error( + request, + "Your selected PowerPoint template could not be found on the server – try uploading it again", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) + except Exception as error: + messages.error( + request, + "Encountered an error generating the document: {}".format(error), + extra_tags="alert-danger", + ) + return HttpResponseRedirect(reverse("reporting:report_detail", kwargs={"pk": pk})) @login_required @@ -995,50 +1208,89 @@ def archive(request, pk): related :model:`reporting.Evidence` and related files, and compress the files into a single Zip file for arhciving. """ - report_instance = Report.objects.select_related("project", "project__client").get( - pk=pk - ) - archive_loc = os.path.join(settings.MEDIA_ROOT, "archives") - evidence_loc = os.path.join(settings.MEDIA_ROOT, "evidence", str(pk)) - docx_template_loc = os.path.join(settings.MEDIA_ROOT, "templates", "template.docx") - pptx_template_loc = os.path.join(settings.MEDIA_ROOT, "templates", "template.pptx") - # Ask Spenny to make us reports with these findings - output_path = os.path.join(settings.MEDIA_ROOT, report_instance.title) - evidence_path = os.path.join(settings.MEDIA_ROOT) - template_loc = os.path.join(settings.MEDIA_ROOT, "templates", "template.docx") - spenny = reportwriter.Reportwriter( - report_instance, output_path, evidence_path, template_loc - ) - json_doc, word_doc, excel_doc, ppt_doc = spenny.generate_all_reports( - docx_template_loc, pptx_template_loc - ) - # Create a zip file in memory and add the reports to it - zip_buffer = io.BytesIO() - zf = zipfile.ZipFile(zip_buffer, "a") - zf.writestr("report.json", json_doc) - zf.writestr("report.docx", word_doc.getvalue()) - zf.writestr("report.xlsx", excel_doc.getvalue()) - zf.writestr("report.pptx", ppt_doc.getvalue()) - zip_directory(evidence_loc, zf) - zf.close() - zip_buffer.seek(0) - with open( - os.path.join(archive_loc, report_instance.title + ".zip"), "wb" - ) as archive_file: - archive_file.write(zip_buffer.read()) - new_archive = Archive( - client=report_instance.project.client, - report_archive=File( - open(os.path.join(archive_loc, report_instance.title + ".zip"), "rb") - ), + try: + report_instance = Report.objects.select_related( + "project", "project__client" + ).get(pk=pk) + output_path = os.path.join(settings.MEDIA_ROOT, report_instance.title) + evidence_path = os.path.join(settings.MEDIA_ROOT) + archive_loc = os.path.join(settings.MEDIA_ROOT, "archives") + evidence_loc = os.path.join(settings.MEDIA_ROOT, "evidence", str(pk)) + report_name = generate_report_name(report_instance) + + # Get the templates for Word and PowerPoint + if report_instance.docx_template: + docx_template = report_instance.docx_template + else: + docx_template = ReportTemplate.objects.get( + default=True, doc_type__doc_type="docx" + ) + if report_instance.pptx_template: + pptx_template = report_instance.pptx_template + else: + pptx_template = ReportTemplate.objects.get( + default=True, doc_type__doc_type="pptx" + ) + + engine = reportwriter.Reportwriter( + report_instance, output_path, evidence_path, template_loc=None ) - new_archive.save() - messages.success( - request, - "{} has been archived successfully.".format(report_instance.title), - extra_tags="alert-success", - ) - return HttpResponseRedirect(reverse("reporting:archived_reports")) + json_doc, word_doc, excel_doc, ppt_doc = engine.generate_all_reports( + docx_template, pptx_template + ) + + # Convert the dict to pretty JSON output for the file + pretty_json = json.dumps(json_doc, indent=4) + + # Create a zip file in memory and add the reports to it + zip_buffer = io.BytesIO() + zf = zipfile.ZipFile(zip_buffer, "a") + zf.writestr("report.json", pretty_json) + zf.writestr("report.docx", word_doc.getvalue()) + zf.writestr("report.xlsx", excel_doc.getvalue()) + zf.writestr("report.pptx", ppt_doc.getvalue()) + zip_directory(evidence_loc, zf) + zf.close() + zip_buffer.seek(0) + with open( + os.path.join(archive_loc, report_name + ".zip"), "wb" + ) as archive_file: + archive_file.write(zip_buffer.read()) + new_archive = Archive( + client=report_instance.project.client, + report_archive=File( + open(os.path.join(archive_loc, report_name + ".zip"), "rb") + ), + ) + new_archive.save() + messages.success( + request, + "Successfully archived {}".format(report_instance.title), + extra_tags="alert-success", + ) + return HttpResponseRedirect(reverse("reporting:archived_reports")) + except Report.DoesNotExist: + messages.error( + request, + "The target report does not exist", + extra_tags="alert-danger", + ) + except ReportTemplate.DoesNotExist: + messages.error( + request, + "You do not have templates selected for Word and PowerPoint and have not selected default templates", + extra_tags="alert-danger", + ) + return HttpResponseRedirect( + reverse("reporting:report_detail", kwargs={"pk": pk}) + ) + except Exception: + messages.error( + request, + "Failed to generate one or more documents for the archive", + extra_tags="alert-danger", + ) + return HttpResponseRedirect(reverse("reporting:report_detail", kwargs={"pk": pk})) @login_required @@ -1121,11 +1373,12 @@ def convert_finding(request, pk): return render(request, "reporting/finding_form.html", {"form": form}) +@login_required def export_findings_to_csv(request): """ Export all :model:`reporting.Finding` to a csv file for download. """ - timestamp = datetime.now().isoformat() + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") fiinding_resource = FindingResource() dataset = fiinding_resource.export() response = HttpResponse(dataset.csv, content_type="text/csv") @@ -1273,6 +1526,20 @@ class ReportDetailView(LoginRequiredMixin, DetailView): model = Report + def get_context_data(self, **kwargs): + ctx = super(ReportDetailView, self).get_context_data(**kwargs) + form = SelectReportTemplateForm(instance=self.object) + form.fields["docx_template"].queryset = ReportTemplate.objects.filter( + Q(doc_type__doc_type="docx") & Q(client=self.object.project.client) + | Q(client__isnull=True) + ) + form.fields["pptx_template"].queryset = ReportTemplate.objects.filter( + Q(doc_type__doc_type="pptx") & Q(client=self.object.project.client) + | Q(client__isnull=True) + ) + ctx["form"] = form + return ctx + class ReportCreate(LoginRequiredMixin, CreateView): """ @@ -1281,7 +1548,7 @@ class ReportCreate(LoginRequiredMixin, CreateView): **Context** ``project`` - Instance of :model:`reporting.Project` associated with this report + Instance of :model:`rolodex.Project` associated with this report ``cancel_link`` Link for the form's Cancel button to return to report list or details page @@ -1300,9 +1567,15 @@ def setup(self, request, *args, **kwargs): # Determine if ``pk`` is in the kwargs if "pk" in self.kwargs: pk = self.kwargs.get("pk") - # Try to get the project from :model:`reporting.Project` + # Try to get the project from :model:`rolodex.Project` if pk: - self.project = get_object_or_404(Project, pk=self.kwargs.get("pk")) + try: + self.project = get_object_or_404(Project, pk=self.kwargs.get("pk")) + except Project.DoesNotExist: + logger.info( + "Received report create request for Project ID %s, but that Project does not exist", + pk, + ) def get_form_kwargs(self): kwargs = super(ReportCreate, self).get_form_kwargs() @@ -1325,14 +1598,12 @@ def get_form(self, form_class=None): if not form.fields["project"].queryset: messages.error( self.request, - "There are no active projects for a new report.", + "There are no active projects for a new report", extra_tags="alert-error", ) return form def form_valid(self, form): - project = get_object_or_404(Project, pk=self.kwargs.get("pk")) - form.instance.project = project form.instance.created_by = self.request.user self.request.session["active_report"] = {} self.request.session["active_report"]["title"] = form.instance.title @@ -1350,7 +1621,7 @@ def get_success_url(self): self.request.session.modified = True messages.success( self.request, - "New report was successfully created and is now your active report.", + "Successfully created new report and set it as your active report", extra_tags="alert-success", ) return reverse("reporting:report_detail", kwargs={"pk": self.object.pk}) @@ -1400,7 +1671,7 @@ def form_valid(self, form): def get_success_url(self): messages.success( - self.request, "Report was updated successfully.", extra_tags="alert-success" + self.request, "Successfully updated the report", extra_tags="alert-success" ) return reverse("reporting:report_detail", kwargs={"pk": self.object.pk}) @@ -1433,7 +1704,7 @@ def get_success_url(self): self.request.session.modified = True messages.warning( self.request, - "Report and associated evidence files were deleted successfully.", + "Successfully deleted the report and associated evidence files", extra_tags="alert-warning", ) return "{}#reports".format( @@ -1451,6 +1722,208 @@ def get_context_data(self, **kwargs): return ctx +class ReportTemplateListView(LoginRequiredMixin, generic.ListView): + """ + Display a list of all :model:`reporting.ReportTemplate`. + + **Template** + + :template:`reporting/report_template_list.html` + """ + + model = ReportTemplate + template_name = "reporting/report_templates_list.html" + + +class ReportTemplateDetailView(LoginRequiredMixin, DetailView): + """ + Display an individual :model:`reporting.ReportTemplate`. + + **Template** + + :template:`reporting/report_template_list.html` + """ + + model = ReportTemplate + template_name = "reporting/report_template_detail.html" + + +class ReportTemplateCreate(LoginRequiredMixin, CreateView): + """ + Create an individual instance of :model:`reporting.ReportTemplate`. + + **Context** + + ``cancel_link`` + Link for the form's Cancel button to return to template list page + + **Template** + + :template:`report_template_form.html` + """ + + model = ReportTemplate + form_class = ReportTemplateForm + template_name = "reporting/report_template_form.html" + + def get_context_data(self, **kwargs): + ctx = super(ReportTemplateCreate, self).get_context_data(**kwargs) + ctx["cancel_link"] = reverse("reporting:templates") + return ctx + + def get_initial(self): + date = datetime.now().strftime("%d %B %Y") + initial_upload = f'

    {date}

    Initial upload

    ' + return {"uploaded_by": self.request.user, "changelog": initial_upload} + + def get_success_url(self): + messages.success( + self.request, + "Template successfully uploaded", + extra_tags="alert-success", + ) + return reverse("reporting:template_detail", kwargs={"pk": self.object.pk}) + + +class ReportTemplateUpdate(LoginRequiredMixin, PermissionRequiredMixin, UpdateView): + """ + Save an individual instance of :model:`reporting.ReportTemplate`. + + **Context** + + ``cancel_link`` + Link for the form's Cancel button to return to template list page + + **Template** + + :template:`report_template_form.html` + """ + + model = ReportTemplate + form_class = ReportTemplateForm + template_name = "reporting/report_template_form.html" + permission_denied_message = "Only an admin can edit this template" + + def has_permission(self): + self.object = self.get_object() + if self.object.protected: + return self.request.user.is_staff + else: + return self.request.user.is_active + + def handle_no_permission(self): + messages.error( + self.request, "That template is protected – only an admin can edit it" + ) + return HttpResponseRedirect( + reverse( + "reporting:template_detail", + args=(self.object.pk,), + ) + ) + + def get_context_data(self, **kwargs): + ctx = super(ReportTemplateUpdate, self).get_context_data(**kwargs) + ctx["cancel_link"] = reverse("reporting:templates") + return ctx + + def get_success_url(self): + messages.success( + self.request, + "Template successfully updated", + extra_tags="alert-success", + ) + return reverse("reporting:template_detail", kwargs={"pk": self.object.pk}) + + +class ReportTemplateDelete(LoginRequiredMixin, DeleteView): + """ + Delete an individual instance of :model:`reporting.ReportTemplate`. + + **Context** + + ``object_type`` + String describing what is to be deleted + ``object_to_be_deleted`` + To-be-deleted instance of :model:`reporting.ReportTemplate` + ``cancel_link`` + Link for the form's Cancel button to return to template's detail page + + **Template** + + :template:`confirm_delete.html` + """ + + model = ReportTemplate + template_name = "confirm_delete.html" + + def get_success_url(self): + messages.success( + self.request, + self.message, + extra_tags="alert-success", + ) + return reverse("reporting:templates") + + def delete(self, request, *args, **kwargs): + self.object = self.get_object() + logger.info( + "Deleted %s %s by request of %s", + self.object.__class__.__name__, + self.object.id, + self.request.user, + ) + self.message = "Successfully deleted the template and associated file" + if os.path.isfile(self.object.document.path): + try: + os.remove(self.object.document.path) + logger.info("Deleted %s", self.object.document.path) + except Exception: + self.message = "Successfully deleted the template, but could not delete the associated file{}" + logger.warning( + "Failed to delete file associated with %s %s: %s", + self.object.__class__.__name__, + self.object.id, + self.object.document.path, + ) + else: + logger.info( + "Tried to delete template file, but path did not exist: %s", + self.object.document.path, + ) + return super(ReportTemplateDelete, self).delete(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + ctx = super(ReportTemplateDelete, self).get_context_data(**kwargs) + queryset = kwargs["object"] + ctx["cancel_link"] = reverse( + "reporting:template_detail", kwargs={"pk": queryset.pk} + ) + ctx["object_type"] = "report template file (and associated file on disk)" + ctx["object_to_be_deleted"] = queryset.filename + return ctx + + +class ReportTemplateDownload(LoginRequiredMixin, SingleObjectMixin, View): + """ + Return the target :model:`reporting.ReportTemplate` template file for download. + """ + + model = ReportTemplate + + def get(self, *args, **kwargs): + self.object = self.get_object() + file_path = os.path.join(settings.MEDIA_ROOT, self.object.document.path) + if os.path.exists(file_path): + return FileResponse( + open(file_path, "rb"), + as_attachment=True, + filename=os.path.basename(file_path), + ) + else: + raise Http404 + + # CBVs related to :model:`reporting.ReportFindingLink` @@ -1494,6 +1967,8 @@ def form_valid(self, form): old_assignee = old_entry.assigned_to # Notify new assignee over WebSockets if "assigned_to" in form.changed_data: + new_users_assignments = {} + old_users_assignments = {} # Only notify if the assignee is not the user who made the change if self.request.user != self.object.assigned_to: # Count the current user's total assignments @@ -1536,7 +2011,7 @@ def form_valid(self, form): "assignments": new_users_assignments, }, ) - if self.request.user != old_assignee: + if self.request.user != old_assignee and old_users_assignments: # Send a message to the unassigned user async_to_sync(channel_layer.group_send)( "notify_{}".format(old_assignee), @@ -1607,7 +2082,7 @@ def get_form(self, form_class=None): def get_success_url(self): messages.success( self.request, - "{} was successfully updated.".format(self.get_object().title), + "Successfully updated {}".format(self.get_object().title), extra_tags="alert-success", ) return reverse("reporting:report_detail", kwargs={"pk": self.object.report.id}) @@ -1627,6 +2102,121 @@ class EvidenceDetailView(LoginRequiredMixin, DetailView): model = Evidence + def get_context_data(self, **kwargs): + ctx = super(EvidenceDetailView, self).get_context_data(**kwargs) + file_content = None + if os.path.isfile(self.object.document.path): + if ( + self.object.document.name.endswith(".txt") + or self.object.document.name.endswith(".log") + or self.object.document.name.endswith(".md") + ): + filetype = "text" + file_content = [] + temp = self.object.document.read().splitlines() + for line in temp: + try: + file_content.append(line.decode()) + except Exception: + file_content.append(line) + + elif ( + self.object.document.name.endswith(".jpg") + or self.object.document.name.endswith(".png") + or self.object.document.name.endswith(".jpeg") + ): + filetype = "image" + else: + filetype = "unknown" + else: + filetype = "text" + file_content = [] + file_content.append("FILE NOT FOUND") + + ctx["filetype"] = filetype + ctx["evidence"] = self.object + ctx["file_content"] = file_content + + return ctx + + +class EvidenceCreate(LoginRequiredMixin, CreateView): + """ + Create an individual :model:`reporting.Evidence` entry linked to an individual + :model:`reporting.ReportFindingLink`. + + **Template** + + :template:`reporting/evidence_form.html` + """ + + model = Evidence + form_class = EvidenceForm + + def get_template_names(self): + if "modal" in self.kwargs: + modal = self.kwargs["modal"] + if modal: + return ["reporting/evidence_form_modal.html"] + else: + return ["reporting/evidence_form.html"] + else: + return ["reporting/evidence_form.html"] + + def get_form_kwargs(self): + kwargs = super(EvidenceCreate, self).get_form_kwargs() + finding_pk = self.kwargs.get("pk") + self.evidence_queryset = Evidence.objects.filter(finding=finding_pk) + kwargs.update({"evidence_queryset": self.evidence_queryset}) + self.finding_instance = get_object_or_404(ReportFindingLink, pk=finding_pk) + if "modal" in self.kwargs: + kwargs.update({"is_modal": True}) + return kwargs + + def get_context_data(self, **kwargs): + ctx = super(EvidenceCreate, self).get_context_data(**kwargs) + ctx["cancel_link"] = reverse( + "reporting:report_detail", kwargs={"pk": self.finding_instance.report.pk} + ) + if "modal" in self.kwargs: + friendly_names = self.evidence_queryset.values_list( + "friendly_name", flat=True + ) + used_friendly_names = [] + # Convert the queryset into a list to pass to JavaScript later + for name in friendly_names: + used_friendly_names.append(name) + ctx["used_friendly_names"] = used_friendly_names + + return ctx + + def form_valid(self, form, **kwargs): + self.object = form.save(commit=False) + self.object.uploaded_by = self.request.user + self.object.finding = self.finding_instance + self.object.save() + if os.path.isfile(self.object.document.path): + messages.success( + self.request, + "Evidence uploaded successfully", + extra_tags="alert-success", + ) + else: + messages.error( + self.request, + "Evidence file failed to upload", + extra_tags="alert-danger", + ) + return HttpResponseRedirect(self.get_success_url()) + + def get_success_url(self): + if "modal" in self.kwargs: + return reverse("reporting:upload_evidence_modal_success") + else: + return reverse( + "reporting:report_detail", args=(self.object.finding.report.pk,) + ) + class EvidenceUpdate(LoginRequiredMixin, UpdateView): """ @@ -1645,17 +2235,24 @@ class EvidenceUpdate(LoginRequiredMixin, UpdateView): model = Evidence form_class = EvidenceForm + def get_form_kwargs(self): + kwargs = super(EvidenceUpdate, self).get_form_kwargs() + evidence_queryset = Evidence.objects.filter(finding=self.object.finding.pk) + kwargs.update({"evidence_queryset": evidence_queryset}) + return kwargs + def get_context_data(self, **kwargs): ctx = super(EvidenceUpdate, self).get_context_data(**kwargs) ctx["cancel_link"] = reverse( - "reporting:evidence_detail", kwargs={"pk": self.object.pk}, + "reporting:evidence_detail", + kwargs={"pk": self.object.pk}, ) return ctx def get_success_url(self): messages.success( self.request, - "{} was successfully updated.".format(self.get_object().friendly_name), + "Successfully updated {}".format(self.get_object().friendly_name), extra_tags="alert-success", ) return reverse( @@ -1686,7 +2283,9 @@ class EvidenceDelete(LoginRequiredMixin, DeleteView): def get_success_url(self): messages.success( - self.request, self.message, extra_tags="alert-success", + self.request, + self.message, + extra_tags="alert-success", ) return reverse( "reporting:report_detail", kwargs={"pk": self.object.finding.report.pk} @@ -1737,7 +2336,7 @@ def get_context_data(self, **kwargs): return ctx -# CBVs related to :model:`reporting.FindingNote` +# CBVs related to :model:`reporting.Finding` class FindingNoteCreate(LoginRequiredMixin, CreateView): @@ -1769,17 +2368,19 @@ def get_context_data(self, **kwargs): def get_success_url(self): messages.success( self.request, - "Note successfully added to this finding.", + "Successfully added your note to this finding", extra_tags="alert-success", ) return "{}#notes".format( reverse("reporting:finding_detail", kwargs={"pk": self.object.finding.id}) ) - def get_initial(self): - finding_instance = get_object_or_404(Finding, pk=self.kwargs.get("pk")) - finding = finding_instance - return {"finding": finding, "operator": self.request.user} + def form_valid(self, form, **kwargs): + self.object = form.save(commit=False) + self.object.operator = self.request.user + self.object.finding_id = self.kwargs.get("pk") + self.object.save() + return super().form_valid(form) class FindingNoteUpdate(LoginRequiredMixin, UpdateView): @@ -1809,7 +2410,7 @@ def get_context_data(self, **kwargs): def get_success_url(self): messages.success( - self.request, "Note successfully updated.", extra_tags="alert-success" + self.request, "Successfully updated the note", extra_tags="alert-success" ) return reverse( "reporting:finding_detail", kwargs={"pk": self.object.finding.pk} @@ -1839,28 +2440,28 @@ class LocalFindingNoteCreate(LoginRequiredMixin, CreateView): def get_context_data(self, **kwargs): ctx = super(LocalFindingNoteCreate, self).get_context_data(**kwargs) - finding_instance = get_object_or_404( + self.finding_instance = get_object_or_404( ReportFindingLink, pk=self.kwargs.get("pk") ) ctx["cancel_link"] = reverse( - "reporting:local_edit", kwargs={"pk": finding_instance.pk} + "reporting:local_edit", kwargs={"pk": self.finding_instance.pk} ) return ctx def get_success_url(self): messages.success( self.request, - "Note successfully added to this finding.", + "Successfully added your note to this finding", extra_tags="alert-success", ) return reverse("reporting:local_edit", kwargs={"pk": self.object.finding.pk}) - def get_initial(self): - finding_instance = get_object_or_404( - ReportFindingLink, pk=self.kwargs.get("pk") - ) - finding = finding_instance - return {"finding": finding, "operator": self.request.user} + def form_valid(self, form, **kwargs): + self.object = form.save(commit=False) + self.object.operator = self.request.user + self.object.finding_id = self.kwargs.get("pk") + self.object.save() + return super().form_valid(form) class LocalFindingNoteUpdate(LoginRequiredMixin, UpdateView): @@ -1891,6 +2492,6 @@ def get_context_data(self, **kwargs): def get_success_url(self): messages.success( - self.request, "Note successfully updated.", extra_tags="alert-success" + self.request, "Successfully updated the note", extra_tags="alert-success" ) return reverse("reporting:local_edit", kwargs={"pk": self.object.finding.pk}) diff --git a/ghostwriter/rolodex/admin.py b/ghostwriter/rolodex/admin.py index 7673de2b9..1b1699311 100644 --- a/ghostwriter/rolodex/admin.py +++ b/ghostwriter/rolodex/admin.py @@ -3,6 +3,7 @@ # Django & Other 3rd Party Libraries from django.contrib import admin +# Ghostwriter Libraries from .models import ( Client, ClientContact, @@ -76,7 +77,10 @@ class ProjectAssignmentAdmin(admin.ModelAdmin): list_display_links = ("operator", "project") fieldsets = ( ("Operator Information", {"fields": ("operator", "role", "project")}), - ("Assignment Dates", {"fields": ("start_date", "end_date")},), + ( + "Assignment Dates", + {"fields": ("start_date", "end_date")}, + ), ("Misc", {"fields": ("note",)}), ) diff --git a/ghostwriter/rolodex/filters.py b/ghostwriter/rolodex/filters.py index 2aeff34c1..e34c96861 100644 --- a/ghostwriter/rolodex/filters.py +++ b/ghostwriter/rolodex/filters.py @@ -8,6 +8,7 @@ from django import forms from django.forms.widgets import TextInput +# Ghostwriter Libraries from .models import Client, Project @@ -18,7 +19,9 @@ class ClientFilter(django_filters.FilterSet): **Fields** ``name`` - Case insensitive search of the name field. + Case insensitive search of the model's ``name`` field + ``codename`` + Case insensitive search of the model's ``codename`` field """ name = django_filters.CharFilter( @@ -30,10 +33,19 @@ class ClientFilter(django_filters.FilterSet): } ), ) + codename = django_filters.CharFilter( + lookup_expr="icontains", + widget=TextInput( + attrs={ + "placeholder": "Enter full or codename...", + "autocomplete": "off", + } + ), + ) class Meta: model = Client - fields = ["name"] + fields = ["name", "codename"] def __init__(self, *args, **kwargs): super(ClientFilter, self).__init__(*args, **kwargs) @@ -47,7 +59,11 @@ def __init__(self, *args, **kwargs): Row( Column( PrependedText("name", ''), - css_class="form-group col-md-6 offset-md-3 mb-0", + css_class="form-group col-md-6 mb-0", + ), + Column( + PrependedText("codename", ''), + css_class="form-group col-md-6 mb-0", ), css_class="form-row", ), @@ -71,15 +87,26 @@ class ProjectFilter(django_filters.FilterSet): **Fields** ``start_date`` - DateFilter for start_date values greater than provided value + Date filter for ``start_date`` values greater than provided value ``end_date`` - DateFilter for end_date values less than provided value + Date filter for ``end_date`` values less than provided value ``start_date_range`` - DateRangeFilter for retrieving entries with matching start_date values + Date range filter for retrieving entries with matching ``start_date`` values ``complete`` - Boolean field for filtering incomplete projects. + Boolean field for filtering incomplete projects based on the ``complete`` field + ``codename`` + Case insensitive search of the model's ``codename`` field """ + codename = django_filters.CharFilter( + lookup_expr="icontains", + widget=TextInput( + attrs={ + "placeholder": "Enter full or codename...", + "autocomplete": "off", + } + ), + ) start_date = django_filters.DateFilter( lookup_expr="gte", field_name="start_date", @@ -97,7 +124,7 @@ class ProjectFilter(django_filters.FilterSet): ), ) start_date_range = django_filters.DateRangeFilter( - field_name="start_date", empty_label="-- Filter by Relative Start Date --" + field_name="start_date", empty_label="-- Relative Start Date --" ) STATUS_CHOICES = ( @@ -124,6 +151,14 @@ def __init__(self, *args, **kwargs): # Layout the form for Bootstrap self.helper.layout = Layout( Div( + Row( + Column( + PrependedText("codename", ''), + css_class="form-group col-md-4 mb-0", + ), + Column("complete", css_class="form-group col-md-4 mb-0"), + Column("start_date_range", css_class="form-group col-md-4 mb-0"), + ), Row( Column( PrependedText( @@ -131,17 +166,13 @@ def __init__(self, *args, **kwargs): ), css_class="form-group col-md-6 mb-0", ), - Column("complete", css_class="form-group col-md-6 mb-0"), - css_class="form-row", - ), - Row( Column( PrependedText( - "end_date", '' + "end_date", + '', ), css_class="form-group col-md-6 mb-0", ), - Column("start_date_range", css_class="form-group col-md-6 mb-0"), css_class="form-row", ), ButtonHolder( diff --git a/ghostwriter/rolodex/forms_client.py b/ghostwriter/rolodex/forms_client.py index b0b8c559e..cd3fb947a 100644 --- a/ghostwriter/rolodex/forms_client.py +++ b/ghostwriter/rolodex/forms_client.py @@ -18,7 +18,7 @@ from django.core.exceptions import ValidationError from django.core.validators import validate_email from django.forms.models import BaseInlineFormSet, inlineformset_factory -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # Ghostwriter Libraries from ghostwriter.modules.custom_layout_object import CustomTab, Formset @@ -86,7 +86,8 @@ def clean(self): form.add_error( "name", ValidationError( - _("Your contact is missing a name"), code="incomplete", + _("Your contact is missing a name"), + code="incomplete", ), ) # Raise an error if a form only has a value for the note @@ -224,7 +225,6 @@ class ClientForm(forms.ModelForm): class Meta: model = Client fields = "__all__" - widgets = {"codename": forms.HiddenInput()} def __init__(self, *args, **kwargs): super(ClientForm, self).__init__(*args, **kwargs) @@ -256,8 +256,8 @@ def __init__(self, *args, **kwargs): Column("short_name", css_class="form-group col-md-6 mb-0"), css_class="form-row", ), - "note", "codename", + "note", link_css_class="client-icon", css_id="client", ), @@ -304,11 +304,7 @@ class ClientNoteForm(forms.ModelForm): class Meta: model = ClientNote - fields = "__all__" - widgets = { - "operator": forms.HiddenInput(), - "client": forms.HiddenInput(), - } + fields = ("note",) def __init__(self, *args, **kwargs): super(ClientNoteForm, self).__init__(*args, **kwargs) @@ -317,7 +313,7 @@ def __init__(self, *args, **kwargs): self.helper.form_class = "newitem" self.helper.form_show_labels = False self.helper.layout = Layout( - Div("note", "operator", "client"), + Div("note"), ButtonHolder( Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), HTML( @@ -333,6 +329,7 @@ def clean_note(self): # Check if note is empty if not note: raise ValidationError( - _("You must provide some content for the note"), code="required", + _("You must provide some content for the note"), + code="required", ) return note diff --git a/ghostwriter/rolodex/forms_project.py b/ghostwriter/rolodex/forms_project.py index da786d95f..ac64262aa 100644 --- a/ghostwriter/rolodex/forms_project.py +++ b/ghostwriter/rolodex/forms_project.py @@ -17,7 +17,7 @@ from django import forms from django.core.exceptions import ValidationError from django.forms.models import BaseInlineFormSet, inlineformset_factory -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # Ghostwriter Libraries from ghostwriter.modules.custom_layout_object import CustomTab, Formset @@ -178,12 +178,9 @@ class Meta: def __init__(self, *args, **kwargs): super(ProjectAssignmentForm, self).__init__(*args, **kwargs) self.fields["operator"].queryset = self.fields["operator"].queryset.order_by( - "name" - ) - self.fields["operator"].label_from_instance = lambda obj: "%s (%s)" % ( - obj.name, - obj.username, + "username", "name" ) + self.fields["operator"].label_from_instance = lambda obj: obj.get_display_name self.fields["start_date"].widget.attrs["placeholder"] = "mm/dd/yyyy" self.fields["start_date"].widget.attrs["autocomplete"] = "off" self.fields["start_date"].widget.input_type = "date" @@ -413,9 +410,16 @@ class ProjectForm(forms.ModelForm): with an individual :model:`rolodex.Client`. """ + update_checkouts = forms.BooleanField( + label="Update Domain & Server Checkouts", + help_text="Update domain and server checkout if the project dates change", + required=False, + initial=True, + ) + class Meta: model = Project - exclude = ("operator", "codename", "complete") + exclude = ("operator", "complete") def __init__(self, *args, **kwargs): super(ProjectForm, self).__init__(*args, **kwargs) @@ -431,11 +435,17 @@ def __init__(self, *args, **kwargs): self.fields["note"].widget.attrs[ "placeholder" ] = "This project is intended to assess ..." + # Hide labels for specific fields because ``form_show_labels`` takes priority + self.fields["start_date"].label = False + self.fields["end_date"].label = False + self.fields["note"].label = False + self.fields["slack_channel"].label = False + self.fields["project_type"].label = False + self.fields["client"].label = False # Design form layout with Crispy FormHelper self.helper = FormHelper() # Turn on tags for this parent form self.helper.form_tag = True - self.helper.form_show_labels = False self.helper.form_class = "form-inline justify-content-center" self.helper.form_method = "post" self.helper.form_class = "newitem" @@ -449,6 +459,7 @@ def __init__(self, *args, **kwargs): """ ), "client", + "codename", Row( Column("start_date", css_class="form-group col-md-6 mb-0"), Column("end_date", css_class="form-group col-md-6 mb-0"), @@ -459,6 +470,7 @@ def __init__(self, *args, **kwargs): Column("slack_channel", css_class="form-group col-md-6 mb-0"), css_class="form-row", ), + "update_checkouts", "note", link_css_class="project-icon", css_id="project", @@ -535,7 +547,7 @@ def clean_slack_channel(self): if not slack_channel.startswith("#"): slack_channel = "#" + slack_channel raise ValidationError( - _("Slack channels should start with '#' – check this channel name"), + _("Slack channels should start with # – check this channel name"), code="invalid_channel", ) return slack_channel @@ -549,11 +561,7 @@ class ProjectNoteForm(forms.ModelForm): class Meta: model = ProjectNote - fields = "__all__" - widgets = { - "operator": forms.HiddenInput(), - "project": forms.HiddenInput(), - } + fields = ("note",) def __init__(self, *args, **kwargs): super(ProjectNoteForm, self).__init__(*args, **kwargs) @@ -562,9 +570,7 @@ def __init__(self, *args, **kwargs): self.helper.form_class = "newitem" self.helper.form_show_labels = False self.helper.layout = Layout( - "note", - "operator", - "project", + Div("note"), ButtonHolder( Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), HTML( @@ -580,6 +586,7 @@ def clean_note(self): # Check if note is empty if not note: raise ValidationError( - _("You must provide some content for the note"), code="required", + _("You must provide some content for the note"), + code="required", ) return note diff --git a/ghostwriter/rolodex/migrations/0007_auto_20201027_1914.py b/ghostwriter/rolodex/migrations/0007_auto_20201027_1914.py new file mode 100644 index 000000000..b59004fa8 --- /dev/null +++ b/ghostwriter/rolodex/migrations/0007_auto_20201027_1914.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.10 on 2020-10-27 19:14 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('rolodex', '0006_auto_20200825_1947'), + ] + + operations = [ + migrations.AlterField( + model_name='client', + name='codename', + field=models.CharField(blank=True, help_text='Give the client a codename (might be a ticket number, CMS reference, or something else)', max_length=255, null=True, verbose_name='Client Codename'), + ), + migrations.AlterField( + model_name='project', + name='codename', + field=models.CharField(blank=True, help_text='Give the project a codename (might be a ticket number, PMO reference, or something else)', max_length=255, null=True, verbose_name='Project Codename'), + ), + ] diff --git a/ghostwriter/rolodex/models.py b/ghostwriter/rolodex/models.py index ab841fffd..72db929e9 100644 --- a/ghostwriter/rolodex/models.py +++ b/ghostwriter/rolodex/models.py @@ -32,7 +32,7 @@ class Client(models.Model): max_length=255, null=True, blank=True, - help_text="A codename for the client that might be used to discuss the client in public", + help_text="Give the client a codename (might be a ticket number, CMS reference, or something else)", ) note = models.TextField( "Client Note", @@ -140,7 +140,7 @@ class Project(models.Model): max_length=255, null=True, blank=True, - help_text="A codename for the client that might be used to discuss the client in public", + help_text="Give the project a codename (might be a ticket number, PMO reference, or something else)", ) start_date = models.DateField( "Start Date", max_length=12, help_text="Enter the start date of this project" @@ -319,8 +319,11 @@ class ProjectObjective(models.Model): def get_status(): """Get the default status for the status field.""" - active_status = ObjectiveStatus.objects.get(objective_status="Active") - return active_status.id + try: + active_status = ObjectiveStatus.objects.get(objective_status="Active") + return active_status.id + except ObjectiveStatus.DoesNotExist: + return 1 objective = models.TextField( "Objective", null=True, blank=True, help_text="Provide a concise objective" diff --git a/ghostwriter/rolodex/signals.py b/ghostwriter/rolodex/signals.py index 65fd32541..f5ff826bb 100644 --- a/ghostwriter/rolodex/signals.py +++ b/ghostwriter/rolodex/signals.py @@ -19,7 +19,7 @@ def update_project(sender, instance, **kwargs): """ Updates dates for :model:`shepherd.History`, :model:`shepherd.ServerHistory`, and - :model:`rolodex.ProjectAssignments whenever :model:`rolodex.Project` is updated. + :model:`rolodex.ProjectAssignments` whenever :model:`rolodex.Project` is updated. """ domain_checkouts = History.objects.filter(project=instance) server_checkouts = ServerHistory.objects.filter(project=instance) diff --git a/ghostwriter/rolodex/tasks.py b/ghostwriter/rolodex/tasks.py index fe035f108..2e1d42d11 100644 --- a/ghostwriter/rolodex/tasks.py +++ b/ghostwriter/rolodex/tasks.py @@ -8,7 +8,9 @@ # Django & Other 3rd Party Libraries import requests -from django.conf import settings + +# Ghostwriter Libraries +from ghostwriter.commandcenter.models import SlackConfiguration from .models import Project @@ -18,51 +20,40 @@ def send_slack_msg(message, slack_channel=None): """ - Accepts message text and sends it to Slack. This requires Slack settings and - a webhook be configured in the application's settings. + Send a basic Slack message using the global Slack configuration. **Parameters** ``message`` A string to be sent as the Slack message ``slack_channel`` - Defaults to using the global setting. Can be set to any Slack channel name. + Defaults to using the global setting. Can be set to any Slack channel name """ - try: - enable_slack = settings.SLACK_CONFIG["enable_slack"] - except KeyError: - enable_slack = False - if enable_slack: - try: - slack_emoji = settings.SLACK_CONFIG["slack_emoji"] - slack_username = settings.SLACK_CONFIG["slack_username"] - slack_webhook_url = settings.SLACK_CONFIG["slack_webhook_url"] - slack_alert_target = settings.SLACK_CONFIG["slack_alert_target"] - if not slack_channel: - slack_channel = settings.SLACK_CONFIG["slack_channel"] - slack_capable = True - except KeyError: - slack_capable = False + slack_config = SlackConfiguration.get_solo() - if slack_capable: - message = slack_alert_target + " " + message - slack_data = { - "username": slack_username, - "icon_emoji": slack_emoji, - "channel": slack_channel, - "text": message, - } - response = requests.post( - slack_webhook_url, - data=json.dumps(slack_data), - headers={"Content-Type": "application/json"}, + if slack_config.enable: + message = slack_config.slack_alert_target + " " + message + slack_data = { + "username": slack_config.slack_username, + "icon_emoji": slack_config.slack_emoji, + "channel": slack_config.slack_channel, + "text": message, + } + response = requests.post( + slack_config.webhook_url, + data=json.dumps(slack_data), + headers={"Content-Type": "application/json"}, + ) + if response.status_code != 200: + logger.warning( + "Request to Slack returned an error %s, the response was: %s", + response.status_code, + response.text, ) - if response.status_code != 200: - logger.warning( - "[!] Request to Slack returned an error %s, the response was: %s", - response.status_code, - response.text, - ) + else: + logger.warning( + "Received request to send Slack message, but Slack notifications are disabled in settings" + ) def check_project_freshness(): diff --git a/ghostwriter/rolodex/templates/rolodex/client_list.html b/ghostwriter/rolodex/templates/rolodex/client_list.html index c8009de28..5c348b4d9 100644 --- a/ghostwriter/rolodex/templates/rolodex/client_list.html +++ b/ghostwriter/rolodex/templates/rolodex/client_list.html @@ -24,19 +24,19 @@ {% if filter.qs|length == 0 %}

    There are no active clients yet or your search returned no results.

    {% else %} - +
    - - - + + + {% for client in filter.qs %} - - - + + + {% endfor %}
    Client
    Codename
    		DescriptionClient
    Codename
    		Projects to Date
    {{ client.name }}{{ client.codename }}{{ client.note|bleach }}{{ client.name }}{{ client.codename }}{{ client.project_set.all.count }}
    diff --git a/ghostwriter/rolodex/templates/rolodex/project_detail.html b/ghostwriter/rolodex/templates/rolodex/project_detail.html index b73c0ce4f..a4e2da57e 100644 --- a/ghostwriter/rolodex/templates/rolodex/project_detail.html +++ b/ghostwriter/rolodex/templates/rolodex/project_detail.html @@ -77,31 +77,42 @@

    Project Description

    Assign an Operator

    {% if project.projectassignment_set.all %} - +
    - - - - - - + + + + {% comment %} + {% endcomment %} + + {% for operator in project.projectassignment_set.all %} - - - {% if operator.start_date|date:"d M Y" %} - - {% else %} - - {% endif %} - {% if operator.end_date|date:"d M Y" %} - - {% else %} - - {% endif %} - - + + + + {% comment %} + {% endcomment %} +
    OperatorRoleStart DateEnd DateNoteOptionsOperatorRoleDatesStart DateEnd DateNoteOptions
    {{ operator.operator.name }}{{ operator.role }}{{ operator.start_date|date:"d M Y" }}{{ operator.end_date|date:"d M Y" }}{{ operator.note|bleach }} + {% if operator.operator.name %} + {{ operator.operator.name }} + {% else %} + Missing Full Name ({{ operator.operator.username }}) + {% endif %} + {{ operator.role }}{{ operator.start_date|date:"d M Y" }} –
    {{ operator.end_date|date:"d M Y" }}    		 + {% if operator.start_date|date:"d M Y" %} + {{ operator.start_date|date:"d M Y" }} + {% else %} + -- + {% endif %} + + {% if operator.end_date|date:"d M Y" %} + {{ operator.end_date|date:"d M Y" }} + {% else %} + -- + {% endif %} + {{ operator.note|bleach }}
    @@ -121,16 +132,16 @@

    Project Description

    Add an Objective

    {% if project.projectobjective_set.all %} - +
    - - - - + + + + {% for objective in project.projectobjective_set.all %} - - - - + +
    StatusObjectiveDeadlineOptionsStatusObjectiveDeadlineOptions
    + Project Description {{ objective.status }} {{ objective.objective|bleach }}{{ objective.deadline }} + {{ objective.objective|bleach }}{{ objective.deadline }}
    @@ -164,6 +175,43 @@

    Project Description

    No objectives have been set for this project yet.

    {% endif %}
    +
    +

    + Start New Operation Log + OR + Import Oplog +

    + {% if project.oplog_set.all %} + + + + + + + + {% for log in project.oplog_set.all %} + + + + + + + {% endfor %} +
    IDNameLast ActivityExport CSV
    {{ log.id }}{{ log.name }} + {% if log.entries.all %} + {% for entry in log.entries.all %} + {% if forloop.last %} + {{ entry.start_date }} + {% endif %} + {% endfor %} + {% else %} + No entries yet + {% endif %} + Export
    + {% else %} +

    An operation logs has not been started for this project yet.

    + {% endif %} +
    {% if project.complete %}

    The project is marked as complete so no additional reports may be created.

    @@ -172,31 +220,31 @@

    Project Description

    {% endif %} {% if project.report_set.all %} - +
    - - - - - - + + + + + + {% for report in project.report_set.all %} - - - + + + {% if report.complete %} {% if report.delivered %} - + {% else %} - + {% endif %} {% else %} - + {% endif %} - - +
    CreationTitleLast UpdateStatusCreated byOptionsCreationTitleLast UpdateStatusCreated byOptions
    {{ report.creation }}{{ report.title }}{{ report.last_update|date:"d M Y" }}{{ report.creation }}{{ report.title }}{{ report.last_update|date:"d M Y" }}DeliveredDeliveredComplete, Awaiting DeliveryComplete, Awaiting DeliveryIn ProgressIn Progress{{ report.created_by }} + {{ report.created_by }}
    @@ -214,40 +262,40 @@

    Project Description

    {% for report in project.report_set.all %} {% if report.reportfindinglink_set.all %}

    {{ report.title }}

    - +
    - - - - + + + + {% for finding in report.reportfindinglink_set.all %} {% if finding.severity.severity == "Critical" %} - + {% elif finding.severity.severity == "High" %} - + {% elif finding.severity.severity == "Medium" %} - + {% elif finding.severity.severity == "Low" %} - + {% else %} - + {% endif %} - + {% if finding.assigned_to == request.user %} {% if finding.complete %} - + {% else %} - + {% endif %} {% else %} - + {% endif %} {% if finding.complete %} - + {% else %} - + {% endif %} {% endfor %} @@ -261,7 +309,8 @@

    {{ report.title }}

    {% endif %} -
    +
    +

    Checked-out domains and servers will appear here. Add cloud servers here as they are created for the project. Associate your domains and subdomains with servers by creating connections.

    Domains

    @@ -279,28 +328,27 @@

    Domains

    {% if project.history_set.all %} -
    SeverityFindingOwnerStatusSeverityFindingOwnerStatus
    {{ finding.severity }}{{ finding.severity }}{{ finding.severity }}{{ finding.severity }}{{ finding.severity }}{{ finding.severity }}{{ finding.severity }}{{ finding.severity }}{{ finding.severity }}{{ finding.severity }}{{ finding.title }}{{ finding.title }}YouYouYouYou{{ finding.assigned_to }}{{ finding.assigned_to }}Ready for ReviewReady for ReviewNeeds EditingNeeds Editing
    +
    - - - - - + + + + + {% for domain in project.history_set.all %} - - - - - + + +
    Domain NamePurposeDate RangeNoteOptionsDomain NamePurposeDate RangeNoteOptions
    - - - {{ domain.domain.name }} - + + + + {{ domain.domain.name }} + + {{ domain.activity_type }}{{ domain.start_date|date:"d M Y" }} – {{ domain.end_date|date:"d M Y" }}{{ domain.note|bleach }} + {{ domain.activity_type }}{{ domain.start_date|date:"d M Y" }} –
    {{ domain.end_date|date:"d M Y" }}    		{{ domain.note|bleach }}
    @@ -317,22 +365,13 @@

    Domains

    {% endif %} -
    -

    - Servers - -

    - -
    -

    Checked-out servers will appear here. Add transient servers (i.e. virtual private servers) here as they are spun up for the project so they can be easily tracked and associated with domains and activities. -

    -
    +

    Servers

    {% csrf_token %} -
    +
    @@ -345,10 +384,10 @@

    Add a Cloud Server

    {% if project.serverhistory_set.all or project.transientserver_set.all %} - +
    - - - - - - - - + + + + + + + {% for server in project.serverhistory_set.all %} {% get_primary_address server.server as primary_address%} - - - - - - - - + + + + + - - - - - - - - + + + + + +
    - Primary IP Address + + Primary IP
    @@ -357,58 +396,41 @@

    		All IP AddressesNamePurposeRoleProviderNoteOptionsOther AddressesNamePurposeRoleProviderNoteOptions
    - - {{ primary_address }} + + + + {{ primary_address }} + + - - {{ server.server.ip_address }} + {% if server.server.auxserveraddress_set %} -
    {% for addy in server.server.auxserveraddress_set.all %} - {% if forloop.last %} - - {{ addy }} - {% elif forloop.first %} - - {{ addy }} -
    - {% else %} - - {{ addy }} -
    - {% endif %} + {% endfor %} + {% else %} + -- {% endif %}     		{{ server.server.name }}{{ server.activity_type }}{{ server.server_role }}{{ server.server.server_provider }}{{ server.note|bleach }} + {{ server.server.name }}{{ server.activity_type }}{{ server.server_role }}{{ server.server.server_provider }}{{ server.note|bleach }}
    @@ -421,19 +443,18 @@

    {% endfor %} {% for server in project.transientserver_set.all %}

    - - {{ server.ip_address }} + + {{ server.name }}{{ server.activity_type }}{{ server.server_role }}{{ server.server_provider }}{{ server.note|bleach }} + -- {{ server.name }}{{ server.activity_type }}{{ server.server_role }}{{ server.server_provider }}{{ server.note|bleach }}
    @@ -450,15 +471,7 @@

    {% endif %} -
    -

    - Domains and Servers - -

    -
    -

    Add an association to track which of this project's currently checked-out domains is being used with which project server.

    -
    -
    +

    Domains and Servers

    {% if project.transientserver_set.all or project.serverhistory_set.all and project.history_set.all %} @@ -466,30 +479,30 @@

    Associate a Domain + Server

    {% if project.domainserverconnection_set.all %} - +
    - - - - + + + + {% for connection in project.domainserverconnection_set.all %} - + {% if connection.static_server %} - - + + {% else %} - - + + {% endif %} {% if connection.endpoint %} - + {% else %} - + {% endif %} - - + {% endif %}
    DomainServer IPServer name - CDN EndpointOptionsDomainServer IPServer name + CDN EndpointOptions
    {{ connection.subdomain }}.{{ connection.domain.domain.name }}{{ connection.subdomain }}.{{ connection.domain.domain.name }}{{ connection.static_server.server.ip_address }}{{ connection.static_server.server.name }}{{ connection.static_server.server.ip_address }}{{ connection.static_server.server.name }}{{ connection.transient_server.ip_address }}{{ connection.transient_server.name }}{{ connection.transient_server.ip_address }}{{ connection.transient_server.name }}{{ connection.endpoint }}{{ connection.endpoint }}No EndpointNo Endpoint +
    @@ -546,6 +559,13 @@

    + + + {% comment %} Include the reusable delete confirmation modal and related scripts {% endcomment %} {% include "confirm_delete_modal.html" %} {% endblock %} diff --git a/ghostwriter/rolodex/templates/rolodex/project_list.html b/ghostwriter/rolodex/templates/rolodex/project_list.html index a08a424f9..3df7e3c65 100644 --- a/ghostwriter/rolodex/templates/rolodex/project_list.html +++ b/ghostwriter/rolodex/templates/rolodex/project_list.html @@ -28,23 +28,34 @@ {% if filter.qs|length == 0 %}

    There are no projects to display.

    {% else %} - +
    - - - - - + + + + + {% for project in filter.qs %} - - - - - + + + + + {% endfor %}
    ClientCodenameProject TypeExecution WindowDescriptionClientCodenameProject TypeExecution WindowTeam
    {{ project.client }}{{ project.codename }}{{ project.project_type }}{{ project.start_date|date:"d M Y" }} - {{ project.end_date|date:"d M Y" }}{{ project.note|bleach }}{{ project.client }}{{ project.codename }}{{ project.project_type }}{{ project.start_date|date:"d M Y" }} -
    {{ project.end_date|date:"d M Y" }}    		 + {% for operator in project.projectassignment_set.all %} + {% if operator.operator.name %} + {{ operator.operator.name }} + {% else %} + Missing Full Name ({{ operator.operator.username }}) + {% endif %} + {% if not forloop.last %} +
    + {% endif %} + {% endfor %} +
    diff --git a/ghostwriter/rolodex/templates/snippets/project_nav_tabs.html b/ghostwriter/rolodex/templates/snippets/project_nav_tabs.html index b6a852ce8..130b2beee 100644 --- a/ghostwriter/rolodex/templates/snippets/project_nav_tabs.html +++ b/ghostwriter/rolodex/templates/snippets/project_nav_tabs.html @@ -8,6 +8,11 @@ Objectives {{ project.projectobjective_set.all.count }} +
  • + + Logs + +
  • Reports & Findings {{ project.count_findings }} diff --git a/ghostwriter/rolodex/urls.py b/ghostwriter/rolodex/urls.py index 5d27c8b0b..23acbb67d 100644 --- a/ghostwriter/rolodex/urls.py +++ b/ghostwriter/rolodex/urls.py @@ -3,6 +3,7 @@ # Django & Other 3rd Party Libraries from django.urls import path +# Ghostwriter Libraries from . import views app_name = "rolodex" diff --git a/ghostwriter/rolodex/views.py b/ghostwriter/rolodex/views.py index 0e13409be..b96cdd70a 100644 --- a/ghostwriter/rolodex/views.py +++ b/ghostwriter/rolodex/views.py @@ -13,7 +13,7 @@ from django.shortcuts import get_object_or_404, render from django.template.loader import render_to_string from django.urls import reverse, reverse_lazy -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.generic.detail import DetailView, SingleObjectMixin from django.views.generic.edit import CreateView, DeleteView, UpdateView, View @@ -60,7 +60,8 @@ def update_project_badges(request, pk): """ project_instance = get_object_or_404(Project, pk=pk) html = render_to_string( - "snippets/project_nav_tabs.html", {"project": project_instance}, + "snippets/project_nav_tabs.html", + {"project": project_instance}, ) return HttpResponse(html) @@ -77,7 +78,8 @@ def update_client_badges(request, pk): """ client_instance = get_object_or_404(Client, pk=pk) html = render_to_string( - "snippets/client_nav_tabs.html", {"client": client_instance}, + "snippets/client_nav_tabs.html", + {"client": client_instance}, ) return HttpResponse(html) @@ -403,7 +405,7 @@ def client_list(request): """ # Check if a search parameter is in the request try: - search_term = request.GET.get("client_search") + search_term = request.GET.get("client_search").strip() except Exception: search_term = "" if search_term: @@ -520,7 +522,9 @@ class ClientCreate(LoginRequiredMixin, CreateView): def get_success_url(self): messages.success( - self.request, "Client successfully saved.", extra_tags="alert-success", + self.request, + "Client successfully saved.", + extra_tags="alert-success", ) return reverse("rolodex:client_detail", kwargs={"pk": self.object.pk}) @@ -534,16 +538,6 @@ def get_context_data(self, **kwargs): return ctx def form_valid(self, form): - # Generate and assign a unique codename to the client - codename_verified = False - while not codename_verified: - new_codename = codenames.codename(uppercase=True) - try: - Client.objects.filter(codename__iequal=new_codename) - except Exception: - codename_verified = True - form.instance.codename = new_codename - # Get form context data – used for validation of inline forms ctx = self.get_context_data() contacts = ctx["contacts"] @@ -572,6 +566,20 @@ def form_valid(self, form): logger.error(message) return super(ClientCreate, self).form_invalid(form) + def get_initial(self): + # Generate and assign a unique codename to the project + codename_verified = False + codename = "" + while not codename_verified: + codename = codenames.codename(uppercase=True) + try: + Project.objects.filter(codename__iequal=codename) + except Exception: + codename_verified = True + return { + "codename": codename, + } + class ClientUpdate(LoginRequiredMixin, UpdateView): """ @@ -595,7 +603,9 @@ class ClientUpdate(LoginRequiredMixin, UpdateView): def get_success_url(self): messages.success( - self.request, "Client successfully saved.", extra_tags="alert-success", + self.request, + "Client successfully saved.", + extra_tags="alert-success", ) return reverse("rolodex:client_detail", kwargs={"pk": self.object.pk}) @@ -613,16 +623,6 @@ def get_context_data(self, **kwargs): return ctx def form_valid(self, form): - # Generate and assign a unique codename to the client - codename_verified = False - while not codename_verified: - new_codename = codenames.codename(uppercase=True) - try: - Client.objects.filter(codename__iequal=new_codename) - except Exception: - codename_verified = True - form.instance.codename = new_codename - # Get form context data – used for validation of inline forms ctx = self.get_context_data() contacts = ctx["contacts"] @@ -729,6 +729,13 @@ def get_context_data(self, **kwargs): ) return ctx + def form_valid(self, form, **kwargs): + self.object = form.save(commit=False) + self.object.operator = self.request.user + self.object.client_id = self.kwargs.get("pk") + self.object.save() + return super().form_valid(form) + class ClientNoteUpdate(LoginRequiredMixin, UpdateView): """ @@ -820,14 +827,21 @@ def setup(self, request, *args, **kwargs): def get_success_url(self): messages.success( - self.request, "Project successfully saved.", extra_tags="alert-success", + self.request, + "Project successfully saved.", + extra_tags="alert-success", ) return reverse("rolodex:project_detail", kwargs={"pk": self.object.pk}) def get_context_data(self, **kwargs): ctx = super(ProjectCreate, self).get_context_data(**kwargs) ctx["client"] = self.client - ctx["cancel_link"] = reverse("rolodex:projects") + if self.client: + ctx["cancel_link"] = reverse( + "rolodex:client_detail", kwargs={"pk": self.client.pk} + ) + else: + ctx["cancel_link"] = reverse("rolodex:projects") if self.request.POST: ctx["objectives"] = ProjectObjectiveFormSet(self.request.POST, prefix="obj") ctx["assignments"] = ProjectAssignmentFormSet( @@ -839,16 +853,6 @@ def get_context_data(self, **kwargs): return ctx def form_valid(self, form): - # Generate and assign a unique codename to the project - codename_verified = False - while not codename_verified: - new_codename = codenames.codename(uppercase=True) - try: - Project.objects.filter(codename__iequal=new_codename) - except Exception: - codename_verified = True - form.instance.codename = new_codename - # Get form context data – used for validation of inline forms ctx = self.get_context_data() objectives = ctx["objectives"] @@ -876,7 +880,7 @@ def form_valid(self, form): else: # Raise an error to rollback transactions raise forms.ValidationError(_("Invalid form data")) - # Otherwise return `form_invalid` and display errors + # Otherwise return ``form_invalid`` and display errors except Exception as exception: template = "An exception of type {0} occurred. Arguments:\n{1!r}" message = template.format(type(exception).__name__, exception.args) @@ -884,8 +888,18 @@ def form_valid(self, form): return super(ProjectCreate, self).form_invalid(form) def get_initial(self): + # Generate and assign a unique codename to the project + codename_verified = False + codename = "" + while not codename_verified: + codename = codenames.codename(uppercase=True) + try: + Project.objects.filter(codename__iequal=codename) + except Exception: + codename_verified = True return { "client": self.client, + "codename": codename, } @@ -941,22 +955,79 @@ def get_success_url(self): ) return reverse("rolodex:project_detail", kwargs={"pk": self.object.pk}) - def get_initial(self): - project_instance = get_object_or_404(Project, pk=self.kwargs.get("pk")) - return { - "codename": project_instance.codename, - } - def form_valid(self, form): # Get form context data – used for validation of inline forms ctx = self.get_context_data() objectives = ctx["objectives"] assignments = ctx["assignments"] + from ghostwriter.shepherd.models import History, ServerHistory + # Now validate inline formsets - # Validation is largely handled by the custom base formset, `BaseProjectInlineFormSet` + # Validation is largely handled by the custom base formset, ``BaseProjectInlineFormSet`` try: with transaction.atomic(): + # Update infrastructure if project's dates changed + update = form.cleaned_data["update_checkouts"] + if update: + if ( + "end_date" in form.changed_data + or "start_date" in form.changed_data + ): + logger.info( + "Date changed on Project %s, so updating domain and server checkouts", + self.object.pk, + ) + # Get the project's current dates + old_project_data = Project.objects.get(pk=self.object.pk) + old_start_date = old_project_data.start_date + old_end_date = old_project_data.end_date + # Form dates + new_start_date = form.cleaned_data["start_date"] + new_end_date = form.cleaned_data["end_date"] + # Timedelta changes + start_timedelta = new_start_date - old_start_date + end_timedelta = new_end_date - old_end_date + try: + # Update checkouts based on deltas + domain_checkouts = History.objects.filter( + project=self.object.pk + ) + server_checkouts = ServerHistory.objects.filter( + project=self.object.pk + ) + for checkout in domain_checkouts: + logger.info( + "Updating checkout for %s from %s - %s to %s - %s", + checkout.domain, + checkout.start_date, + checkout.end_date, + checkout.start_date + start_timedelta, + checkout.end_date + end_timedelta, + ) + checkout.start_date = ( + checkout.start_date + start_timedelta + ) + checkout.end_date = checkout.end_date + end_timedelta + checkout.save() + for checkout in server_checkouts: + logger.info( + "Updating checkout for %s from %s-%s to %s-%s", + checkout.server, + checkout.start_date, + checkout.end_date, + checkout.start_date + start_timedelta, + checkout.end_date + end_timedelta, + ) + checkout.start_date = ( + checkout.start_date + start_timedelta + ) + checkout.end_date = checkout.end_date + end_timedelta + checkout.save() + except Exception: + message = "Could not update checkouts with your changed project dates. Review your checkouts or uncheck the box for automatic updates." + form.add_error("update_checkouts", message) + # Save the parent form – will rollback if a child fails validation self.object = form.save() @@ -969,17 +1040,15 @@ def form_valid(self, form): if assignments_valid: assignments.instance = self.object assignments.save() - + # Proceed with form submission if form.is_valid() and objectives_valid and assignments_valid: return super().form_valid(form) else: # Raise an error to rollback transactions raise forms.ValidationError(_("Invalid form data")) - # Otherwise return `form_invalid` and display errors - except Exception as exception: - template = "An exception of type {0} occurred. Arguments:\n{1!r}" - message = template.format(type(exception).__name__, exception.args) - logger.error(message) + # Otherwise return ``form_invalid`` and display errors + except Exception: + logger.exception("Failed to update the project") return super(ProjectUpdate, self).form_invalid(form) @@ -1064,6 +1133,13 @@ def get_context_data(self, **kwargs): ) return ctx + def form_valid(self, form, **kwargs): + self.object = form.save(commit=False) + self.object.operator = self.request.user + self.object.project_id = self.kwargs.get("pk") + self.object.save() + return super().form_valid(form) + class ProjectNoteUpdate(LoginRequiredMixin, UpdateView): """ diff --git a/ghostwriter/routing.py b/ghostwriter/routing.py index 8105f7f06..c655a0f88 100644 --- a/ghostwriter/routing.py +++ b/ghostwriter/routing.py @@ -12,8 +12,10 @@ { # http->django views is added by default "websocket": AuthMiddlewareStack( - URLRouter(ghostwriter.home.routing.websocket_urlpatterns + - ghostwriter.oplog.routing.websocket_urlpatterns) + URLRouter( + ghostwriter.home.routing.websocket_urlpatterns + + ghostwriter.oplog.routing.websocket_urlpatterns + ) ), } ) diff --git a/ghostwriter/shepherd/admin.py b/ghostwriter/shepherd/admin.py index c303e2b97..b058bf6f3 100644 --- a/ghostwriter/shepherd/admin.py +++ b/ghostwriter/shepherd/admin.py @@ -4,6 +4,7 @@ from django.contrib import admin from import_export.admin import ImportExportModelAdmin +# Ghostwriter Libraries from .models import ( ActivityType, AuxServerAddress, @@ -48,8 +49,14 @@ class DomainServerConnectionAdmin(admin.ModelAdmin): ) list_display_links = ("subdomain", "domain_name", "endpoint") fieldsets = ( - ("Domain Information", {"fields": ("domain", "subdomain", "endpoint")},), - ("Server Connection", {"fields": ("static_server", "transient_server")},), + ( + "Domain Information", + {"fields": ("domain", "subdomain", "endpoint")}, + ), + ( + "Server Connection", + {"fields": ("static_server", "transient_server")}, + ), ) @@ -59,7 +66,10 @@ class HistoryAdmin(admin.ModelAdmin): list_filter = ("activity_type", "client") list_display_links = ("domain", "client") fieldsets = ( - ("Domain Checkout Information", {"fields": ("domain", "client", "project")},), + ( + "Domain Checkout Information", + {"fields": ("domain", "client", "project")}, + ), ( "Domain Use Information", {"fields": ("operator", "activity_type", "start_date", "end_date")}, @@ -146,7 +156,10 @@ class ServerHistoryRoleAdmin(admin.ModelAdmin): list_filter = ("activity_type", "client") list_display_links = ("server_name", "ip_address", "client") fieldsets = ( - ("Server Checkout Information", {"fields": ("server", "client", "project")},), + ( + "Server Checkout Information", + {"fields": ("server", "client", "project")}, + ), ( "Server Use Information", { @@ -196,7 +209,15 @@ class StaticServerAdmin(ImportExportModelAdmin): "Basic Server Information", {"fields": ("ip_address", "name", "server_status", "server_provider")}, ), - ("Misc", {"fields": ("last_used_by", "note",)}), + ( + "Misc", + { + "fields": ( + "last_used_by", + "note", + ) + }, + ), ) diff --git a/ghostwriter/shepherd/filters.py b/ghostwriter/shepherd/filters.py index c0553c383..790071b75 100644 --- a/ghostwriter/shepherd/filters.py +++ b/ghostwriter/shepherd/filters.py @@ -13,6 +13,7 @@ from django import forms from django.forms.widgets import TextInput +# Ghostwriter Libraries from .models import Domain, DomainStatus, HealthStatus, ServerStatus @@ -87,8 +88,8 @@ def __init__(self, *args, **kwargs): css_class="form-row", ), Accordion( - AccordionGroup("Health Statuses", InlineCheckboxes("health_status")), AccordionGroup("Domain Statuses", InlineCheckboxes("domain_status")), + AccordionGroup("Health Statuses", InlineCheckboxes("health_status")), ), ButtonHolder( Submit("submit", "Filter", css_class="btn btn-primary col-md-2"), diff --git a/ghostwriter/shepherd/forms.py b/ghostwriter/shepherd/forms.py index a38709481..c17fd4d67 100644 --- a/ghostwriter/shepherd/forms.py +++ b/ghostwriter/shepherd/forms.py @@ -9,7 +9,7 @@ from django import forms from django.core.exceptions import ValidationError from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # Ghostwriter Libraries from ghostwriter.rolodex.models import Project @@ -40,6 +40,7 @@ def __init__(self, *args, **kwargs): super(CheckoutForm, self).__init__(*args, **kwargs) data_projects_url = reverse("shepherd:ajax_load_projects") data_project_url = reverse("shepherd:ajax_load_project") + overwatch_url = reverse("shepherd:ajax_domain_overwatch") for field in self.fields: self.fields[field].widget.attrs["autocomplete"] = "off" self.fields["client"].empty_label = "-- Select a Client --" @@ -64,6 +65,7 @@ def __init__(self, *args, **kwargs): self.helper.attrs = { "data-projects-url": data_projects_url, "data-project-url": data_project_url, + "overwatch-url": overwatch_url, } self.helper.form_id = "checkout-form" self.helper.layout = Layout( @@ -314,16 +316,34 @@ def __init__(self, project=None, *args, **kwargs): if project: self.fields["static_server"].queryset = ServerHistory.objects.filter( project=project - ) + ).order_by("activity_type", "server_role") self.fields["transient_server"].queryset = TransientServer.objects.filter( project=project - ) + ).order_by("activity_type", "server_role") for field in self.fields: self.fields[field].widget.attrs["autocomplete"] = "off" - self.fields["domain"].queryset = History.objects.filter(project=project) + self.fields["domain"].queryset = History.objects.filter( + project=project + ).order_by("activity_type") self.fields["domain"].empty_label = "-- Select a Domain [Required] --" + self.fields[ + "domain" + ].label_from_instance = lambda obj: f"{obj.domain.name} ({obj.activity_type})" + self.fields["static_server"].empty_label = "-- Select Static Server --" + self.fields[ + "static_server" + ].label_from_instance = ( + lambda obj: f"{obj.server.ip_address} ({obj.server_role} | {obj.activity_type})" + ) + self.fields["transient_server"].empty_label = "-- Select VPS --" + self.fields[ + "transient_server" + ].label_from_instance = ( + lambda obj: f"{obj.ip_address} ({obj.server_role} | {obj.activity_type})" + ) + self.helper = FormHelper() self.helper.form_method = "post" self.helper.form_class = "newitem" @@ -387,12 +407,7 @@ class DomainNoteForm(forms.ModelForm): class Meta: model = DomainNote - fields = "__all__" - widgets = { - "timestamp": forms.HiddenInput(), - "operator": forms.HiddenInput(), - "domain": forms.HiddenInput(), - } + fields = ("note",) def __init__(self, *args, **kwargs): super(DomainNoteForm, self).__init__(*args, **kwargs) @@ -401,7 +416,7 @@ def __init__(self, *args, **kwargs): self.helper.form_class = "newitem" self.helper.form_show_labels = False self.helper.layout = Layout( - Div("note", "operator", "domain"), + Div("note"), ButtonHolder( Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), HTML( @@ -417,7 +432,8 @@ def clean_note(self): # Check if note is empty if not note: raise ValidationError( - _("You must provide some content for the note"), code="required", + _("You must provide some content for the note"), + code="required", ) return note diff --git a/ghostwriter/shepherd/forms_server.py b/ghostwriter/shepherd/forms_server.py index ff21803bb..a1f88d509 100644 --- a/ghostwriter/shepherd/forms_server.py +++ b/ghostwriter/shepherd/forms_server.py @@ -18,7 +18,7 @@ from django.core.exceptions import ValidationError from django.forms.models import BaseInlineFormSet, inlineformset_factory from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ # Ghostwriter Libraries from ghostwriter.modules.custom_layout_object import CustomTab, Formset @@ -335,12 +335,7 @@ class ServerNoteForm(forms.ModelForm): class Meta: model = ServerNote - fields = "__all__" - widgets = { - "timestamp": forms.HiddenInput(), - "operator": forms.HiddenInput(), - "server": forms.HiddenInput(), - } + fields = ("note",) def __init__(self, *args, **kwargs): super(ServerNoteForm, self).__init__(*args, **kwargs) @@ -349,7 +344,7 @@ def __init__(self, *args, **kwargs): self.helper.form_class = "newitem" self.helper.form_show_labels = False self.helper.layout = Layout( - Div("note", "operator", "server"), + Div("note"), ButtonHolder( Submit("submit", "Submit", css_class="btn btn-primary col-md-4"), HTML( @@ -365,7 +360,8 @@ def clean_note(self): # Check if note is empty if not note: raise ValidationError( - _("You must provide some content for the note"), code="required", + _("You must provide some content for the note"), + code="required", ) return note diff --git a/ghostwriter/shepherd/migrations/0015_auto_20201120_0620.py b/ghostwriter/shepherd/migrations/0015_auto_20201120_0620.py new file mode 100644 index 000000000..a0ceecb3c --- /dev/null +++ b/ghostwriter/shepherd/migrations/0015_auto_20201120_0620.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.10 on 2020-11-20 06:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('shepherd', '0014_auto_20200909_1804'), + ] + + operations = [ + migrations.AddField( + model_name='domain', + name='last_health_check', + field=models.DateField(blank=True, help_text='The date and time of the latest health check for this domain name', null=True, verbose_name='Last Health Check'), + ), + migrations.AddField( + model_name='domain', + name='vt_permalink', + field=models.CharField(blank=True, help_text="VirusTotal's permalink for scan results of this domain", max_length=255, null=True, verbose_name='VirusTotal Permalink'), + ), + ] diff --git a/ghostwriter/shepherd/models.py b/ghostwriter/shepherd/models.py index a7cdd0815..f6ccc1b73 100644 --- a/ghostwriter/shepherd/models.py +++ b/ghostwriter/shepherd/models.py @@ -116,6 +116,11 @@ def __str__(self): return self.activity +class DomainManager(models.Manager): + def get_by_natural_key(self, name): + return self.get(name=name) + + class Domain(models.Model): """ Stores an individual domain, related to :model:`shepherd.WhoisStatus`, @@ -152,6 +157,19 @@ class Domain(models.Model): expiration = models.DateField( "Expiration Date", help_text="Select the date the domain will expire" ) + last_health_check = models.DateField( + "Last Health Check", + help_text="The date and time of the latest health check for this domain name", + blank=True, + null=True, + ) + vt_permalink = models.CharField( + "VirusTotal Permalink", + max_length=255, + null=True, + blank=True, + help_text="VirusTotal's permalink for scan results of this domain", + ) all_cat = models.TextField( "All Categories", null=True, @@ -259,11 +277,16 @@ class Domain(models.Model): help_text="The last user to checkout this domain", ) + objects = DomainManager() + class Meta: ordering = ["health_status", "name", "expiration"] verbose_name = "Domain" verbose_name_plural = "Domains" + def natural_key(self): + return (self.name,) + def get_absolute_url(self): return reverse("shepherd:domain_detail", args=[str(self.id)]) @@ -274,7 +297,7 @@ def get_domain_age(self): if self.is_expired: time_delta = self.expiration - self.creation else: - time_delta = datetime.date.today() - self.creation + time_delta = date.today() - self.creation return "{} days".format(time_delta.days) def is_expired(self): @@ -282,19 +305,31 @@ def is_expired(self): Check if the domain's expiration DateField value is in the past. """ expired = False - if datetime.date.today() > self.expiration: + if date.today() > self.expiration: if not self.auto_renew: expired = True return expired - @property + def is_expiring_soon(self): + """ + Check if the domain's expiration DateField value is in the near future. + """ + expiring_soon = False + delta = date.today() + datetime.timedelta(days=30) + if self.expiration <= delta: + if not self.auto_renew: + expiring_soon = True + return expiring_soon + def get_list(self): """ Return an instance's dns_record field value as a list. """ if self.dns_record: try: - json_acceptable_string = self.dns_record.replace("'", '"') + json_acceptable_string = self.dns_record.replace('"', "").replace( + "'", '"' + ) if json_acceptable_string: return json.loads(json_acceptable_string) else: diff --git a/ghostwriter/shepherd/resources.py b/ghostwriter/shepherd/resources.py index 291dc138b..2de4a9842 100644 --- a/ghostwriter/shepherd/resources.py +++ b/ghostwriter/shepherd/resources.py @@ -5,6 +5,7 @@ from import_export.fields import Field from import_export.widgets import ForeignKeyWidget +# Ghostwriter Libraries from .models import ( Domain, DomainStatus, diff --git a/ghostwriter/shepherd/tasks.py b/ghostwriter/shepherd/tasks.py index f2360a5fd..1f47bb0cf 100644 --- a/ghostwriter/shepherd/tasks.py +++ b/ghostwriter/shepherd/tasks.py @@ -2,28 +2,32 @@ # Standard Libraries import datetime -import io import json import logging import logging.config -import os -import zipfile +import traceback from collections import defaultdict from datetime import date # Django & Other 3rd Party Libraries +from asgiref.sync import async_to_sync +from channels.layers import get_channel_layer import boto3 import nmap import pytz import requests from botocore.exceptions import ClientError -from django.conf import settings -from django.core.files import File from django.db.models import Q from lxml import objectify # Ghostwriter Libraries -from ghostwriter.modules.dns import DNSCollector +from ghostwriter.commandcenter.models import ( + CloudServicesConfiguration, + NamecheapConfiguration, + SlackConfiguration, + VirusTotalConfiguration, +) +from ghostwriter.modules.dns_toolkit import DNSCollector from ghostwriter.modules.review import DomainReview from .models import ( @@ -42,6 +46,8 @@ # Using __name__ resolves to ghostwriter.shepherd.tasks logger = logging.getLogger(__name__) +channel_layer = get_channel_layer() + def craft_cloud_message( username, @@ -52,10 +58,11 @@ def craft_cloud_message( end_date, cloud_provider, vps_name, + ip_address, tags, ): - """Function to craft a nicely formatted Slack message using blocks - for cloud asset notifications. + """ + Craft a nicely formatted Slack message using blocks for cloud asset notifications. """ CLOUD_ASSET_MESSAGE = { "username": username, @@ -82,6 +89,10 @@ def craft_cloud_message( "text": "*Cloud Provider:*\n{}".format(cloud_provider), }, {"type": "mrkdwn", "text": "*Instance Name:*\n{}".format(vps_name)}, + { + "type": "mrkdwn", + "text": "*Ext IP Address:*\n{}".format(ip_address), + }, { "type": "mrkdwn", "text": "*Launch Date:*\n{}".format(launch_time), @@ -99,22 +110,22 @@ def craft_cloud_message( def craft_unknown_asset_message( - username, emoji, channel, launch_time, cloud_provider, vps_name, tags + username, emoji, channel, launch_time, cloud_provider, vps_name, ip_address, tags ): - """Function to craft a nicely formatted Slack message using blocks - for cloud assets not found in Ghostwriter. + """ + Craft a nicely formatted Slack message using blocks for cloud assets not found in Ghostwriter. """ UNKNOWN_ASSET_MESSAGE = { "username": username, "icon_emoji": emoji, "channel": channel, - "text": "An *untracked* cloud asset is running without being attached to a project. If this asset should be ignored, add the `gw_ignore` tag.", + "text": "An *untracked* cloud asset is running without being attached to a project. If this asset should be ignored, tag it with one of the configured `Ignore Tags` in settings.", "blocks": [ { "type": "section", "text": { "type": "mrkdwn", - "text": "An *untracked* cloud asset is running without being attached to a project. If this asset should be ignored, add the `gw_ignore` tag.", + "text": "An *untracked* cloud asset is running without being attached to a project. If this asset should be ignored, tag it with one of the configured `Ignore Tags` in settings.", }, }, { @@ -125,6 +136,10 @@ def craft_unknown_asset_message( "text": "*Cloud Provider:*\n{}".format(cloud_provider), }, {"type": "mrkdwn", "text": "*Instance Name:*\n{}".format(vps_name)}, + { + "type": "mrkdwn", + "text": "*Ext IP Address:*\n{}".format(ip_address), + }, { "type": "mrkdwn", "text": "*Launch Date:*\n{}".format(launch_time), @@ -137,37 +152,53 @@ def craft_unknown_asset_message( return json.dumps(UNKNOWN_ASSET_MESSAGE) -def get_slack_config(): - """Function to determine if Slack settings are configured and return the - settings if available. +def craft_burned_message( + username, emoji, channel, domain, categories, burned_explanation +): """ - try: - enable_slack = settings.SLACK_CONFIG["enable_slack"] - except KeyError: - enable_slack = False - return enable_slack - if enable_slack: - try: - slack_config = {} - slack_config["slack_emoji"] = settings.SLACK_CONFIG["slack_emoji"] - slack_config["slack_username"] = settings.SLACK_CONFIG["slack_username"] - slack_config["slack_webhook_url"] = settings.SLACK_CONFIG[ - "slack_webhook_url" - ] - slack_config["slack_alert_target"] = settings.SLACK_CONFIG[ - "slack_alert_target" - ] - slack_config["slack_channel"] = settings.SLACK_CONFIG["slack_channel"] - slack_capable = True - except KeyError: - slack_capable = False - return slack_config - else: - return enable_slack + Craft a nicely formatted Slack message using blocks for newly burned domain names. + """ + BURNED_DOMAIN_MESSAGE = { + "username": username, + "icon_emoji": emoji, + "channel": channel, + "text": "This domain name is now considered *Burned* :fire:", + "blocks": [ + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "This domain name is now considered *Burned* :fire:", + }, + }, + { + "type": "section", + "fields": [ + { + "type": "mrkdwn", + "text": "*Domain Name:*\n{}".format(domain), + }, + { + "type": "mrkdwn", + "text": "*Categories:*\n{}".format(", ".join(categories)), + }, + { + "type": "mrkdwn", + "text": "*Explanation:*\n{}".format( + ", ".join(burned_explanation) + ), + }, + ], + }, + ], + } + return json.dumps(BURNED_DOMAIN_MESSAGE) class BearerAuth(requests.auth.AuthBase): - """Helper class for providing the Authorization header with Requests.""" + """ + Helper class for providing the ``Authorization`` header with ``Requests``. + """ token = None @@ -180,66 +211,58 @@ def __call__(self, r): def send_slack_msg(message, slack_channel=None): - """Accepts message text and sends it to Slack. This requires Slack - settings and a webhook be configured in the application's settings. + """ + Send a basic Slack message using the global Slack configuration. - Parameters: + **Parameters** - message A string to be sent as the Slack message - slack_channel Defaults to using the global setting. Can be set to any - Slack channel name. + ``message`` + A string to be sent as the Slack message + ``slack_channel`` + Defaults to using the global setting. Can be set to any Slack channel name """ - try: - enable_slack = settings.SLACK_CONFIG["enable_slack"] - except KeyError: - enable_slack = False - if enable_slack: - try: - slack_emoji = settings.SLACK_CONFIG["slack_emoji"] - slack_username = settings.SLACK_CONFIG["slack_username"] - slack_webhook_url = settings.SLACK_CONFIG["slack_webhook_url"] - slack_alert_target = settings.SLACK_CONFIG["slack_alert_target"] - if not slack_channel: - slack_channel = settings.SLACK_CONFIG["slack_channel"] - slack_capable = True - except KeyError: - slack_capable = False - - if slack_capable: - message = slack_alert_target + " " + message - slack_data = { - "username": slack_username, - "icon_emoji": slack_emoji, - "channel": slack_channel, - "text": message, - } - response = requests.post( - slack_webhook_url, - data=json.dumps(slack_data), - headers={"Content-Type": "application/json"}, + slack_config = SlackConfiguration.get_solo() + + if slack_config.enable: + message = slack_config.slack_alert_target + " " + message + slack_data = { + "username": slack_config.slack_username, + "icon_emoji": slack_config.slack_emoji, + "channel": slack_config.slack_channel, + "text": message, + } + response = requests.post( + slack_config.webhook_url, + data=json.dumps(slack_data), + headers={"Content-Type": "application/json"}, + ) + if response.status_code != 200: + logger.warning( + "Request to Slack returned an error %s, the response was: %s", + response.status_code, + response.text, ) - if response.status_code != 200: - logger.error( - "Request to Slack returned an error %s, the response is:\n%s", - response.status_code, - response.text, - ) + else: + logger.warning( + "Received request to send Slack message, but Slack notifications are disabled in settings" + ) -def send_slack_test_msg(slack_channel=None): - """Use `send_slack_msg` to send a test message using the configured Slack webhook.""" - message = "This is a test of your notification system." - send_slack_msg(message, slack_channel) +def send_slack_complete_msg(task): + """ + Send a basic Slack message using the global Slack configuration upon completion + of an ``async_task()``. + **Parameters** -def send_slack_complete_msg(task): - """Function to send a Slack message for a task. Meant to be used as a hook - for an async_task(). + ``task`` + Instance of :model:`django_q.Task` """ if task.success: send_slack_msg( - "{} task has completed its run. It completed " - "successfully with no additional result data.".format(task.group) + "{} task has completed its run. It completed successfully with no additional result data.".format( + task.group + ) ) else: if task.result: @@ -248,55 +271,53 @@ def send_slack_complete_msg(task): ) else: send_slack_msg( - "{} task failed with no result/error data. Check " - "the Django Q admin panel.".format(task.group) + "{} task failed with no result/error data. Check the Django Q admin panel.".format( + task.group + ) ) def release_domains(no_action=False, reset_dns=False): - """Pull all domains currently checked-out in Shepherd and update the - status to Available if the project's end date is today or in the past. + """ + Pull all :model:`shepherd.Domain` currently checked-out in Shepherd and update the + status to ``Available`` if the project's ``end_date`` value is today or in the past. - Parameters: + **Parameters** - no_action Defaults to False. Set to True to take no action and just - return a list of domains that should be released now. - reset_dns Defaults to False. Set to True to reset the DNS records for - Namecheap domains. This sets the DNS records to the defaults. + ``no_action`` + Set to True to take no action and just return a list of domains that should + be released (Default: False) + ``reset_dns`` + Set to True to reset the DNS records for Namecheap domains (Default: False) """ domain_updates = {} domain_updates["errors"] = {} + # Configure Namecheap API requests - namecheap_ready = False session = requests.Session() reset_records_endpoint = "https://api.namecheap.com/xml.response?apiuser={}&apikey={}&username={}&Command=namecheap.domains.dns.setHosts&ClientIp={}&SLD={}&TLD={}" reset_record_template = ( "&HostName1=@&RecordType1=URL&Address1=http://www.namecheap.com&TTL1=100" ) - try: - client_ip = settings.NAMECHEAP_CONFIG["client_ip"] - enable_namecheap = settings.NAMECHEAP_CONFIG["enable_namecheap"] - namecheap_api_key = settings.NAMECHEAP_CONFIG["namecheap_api_key"] - namecheap_username = settings.NAMECHEAP_CONFIG["namecheap_username"] - namecheap_api_username = settings.NAMECHEAP_CONFIG["namecheap_api_username"] - namecheap_ready = True - logger.info("Successfully pulled Namecheap API configuration") - except KeyError as error: - logger.error("Could not retrieve API configuration: %s", error) - domain_updates["errors"][ - "namecheap" - ] = "Could not retrieve API configuration: {}".format(error) + namecheap_config = NamecheapConfiguration.get_solo() + if reset_dns is True and namecheap_config.enable is False: + logger.warning( + "Received request to reset Namecheap DNS records for released domains, but Namecheap API is disabled in settings" + ) + # Start tracking domain releases domains_to_be_released = [] - # First, get all domains set to `Unavailable` + + # First, get all domains set to ``Unavailable`` queryset = Domain.objects.filter(domain_status__domain_status="Unavailable") - # Go through each `Unavailable` domain and check it against projects + + # Go through each ``Unavailable`` domain and check it against projects logger.info("Starting domain release task at %s", datetime.datetime.now()) for domain in queryset: - # Get all projects for the domain release_me = True slack_channel = None try: + # Get latest project checkout for domain project_queryset = History.objects.filter(domain__name=domain.name).latest( "end_date" ) @@ -310,20 +331,17 @@ def release_domains(no_action=False, reset_dns=False): # Check if tomorrow is the end date if date.today() == warning_date: release_me = False - message = ( - "Your domain, {}, will be released tomorrow! " - "Modify the project's end date as needed.".format(domain.name) + message = "Your domain, {}, will be released tomorrow! Modify the project's end date as needed.".format( + domain.name ) - if project_queryset.project.slack_channel: - send_slack_msg(message, project_queryset.project.slack_channel) - else: - send_slack_msg(message) + send_slack_msg(message, slack_channel) except History.DoesNotExist: logger.warning( "The domain %s has no project history, so releasing it", domain.name ) release_date = datetime.datetime.today() - # If release_me is still true, release the domain + + # If ``release_me`` is still ``True``, release the domain if release_me: logger.warning("The domain %s is marked for release", domain.name) domains_to_be_released.append(domain) @@ -334,20 +352,18 @@ def release_domains(no_action=False, reset_dns=False): if no_action: domain_updates[domain.id]["change"] = "no action" else: + logger.info("Releasing %s back into the pool.", domain.name) message = "Your domain, {}, has been released.".format(domain.name) - if slack_channel: - send_slack_msg(message, slack_channel) - else: - send_slack_msg(message) + send_slack_msg(message, slack_channel) domain.domain_status = DomainStatus.objects.get( domain_status="Available" ) domain.save() domain_updates[domain.id]["change"] = "released" # Make sure the Namecheap API config is good and reg is Namecheap - # Most importantly, check the `reset_dns` flag is True in kwargs + # Most importantly, check the ``reset_dns`` flag is ``True`` in kwargs if ( - namecheap_ready + namecheap_config.enable and domain.registrar.lower() == "namecheap" and reset_dns ): @@ -363,10 +379,10 @@ def release_domains(no_action=False, reset_dns=False): # The Namecheap API call requires both usernames, a key, and a whitelisted IP req = session.get( reset_records_endpoint.format( - namecheap_api_username, - namecheap_api_key, - namecheap_username, - client_ip, + namecheap_config.api_username, + namecheap_config.api_key, + namecheap_config.username, + namecheap_config.client_ip, sld, tld, ) @@ -379,9 +395,11 @@ def release_domains(no_action=False, reset_dns=False): # Check the status to make sure it says "OK" namecheap_api_result = root.attrib["Status"] if namecheap_api_result == "OK": - is_success = root.CommandResponse.DomainDNSSetHostsResult.attrib[ - "IsSuccess" - ] + is_success = ( + root.CommandResponse.DomainDNSSetHostsResult.attrib[ + "IsSuccess" + ] + ) warnings = ( root.CommandResponse.DomainDNSSetHostsResult.Warnings ) @@ -438,154 +456,190 @@ def release_domains(no_action=False, reset_dns=False): ] = "Namecheap API request failed with error: {}".format(error) else: domain_updates[domain.id]["dns"] = "no action" + logger.info("Domain release task completed at %s", datetime.datetime.now()) return domain_updates def release_servers(no_action=False): - """Pull all servers currently checked-out in Shepherd and update the - status to Available if the project's end date is today or in the past. + """ + Pull all :model:`shepherd.StaticServer` currently checked-out in Shepherd and + update the ``server_status`` to ``Available`` if the project's end date is today + or in the past. - Parameters: + **Parameters** - no_action Defaults to False. Set to True to take no action and just - return a list of servers that should be released now. + ``no_action`` + Set to True to take no action and just return a list of servers that + should be released now (Default: False) """ + server_updates = {} + server_updates["errors"] = {} servers_to_be_released = [] - # First get all server set to `Unavailable` + + # First get all server set to ``Unavailable`` queryset = StaticServer.objects.filter(server_status__server_status="Unavailable") - # Go through each `Unavailable` server and check it against projects + + # Go through each ``Unavailable`` server and check it against projects for server in queryset: - # Get all projects for the server + release_me = True + slack_channel = None + + # Get latest project checkout for the server try: project_queryset = ServerHistory.objects.filter( server__ip_address=server.ip_address ).latest("end_date") + release_date = project_queryset.end_date + warning_date = release_date - datetime.timedelta(1) + if project_queryset.project.slack_channel: + slack_channel = project_queryset.project.slack_channel + # Check if date is before or is the end date + if date.today() <= release_date: + release_me = False + # Check if tomorrow is the end date + if date.today() == warning_date: + release_me = False + message = "Your server, {}, will be released tomorrow! Modify the project's end date as needed.".format( + server.ip_address + ) + send_slack_msg(message, slack_channel) except ServerHistory.DoesNotExist: - continue - release_me = True - # Check each project's end date to determine if all are in the past - release_date = project_queryset.end_date - warning_date = release_date - datetime.timedelta(1) - # Check if date is before or is the end date - if date.today() <= release_date: - release_me = False - # Check if tomorrow is the end date - if date.today() == warning_date: - release_me = False - message = ( - "Your server, {}, will be released tomorrow! " - "Modify the project's end date as needed.".format(server.ip_address) + logger.warning( + "The server %s has no project history, so releasing it", + server.ip_address, ) - if project_queryset.project.slack_channel: - send_slack_msg(message, project_queryset.project.slack_channel) - else: - send_slack_msg(message) - # If release_me is still true, release the server + release_date = datetime.datetime.today() + + # If ``release_me`` is still ``True``, release the server if release_me: + logger.warning("The server %s is marked for release", server.ip_address) servers_to_be_released.append(server) - # Check no_action and just return list if it is set to True + server_updates[server.id] = {} + server_updates[server.id]["server"] = server.ip_address + server_updates[server.id]["hostname"] = server.name + server_updates[server.id]["release_date"] = release_date + + # Check ``no_action`` and just return list if it is set to ``True`` if no_action: - return servers_to_be_released + server_updates[server.id]["change"] = "no action" else: + logger.info("Releasing %s back into the pool.", server.ip_address) message = "Your server, {}, has been released.".format( server.ip_address ) - logger.info("Releasing %s back into the pool.", server.ip_address) - if project_queryset.project.slack_channel: - send_slack_msg(message, project_queryset.project.slack_channel) - else: - send_slack_msg(message) + send_slack_msg(message, slack_channel) server.server_status = ServerStatus.objects.get( server_status="Available" ) server.save() - return servers_to_be_released + server_updates[server.id]["change"] = "released" + + logger.info("Server release task completed at %s", datetime.datetime.now()) + return server_updates def check_domains(domain=None): - """Initiate a check of all domains in the Domain model and update each - domain status. + """ + Initiate a check of all :model:`shepherd.Domain` and update the ``domain_status`` values. - Parameters: + **Parameters** - domain Defaults to None. Provide a domain name's primary key to - update only that domain. This arg is used when a user - requests a specific domain be updated via the web - interface. + ``domain`` + Individual domain's primary key to update only that domain (Default: None) """ - # Get all domains from the database + # Fetch Slack configuration information + slack_config = SlackConfiguration.get_solo() + + # Get target domain(s) from the database domain_list = [] + sleep_time_override = None if domain: domain_queryset = Domain.objects.get(pk=domain) domain_list.append(domain_queryset) + logger.info("Checking only one domain, so disabling sleep time for VirusTotal") + sleep_time_override = 0 else: domain_queryset = Domain.objects.all() for result in domain_queryset: domain_list.append(result) - domain_review = DomainReview(domain_list) + # Execute ``DomainReview`` to check categories + domain_review = DomainReview(domain_list, sleep_time_override) lab_results = domain_review.check_domain_status() + # Update the domains as needed + domain_updates = {} + domain_updates["errors"] = {} for domain in lab_results: + change = "no action" + domain_updates[domain.id] = {} + domain_updates[domain.id]["domain"] = domain.name + if "vt_results" in lab_results[domain]: + domain_updates[domain.id]["vt_results"] = lab_results[domain]["vt_results"] try: - # The `domain` is already a Domain object so this query might be - # unnecessary :thinking_emoji: - domain_instance = Domain.objects.get(name=domain.name) # Flip status if a domain has been flagged as burned if lab_results[domain]["burned"]: - logger.warning("Domain %s is burned", domain.name) - domain_instance.health_status = HealthStatus.objects.get( - health_status="Burned" - ) - domain_instance.domain_status = DomainStatus.objects.get( - domain_status="Burned" - ) - message = "*{}* has been flagged as burned because: {}".format( - domain.name, lab_results[domain]["burned_explanation"] - ) - if lab_results[domain]["categories"]["bad"]: - message = message + " (Bad categories: {})".format( - lab_results[domain]["categories"]["bad"] + domain.health_status = HealthStatus.objects.get(health_status="Burned") + domain.domain_status = DomainStatus.objects.get(domain_status="Burned") + change = "burned" + if slack_config.enable: + slack_data = craft_burned_message( + slack_config.slack_username, + slack_config.slack_emoji, + slack_config.slack_channel, + domain.name, + lab_results[domain]["categories"], + lab_results[domain]["burned_explanation"], + ) + response = requests.post( + slack_config.webhook_url, + data=slack_data, + headers={"Content-Type": "application/json"}, ) - send_slack_msg(message) # Update other fields for the domain object - domain_instance.health_dns = lab_results[domain]["health_dns"] - domain_instance.burned_explanation = lab_results[domain][ - "burned_explanation" - ] - domain_instance.all_cat = lab_results[domain]["categories"]["all"] - domain_instance.talos_cat = lab_results[domain]["categories"]["talos"] - domain_instance.opendns_cat = lab_results[domain]["categories"]["opendns"] - domain_instance.bluecoat_cat = lab_results[domain]["categories"]["bluecoat"] - domain_instance.ibm_xforce_cat = lab_results[domain]["categories"]["xforce"] - domain_instance.trendmicro_cat = lab_results[domain]["categories"][ - "trendmicro" - ] - domain_instance.fortiguard_cat = lab_results[domain]["categories"][ - "fortiguard" - ] - domain_instance.mx_toolbox_status = lab_results[domain]["categories"][ - "mxtoolbox" - ] - domain_instance.save() - except Exception as error: - logger.error('Error updating "%s" – %s', domain.name, error) + if ( + lab_results[domain]["burned"] + and "burned_explanation" in lab_results[domain] + ): + if lab_results[domain]["burned_explanation"]: + domain.burned_explanation = "\n".join( + lab_results[domain]["burned_explanation"] + ) + if lab_results[domain]["categories"] != domain.all_cat: + change = "categories updated" + if lab_results[domain]["categories"]: + domain.all_cat = ", ".join(lab_results[domain]["categories"]).title() + else: + domain.all_cat = "Uncategorized" + domain.last_health_check = datetime.datetime.now() + domain.save() + domain_updates[domain.id]["change"] = change + except Exception: + trace = traceback.format_exc() + domain_updates[domain.id]["change"] = "error" + domain_updates["errors"][domain.name] = {} + domain_updates["errors"][domain.name] = trace + logger.exception('Error updating "%s"', domain.name) pass + return domain_updates + def update_dns(domain=None): - """Initiate a check of all domains in the Domain model and update each - domain's DNS records. + """ + Initiate a check of all :model:`shepherd.Domain` and update each domain's DNS records. - Parameters: + **Parameters** - domain Defaults to None. Provide a domain name's primary key to - update only that domain. This arg is used when a user - requests a specific domain be updated via the web - interface. + ``domain`` + Individual domain name's primary key to update only that domain (Default: None) """ domain_list = [] dns_toolkit = DNSCollector() - # Get all domains from the database + + domain_updates = {} + domain_updates["errors"] = {} + + # Get the target domain(s) from the database if domain: domain_queryset = Domain.objects.get(pk=domain) domain_list.append(domain_queryset) @@ -596,84 +650,104 @@ def update_dns(domain=None): ) else: logger.info("Starting mass DNS record update at %s", datetime.datetime.now()) - domain_queryset = Domain.objects.all() + domain_queryset = Domain.objects.filter( + ~Q(domain_status=DomainStatus.objects.get(domain_status="Expired")) + ) for result in domain_queryset: domain_list.append(result) + + record_types = ["A", "NS", "MX", "TXT", "CNAME", "SOA", "DMARC"] + dns_records = dns_toolkit.run_async_dns( + domains=domain_list, record_types=record_types + ) + for domain in domain_list: - # Get each type of DNS record for the domain - try: - try: - temp = [] - ns_records_list = dns_toolkit.get_dns_record(domain.name, "NS") - for rdata in ns_records_list.response.answer: - for item in rdata.items: - temp.append(item.to_text()) - ns_records = ", ".join(x.strip(".") for x in temp) - except Exception: - ns_records = None - try: - temp = [] - a_records = dns_toolkit.get_dns_record(domain.name, "A") - for rdata in a_records.response.answer: - for item in rdata.items: - temp.append(item.to_text()) - a_records = ", ".join(temp) - except Exception: - a_records = None - try: - mx_records = dns_toolkit.return_dns_record_list(domain.name, "MX") - except Exception: - mx_records = None - try: - txt_records = dns_toolkit.return_dns_record_list(domain.name, "TXT") - except Exception: - txt_records = None - try: - soa_records = dns_toolkit.return_dns_record_list(domain.name, "SOA") - except Exception: - soa_records = None + domain_updates[domain.id] = {} + domain_updates[domain.id]["domain"] = domain.name + + if domain.name in dns_records: try: - dmarc_record = dns_toolkit.return_dns_record_list( - "_dmarc." + domain.name, "TXT" - ) - except Exception: - dmarc_record = None - # Assemble the dict to be stored in the database - dns_records_dict = {} - if ns_records: - dns_records_dict["ns"] = ns_records - if a_records: - dns_records_dict["a"] = a_records - if mx_records: - dns_records_dict["mx"] = mx_records - if dmarc_record: - dns_records_dict["dmarc"] = dmarc_record + a_record = dns_records[domain.name]["a_record"] + mx_record = dns_records[domain.name]["mx_record"] + ns_record = dns_records[domain.name]["ns_record"] + txt_record = dns_records[domain.name]["txt_record"] + soa_record = dns_records[domain.name]["soa_record"] + cname_record = dns_records[domain.name]["cname_record"] + dmarc_record = dns_records[domain.name]["dmarc_record"] + + # Format any lists as strings for storage + if isinstance(a_record, list): + a_record = ", ".join(a_record).replace('"', "") else: - dns_records_dict["dmarc"] = "No DMARC record!" - if txt_records: - dns_records_dict["txt"] = txt_records - if soa_records: - dns_records_dict["soa"] = soa_records - except Exception as error: - logger.error( - "Encountered an error processing records for %s - %s", domain, error + a_record = a_record.replace('"', "") + if isinstance(mx_record, list): + mx_record = ", ".join(mx_record).replace('"', "") + else: + mx_record = mx_record.replace('"', "") + if isinstance(ns_record, list): + ns_record = ", ".join(ns_record).replace('"', "") + else: + ns_record = ns_record.replace('"', "") + if isinstance(txt_record, list): + txt_record = ", ".join(txt_record).replace('"', "") + else: + txt_record = txt_record.replace('"', "") + if isinstance(soa_record, list): + soa_record = ", ".join(soa_record).replace('"', "") + else: + soa_record = soa_record.replace('"', "") + if isinstance(cname_record, list): + cname_record = ", ".join(cname_record).replace('"', "") + else: + cname_record = cname_record.replace('"', "") + if isinstance(dmarc_record, list): + dmarc_record = ", ".join(dmarc_record).replace('"', "") + else: + dmarc_record = dmarc_record.replace('"', "") + + # Assemble the dict to be stored in the database + dns_records_dict = {} + dns_records_dict["ns"] = ns_record + dns_records_dict["a"] = a_record + dns_records_dict["mx"] = mx_record + dns_records_dict["cname"] = cname_record + dns_records_dict["dmarc"] = dmarc_record + dns_records_dict["txt"] = txt_record + dns_records_dict["soa"] = soa_record + + logger.info(dns_records_dict) + + # Look-up the individual domain and save the new record string + domain_instance = Domain.objects.get(name=domain.name) + domain_instance.dns_record = dns_records_dict + domain_instance.save() + domain_updates[domain.id]["result"] = "updated" + except Exception: + trace = traceback.format_exc() + logger.exception("Failed updating DNS records for %s", domain.name) + domain_updates["errors"][ + domain.name + ] = "Failed updating DNS records: {traceback}".format(traceback=trace) + else: + logger.warning( + "The domain %s was not found in the returned DNS records", domain.name ) - dns_records_dict = "None" - # Look-up the individual domain and save the new record string - domain_instance = Domain.objects.get(name=domain.name) - domain_instance.dns_record = dns_records_dict - domain_instance.save() + domain_updates[domain.id]["result"] = "no results" + # Log task completed logger.info("DNS update completed at %s", datetime.datetime.now()) + return domain_updates def scan_servers(only_active=False): - """Uses `python-nmap` to scan servers in the `StaticServer` model to - report open ports. + """ + Uses ``python-nmap`` to scan individual :model:`shepherd.StaticServer` + and :model:`shepherd.TransientServer` to identify open ports. - Parameters: + **Parameters** - only_active Defaults to False. set to True to restrict + ``only_active`` + Only scan servers marked as in-use (Default: False) """ # Create the scanner scanner = nmap.PortScanner() @@ -711,7 +785,8 @@ def scan_servers(only_active=False): def fetch_namecheap_domains(): - """Fetch a list of registered domains for the specified Namecheap account. A valid API key, + """ + Fetch a list of registered domains for the specified Namecheap account. A valid API key, username, and whitelisted IP address must be used. Returns a dictionary containing errors and each domain name paired with change status. @@ -746,31 +821,17 @@ def fetch_namecheap_domains(): "Starting Namecheap synchronization task at %s", datetime.datetime.now() ) - try: - client_ip = settings.NAMECHEAP_CONFIG["client_ip"] - enable_namecheap = settings.NAMECHEAP_CONFIG["enable_namecheap"] - namecheap_api_key = settings.NAMECHEAP_CONFIG["namecheap_api_key"] - namecheap_username = settings.NAMECHEAP_CONFIG["namecheap_username"] - namecheap_page_size = settings.NAMECHEAP_CONFIG["namecheap_page_size"] - namecheap_api_username = settings.NAMECHEAP_CONFIG["namecheap_api_username"] - except KeyError as e: - logger.error( - "Encountered an error when fetching the Namecheap API configuration. Check your .django env file. Error: %s", - e, - ) - return "[!] Encountered an error when fetching the Namecheap API configuration. Check your .django file. Error: {}".format( - e - ) + namecheap_config = NamecheapConfiguration.get_solo() try: # The Namecheap API call requires both usernames, a key, and a whitelisted IP req = session.get( get_domain_list_endpoint.format( - namecheap_api_username, - namecheap_api_key, - namecheap_username, - client_ip, - namecheap_page_size, + namecheap_config.api_username, + namecheap_config.api_key, + namecheap_config.username, + namecheap_config.client_ip, + namecheap_config.page_size, ) ) # Check if request returned a 200 OK @@ -784,35 +845,43 @@ def fetch_namecheap_domains(): for domain in root.CommandResponse.DomainGetListResult.Domain: domains_list.append(domain.attrib) elif namecheap_api_result == "ERROR": - error_message = 'Namecheap returned an "ERROR" response, so no domains were returned.' - if "Invalid request IP" in req.text: - error_message = ( - "[!] " - + error_message - + "\n" - + "L.. You are not connecting to Namecheap using your whitelisted IP address." - ) - logger.error(error_message + "\nFull Response:\n%s", req.text) - return "[!] " + error_message + "\nFull Response:\n{}".format(req.text) + error_id = root.Errors[0].Error[0].attrib["Number"] + error_msg = root.Errors[0].Error[0].text + logger.error( + "Namecheap API returned error #%s: %s", error_id, error_msg + ) + domain_changes["errors"][ + "namecheap" + ] = f"Namecheap API returned error #{error_id}: {error_msg} (see https://www.namecheap.com/support/api/error-codes/)" + return domain_changes else: logger.error( - 'Namecheap did not return an "OK" response, so no domains were returned.\nFull Response:\n%s', - req.text, + 'Namecheap did not return an "OK" or "ERROR" response: %s', req.text ) - return '[!] Namecheap did not return an "OK" response, so no domains were returned.\nFull Response:\n{}'.format( - req.text + domain_changes["errors"][ + "namecheap" + ] = 'Namecheap did not return an "OK" or "ERROR" response: {response}'.format( + response=req.text ) + return domain_changes else: logger.error( - 'Namecheap API request failed. Namecheap did not return a 200 response.\nL.. API request returned status "%s"', - req.status_code, + "Namecheap returned a %s response: %s", req.status_code, req.text ) - return '[!] Namecheap API request failed. Namecheap did not return a 200 response.\nL.. API request returned status "{}"'.format( - req.status_code + domain_changes["errors"][ + "namecheap" + ] = "Namecheap returned a {status_code} response: {text}".format( + status_code=req.status_code, text=req.text ) - except Exception as error: - logger.error("Namecheap API request failed with error: %s", error) - return "Namecheap API request failed with error: {}".format(error) + return domain_changes + except Exception: + trace = traceback.format_exc() + logger.exception("Namecheap API request failed") + domain_changes["errors"][ + "namecheap" + ] = "The Namecheap API request failed: {traceback}".format(traceback=trace) + return domain_changes + # There's a chance no domains are returned if the provided usernames don't have any domains if domains_list: # Get the current list of Namecheap domains in the library @@ -844,7 +913,16 @@ def fetch_namecheap_domains(): for attr, value in entry.items(): setattr(domain, attr, value) domain.save() - except Exception as e: + except Exception: + trace = traceback.format_exc() + domain_changes["errors"][ + domain + ] = "Failed to update the entry for {domain}: {traceback}".format( + domain=domain, traceback=trace + ) + logger.exception( + "Failed to update the entry for %s", domain.name + ) pass instance = DomainNote.objects.create( domain=domain, @@ -852,48 +930,52 @@ def fetch_namecheap_domains(): ) # Now, loop over every domain returned by Namecheap for domain in domains_list: - logger.info("Domain {} is now being processed".format(domain["Name"])) + logger.info("Domain %s is now being processed", domain["Name"]) + # Prepare domain attributes for Domain model entry = {} entry["name"] = domain["Name"] entry["registrar"] = "Namecheap" + # Set the WHOIS status based on WhoisGuard if domain["IsExpired"] == "true": entry["expired"] = True - # Expired domains have WhoisGuard set to `NOTPRESENT` + # Expired domains have WhoisGuard set to ``NOTPRESENT`` entry["whois_status"] = WhoisStatus.objects.get(pk=2) else: try: entry["whois_status"] = WhoisStatus.objects.get( whois_status__iexact=domain["WhoisGuard"].capitalize() ) - # Anything not `Enabled` or `Disabled`, set to `Unknown` - except: + # Anything not ``Enabled`` or ``Disabled``, set to ``Unknown`` + except Exception: + logger.exception( + "Namecheap WHOIS status (%s) was not found in the database, so defaulted to `Unknown`", + domain["WhoisGuard"].capitalize(), + ) entry["whois_status"] = WhoisStatus.objects.get(pk=3) + # Check if the domain is locked - locked generally means it's burned newly_burned = False if domain["IsLocked"] == "true": logger.warning( "Domain %s is marked as LOCKED by Namecheap", domain["Name"] ) - domain_instance = Domain.objects.get(name=domain["Name"]) - health_burned = HealthStatus.objects.get(health_status="Burned") - domain_burned = DomainStatus.objects.get(domain_status="Burned") - # Even if already set to Burned, add some explanation if missing - if not domain_instance.burned_explanation: - entry[ - "burned_explanation" - ] = "

    Namecheap has locked the domain. This is usually the result of a legal complaint related to phishing/malicious activities.

    " - # Update statuses if set to something else - if not domain_instance.health_status == health_burned: - newly_burned = True - entry["health_status"] = health_burned - if not domain_instance.domain_status == domain_burned: - newly_burned = True - entry["domain_status"] = domain_burned + newly_burned = True + entry["health_status"] = HealthStatus.objects.get( + health_status="Burned" + ) + entry["domain_status"] = DomainStatus.objects.get( + domain_status="Burned" + ) + entry[ + "burned_explanation" + ] = "

    Namecheap has locked the domain. This is usually the result of a legal complaint related to phishing/malicious activities.

    " + # Set AutoRenew status if domain["AutoRenew"] == "false": entry["auto_renew"] = False + # Convert Namecheap dates to Django entry["creation"] = datetime.datetime.strptime( domain["Created"], "%m/%d/%Y" @@ -901,18 +983,20 @@ def fetch_namecheap_domains(): entry["expiration"] = datetime.datetime.strptime( domain["Expires"], "%m/%d/%Y" ).strftime("%Y-%m-%d") + + # Update or create the domain record with assigned attrs try: - # Update or create the domain record with assigned attrs instance, created = Domain.objects.update_or_create( name=domain.get("Name"), defaults=entry ) for attr, value in entry.items(): setattr(instance, attr, value) + logger.debug( "Domain %s is being saved with this data: %s", domain["Name"], entry ) instance.save() - # Update the change tracking dict + # Add entry to domain change tracking dict domain_changes["updates"][instance.id] = {} domain_changes["updates"][instance.id]["domain"] = domain["Name"] @@ -927,22 +1011,36 @@ def fetch_namecheap_domains(): domain_changes["updates"][instance.id]["change"] = "burned" else: domain_changes["updates"][instance.id]["change"] = "updated" - except Exception as e: + except Exception: + trace = traceback.format_exc() + logger.exception( + "Encountered an exception while trying to create or update %s", + domain["Name"], + ) domain_changes["errors"][domain["Name"]] = {} - domain_changes["errors"][domain["Name"]]["error"] = "{}".format(e) + domain_changes["errors"][domain["Name"]]["error"] = trace logger.info( "Namecheap synchronization completed at %s with these changes:\n%s", datetime.datetime.now(), domain_changes, ) - return domain_changes else: logger.warning("No domains were returned for the provided Namecheap account!") - return "[!] No domains were returned for the provided Namecheap account!" + + return domain_changes def months_between(date1, date2): - """Compare two dates and return the number of months beetween them.""" + """ + Compare two dates and return the number of months beetween them. + + **Parameters** + + ``date1`` + First date for the comparison + ``date2`` + Second date for the comparison + """ if date1 > date2: date1, date2 = date2, date1 m1 = date1.year * 12 + date1.month @@ -958,14 +1056,23 @@ def months_between(date1, date2): return months -def json_datetime_converter(o): - """Function to convert datetime objects to strings for json.dumps().""" - if isinstance(o, datetime.datetime): - return o.__str__() +def json_datetime_converter(dt): + """ + Convert datetime objects to strings for json.dumps(). + + **Parameters** + + ``dt`` + Datetime object to convert to a string + + """ + if isinstance(dt, datetime.datetime): + return dt.__str__() def review_cloud_infrastructure(): - """Fetch active virtual machines/instances in Digital Ocean, Azure, and AWS and + """ + Fetch active virtual machines/instances in Digital Ocean, Azure, and AWS and compare IP addresses to project infrastructure. Send a report to Slack if any instances are still alive after project end date or if an IP address is not found for a project. @@ -974,22 +1081,16 @@ def review_cloud_infrastructure(): """ # Digital Ocean API endpoint for droplets DIGITAL_OCEAN_ENDPOINT = "https://api.digitalocean.com/v2/droplets" + # Fetch cloud API keys and tokens - aws_key = settings.CLOUD_SERVICE_CONFIG["aws_key"] - aws_secret = settings.CLOUD_SERVICE_CONFIG["aws_secret"] - do_api_key = settings.CLOUD_SERVICE_CONFIG["do_api_key"] + cloud_config = CloudServicesConfiguration.get_solo() # Fetch Slack configuration information - slack_config = get_slack_config() - if slack_config: - slack_emoji = slack_config["slack_emoji"] - slack_username = slack_config["slack_username"] - slack_default_channel = slack_config["slack_channel"] - slack_webhook_url = slack_config["slack_webhook_url"] - slack_alert_target = slack_config["slack_alert_target"] + slack_config = SlackConfiguration.get_solo() # Set timezone for dates to UTC utc = pytz.UTC + # Create info dict vps_info = defaultdict() vps_info["errors"] = {} @@ -1005,20 +1106,28 @@ def review_cloud_infrastructure(): # Create AWS client for EC2 using a default region and get a list of all regions aws_capable = True + regions = [] try: client = boto3.client( "ec2", region_name="us-west-2", - aws_access_key_id=aws_key, - aws_secret_access_key=aws_secret, + aws_access_key_id=cloud_config.aws_key, + aws_secret_access_key=cloud_config.aws_secret, ) regions = [ region["RegionName"] for region in client.describe_regions()["Regions"] ] - except ClientError as e: + except ClientError: logger.error("AWS could not validate the provided credentials") aws_capable = False - vps_info["errors"]["aws"] = "AWS could not validate the provided credentials." + vps_info["errors"]["aws"] = "AWS could not validate the provided credentials" + except Exception: + trace = traceback.format_exc() + logger.exception("Testing authentication to AWS failed") + aws_capable = False + vps_info["errors"][ + "aws" + ] = "Testing authentication to AWS failed: {traceback}".format(traceback=trace) if aws_capable: logger.info("AWS credentials are functional, beginning AWS review") # Loop over the regions to check each one for EC2 instances @@ -1028,8 +1137,8 @@ def review_cloud_infrastructure(): ec2 = boto3.resource( "ec2", region_name=region, - aws_access_key_id=aws_key, - aws_secret_access_key=aws_secret, + aws_access_key_id=cloud_config.aws_key, + aws_secret_access_key=cloud_config.aws_secret, ) # Get all EC2 instances that are running running_instances = ec2.instances.filter( @@ -1053,9 +1162,9 @@ def review_cloud_infrastructure(): else: tags.append("{}: {}".format(tag["Key"], tag["Value"])) pub_addresses = [] - pub_addresses.append(instance.private_ip_address) + pub_addresses.append(instance.public_ip_address) priv_addresses = [] - priv_addresses.append(instance.public_ip_address) + priv_addresses.append(instance.private_ip_address) # Add instance info to a dictionary vps_info["instances"][instance.id] = { "id": instance.id, @@ -1077,25 +1186,57 @@ def review_cloud_infrastructure(): ############### # Get all Digital Ocean droplets for the account + active_droplets = {} do_capable = True headers = {"Content-Type": "application/json"} try: active_droplets = requests.get( - DIGITAL_OCEAN_ENDPOINT, headers=headers, auth=BearerAuth(do_api_key) - ).json() - logger.info( - "Digital Ocean credentials are functional, beginning droplet review" + DIGITAL_OCEAN_ENDPOINT, + headers=headers, + auth=BearerAuth(cloud_config.do_api_key), ) - except: - logger.error( + if active_droplets.status_code == 200: + active_droplets = active_droplets.json() + logger.info( + "Digital Ocean credentials are functional, beginning droplet review" + ) + else: + do_capable = False + logger.info( + "Digital Ocean denied access with HTTP code %s and this message: %s", + active_droplets.status_code, + active_droplets.text, + ) + try: + error_message = active_droplets.json() + api_response = error_message + if "message" in error_message: + api_response = error_message["message"] + except ValueError: + api_response = active_droplets.text + vps_info["errors"][ + "digital_ocean" + ] = f"Digital Ocean API request failed with this response: {api_response}" + # Catch a JSON decoding error with the response + except ValueError: + logger.exception("Could not decode the response from Digital Ocean") + vps_info["errors"][ + "digital_ocean" + ] = f"Could not decode this response from Digital Ocean: {active_droplets.text}" + # Catch any other errors related to the web request + except Exception: + trace = traceback.format_exc() + logger.exception( "Could not retrieve content from Digital Ocean with the provided API key" ) do_capable = False vps_info["errors"][ "digital_ocean" - ] = "Could not retrieve content from Digital Ocean with the provided API key." + ] = "Could not retrieve content from Digital Ocean with the provided API key: {traceback}".format( + traceback=trace + ) # Loop over the droplets to generate the info dict - if do_capable: + if do_capable and "droplets" in active_droplets: for droplet in active_droplets["droplets"]: # Get the networking info if "v4" in droplet["networks"]: @@ -1174,39 +1315,41 @@ def review_cloud_infrastructure(): if queryset: for result in queryset: if result.project.end_date < instance["launch_time"].date(): - if slack_config: + if slack_config.enable: if result.project.slack_channel: slack_data = craft_cloud_message( - slack_username, - slack_emoji, - slack_default_channel, + slack_config.slack_username, + slack_config.slack_emoji, + result.project.slack_channel, instance["launch_time"], result.project, result.project.end_date, instance["provider"], instance_name, + ", ".join(pub_addresses), instance["tags"], ) # send_slack_msg(message, slack_channel=result.project.slack_channel) response = requests.post( - slack_webhook_url, + slack_config.webhook_url, data=slack_data, headers={"Content-Type": "application/json"}, ) else: slack_data = craft_cloud_message( - slack_username, - slack_emoji, - slack_default_channel, + slack_config.slack_username, + slack_config.slack_emoji, + slack_config.slack_channel, instance["launch_time"], result.project, result.project.end_date, instance["provider"], instance_name, + ", ".join(pub_addresses), instance["tags"], ) response = requests.post( - slack_webhook_url, + slack_config.webhook_url, data=slack_data, headers={"Content-Type": "application/json"}, ) @@ -1214,27 +1357,36 @@ def review_cloud_infrastructure(): # Project is still active, so track these assets for later assets_in_use.append(instance_id) else: - if "gw_ignore" in instance["tags"]: + ignore_tags = [] + instance_tags = [] + for tag in cloud_config.ignore_tag.split(","): + ignore_tags.append(tag.strip()) + for tag in instance["tags"].split(","): + instance_tags.append(tag.strip()) + if any(tag in ignore_tags for tag in instance_tags): + logger.info( + "Ignoring %s because it is tagged with a configured ignore tag", + instance_name, + ) assets_in_use.append(instance_id) else: - if slack_config: + if slack_config.enable: slack_data = craft_unknown_asset_message( - slack_username, - slack_emoji, - slack_default_channel, + slack_config.slack_username, + slack_config.slack_emoji, + slack_config.slack_channel, instance["launch_time"], instance["provider"], instance_name, + pub_addresses, instance["tags"], ) response = requests.post( - slack_webhook_url, + slack_config.webhook_url, data=slack_data, headers={"Content-Type": "application/json"}, ) - # Drop active assets from the dict - for instance_id in assets_in_use: - del vps_info[instance_id] + # Return the stale cloud asset data in JSON for the task results json_data = json.dumps(dict(vps_info), default=json_datetime_converter, indent=2) logger.info("Cloud review completed at %s", datetime.datetime.now()) @@ -1243,17 +1395,24 @@ def review_cloud_infrastructure(): def check_expiration(): - """Fetch all domains from the library and check the expiration dates. If - the expiration date is less than or equal to the current date, check the - auto-renew status. Then either expire the domain or add one year to the - expiration date. """ - domain_queryset = Domain.objects.all() + Update expiration statuses for all :model:`shepherd.Domain`. + """ + domain_changes = {} + domain_changes["errors"] = {} + domain_changes["updates"] = {} expired_status = DomainStatus.objects.get(domain_status="Expired") + domain_queryset = Domain.objects.filter(~Q(domain_status=expired_status)) for domain in domain_queryset: + logger.info("Checking %s", domain) + domain_changes["updates"][domain.id] = {} + domain_changes["updates"][domain.id]["change"] = "no change" if domain.expiration <= date.today(): + domain_changes["updates"][domain.id]["domain"] = domain.name + if domain.auto_renew: logger.info("Adding one year to %s's expiration date", domain.name) + domain_changes["updates"][domain.id]["change"] = "auto-renewed" domain.expiration = domain.expiration + datetime.timedelta(days=365) domain.expired = False domain.save() @@ -1263,6 +1422,332 @@ def check_expiration(): domain.name, domain.expiration, ) + domain_changes["updates"][domain.id]["change"] = "expired" domain.expired = True domain.domain_status = expired_status domain.save() + + logger.info("Domain expiration update completed at %s", datetime.datetime.now()) + return domain_changes + + +def test_aws_keys(user): + """ + Test the AWS access keys configured in :model:`commandcenter.CloudServicesConfiguration`. + """ + cloud_config = CloudServicesConfiguration.get_solo() + level = "error" + logger.info("Starting a test of the AWS keys at %s", datetime.datetime.now()) + try: + # Send the STS ``get_caller_identity`` API call to test keys + client = boto3.client( + "sts", + region_name="us-west-2", + aws_access_key_id=cloud_config.aws_key, + aws_secret_access_key=cloud_config.aws_secret, + ) + client.get_caller_identity() + logger.info("Successfully verified the AWS keys") + message = "Successfully verified the AWS keys" + level = "success" + except ClientError: + logger.error("AWS could not validate the provided credentials") + message = "AWS could not validate the provided credentials" + except Exception: + logger.exception("Testing authentication to AWS failed") + message = "Testing authentication to AWS failed" + + # Send a message to the requesting user + async_to_sync(channel_layer.group_send)( + "notify_{}".format(user), + { + "type": "message", + "message": { + "message": message, + "level": level, + "title": "AWS Test Complete", + }, + }, + ) + + logger.info("Test of the AWS access keys completed at %s", datetime.datetime.now()) + return {"result": level, "message": message} + + +def test_digital_ocean(user): + """ + Test the Digital Ocean API key configured in + :model:`commandcenter.CloudServicesConfiguration`. + """ + DIGITAL_OCEAN_ENDPOINT = "https://api.digitalocean.com/v2/droplets" + cloud_config = CloudServicesConfiguration.get_solo() + level = "error" + logger.info( + "Starting a test of the Digital Ocean API key at %s", datetime.datetime.now() + ) + try: + # Request all active droplets (as done in the real task) + headers = {"Content-Type": "application/json"} + active_droplets = requests.get( + DIGITAL_OCEAN_ENDPOINT, + headers=headers, + auth=BearerAuth(cloud_config.do_api_key), + ) + if active_droplets.status_code == 200: + logger.info( + "Digital Ocean credentials are functional, beginning droplet review" + ) + logger.info("Successfully verified the Digital Ocean API key") + message = "Successfully verified the Digital Ocean API key" + level = "success" + else: + logger.info( + "Digital Ocean denied access with HTTP code %s and this message: %s", + active_droplets.status_code, + active_droplets.text, + ) + api_response = active_droplets.text + try: + error_message = active_droplets.json() + if "message" in error_message: + api_response = error_message["message"] + except ValueError: + pass + message = f"Digital Ocean denied access with HTTP code {active_droplets.status_code} and this message: {api_response}" + except ClientError: + logger.error("Digital Ocean could not validate the provided API key") + message = "Digital Ocean could not validate the provided API key" + except Exception: + logger.exception("Testing authentication to Digital Ocean failed") + message = "Testing authentication to Digital Ocean failed" + + # Send a message to the requesting user + async_to_sync(channel_layer.group_send)( + "notify_{}".format(user), + { + "type": "message", + "message": { + "message": message, + "level": level, + "title": "Digital Ocean Test Complete", + }, + }, + ) + + logger.info( + "Test of the Digital Ocean API key completed at %s", datetime.datetime.now() + ) + return {"result": level, "message": message} + + +def test_namecheap(user): + """ + Test the Namecheap API configuration stored in :model:`commandcenter.NamecheapConfiguration`. + """ + session = requests.Session() + get_domain_list_endpoint = "https://api.namecheap.com/xml.response?ApiUser={}&ApiKey={}&UserName={}&Command=namecheap.domains.getList&ClientIp={}&PageSize={}" + namecheap_config = NamecheapConfiguration.get_solo() + level = "error" + logger.info("Starting Namecheap API test at %s", datetime.datetime.now()) + try: + # The Namecheap API call requires both usernames, a key, and a whitelisted IP + req = session.get( + get_domain_list_endpoint.format( + namecheap_config.api_username, + namecheap_config.api_key, + namecheap_config.username, + namecheap_config.client_ip, + namecheap_config.page_size, + ) + ) + # Check if request returned a 200 OK + if req.ok: + # Convert Namecheap XML into an easy to use object for iteration + root = objectify.fromstring(req.content) + # Check the status to make sure it says "OK" + namecheap_api_result = root.attrib["Status"] + if namecheap_api_result == "OK": + level = "success" + message = "Successfully authenticated to Namecheap" + elif namecheap_api_result == "ERROR": + error_id = root.Errors[0].Error[0].attrib["Number"] + error_msg = root.Errors[0].Error[0].text + logger.error( + "Namecheap API returned error #%s: %s", error_id, error_msg + ) + message = f"Namecheap API returned error #{error_id}: {error_msg} (see https://www.namecheap.com/support/api/error-codes/)" + else: + logger.error( + "Namecheap did not return an response: %s", + namecheap_api_result, + req.text, + ) + message = ( + "Namecheap returned a {namecheap_api_result} response: {req.text}" + ) + else: + logger.error( + "Namecheap returned HTTP code %s in its response: %s", + req.status_code, + req.text, + ) + message = f"Namecheap returned HTTP code {req.status_code} in its response: {req.text}" + except Exception: + trace = traceback.format_exc() + logger.exception("Namecheap API request failed") + message = f"The Namecheap API request failed: {trace}" + + # Send a message to the requesting user + async_to_sync(channel_layer.group_send)( + "notify_{}".format(user), + { + "type": "message", + "message": { + "message": message, + "level": level, + "title": "Namecheap Test Complete", + }, + }, + ) + + logger.info( + "Test of the Namecheap API configuration completed at %s", + datetime.datetime.now(), + ) + return {"result": level, "message": message} + + +def test_slack_webhook(user): + """ + Test the Slack Webhook configuration stored in :model:`commandcenter.SlackConfiguration`. + """ + slack_config = SlackConfiguration.get_solo() + level = "error" + logger.info("Starting Slack Webhook test at %s", datetime.datetime.now()) + try: + if slack_config.enable: + slack_data = { + "username": slack_config.slack_username, + "icon_emoji": slack_config.slack_emoji, + "channel": slack_config.slack_channel, + "text": f"{slack_config.slack_alert_target} Hello from Ghostwriter :wave:", + } + response = requests.post( + slack_config.webhook_url, + data=json.dumps(slack_data), + headers={"Content-Type": "application/json"}, + ) + if response.ok: + level = "success" + message = f"Slack accepted the request and you should see a message posted in {slack_config.slack_channel}" + elif response.status_code == 400: + message = f"Slack accepted the request, but said the user {slack_config.slack_channel} does not exist" + elif response.status_code == 403: + if "invalid_token" in response.text: + message = "Slack accepted the request, but said your Webhook token is invalid" + elif "action_prohibited" in response.text: + message = f"Slack accepted the request, but said your Webhook token cannot send messages to {slack_config.slack_channel}, or is otherwise restricted" + else: + message = f"Slack accepted the request, but said your Webhook token is not permitted to send messages" + elif response.status_code == 404: + if "channel_not_found" in response.text: + message = f"Slack accepted the request, but said it could not find the {slack_config.slack_channel} channel" + else: + message = f"Slack accepted the request, but said it could not find the {slack_config.slack_channel} channel" + elif response.status_code == 410: + if "channel_is_archived" in response.text: + message = f"Slack accepted the request, but said the {slack_config.slack_channel} channel is archived" + else: + message = f"Slack accepted the request, but said the {slack_config.slack_channel} channel is unavailable - possibly archived?" + else: + logger.warning( + "Request to Slack returned HTTP code %s with this message: %s", + response.status_code, + response.text, + ) + message = f"Request to Slack returned HTTP code {response.status_code} with this message: {response.text}" + else: + logger.warning( + "Received request to send Slack message, but Slack notifications are disabled in settings" + ) + message = "Received request to send Slack message, but Slack notifications are disabled in settings" + except Exception: + trace = traceback.format_exc() + logger.exception("Slack Webhook API request failed") + message = f"Slack Webhook API request failed: {trace}" + + # Send a message to the requesting user + async_to_sync(channel_layer.group_send)( + "notify_{}".format(user), + { + "type": "message", + "message": { + "message": message, + "level": level, + "title": "Slack Test Complete", + }, + }, + ) + + logger.info("Test of the Slack Webhook completed at %s", datetime.datetime.now()) + return {"result": level, "message": message} + + +def test_virustotal(user): + """ + Test the VirusTotal API key stored in :model:`commandcenter.VirusTotalConfiguration`. + """ + virustotal_config = VirusTotalConfiguration.get_solo() + level = "error" + logger.info("Starting VirusTotal API test at %s", datetime.datetime.now()) + try: + if virustotal_config.enable: + virustotal_url = "https://www.virustotal.com/vtapi/v2/url/report" + params = {"apikey": virustotal_config.api_key, "resource": "google.com"} + response = requests.get(virustotal_url, params=params) + logger.info(response.status_code) + + if response.ok: + message = "Successfully authenticated to the VirusTotal API" + level = "success" + logger.info("VT Test Succeeded") + elif response.status_code == 204: + message = "Successfully authenticated to the VirusTotal API, but response indicated the API key has hit its rate limit for now" + level = "warning" + elif response.status_code == 400: + message = "VirusTotal did not accept the API request (Bad Request)" + elif response.status_code == 403: + message = "Successfully authenticated to the VirusTotal API, but response indicated the configured key is restricted" + level = "warning" + else: + logger.warning( + "Request to VirusTotal API returned HTTP code %s with this message: %s", + response.status_code, + response.text, + ) + message = f"Request to VirusTotal API returned HTTP code {response.status_code} with this message: {response.text}" + else: + logger.warning( + "Received request to test the VirusTotal API key, but VirusTotal is disabled in settings" + ) + message = "Received request to test the VirusTotal API key, but VirusTotal is disabled in settings" + except Exception: + trace = traceback.format_exc() + logger.exception("VirusTotal API request failed") + message = f"VirusTotal API request failed: {trace}" + + # Send a message to the requesting user + async_to_sync(channel_layer.group_send)( + "notify_{}".format(user), + { + "type": "message", + "message": { + "message": message, + "level": level, + "title": "VirusTotal Test Complete", + }, + }, + ) + + logger.info("Test of the VirusTotal completed at %s", datetime.datetime.now()) + return {"result": level, "message": message} diff --git a/ghostwriter/shepherd/templates/shepherd/checkout.html b/ghostwriter/shepherd/templates/shepherd/checkout.html index b9d8ff012..0302aa095 100644 --- a/ghostwriter/shepherd/templates/shepherd/checkout.html +++ b/ghostwriter/shepherd/templates/shepherd/checkout.html @@ -36,6 +36,39 @@ {% endblock %} {% block morescripts %} + + {% if domain.health_status.health_status == "Burned" %} + + {% endif %} + + {% if domain.is_expiring_soon %} + + {% endif %} + + + + + + {% endblock %} diff --git a/ghostwriter/shepherd/templates/shepherd/server_checkout.html b/ghostwriter/shepherd/templates/shepherd/server_checkout.html index 0c2c0a03f..0cbcb5da4 100644 --- a/ghostwriter/shepherd/templates/shepherd/server_checkout.html +++ b/ghostwriter/shepherd/templates/shepherd/server_checkout.html @@ -8,7 +8,7 @@     diff --git a/ghostwriter/shepherd/templates/shepherd/update.html b/ghostwriter/shepherd/templates/shepherd/update.html index 981028510..1546bd94c 100644 --- a/ghostwriter/shepherd/templates/shepherd/update.html +++ b/ghostwriter/shepherd/templates/shepherd/update.html @@ -55,6 +55,8 @@
    Pull Domains from Namecheap
    {% endif %} + +
    Domain Categories
    @@ -82,7 +84,7 @@
    Domain Categories
    {% endif %} {% endif %} -

    Note that updates will require at least {{ update_time }} minutes ({{ total_domains }} domains * {{ sleep_time }} second sleep configured in settings).

    +

    Note that updates will require at least {{ update_time }} minutes ({{ total_domains }} non-expired domains * {{ sleep_time }} second sleep configured in settings).

    {% csrf_token %} @@ -90,6 +92,8 @@
    Domain Categories
    + +
    DNS Records
    @@ -118,14 +122,14 @@
    DNS Records
    {% endif %} {% endif %} -

    An individual update may take up to 60 seconds, so expect this to take some time.

    -
    {% csrf_token %}
    + + {% if enable_cloud_monitor %}
    Cloud Infrastructure
    @@ -165,7 +169,6 @@
    Cloud Infrastructure
    {% endblock %} - {% block morescripts %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ghostwriter/shepherd/urls.py b/ghostwriter/shepherd/urls.py index 804474e83..ebe2a11f6 100644 --- a/ghostwriter/shepherd/urls.py +++ b/ghostwriter/shepherd/urls.py @@ -3,6 +3,7 @@ # Django & Other 3rd Party Libraries from django.urls import path +# Ghostwriter Libraries from . import views app_name = "shepherd" @@ -85,6 +86,16 @@ views.update_domain_badges, name="ajax_update_domain_badges", ), + path( + "ajax/domain/overwatch", + views.ajax_domain_overwatch, + name="ajax_domain_overwatch", + ), + path( + "ajax/project//domains", + views.ajax_project_domains, + name="ajax_project_domains", + ), ] # URLs for domain status change functions diff --git a/ghostwriter/shepherd/views.py b/ghostwriter/shepherd/views.py index 0bdcd5b29..100df0c05 100644 --- a/ghostwriter/shepherd/views.py +++ b/ghostwriter/shepherd/views.py @@ -7,7 +7,6 @@ # Django & Other 3rd Party Libraries from django import forms -from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin @@ -18,13 +17,18 @@ from django.shortcuts import get_object_or_404, render from django.template.loader import render_to_string from django.urls import reverse, reverse_lazy -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.generic.detail import DetailView, SingleObjectMixin from django.views.generic.edit import CreateView, DeleteView, UpdateView, View from django_q.models import Task from django_q.tasks import async_task # Ghostwriter Libraries +from ghostwriter.commandcenter.models import ( + CloudServicesConfiguration, + NamecheapConfiguration, + VirusTotalConfiguration, +) from ghostwriter.rolodex.models import Project from .filters import DomainFilter, ServerFilter @@ -73,7 +77,8 @@ def update_domain_badges(request, pk): """ domain_instance = get_object_or_404(Domain, pk=pk) html = render_to_string( - "snippets/domain_nav_tabs.html", {"domain": domain_instance}, + "snippets/domain_nav_tabs.html", + {"domain": domain_instance}, ) return HttpResponse(html) @@ -94,6 +99,7 @@ def ajax_load_projects(request): """ client_id = request.GET.get("client") projects = Project.objects.filter(client_id=client_id).order_by("codename") + return render( request, "shepherd/project_dropdown_list.html", {"projects": projects} ) @@ -115,6 +121,49 @@ def ajax_load_project(request): return HttpResponse(data, content_type="application/json") +@login_required +def ajax_domain_overwatch(request): + """ + Retrieve an individual :model:`shepherd.History` to check domain's past history + prior to checkout. + """ + client_id = None + domain_id = None + try: + client_id = int(request.GET.get("client")) + domain_id = int(request.GET.get("domain")) + except Exception: + logger.exception("Received bad primary key values") + + if client_id and domain_id: + domain_history = History.objects.filter( + Q(domain=domain_id) & Q(client=client_id) + ) + if domain_history: + data = { + "result": "warning", + "message": "Domain has been used with this client in the past", + } + else: + data = {"result": "success", "message": ""} + else: + data = {"result": "error"} + + return JsonResponse(data) + + +@login_required +def ajax_project_domains(request, pk): + """ + Retrieve all :model:`shepherd.History` related to an individual + :model:`rolodex.Project`. + """ + domain_history = History.objects.filter(project=pk) + data = serializers.serialize("json", domain_history, use_natural_foreign_keys=True) + + return HttpResponse(data, content_type="application/json") + + class DomainRelease(LoginRequiredMixin, SingleObjectMixin, View): """ Set the ``domain_status`` field of an individual :model:`shepherd.Domain` to @@ -220,7 +269,7 @@ def post(self, request, *args, **kwargs): group="Domain Updates", hook="ghostwriter.shepherd.tasks.send_slack_complete_msg", ) - message = "Domain category update task (Task ID {task}) has been successfully queued".format( + message = "Successfully queued domain category update task (Task ID {task}) ".format( task=task_id ) except Exception: @@ -267,7 +316,7 @@ def post(self, request, *args, **kwargs): group="DNS Updates", hook="ghostwriter.shepherd.tasks.send_slack_complete_msg", ) - message = "DNS update task (Task ID {task}) has been successfully queued".format( + message = "Successfully queued DNS update task (Task ID {task})".format( task=task_id ) except Exception: @@ -296,8 +345,10 @@ def post(self, request, *args, **kwargs): "ghostwriter.shepherd.tasks.fetch_namecheap_domains", group="Namecheap Update", ) - message = "Namecheap update task (Task ID {task}) has been successfully queued.".format( - task=task_id + message = ( + "Successfully queued Namecheap update task (Task ID {task})".format( + task=task_id + ) ) except Exception: result = "error" @@ -324,8 +375,10 @@ def post(self, request, *args, **kwargs): "ghostwriter.shepherd.tasks.review_cloud_infrastructure", group="Cloud Infrastructure Review", ) - message = "Cloud monitor task (Task ID {task}) has been successfully queued.".format( - task=task_id + message = ( + "Successfully queued the cloud monitor task (Task ID {task})".format( + task=task_id + ) ) except Exception: result = "error" @@ -448,7 +501,7 @@ def domain_list(request): # Check if a search parameter is in the request search_term = "" if "domain_search" in request.GET: - search_term = request.GET.get("domain_search") + search_term = request.GET.get("domain_search").strip() if search_term is None or search_term == "": search_term = "" # If there is a search term, filter the query by domain name or category @@ -655,7 +708,7 @@ def burn(request, pk): **Context** ``form`` - Instance of :form:`shepjerd.BurnForm` + Instance of :form:`shepherd.BurnForm` ``domain_instance`` Instance of :model:`shepherd.Domain` to be updated ``domain_name`` @@ -686,7 +739,7 @@ def burn(request, pk): domain_instance.save() # Redirect to the user's checked-out domains messages.warning( - request, "Domain has been marked as burned.", extra_tags="alert-warning" + request, "Domain has been marked as burned", extra_tags="alert-warning" ) return HttpResponseRedirect( "{}#health".format(reverse("shepherd:domain_detail", kwargs={"pk": pk})) @@ -720,7 +773,7 @@ def update(request): ``update_time`` Calculated time estimate for updating health of all :model:`shepherd.Domain` ``sleep_time`` - The associated value from ``settings.DOMAINCHECK_CONFIG`` + The associated value from :model:`commandcenter.VirusTotalConfiguration` ``cat_last_update_requested`` Start time of latest :model:`django_q.Task` for group "Domain Updates" ``cat_last_update_completed`` @@ -738,7 +791,7 @@ def update(request): ``dns_last_result`` Result of latest :model:`django_q.Task` for group "DNS Updates" ``enable_namecheap`` - The associated value from ``settings.NAMECHEAP_CONFIG`` + The associated value from :model:`commandcenter.NamecheapConfiguration` ``namecheap_last_update_requested`` Start time of latest :model:`django_q.Task` for group "Namecheap Update" ``namecheap_last_update_completed`` @@ -748,7 +801,7 @@ def update(request): ``namecheap_last_result`` Result of latest :model:`django_q.Task` for group "Namecheap Update" ``enable_cloud_monitor`` - The associated value from ``settings.CLOUD_SERVICE_CONFIG`` + The associated value from :model:`commandcenter.CloudServicesConfiguration` ``cloud_last_update_requested`` Start time of latest :model:`django_q.Task` for group "Cloud Infrastructure Review" ``cloud_last_update_completed`` @@ -764,10 +817,26 @@ def update(request): """ # Check if the request is a GET if request.method == "GET": + # Get relevant configuration settings + vt_config = VirusTotalConfiguration.get_solo() + sleep_time = vt_config.sleep_time + cloud_config = CloudServicesConfiguration.get_solo() + enable_cloud_monitor = cloud_config.enable + namecheap_config = NamecheapConfiguration.get_solo() + enable_namecheap = namecheap_config.enable + # Collect data for category updates - total_domains = Domain.objects.all().count() + cat_last_update_completed = "" + cat_last_update_time = "" + cat_last_result = "" + try: + expired_status = DomainStatus.objects.get(domain_status="Expired") + except DomainStatus.DoesNotExist: + expired_status = None + total_domains = ( + Domain.objects.all().exclude(domain_status=expired_status).count() + ) try: - sleep_time = settings.DOMAINCHECK_CONFIG["sleep_time"] update_time = round(total_domains * sleep_time / 60, 2) except Exception: sleep_time = 20 @@ -785,13 +854,13 @@ def update(request): cat_last_update_time = round(queryset.time_taken() / 60, 2) else: cat_last_update_completed = "Failed" - cat_last_update_time = "" except Exception: cat_last_update_requested = "Updates Have Not Been Run Yet" - cat_last_update_completed = "" - cat_last_update_time = "" - cat_last_result = "" + # Collect data for DNS updates + dns_last_update_completed = "" + dns_last_update_time = "" + dns_last_result = "" try: queryset = Task.objects.filter(group="DNS Updates")[0] dns_last_update_requested = queryset.started @@ -801,46 +870,46 @@ def update(request): dns_last_update_time = round(queryset.time_taken() / 60, 2) else: dns_last_update_completed = "Failed" - dns_last_update_time = "" except Exception: dns_last_update_requested = "Updates Have Not Been Run Yet" - dns_last_update_completed = "" - dns_last_update_time = "" - dns_last_result = "" + # Collect data for Namecheap updates - enable_namecheap = settings.NAMECHEAP_CONFIG["enable_namecheap"] - try: - queryset = Task.objects.filter(group="Namecheap Update")[0] - namecheap_last_update_requested = queryset.started - namecheap_last_result = queryset.result - if queryset.success: - namecheap_last_update_completed = queryset.stopped - namecheap_last_update_time = round(queryset.time_taken() / 60, 2) - else: - namecheap_last_update_completed = "Failed" - namecheap_last_update_time = "" - except Exception: - namecheap_last_update_requested = "A Namecheap Update Has Not Been Run Yet" - namecheap_last_update_completed = "" - namecheap_last_update_time = "" - namecheap_last_result = "" + namecheap_last_update_completed = "" + namecheap_last_update_time = "" + namecheap_last_result = "" + if enable_namecheap: + try: + queryset = Task.objects.filter(group="Namecheap Update")[0] + namecheap_last_update_requested = queryset.started + namecheap_last_result = queryset.result + if queryset.success: + namecheap_last_update_completed = queryset.stopped + namecheap_last_update_time = round(queryset.time_taken() / 60, 2) + else: + namecheap_last_update_completed = "Failed" + except Exception: + namecheap_last_update_requested = "Namecheap Sync Has Not Been Run Yet" + else: + namecheap_last_update_requested = "Namecheap Syncing is Disabled" + # Collect data for cloud monitoring - enable_cloud_monitor = settings.CLOUD_SERVICE_CONFIG["enable_cloud_monitor"] - try: - queryset = Task.objects.filter(group="Cloud Infrastructure Review")[0] - cloud_last_update_requested = queryset.started - cloud_last_result = queryset.result - if queryset.success: - cloud_last_update_completed = queryset.stopped - cloud_last_update_time = round(queryset.time_taken() / 60, 2) - else: - cloud_last_update_completed = "Failed" - cloud_last_update_time = "" - except Exception: - cloud_last_update_requested = "A Namecheap Update Has Not Been Run Yet" - cloud_last_update_completed = "" - cloud_last_update_time = "" - cloud_last_result = "" + cloud_last_update_completed = "" + cloud_last_update_time = "" + cloud_last_result = "" + if enable_cloud_monitor: + try: + queryset = Task.objects.filter(group="Cloud Infrastructure Review")[0] + cloud_last_update_requested = queryset.started + cloud_last_result = queryset.result + if queryset.success: + cloud_last_update_completed = queryset.stopped + cloud_last_update_time = round(queryset.time_taken() / 60, 2) + else: + cloud_last_update_completed = "Failed" + except Exception: + cloud_last_update_requested = "Cloud Review Has Not Been Run Yet" + else: + cloud_last_update_requested = "Cloud Services are Disabled" # Assemble context for the page context = { "total_domains": total_domains, @@ -874,7 +943,7 @@ def export_domains_to_csv(request): """ Export all :model:`shepherd.Domain` to a csv file for download. """ - timestamp = datetime.now().isoformat() + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") domain_resource = DomainResource() dataset = domain_resource.export() response = HttpResponse(dataset.csv, content_type="text/csv") @@ -887,7 +956,7 @@ def export_servers_to_csv(request): """ Export all :model:`shepherd.Server` to a csv file for download. """ - timestamp = datetime.now().isoformat() + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") server_resource = StaticServerResource() dataset = server_resource.export() response = HttpResponse(dataset.csv, content_type="text/csv") @@ -952,7 +1021,7 @@ def form_valid(self, form): def get_success_url(self): messages.success( - self.request, "Domain successfully checked-out.", extra_tags="alert-success" + self.request, "Domain successfully checked-out", extra_tags="alert-success" ) return "{}#infrastructure".format( reverse("rolodex:project_detail", kwargs={"pk": self.object.project.pk}) @@ -961,6 +1030,7 @@ def get_success_url(self): def get_context_data(self, **kwargs): ctx = super(HistoryCreate, self).get_context_data(**kwargs) ctx["domain_name"] = self.domain.name.upper() + ctx["domain"] = self.domain ctx["cancel_link"] = reverse( "shepherd:domain_detail", kwargs={"pk": self.kwargs.get("pk")} ) @@ -990,7 +1060,7 @@ class HistoryUpdate(LoginRequiredMixin, UpdateView): def get_success_url(self): messages.success( self.request, - "Domain history successfully updated.", + "Domain history successfully updated", extra_tags="alert-success", ) return "{}#history".format( @@ -1030,7 +1100,7 @@ class HistoryDelete(LoginRequiredMixin, DeleteView): def get_success_url(self): messages.warning( self.request, - "Project history successfully deleted.", + "Project history successfully deleted", extra_tags="alert-warning", ) return "{}#history".format( @@ -1080,7 +1150,7 @@ class DomainCreate(LoginRequiredMixin, CreateView): def get_success_url(self): messages.success( - self.request, "Domain successfully created.", extra_tags="alert-success" + self.request, "Domain successfully created", extra_tags="alert-success" ) return reverse("shepherd:domain_detail", kwargs={"pk": self.object.pk}) @@ -1109,7 +1179,7 @@ class DomainUpdate(LoginRequiredMixin, UpdateView): def get_success_url(self): messages.success( - self.request, "Domain successfully updated.", extra_tags="alert-success" + self.request, "Domain successfully updated", extra_tags="alert-success" ) return reverse("shepherd:domain_detail", kwargs={"pk": self.object.id}) @@ -1144,7 +1214,7 @@ class DomainDelete(LoginRequiredMixin, DeleteView): def get_success_url(self): messages.warning( - self.request, "Domain successfully deleted.", extra_tags="alert-warning" + self.request, "Domain successfully deleted", extra_tags="alert-warning" ) return reverse("shepherd:domains") @@ -1342,7 +1412,7 @@ class ServerDelete(LoginRequiredMixin, DeleteView): def get_success_url(self): messages.warning( - self.request, "Server successfully deleted.", extra_tags="alert-warning" + self.request, "Server successfully deleted", extra_tags="alert-warning" ) return reverse("shepherd:servers") @@ -1394,7 +1464,7 @@ def form_valid(self, form): def get_success_url(self): messages.success( - self.request, "Server successfully checked-out.", extra_tags="alert-success" + self.request, "Server successfully checked-out", extra_tags="alert-success" ) # return reverse('shepherd:user_assets') return "{}#infrastructure".format( @@ -1403,8 +1473,9 @@ def get_success_url(self): def get_context_data(self, **kwargs): ctx = super(ServerHistoryCreate, self).get_context_data(**kwargs) + ctx["server_instance"] = self.server ctx["cancel_link"] = reverse( - "shepherd:server_detail", kwargs={"pk": self.get_object().id} + "shepherd:server_detail", kwargs={"pk": self.kwargs.get("pk")} ) return ctx @@ -1430,7 +1501,7 @@ class ServerHistoryUpdate(LoginRequiredMixin, UpdateView): def get_success_url(self): messages.success( self.request, - "Server history successfully updated.", + "Server history successfully updated", extra_tags="alert-success", ) return "{}#infrastructure".format( @@ -1470,7 +1541,7 @@ class ServerHistoryDelete(LoginRequiredMixin, DeleteView): def get_success_url(self): messages.warning( self.request, - "Server history successfully deleted.", + "Server history successfully deleted", extra_tags="alert-warning", ) return "{}#infrastructure".format( @@ -1511,7 +1582,7 @@ class TransientServerCreate(LoginRequiredMixin, CreateView): def get_success_url(self): messages.success( self.request, - "Server successfully added to the project.", + "Server successfully added to the project", extra_tags="alert-success", ) return "{}#infrastructure".format( @@ -1551,7 +1622,7 @@ class TransientServerUpdate(LoginRequiredMixin, UpdateView): def get_success_url(self): messages.success( self.request, - "Server information successfully updated.", + "Server information successfully updated", extra_tags="alert-success", ) return "{}#infrastructure".format( @@ -1587,7 +1658,7 @@ class DomainServerConnectionCreate(LoginRequiredMixin, CreateView): def get_success_url(self): messages.success( self.request, - "Server successfully associated with domain.", + "Server successfully associated with domain", extra_tags="alert-success", ) return "{}#infrastructure".format( @@ -1634,7 +1705,7 @@ class DomainServerConnectionUpdate(LoginRequiredMixin, UpdateView): def get_success_url(self): messages.success( self.request, - "Connection information successfully updated.", + "Connection information successfully updated", extra_tags="alert-success", ) return "{}#infrastructure".format( @@ -1677,7 +1748,6 @@ class DomainNoteCreate(LoginRequiredMixin, CreateView): template_name = "note_form.html" def get_success_url(self): - """Override the function to return to the new record after creation.""" messages.success( self.request, "Note successfully added to this domain", @@ -1699,6 +1769,13 @@ def get_context_data(self, **kwargs): ) return ctx + def form_valid(self, form, **kwargs): + self.object = form.save(commit=False) + self.object.operator = self.request.user + self.object.domain_id = self.kwargs.get("pk") + self.object.save() + return super().form_valid(form) + class DomainNoteUpdate(LoginRequiredMixin, UpdateView): """ @@ -1777,6 +1854,13 @@ def get_context_data(self, **kwargs): ) return ctx + def form_valid(self, form, **kwargs): + self.object = form.save(commit=False) + self.object.operator = self.request.user + self.object.server_id = self.kwargs.get("pk") + self.object.save() + return super().form_valid(form) + class ServerNoteUpdate(LoginRequiredMixin, UpdateView): """ diff --git a/ghostwriter/singleton/LICENSE b/ghostwriter/singleton/LICENSE new file mode 100644 index 000000000..39a05b51f --- /dev/null +++ b/ghostwriter/singleton/LICENSE @@ -0,0 +1,14 @@ +License and Credits +=================== + +This application is based on code written by [lazybird](https://github.com/lazybird) under their [django-solo](https://github.com/lazybird/django-solo/blob/master/LICENSE) project. + +It was originally licensed under a (Creative Commons Attribution 3.0 Unported[http://creativecommons.org/licenses/by/3.0/]. + +django-solo (c) by lazybird + +django-solo is licensed under a +Creative Commons Attribution 3.0 Unported License. + +You should have received a copy of the license along with this +work. If not, see . diff --git a/ghostwriter/singleton/__init__.py b/ghostwriter/singleton/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ghostwriter/singleton/admin.py b/ghostwriter/singleton/admin.py new file mode 100644 index 000000000..8d378d3cf --- /dev/null +++ b/ghostwriter/singleton/admin.py @@ -0,0 +1,79 @@ +# Django & Other 3rd Party Libraries +from django.conf.urls import url +from django.contrib import admin +from django.http import HttpResponseRedirect +from django.utils.translation import gettext_lazy as _ + +# Ghostwriter Libraries +from .models import DEFAULT_SINGLETON_INSTANCE_ID + +try: + from django.utils.encoding import force_unicode +except ImportError: + from django.utils.encoding import force_text as force_unicode + + +class SingletonModelAdmin(admin.ModelAdmin): + object_history_template = "admin/object_history.html" + change_form_template = "admin/change_form.html" + + def has_add_permission(self, request): + return False + + def has_delete_permission(self, request, obj=None): + return False + + def get_urls(self): + urls = super(SingletonModelAdmin, self).get_urls() + + try: + model_name = self.model._meta.model_name + except AttributeError: + model_name = self.model._meta.module_name.lower() + + self.model._meta.verbose_name_plural = self.model._meta.verbose_name + url_name_prefix = "%(app_name)s_%(model_name)s" % { + "app_name": self.model._meta.app_label, + "model_name": model_name, + } + custom_urls = [ + url( + r"^history/$", + self.admin_site.admin_view(self.history_view), + {"object_id": str(self.singleton_instance_id)}, + name="%s_history" % url_name_prefix, + ), + url( + r"^$", + self.admin_site.admin_view(self.change_view), + {"object_id": str(self.singleton_instance_id)}, + name="%s_change" % url_name_prefix, + ), + ] + # By inserting the custom URLs first, we overwrite the standard URLs. + return custom_urls + urls + + def response_change(self, request, obj): + msg = _("%(obj)s was changed successfully.") % {"obj": force_unicode(obj)} + if "_continue" in request.POST: + self.message_user(request, msg + " " + _("You may edit it again below.")) + return HttpResponseRedirect(request.path) + else: + self.message_user(request, msg) + return HttpResponseRedirect("../../") + + def change_view(self, request, object_id, form_url="", extra_context=None): + if object_id == str(self.singleton_instance_id): + self.model.objects.get_or_create(pk=self.singleton_instance_id) + return super(SingletonModelAdmin, self).change_view( + request, + object_id, + form_url=form_url, + extra_context=extra_context, + ) + + @property + def singleton_instance_id(self): + return getattr( + self.model, "singleton_instance_id", DEFAULT_SINGLETON_INSTANCE_ID + ) diff --git a/ghostwriter/singleton/apps.py b/ghostwriter/singleton/apps.py new file mode 100644 index 000000000..b2208ebb6 --- /dev/null +++ b/ghostwriter/singleton/apps.py @@ -0,0 +1,14 @@ +"""This contains the configuration of the Singleton application.""" + +# Django & Other 3rd Party Libraries +from django.apps import AppConfig + + +class SingletonConfig(AppConfig): + name = "ghostwriter.singleton" + + def ready(self): + try: + import ghostwriter.singleton.signals # noqa F401 + except ImportError: + pass diff --git a/ghostwriter/singleton/models.py b/ghostwriter/singleton/models.py new file mode 100644 index 000000000..54d18b26b --- /dev/null +++ b/ghostwriter/singleton/models.py @@ -0,0 +1,69 @@ +# Django & Other 3rd Party Libraries +from django.conf import settings +from django.db import models + +try: + from django.core.cache import caches + + get_cache = lambda cache_name: caches[cache_name] +except ImportError: + from django.core.cache import get_cache + + +# Default ID for each singleton model +DEFAULT_SINGLETON_INSTANCE_ID = 1 + + +class SingletonModel(models.Model): + """ + Sub-class of ``models.Model`` for models that will only have a single entry. + """ + + singleton_instance_id = DEFAULT_SINGLETON_INSTANCE_ID + + class Meta: + abstract = True + + def save(self, *args, **kwargs): + self.pk = self.singleton_instance_id + super(SingletonModel, self).save(*args, **kwargs) + self.set_to_cache() + + def delete(self, *args, **kwargs): + self.clear_cache() + super(SingletonModel, self).delete(*args, **kwargs) + + def clear_cache(self): + cache_name = getattr(settings, "SOLO_CACHE", settings.SOLO_CACHE) + if cache_name: + cache = get_cache(cache_name) + cache_key = self.get_cache_key() + cache.delete(cache_key) + + def set_to_cache(self): + cache_name = getattr(settings, "SOLO_CACHE", settings.SOLO_CACHE) + if not cache_name: + return None + cache = get_cache(cache_name) + cache_key = self.get_cache_key() + timeout = getattr(settings, "SOLO_CACHE_TIMEOUT", settings.SOLO_CACHE_TIMEOUT) + cache.set(cache_key, self, timeout) + + @classmethod + def get_cache_key(cls): + prefix = getattr(settings, "SOLO_CACHE_PREFIX", settings.SOLO_CACHE_PREFIX) + return "%s:%s" % (prefix, cls.__name__.lower()) + + @classmethod + def get_solo(cls): + cache_name = getattr(settings, "SOLO_CACHE", settings.SOLO_CACHE) + if not cache_name: + obj, created = cls.objects.get_or_create(pk=cls.singleton_instance_id) + return obj + cache = get_cache(cache_name) + cache_key = cls.get_cache_key() + obj = cache.get(cache_key) + if not obj: + obj, created = cls.objects.get_or_create(pk=cls.singleton_instance_id) + obj.set_to_cache() + return obj diff --git a/ghostwriter/singleton/templates/admin/change_form.html b/ghostwriter/singleton/templates/admin/change_form.html new file mode 100644 index 000000000..5f285651a --- /dev/null +++ b/ghostwriter/singleton/templates/admin/change_form.html @@ -0,0 +1,16 @@ +{% extends "admin/change_form.html" %} +{% load i18n %} +{% load admin_urls %} + +{% block breadcrumbs %} +
    + {% trans 'Home' %} › + {{ opts.app_config.verbose_name }} › + {{ opts.verbose_name|capfirst }} +
    +{% endblock %} + +{% block object-tools-items %} +
  • {% trans "History" %}
    • +{% if has_absolute_url %}
  • {% trans "View on site" %}
    • {% endif%} +{% endblock %} diff --git a/ghostwriter/singleton/templates/admin/object_history.html b/ghostwriter/singleton/templates/admin/object_history.html new file mode 100644 index 000000000..f529cc432 --- /dev/null +++ b/ghostwriter/singleton/templates/admin/object_history.html @@ -0,0 +1,12 @@ +{% extends "admin/object_history.html" %} +{% load i18n %} + +{% block breadcrumbs %} +
    +{% trans 'Home' %} › +{{ opts.app_config.verbose_name }} › +{{ object|truncatewords:"18" }} › +{% trans 'History' %} + +
    +{% endblock %} diff --git a/ghostwriter/singleton/templatetags/__init__.py b/ghostwriter/singleton/templatetags/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/ghostwriter/singleton/templatetags/settings_tags.py b/ghostwriter/singleton/templatetags/settings_tags.py new file mode 100644 index 000000000..b459218d9 --- /dev/null +++ b/ghostwriter/singleton/templatetags/settings_tags.py @@ -0,0 +1,35 @@ +# Django & Other 3rd Party Libraries +from django import template +from django.utils.translation import gettext_lazy as _ + +try: + from django.apps import apps + + get_model = apps.get_model +except ImportError: + from django.db.models.loading import get_model + + +register = template.Library() + + +@register.simple_tag() +def get_solo(model_path): + try: + app_label, model_name = model_path.rsplit(".", 1) + except ValueError: + raise template.TemplateSyntaxError( + _( + "Templatetag requires the model dotted path: 'app_label.ModelName'. " + "Received '%s'." % model_path + ) + ) + model_class = get_model(app_label, model_name) + if not model_class: + raise template.TemplateSyntaxError( + _( + "Could not get the model name '%(model)s' from the application " + "named '%(app)s'" % {"model": model_name, "app": app_label,} + ) + ) + return model_class.get_solo() diff --git a/ghostwriter/static/css/styles.css b/ghostwriter/static/css/styles.css index d12dee5db..49d6eb5d5 100644 --- a/ghostwriter/static/css/styles.css +++ b/ghostwriter/static/css/styles.css @@ -1,4 +1,4 @@ -@import "https://fonts.googleapis.com/css?family=Poppins:300,400,500,600,700"; +@import 'https://fonts.googleapis.com/css?family=Poppins:300,400,500,600,700'; /* --------------------------------------------------- GENERAL STYLES @@ -10,29 +10,48 @@ --secondary-color: #2B2C66; --secondary-color-fade: #7386D5; + /* TEXT */ --h1-font-color: #2B2C66; --h2-font-color: #51B071; --main-txt-color: #666; --main-bg-color: #fafafa; - --main-footer-background-color: #51B071; + /* BUTTONS */ --button-primary-color: #51B071; - --button-hover-color: #2B2C66; + --button-hover-color: #6d7fcc; --button-warning-color: red; --button-border-radius: 4px; + /* LINKS */ --active-link-color: #51b071; + --clickable-underline-color: #51B071; + + /* SIDEBAR */ + --sidebar-text-color: #3B454E; + --sidebar-active-text-color: #7386D5; + --sidebar-header-text-color: #fff; + --sidebar-link-text-color: #9EABB7; + + --sidebar-background-color: #F5F7F9; + --sidebar-header-background-color: #2B2C66; + --sidebar-hover-background-color: #E6ECF1; + --sidebar-link-background-color: #9EABB7; - --sidebar-text-color: #fff; - --sidebar-border-color: #47748b; - --sidebar-select-color: #7386D5; - --sidebar-active-color: #51B071; - --sidebar-background-color: #2B2C66; - --sidebar-submenu-active-color: #f5f5f5; - --sidebar-link-background-color: #6d7fcc; + /* SIDEBAR BUTTON */ --sidebar-collapse-button-color: #51B071; --sidebar-collapse-button-text: #fff; + /* NAVBAR */ + --navbar-background-color: #fff; + --navbar-breadcrumb-active-color: #7386D5; + --navbar-breadcrumb-background-color: #f8f9fa; + --navbar-breadcrumb-active-background-color: #f8f9fa; + --navbar-report-selected-font-color: #21612e; + --navbar-report-selected-background-color: #d4eed9; + --navbar-report-empty-font-color: #721c23; + --navbar-report-empty-background-color: #f7d7d9; + + /* ACCORDIONS, COLLAPSIBLES, & DROPDOWNS */ --accordion-primary-color: #2B2C66; --accordion-expanded-color: #51B071; @@ -42,11 +61,16 @@ --dropdown-button-main-color: #2B2C66; --dropdown-button-hover-color: #6d7fcc; + /* SEARCH BOXES */ + --search-text-color: #aaa; + --search-box-shadow: #ccc; + --search-input-background-color: #fcfcfc; + + /* TABLES */ --table-header-color: #2B2C66; --secondary-table-header-color: rgba(170, 170, 170, 0.2); - --clickable-underline-color: #51B071; - + /* CUSTOM TEXT ELEMENTS */ --burned-color: #FF7E79; --healthy-color: #A8D08D; --info-rating-color: #8EAADB; @@ -55,28 +79,15 @@ --high-rating-color: #FF7E79; --crit-rating-color: #966FD6; --neutral-rating-color: #7a7a7a; - - --navbar-background-color: #fff; - --navbar-breadcrumb-active-color: #7386D5; - --navbar-breadcrumb-background-color: #f8f9fa; - --navbar-breadcrumb-active-background-color: #f8f9fa; - --navbar-report-selected-font-color: #21612e; - --navbar-report-selected-background-color: #d4eed9; - --navbar-report-empty-font-color: #721c23; - --navbar-report-empty-background-color: #f7d7d9; - - --search-text-color: #aaa; - --search-box-shadow: #ccc; - --search-input-background-color: #fcfcfc; } body { - font-family: 'Poppins', "Font Awesome 5 Free", sans-serif; + font-family: 'Poppins', 'Font Awesome 5 Free', sans-serif; background: var(--main-bg-color); } p { - font-family: 'Poppins', "Font Awesome 5 Free", sans-serif; + font-family: 'Poppins', 'Font Awesome 5 Free', sans-serif; font-size: var(--main-font-size); font-weight: 300; line-height: 1.7em; @@ -157,6 +168,22 @@ h6 { cursor: pointer; } +.copy-link:before { + font-family: 'Font Awesome 5 Free'; + display: inline-block; + padding-right: 10px; + vertical-align: middle; + font-weight: 600; + content: '\f0c5'; + color: #51B071; + padding-left: 5px; + font-size: 14px; +} + +.copy-link { + cursor: copy; +} + #scroll-button { display: inline-block; background-color: var(--button-primary-color); @@ -182,11 +209,11 @@ h6 { } #scroll-button::before { - font-family: "Font Awesome 5 Free"; - content: "\f062"; + font-family: 'Font Awesome 5 Free'; + content: '\f062'; font-weight: 600; font-style: normal; - text-rendering: normal; + text-rendering: auto; font-size: 2em; line-height: 50px; color: #fff; @@ -202,11 +229,30 @@ h6 { margin: 20px; } +.please-wait-modal { + border: 16px solid var(--secondary-color); + border-top: 16px solid var(--primary-color); + border-radius: 50%; + width: 120px; + height: 120px; + animation: spin 2s linear infinite; +} + +@keyframes spin { + 0% { + transform: rotate(0deg); + } + + 100% { + transform: rotate(360deg); + } +} + /* --------------------------------------------------- FILE UPLOAD STYLES ----------------------------------------------------- */ -input[type="file"] { +input[type='file'] { position: absolute; left: 0; opacity: 0; @@ -278,12 +324,12 @@ input[type="file"] { ----------------------------------------------------- */ .navbar { - padding: 15px 10px; + /* padding: 15px 10px; */ background: var(--navbar-background-color); border: none; border-radius: 0; margin-bottom: 40px; - box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.1); + box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.3); font-size: 15px; } @@ -331,7 +377,7 @@ input[type="file"] { .breadcrumb li { display: inline; - max-width: 200px; + max-width: 185px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; @@ -351,16 +397,14 @@ input[type="file"] { margin-top: 0px; margin-bottom: 5px; margin-left: 20px; - margin-right: 20px; font-weight: 500; - text-align: center; + text-align: left; } .search { position: relative; color: var(--search-text-color); font-size: 16px; - padding-bottom: 10px; } .search input { @@ -404,6 +448,11 @@ input[type="file"] { color: var(--sidebar-text-color); transition: all 0.6s cubic-bezier(0.945, 0.020, 0.270, 0.665); transform-origin: bottom left; + box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.3); +} + +.sidebar-body { + border-top-right-radius: 5px; } #sidebar.active { @@ -412,18 +461,29 @@ input[type="file"] { } #sidebar .sidebar-header { - padding: 10px; - background: var(--sidebar-background-color); + min-width: 255px; + max-width: 255px; + min-height: 100px; + max-height: 100px; + background: var(--sidebar-header-background-color); + color: var(--sidebar-header-text-color); + border-top-right-radius: 5px; + border-bottom-right-radius: 5px; + box-shadow: 1px 1px 3px rgba(0, 0, 0, 0.3); } -#sidebar .sidebar-header p { - margin: 0px; +/* #sidebar .sidebar-header-img { + padding-top: ; +} */ + +#sidebar .components { + padding: 0; + font-weight: 100; } #sidebar ul.components { - padding: 10px 0; - border-bottom: 1px solid var(--sidebar-border-color); - font-weight: 100; + margin-block-start: 0; + margin-block-end: 0.5rem; } #sidebar ul p { @@ -432,32 +492,49 @@ input[type="file"] { } #sidebar ul li a { - padding: 10px; + padding-top: 10px; + padding-bottom: 10px; font-size: 1.1em; display: block; } +#sidebar ul li { + text-align: left; +} + +#sidebar ul li a { + padding-left: 30px; +} + #sidebar ul li a:hover { - color: var(--sidebar-select-color); - background: var(--sidebar-text-color); + background: var(--sidebar-hover-background-color); } #sidebar ul li.active>#sidebar a, -#sidebar a[aria-expanded="true"] { - color: var(--sidebar-text-color); - background: var(--sidebar-active-color); +#sidebar a[aria-expanded='true'] { + color: var(--sidebar-active-text-color); + background: var(--sidebar-header-text-color); + border-radius: 2px; + -webkit-box-shadow: 1px 1px 1px 2px rgba(0, 0, 0, 0.3); + box-shadow: 1px 1px 1px 2px rgba(0, 0, 0, 0.3); } #sidebar form { - width: 100%; + width: 90%; margin: 0; + margin-left: 5%; + padding-bottom: 10px; } -a[data-toggle="collapse"] { +.sidebar-item { + background: var(--sidebar-link-background-color); +} + +a[data-toggle='collapse'] { position: relative; } -.dropdown-toggle[aria-expanded="true"]:after { +.dropdown-toggle[aria-expanded='true']:after { transform: rotate(180deg); } @@ -470,10 +547,15 @@ a[data-toggle="collapse"] { transition: 0.5s; } +#sidebar ul li ul { + -webkit-box-shadow: inset 1px 1px 3px 1px rgba(0, 0, 0, .3); + box-shadow: inset 1px 1px 3px 1px rgba(0, 0, 0, .3); +} + ul ul a { font-size: 0.9em !important; padding-left: 30px !important; - background: var(--sidebar-link-background-color); + color: var(--sidebar-link-text-color); } ul.CTAs { @@ -505,7 +587,7 @@ a.article:hover { #content { width: 100%; - padding: 20px; + padding: 10px 20px 20px 20px; min-height: 100vh; transition: all 0.3s; text-align: center; @@ -558,7 +640,7 @@ h1 { .h1content { text-align: center; - color: white; + color: #fff; font-family: 'Trebuchet MS', 'Lucida Sans Unicode', 'Lucida Grande', 'Lucida Sans', Arial, sans-serif; font-size: 2.5em; } @@ -571,18 +653,6 @@ h2 { font-size: 2.5em } -footer { - position: absolute; - bottom: 0; - width: 100%; - height: 125px; - background-color: var(--main-footer-background-color); - text-align: center; - color: white; - font-family: 'Trebuchet MS', 'Lucida Sans Unicode', 'Lucida Grande', 'Lucida Sans', Arial, sans-serif; - font-size: 1em -} - .center { display: block; margin-left: auto; @@ -606,9 +676,9 @@ footer { } p.severity-box { - color: white; + color: #fff; border-style: solid; - border-color: black; + border-color: #000; border-width: 1px; text-align: center; } @@ -707,7 +777,7 @@ p.severity-box.info { /* Green */ border: none; border-radius: var(--button-border-radius); - color: white; + color: #fff; padding: 15px 32px; text-align: center; text-decoration: none; @@ -721,12 +791,12 @@ p.severity-box.info { .button:hover { background-color: var(--button-hover-color); - color: white; + color: #fff; } .delete:hover { background-color: var(--button-warning-color); - color: white; + color: #fff; } .disabled { @@ -767,7 +837,7 @@ p.severity-box.info { .btn-primary:hover { background-color: var(--button-hover-color); border-color: var(--button-hover-color); - color: white; + color: #fff; } /* --------------------------------------------------- @@ -798,16 +868,38 @@ p .clickable { th { background-color: var(--table-header-color); - color: white; + color: #fff; } th.severity_header { background-color: var(--secondary-table-header-color); - text-align: justified; + text-align: justify; padding-top: 10px; padding-bottom: 10px; border-bottom: none; pointer-events: none; + opacity: 0.8; + color: #fff; +} + +th.severity_header.critical { + background-color: var(--crit-rating-color); +} + +th.severity_header.high { + background-color: var(--high-rating-color); +} + +th.severity_header.medium { + background-color: var(--med-rating-color); +} + +th.severity_header.low { + background-color: var(--low-rating-color); +} + +th.severity_header.informational { + background-color: var(--info-rating-color); } .severity_header:hover { @@ -891,7 +983,7 @@ tr:hover { .form-th { background-color: #fafafa; - color: black; + color: #000; } .form-th:hover { @@ -932,8 +1024,8 @@ td p { } .holdme:before { - font-family: "Font Awesome 5 Free"; - content: "\f7a4"; + font-family: 'Font Awesome 5 Free'; + content: '\f7a4'; font-weight: 900; font-size: 18px; } @@ -952,7 +1044,7 @@ td p { .dropbtn-info { background-color: var(--dropdown-button-main-color); - color: white; + color: #fff; } .dropdown { @@ -981,7 +1073,7 @@ td p { } .dropdown-content a { - color: black; + color: #000; font-size: 12px; padding: 8px 10px; text-align: left; @@ -1021,12 +1113,12 @@ td p { .nav-tabs .nav-link.active { background-color: var(--primary-color); - color: white; + color: #fff; } .nav-tabs .nav-link { background-color: var(--neutral-rating-color); - color: white; + color: #fff; } .nav-link { @@ -1034,7 +1126,7 @@ td p { } .tab-icon:before { - font-family: "Font Awesome 5 Free"; + font-family: 'Font Awesome 5 Free'; display: inline-block; padding-right: 10px; vertical-align: middle; @@ -1048,100 +1140,100 @@ td p { } .search-icon:before { - content: "\f002"; + content: '\f002'; } .pencil-icon:before { - content: "\f303"; + content: '\f303'; } .shield-icon:before { - content: "\f3ed"; + content: '\f3ed'; } .link-icon:before { - content: "\f0c1"; + content: '\f0c1'; } .server-icon:before { - content: "\f233"; + content: '\f233'; } .list-icon:before { - content: "\f0ca"; + content: '\f0ca'; } .file-icon:before { - content: "\f15b"; + content: '\f15b'; } .wireless-icon:before { - content: "\f1eb"; + content: '\f1eb'; } .cloud-icon:before { - content: "\f0c2"; + content: '\f0c2'; } .physical-icon:before { - content: "\f084"; + content: '\f084'; } .network-icon:before { - content: "\f6ff"; + content: '\f6ff'; } .mobile-icon:before { - content: "\f3cd"; + content: '\f3cd'; } .host-icon:before { - content: "\f109"; + content: '\f109'; } .bug-icon:before { - content: "\f188"; + content: '\f188'; } .code-icon:before { - content: "\f121"; + content: '\f121'; } .grip-icon:before { - content: "\f7a4"; + content: '\f7a4'; } .comment-icon:before { - content: "\f27a"; + content: '\f27a'; } .ol-list-icon:before { - content: "\f0cb"; + content: '\f0cb'; } .help-icon:before { - content: "\f4c4"; + content: '\f4c4'; } .impact-icon:before { - content: "\f1e2"; + content: '\f1e2'; } .writing-icon:before { - content: "\f5b7"; + content: '\f5b7'; } .server-icon:before { - content: "\f233"; + content: '\f233'; } .route-icon:before { - content: "\f4d7"; + content: '\f4d7'; } .project-icon:before { - font-family: "Font Awesome 5 Free"; - content: "\f542"; + font-family: 'Font Awesome 5 Free'; + content: '\f542'; display: inline-block; padding-right: 10px; vertical-align: middle; @@ -1149,8 +1241,8 @@ td p { } .assignment-icon:before { - font-family: "Font Awesome 5 Free"; - content: "\f0c0"; + font-family: 'Font Awesome 5 Free'; + content: '\f0c0'; display: inline-block; padding-right: 10px; vertical-align: middle; @@ -1158,8 +1250,8 @@ td p { } .objective-icon:before { - font-family: "Font Awesome 5 Free"; - content: "\f05b"; + font-family: 'Font Awesome 5 Free'; + content: '\f05b'; display: inline-block; padding-right: 10px; vertical-align: middle; @@ -1167,8 +1259,8 @@ td p { } .poc-icon:before { - font-family: "Font Awesome 5 Free"; - content: "\f2bb"; + font-family: 'Font Awesome 5 Free'; + content: '\f2bb'; display: inline-block; padding-right: 10px; vertical-align: middle; @@ -1176,8 +1268,8 @@ td p { } .client-icon:before { - font-family: "Font Awesome 5 Free"; - content: "\f1ad"; + font-family: 'Font Awesome 5 Free'; + content: '\f1ad'; display: inline-block; padding-right: 10px; vertical-align: middle; @@ -1211,7 +1303,7 @@ td p { border: none; border-bottom: 1px solid; border-color: #f5f5f5; - color: white; + color: #fff; cursor: pointer; padding: 18px; width: 100%; @@ -1227,15 +1319,15 @@ td p { } .collapsible:after { - content: "\002B"; - color: white; + content: '\002B'; + color: #fff; font-weight: 600; float: right; margin-left: 5px; } .collapsible.active:after { - content: "\2212"; + content: '\2212'; } .collapsiblecontent { @@ -1269,7 +1361,7 @@ td p { border: none; } -.myaccordion .card { +.myaccordion>.card { padding: 0 0 0 0; border-radius: 5px; overflow: hidden; @@ -1287,7 +1379,7 @@ td p { border: none; border-bottom: 1px solid; border-color: #f5f5f5; - color: white; + color: #fff; cursor: pointer; padding: 10px; text-align: left; @@ -1306,7 +1398,7 @@ td p { .myaccordion .btn { font-weight: 600; - color: white; + color: #fff; padding: 0; } @@ -1316,20 +1408,31 @@ td p { } .myaccordion .card-header:hover, -.card-header[aria-expanded=true] { +.myaccordion .card-header[aria-expanded=true] { background-color: var(--accordion-expanded-color); } -.card-header:after { - content: "\002B"; - color: white; +.myaccordion .card-header:after { + content: '\002B'; + color: #fff; font-weight: 600; float: right; margin-left: 5px; } -.card-header[aria-expanded=true]:after { - content: "\2212"; +h5 a:after { + content: '\002B'; + color: #000; + font-weight: 600; + float: right; +} + +h5 a[aria-expanded=true]:after { + content: '\2212'; +} + +.myaccordion .card-header[aria-expanded=true]:after { + content: '\2212'; } .finding-card ul, @@ -1347,14 +1450,14 @@ ol { .finding-card pre { text-align: justify; - font-family: "Courier New", Courier, monospace; + font-family: 'Courier New', Courier, monospace; padding-left: 20px; white-space: pre-wrap; } .finding-card code { font-size: var(--main-font-size); - font-family: "Courier New", Courier, monospace; + font-family: 'Courier New', Courier, monospace; color: var(--main-txt-color); } @@ -1379,7 +1482,7 @@ ol { } .container::after { - content: ""; + content: ''; clear: both; display: table; } @@ -1508,7 +1611,7 @@ ol { /* height: 100vh; */ grid-template-rows: 70px 1fr 70px; grid-template-columns: 250px 1fr; - grid-template-areas: "header header""nav main""footer footer"; + grid-template-areas: 'header header''nav main''footer footer'; } .admin__main { @@ -1572,7 +1675,7 @@ img { height: auto; } -.card { +.dashboard__item>.card { height: 100%; padding: 1rem; font-size: 2rem; @@ -1582,14 +1685,7 @@ img { transition: all 0.3s cubic-bezier(0.25, 0.8, 0.25, 1); } -.card table { - margin: 0px; - width: 90%; - margin-left: 5%; - border: none; -} - -.card td { +.dashboard__item>.card td { border-bottom: 0; text-align: left; } @@ -1610,7 +1706,7 @@ img { #toast-container>.toast:before { position: fixed; - font-family: "Font Awesome 5 Free"; + font-family: 'Font Awesome 5 Free'; font-size: 24px; line-height: 18px; float: left; @@ -1620,7 +1716,7 @@ img { } #toast-container>.toast-success:before { - content: "\f00c"; + content: '\f00c'; font-weight: 600; color: #2a562a; } @@ -1632,7 +1728,7 @@ img { } #toast-container>.toast-warning:before { - content: "\f071"; + content: '\f071'; font-weight: 600; color: #806620; } @@ -1644,7 +1740,7 @@ img { } #toast-container>.toast-error:before { - content: "\f06a"; + content: '\f06a'; font-weight: 600; color: #692326; } @@ -1656,7 +1752,7 @@ img { } #toast-container>.toast-info:before { - content: "\f129"; + content: '\f129'; font-weight: 600; color: #51768F; } diff --git a/ghostwriter/static/files/template_domain.csv b/ghostwriter/static/files/template_domain.csv deleted file mode 100644 index 8c2ee6922..000000000 --- a/ghostwriter/static/files/template_domain.csv +++ /dev/null @@ -1,2 +0,0 @@ -name,registrar,auto_renew,health_status,health_dns,dns_record,whois_status,creation,expiration,ibm_xforce_cat,talos_cat,bluecoat_cat,fortiguard_cat,opendns_cat,trendmicro_cat,mx_toolbox_status,note,domain_status -specterops.io, SpecterHops Domain Names,Enabled,Healthy, Healthy,127.0.0.1, Enabled,2017-01-26,2019-01-26, Technology, Technology, Uncategorized, Uncategorized, Uncategorized, Uncategorized, No Issues, This is a note, Available \ No newline at end of file diff --git a/ghostwriter/static/files/template_findings.csv b/ghostwriter/static/files/template_findings.csv deleted file mode 100644 index df35af680..000000000 --- a/ghostwriter/static/files/template_findings.csv +++ /dev/null @@ -1,2 +0,0 @@ -title,description,severity,impact,mitigation,replication_steps,host_detection_techniques,network_detection_techniques,references,finding_type,finding_guidance -Test finding,aze,Low,qsdq,azae,aze,daqd,qsd,dqsdq,Mobile,qsd diff --git a/ghostwriter/static/files/template_server.csv b/ghostwriter/static/files/template_server.csv deleted file mode 100644 index bebd04e55..000000000 --- a/ghostwriter/static/files/template_server.csv +++ /dev/null @@ -1,2 +0,0 @@ -ip_address,name,server_status,server_provider,note -127.0.0.1,localhost,Available,Microsoft Azure,Test server \ No newline at end of file diff --git a/ghostwriter/static/images/all_white_header.png b/ghostwriter/static/images/all_white_header.png new file mode 100644 index 000000000..5c052346e Binary files /dev/null and b/ghostwriter/static/images/all_white_header.png differ diff --git a/ghostwriter/static/images/all_white_logo.png b/ghostwriter/static/images/all_white_logo.png new file mode 100644 index 000000000..f960ba8e7 Binary files /dev/null and b/ghostwriter/static/images/all_white_logo.png differ diff --git a/ghostwriter/static/images/dashboard.png b/ghostwriter/static/images/dashboard.png new file mode 100644 index 000000000..ca0827e1b Binary files /dev/null and b/ghostwriter/static/images/dashboard.png differ diff --git a/ghostwriter/static/images/domain.png b/ghostwriter/static/images/domain.png index 53181fc2a..215b9e48b 100644 Binary files a/ghostwriter/static/images/domain.png and b/ghostwriter/static/images/domain.png differ diff --git a/ghostwriter/static/images/logo_name.png b/ghostwriter/static/images/logo_name.png new file mode 100644 index 000000000..3c68b82ca Binary files /dev/null and b/ghostwriter/static/images/logo_name.png differ diff --git a/ghostwriter/static/js/project.js b/ghostwriter/static/js/project.js index 2846ee23c..4bead8ec2 100644 --- a/ghostwriter/static/js/project.js +++ b/ghostwriter/static/js/project.js @@ -4,56 +4,55 @@ function displayToastTop({ type, string, - title = "", + title = '', delay = 4, escapeHTML = true, context = null, - singleton = false }) { if (context !== null) { - if (context == "form") { - title = "Form Validation Error"; + if (context == 'form') { + title = 'Form Validation Error'; } } delay = delay * 1000; - if (type === "error" && delay === 4000) { + if (type === 'error' && delay === 4000) { delay = 0; } toastr.options.timeOut = delay.toString(); toastr.options.extendedTimeOut = delay.toString(); toastr.options.escapeHtml = escapeHTML; let msg; - if (type === "success") { - if (title == "") { - title = "Great Success"; + if (type === 'success') { + if (title == '') { + title = 'Great Success'; } msg = toastr.success(string, title); - } else if (type === "warning") { - if (title == "") { - title = "Beware"; + } else if (type === 'warning') { + if (title == '') { + title = 'Beware'; } msg = toastr.warning(string, title); - } else if (type === "error") { - if (title == "") { - title = "Great Failure"; + } else if (type === 'error') { + if (title == '') { + title = 'Great Failure'; } msg = toastr.error(string, title); - } else if (type === "info") { - if (title == "") { - title = "FYI"; + } else if (type === 'info') { + if (title == '') { + title = 'FYI'; } msg = toastr.info(string, title); } else { - if (title == "") { - title = "Beware"; + if (title == '') { + title = 'Beware'; } msg = toastr.warning(string, title); } if (msg !== undefined) { msg.css({ - width: "100%", - "min-width": "400px", - "white-space": "pre-wrap", + width: '100%', + 'min-width': '400px', + 'white-space': 'pre-wrap', }); } } @@ -63,3 +62,21 @@ function csrfSafeMethod(method) { // These HTTP methods do not require CSRF protection return /^(GET|HEAD|OPTIONS|TRACE)$/.test(method); } + +function generateDownloadName(name) { + var d = new Date(); + var year = d.getFullYear() + var month = d.getMonth() + var day = d.getDay() + var hour = d.getHours() + var minutes = d.getMinutes() + var sec = d.getSeconds() + if (hour < 10) { + hour = '0' + hour; + } + if (minutes < 10) { + minutes = '0' + minutes; + } + filename = year + month + day + '_' + hour + minutes + day + '_' + name + return filename +} \ No newline at end of file diff --git a/ghostwriter/static/js/tinymce/config.js b/ghostwriter/static/js/tinymce/config.js index fb01707d2..c8cf69b5a 100644 --- a/ghostwriter/static/js/tinymce/config.js +++ b/ghostwriter/static/js/tinymce/config.js @@ -24,15 +24,27 @@ function adjust_dialog() { // Default config for all TinyMCE editors var default_config = { + branding: false, width: '100%', height: '300', theme: 'silver', skin: 'Ghostwriter', selector: 'textarea', editor_deselector: 'empty-form', - menubar: 'file edit tools', - plugins: 'lists image hr autosave advlist code wordcount codesample searchreplace paste link', - toolbar: 'bold italic link bullist numlist codesample codeBlock codeInline evidenceUpload removeformat searchreplace', + menubar: 'file edit insert view format tools', + visualchars_default_state: false, + menu: { + file: { title: 'File', items: 'newdocument restoredraft' }, + edit: { title: 'Edit', items: 'undo redo | cut copy paste | selectall | searchreplace' }, + view: { title: 'View', items: 'code | visualchars visualblocks | preview' }, + insert: { title: 'Insert', items: 'evidenceUpload codesample link' }, + format: { title: 'Format', items: 'bold italic underline strikethrough superscript subscript codeformat | formats fontformats fontsizes align | forecolor | removeformat' }, + tools: { title: 'Tools', items: 'code wordcount' }, + }, + toolbar_mode: 'floating', + plugins: 'visualchars visualblocks save preview lists image hr autosave advlist code wordcount codesample searchreplace paste link', + toolbar: 'subscript superscript bold italic underline link | bullist numlist | codesample codeInline | evidenceUpload | removeformat save', + contextmenu: 'bold italic link removeformat', paste_as_text: true, paste_data_images: false, browser_spellcheck: true, @@ -65,9 +77,9 @@ var default_config = { selector: 'p,h1,h2,h3,h4,h5,h6,td,th,div,ul,ol,li,table,img', classes: 'right' }, - alignfull: { + alignjustify: { selector: 'p,h1,h2,h3,h4,h5,h6,td,th,div,ul,ol,li,table,img', - classes: 'full' + classes: 'justify' }, bold: { inline: 'span', @@ -84,8 +96,47 @@ var default_config = { }, strikethrough: { inline: 'del' - } + }, + subscript: { + inline: 'sub' + }, + superscript: { + inline: 'sup' + }, + highlight: { + inline : 'span', + classes : 'highlight', + styles : { + backgroundColor : 'yellow' + } + }, }, + style_formats: [ + { title: 'Headings', items: [ + { title: 'Heading 1', format: 'h1' }, + { title: 'Heading 2', format: 'h2' }, + { title: 'Heading 3', format: 'h3' }, + { title: 'Heading 4', format: 'h4' }, + { title: 'Heading 5', format: 'h5' }, + { title: 'Heading 6', format: 'h6' } + ]}, + { title: 'Inline', items: [ + { title: 'Bold', format: 'bold' }, + { title: 'Italic', format: 'italic' }, + { title: 'Underline', format: 'underline' }, + { title: 'Strikethrough', format: 'strikethrough' }, + { title: 'Superscript', format: 'superscript' }, + { title: 'Subscript', format: 'subscript' }, + { title: 'Code', format: 'code' }, + { title: 'Highlight', format: 'highlight' } + ]}, + { title: 'Align', items: [ + { title: 'Left', format: 'alignleft' }, + { title: 'Center', format: 'aligncenter' }, + { title: 'Right', format: 'alignright' }, + { title: 'Justify', format: 'alignjustify' } + ]} + ] } /* @@ -98,7 +149,7 @@ basic_config = { setup: function (editor) { editor.ui.registry.addButton('codeInline', { icon: 'sourcecode', - text: 'Inline Code', + text: '', tooltip: 'Format selected text as inline code', onAction: function (_) { tinymce.activeEditor.formatter.toggle('code') @@ -120,7 +171,6 @@ Combine it with the default config above finding_config = { selector: "textarea.enable-evidence-upload", - toolbar: 'bold italic link bullist numlist codesample codeBlock codeInline evidenceUpload removeformat searchreplace', setup: function (editor) { editor.ui.registry.addButton('codeInline', { icon: 'sourcecode', diff --git a/ghostwriter/templates/400.html b/ghostwriter/templates/400.html new file mode 100644 index 000000000..f6dc07f78 --- /dev/null +++ b/ghostwriter/templates/400.html @@ -0,0 +1,7 @@ +{% extends "base_generic.html" %} + +{% block title %}Bad Request (400){% endblock %} + +{% block content %} +

    Bad Request (400)

    +{% endblock content %} diff --git a/ghostwriter/templates/403.html b/ghostwriter/templates/403.html index 42d89f01d..ed84405d7 100644 --- a/ghostwriter/templates/403.html +++ b/ghostwriter/templates/403.html @@ -4,6 +4,4 @@ {% block content %}

    Forbidden (403)

    - -

    CSRF verification failed so the request aborted.

    {% endblock content %} diff --git a/ghostwriter/templates/account/base.html b/ghostwriter/templates/account/base.html index 674f7945d..b6b4e31b2 100644 --- a/ghostwriter/templates/account/base.html +++ b/ghostwriter/templates/account/base.html @@ -3,7 +3,7 @@ {% block content %}
    -
    +
    {% block inner %}{% endblock %}
    diff --git a/ghostwriter/templates/account/email.html b/ghostwriter/templates/account/email.html index f65008c5c..1a672914d 100644 --- a/ghostwriter/templates/account/email.html +++ b/ghostwriter/templates/account/email.html @@ -32,10 +32,10 @@

    {% trans "Email Addresses" %}

    {% endfor %} -
    +
    - - + +
    @@ -50,7 +50,7 @@

    {% trans "Add Email Address" %}

    {% csrf_token %} {{ form|crispy }} - +
    {% endblock %} diff --git a/ghostwriter/templates/account/email_confirm.html b/ghostwriter/templates/account/email_confirm.html index c7e5c5fdc..3ff42b580 100644 --- a/ghostwriter/templates/account/email_confirm.html +++ b/ghostwriter/templates/account/email_confirm.html @@ -15,7 +15,7 @@

    {% trans "Confirm Email Address" %}

    {% csrf_token %} - +
    {% else %} {% url 'account_email' as email_url %} diff --git a/ghostwriter/templates/account/login.html b/ghostwriter/templates/account/login.html index 3e9071801..a8717b3c9 100644 --- a/ghostwriter/templates/account/login.html +++ b/ghostwriter/templates/account/login.html @@ -63,7 +63,7 @@

    {% trans "Sign In" %}

    {% if redirect_field_value %} {% endif %} - +
    diff --git a/ghostwriter/templates/account/logout.html b/ghostwriter/templates/account/logout.html index a83898a9d..e11c1de08 100644 --- a/ghostwriter/templates/account/logout.html +++ b/ghostwriter/templates/account/logout.html @@ -14,6 +14,6 @@

    {% trans "Sign Out" %}

    {% if redirect_field_value %} {% endif %} - + {% endblock %} diff --git a/ghostwriter/templates/account/password_change.html b/ghostwriter/templates/account/password_change.html index 648dc4e07..2975d7c1e 100644 --- a/ghostwriter/templates/account/password_change.html +++ b/ghostwriter/templates/account/password_change.html @@ -31,8 +31,8 @@

    {% trans "Change Password" %}

    {% csrf_token %} {% for field in form %} - {{ field|as_crispy_field }} + {{ field|as_crispy_field }} {% endfor %} - +
    {% endblock %} diff --git a/ghostwriter/templates/account/password_reset.html b/ghostwriter/templates/account/password_reset.html index 71f5317df..220e92496 100644 --- a/ghostwriter/templates/account/password_reset.html +++ b/ghostwriter/templates/account/password_reset.html @@ -15,7 +15,7 @@

    {% trans "Password Reset" %}

    {% csrf_token %} {{ form|crispy }} - +

    {% blocktrans %}Please contact your admin if you have any trouble resetting your password.{% endblocktrans %}

    diff --git a/ghostwriter/templates/account/password_reset_from_key.html b/ghostwriter/templates/account/password_reset_from_key.html index 8aafd422d..6e800cb77 100644 --- a/ghostwriter/templates/account/password_reset_from_key.html +++ b/ghostwriter/templates/account/password_reset_from_key.html @@ -15,10 +15,10 @@

    {% if token_fail %}{% trans "Bad Token" %}{% else %}{% trans "Change Passwor
    {% csrf_token %} {{ form|crispy }} - +
    {% else %} -

    {% trans 'Your password is now changed.' %}

    +

    {% trans 'Your password has been changed.' %}

    {% endif %} {% endif %} {% endblock %} diff --git a/ghostwriter/templates/account/password_reset_from_key_done.html b/ghostwriter/templates/account/password_reset_from_key_done.html index b3805c698..0a2819f65 100644 --- a/ghostwriter/templates/account/password_reset_from_key_done.html +++ b/ghostwriter/templates/account/password_reset_from_key_done.html @@ -5,5 +5,5 @@ {% block inner %}

    {% trans "Change Password" %}

    -

    {% trans 'Your password is now changed.' %}

    +

    {% trans 'Your password has been changed.' %}

    {% endblock %} diff --git a/ghostwriter/templates/account/password_set.html b/ghostwriter/templates/account/password_set.html index b325b5baf..8c2a7c59d 100644 --- a/ghostwriter/templates/account/password_set.html +++ b/ghostwriter/templates/account/password_set.html @@ -9,6 +9,6 @@

    {% trans "Set Password" %}

    {% csrf_token %} {{ form|crispy }} - +
    {% endblock %} diff --git a/ghostwriter/templates/account/signup.html b/ghostwriter/templates/account/signup.html index 6a183507c..49f20aa36 100644 --- a/ghostwriter/templates/account/signup.html +++ b/ghostwriter/templates/account/signup.html @@ -14,6 +14,6 @@

    {% trans "Sign Up" %}

    {% if redirect_field_value %} {% endif %} - + {% endblock %} diff --git a/ghostwriter/templates/account/signup_closed.html b/ghostwriter/templates/account/signup_closed.html index baf28aedf..a37a6c50f 100644 --- a/ghostwriter/templates/account/signup_closed.html +++ b/ghostwriter/templates/account/signup_closed.html @@ -7,5 +7,5 @@ {% block inner %}

    {% trans "Sign Up Closed" %}

    -

    {% trans "We are sorry, but the sign up is currently closed." %}

    +

    {% trans "We are sorry, but the sign up is currently closed. Contact your administrator for an account." %}

    {% endblock %} diff --git a/ghostwriter/templates/base_generic.html b/ghostwriter/templates/base_generic.html index 9573235dc..0690f0088 100644 --- a/ghostwriter/templates/base_generic.html +++ b/ghostwriter/templates/base_generic.html @@ -54,164 +54,154 @@ {% endif %} @@ -329,7 +319,7 @@

    SpecterOps"); $("body").append($temp); - $temp.val($(element).text()).select(); + $temp.val($(element).text().trim()).select(); document.execCommand("copy"); $temp.remove(); } @@ -414,16 +404,18 @@

    SpecterOps @@ -432,14 +424,18 @@

    SpecterOpsSpecterOps
      -
    • Hi, {{ user }}!
      • +
    • Welcome, {{ user|capfirst }}!
    {% endblock %} @@ -15,7 +15,7 @@ {% block content %} {% load static %}
    - Ghostwriter Logo + Ghostwriter Logo
    @@ -52,7 +52,7 @@ {% endfor %} {% else %}

    You are not currently assigned to any projects in the next week.You are not currently assigned to any future projects.
    diff --git a/ghostwriter/users/admin.py b/ghostwriter/users/admin.py index e04367d5d..1d803ebd5 100644 --- a/ghostwriter/users/admin.py +++ b/ghostwriter/users/admin.py @@ -1,12 +1,14 @@ """This contains customizations for displaying the Users application models in the admin panel.""" +# Django & Other 3rd Party Libraries from django.contrib import admin from django.contrib.auth import admin as auth_admin from django.contrib.auth import get_user_model -from django.utils.translation import gettext +from django.contrib.auth.models import Group from django.utils.translation import gettext_lazy as _ -from ghostwriter.users.forms import UserChangeForm, UserCreationForm +# Ghostwriter libaries +from ghostwriter.users.forms import GroupAdminForm User = get_user_model() @@ -32,14 +34,23 @@ class UserAdmin(auth_admin.UserAdmin): (_("Personal Information"), {"fields": ("name", "email")}), ( _("User Permissions"), - {"fields": ("is_active", "is_staff", "is_superuser",),}, + { + "fields": ( + "is_active", + "is_staff", + "is_superuser", + ), + }, ), (_("Important Dates"), {"fields": ("last_login", "date_joined")}), ) add_fieldsets = ( ( None, - {"classes": ("wide",), "fields": ("username", "password1", "password2"),}, + { + "classes": ("wide",), + "fields": ("username", "password1", "password2"), + }, ), ) search_fields = ("username", "name", "email") @@ -48,3 +59,16 @@ class UserAdmin(auth_admin.UserAdmin): admin.site.register(User, UserAdmin) + + +# Unregister the original Group admin +admin.site.unregister(Group) + + +class GroupAdmin(admin.ModelAdmin): + form = GroupAdminForm + filter_horizontal = ["permissions"] + + +# Register the new Group ModelAdmin +admin.site.register(Group, GroupAdmin) diff --git a/ghostwriter/users/forms.py b/ghostwriter/users/forms.py index d106cd2dd..ff264d286 100644 --- a/ghostwriter/users/forms.py +++ b/ghostwriter/users/forms.py @@ -1,8 +1,12 @@ """This contains all of the forms used by the Users application.""" +# Django & Other 3rd Party Libraries +from django.forms import ModelForm, ModelMultipleChoiceField +from django.contrib.admin.widgets import FilteredSelectMultiple from django.contrib.auth import forms, get_user_model +from django.contrib.auth.models import Group from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ User = get_user_model() @@ -37,3 +41,40 @@ def clean_username(self): return username raise ValidationError(self.error_messages["duplicate_username"]) + + +# Create ModelForm based on the Group model. +class GroupAdminForm(ModelForm): + class Meta: + model = Group + exclude = [] + + # Add the users field + users = ModelMultipleChoiceField( + queryset=User.objects.all(), + required=False, + # Use the pretty ``filter_horizontal`` widget + widget=FilteredSelectMultiple("users", False), + label=_( + "Users", + ), + ) + + def __init__(self, *args, **kwargs): + # Do the normal form initialisation + super(GroupAdminForm, self).__init__(*args, **kwargs) + # If it is an existing group (saved objects have a pk) + if self.instance.pk: + # Populate the users field with the current Group users + self.fields["users"].initial = self.instance.user_set.all() + + def save_m2m(self): + # Add the users to the Group + self.instance.user_set.set(self.cleaned_data["users"]) + + def save(self, *args, **kwargs): + # Default save + instance = super(GroupAdminForm, self).save() + # Save many-to-many data + self.save_m2m() + return instance diff --git a/ghostwriter/users/models.py b/ghostwriter/users/models.py index 6cd221df7..0c6a5d446 100644 --- a/ghostwriter/users/models.py +++ b/ghostwriter/users/models.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import AbstractUser from django.db.models import CharField from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class User(AbstractUser): @@ -18,3 +18,15 @@ class User(AbstractUser): def get_absolute_url(self): return reverse("users:detail", kwargs={"username": self.username}) + + def get_display_name(self): + """ + Return a display name appropriate for dropdown menus. + """ + if self.name: + display_name = "{full_name} ({username})".format( + full_name=self.name, username=self.username + ) + else: + display_name = self.username.capitalize() + return display_name diff --git a/ghostwriter/users/views.py b/ghostwriter/users/views.py index 02f0a7338..b6716ee20 100644 --- a/ghostwriter/users/views.py +++ b/ghostwriter/users/views.py @@ -6,7 +6,7 @@ from django.contrib.auth import get_user_model from django.contrib.auth.mixins import LoginRequiredMixin from django.urls import reverse, reverse_lazy -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.generic import DetailView, RedirectView, UpdateView User = get_user_model() diff --git a/production.yml b/production.yml index a3b7b105b..08e738806 100644 --- a/production.yml +++ b/production.yml @@ -43,6 +43,7 @@ services: depends_on: - django volumes: + - production_data:/app/media - production_staticfiles:/app/staticfiles - ./compose/production/nginx/nginx.conf:/etc/nginx/nginx.conf - ./ssl:/ssl @@ -53,7 +54,7 @@ services: redis: image: redis:5-alpine restart: unless-stopped - + queue: <<: *django image: ghostwriter_production_queue diff --git a/requirements/base.txt b/requirements/base.txt index 1350f4f97..af7b313da 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,39 +1,39 @@ -pytz==2019.1 # https://github.com/stub42/pytz -python-slugify==3.0.2 # https://github.com/un33k/python-slugify -Pillow==6.1.0 # https://github.com/python-pillow/Pillow +pytz==2020.1 # https://github.com/stub42/pytz +python-slugify==4.0.1 # https://github.com/un33k/python-slugify +Pillow==7.2.0 # https://github.com/python-pillow/Pillow rcssmin==1.0.6 # https://github.com/ndparker/rcssmin argon2-cffi==20.1.0 # https://github.com/hynek/argon2_cffi -redis==3.2.1 # https://github.com/antirez/redis +redis==3.5.3 # https://github.com/antirez/redis # Django # ------------------------------------------------------------------------------ -django==2.2.3 # pyup: < 3.0 # https://www.djangoproject.com/ +django==3.0.10 # pyup: < 3.0 # https://www.djangoproject.com/ django-environ==0.4.5 # https://github.com/joke2k/django-environ -django-model-utils==3.2.0 # https://github.com/jazzband/django-model-utils -django-allauth==0.39.1 # https://github.com/pennersr/django-allauth -django-crispy-forms==1.7.2 # https://github.com/django-crispy-forms/django-crispy-forms -django-compressor==2.3 # https://github.com/django-compressor/django-compressor -django-redis==4.10.0 # https://github.com/niwinz/django-redis +django-model-utils==4.0.0 # https://github.com/jazzband/django-model-utils +django-allauth==0.42.0 # https://github.com/pennersr/django-allauth +django-crispy-forms==1.9.2 # https://github.com/django-crispy-forms/django-crispy-forms +django-compressor==2.4 # https://github.com/django-compressor/django-compressor +django-redis==4.12.1 # https://github.com/niwinz/django-redis channels==2.4.0 # https://channels.readthedocs.io/en/stable/index.html channels-redis==2.4.2 # https://github.com/django/channels_redis/ +djangorestframework==3.12.1 # https://github.com/encode/django-rest-framework # Ghostwriter specific +boto3==1.15.16 +botocore==1.18.16 bs4==0.0.1 -django-filter==2.2.0 -django-q==1.0.2 -dnspython==1.16.0 -pytesseract==0.2.7 -python-nmap==0.6.1 -python-docx==0.8.7 -python-pptx==0.6.18 -redis==3.2.1 -requests==2.22.0 -boto3==1.10.8 -botocore==1.13.8 -djangorestframework==3.11.0 +django-bleach==0.6.1 +django-filter==2.4.0 +django-import-export==2.4.0 +django-q==1.3.3 djangorestframework-api-key==2.0.0 -django-import-export==2.2.0 -django-tinymce==2.8.0 -docxtpl==0.6.3 +django-tinymce==2.8.0 # Deprecated, but kept here for legacy migrations docutils==0.16 -django-bleach==0.6.1 +docxtpl==0.10.0 +dnspython==2.0.0 +jinja2==2.11.2 +python-docx==0.8.10 +python-nmap==0.6.1 +python-pptx==0.6.18 +redis==3.5.3 +requests==2.24.0 diff --git a/requirements/local.txt b/requirements/local.txt index 3a67e740d..9974fecb0 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -1,29 +1,29 @@ -r ./base.txt -Werkzeug==0.14.1 # pyup: < 0.15 # https://github.com/pallets/werkzeug -ipdb==0.12 # https://github.com/gotcha/ipdb -Sphinx==2.1.2 # https://github.com/sphinx-doc/sphinx -psycopg2==2.8.3 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 +Werkzeug==1.0.1 # pyup: < 0.15 # https://github.com/pallets/werkzeug +ipdb==0.13.4 # https://github.com/gotcha/ipdb +Sphinx==3.2.1 # https://github.com/sphinx-doc/sphinx +psycopg2==2.8.6 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 # Testing # ------------------------------------------------------------------------------ -mypy==0.720 # https://github.com/python/mypy -pytest==5.0.1 # https://github.com/pytest-dev/pytest -pytest-sugar==0.9.2 # https://github.com/Frozenball/pytest-sugar +mypy==0.782 # https://github.com/python/mypy +pytest==6.1.1 # https://github.com/pytest-dev/pytest +pytest-sugar==0.9.4 # https://github.com/Frozenball/pytest-sugar model_bakery==1.1.1 # https://github.com/model-bakers/model_bakery django-webtest==1.9.7 # https://pypi.org/project/django-webtest/ factory-boy==3.0.1 # https://github.com/FactoryBoy/factory_boy # Code quality # ------------------------------------------------------------------------------ -flake8==3.8.3 # https://github.com/PyCQA/flake8 -coverage==5.2.1 # https://github.com/nedbat/coveragepy -black==19.10b0 # https://github.com/ambv/black -pylint-django==2.0.11 # https://github.com/PyCQA/pylint-django +flake8==3.8.4 # https://github.com/PyCQA/flake8 +coverage==5.3 # https://github.com/nedbat/coveragepy +black==20.8b1 # https://github.com/ambv/black +pylint-django==2.3.0 # https://github.com/PyCQA/pylint-django # Django # ------------------------------------------------------------------------------ -django-debug-toolbar==2.0 # https://github.com/jazzband/django-debug-toolbar -django-extensions==2.1.9 # https://github.com/django-extensions/django-extensions -django-coverage-plugin==1.6.0 # https://github.com/nedbat/django_coverage_plugin -pytest-django==3.5.1 # https://github.com/pytest-dev/pytest-django +django-debug-toolbar==3.1.1 # https://github.com/jazzband/django-debug-toolbar +django-extensions==3.0.9 # https://github.com/django-extensions/django-extensions +django-coverage-plugin==1.8.0 # https://github.com/nedbat/django_coverage_plugin +pytest-django==4.0.0 # https://github.com/pytest-dev/pytest-django diff --git a/requirements/production.txt b/requirements/production.txt index db69cc178..f8ec404fd 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -2,10 +2,10 @@ -r ./base.txt -gunicorn==19.9.0 # https://github.com/benoitc/gunicorn -psycopg2==2.8.3 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 -Collectfast==0.6.2 # https://github.com/antonagestam/collectfast +gunicorn==20.0.4 # https://github.com/benoitc/gunicorn +psycopg2==2.8.6 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 +Collectfast==2.2.0 # https://github.com/antonagestam/collectfast # Django # ------------------------------------------------------------------------------ -django-anymail[mailgun]==6.1.0 # https://github.com/anymail/django-anymail +django-anymail[mailgun]==8.1 # https://github.com/anymail/django-anymail