Skip to content

Commit

Permalink
feat: Add HTML report output, report docs
Browse files Browse the repository at this point in the history
  • Loading branch information
bmtcril committed Dec 19, 2024
1 parent e6f63e6 commit d082c46
Show file tree
Hide file tree
Showing 12 changed files with 253 additions and 37 deletions.
28 changes: 0 additions & 28 deletions README.rst
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
code-annotations
=============================

|pypi-badge| |CI| |codecov-badge| |doc-badge| |pyversions-badge|
|license-badge|

Extensible tools for parsing annotations in codebases

Overview
Expand Down Expand Up @@ -55,28 +52,3 @@ Have a question about this repository, or about Open edX in general? Please
refer to this `list of resources`_ if you need any assistance.

.. _list of resources: https://open.edx.org/getting-help


.. |pypi-badge| image:: https://img.shields.io/pypi/v/code-annotations.svg
:target: https://pypi.python.org/pypi/code-annotations/
:alt: PyPI

.. |CI| image:: https://github.com/openedx/code-annotations/workflows/Python%20CI/badge.svg?branch=master
:target: https://github.com/openedx/code-annotations/actions?query=workflow%3A%22Python+CI%22
:alt: CI

.. |codecov-badge| image:: http://codecov.io/github/edx/code-annotations/coverage.svg?branch=master
:target: http://codecov.io/github/edx/code-annotations?branch=master
:alt: Codecov

.. |doc-badge| image:: https://readthedocs.org/projects/code-annotations/badge/?version=latest
:target: http://code-annotations.readthedocs.io/en/latest/
:alt: Documentation

.. |pyversions-badge| image:: https://img.shields.io/pypi/pyversions/code-annotations.svg
:target: https://pypi.python.org/pypi/code-annotations/
:alt: Supported Python versions

.. |license-badge| image:: https://img.shields.io/github/license/edx/code-annotations.svg
:target: https://github.com/openedx/code-annotations/blob/master/LICENSE.txt
:alt: License
5 changes: 3 additions & 2 deletions code_annotations/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
PACKAGE_DIR = os.path.dirname(os.path.realpath(__file__))
DEFAULT_TEMPLATE_DIR = os.path.join(PACKAGE_DIR, "report_templates")


class AnnotationConfig:
"""
Configuration shared among all Code Annotations commands.
Expand Down Expand Up @@ -67,9 +68,9 @@ def __init__(self, config_file_path, report_path_override=None, verbosity=1, sou
os.path.join(DEFAULT_TEMPLATE_DIR, self.rendered_report_format)
)
self.rendered_report_dir = raw_config.get('rendered_report_dir', 'annotation_reports')
self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix')
self.rendered_report_source_link_prefix = raw_config.get('rendered_report_source_link_prefix', None)
self.trim_filename_prefixes = raw_config.get('trim_filename_prefixes', [])
self.third_party_package_location = raw_config.get('third_party_package_location', None)
self.third_party_package_location = raw_config.get('third_party_package_location', "site-packages")
self.rendered_report_file_extension = f".{self.rendered_report_format}"

self._configure_annotations(raw_config)
Expand Down
9 changes: 5 additions & 4 deletions code_annotations/generate_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ def _add_report_file_to_full_report(self, report_file, report):
else:
report[trimmed_filename] = loaded_report[filename]


def _aggregate_reports(self):
"""
Combine all of the given report files into a single report object.
Expand All @@ -105,6 +104,7 @@ def _write_doc_file(self, doc_title, doc_filename, doc_data):
Write out a single report file with the given data. This is rendered using the configured top level template.
Args:
doc_title: Title to use for the document.
doc_filename: Filename to write to.
doc_data: Dict of reporting data to use, in the {'file name': [list, of, annotations,]} style.
"""
Expand All @@ -128,8 +128,7 @@ def _write_doc_file(self, doc_title, doc_filename, doc_data):
slugify=slugify,
source_link_prefix=self.config.rendered_report_source_link_prefix,
third_party_package_location=self.config.third_party_package_location,
),
)
))

def _generate_per_choice_docs(self):
"""
Expand All @@ -155,7 +154,9 @@ def _generate_per_annotation_docs(self):
if report_annotation['annotation_token'] == annotation:
annotation_report[filename].append(report_annotation)

self._write_doc_file(f"All References to Annotation '{annotation}'", f'annotation_{annotation}', annotation_report)
self._write_doc_file(
f"All References to Annotation '{annotation}'", f'annotation_{annotation}', annotation_report
)

def render(self):
"""
Expand Down
14 changes: 14 additions & 0 deletions code_annotations/report_templates/html/annotation.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
{% if is_third_party %}
{# no links for third party code since we don't know where to link to #}
{% if annotation.extra and annotation.extra.object_id %}
{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %}
{% else %}
{% if loop.changed(annotation.line_number)%}{{ filename }}:{{ annotation.line_number }}<br />{% endif %}:
{{ annotation.annotation_token }} {% include "annotation_data.tpl" %}
{% endif %}
{% elif annotation.extra and annotation.extra.object_id %}
<a href="{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}" target="_blank">{{ annotation.extra.object_id }} {% if annotation.line_number > 0 %}line {{ annotation.line_number }}{% endif %}</a>: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %}
{% else %}
<a href="{{ source_link_prefix }}{{ filename }}#L{{ annotation.line_number }}" target="_blank">`{{ filename }}:{{ annotation.line_number }}: {{ annotation.annotation_token }} {% include "annotation_data.tpl" %}
{% endif %}
8 changes: 8 additions & 0 deletions code_annotations/report_templates/html/annotation_data.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{% if annotation.annotation_data is sequence and annotation.annotation_data is not string %}
{% for a in annotation.annotation_data %}
<a href="choice-{{ slugify(a) }}.html">{{ a }}</a>{% if not loop.last %}, {% endif %}
{% endfor %}

{% else %}
{{ annotation.annotation_data }}
{% endif %}
27 changes: 27 additions & 0 deletions code_annotations/report_templates/html/annotation_list.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{% extends "base.tpl" %}
{% block content %}
Annotations found in {{ report|length }} files.

{% for filename in report %}
{% set is_third_party = third_party_package_location in filename %}

<h2 id="file-{{ slugify(filename) }}">{{ filename }}</h2>
<div class="file-annotations">
{{ report[filename]|length }} annotations {% if is_third_party %}(installed package){% endif %}<br />
</div>

{% for annotation in report[filename] %}
{% if loop.changed(annotation.report_group_id) %}
{% if not loop.first %}</ul></div>{% endif %}
<div class="group-annotations"><ul>
{% endif %}
<li>{% include 'annotation.tpl' %}</li>
{% if loop.last %}
</ul></div>
{% endif %}
{% endfor %}


{% endfor %}

{% endblock %}
92 changes: 92 additions & 0 deletions code_annotations/report_templates/html/base.tpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
<html>
<head>
<title>{{ doc_title }}</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<style>
body {
font-family: 'Trebuchet MS', sans-serif;
}
.title {
text-align: center;
}
.table {
display: table;
border-spacing: 12px;
}
.row {
display: table-row;
margin-bottom: 0;
margin-top: 0;
width: 100%;
}
.cell1 {
display: table-cell;
width: 20%;
margin-right: 1%;
border: 1px solid #ccc;
margin 12px;
background-color: #ffffee;
}
.cell2 {
display: table-cell;
width: 79%;
margin-right: 1%;
margin 12px;
}
.group-annotations {
border: 1px solid #ccc;
margin: 10px;
}
</style>
</head>
<body>
<h1 class="title">{{ doc_title }}</h1>

<div class="table">
<div class="row">
<div class="cell1">
<h3><a href="index.html">Home</a></h3>

<h3>Annotations</h3>

<ul>
{% for a in all_annotations %}
<li><a href="annotation-{{ slugify(a) }}.html">annotation_{{ slugify(a) }}</a></li>
{% endfor %}
</ul>

<h3>Choices</h3>

<ul>
{% for choice in all_choices %}
<li><a href="choice-{{ slugify(choice) }}.html">choice_{{ slugify(choice) }}</a></li>
{% endfor %}
</ul>
</div>
<div class="cell2">
<h2>Files in this page</h2>
<ul>
{% for filename in report %}
<li><a href="#file-{{ slugify(filename) }}">{{ filename }}</a></li>
{% endfor %}
</ul>

{% block content %}{% endblock %}
</div>
</div>
</div>
{% block footer %}
<div class="footer">
<br /><br />
<hr />
Built at {{ create_time.strftime('%Y-%m-%d %H:%M:%S %Z') }}
</div>
{% endblock %}
</body>
</html>
27 changes: 27 additions & 0 deletions docs/configuration.rst
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,15 @@ Configuring Code Annotations is a pretty simple affair. Here is an example showi
- py3
javascript:
- js
# Options only used for human readable reports
report_template_dir: /my_custom_templates/report_templates/
rendered_report_dir: rendered_reports
rendered_report_format: html
rendered_report_source_link_prefix: https://github.com/my_org/my_project/blob/master/
trim_filename_prefixes:
- /my_org/my_project/
- /my_project/venv/lib/python3.11/
third_party_package_location: site-packages
``source_path``
The file or directory to be searched for annotations. If a directory, it will be searched recursively. This can be
Expand Down Expand Up @@ -68,3 +77,21 @@ Configuring Code Annotations is a pretty simple affair. Here is an example showi
extensions are turned on here. The key is the extension name, as given in the ``setup.py`` or ``setup.cfg`` of the
package that installed the extension. The values are a list of file extensions that, when found, will be passed to
the extension for annotation searching. See :doc:`extensions` for details.

``report_template_dir`` (optional)
When running the ``generate_docs`` comman you can specify a custom template directory here to override the default templates if you would like a different look.

``rendered_report_dir`` (optional)
When running the ``generate_docs`` command, this option specifies the directory where the rendered report will be written. The default is ``annotation_reports`` in the current working directory.

``rendered_report_format`` (optional)
When running the ``generate_docs`` command, this option specifies the format of the rendered report. Options are ``html`` and ``rst``. The default is ``rst``.

``rendered_report_source_link_prefix`` (optional)
When running the ``generate_docs`` command, this option specifies the URL prefix to use when linking to source files in the rendered report. When specified, "local" source files (those not found in site-packages) will be appended to this setting to create hyperlinks to the lines in source files online. For Github links this is the correct format: ``https://github.com/openedx/edx-platform/blob/master/``. The default is an None.

``trim_filename_prefixes`` (optional)
When running the ``generate_docs`` command, this option specifies a list of prefixes to remove from the beginning of filenames in the rendered report. This is useful for making the report more readable by removing long, repetitive prefixes of the type often found in a Django search. The default is an empty list.

``third_party_package_location`` (optional)
When running the ``generate_docs`` command, this option specifies the location of third party packages that may have been found in a Django search. This is used to determine if a file is a third party file or not. The default is ``site-packages``.
14 changes: 14 additions & 0 deletions docs/getting_started.rst
Original file line number Diff line number Diff line change
Expand Up @@ -126,3 +126,17 @@ Add more structure to your annotations
Annotations can be more than simple messages. They can enforce the use of choices from a fixed list, and can be grouped
to provide more context-aware information. See :doc:`configuration` and :doc:`writing_annotations` for more information
on how to use those options.

Get a human readable report
---------------------------
The output generated by the search commands is a YAML file. To get a human readable report generated from one of those files in either rst or html format, you can use the ``generate_docs`` command.

There are several configuration options available for this command, including the ability to specify the output format, the output directory, and create links to the source files on sites like Github. For more information, see :doc:`configuration`. Once your configuration is set, you can run:

.. code-block:: bash
$ code_annotations generate_docs --config_file /path/to/your/config /path/to/your/report.yaml
Which will generate files in the output directory you configured. From there you can open the files in your browser to see the report, if you chose HTML, or use a tool like `restview`_ to render the RST files to your browser.

.. _restview: https://pypi.python.org/pypi/restview
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
source_path: tests/extensions/javascript_test_files/
report_path: test_reports
safelist_path: .annotation_safe_list.yml
report_template_dir: code_annotations/report_templates/
report_template_dir: code_annotations/report_templates/rst
rendered_report_dir: test_reports/
rendered_report_file_extension: .rst
rendered_report_format: rst
rendered_report_source_link_prefix: https://github.com/openedx/edx-platform/tree/master/

coverage_target: 50.0
annotations:
".. no_pii:":
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
source_path: tests/extensions/javascript_test_files/
report_path: test_reports
safelist_path: .annotation_safe_list.yml
report_template_dir: code_annotations/report_templates/rst
rendered_report_dir: test_reports/
rendered_report_format: html
rendered_report_source_link_prefix: https://github.com/openedx/edx-platform/tree/master/
trim_filename_prefixes:
- /foo/bar
third_party_package_location: site-packages

coverage_target: 50.0
annotations:
".. no_pii:":
".. ignored:":
choices: [irrelevant, terrible, silly-silly]
"pii_group":
- ".. pii:":
- ".. pii_types:":
choices: [id, name, other]
- ".. pii_retirement:":
choices: [retained, local_api, consumer_api, third_party]
extensions:
python:
- pyt
36 changes: 35 additions & 1 deletion tests/test_generate_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,40 @@ def test_generate_report_simple():
assert os.path.exists(created_doc)


def test_generate_report_simple_html():
find_result = call_script(
(
'static_find_annotations',
'--config_file',
'tests/test_configurations/.annotations_test_python_only',
'--source_path=tests/extensions/python_test_files/simple_success.pyt',
'--no_lint',
),
delete_test_reports=False)

assert find_result.exit_code == EXIT_CODE_SUCCESS
assert "Writing report..." in find_result.output
report_file = get_report_filename_from_output(find_result.output)

report_result = call_script(
(
'generate_docs',
report_file,
'--config_file',
'tests/test_configurations/.annotations_test_success_with_report_docs_html',
'-vv'
),
delete_test_docs=False
)

assert find_result.exit_code == EXIT_CODE_SUCCESS
assert "Report rendered in" in report_result.output

# All file types are created
for created_doc in ('test_reports/index.html', 'test_reports/choice-id.html', 'test_reports/annotation-pii.html'):
assert os.path.exists(created_doc)


def _do_find(source_path, new_report_path):
"""
Do a static annotation search with report, rename the report to a distinct name.
Expand Down Expand Up @@ -167,4 +201,4 @@ def test_generate_report_missing_key():
))

assert report_result.exit_code == EXIT_CODE_FAILURE
assert "No report_template_dir key in tests/test_configurations/" in report_result.output
assert "No rendered_report_source_link_prefix key in" in report_result.output

0 comments on commit d082c46

Please sign in to comment.