From 35a7503596de2ee6cc307c5ce4ca7795214f2115 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Sat, 2 Sep 2023 09:37:00 -0300 Subject: [PATCH 1/7] Add line numbers for detailed metrics in Cyclomatic and Halstead operators. --- src/wily/operators/cyclomatic.py | 4 ++ src/wily/operators/halstead.py | 68 +++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/wily/operators/cyclomatic.py b/src/wily/operators/cyclomatic.py index 19a62b97..b75c0dbc 100644 --- a/src/wily/operators/cyclomatic.py +++ b/src/wily/operators/cyclomatic.py @@ -105,6 +105,8 @@ def _dict_from_function(l): "complexity": l.complexity, "fullname": l.fullname, "loc": l.endline - l.lineno, + "lineno": l.lineno, + "endline": l.endline, } @staticmethod @@ -116,4 +118,6 @@ def _dict_from_class(l): "complexity": l.complexity, "fullname": l.fullname, "loc": l.endline - l.lineno, + "lineno": l.lineno, + "endline": l.endline, } diff --git a/src/wily/operators/halstead.py b/src/wily/operators/halstead.py index 9bcc03bc..d8f7be50 100644 --- a/src/wily/operators/halstead.py +++ b/src/wily/operators/halstead.py @@ -3,13 +3,77 @@ Measures all of the halstead metrics (volume, vocab, difficulty) """ +import ast +import collections + import radon.cli.harvest as harvesters from radon.cli import Config +from radon.metrics import Halstead, HalsteadReport, halstead_visitor_report +from radon.visitors import HalsteadVisitor from wily import logger from wily.lang import _ from wily.operators import BaseOperator, Metric, MetricType +NumberedHalsteadReport = collections.namedtuple( + "NumberedHalsteadReport", + HalsteadReport._fields + ("lineno", "endline"), +) + + +class NumberedHalsteadVisitor(HalsteadVisitor): + """HalsteadVisitor that adds class name, lineno and endline for code blocks.""" + + def __init__(self, context=None, lineno=None, endline=None, classname=None): + """ + Initialize the numbered visitor. + + :param context: Function/method name. + :param lineno: The starting line of the code block, if any. + :param endline: The ending line of the code block, if any. + :param classname: The class name for a method. + """ + super().__init__(context) + self.lineno = lineno + self.endline = endline + self.class_name = classname + + def visit_FunctionDef(self, node): + """Visit functions and methods, adding class name if any, lineno and endline.""" + if self.class_name: + node.name = f"{self.class_name}.{node.name}" + super().visit_FunctionDef(node) + self.function_visitors[-1].lineno = node.lineno + self.function_visitors[-1].endline = node.end_lineno + + def visit_ClassDef(self, node): + """Visit classes, adding class name and creating visitors for methods.""" + self.class_name = node.name + for child in node.body: + visitor = NumberedHalsteadVisitor(classname=self.class_name) + visitor.visit(child) + self.function_visitors.extend(visitor.function_visitors) + self.class_name = None + + +def number_report(visitor): + """Create a report with added lineno and endline.""" + return NumberedHalsteadReport( + *(halstead_visitor_report(visitor) + (visitor.lineno, visitor.endline)) + ) + + +class NumberedHCHarvester(harvesters.HCHarvester): + """Version of HCHarvester that adds lineno and endline.""" + + def gobble(self, fobj): + """Analyze the content of the file object, adding line numbers for blocks.""" + code = fobj.read() + visitor = NumberedHalsteadVisitor.from_ast(ast.parse(code)) + total = number_report(visitor) + functions = [(v.context, number_report(v)) for v in visitor.function_visitors] + return Halstead(total, functions) + class HalsteadOperator(BaseOperator): """Halstead Operator.""" @@ -54,7 +118,7 @@ def __init__(self, config, targets): # TODO : Import config from wily.cfg logger.debug(f"Using {targets} with {self.defaults} for HC metrics") - self.harvester = harvesters.HCHarvester(targets, config=Config(**self.defaults)) + self.harvester = NumberedHCHarvester(targets, config=Config(**self.defaults)) def run(self, module, options): """ @@ -100,4 +164,6 @@ def _report_to_dict(self, report): "length": report.length, "effort": report.effort, "difficulty": report.difficulty, + "lineno": report.lineno, + "endline": report.endline, } From a26263a25e2ef095e42eeded7619928294f1ff7d Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Sat, 23 Sep 2023 20:15:10 -0300 Subject: [PATCH 2/7] Fix typing of HalsteadOperator._report_to_dict(). --- src/wily/operators/halstead.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/wily/operators/halstead.py b/src/wily/operators/halstead.py index 6f0ea224..c1d32eb7 100644 --- a/src/wily/operators/halstead.py +++ b/src/wily/operators/halstead.py @@ -138,7 +138,7 @@ def run(self, module: str, options: Dict[str, Any]) -> Dict[Any, Any]: if isinstance(instance, list): for item in instance: function, report = item - assert isinstance(report, HalsteadReport) + assert isinstance(report, NumberedHalsteadReport) results[filename]["detailed"][function] = self._report_to_dict( report ) @@ -150,11 +150,11 @@ def run(self, module: str, options: Dict[str, Any]) -> Dict[Any, Any]: details["error"], ) continue - assert isinstance(instance, HalsteadReport) + assert isinstance(instance, NumberedHalsteadReport) results[filename]["total"] = self._report_to_dict(instance) return results - def _report_to_dict(self, report: HalsteadReport) -> Dict[str, Any]: + def _report_to_dict(self, report: NumberedHalsteadReport) -> Dict[str, Any]: return { "h1": report.h1, "h2": report.h2, From 324ff7d96aeced7b7c1012fc4a7e0c3177ffe0ec Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Sun, 24 Sep 2023 09:46:38 -0300 Subject: [PATCH 3/7] Add a test for metric fields and values after building, covers the new lineno and endline entries for Halstead and Cyclomatic. --- test/integration/test_complex_commits.py | 146 +++++++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/test/integration/test_complex_commits.py b/test/integration/test_complex_commits.py index f6885e18..d89bb531 100644 --- a/test/integration/test_complex_commits.py +++ b/test/integration/test_complex_commits.py @@ -112,3 +112,149 @@ def test_skip_files(tmpdir, cache_path): assert "raw" in data3["operator_data"] assert _path1 in data3["operator_data"]["raw"] assert _path2 in data3["operator_data"]["raw"] + + +complex_test = """ +import abc +foo = 1 +def function1(): + a = 1 + 1 + if a == 2: + print(1) +class Class1(object): + def method(self): + b = 1 + 5 + if b == 6: + if 1==2: + if 2==3: + print(1) +""" + + +def test_metric_entries(tmpdir, cache_path): + """Test that the expected fields and values are present in metric results.""" + repo = Repo.init(path=tmpdir) + tmppath = pathlib.Path(tmpdir) / "src" + tmppath.mkdir() + + # Write and commit one test file to the repo + with open(tmppath / "test1.py", "w") as test1_txt: + test1_txt.write(complex_test) + index = repo.index + index.add([str(tmppath / "test1.py")]) + author = Actor("An author", "author@example.com") + committer = Actor("A committer", "committer@example.com") + commit = index.commit("commit one file", author=author, committer=committer) + repo.close() + + # Build the wily cache + runner = CliRunner() + result = runner.invoke( + main.cli, + ["--debug", "--path", tmpdir, "--cache", cache_path, "build", str(tmppath)], + ) + assert result.exit_code == 0, result.stdout + + # Get the revision path and the revision data + cache_path = pathlib.Path(cache_path) + rev_path = cache_path / "git" / (commit.name_rev.split(" ")[0] + ".json") + assert rev_path.exists() + with open(rev_path) as rev_file: + data = json.load(rev_file) + + # Check that basic data format is correct + assert "cyclomatic" in data["operator_data"] + assert _path1 in data["operator_data"]["cyclomatic"] + assert "detailed" in data["operator_data"]["cyclomatic"][_path1] + assert "total" in data["operator_data"]["cyclomatic"][_path1] + + # Test total and detailed metrics + expected_cyclomatic_total = {"complexity": 11} + total_cyclomatic = data["operator_data"]["cyclomatic"][_path1]["total"] + assert total_cyclomatic == expected_cyclomatic_total + + detailed_cyclomatic = data["operator_data"]["cyclomatic"][_path1]["detailed"] + assert "function1" in detailed_cyclomatic + assert "lineno" in detailed_cyclomatic["function1"] + assert "endline" in detailed_cyclomatic["function1"] + expected_cyclomatic_function1 = { + "name": "function1", + "is_method": False, + "classname": None, + "closures": [], + "complexity": 2, + "loc": 3, + "lineno": 4, + "endline": 7, + } + assert detailed_cyclomatic["function1"] == expected_cyclomatic_function1 + + expected_cyclomatic_Class1 = { + "name": "Class1", + "inner_classes": [], + "real_complexity": 5, + "complexity": 5, + "loc": 6, + "lineno": 8, + "endline": 14, + } + assert detailed_cyclomatic["Class1"] == expected_cyclomatic_Class1 + + expected_cyclomatic_method = { + "name": "method", + "is_method": True, + "classname": "Class1", + "closures": [], + "complexity": 4, + "loc": 5, + "lineno": 9, + "endline": 14, + } + assert detailed_cyclomatic["Class1.method"] == expected_cyclomatic_method + + expected_halstead_total = { + "h1": 2, + "h2": 3, + "N1": 2, + "N2": 4, + "vocabulary": 5, + "volume": 13.931568569324174, + "length": 6, + "effort": 18.575424759098897, + "difficulty": 1.3333333333333333, + "lineno": None, + "endline": None, + } + total_halstead = data["operator_data"]["halstead"][_path1]["total"] + assert total_halstead == expected_halstead_total + + detailed_halstead = data["operator_data"]["halstead"][_path1]["detailed"] + assert "function1" in detailed_halstead + assert "lineno" in detailed_halstead["function1"] + assert detailed_halstead["function1"]["lineno"] is not None + assert "endline" in detailed_halstead["function1"] + assert detailed_halstead["function1"]["endline"] is not None + + assert "Class1" not in detailed_halstead + + assert "Class1.method" in detailed_halstead + assert "lineno" in detailed_halstead["Class1.method"] + assert detailed_halstead["Class1.method"]["lineno"] is not None + assert "endline" in detailed_halstead["Class1.method"] + assert detailed_halstead["Class1.method"]["endline"] is not None + + expected_raw_total = { + "loc": 14, + "lloc": 13, + "sloc": 13, + "comments": 0, + "multi": 0, + "blank": 1, + "single_comments": 0, + } + total_raw = data["operator_data"]["raw"][_path1]["total"] + assert total_raw == expected_raw_total + + expected_maintainability = {"mi": 62.3299092923013, "rank": "A"} + total_maintainability = data["operator_data"]["maintainability"][_path1]["total"] + assert total_maintainability == expected_maintainability From fb6751f39bdaa84731d3c8e9e6891d553293f665 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Sun, 24 Sep 2023 10:20:37 -0300 Subject: [PATCH 4/7] Guard against FuncDef missing end_lineno in Python 3.7. --- src/wily/operators/halstead.py | 4 +++- test/integration/test_complex_commits.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/wily/operators/halstead.py b/src/wily/operators/halstead.py index c1d32eb7..c066e30a 100644 --- a/src/wily/operators/halstead.py +++ b/src/wily/operators/halstead.py @@ -46,7 +46,9 @@ def visit_FunctionDef(self, node): node.name = f"{self.class_name}.{node.name}" super().visit_FunctionDef(node) self.function_visitors[-1].lineno = node.lineno - self.function_visitors[-1].endline = node.end_lineno + # FuncDef is missing end_lineno in Python 3.7 + endline = node.end_lineno if hasattr(node, "end_lineno") else None + self.function_visitors[-1].endline = endline def visit_ClassDef(self, node): """Visit classes, adding class name and creating visitors for methods.""" diff --git a/test/integration/test_complex_commits.py b/test/integration/test_complex_commits.py index d89bb531..d32363ad 100644 --- a/test/integration/test_complex_commits.py +++ b/test/integration/test_complex_commits.py @@ -233,7 +233,9 @@ def test_metric_entries(tmpdir, cache_path): assert "lineno" in detailed_halstead["function1"] assert detailed_halstead["function1"]["lineno"] is not None assert "endline" in detailed_halstead["function1"] - assert detailed_halstead["function1"]["endline"] is not None + if sys.version_info > (3, 7): + # FuncDef is missing end_lineno in Python 3.7 + assert detailed_halstead["function1"]["endline"] is not None assert "Class1" not in detailed_halstead From f007b7c21a543b07e5259f0b5f9842f1695a00e7 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Sun, 24 Sep 2023 10:31:51 -0300 Subject: [PATCH 5/7] Silence a ruff error for checking Python version. --- test/integration/test_complex_commits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_complex_commits.py b/test/integration/test_complex_commits.py index d32363ad..c805afb7 100644 --- a/test/integration/test_complex_commits.py +++ b/test/integration/test_complex_commits.py @@ -233,7 +233,7 @@ def test_metric_entries(tmpdir, cache_path): assert "lineno" in detailed_halstead["function1"] assert detailed_halstead["function1"]["lineno"] is not None assert "endline" in detailed_halstead["function1"] - if sys.version_info > (3, 7): + if sys.version_info > (3, 7): # noqa: UP036 # FuncDef is missing end_lineno in Python 3.7 assert detailed_halstead["function1"]["endline"] is not None From 0e1a50b7f0985115c7a8733881de9f59cafe87e0 Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Sun, 24 Sep 2023 10:37:18 -0300 Subject: [PATCH 6/7] Correctly check for Python version < 3.8. --- test/integration/test_complex_commits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/test_complex_commits.py b/test/integration/test_complex_commits.py index c805afb7..1493a8ea 100644 --- a/test/integration/test_complex_commits.py +++ b/test/integration/test_complex_commits.py @@ -233,7 +233,7 @@ def test_metric_entries(tmpdir, cache_path): assert "lineno" in detailed_halstead["function1"] assert detailed_halstead["function1"]["lineno"] is not None assert "endline" in detailed_halstead["function1"] - if sys.version_info > (3, 7): # noqa: UP036 + if sys.version_info >= (3, 8): # FuncDef is missing end_lineno in Python 3.7 assert detailed_halstead["function1"]["endline"] is not None From e4ae10d0bda1c9a5958f1c70fdac20aa8b3c295b Mon Sep 17 00:00:00 2001 From: devdanzin <74280297+devdanzin@users.noreply.github.com> Date: Sun, 24 Sep 2023 10:58:45 -0300 Subject: [PATCH 7/7] Add missing version check for test of Halstead endline. --- test/integration/test_complex_commits.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/test_complex_commits.py b/test/integration/test_complex_commits.py index 1493a8ea..5b25586a 100644 --- a/test/integration/test_complex_commits.py +++ b/test/integration/test_complex_commits.py @@ -243,7 +243,8 @@ def test_metric_entries(tmpdir, cache_path): assert "lineno" in detailed_halstead["Class1.method"] assert detailed_halstead["Class1.method"]["lineno"] is not None assert "endline" in detailed_halstead["Class1.method"] - assert detailed_halstead["Class1.method"]["endline"] is not None + if sys.version_info >= (3, 8): + assert detailed_halstead["Class1.method"]["endline"] is not None expected_raw_total = { "loc": 14,