diff --git a/tests/Tester.py b/tests/Tester.py index 3b04cc593c1..2d374889270 100644 --- a/tests/Tester.py +++ b/tests/Tester.py @@ -1,5 +1,6 @@ import random import textwrap +from dataclasses import dataclass import pickle import hashlib import hedy @@ -12,6 +13,7 @@ import inspect import unittest import utils +import typing from hedy_content import ALL_KEYWORD_LANGUAGES, KEYWORDS from hedy_sourcemap import SourceRange @@ -48,6 +50,52 @@ def __repr__(self): return f'Snippet({self.name})' +@dataclass +class YamlSnippet: + """A snippet found in one of the YAML files. + + This is a replacement of 'Snippet' with fewer fields, only the fields that + are used in the snippet tests. + + `yaml_path` is the path in the YAML where this snippet was found, as an + array of either strings or ints. + + For example, in a YAML file that looks like: + + ``` + adventures: + 1: + code: | + print hello + ``` + + The `yaml_path` would be `['adventures', 1, 'code']`. + """ + filename: str + yaml_path: typing.List + code: str + language: str + level: int + + def __post_init__(self): + # 'code' may be replaced later on when translating keywords + self.original_code = self.code + self.name = f'{self.relative_filename}-{self.yaml_path_str}' + + @property + def relative_filename(self): + return os.path.relpath(self.filename, ROOT_DIR) + + @property + def yaml_path_str(self): + return '.'.join(str(x) for x in self.yaml_path) + + @property + def location(self): + """Returns a description of the location.""" + return f'{self.relative_filename} at {self.yaml_path_str}' + + class SkippedMapping: """ Class used to test if a certain source mapping contains an exception type """ @@ -473,6 +521,8 @@ def indent(code, spaces_amount=2, skip_first_line=False): @staticmethod def translate_keywords_in_snippets(snippets): + """Mutates the snippets in-place.""" + # fill keyword dict for all keyword languages keyword_dict = {} for lang in ALL_KEYWORD_LANGUAGES: @@ -487,20 +537,18 @@ def translate_keywords_in_snippets(snippets): # NOTE: .format() instead of safe_format() on purpose! for snippet in snippets: # store original code - snippet[1].original_code = snippet[1].code + snippet.original_code = snippet.code try: - if snippet[1].language in ALL_KEYWORD_LANGUAGES.keys(): - snippet[1].code = snippet[1].code.format(**keyword_dict[snippet[1].language]) + if snippet.language in ALL_KEYWORD_LANGUAGES.keys(): + snippet.code = snippet.code.format(**keyword_dict[snippet.language]) else: - snippet[1].code = snippet[1].code.format(**english_keywords) + snippet.code = snippet.code.format(**english_keywords) except KeyError: print("This following snippet contains an invalid placeholder...") - print(snippet[1].code) + print(snippet.code) except ValueError: print("This following snippet contains an unclosed invalid placeholder...") - print(snippet[1].code) - - return snippets + print(snippet.code) def format_test_error_md(self, E, snippet: Snippet): """Given a snippet and an exception, return a Markdown string describing the problem.""" @@ -530,7 +578,7 @@ def add_arrow(code): rel_file = os.path.relpath(snippet.filename, ROOT_DIR) message.append(f'## {rel_file}') - message.append(f'There was a problem in a level {snippet.level} snippet:') + message.append(f'There was a problem in a level {snippet.level} snippet, at {snippet.yaml_path_str}:') # Use a 'caution' admonition because it renders in red message.append('> [!CAUTION]') diff --git a/tests/test_snippets/snippet_tester.py b/tests/test_snippets/snippet_tester.py index 526859f176c..fab0a217d0b 100644 --- a/tests/test_snippets/snippet_tester.py +++ b/tests/test_snippets/snippet_tester.py @@ -9,18 +9,17 @@ - Snippets can hold on to the information where in the YAML file they were discovered. That way, individual test suites don't have to supply a 'yaml_locator'. - - """ import os from os import path from dataclasses import dataclass +import functools import exceptions import hedy import utils -from tests.Tester import HedyTester, Snippet +from tests.Tester import HedyTester, YamlSnippet from website.yaml_file import YamlFile fix_error = False @@ -30,150 +29,123 @@ fix_error = True -check_stories = False - - def rootdir(): """Return the repository root directory.""" return os.path.join(os.path.dirname(__file__), '..', '..') -def collect_adventures_snippets(path, filtered_language=None): - Hedy_snippets = [] +def listify(fn): + """Turns a function written as a generator into a function that returns a list. + + Writing a function that produces elements one by one is convenient to write + as a generator (using `yield`), but the return value can only be iterated once. + The caller needs to know that the function is a generator and call `list()` on + the result. + + This decorator does that from the function side: `list()` is automatically + called, so the caller doesn't need to know anything, yet the function is still + nice to read and write. + """ + @functools.wraps(fn) + def wrapper(*args, **kwargs): + return list(fn(*args, **kwargs)) + return wrapper + + +@listify +def collect_adventures_snippets(): + """Find the snippets for adventures.""" + for filename, language, yaml in find_yaml_files('content/adventures'): + for adventure_key, adventure in yaml['adventures'].items(): + # the default tab sometimes contains broken code to make a point to learners about changing syntax. + if adventure_key in ['default', 'debugging']: + continue + + for level_number, level in adventure['levels'].items(): + for markdown_text in level.values(): + for code in markdown_code_blocks(markdown_text.as_string()): + yield YamlSnippet( + code=code, + filename=filename, + language=language, + level=level_number, + yaml_path=markdown_text.yaml_path) + + +@listify +def collect_cheatsheet_snippets(): + """Find the snippets in cheatsheets.""" + for filename, language, yaml in find_yaml_files('content/cheatsheets'): + for level_number, level in yaml.items(): + for command in level: + if code := command.get('demo_code'): + yield YamlSnippet( + code=code.as_string(), + filename=filename, + language=language, + level=level_number, + yaml_path=code.yaml_path) + + +@listify +def collect_parsons_snippets(): + """Find the snippets in Parsons YAMLs.""" + for filename, language, yaml in find_yaml_files('content/parsons'): + for level_number, level in yaml['levels'].items(): + for exercise in level: + code = exercise['code'] + yield YamlSnippet( + code=code.as_string(), + filename=filename, + language=language, + level=level_number, + yaml_path=code.yaml_path) + + +@listify +def collect_slides_snippets(): + """Find the snippets in slides YAMLs.""" + for filename, language, yaml in find_yaml_files('content/slides'): + for level_number, level in yaml['levels'].items(): + for slide in level: + # Some slides have code that is designed to fail + if slide.get('debug'): + continue + + if code := slide.get('code'): + yield YamlSnippet( + code=code.as_string(), + filename=filename, + language=language, + # Level 0 needs to be treated as level 1 + level=max(1, level_number), + yaml_path=code.yaml_path) + + +def find_yaml_files(repository_path): + """Find all YAML files in a given directory, relative to the repository root. + + Returns an iterator of (filename, language, located_yaml_object). + + The `located_yaml_object` is a `LocatedYamlValue` representing the root of the + YAML file, which can be indexed using `[]` and by using `.items()` and `.values()`. + """ + path = os.path.join(rootdir(), repository_path) files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith('.yaml')] + for f in files: + filename = os.path.join(path, f) lang = f.split(".")[0] - # we always store the English snippets, to use them if we need to restore broken code - if not filtered_language or (filtered_language and (lang == filtered_language or lang == 'en')): - f = os.path.join(path, f) - yaml = YamlFile.for_file(f) - - for key, adventure in yaml['adventures'].items(): - # the default tab sometimes contains broken code to make a point to learners about changing syntax. - if not key == 'default' and not key == 'debugging': - for level_number in adventure['levels']: - if level_number > hedy.HEDY_MAX_LEVEL: - print('content above max level!') - else: - level = adventure['levels'][level_number] - adventure_name = adventure['name'] - - for adventure_part, text in level.items(): - # This block is markdown, and there can be multiple code blocks inside it - codes = [tag.contents[0].contents[0] - for tag in utils.markdown_to_html_tags(text) - if tag.name == 'pre' and tag.contents and tag.contents[0].contents] - - if check_stories and adventure_part == 'story_text' and codes != []: - # Can be used to catch languages with example codes in the story_text - # at once point in time, this was the default and some languages still use this old - # structure - - feedback = f"Example code in story text {lang}, {adventure_name},\ - {level_number}, not recommended!" - raise Exception(feedback) - - for i, code in enumerate(codes): - Hedy_snippets.append(Snippet( - filename=f, - level=level_number, - field_name=adventure_part, - code=code, - adventure_name=adventure_name, - key=key, - counter=1)) - - return Hedy_snippets - - -def collect_cheatsheet_snippets(path): - Hedy_snippets = [] - files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith('.yaml')] - for file in files: - lang = file.split(".")[0] - file = os.path.join(path, file) - yaml = YamlFile.for_file(file) - - for level in yaml: - level_number = int(level) - if level_number > hedy.HEDY_MAX_LEVEL: - print('content above max level!') - else: - try: - # commands.k.demo_code - for k, command in enumerate(yaml[level]): - snippet = Snippet( - filename=file, - level=level, - field_name=str(k), - code=command['demo_code']) - Hedy_snippets.append(snippet) - except BaseException: - print(f'Problem reading commands yaml for {lang} level {level}') - - return Hedy_snippets - - -def collect_parsons_snippets(path): - Hedy_snippets = [] - files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith('.yaml')] - for file in files: - lang = file.split(".")[0] - file = os.path.join(path, file) - yaml = YamlFile.for_file(file) - levels = yaml.get('levels') - - for level, content in levels.items(): - level_number = int(level) - if level_number > hedy.HEDY_MAX_LEVEL: - print('content above max level!') - else: - try: - for exercise_id, exercise in levels[level].items(): - code = exercise.get('code') - snippet = Snippet( - filename=file, - level=level, - field_name=f"{exercise_id}", - code=code) - Hedy_snippets.append(snippet) - except BaseException: - print(f'Problem reading commands yaml for {lang} level {level}') - - return Hedy_snippets - - -def collect_slides_snippets(path): - Hedy_snippets = [] - files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path, f)) and f.endswith('.yaml')] - for file in files: - lang = file.split(".")[0] - file = os.path.join(path, file) - yaml = YamlFile.for_file(file) - levels = yaml.get('levels') - - for level, content in levels.items(): - level_number = int(level) - if level_number > hedy.HEDY_MAX_LEVEL: - print('content above max level!') - else: - try: - number = 0 - # commands.k.demo_code - for x, y in content.items(): - if 'code' in y.keys() and 'debug' not in y.keys(): - snippet = Snippet( - filename=file, - level=level_number if level_number > 0 else 1, - language=lang, - field_name=x, - code=y['code']) - Hedy_snippets.append(snippet) - number += 1 - except BaseException: - print(f'Problem reading commands yaml for {lang} level {level}') - - return Hedy_snippets + yaml = YamlFile.for_file(os.path.join(path, f)) + + yield (filename, lang, LocatedYamlValue(yaml, [])) + + +def markdown_code_blocks(text): + """Parse the text as MarkDown and return all code blocks in here.""" + return [tag.contents[0].contents[0] + for tag in utils.markdown_to_html_tags(text) + if tag.name == 'pre' and tag.contents and tag.contents[0].contents] def filter_snippets(snippets, level=None, lang=None): @@ -181,14 +153,22 @@ def filter_snippets(snippets, level=None, lang=None): raise RuntimeError('Whoops, it looks like you left a snippet filter in!') if lang: - snippets = [(name, snippet) for (name, snippet) in snippets if snippet.language[:2] == lang] + snippets = [s for s in snippets if s.language[:len(lang)] == lang] if level: - snippets = [(name, snippet) for (name, snippet) in snippets if snippet.level == level] + snippets = [s for s in snippets if s.level == level] return snippets +def snippets_with_names(snippets): + """Expand a set of snippets to pairs of (name, snippet). + + This is necessary to stick it into @parameterized.expand. + """ + return ((s.name, s) for s in snippets) + + @dataclass class YamlLocation: dict: dict @@ -199,12 +179,9 @@ class HedySnippetTester(HedyTester): """Base class for all other snippet testers. The logic is the same between all of them, so we can combine it. - - 'yaml_locator' is a function that, given a snippet, will tell us where - in the file it was found, by returning a pair of `(containing_dict, """ - def do_snippet(self, snippet, yaml_locator=None): + def do_snippet(self, snippet): if snippet is None or len(snippet.code) == 0: return @@ -225,8 +202,8 @@ def do_snippet(self, snippet, yaml_locator=None): except exceptions.HedyException as E: error_message = self.format_test_error_md(E, snippet) - if fix_error and yaml_locator: - self.restore_snippet_to_english(snippet, yaml_locator) + if fix_error and isinstance(snippet, YamlSnippet): + self.restore_snippet_to_english(snippet) with open(path.join(rootdir(), 'snippet-report.md.tmp'), 'a') as f: f.write(error_message + '\n') @@ -235,19 +212,112 @@ def do_snippet(self, snippet, yaml_locator=None): print(error_message) raise E - def restore_snippet_to_english(self, snippet, yaml_locator): + def restore_snippet_to_english(self, snippet): # English file is always 'en.yaml' in the same dir en_file = path.join(path.dirname(snippet.filename), 'en.yaml') # Read English yaml file original_yaml = YamlFile.for_file(en_file) - original_loc = yaml_locator(snippet, original_yaml) + original_loc = locate_snippet_in_yaml(original_yaml, snippet) # Read broken yaml file broken_yaml = utils.load_yaml_rt(snippet.filename) - broken_loc = yaml_locator(snippet, broken_yaml) + broken_loc = locate_snippet_in_yaml(broken_yaml, snippet) # Restore to English version broken_loc.dict[broken_loc.key] = original_loc.dict[original_loc.key] with open(snippet.filename, 'w') as file: file.write(utils.dump_yaml_rt(broken_yaml)) + + +def locate_snippet_in_yaml(root, snippet): + """Given a YamlSnippet, locate its containing object and key. + + This uses the `yaml_path` to descend into the given YAML object bit + by bit (by indexing the dictionary or list with the next string + or int) until we arrive at the parent object of the string we're + looking for. + """ + path = snippet.yaml_path.copy() + while len(path) > 1: + root = root[path[0]] + path = path[1:] + return YamlLocation(dict=root, key=path[0]) + + +class LocatedYamlValue: + """A value in a YAML file, along with its path inside that YAML file. + + Has features to descend into children of the wrapped value, which also + emits `LocatedYamlValue`s with their paths automatically baked in. + + For example, if we have a referece to the adventure `1` (with path + `['adventures', 1]` in the following YAML: + + adventures: + 1: + code: | + print hello + + And we would write: + + code = level['code'] + + `code` would be a string that knew its path is `['adventures', 1, 'code']`. + + This makes it so that the authors of YAML traversal functions don't have to + keep track of the path of strings, when they finally construct a + `YamlSnippet`. + """ + + def __init__(self, inner, yaml_path): + self.inner = inner + self.yaml_path = yaml_path + + def items(self): + """Returns a set of (key, LocatedYamlValue) values, one for every element of the inner value. + + The inner value must be a dict-like or list. For a list, the indexes will be returned as keys. + """ + if hasattr(self.inner, 'items'): + # Dict-like + return [(k, LocatedYamlValue(v, self.yaml_path + [k])) for k, v in self.inner.items()] + if isinstance(self.inner, list): + # A list can be treated as a dict-like using integer indexes + return [(i, LocatedYamlValue(v, self.yaml_path + [i])) for i, v in enumerate(self.inner)] + raise TypeError('Can only call .items() on a value of type dict or list, got %r' % self.inner) + + def values(self): + """Returns a list of `LocatedYamlValue`s, one for every element in this collection. + + Ignores keys. + """ + return [v for _, v in self.items()] + + def __getitem__(self, key): + """Retrieve a single item from the inner value.""" + ret = self.inner[key] + return LocatedYamlValue(ret, self.yaml_path + [key]) + + def get(self, key, default=None): + """Retrieve a single item from the inner value.""" + ret = self.inner.get(key, default) + if ret is None: + return None + return LocatedYamlValue(ret, self.yaml_path + [key]) + + def __iter__(self): + """Returns an iterator over the values of this container. + + Note that this function behaves differently from a normal dict for dict values: + normal dicts will iterate over dictionary keys, while this type of dict will + iterate over dictionary values. + """ + return iter(self.values()) + + def as_string(self): + """Returns the inner value, failing if it's not a string.""" + if not isinstance(self.inner, str): + raise TypeError('as_string(): expect inner value to be a string, got a %s: %r' % + (type(self.inner), self.inner)) + return self.inner diff --git a/tests/test_snippets/test_adventures.py b/tests/test_snippets/test_adventures.py index 3d6a7a4baf3..d9a2b27fd74 100644 --- a/tests/test_snippets/test_adventures.py +++ b/tests/test_snippets/test_adventures.py @@ -6,26 +6,27 @@ from . import snippet_tester +# Can be used to catch languages with example codes in the story_text +# at once point in time, this was the default and some languages still use this old +# structure check_stories = False -Hedy_snippets = [(s.name, s) for s in snippet_tester.collect_adventures_snippets( - path=path.join(snippet_tester.rootdir(), 'content/adventures'))] -Hedy_snippets = HedyTester.translate_keywords_in_snippets(Hedy_snippets) +snippets = snippet_tester.collect_adventures_snippets() + +if check_stories: + for snippet in snippets: + if snippet.field_path[-1].startswith('story_text'): + raise Exception(f"Example code in story text: {snippet.location}, not recommended!") + +HedyTester.translate_keywords_in_snippets(snippets) # lang = 'zh_hans' #useful if you want to test just 1 language lang = None level = None -Hedy_snippets = snippet_tester.filter_snippets(Hedy_snippets, lang=lang, level=level) +snippets = snippet_tester.filter_snippets(snippets, lang=lang, level=level) class TestsAdventurePrograms(snippet_tester.HedySnippetTester): - @parameterized.expand(Hedy_snippets, skip_on_empty=True) + @parameterized.expand(snippet_tester.snippets_with_names(snippets), skip_on_empty=True) def test_adventures(self, name, snippet): - self.do_snippet(snippet, yaml_locator=adventure_locator) - - -def adventure_locator(snippet, yaml): - """Returns where in the adventures YAML we found an adventure snippet.""" - return snippet_tester.YamlLocation( - dict=yaml['adventures'][snippet.key]['levels'][snippet.level], - key=snippet.field_name) + self.do_snippet(snippet) diff --git a/tests/test_snippets/test_cheatsheets.py b/tests/test_snippets/test_cheatsheets.py index e3697b96985..4503a268d0b 100644 --- a/tests/test_snippets/test_cheatsheets.py +++ b/tests/test_snippets/test_cheatsheets.py @@ -5,25 +5,18 @@ from tests.Tester import HedyTester from . import snippet_tester -Hedy_snippets = [(s.name, s) for s in snippet_tester.collect_cheatsheet_snippets( - path=path.join(snippet_tester.rootdir(), 'content/cheatsheets'))] -Hedy_snippets = HedyTester.translate_keywords_in_snippets(Hedy_snippets) +snippets = snippet_tester.collect_cheatsheet_snippets() + +HedyTester.translate_keywords_in_snippets(snippets) # lang = 'zh_hans' #useful if you want to test just 1 language lang = None level = None -Hedy_snippets = snippet_tester.filter_snippets(Hedy_snippets, lang=lang, level=level) +snippets = snippet_tester.filter_snippets(snippets, lang=lang, level=level) class TestsCheatsheetPrograms(snippet_tester.HedySnippetTester): - @parameterized.expand(Hedy_snippets, skip_on_empty=True) + @parameterized.expand(snippet_tester.snippets_with_names(snippets), skip_on_empty=True) def test_cheatsheets_programs(self, name, snippet): - self.do_snippet(snippet, yaml_locator=cheatsheet_locator) - - -def cheatsheet_locator(snippet, yaml): - """Returns where in the cheatsheet YAML we found a cheatsheet snippet.""" - return snippet_tester.YamlLocation( - dict=yaml[snippet.level][int(snippet.field_name)], - key='demo_code') + self.do_snippet(snippet) diff --git a/tests/test_snippets/test_parsons.py b/tests/test_snippets/test_parsons.py index e24de351723..27b03f448df 100644 --- a/tests/test_snippets/test_parsons.py +++ b/tests/test_snippets/test_parsons.py @@ -5,24 +5,18 @@ from . import snippet_tester -Hedy_snippets = [(s.name, s) for s in snippet_tester.collect_parsons_snippets( - path=path.join(snippet_tester.rootdir(), 'content/parsons'))] -Hedy_snippets = HedyTester.translate_keywords_in_snippets(Hedy_snippets) +snippets = snippet_tester.collect_parsons_snippets() + +HedyTester.translate_keywords_in_snippets(snippets) # lang = 'zh_hans' #useful if you want to test just 1 language lang = None level = None -Hedy_snippets = snippet_tester.filter_snippets(Hedy_snippets, lang=lang, level=level) +snippets = snippet_tester.filter_snippets(snippets, lang=lang, level=level) class TestsParsonsPrograms(snippet_tester.HedySnippetTester): - @parameterized.expand(Hedy_snippets, skip_on_empty=True) - def test_parsons(self, name, snippet): - self.do_snippet(snippet, yaml_locator=parsons_locator) - -def parsons_locator(snippet, yaml): - """Returns where in the Parsons YAML we found a Parsons snippet.""" - return snippet_tester.YamlLocation( - dict=yaml['levels'][snippet.level], - key=int(snippet.field_name)) + @parameterized.expand(snippet_tester.snippets_with_names(snippets), skip_on_empty=True) + def test_parsons(self, name, snippet): + self.do_snippet(snippet) diff --git a/tests/test_snippets/test_slides.py b/tests/test_snippets/test_slides.py index 9cc8d513fae..90f1de0f566 100644 --- a/tests/test_snippets/test_slides.py +++ b/tests/test_snippets/test_slides.py @@ -5,27 +5,18 @@ from tests.Tester import HedyTester from . import snippet_tester -Hedy_snippets = [(s.name, s) for s in snippet_tester.collect_slides_snippets( - path=path.join(snippet_tester.rootdir(), 'content/slides'))] -Hedy_snippets = HedyTester.translate_keywords_in_snippets(Hedy_snippets) + +snippets = snippet_tester.collect_slides_snippets() + +HedyTester.translate_keywords_in_snippets(snippets) # lang = 'zh_hans' #useful if you want to test just 1 language lang = None level = None -Hedy_snippets = snippet_tester.filter_snippets(Hedy_snippets, lang=lang, level=level) - -if lang: - Hedy_snippets = [(name, snippet) for (name, snippet) in Hedy_snippets if snippet.language[:2] == lang] +snippets = snippet_tester.filter_snippets(snippets, lang=lang, level=level) class TestsSlidesPrograms(snippet_tester.HedySnippetTester): - @parameterized.expand(Hedy_snippets, skip_on_empty=True) + @parameterized.expand(snippet_tester.snippets_with_names(snippets), skip_on_empty=True) def test_slide_programs(self, name, snippet): - self.do_snippet(snippet, yaml_locator=slides_locator) - - -def slides_locator(snippet, yaml): - """Returns where in the Slides YAML we found a Slides snippet.""" - return snippet_tester.YamlLocation( - dict=yaml['levels'][snippet.level][snippet.field_name], - key='code') + self.do_snippet(snippet)