From cf53a7c678ceb04a70855de45f1cbf9eb524d096 Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 15 Mar 2024 18:26:56 +0000 Subject: [PATCH 01/57] [tests] JavaScript: extract searchindex.js-format test fixtures --- karma.conf.js | 3 ++- tests/js/fixtures/index.cpp.js | 15 +++++++++++ tests/js/fixtures/index.multiterm.js | 22 ++++++++++++++++ tests/js/searchtools.js | 39 ++++++++++++---------------- 4 files changed, 55 insertions(+), 24 deletions(-) create mode 100644 tests/js/fixtures/index.cpp.js create mode 100644 tests/js/fixtures/index.multiterm.js diff --git a/karma.conf.js b/karma.conf.js index 8a18e80ba7a..5f20030596a 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -19,7 +19,8 @@ module.exports = function(config) { 'sphinx/themes/basic/static/doctools.js', 'sphinx/themes/basic/static/searchtools.js', 'sphinx/themes/basic/static/sphinx_highlight.js', - 'tests/js/*.js' + 'tests/js/*.js', + { pattern: 'tests/js/fixtures/*.js', included: false, served: true } ], diff --git a/tests/js/fixtures/index.cpp.js b/tests/js/fixtures/index.cpp.js new file mode 100644 index 00000000000..8e8cb65e91e --- /dev/null +++ b/tests/js/fixtures/index.cpp.js @@ -0,0 +1,15 @@ +Search.setIndex({ + docnames: [ + "index" + ], + filenames: [ + "index.rst" + ], + terms: { + "c++" : 0 + }, + titles: [ + "<no title>" + ], + titleterms: {} +}); diff --git a/tests/js/fixtures/index.multiterm.js b/tests/js/fixtures/index.multiterm.js new file mode 100644 index 00000000000..d1290e71827 --- /dev/null +++ b/tests/js/fixtures/index.multiterm.js @@ -0,0 +1,22 @@ +Search.setIndex({ + alltitles: { + "Main Page": [[0, "main-page"]] + }, + docnames: [ + "index" + ], + filenames: [ + "index.rst" + ], + terms: { + main: 0, + page: 0 + }, + titles: [ + "Main Page" + ], + titleterms : { + main: 0, + page: 0 + } +}); diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index 91c35a6ba14..47d5e6efb43 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -1,20 +1,22 @@ describe('Basic html theme search', function() { + function loadFixture(name) { + req = new XMLHttpRequest(); + req.open("GET", `base/tests/js/fixtures/${name}`, false); + req.send(null); + return req.responseText; + } + describe('terms search', function() { it('should find "C++" when in index', function() { - index = { - docnames:["index"], - filenames:["index.rst"], - terms:{'c++':0}, - titles:["<no title>"], - titleterms:{} - } - Search.setIndex(index); + searchindex = loadFixture("index.cpp.js"); + eval(searchindex); + searchterms = ['c++']; excluded = []; - terms = index.terms; - titleterms = index.titleterms; + terms = Search._index.terms; + titleterms = Search._index.titleterms; hits = [[ "index", @@ -28,22 +30,13 @@ describe('Basic html theme search', function() { }); it('should be able to search for multiple terms', function() { - index = { - alltitles: { - 'Main Page': [[0, 'main-page']], - }, - docnames:["index"], - filenames:["index.rst"], - terms:{main:0, page:0}, - titles:["Main Page"], - titleterms:{ main:0, page:0 } - } - Search.setIndex(index); + searchindex = loadFixture("index.multiterm.js"); + eval(searchindex); searchterms = ['main', 'page']; excluded = []; - terms = index.terms; - titleterms = index.titleterms; + terms = Search._index.terms; + titleterms = Search._index.titleterms; hits = [[ 'index', 'Main Page', From 311c4f05f8f9a8de1c5d1b0beae829bb91cbd8a5 Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 15 Mar 2024 22:23:25 +0000 Subject: [PATCH 02/57] [tests] JavaScript: write Sphinx sources that produce acceptable searchindex.js test fixtures --- .gitignore | 1 + tests/js/fixtures/index.cpp.js | 16 +--------------- tests/js/fixtures/index.multiterm.js | 23 +---------------------- tests/js/roots/cpp/conf.py | 14 ++++++++++++++ tests/js/roots/cpp/index.rst | 5 +++++ tests/js/roots/multiterm/conf.py | 0 tests/js/roots/multiterm/index.rst | 4 ++++ 7 files changed, 26 insertions(+), 37 deletions(-) create mode 100644 tests/js/roots/cpp/conf.py create mode 100644 tests/js/roots/cpp/index.rst create mode 100644 tests/js/roots/multiterm/conf.py create mode 100644 tests/js/roots/multiterm/index.rst diff --git a/.gitignore b/.gitignore index a2f7d63b3ad..3c87572d26f 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ doc/_build/ doc/locale/ tests/.coverage tests/build/ +tests/js/roots/*/build tests/test-server.lock utils/regression_test.js diff --git a/tests/js/fixtures/index.cpp.js b/tests/js/fixtures/index.cpp.js index 8e8cb65e91e..8be181211a9 100644 --- a/tests/js/fixtures/index.cpp.js +++ b/tests/js/fixtures/index.cpp.js @@ -1,15 +1 @@ -Search.setIndex({ - docnames: [ - "index" - ], - filenames: [ - "index.rst" - ], - terms: { - "c++" : 0 - }, - titles: [ - "<no title>" - ], - titleterms: {} -}); +Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx"]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"a": 0, "c++": 0, "class": 0, "class.": 0, "description": 0, "engine": 0, "fixture.": 0, "generate": 0, "index": 0, "is": 0, "of": 0, "project": 0, "sample": 0, "search": 0, "sphinx": 0, "the": 0, "this": 0, "to": 0, "used": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file diff --git a/tests/js/fixtures/index.multiterm.js b/tests/js/fixtures/index.multiterm.js index d1290e71827..dd88223a1fd 100644 --- a/tests/js/fixtures/index.multiterm.js +++ b/tests/js/fixtures/index.multiterm.js @@ -1,22 +1 @@ -Search.setIndex({ - alltitles: { - "Main Page": [[0, "main-page"]] - }, - docnames: [ - "index" - ], - filenames: [ - "index.rst" - ], - terms: { - main: 0, - page: 0 - }, - titles: [ - "Main Page" - ], - titleterms : { - main: 0, - page: 0 - } -}); +Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"a": [], "c++": [], "class": [], "class.": [], "description": [], "engine": [], "fixture.": [], "generate": [], "index": [], "is": [], "of": [], "project": [], "sample": [], "search": [], "sphinx": [], "the": [], "this": [], "to": [], "used": [], "welcom": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file diff --git a/tests/js/roots/cpp/conf.py b/tests/js/roots/cpp/conf.py new file mode 100644 index 00000000000..5d547e93e57 --- /dev/null +++ b/tests/js/roots/cpp/conf.py @@ -0,0 +1,14 @@ +from sphinx.search import SearchLanguage, languages + + +class NullStemmedLanguage(SearchLanguage): + + def split(self, input: str) -> list[str]: + return input.split() + + def stem(self, word: str) -> str: + return word.lower() + + +languages["xx"] = NullStemmedLanguage +html_search_language = "xx" diff --git a/tests/js/roots/cpp/index.rst b/tests/js/roots/cpp/index.rst new file mode 100644 index 00000000000..c6cb1e12e5a --- /dev/null +++ b/tests/js/roots/cpp/index.rst @@ -0,0 +1,5 @@ +This is a sample C++ project used to generate a search engine index fixture. + +.. cpp:class:: public Sphinx + + The description of Sphinx class. diff --git a/tests/js/roots/multiterm/conf.py b/tests/js/roots/multiterm/conf.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/js/roots/multiterm/index.rst b/tests/js/roots/multiterm/index.rst new file mode 100644 index 00000000000..2e9f4967893 --- /dev/null +++ b/tests/js/roots/multiterm/index.rst @@ -0,0 +1,4 @@ +Main Page +========= + +Welcome to the... main page! From e0bee37646dff04aa844a99d4a44c5a025d9ab3a Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 15 Mar 2024 22:31:18 +0000 Subject: [PATCH 03/57] [tests] linting: add a ruff exclude filter for the JS fixture source roots --- .ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/.ruff.toml b/.ruff.toml index 0d34a34b03e..fb84fefd11b 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -9,6 +9,7 @@ exclude = [ ".tox", ".venv", "tests/roots/*", + "tests/js/roots/*", "build/*", "doc/_build/*", "sphinx/search/*", From 4340bbed2d7f43b1fde1d798cc8db4ff4a2828fe Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 10:34:36 +0000 Subject: [PATCH 04/57] [tests] nitpick: reduce the changeset diff by retaining an index variable in test cases --- tests/js/searchtools.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index 47d5e6efb43..1ad589cf16b 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -12,11 +12,12 @@ describe('Basic html theme search', function() { it('should find "C++" when in index', function() { searchindex = loadFixture("index.cpp.js"); eval(searchindex); + index = Search._index; searchterms = ['c++']; excluded = []; - terms = Search._index.terms; - titleterms = Search._index.titleterms; + terms = index.terms; + titleterms = index.titleterms; hits = [[ "index", @@ -32,11 +33,12 @@ describe('Basic html theme search', function() { it('should be able to search for multiple terms', function() { searchindex = loadFixture("index.multiterm.js"); eval(searchindex); + index = Search._index; searchterms = ['main', 'page']; excluded = []; - terms = Search._index.terms; - titleterms = Search._index.titleterms; + terms = index.terms; + titleterms = index.titleterms; hits = [[ 'index', 'Main Page', From 40fc985bfdc55d4a7977fb5a000c76f03193c588 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 10:41:51 +0000 Subject: [PATCH 05/57] Add CHANGES.rst entry --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index d49fdd9612e..973c4bb45d7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -104,6 +104,7 @@ Testing * pytest: report the result of ``test_run_epubcheck`` as ``skipped`` instead of ``success`` when Java and/or the ``epubcheck.jar`` code are not available. +* karma: refactor HTML search tests to use fixtures generated by Sphinx. Release 7.2.6 (released Sep 13, 2023) ===================================== From 7e22d78d5f051d9f836c0076946beac2884d9ba1 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 10:48:55 +0000 Subject: [PATCH 06/57] [tests] fixup: regenerate multiterm test search index --- tests/js/fixtures/index.multiterm.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/js/fixtures/index.multiterm.js b/tests/js/fixtures/index.multiterm.js index dd88223a1fd..282549de340 100644 --- a/tests/js/fixtures/index.multiterm.js +++ b/tests/js/fixtures/index.multiterm.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"a": [], "c++": [], "class": [], "class.": [], "description": [], "engine": [], "fixture.": [], "generate": [], "index": [], "is": [], "of": [], "project": [], "sample": [], "search": [], "sphinx": [], "the": [], "this": [], "to": [], "used": [], "welcom": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file +Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"welcom": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file From 8f753a865ace1998085e4b1285562129eee80746 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 10:52:21 +0000 Subject: [PATCH 07/57] [tests] refactor: use a separate index fixture directory per root --- karma.conf.js | 2 +- tests/js/fixtures/{index.cpp.js => cpp/searchindex.js} | 0 .../fixtures/{index.multiterm.js => multiterm/searchindex.js} | 0 tests/js/searchtools.js | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename tests/js/fixtures/{index.cpp.js => cpp/searchindex.js} (100%) rename tests/js/fixtures/{index.multiterm.js => multiterm/searchindex.js} (100%) diff --git a/karma.conf.js b/karma.conf.js index 5f20030596a..1ecdfd9fa7d 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -20,7 +20,7 @@ module.exports = function(config) { 'sphinx/themes/basic/static/searchtools.js', 'sphinx/themes/basic/static/sphinx_highlight.js', 'tests/js/*.js', - { pattern: 'tests/js/fixtures/*.js', included: false, served: true } + { pattern: 'tests/js/fixtures/**/*.js', included: false, served: true } ], diff --git a/tests/js/fixtures/index.cpp.js b/tests/js/fixtures/cpp/searchindex.js similarity index 100% rename from tests/js/fixtures/index.cpp.js rename to tests/js/fixtures/cpp/searchindex.js diff --git a/tests/js/fixtures/index.multiterm.js b/tests/js/fixtures/multiterm/searchindex.js similarity index 100% rename from tests/js/fixtures/index.multiterm.js rename to tests/js/fixtures/multiterm/searchindex.js diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index 1ad589cf16b..f994e5890b6 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -10,7 +10,7 @@ describe('Basic html theme search', function() { describe('terms search', function() { it('should find "C++" when in index', function() { - searchindex = loadFixture("index.cpp.js"); + searchindex = loadFixture("cpp/searchindex.js"); eval(searchindex); index = Search._index; @@ -31,7 +31,7 @@ describe('Basic html theme search', function() { }); it('should be able to search for multiple terms', function() { - searchindex = loadFixture("index.multiterm.js"); + searchindex = loadFixture("multiterm/searchindex.js"); eval(searchindex); index = Search._index; From 695037d9b77fed334e2c7c6e265922020a6b74d1 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 12:22:21 +0000 Subject: [PATCH 08/57] [search] Refactor search code to improve testability --- sphinx/themes/basic/static/searchtools.js | 32 +++++++++++++++-------- tests/js/fixtures/cpp/searchindex.js | 2 +- tests/js/language_data.js | 26 ++++++++++++++++++ tests/js/roots/cpp/conf.py | 14 ---------- tests/js/searchtools.js | 25 ++++++------------ 5 files changed, 56 insertions(+), 43 deletions(-) create mode 100644 tests/js/language_data.js diff --git a/sphinx/themes/basic/static/searchtools.js b/sphinx/themes/basic/static/searchtools.js index 1197fa9f4a0..db25cfe41a3 100644 --- a/sphinx/themes/basic/static/searchtools.js +++ b/sphinx/themes/basic/static/searchtools.js @@ -250,16 +250,7 @@ const Search = { else Search.deferQuery(query); }, - /** - * execute search (requires search index to be loaded) - */ - query: (query) => { - const filenames = Search._index.filenames; - const docNames = Search._index.docnames; - const titles = Search._index.titles; - const allTitles = Search._index.alltitles; - const indexEntries = Search._index.indexentries; - + _parseQuery: (query) => { // stem the search terms and add them to the correct list const stemmer = new Stemmer(); const searchTerms = new Set(); @@ -295,6 +286,19 @@ const Search = { // console.info("required: ", [...searchTerms]); // console.info("excluded: ", [...excludedTerms]); + return [query, searchTerms, excludedTerms, highlightTerms, objectTerms]; + }, + + /** + * execute search (requires search index to be loaded) + */ + _performSearch: (query, searchTerms, excludedTerms, highlightTerms, objectTerms) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + const allTitles = Search._index.alltitles; + const indexEntries = Search._index.indexentries; + // array of [docname, title, anchor, descr, score, filename] let results = []; _removeChildren(document.getElementById("search-progress")); @@ -372,7 +376,13 @@ const Search = { return acc; }, []); - results = results.reverse(); + return results.reverse(); + }, + + query: (query) => { + + const searchTerms = Search._parseQuery(query); + const results = Search._performSearch(...searchTerms); // for debugging //Search.lastresults = results.slice(); // a copy diff --git a/tests/js/fixtures/cpp/searchindex.js b/tests/js/fixtures/cpp/searchindex.js index 8be181211a9..3a42936c501 100644 --- a/tests/js/fixtures/cpp/searchindex.js +++ b/tests/js/fixtures/cpp/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx"]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"a": 0, "c++": 0, "class": 0, "class.": 0, "description": 0, "engine": 0, "fixture.": 0, "generate": 0, "index": 0, "is": 0, "of": 0, "project": 0, "sample": 0, "search": 0, "sphinx": 0, "the": 0, "this": 0, "to": 0, "used": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file +Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx"]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "a": [], "c": 0, "c++": [], "class": 0, "class.": [], "descript": 0, "description": [], "engin": 0, "engine": [], "fixtur": 0, "fixture.": [], "gener": 0, "generate": [], "i": 0, "index": 0, "is": [], "of": [], "project": 0, "sampl": 0, "sample": [], "search": 0, "sphinx": 0, "the": [], "thi": 0, "this": [], "to": [], "us": 0, "used": []}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file diff --git a/tests/js/language_data.js b/tests/js/language_data.js new file mode 100644 index 00000000000..db395c75dc0 --- /dev/null +++ b/tests/js/language_data.js @@ -0,0 +1,26 @@ +/* + * language_data.js + * ~~~~~~~~~~~~~~~~ + * + * This script contains the language-specific data used by searchtools.js, + * namely the list of stopwords, stemmer, scorer and splitter. + * + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +var stopwords = []; + + +/* Non-minified version is copied as a separate JS file, is available */ + +/** + * Dummy stemmer for languages without stemming rules. + */ +var Stemmer = function() { + this.stemWord = function(w) { + return w; + } +} + diff --git a/tests/js/roots/cpp/conf.py b/tests/js/roots/cpp/conf.py index 5d547e93e57..e69de29bb2d 100644 --- a/tests/js/roots/cpp/conf.py +++ b/tests/js/roots/cpp/conf.py @@ -1,14 +0,0 @@ -from sphinx.search import SearchLanguage, languages - - -class NullStemmedLanguage(SearchLanguage): - - def split(self, input: str) -> list[str]: - return input.split() - - def stem(self, word: str) -> str: - return word.lower() - - -languages["xx"] = NullStemmedLanguage -html_search_language = "xx" diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index f994e5890b6..e1bd0827b88 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -10,14 +10,9 @@ describe('Basic html theme search', function() { describe('terms search', function() { it('should find "C++" when in index', function() { - searchindex = loadFixture("cpp/searchindex.js"); - eval(searchindex); - index = Search._index; + eval(loadFixture("cpp/searchindex.js")); - searchterms = ['c++']; - excluded = []; - terms = index.terms; - titleterms = index.titleterms; + searchTerms = Search._parseQuery('C++'); hits = [[ "index", @@ -27,18 +22,14 @@ describe('Basic html theme search', function() { 5, "index.rst" ]]; - expect(Search.performTermsSearch(searchterms, excluded, terms, titleterms)).toEqual(hits); + expect(Search._performSearch(...searchTerms)).toEqual(hits); }); it('should be able to search for multiple terms', function() { - searchindex = loadFixture("multiterm/searchindex.js"); - eval(searchindex); - index = Search._index; - - searchterms = ['main', 'page']; - excluded = []; - terms = index.terms; - titleterms = index.titleterms; + eval(loadFixture("multiterm/searchindex.js")); + + searchTerms = Search._parseQuery('main page'); + hits = [[ 'index', 'Main Page', @@ -46,7 +37,7 @@ describe('Basic html theme search', function() { null, 15, 'index.rst']]; - expect(Search.performTermsSearch(searchterms, excluded, terms, titleterms)).toEqual(hits); + expect(Search._performSearch(...searchTerms)).toEqual(hits); }); }); From 58b5e901dbd9be52fc139fd0d8ef898d9756e9a9 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 12:48:08 +0000 Subject: [PATCH 09/57] [tests] expect duplicate result due to #11961 --- tests/js/searchtools.js | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index e1bd0827b88..5540be1f0de 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -30,13 +30,25 @@ describe('Basic html theme search', function() { searchTerms = Search._parseQuery('main page'); - hits = [[ - 'index', - 'Main Page', - '', - null, - 15, - 'index.rst']]; + // fixme: duplicate result due to https://github.com/sphinx-doc/sphinx/issues/11961 + hits = [ + [ + 'index', + 'Main Page', + '', + null, + 15, + 'index.rst' + ], + [ + 'index', + 'Main Page', + '#main-page', + null, + 100, + 'index.rst' + ] + ]; expect(Search._performSearch(...searchTerms)).toEqual(hits); }); From c37bb53bf61dec291aa16bedb9d364718130b0db Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 15:08:24 +0000 Subject: [PATCH 10/57] [tests] run 'js-beautify -r ...' on each searchindex.js file --- tests/js/fixtures/cpp/searchindex.js | 58 +++++++++++++++++++++- tests/js/fixtures/multiterm/searchindex.js | 35 ++++++++++++- 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/tests/js/fixtures/cpp/searchindex.js b/tests/js/fixtures/cpp/searchindex.js index 8be181211a9..a73122672c2 100644 --- a/tests/js/fixtures/cpp/searchindex.js +++ b/tests/js/fixtures/cpp/searchindex.js @@ -1 +1,57 @@ -Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx"]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"a": 0, "c++": 0, "class": 0, "class.": 0, "description": 0, "engine": 0, "fixture.": 0, "generate": 0, "index": 0, "is": 0, "of": 0, "project": 0, "sample": 0, "search": 0, "sphinx": 0, "the": 0, "this": 0, "to": 0, "used": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file +Search.setIndex({ + "alltitles": {}, + "docnames": ["index"], + "envversion": { + "sphinx": 61, + "sphinx.domains.c": 3, + "sphinx.domains.changeset": 1, + "sphinx.domains.citation": 1, + "sphinx.domains.cpp": 9, + "sphinx.domains.index": 1, + "sphinx.domains.javascript": 3, + "sphinx.domains.math": 2, + "sphinx.domains.python": 4, + "sphinx.domains.rst": 2, + "sphinx.domains.std": 2 + }, + "filenames": ["index.rst"], + "indexentries": { + "sphinx (c++ class)": [ + [0, "_CPPv46Sphinx"] + ] + }, + "objects": { + "": [ + [0, 0, 1, "_CPPv46Sphinx", "Sphinx"] + ] + }, + "objnames": { + "0": ["cpp", "class", "C++ class"] + }, + "objtypes": { + "0": "cpp:class" + }, + "terms": { + "a": 0, + "c++": 0, + "class": 0, + "class.": 0, + "description": 0, + "engine": 0, + "fixture.": 0, + "generate": 0, + "index": 0, + "is": 0, + "of": 0, + "project": 0, + "sample": 0, + "search": 0, + "sphinx": 0, + "the": 0, + "this": 0, + "to": 0, + "used": 0 + }, + "titles": ["<no title>"], + "titleterms": {} +}) \ No newline at end of file diff --git a/tests/js/fixtures/multiterm/searchindex.js b/tests/js/fixtures/multiterm/searchindex.js index 282549de340..976679c9a04 100644 --- a/tests/js/fixtures/multiterm/searchindex.js +++ b/tests/js/fixtures/multiterm/searchindex.js @@ -1 +1,34 @@ -Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"welcom": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file +Search.setIndex({ + "alltitles": { + "Main Page": [ + [0, "main-page"] + ] + }, + "docnames": ["index"], + "envversion": { + "sphinx": 61, + "sphinx.domains.c": 3, + "sphinx.domains.changeset": 1, + "sphinx.domains.citation": 1, + "sphinx.domains.cpp": 9, + "sphinx.domains.index": 1, + "sphinx.domains.javascript": 3, + "sphinx.domains.math": 2, + "sphinx.domains.python": 4, + "sphinx.domains.rst": 2, + "sphinx.domains.std": 2 + }, + "filenames": ["index.rst"], + "indexentries": {}, + "objects": {}, + "objnames": {}, + "objtypes": {}, + "terms": { + "welcom": 0 + }, + "titles": ["Main Page"], + "titleterms": { + "main": 0, + "page": 0 + } +}) \ No newline at end of file From 8ac1b6ccdaea6ba52eae214c52d29bf904380fe9 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 15:13:46 +0000 Subject: [PATCH 11/57] [tests] Fixup: regenerate cpp searchindex.js --- tests/js/fixtures/cpp/searchindex.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/js/fixtures/cpp/searchindex.js b/tests/js/fixtures/cpp/searchindex.js index 3a42936c501..e110a2853e4 100644 --- a/tests/js/fixtures/cpp/searchindex.js +++ b/tests/js/fixtures/cpp/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx"]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "a": [], "c": 0, "c++": [], "class": 0, "class.": [], "descript": 0, "description": [], "engin": 0, "engine": [], "fixtur": 0, "fixture.": [], "gener": 0, "generate": [], "i": 0, "index": 0, "is": [], "of": [], "project": 0, "sampl": 0, "sample": [], "search": 0, "sphinx": 0, "the": [], "thi": 0, "this": [], "to": [], "us": 0, "used": []}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file +Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx"]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "c": 0, "class": 0, "descript": 0, "engin": 0, "fixtur": 0, "gener": 0, "i": 0, "index": 0, "project": 0, "sampl": 0, "search": 0, "sphinx": 0, "thi": 0, "us": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file From fbaff49c264640caefd190db3afc10b4cea6e406 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 15:59:31 +0000 Subject: [PATCH 12/57] [utils] Add script to regenerate JavaScript test fixtures --- utils/generate_js_fixtures.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 utils/generate_js_fixtures.py diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py new file mode 100755 index 00000000000..d9afa029037 --- /dev/null +++ b/utils/generate_js_fixtures.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +import subprocess +from pathlib import Path + +SPHINX_ROOT = Path(__file__).resolve().parents[1] +TEST_JS_FIXTURES = SPHINX_ROOT / "tests" / "js" / "fixtures" +TEST_JS_ROOTS = SPHINX_ROOT / "tests" / "js" / "roots" + + +def build(srcdir: Path) -> None: + cmd = ("sphinx-build", "-E", "-q", "-b", "html", f"{srcdir}", f"{srcdir}/build") + subprocess.run(cmd, check=True, capture_output=True) + + +def beautify(filename: Path) -> None: + cmd = ("js-beautify", "-r", filename) + subprocess.run(cmd, check=True, capture_output=True) + + +for directory in TEST_JS_ROOTS.iterdir(): + searchindex = directory / "build" / "searchindex.js" + destination = TEST_JS_FIXTURES / directory.name / "searchindex.js" + + print(f"Building {directory} ... ", end="") + build(directory) + print("done") + + print(f"Beautifying {searchindex} ... ", end="") + beautify(searchindex) + print("done") + + print(f"Moving {searchindex} to {destination} ... ", end="") + searchindex.replace(destination) + print("done") From d619a5211a5b863896996e71ee92ffb7f60866d5 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 16:38:15 +0000 Subject: [PATCH 13/57] [tests] nitpick: relocate project-related JavaScript files away from directory containing test case files --- karma.conf.js | 3 ++- tests/js/{ => project}/documentation_options.js | 0 tests/js/{ => project}/language_data.js | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename tests/js/{ => project}/documentation_options.js (100%) rename tests/js/{ => project}/language_data.js (100%) diff --git a/karma.conf.js b/karma.conf.js index 1ecdfd9fa7d..b2bd6d2350e 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -15,7 +15,8 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ - 'tests/js/documentation_options.js', + 'tests/js/project/documentation_options.js', + 'tests/js/project/language_data.js', 'sphinx/themes/basic/static/doctools.js', 'sphinx/themes/basic/static/searchtools.js', 'sphinx/themes/basic/static/sphinx_highlight.js', diff --git a/tests/js/documentation_options.js b/tests/js/project/documentation_options.js similarity index 100% rename from tests/js/documentation_options.js rename to tests/js/project/documentation_options.js diff --git a/tests/js/language_data.js b/tests/js/project/language_data.js similarity index 100% rename from tests/js/language_data.js rename to tests/js/project/language_data.js From ee4316d80bcc93c230b410edf508c2f5e70571a4 Mon Sep 17 00:00:00 2001 From: James Addison Date: Mon, 18 Mar 2024 19:32:37 +0000 Subject: [PATCH 14/57] [tests] Regenerate test fixtures by running 'python utils/generate_js_fixtures.py' --- tests/js/fixtures/cpp/searchindex.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/js/fixtures/cpp/searchindex.js b/tests/js/fixtures/cpp/searchindex.js index 3948913c309..17f1ae8d9b8 100644 --- a/tests/js/fixtures/cpp/searchindex.js +++ b/tests/js/fixtures/cpp/searchindex.js @@ -17,7 +17,7 @@ Search.setIndex({ "filenames": ["index.rst"], "indexentries": { "sphinx (c++ class)": [ - [0, "_CPPv46Sphinx"] + [0, "_CPPv46Sphinx", false] ] }, "objects": { From 0bc558abddc362be3e714920ac73c6b045217a30 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 16 Mar 2024 15:59:31 +0000 Subject: [PATCH 15/57] [utils] Add script to regenerate JavaScript test fixtures --- utils/generate_js_fixtures.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 utils/generate_js_fixtures.py diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py new file mode 100755 index 00000000000..d9afa029037 --- /dev/null +++ b/utils/generate_js_fixtures.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 + +import subprocess +from pathlib import Path + +SPHINX_ROOT = Path(__file__).resolve().parents[1] +TEST_JS_FIXTURES = SPHINX_ROOT / "tests" / "js" / "fixtures" +TEST_JS_ROOTS = SPHINX_ROOT / "tests" / "js" / "roots" + + +def build(srcdir: Path) -> None: + cmd = ("sphinx-build", "-E", "-q", "-b", "html", f"{srcdir}", f"{srcdir}/build") + subprocess.run(cmd, check=True, capture_output=True) + + +def beautify(filename: Path) -> None: + cmd = ("js-beautify", "-r", filename) + subprocess.run(cmd, check=True, capture_output=True) + + +for directory in TEST_JS_ROOTS.iterdir(): + searchindex = directory / "build" / "searchindex.js" + destination = TEST_JS_FIXTURES / directory.name / "searchindex.js" + + print(f"Building {directory} ... ", end="") + build(directory) + print("done") + + print(f"Beautifying {searchindex} ... ", end="") + beautify(searchindex) + print("done") + + print(f"Moving {searchindex} to {destination} ... ", end="") + searchindex.replace(destination) + print("done") From a74bc156d5f10ce3e5f87cfc56f51d7100976e9b Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 22 Mar 2024 13:44:33 +0000 Subject: [PATCH 16/57] [utils] Remove beautification step for generated searchindex.js files. Ref: https://github.com/sphinx-doc/sphinx/pull/12102#discussion_r1535606662 --- utils/generate_js_fixtures.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py index d9afa029037..beb5254393a 100755 --- a/utils/generate_js_fixtures.py +++ b/utils/generate_js_fixtures.py @@ -26,10 +26,6 @@ def beautify(filename: Path) -> None: build(directory) print("done") - print(f"Beautifying {searchindex} ... ", end="") - beautify(searchindex) - print("done") - print(f"Moving {searchindex} to {destination} ... ", end="") searchindex.replace(destination) print("done") From 68a6504c8922652a6c389f0934d3958b0810e26d Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 22 Mar 2024 13:50:47 +0000 Subject: [PATCH 17/57] [tests] Regenerate searchindex.js files. --- tests/js/fixtures/cpp/searchindex.js | 54 +--------------------- tests/js/fixtures/multiterm/searchindex.js | 35 +------------- 2 files changed, 2 insertions(+), 87 deletions(-) diff --git a/tests/js/fixtures/cpp/searchindex.js b/tests/js/fixtures/cpp/searchindex.js index 17f1ae8d9b8..b829f7cc320 100644 --- a/tests/js/fixtures/cpp/searchindex.js +++ b/tests/js/fixtures/cpp/searchindex.js @@ -1,53 +1 @@ -Search.setIndex({ - "alltitles": {}, - "docnames": ["index"], - "envversion": { - "sphinx": 61, - "sphinx.domains.c": 3, - "sphinx.domains.changeset": 1, - "sphinx.domains.citation": 1, - "sphinx.domains.cpp": 9, - "sphinx.domains.index": 1, - "sphinx.domains.javascript": 3, - "sphinx.domains.math": 2, - "sphinx.domains.python": 4, - "sphinx.domains.rst": 2, - "sphinx.domains.std": 2 - }, - "filenames": ["index.rst"], - "indexentries": { - "sphinx (c++ class)": [ - [0, "_CPPv46Sphinx", false] - ] - }, - "objects": { - "": [ - [0, 0, 1, "_CPPv46Sphinx", "Sphinx"] - ] - }, - "objnames": { - "0": ["cpp", "class", "C++ class"] - }, - "objtypes": { - "0": "cpp:class" - }, - "terms": { - "The": 0, - "c": 0, - "class": 0, - "descript": 0, - "engin": 0, - "fixtur": 0, - "gener": 0, - "i": 0, - "index": 0, - "project": 0, - "sampl": 0, - "search": 0, - "sphinx": 0, - "thi": 0, - "us": 0 - }, - "titles": ["<no title>"], - "titleterms": {} -}) \ No newline at end of file +Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx", false]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "c": 0, "class": 0, "descript": 0, "engin": 0, "fixtur": 0, "gener": 0, "i": 0, "index": 0, "project": 0, "sampl": 0, "search": 0, "sphinx": 0, "thi": 0, "us": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file diff --git a/tests/js/fixtures/multiterm/searchindex.js b/tests/js/fixtures/multiterm/searchindex.js index 976679c9a04..282549de340 100644 --- a/tests/js/fixtures/multiterm/searchindex.js +++ b/tests/js/fixtures/multiterm/searchindex.js @@ -1,34 +1 @@ -Search.setIndex({ - "alltitles": { - "Main Page": [ - [0, "main-page"] - ] - }, - "docnames": ["index"], - "envversion": { - "sphinx": 61, - "sphinx.domains.c": 3, - "sphinx.domains.changeset": 1, - "sphinx.domains.citation": 1, - "sphinx.domains.cpp": 9, - "sphinx.domains.index": 1, - "sphinx.domains.javascript": 3, - "sphinx.domains.math": 2, - "sphinx.domains.python": 4, - "sphinx.domains.rst": 2, - "sphinx.domains.std": 2 - }, - "filenames": ["index.rst"], - "indexentries": {}, - "objects": {}, - "objnames": {}, - "objtypes": {}, - "terms": { - "welcom": 0 - }, - "titles": ["Main Page"], - "titleterms": { - "main": 0, - "page": 0 - } -}) \ No newline at end of file +Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"welcom": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file From 64ecd41b3d4a4e3f683245488b29f8ed37e1f490 Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 22 Mar 2024 14:27:35 +0000 Subject: [PATCH 18/57] [linting] Reformat using 'ruff format'. (cherry picked from commit 9bed49541ec660420273b2e02724b15d09c9fd07) --- utils/generate_js_fixtures.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py index beb5254393a..7d452aad5c7 100755 --- a/utils/generate_js_fixtures.py +++ b/utils/generate_js_fixtures.py @@ -4,28 +4,28 @@ from pathlib import Path SPHINX_ROOT = Path(__file__).resolve().parents[1] -TEST_JS_FIXTURES = SPHINX_ROOT / "tests" / "js" / "fixtures" -TEST_JS_ROOTS = SPHINX_ROOT / "tests" / "js" / "roots" +TEST_JS_FIXTURES = SPHINX_ROOT / 'tests' / 'js' / 'fixtures' +TEST_JS_ROOTS = SPHINX_ROOT / 'tests' / 'js' / 'roots' def build(srcdir: Path) -> None: - cmd = ("sphinx-build", "-E", "-q", "-b", "html", f"{srcdir}", f"{srcdir}/build") + cmd = ('sphinx-build', '-E', '-q', '-b', 'html', f'{srcdir}', f'{srcdir}/build') subprocess.run(cmd, check=True, capture_output=True) def beautify(filename: Path) -> None: - cmd = ("js-beautify", "-r", filename) + cmd = ('js-beautify', '-r', filename) subprocess.run(cmd, check=True, capture_output=True) for directory in TEST_JS_ROOTS.iterdir(): - searchindex = directory / "build" / "searchindex.js" - destination = TEST_JS_FIXTURES / directory.name / "searchindex.js" + searchindex = directory / 'build' / 'searchindex.js' + destination = TEST_JS_FIXTURES / directory.name / 'searchindex.js' - print(f"Building {directory} ... ", end="") + print(f'Building {directory} ... ', end='') build(directory) - print("done") + print('done') - print(f"Moving {searchindex} to {destination} ... ", end="") + print(f'Moving {searchindex} to {destination} ... ', end='') searchindex.replace(destination) - print("done") + print('done') From 1addb80920ee819fbf30932b31217b6b594e511a Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 22 Mar 2024 14:32:37 +0000 Subject: [PATCH 19/57] [tests] Regenerate searchindex.js files. --- tests/js/fixtures/cpp/searchindex.js | 54 +--------------------- tests/js/fixtures/multiterm/searchindex.js | 35 +------------- 2 files changed, 2 insertions(+), 87 deletions(-) diff --git a/tests/js/fixtures/cpp/searchindex.js b/tests/js/fixtures/cpp/searchindex.js index 17f1ae8d9b8..b829f7cc320 100644 --- a/tests/js/fixtures/cpp/searchindex.js +++ b/tests/js/fixtures/cpp/searchindex.js @@ -1,53 +1 @@ -Search.setIndex({ - "alltitles": {}, - "docnames": ["index"], - "envversion": { - "sphinx": 61, - "sphinx.domains.c": 3, - "sphinx.domains.changeset": 1, - "sphinx.domains.citation": 1, - "sphinx.domains.cpp": 9, - "sphinx.domains.index": 1, - "sphinx.domains.javascript": 3, - "sphinx.domains.math": 2, - "sphinx.domains.python": 4, - "sphinx.domains.rst": 2, - "sphinx.domains.std": 2 - }, - "filenames": ["index.rst"], - "indexentries": { - "sphinx (c++ class)": [ - [0, "_CPPv46Sphinx", false] - ] - }, - "objects": { - "": [ - [0, 0, 1, "_CPPv46Sphinx", "Sphinx"] - ] - }, - "objnames": { - "0": ["cpp", "class", "C++ class"] - }, - "objtypes": { - "0": "cpp:class" - }, - "terms": { - "The": 0, - "c": 0, - "class": 0, - "descript": 0, - "engin": 0, - "fixtur": 0, - "gener": 0, - "i": 0, - "index": 0, - "project": 0, - "sampl": 0, - "search": 0, - "sphinx": 0, - "thi": 0, - "us": 0 - }, - "titles": ["<no title>"], - "titleterms": {} -}) \ No newline at end of file +Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx", false]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "c": 0, "class": 0, "descript": 0, "engin": 0, "fixtur": 0, "gener": 0, "i": 0, "index": 0, "project": 0, "sampl": 0, "search": 0, "sphinx": 0, "thi": 0, "us": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file diff --git a/tests/js/fixtures/multiterm/searchindex.js b/tests/js/fixtures/multiterm/searchindex.js index 976679c9a04..282549de340 100644 --- a/tests/js/fixtures/multiterm/searchindex.js +++ b/tests/js/fixtures/multiterm/searchindex.js @@ -1,34 +1 @@ -Search.setIndex({ - "alltitles": { - "Main Page": [ - [0, "main-page"] - ] - }, - "docnames": ["index"], - "envversion": { - "sphinx": 61, - "sphinx.domains.c": 3, - "sphinx.domains.changeset": 1, - "sphinx.domains.citation": 1, - "sphinx.domains.cpp": 9, - "sphinx.domains.index": 1, - "sphinx.domains.javascript": 3, - "sphinx.domains.math": 2, - "sphinx.domains.python": 4, - "sphinx.domains.rst": 2, - "sphinx.domains.std": 2 - }, - "filenames": ["index.rst"], - "indexentries": {}, - "objects": {}, - "objnames": {}, - "objtypes": {}, - "terms": { - "welcom": 0 - }, - "titles": ["Main Page"], - "titleterms": { - "main": 0, - "page": 0 - } -}) \ No newline at end of file +Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"welcom": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file From 52ee7c455c24a38aa2e05e7992fde9735bc8022e Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 10:34:55 +0100 Subject: [PATCH 20/57] [utils] Cleanup: remove unused function. Relates-to commit a74bc156d5f10ce3e5f87cfc56f51d7100976e9b. --- utils/generate_js_fixtures.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py index 7d452aad5c7..0ba3d5132a5 100755 --- a/utils/generate_js_fixtures.py +++ b/utils/generate_js_fixtures.py @@ -13,11 +13,6 @@ def build(srcdir: Path) -> None: subprocess.run(cmd, check=True, capture_output=True) -def beautify(filename: Path) -> None: - cmd = ('js-beautify', '-r', filename) - subprocess.run(cmd, check=True, capture_output=True) - - for directory in TEST_JS_ROOTS.iterdir(): searchindex = directory / 'build' / 'searchindex.js' destination = TEST_JS_FIXTURES / directory.name / 'searchindex.js' From 916e5afe64f7a4a6f4be9f442012364afe4f1039 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 11:05:02 +0100 Subject: [PATCH 21/57] [tests] Add test case to confirm that JavaScript search fixtures remain the same when regenerated. --- tests/test_search.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/test_search.py b/tests/test_search.py index fc50162e56e..5d7863cc13e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -2,7 +2,9 @@ import json import warnings +from hashlib import sha256 from io import BytesIO +from pathlib import Path import pytest from docutils import frontend, utils @@ -341,3 +343,20 @@ def assert_is_sorted(item, path): # Pretty print the index. Only shown by pytest on failure. print(f'searchindex.js contents:\n\n{json.dumps(index, indent=2)}') assert_is_sorted(index, '') + + +def test_check_js_search_indexes(make_app): + projects = {} + + TEST_JS_ROOTS = Path(__file__).resolve().parent / 'js' / 'roots' + for directory in TEST_JS_ROOTS.iterdir(): + app = make_app('html', srcdir=directory) + app.build() + searchindex = (app.outdir / 'searchindex.js').read_bytes() + projects[directory.name] = sha256(searchindex).digest() + + TEST_JS_FIXTURES = Path(__file__).resolve().parent / 'js' / 'fixtures' + for directory in TEST_JS_FIXTURES.iterdir(): + msg = f"Search index fixture in {directory} does not match regenerated copy." + searchindex = (directory / 'searchindex.js').read_bytes() + assert projects[directory.name] == sha256(searchindex).digest(), msg From de6f7a04b5627bda9191ba789c7f83f982494b8e Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 11:24:11 +0100 Subject: [PATCH 22/57] [docs] Add debugging tip for JavaScript searchindex.js fixtures. --- doc/internals/contributing.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/internals/contributing.rst b/doc/internals/contributing.rst index 207374e8f87..f8dc754aaf4 100644 --- a/doc/internals/contributing.rst +++ b/doc/internals/contributing.rst @@ -338,3 +338,8 @@ Debugging tips Minified files in ``sphinx/search/minified-js/*.js`` are generated from non-minified ones using ``uglifyjs`` (installed via npm), with ``-m`` option to enable mangling. + +* The ``searchindex.js`` files found in the ``tests/js/roots/*`` directories + are generated using the Sphinx HTML builder, and are provided as test + fixtures to the Sphinx JavaScript test suite. A utility script found at + ``utils/generate_js_fixtures.py`` can be used to regenerate them. From ef8e4b091752546661816362b473c792fe759362 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 12:07:14 +0100 Subject: [PATCH 23/57] [tests] Refactor: rewrite searchindex test using pytest parametrize. --- tests/test_search.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index 5d7863cc13e..9748ee91af5 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -345,18 +345,18 @@ def assert_is_sorted(item, path): assert_is_sorted(index, '') -def test_check_js_search_indexes(make_app): - projects = {} - - TEST_JS_ROOTS = Path(__file__).resolve().parent / 'js' / 'roots' - for directory in TEST_JS_ROOTS.iterdir(): - app = make_app('html', srcdir=directory) - app.build() - searchindex = (app.outdir / 'searchindex.js').read_bytes() - projects[directory.name] = sha256(searchindex).digest() - - TEST_JS_FIXTURES = Path(__file__).resolve().parent / 'js' / 'fixtures' - for directory in TEST_JS_FIXTURES.iterdir(): - msg = f"Search index fixture in {directory} does not match regenerated copy." - searchindex = (directory / 'searchindex.js').read_bytes() - assert projects[directory.name] == sha256(searchindex).digest(), msg +@pytest.mark.parametrize('directory', ( + directory for directory in (Path(__file__).resolve().parent / 'js' / 'roots').iterdir() +)) +def test_check_js_search_indexes(make_app, directory): + app = make_app('html', srcdir=directory) + app.build() + + fresh_searchindex = (app.outdir / 'searchindex.js').read_bytes() + fresh_hash = sha256(fresh_searchindex).digest() + + existing_searchindex = (directory / '..' / '..' / 'fixtures' / directory.name / 'searchindex.js').read_bytes() + existing_hash = sha256(existing_searchindex).digest() + + msg = f"Search index fixture in {directory} does not match regenerated copy." + assert fresh_hash == existing_hash, msg From 450aafedc7b85563b32f6ac6f554be5810fedcf4 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 12:28:45 +0100 Subject: [PATCH 24/57] [tests] Refactor: use temporary directory when regenerating search indexes. --- tests/test_search.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index 9748ee91af5..692bf3c3111 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -348,8 +348,8 @@ def assert_is_sorted(item, path): @pytest.mark.parametrize('directory', ( directory for directory in (Path(__file__).resolve().parent / 'js' / 'roots').iterdir() )) -def test_check_js_search_indexes(make_app, directory): - app = make_app('html', srcdir=directory) +def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): + app = make_app('html', srcdir=directory, builddir=sphinx_test_tempdir / directory.name) app.build() fresh_searchindex = (app.outdir / 'searchindex.js').read_bytes() From d885288b95bf83190b52f4da3828a8be1f8527c3 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:09:53 +0100 Subject: [PATCH 25/57] Revert "[tests] nitpick: relocate project-related JavaScript files away from directory containing test case files" This reverts commit d619a5211a5b863896996e71ee92ffb7f60866d5. --- karma.conf.js | 3 +-- tests/js/{project => }/documentation_options.js | 0 tests/js/{project => }/language_data.js | 0 3 files changed, 1 insertion(+), 2 deletions(-) rename tests/js/{project => }/documentation_options.js (100%) rename tests/js/{project => }/language_data.js (100%) diff --git a/karma.conf.js b/karma.conf.js index b2bd6d2350e..1ecdfd9fa7d 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -15,8 +15,7 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ - 'tests/js/project/documentation_options.js', - 'tests/js/project/language_data.js', + 'tests/js/documentation_options.js', 'sphinx/themes/basic/static/doctools.js', 'sphinx/themes/basic/static/searchtools.js', 'sphinx/themes/basic/static/sphinx_highlight.js', diff --git a/tests/js/project/documentation_options.js b/tests/js/documentation_options.js similarity index 100% rename from tests/js/project/documentation_options.js rename to tests/js/documentation_options.js diff --git a/tests/js/project/language_data.js b/tests/js/language_data.js similarity index 100% rename from tests/js/project/language_data.js rename to tests/js/language_data.js From 6987c485dc67916898c3913692312ae75121aafa Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:12:20 +0100 Subject: [PATCH 26/57] [tests] karma config: explicitly add language_data.js alongside documentation_options.js entry. --- karma.conf.js | 1 + 1 file changed, 1 insertion(+) diff --git a/karma.conf.js b/karma.conf.js index 1ecdfd9fa7d..e24d3509c94 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -16,6 +16,7 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ 'tests/js/documentation_options.js', + 'tests/js/language_data.js', 'sphinx/themes/basic/static/doctools.js', 'sphinx/themes/basic/static/searchtools.js', 'sphinx/themes/basic/static/sphinx_highlight.js', From fac7694ef6a650d69d4ad408e7f76bedbfc52af0 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:17:33 +0100 Subject: [PATCH 27/57] [tests] nitpick: relocate file-read operations to same-code-line as hashing operation. --- tests/test_search.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index 692bf3c3111..b796978d24e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -352,11 +352,11 @@ def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): app = make_app('html', srcdir=directory, builddir=sphinx_test_tempdir / directory.name) app.build() - fresh_searchindex = (app.outdir / 'searchindex.js').read_bytes() - fresh_hash = sha256(fresh_searchindex).digest() + fresh_searchindex = (app.outdir / 'searchindex.js') + fresh_hash = sha256(fresh_searchindex.read_bytes()).digest() - existing_searchindex = (directory / '..' / '..' / 'fixtures' / directory.name / 'searchindex.js').read_bytes() - existing_hash = sha256(existing_searchindex).digest() + existing_searchindex = (directory / '..' / '..' / 'fixtures' / directory.name / 'searchindex.js') + existing_hash = sha256(existing_searchindex.read_bytes()).digest() msg = f"Search index fixture in {directory} does not match regenerated copy." assert fresh_hash == existing_hash, msg From a464bdf4c7b9a1a97b1bb87ba60f66607697f720 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:21:31 +0100 Subject: [PATCH 28/57] [tests] nitpick / consistency: use current-file as base for both project and fixture path lookups. --- tests/test_search.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_search.py b/tests/test_search.py index b796978d24e..e93cd4061da 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -12,6 +12,8 @@ from sphinx.search import IndexBuilder +from tests.utils import TESTS_ROOT + class DummyEnvironment: def __init__(self, version, domains): @@ -355,7 +357,7 @@ def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): fresh_searchindex = (app.outdir / 'searchindex.js') fresh_hash = sha256(fresh_searchindex.read_bytes()).digest() - existing_searchindex = (directory / '..' / '..' / 'fixtures' / directory.name / 'searchindex.js') + existing_searchindex = (Path(__file__).resolve().parent / 'js' / 'fixtures' / directory.name / 'searchindex.js') existing_hash = sha256(existing_searchindex.read_bytes()).digest() msg = f"Search index fixture in {directory} does not match regenerated copy." From 5b6a479070030c42d881a24b7a24b8e3e6dc3b04 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:22:39 +0100 Subject: [PATCH 29/57] [tests] refactor / cleanup: re-use existing TESTS_ROOT testutil constant. --- tests/test_search.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index e93cd4061da..416f8ddf0e1 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -4,7 +4,6 @@ import warnings from hashlib import sha256 from io import BytesIO -from pathlib import Path import pytest from docutils import frontend, utils @@ -348,7 +347,7 @@ def assert_is_sorted(item, path): @pytest.mark.parametrize('directory', ( - directory for directory in (Path(__file__).resolve().parent / 'js' / 'roots').iterdir() + directory for directory in (TESTS_ROOT / 'js' / 'roots').iterdir() )) def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): app = make_app('html', srcdir=directory, builddir=sphinx_test_tempdir / directory.name) @@ -357,7 +356,7 @@ def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): fresh_searchindex = (app.outdir / 'searchindex.js') fresh_hash = sha256(fresh_searchindex.read_bytes()).digest() - existing_searchindex = (Path(__file__).resolve().parent / 'js' / 'fixtures' / directory.name / 'searchindex.js') + existing_searchindex = (TESTS_ROOT / 'js' / 'fixtures' / directory.name / 'searchindex.js') existing_hash = sha256(existing_searchindex.read_bytes()).digest() msg = f"Search index fixture in {directory} does not match regenerated copy." From fc76aef92bb0430b426047ea1d5e9b1051b5044b Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:26:16 +0100 Subject: [PATCH 30/57] [tests] search: indicate full path of search index file when it fails a regeneration check. --- tests/test_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_search.py b/tests/test_search.py index 416f8ddf0e1..39beedae1dd 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -359,5 +359,5 @@ def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): existing_searchindex = (TESTS_ROOT / 'js' / 'fixtures' / directory.name / 'searchindex.js') existing_hash = sha256(existing_searchindex.read_bytes()).digest() - msg = f"Search index fixture in {directory} does not match regenerated copy." + msg = f"Search index fixture {existing_searchindex} does not match regenerated copy." assert fresh_hash == existing_hash, msg From e9614b91943d796eb3ddec30cd880e0eaef10594 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:37:46 +0100 Subject: [PATCH 31/57] [docs] fixup for JS search-index fixture paths. --- doc/internals/contributing.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/internals/contributing.rst b/doc/internals/contributing.rst index f8dc754aaf4..6ba69948ea9 100644 --- a/doc/internals/contributing.rst +++ b/doc/internals/contributing.rst @@ -339,7 +339,7 @@ Debugging tips non-minified ones using ``uglifyjs`` (installed via npm), with ``-m`` option to enable mangling. -* The ``searchindex.js`` files found in the ``tests/js/roots/*`` directories +* The ``searchindex.js`` files found in the ``tests/js/fixtures/*`` directories are generated using the Sphinx HTML builder, and are provided as test fixtures to the Sphinx JavaScript test suite. A utility script found at ``utils/generate_js_fixtures.py`` can be used to regenerate them. From 4d62b1542925c9c9d13e98373ab9c006e94e1bb6 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:39:40 +0100 Subject: [PATCH 32/57] [docs] contributing: rewrite paragraph about JavaScript test fixtures. --- doc/internals/contributing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/internals/contributing.rst b/doc/internals/contributing.rst index 6ba69948ea9..3bdde63aea8 100644 --- a/doc/internals/contributing.rst +++ b/doc/internals/contributing.rst @@ -340,6 +340,6 @@ Debugging tips option to enable mangling. * The ``searchindex.js`` files found in the ``tests/js/fixtures/*`` directories - are generated using the Sphinx HTML builder, and are provided as test - fixtures to the Sphinx JavaScript test suite. A utility script found at - ``utils/generate_js_fixtures.py`` can be used to regenerate them. + are generated by the Sphinx HTML builder, and provide test data used by the + Sphinx JavaScript unit tests. The ``utils/generate_js_fixtures.py`` script + can be used to regenerate the fixtures. From 38bfc341882d4b9f2265adc2c59ea75f1a8bc628 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:43:41 +0100 Subject: [PATCH 33/57] [docs] contributing: rewrite (again) paragraph about JavaScript test fixtures. --- doc/internals/contributing.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/internals/contributing.rst b/doc/internals/contributing.rst index 3bdde63aea8..92cd1810f64 100644 --- a/doc/internals/contributing.rst +++ b/doc/internals/contributing.rst @@ -340,6 +340,7 @@ Debugging tips option to enable mangling. * The ``searchindex.js`` files found in the ``tests/js/fixtures/*`` directories - are generated by the Sphinx HTML builder, and provide test data used by the - Sphinx JavaScript unit tests. The ``utils/generate_js_fixtures.py`` script - can be used to regenerate the fixtures. + are generated by using the standard Sphinx HTML builder on the corresponding + input projects found in ``tests/js/roots/*``. The fixtures provide test data + used by the Sphinx JavaScript unit tests, and can be regenerated by running + the ``utils/generate_js_fixtures.py`` script. From 51f9a4ddd448262468e2e55314eb1112774219bf Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 14:47:04 +0100 Subject: [PATCH 34/57] [tests] karma config: nitpick: move served files to top of files option. --- karma.conf.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/karma.conf.js b/karma.conf.js index e24d3509c94..4f1b9c616e4 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -15,13 +15,13 @@ module.exports = function(config) { // list of files / patterns to load in the browser files: [ + { pattern: 'tests/js/fixtures/**/*.js', included: false, served: true }, 'tests/js/documentation_options.js', 'tests/js/language_data.js', 'sphinx/themes/basic/static/doctools.js', 'sphinx/themes/basic/static/searchtools.js', 'sphinx/themes/basic/static/sphinx_highlight.js', - 'tests/js/*.js', - { pattern: 'tests/js/fixtures/**/*.js', included: false, served: true } + 'tests/js/*.js' ], From 8c6c66969fec35100b2af362dc1350f261d5cc73 Mon Sep 17 00:00:00 2001 From: James Addison <55152140+jayaddison@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:04:13 +0100 Subject: [PATCH 35/57] [search] nitpick: remove empty line from query function. --- sphinx/themes/basic/static/searchtools.js | 1 - 1 file changed, 1 deletion(-) diff --git a/sphinx/themes/basic/static/searchtools.js b/sphinx/themes/basic/static/searchtools.js index ee3b0e66cdf..557f521f1a8 100644 --- a/sphinx/themes/basic/static/searchtools.js +++ b/sphinx/themes/basic/static/searchtools.js @@ -402,7 +402,6 @@ const Search = { }, query: (query) => { - const searchTerms = Search._parseQuery(query); const results = Search._performSearch(...searchTerms); From 58275765370b09d3df8fa07a9416b07d4bff0150 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 16:31:24 +0100 Subject: [PATCH 36/57] [search] typo fixup: 'if' instead of 'is'. --- sphinx/themes/basic/static/language_data.js_t | 2 +- tests/js/language_data.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/themes/basic/static/language_data.js_t b/sphinx/themes/basic/static/language_data.js_t index 76bf3718701..3666e5e6f03 100644 --- a/sphinx/themes/basic/static/language_data.js_t +++ b/sphinx/themes/basic/static/language_data.js_t @@ -13,7 +13,7 @@ var stopwords = {{ search_language_stop_words }}; {% if search_language_stemming_code %} -/* Non-minified version is copied as a separate JS file, is available */ +/* Non-minified version is copied as a separate JS file, if available */ {{ search_language_stemming_code|safe }} {% endif -%} diff --git a/tests/js/language_data.js b/tests/js/language_data.js index db395c75dc0..89083d9ec7c 100644 --- a/tests/js/language_data.js +++ b/tests/js/language_data.js @@ -13,7 +13,7 @@ var stopwords = []; -/* Non-minified version is copied as a separate JS file, is available */ +/* Non-minified version is copied as a separate JS file, if available */ /** * Dummy stemmer for languages without stemming rules. From 9773c0c8ab07418d324044edcc475f6230d76a29 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 16:33:25 +0100 Subject: [PATCH 37/57] [utils] js fixtures: use '_build' as output dir instead of 'build'. --- .gitignore | 2 +- utils/generate_js_fixtures.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 22bb011d930..35fd23178f5 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,7 @@ doc/_build/ doc/locale/ tests/.coverage tests/build/ -tests/js/roots/*/build +tests/js/roots/*/_build tests/test-server.lock utils/regression_test.js diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py index 0ba3d5132a5..f6fae8dc288 100755 --- a/utils/generate_js_fixtures.py +++ b/utils/generate_js_fixtures.py @@ -9,12 +9,12 @@ def build(srcdir: Path) -> None: - cmd = ('sphinx-build', '-E', '-q', '-b', 'html', f'{srcdir}', f'{srcdir}/build') + cmd = ('sphinx-build', '-E', '-q', '-b', 'html', f'{srcdir}', f'{srcdir}/_build') subprocess.run(cmd, check=True, capture_output=True) for directory in TEST_JS_ROOTS.iterdir(): - searchindex = directory / 'build' / 'searchindex.js' + searchindex = directory / '_build' / 'searchindex.js' destination = TEST_JS_FIXTURES / directory.name / 'searchindex.js' print(f'Building {directory} ... ', end='') From 66328bc1d9f035a6dd83b9d3612bf7c1b9e730a9 Mon Sep 17 00:00:00 2001 From: James Addison <55152140+jayaddison@users.noreply.github.com> Date: Thu, 4 Apr 2024 16:51:45 +0100 Subject: [PATCH 38/57] [utils] update path navigation steps during JavaScript fixture generation code. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- utils/generate_js_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py index f6fae8dc288..7dcdf3569a6 100755 --- a/utils/generate_js_fixtures.py +++ b/utils/generate_js_fixtures.py @@ -3,7 +3,7 @@ import subprocess from pathlib import Path -SPHINX_ROOT = Path(__file__).resolve().parents[1] +SPHINX_ROOT = Path(__file__).resolve().parent.parent TEST_JS_FIXTURES = SPHINX_ROOT / 'tests' / 'js' / 'fixtures' TEST_JS_ROOTS = SPHINX_ROOT / 'tests' / 'js' / 'roots' From 7370d2056d61225a47d50ac279bbb1a12caa6227 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 17:37:44 +0100 Subject: [PATCH 39/57] [tests] search: re-constrain test coverage for multiterm test case to exclude titles. --- tests/js/searchtools.js | 34 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index f157f9a277b..f4c12284351 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -28,28 +28,18 @@ describe('Basic html theme search', function() { it('should be able to search for multiple terms', function() { eval(loadFixture("multiterm/searchindex.js")); - searchTerms = Search._parseQuery('main page'); - - // fixme: duplicate result due to https://github.com/sphinx-doc/sphinx/issues/11961 - hits = [ - [ - 'index', - 'Main Page', - '', - null, - 15, - 'index.rst' - ], - [ - 'index', - 'Main Page', - '#main-page', - null, - 100, - 'index.rst' - ] - ]; - expect(Search._performSearch(...searchTerms)).toEqual(hits); + searchterms = ['main', 'page']; + excluded = []; + terms = index.terms; + titleterms = index.titleterms; + hits = [[ + 'index', + 'Main Page', + '', + null, + 15, + 'index.rst']]; + expect(Search.performTermsSearch(searchterms, excluded, terms, titleterms)).toEqual(hits); }); it('should partially-match "sphinx" when in title index', function() { From 2ef85babf3e96b8266a955cdd52d2416651013a9 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 18:35:47 +0100 Subject: [PATCH 40/57] [search] Rectification: rename misleadingly-named 'searchTerms' variable to 'searchParameters'. (cherry picked from commit 7c151e4cf9f0bca8a08fe459a67f547b6ae9f878) --- sphinx/themes/basic/static/searchtools.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/themes/basic/static/searchtools.js b/sphinx/themes/basic/static/searchtools.js index 557f521f1a8..fadf7fa45b6 100644 --- a/sphinx/themes/basic/static/searchtools.js +++ b/sphinx/themes/basic/static/searchtools.js @@ -402,8 +402,8 @@ const Search = { }, query: (query) => { - const searchTerms = Search._parseQuery(query); - const results = Search._performSearch(...searchTerms); + const searchParameters = Search._parseQuery(query); + const results = Search._performSearch(...searchParameters); // for debugging //Search.lastresults = results.slice(); // a copy From cd1a368e120ab153ef3b9d508952aa70c7d25543 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 18:37:59 +0100 Subject: [PATCH 41/57] [search] Rectification: test cases: rename misleadingly-named 'searchTerms' variable to 'searchParameters'. Relates-to commit 2ef85babf3e96b8266a955cdd52d2416651013a9. --- tests/js/searchtools.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index f4c12284351..f583f567624 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -12,7 +12,7 @@ describe('Basic html theme search', function() { it('should find "C++" when in index', function() { eval(loadFixture("cpp/searchindex.js")); - searchTerms = Search._parseQuery('C++'); + searchParameters = Search._parseQuery('C++'); hits = [[ "index", @@ -22,7 +22,7 @@ describe('Basic html theme search', function() { 5, "index.rst" ]]; - expect(Search._performSearch(...searchTerms)).toEqual(hits); + expect(Search._performSearch(...searchParameters)).toEqual(hits); }); it('should be able to search for multiple terms', function() { From 7e4b56259214127a20645e4ca63c49c4fc95a912 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 18:39:14 +0100 Subject: [PATCH 42/57] [tests] search: add end-to-end / search-result-aggregation test. --- tests/js/searchtools.js | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index f583f567624..c2c401509b1 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -69,6 +69,37 @@ describe('Basic html theme search', function() { }); + describe('aggregation of search results', function() { + + it('should combine document title and document term matches', function() { + eval(loadFixture("multiterm/searchindex.js")); + + searchParameters = Search._parseQuery('main page'); + + // fixme: duplicate result due to https://github.com/sphinx-doc/sphinx/issues/11961 + hits = [ + [ + 'index', + 'Main Page', + '', + null, + 15, + 'index.rst' + ], + [ + 'index', + 'Main Page', + '#main-page', + null, + 100, + 'index.rst' + ] + ]; + expect(Search._performSearch(...searchParameters)).toEqual(hits); + }); + + }); + }); describe("htmlToText", function() { From ec4f236d0074cf87f32bda88352953713b2322c5 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 18:56:44 +0100 Subject: [PATCH 43/57] [tests] search: fixup: initialize index-term variables from Search._index The Search._index property is assigned during the setIndex(...) function call, as performed by our evaluation of searchindex.js content. Fixup-for / relates-to commit 7370d2056d61225a47d50ac279bbb1a12caa6227. --- tests/js/searchtools.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index c2c401509b1..0355320adde 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -30,8 +30,8 @@ describe('Basic html theme search', function() { searchterms = ['main', 'page']; excluded = []; - terms = index.terms; - titleterms = index.titleterms; + terms = Search._index.terms; + titleterms = Search._index.titleterms; hits = [[ 'index', 'Main Page', From 9b709d7c24300b7b2be8d95f4da7c459f21df801 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 19:10:11 +0100 Subject: [PATCH 44/57] [tests] search: migrate partial-match search to use fixture data. Post-merge / relates-to commit 94b9fc5698d5f7216044783d14113c35efcbfeec. --- tests/js/fixtures/partial/searchindex.js | 1 + tests/js/roots/partial/conf.py | 0 tests/js/roots/partial/index.rst | 4 ++++ tests/js/searchtools.js | 17 ++++------------- 4 files changed, 9 insertions(+), 13 deletions(-) create mode 100644 tests/js/fixtures/partial/searchindex.js create mode 100644 tests/js/roots/partial/conf.py create mode 100644 tests/js/roots/partial/index.rst diff --git a/tests/js/fixtures/partial/searchindex.js b/tests/js/fixtures/partial/searchindex.js new file mode 100644 index 00000000000..1737df7b6f8 --- /dev/null +++ b/tests/js/fixtures/partial/searchindex.js @@ -0,0 +1 @@ +Search.setIndex({"alltitles": {"sphinx_utils module": [[0, "sphinx-utils-module"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"us": 0, "util": 0}, "titles": ["sphinx_utils module"], "titleterms": {"modul": 0, "sphinx_util": 0}}) \ No newline at end of file diff --git a/tests/js/roots/partial/conf.py b/tests/js/roots/partial/conf.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/js/roots/partial/index.rst b/tests/js/roots/partial/index.rst new file mode 100644 index 00000000000..c0ac6fdab95 --- /dev/null +++ b/tests/js/roots/partial/index.rst @@ -0,0 +1,4 @@ +sphinx_utils module +=================== + +Useful utilities. diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index 0355320adde..a5cd5130351 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -43,18 +43,9 @@ describe('Basic html theme search', function() { }); it('should partially-match "sphinx" when in title index', function() { - index = { - docnames:["index"], - filenames:["index.rst"], - terms:{'useful': 0, 'utilities': 0}, - titles:["sphinx_utils module"], - titleterms:{'sphinx_utils': 0} - } - Search.setIndex(index); - searchterms = ['sphinx']; - excluded = []; - terms = index.terms; - titleterms = index.titleterms; + eval(loadFixture("partial/searchindex.js")); + + searchParameters = Search._parseQuery('sphinx'); hits = [[ "index", @@ -64,7 +55,7 @@ describe('Basic html theme search', function() { 7, "index.rst" ]]; - expect(Search.performTermsSearch(searchterms, excluded, terms, titleterms)).toEqual(hits); + expect(Search._performSearch(...searchParameters)).toEqual(hits); }); }); From 3d108c1d4328604f827b389a91ed3b96be0c1047 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 19:15:13 +0100 Subject: [PATCH 45/57] [tests] search: re-constrain cpp and partial test cases to term-search-only. Follows-on-from commit 7370d2056d61225a47d50ac279bbb1a12caa6227. --- tests/js/searchtools.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index a5cd5130351..7d00166c81d 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -12,7 +12,10 @@ describe('Basic html theme search', function() { it('should find "C++" when in index', function() { eval(loadFixture("cpp/searchindex.js")); - searchParameters = Search._parseQuery('C++'); + searchterms = ['c++']; + excluded = []; + terms = Search._index.terms; + titleterms = Search._index.titleterms; hits = [[ "index", @@ -22,7 +25,7 @@ describe('Basic html theme search', function() { 5, "index.rst" ]]; - expect(Search._performSearch(...searchParameters)).toEqual(hits); + expect(Search.performTermsSearch(searchterms, excluded, terms, titleterms)).toEqual(hits); }); it('should be able to search for multiple terms', function() { @@ -45,7 +48,10 @@ describe('Basic html theme search', function() { it('should partially-match "sphinx" when in title index', function() { eval(loadFixture("partial/searchindex.js")); - searchParameters = Search._parseQuery('sphinx'); + searchterms = ['sphinx']; + excluded = []; + terms = Search._index.terms; + titleterms = Search._index.titleterms; hits = [[ "index", @@ -55,7 +61,7 @@ describe('Basic html theme search', function() { 7, "index.rst" ]]; - expect(Search._performSearch(...searchParameters)).toEqual(hits); + expect(Search.performTermsSearch(searchterms, excluded, terms, titleterms)).toEqual(hits); }); }); From d0125c7160e5b2346c2b659e5affc196bd7aa03c Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 4 Apr 2024 20:02:12 +0100 Subject: [PATCH 46/57] [tests] search: use Search.parseQuery function to match query terms to indexed terms. --- tests/js/searchtools.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index 7d00166c81d..35bed0a5c54 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -12,8 +12,7 @@ describe('Basic html theme search', function() { it('should find "C++" when in index', function() { eval(loadFixture("cpp/searchindex.js")); - searchterms = ['c++']; - excluded = []; + [/* first-ignored */, searchterms, excluded, /*rest-ignored*/] = Search._parseQuery('C++'); terms = Search._index.terms; titleterms = Search._index.titleterms; @@ -31,8 +30,7 @@ describe('Basic html theme search', function() { it('should be able to search for multiple terms', function() { eval(loadFixture("multiterm/searchindex.js")); - searchterms = ['main', 'page']; - excluded = []; + [/* first-ignored */, searchterms, excluded, /*rest-ignored*/] = Search._parseQuery('main page'); terms = Search._index.terms; titleterms = Search._index.titleterms; hits = [[ @@ -48,8 +46,7 @@ describe('Basic html theme search', function() { it('should partially-match "sphinx" when in title index', function() { eval(loadFixture("partial/searchindex.js")); - searchterms = ['sphinx']; - excluded = []; + [/* first-ignored */, searchterms, excluded, /*rest-ignored*/] = Search._parseQuery('sphinx'); terms = Search._index.terms; titleterms = Search._index.titleterms; From 3a6c784456c8f72b77ae73542480e83155f6f405 Mon Sep 17 00:00:00 2001 From: James Addison Date: Fri, 5 Apr 2024 16:23:24 +0100 Subject: [PATCH 47/57] [utils] js fixtures: create destination parent directory if it does not already exist. --- utils/generate_js_fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py index 7dcdf3569a6..db800b0ce6b 100755 --- a/utils/generate_js_fixtures.py +++ b/utils/generate_js_fixtures.py @@ -22,5 +22,6 @@ def build(srcdir: Path) -> None: print('done') print(f'Moving {searchindex} to {destination} ... ', end='') + destination.parent.mkdir(exist_ok=True) searchindex.replace(destination) print('done') From cc8ee126db7cdfb74898a15696333bbb3ef137fe Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 4 May 2024 17:49:18 +0100 Subject: [PATCH 48/57] Fixup: relocate CHANGES entry for this feature to in-development section. --- CHANGES.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.rst b/CHANGES.rst index 9ca269bbc21..f4248029709 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -38,6 +38,8 @@ Bugs fixed Testing ------- +* karma: refactor HTML search tests to use fixtures generated by Sphinx. + Release 7.3.7 (released Apr 19, 2024) ===================================== @@ -313,7 +315,6 @@ Testing As a side-effect, this removes the need for test server lockfiles, meaning that any remaining ``tests/test-server.lock`` files can safely be deleted. -* karma: refactor HTML search tests to use fixtures generated by Sphinx. Release 7.2.6 (released Sep 13, 2023) ===================================== From 2cc14764bfbfa967104546f616614d03349220a8 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 4 May 2024 17:49:55 +0100 Subject: [PATCH 49/57] Fixup: add self-credit to CHANGES entry. --- CHANGES.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.rst b/CHANGES.rst index f4248029709..f807fa575c9 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -39,6 +39,7 @@ Testing ------- * karma: refactor HTML search tests to use fixtures generated by Sphinx. + Patch by James Addison. Release 7.3.7 (released Apr 19, 2024) ===================================== From 7e92842794b0e223fd1978c6f920b9fe2be643c8 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 4 May 2024 19:25:46 +0100 Subject: [PATCH 50/57] Tests: attempt to add explanatory content to JS test fixture source projects. --- tests/js/roots/cpp/index.rst | 5 +++++ tests/js/roots/multiterm/index.rst | 11 ++++++++++- tests/js/roots/partial/index.rst | 7 ++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/tests/js/roots/cpp/index.rst b/tests/js/roots/cpp/index.rst index c6cb1e12e5a..d731343dca6 100644 --- a/tests/js/roots/cpp/index.rst +++ b/tests/js/roots/cpp/index.rst @@ -3,3 +3,8 @@ This is a sample C++ project used to generate a search engine index fixture. .. cpp:class:: public Sphinx The description of Sphinx class. + +Indexing and querying the term C++ can be challenging, because search-related +tokenization often drops punctuation and mathematical characters (they occur +frequently on the web and would inflate the cardinality and size of web search +indexes). diff --git a/tests/js/roots/multiterm/index.rst b/tests/js/roots/multiterm/index.rst index 2e9f4967893..495e5ce858c 100644 --- a/tests/js/roots/multiterm/index.rst +++ b/tests/js/roots/multiterm/index.rst @@ -1,4 +1,13 @@ Main Page ========= -Welcome to the... main page! +This is the main page of the ``multiterm`` test project. + +This document is used as a test fixture to check that the search functionality +included when projects are built into an HTML output format can successfully +match this document when a search query containing multiple terms is performed. + +At the time-of-writing this message, the application doesn't support "phrase +queries" -- queries that require all of the contained terms to appear adjacent +to each other and in the same order in the document as in the query; perhaps it +will do in future? diff --git a/tests/js/roots/partial/index.rst b/tests/js/roots/partial/index.rst index c0ac6fdab95..6a9561b3994 100644 --- a/tests/js/roots/partial/index.rst +++ b/tests/js/roots/partial/index.rst @@ -1,4 +1,9 @@ sphinx_utils module =================== -Useful utilities. +Partial (also known as "prefix") matches on document titles should be possible +using the JavaScript search functionality included when HTML documentation +projects are built. + +This document provides a sample reStructuredText input to confirm that partial +title matching is possible. From 9c0f2b2a3a7da0fa54a9bc57ed703763a7681705 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 4 May 2024 20:28:19 +0100 Subject: [PATCH 51/57] Tests: regenerate JavaScript fixtures by running 'python utils/generate_js_fixtures.py' --- tests/js/fixtures/cpp/searchindex.js | 2 +- tests/js/fixtures/multiterm/searchindex.js | 2 +- tests/js/fixtures/partial/searchindex.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/js/fixtures/cpp/searchindex.js b/tests/js/fixtures/cpp/searchindex.js index b829f7cc320..f704f7aa7e2 100644 --- a/tests/js/fixtures/cpp/searchindex.js +++ b/tests/js/fixtures/cpp/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx", false]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "c": 0, "class": 0, "descript": 0, "engin": 0, "fixtur": 0, "gener": 0, "i": 0, "index": 0, "project": 0, "sampl": 0, "search": 0, "sphinx": 0, "thi": 0, "us": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file +Search.setIndex({"alltitles": {}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {"sphinx (c++ class)": [[0, "_CPPv46Sphinx", false]]}, "objects": {"": [[0, 0, 1, "_CPPv46Sphinx", "Sphinx"]]}, "objnames": {"0": ["cpp", "class", "C++ class"]}, "objtypes": {"0": "cpp:class"}, "terms": {"The": 0, "becaus": 0, "c": 0, "can": 0, "cardin": 0, "challeng": 0, "charact": 0, "class": 0, "descript": 0, "drop": 0, "engin": 0, "fixtur": 0, "frequent": 0, "gener": 0, "i": 0, "index": 0, "inflat": 0, "mathemat": 0, "occur": 0, "often": 0, "project": 0, "punctuat": 0, "queri": 0, "relat": 0, "sampl": 0, "search": 0, "size": 0, "sphinx": 0, "term": 0, "thei": 0, "thi": 0, "token": 0, "us": 0, "web": 0, "would": 0}, "titles": ["<no title>"], "titleterms": {}}) \ No newline at end of file diff --git a/tests/js/fixtures/multiterm/searchindex.js b/tests/js/fixtures/multiterm/searchindex.js index 282549de340..b791df93d11 100644 --- a/tests/js/fixtures/multiterm/searchindex.js +++ b/tests/js/fixtures/multiterm/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"welcom": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file +Search.setIndex({"alltitles": {"Main Page": [[0, "main-page"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"At": 0, "adjac": 0, "all": 0, "an": 0, "appear": 0, "applic": 0, "ar": 0, "built": 0, "can": 0, "check": 0, "contain": 0, "do": 0, "document": 0, "doesn": 0, "each": 0, "fixtur": 0, "format": 0, "function": 0, "futur": 0, "html": 0, "i": 0, "includ": 0, "match": 0, "messag": 0, "multipl": 0, "multiterm": 0, "order": 0, "other": 0, "output": 0, "perform": 0, "perhap": 0, "phrase": 0, "project": 0, "queri": 0, "requir": 0, "same": 0, "search": 0, "successfulli": 0, "support": 0, "t": 0, "term": 0, "test": 0, "thi": 0, "time": 0, "us": 0, "when": 0, "write": 0}, "titles": ["Main Page"], "titleterms": {"main": 0, "page": 0}}) \ No newline at end of file diff --git a/tests/js/fixtures/partial/searchindex.js b/tests/js/fixtures/partial/searchindex.js index 1737df7b6f8..6ccfbd6d07e 100644 --- a/tests/js/fixtures/partial/searchindex.js +++ b/tests/js/fixtures/partial/searchindex.js @@ -1 +1 @@ -Search.setIndex({"alltitles": {"sphinx_utils module": [[0, "sphinx-utils-module"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"us": 0, "util": 0}, "titles": ["sphinx_utils module"], "titleterms": {"modul": 0, "sphinx_util": 0}}) \ No newline at end of file +Search.setIndex({"alltitles": {"sphinx_utils module": [[0, "sphinx-utils-module"]]}, "docnames": ["index"], "envversion": {"sphinx": 61, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2}, "filenames": ["index.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"also": 0, "ar": 0, "built": 0, "confirm": 0, "document": 0, "function": 0, "html": 0, "i": 0, "includ": 0, "input": 0, "javascript": 0, "known": 0, "match": 0, "partial": 0, "possibl": 0, "prefix": 0, "project": 0, "provid": 0, "restructuredtext": 0, "sampl": 0, "search": 0, "should": 0, "thi": 0, "titl": 0, "us": 0, "when": 0}, "titles": ["sphinx_utils module"], "titleterms": {"modul": 0, "sphinx_util": 0}}) \ No newline at end of file From 7acb3cd5a9db3d0fcff6e414031562fdda113f76 Mon Sep 17 00:00:00 2001 From: James Addison Date: Sat, 4 May 2024 20:59:06 +0100 Subject: [PATCH 52/57] Tests: compare regenerated-vs-existing search indices byte-for-byte instead of comparing their sha256 hash values. --- tests/test_search.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index dd0f1da0dde..d36bb57743c 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -3,7 +3,6 @@ import json import warnings -from hashlib import sha256 from io import BytesIO import pytest @@ -359,10 +358,7 @@ def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): app.build() fresh_searchindex = (app.outdir / 'searchindex.js') - fresh_hash = sha256(fresh_searchindex.read_bytes()).digest() - existing_searchindex = (TESTS_ROOT / 'js' / 'fixtures' / directory.name / 'searchindex.js') - existing_hash = sha256(existing_searchindex.read_bytes()).digest() msg = f"Search index fixture {existing_searchindex} does not match regenerated copy." - assert fresh_hash == existing_hash, msg + assert fresh_searchindex.read_bytes() == existing_searchindex.read_bytes(), msg From de00490cfecb0d42fc15874c41b87f619ada605b Mon Sep 17 00:00:00 2001 From: James Addison Date: Tue, 11 Jun 2024 18:02:52 +0100 Subject: [PATCH 53/57] [tests] search: code review: add variable names for parsed-but-unused query terms items. --- tests/js/searchtools.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/js/searchtools.js b/tests/js/searchtools.js index 7efbb6974a2..5e97572fb3e 100644 --- a/tests/js/searchtools.js +++ b/tests/js/searchtools.js @@ -12,7 +12,7 @@ describe('Basic html theme search', function() { it('should find "C++" when in index', function() { eval(loadFixture("cpp/searchindex.js")); - [/* first-ignored */, searchterms, excluded, /*rest-ignored*/] = Search._parseQuery('C++'); + [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('C++'); terms = Search._index.terms; titleterms = Search._index.titleterms; @@ -30,7 +30,7 @@ describe('Basic html theme search', function() { it('should be able to search for multiple terms', function() { eval(loadFixture("multiterm/searchindex.js")); - [/* first-ignored */, searchterms, excluded, /*rest-ignored*/] = Search._parseQuery('main page'); + [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('main page'); terms = Search._index.terms; titleterms = Search._index.titleterms; hits = [[ @@ -46,7 +46,7 @@ describe('Basic html theme search', function() { it('should partially-match "sphinx" when in title index', function() { eval(loadFixture("partial/searchindex.js")); - [/* first-ignored */, searchterms, excluded, /*rest-ignored*/] = Search._parseQuery('sphinx'); + [_searchQuery, searchterms, excluded, ..._remainingItems] = Search._parseQuery('sphinx'); terms = Search._index.terms; titleterms = Search._index.titleterms; From b8ab726b2336199af527067948b8e12f32a7f086 Mon Sep 17 00:00:00 2001 From: James Addison Date: Tue, 11 Jun 2024 18:05:47 +0100 Subject: [PATCH 54/57] [utils] js fixtures: code review: use fully-expanded option names when invoking sphinx command. --- utils/generate_js_fixtures.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py index db800b0ce6b..16a173d9a26 100755 --- a/utils/generate_js_fixtures.py +++ b/utils/generate_js_fixtures.py @@ -9,7 +9,14 @@ def build(srcdir: Path) -> None: - cmd = ('sphinx-build', '-E', '-q', '-b', 'html', f'{srcdir}', f'{srcdir}/_build') + cmd = ( + 'sphinx-build', + '--fresh-env', + '--quiet', + '--builder', 'html', + f'{srcdir}', + f'{srcdir}/_build', + ) subprocess.run(cmd, check=True, capture_output=True) From 5c231577106bd73785acaccb870f1cbb8e76693f Mon Sep 17 00:00:00 2001 From: James Addison Date: Tue, 11 Jun 2024 18:13:55 +0100 Subject: [PATCH 55/57] [utils] js fixtures: lint fixup: use argument expansion to populate builder option. --- utils/generate_js_fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/generate_js_fixtures.py b/utils/generate_js_fixtures.py index 16a173d9a26..37e844f1a80 100755 --- a/utils/generate_js_fixtures.py +++ b/utils/generate_js_fixtures.py @@ -13,7 +13,7 @@ def build(srcdir: Path) -> None: 'sphinx-build', '--fresh-env', '--quiet', - '--builder', 'html', + *('--builder', 'html'), f'{srcdir}', f'{srcdir}/_build', ) From 83fe92ab67867b13c5b6c577e7d8f3e0ad88b80e Mon Sep 17 00:00:00 2001 From: James Addison <55152140+jayaddison@users.noreply.github.com> Date: Thu, 13 Jun 2024 11:39:14 +0100 Subject: [PATCH 56/57] [tests] search: provide eager-evaluated JavaScript test roots instead of reading them from an iterator at test-time. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- tests/test_search.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_search.py b/tests/test_search.py index d36bb57743c..f2d27d80c01 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -350,9 +350,8 @@ def assert_is_sorted(item, path: str): assert_is_sorted(child, f'{path}[{i}]') -@pytest.mark.parametrize('directory', ( - directory for directory in (TESTS_ROOT / 'js' / 'roots').iterdir() -)) +JAVASCRIPT_TEST_ROOTS = list((TESTS_ROOT / 'js' / 'roots').iterdir()) +@pytest.mark.parametrize('directory', JAVASCRIPT_TEST_ROOTS) def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): app = make_app('html', srcdir=directory, builddir=sphinx_test_tempdir / directory.name) app.build() From f9d6191728324fa087a67d21fdd0bfc78cf92e51 Mon Sep 17 00:00:00 2001 From: James Addison Date: Thu, 13 Jun 2024 11:42:42 +0100 Subject: [PATCH 57/57] [tests] search: relocate constant declaration to appease linters. --- tests/test_search.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_search.py b/tests/test_search.py index f2d27d80c01..3b3413db8d9 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -13,6 +13,8 @@ from tests.utils import TESTS_ROOT +JAVASCRIPT_TEST_ROOTS = list((TESTS_ROOT / 'js' / 'roots').iterdir()) + class DummyEnvironment: def __init__(self, version, domains): @@ -350,7 +352,6 @@ def assert_is_sorted(item, path: str): assert_is_sorted(child, f'{path}[{i}]') -JAVASCRIPT_TEST_ROOTS = list((TESTS_ROOT / 'js' / 'roots').iterdir()) @pytest.mark.parametrize('directory', JAVASCRIPT_TEST_ROOTS) def test_check_js_search_indexes(make_app, sphinx_test_tempdir, directory): app = make_app('html', srcdir=directory, builddir=sphinx_test_tempdir / directory.name)