From 0aa2652333ccd940a0613db72471431f1c0ae570 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Wed, 18 Dec 2024 11:19:07 +0100 Subject: [PATCH] Add test coverage for product-details rebuild (#1595) * product-details: move git clone / push to separate functions for easier mocking * tests: add coverage for product-details rebuild --- api/src/shipit_api/admin/product_details.py | 38 ++++---- api/tests/test_product_details.py | 97 ++++++++++++++++++++- 2 files changed, 118 insertions(+), 17 deletions(-) diff --git a/api/src/shipit_api/admin/product_details.py b/api/src/shipit_api/admin/product_details.py index bc87bb568..453477f3c 100644 --- a/api/src/shipit_api/admin/product_details.py +++ b/api/src/shipit_api/admin/product_details.py @@ -1106,20 +1106,7 @@ def run_check(*arg, **kw): return cli_common.utils.retry(lambda: cli_common.command.run_check(*arg, **kw)) -async def rebuild( - db_session: sqlalchemy.orm.Session, - git_branch: str, - git_repo_url: str, - folder_in_repo: str, - breakpoint_version: typing.Optional[int], - clean_working_copy: bool = True, -): - secrets = [urllib.parse.urlparse(git_repo_url).password] - - # Sometimes we want to work from a clean working copy - if clean_working_copy and shipit_api.common.config.PRODUCT_DETAILS_DIR.exists(): - shutil.rmtree(shipit_api.common.config.PRODUCT_DETAILS_DIR) - +def setup_working_copy(git_branch, git_repo_url, secrets): # Clone/pull latest product details logger.info(f"Getting latest product details from {cli_common.command.hide_secrets(git_repo_url, secrets)}.") if shipit_api.common.config.PRODUCT_DETAILS_DIR.exists(): @@ -1141,6 +1128,23 @@ async def rebuild( run_check(["git", "config", "user.email", "release-services+robot@mozilla.com"], cwd=shipit_api.common.config.PRODUCT_DETAILS_DIR, secrets=secrets) run_check(["git", "config", "user.name", "Release Services Robot"], cwd=shipit_api.common.config.PRODUCT_DETAILS_DIR, secrets=secrets) + +async def rebuild( + db_session: sqlalchemy.orm.Session, + git_branch: str, + git_repo_url: str, + folder_in_repo: str, + breakpoint_version: typing.Optional[int], + clean_working_copy: bool = True, +): + secrets = [urllib.parse.urlparse(git_repo_url).password] + + # Sometimes we want to work from a clean working copy + if clean_working_copy and shipit_api.common.config.PRODUCT_DETAILS_DIR.exists(): + shutil.rmtree(shipit_api.common.config.PRODUCT_DETAILS_DIR) + + setup_working_copy(git_branch, git_repo_url, secrets) + # XXX: we need to implement how to figure out breakpoint_version from old_product_details # if breakpoint_version is not provided we should figure it out from old_product_details # and if we can not figure it out we should use shipit_api.common.config.BREAKPOINT_VERSION @@ -1303,4 +1307,8 @@ async def rebuild( # XXX: we need a better commmit message, maybe mention what triggered this update commit_message = "Updating product details" run_check(["git", "commit", "-m", commit_message], cwd=shipit_api.common.config.PRODUCT_DETAILS_DIR, secrets=secrets) - run_check(["git", "push", "origin", git_branch], cwd=shipit_api.common.config.PRODUCT_DETAILS_DIR, secrets=secrets) + git_push(git_branch, secrets) + + +def git_push(git_branch, secrets): + run_check(["git", "push", "origin", git_branch], cwd=shipit_api.common.config.PRODUCT_DETAILS_DIR, secrets=secrets) diff --git a/api/tests/test_product_details.py b/api/tests/test_product_details.py index 67907c8ca..599bfcd16 100644 --- a/api/tests/test_product_details.py +++ b/api/tests/test_product_details.py @@ -3,16 +3,33 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +import json import pathlib import re +import subprocess +from unittest import mock import aiohttp import pytest from aioresponses import aioresponses +from sqlalchemy import engine, event import shipit_api.admin.product_details -from shipit_api.admin.product_details import fetch_l10n_data -from shipit_api.common.models import Release +from shipit_api.admin.product_details import fetch_l10n_data, rebuild +from shipit_api.common.models import Release, Version + + +# product_details uses a postgresql-specific "split_part" sql function +@event.listens_for(engine.Engine, "connect") +def setup_split_part(dbapi_connection, conn_rec): + def split_part(string, delimiter, position): + return string.split(delimiter, position)[position - 1] + + dbapi_connection.create_function( + "split_part", + 3, + split_part, + ) def create_html(folder, items): @@ -72,3 +89,79 @@ async def test_fetch_l10n_data(): m.get(url, status=200, payload=dict(a="a")) (_, changesets) = await fetch_l10n_data(session, release, raise_on_failure=True, use_cache=False) assert changesets == {"a": "a"} + + +def mock_setup_working_copy(branch, url, secrets): + subprocess.check_call(["git", "clone", "-n", url, str(shipit_api.common.config.PRODUCT_DETAILS_DIR)]) + subprocess.check_call(["git", "checkout", "-b", branch, "906b7cd284728a2acec695ddcba9193b44d38982"], cwd=shipit_api.common.config.PRODUCT_DETAILS_DIR) + subprocess.check_call(["git", "config", "user.email", "release-services+robot@mozilla.com"], cwd=shipit_api.common.config.PRODUCT_DETAILS_DIR) + subprocess.check_call(["git", "config", "user.name", "Release Services Robot"], cwd=shipit_api.common.config.PRODUCT_DETAILS_DIR) + + +def mock_git_push(branch, secrets): + return + + +@pytest.mark.asyncio +@mock.patch("shipit_api.admin.product_details.setup_working_copy", mock_setup_working_copy) +@mock.patch("shipit_api.admin.product_details.git_push", mock_git_push) +async def test_rebuild(app, tmp_path): + fxnightly = Version(product_name="firefox", current_version="135.0a1", product_channel="nightly") + tbnightly = Version(product_name="thunderbird", current_version="135.0a1", product_channel="nightly") + deved = Release( + product="devedition", + branch="releases/mozilla-beta", + version="134.0b9", + revision="615791f9752c70ef1757abf68544c8275f219ce3", + build_number=1, + release_eta=None, + status="shipped", + partial_updates=None, + ) + beta = Release( + product="firefox", + branch="releases/mozilla-beta", + version="134.0b8", + revision="9fb87e89c26069198ce2a59a0a790a264d225169", + build_number=1, + release_eta=None, + status="shipped", + partial_updates=None, + ) + release = Release( + product="firefox", + branch="releases/mozilla-release", + version="133.0", + revision="8141aab3ba856d7cbae6c851dd71f2e0cb69649c", + build_number=2, + release_eta=None, + status="shipped", + partial_updates=None, + ) + app.db.session.add(fxnightly) + app.db.session.add(tbnightly) + app.db.session.add(deved) + app.db.session.add(beta) + app.db.session.add(release) + app.db.session.commit() + with ( + mock.patch("shipit_api.common.config.PRODUCT_DETAILS_DIR", tmp_path / "product-details"), + mock.patch("shipit_api.common.config.PRODUCT_DETAILS_NEW_DIR", tmp_path / "product-details-new"), + mock.patch("shipit_api.common.config.PRODUCT_DETAILS_CACHE_DIR", tmp_path / "product-details-cache"), + ): + await rebuild(app.db.session, "testing", "https://github.com/mozilla-releng/product-details", "public", 130) + + parent = tmp_path / "product-details" / "public" / "1.0" + with (parent / "firefox_versions.json").open() as f: + versions = json.load(f) + assert versions["FIREFOX_NIGHTLY"] == "135.0a1" + assert versions["FIREFOX_DEVEDITION"] == "134.0b9" + assert versions["LATEST_FIREFOX_DEVEL_VERSION"] == "134.0b8" + assert versions["LATEST_FIREFOX_VERSION"] == "133.0" + + with (parent / "firefox_primary_builds.json").open() as f: + primary_builds = json.load(f) + assert set(primary_builds["ach"].keys()) == {"133.0", "134.0b8", "134.0b9", "135.0a1"} + + assert not list(parent.glob("l10n/Devedition-*")) + assert next(parent.glob("l10n/Firefox-134.0b8-*")).name == "Firefox-134.0b8-build1.json"