Update tests

Signed-off-by: Zelin Hao <[email protected]>

src/build_workflow/build_incremental.py

@@ -81,8 +81,8 @@ def build_incremental(self, args: BuildArgs, input_manifest: InputManifest, comp
 
         build_manifest_data = BuildManifest.from_path(build_manifest_path).__to_dict__()
 
-        # output_dir = BuildOutputDir(input_manifest.build.filename, args.distribution).dir
-        output_dir = BuildOutputDir("temp", args.distribution).dir
+        output_dir = BuildOutputDir(input_manifest.build.filename, args.distribution).dir
+        # output_dir = BuildOutputDir("temp", args.distribution).dir
         failed_plugins = []
 
         with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
@@ -100,15 +100,7 @@ def build_incremental(self, args: BuildArgs, input_manifest: InputManifest, comp
             )
 
             build_recorder_incremental = BuildRecorder(target, build_manifest_data)
-            # build_recorder = BuildRecorder(target)
-            # git_repo = GitRepository(
-            #     "https://github.com/zelinh/common-utils",
-            #     "main",
-            # )
-            #
-            # build_recorder.record_component("common-utils", git_repo)
-            # build_recorder.write_manifest()
-        #
+
             logging.info(f"Building {input_manifest.build.name} ({target.architecture}) into {target.output_dir}")
 
             for component in input_manifest.components.select(focus=components, platform=target.platform):
@@ -127,50 +119,10 @@ def build_incremental(self, args: BuildArgs, input_manifest: InputManifest, comp
                         continue
                     else:
                         raise
+
             build_recorder_incremental.write_manifest()
 
         if len(failed_plugins) > 0:
             logging.error(f"Failed plugins are {failed_plugins}")
         logging.info("Done.")
 
-
-
-        # output_dir = BuildOutputDir(input_manifest.build.filename, args.distribution).dir
-        #
-        # with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
-        #     logging.info(f"Building in {work_dir.name}")
-        #
-        # target = BuildTarget(
-        #     name=input_manifest.build.name,
-        #     version=input_manifest.build.version,
-        #     qualifier=input_manifest.build.qualifier,
-        #     patches=input_manifest.build.patches,
-        #     snapshot=args.snapshot if args.snapshot is not None else input_manifest.build.snapshot,
-        #     output_dir=output_dir,
-        #     distribution=args.distribution,
-        #     platform=args.platform or input_manifest.build.platform,
-        #     architecture=args.architecture or input_manifest.build.architecture,
-        # )
-        #
-        # build_recorder = BuildRecorder(target, build_manifest_data)
-        #
-        # logging.info(f"Building {input_manifest.build.name} ({target.architecture}) into {target.output_dir}")
-        #
-        # for component in input_manifest.components.select(focus=components, platform=target.platform):
-        #     logging.info(f"Building {component.name}")
-        #
-        #     builder = Builders.builder_from(component, target)
-        #     try:
-        #         builder.checkout(work_dir.name)
-        #         builder.build(build_recorder)
-        #         builder.export_artifacts(build_recorder)
-        #         logging.info(f"Successfully built {component.name}")
-        #     except:
-        #         logging.error(f"Error building {component.name}, retry with: {args.component_command(component.name)}")
-        #         # if args.continue_on_error and component.name not in ['OpenSearch', 'job-scheduler', 'common-utils', 'OpenSearch-Dashboards']:
-        #         #     failed_plugins.append(component.name)
-        #         #     continue
-        #         # else:
-        #         #     raise
-        #
-        # build_recorder.write_manifest()

tests/tests_build_workflow/test_build_incremental.py

@@ -5,10 +5,13 @@
 # this file be licensed under the Apache-2.0 license or a
 # compatible open source license.
 
+import logging
 import os
+import tempfile
 import unittest
-from typing import List
-from unittest.mock import MagicMock, patch
+from typing import List, Any
+from unittest.mock import MagicMock, patch, call
+import yaml
 
 from build_workflow.build_incremental import BuildIncremental
 from manifests.build_manifest import BuildManifest
@@ -20,6 +23,7 @@ class TestBuildIncremental(unittest.TestCase):
         os.path.join(os.path.dirname(__file__), "data", "opensearch-input-2.12.0.yml"))
     BUILD_MANIFEST = BuildManifest.from_path(
         os.path.join(os.path.dirname(__file__), "data", "opensearch-build-tar-2.12.0.yml"))
+    BUILD_MANIFEST_PATH = os.path.join(os.path.dirname(__file__), "data", "opensearch-build-tar-2.12.0.yml")
     INPUT_MANIFEST_DASHBOARDS = InputManifest.from_path(
         os.path.join(os.path.dirname(__file__), "data", "opensearch-dashboards-input-2.12.0.yml"))
     BUILD_MANIFEST_DASHBOARDS = BuildManifest.from_path(
@@ -158,3 +162,52 @@ def test_rebuild_plugins_with_dashboards(self) -> None:
         self.assertEqual(len(rebuild_list), 2)
         self.assertTrue("OpenSearch-Dashboards" in rebuild_list)
         self.assertTrue("observabilityDashboards" in rebuild_list)
+
+    # @patch("argparse._sys.argv", ["run_build.py", INPUT_MANIFEST, "-p", "linux"])
+    @patch("os.path.join")
+    @patch("build_workflow.build_incremental.Builders.builder_from", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.BuildRecorder", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.TemporaryDirectory")
+    def test_build_incremental_no_prebuild_manifest(self, mock_temp: MagicMock, mock_recorder: MagicMock, mock_builder: MagicMock, mock_join_path: MagicMock, *mocks: Any) -> None:
+        mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir()
+        args = MagicMock()
+        mock_join_path.return_value = "non_exist_path"
+        try:
+            self.buildIncremental.build_incremental(args, self.INPUT_MANIFEST, ["common-utils"])
+            self.assertRaises(FileNotFoundError)
+        except FileNotFoundError:
+            pass
+
+    @patch("build_workflow.build_incremental.logging.info")
+    @patch("paths.build_output_dir")
+    @patch("os.path.exists")
+    @patch("manifests.build_manifest.BuildManifest.from_path")
+    @patch("build_workflow.build_incremental.Builders.builder_from", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.BuildRecorder", return_value=MagicMock())
+    @patch("build_workflow.build_incremental.TemporaryDirectory")
+    def test_build_incremental_with_prebuild_manifest(self, mock_temp: MagicMock, mock_recorder: MagicMock, mock_builder: MagicMock, mock_build_manifest: MagicMock, mock_path_exist: MagicMock,
+                                                      mock_build_output_dir: MagicMock, mock_logging_info: MagicMock, *mocks: Any) -> None:
+        mock_temp.return_value.__enter__.return_value.name = tempfile.gettempdir()
+        args = MagicMock()
+        args.distribution = "tar"
+        args.keep = False
+        args.snapshot = None
+        args.platform = "linux"
+        args.architecture = "x64"
+        mock_path_exist.return_value = True
+        mock_build_manifest.return_value = self.BUILD_MANIFEST
+        self.buildIncremental.build_incremental(args, self.INPUT_MANIFEST, ["common-utils", "opensearch-observability"])
+        mock_build_manifest.assert_called_once()
+        mock_build_manifest.assert_called_with(os.path.join("tar", "builds", "opensearch", "manifest.yml"))
+        self.assertTrue(args.platform == "linux")
+        self.assertNotEqual(mock_builder.return_value.build.call_count, 0)
+        self.assertEqual(mock_builder.return_value.build.call_count, 2)
+        self.assertEqual(mock_builder.return_value.build.call_count, mock_builder.return_value.export_artifacts.call_count)
+
+        mock_logging_info.assert_has_calls([
+            call('Rebuilding common-utils'),
+            call('Rebuilding opensearch-observability'),
+        ], any_order=True)
+
+        mock_recorder.assert_called_once()
+        mock_recorder.return_value.write_manifest.assert_called()

tests/tests_build_workflow/test_build_recorder.py

@@ -246,9 +246,7 @@ def test_record_artifact_check_plugin_version_properties_snapshot(self, mock_plu
         mock_copyfile.assert_called()
         mock_makedirs.assert_called()
 
-    @patch("shutil.copyfile")
-    @patch("os.makedirs")
-    def test_append_component_with_existing_manifest(self, mock_makedirs: Mock, mock_copyfile: Mock) -> None:
+    def test_append_component_with_existing_manifest(self) -> None:
         mock = self.__mock_with_manifest(snapshot=False)
 
         self.assertEqual(mock.build_manifest.components_hash.get("job-scheduler").get("commit_id"), "aaf09b0211df15dd74ff2756f2590c360b03486b")
@@ -271,18 +269,3 @@ def test_append_component_with_existing_manifest(self, mock_makedirs: Mock, mock
         self.assertEqual(mock.build_manifest.components_hash.get("job-scheduler").get("repository"), "https://github.com/opensearch-project/job-scheduler.git")
         self.assertEqual(mock.build_manifest.components_hash.get("geospatial").get("commit_id"), "8776900f2f26312b4d3a08e4343f3e3f7bdde536")
         self.assertEqual(mock.build_manifest.components_hash.get("security").get("commit_id"), "e3c8902dea26fd20f56a6f144042b2623f652e3e")
-    #
-        # recorder.record_component(
-        #     "common-utils",
-        #     MagicMock(
-        #         url="https://github.com/opensearch-project/common-utils",
-        #         ref="main",
-        #         sha="3913d7097934cbfe1fdcf919347f22a597d00b76",
-        #     ),
-        # )
-        #
-        # recorder.record_artifact("common-utils", "files", os.path.join("..", "file1.jar"), __file__)
-        #
-        # output_dir = os.path.join("output_dir", "..")
-        # mock_makedirs.assert_called_with(output_dir, exist_ok=True)
-        # mock_copyfile.assert_called_with(__file__, os.path.join(output_dir, "file1.jar"))