From 1c1e7484369449c9849adbfb6d6c348f00793c79 Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:26:39 -0700 Subject: [PATCH 1/9] Added nodejs tests --- .../buildcmd/test_build_in_source.py | 77 ++++++++++++++++++- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index f7fd6c9c5a..9d6d1ef942 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -1,11 +1,16 @@ import os +from pathlib import Path import shutil import pytest import logging -from unittest import skip from parameterized import parameterized -from tests.integration.buildcmd.build_integ_base import BuildIntegProvidedBase, BuildIntegEsbuildBase +from tests.integration.buildcmd.build_integ_base import ( + BuildIntegNodeBase, + BuildIntegProvidedBase, + BuildIntegEsbuildBase, +) +from tests.testing_utils import run_command LOG = logging.getLogger(__name__) @@ -107,3 +112,71 @@ def test_builds_successfully_with_local_dependency(self): # check whether dependencies were installed in source dir self.assertEqual(os.path.isdir(os.path.join(codeuri, "node_modules")), True) + + +class TestBuildCommand_BuildInSource_Nodejs(BuildIntegNodeBase): + is_nested_parent = False + template = "template.yaml" + + def setUp(self): + super().setUp() + self.source_directories = [] + + def tearDown(self): + super().tearDown() + # clean up dependencies installed in source directories + for source in self.source_directories: + shutil.rmtree(Path(source, "node_modules"), ignore_errors=True) + + def validate_node_modules_folder(self, expected_result: bool): + source_node_modules = Path(self.codeuri_path, "node_modules") + + self.assertEqual(source_node_modules.is_dir(), expected_result, "node_modules not found in source folder") + + def validate_node_modules_linked(self): + built_node_modules = Path(self.default_build_dir, "Function", "node_modules") + + self.assertEqual(built_node_modules.is_dir(), True, "node_modules not found in artifact folder") + self.assertEqual(built_node_modules.is_symlink(), True, "node_modules not a symlink in artifact folder") + + @parameterized.expand( + [ + (True, True), # build in source + (False, False), # don't build in source + (None, False), # use default for workflow (don't build in source) + ] + ) + # @pytest.mark.flaky(reruns=3) + def test_builds_successfully_without_local_dependencies(self, build_in_source, dependencies_expected_in_source): + self.codeuri_path = Path(self.test_data_path, "Node") + self.source_directories = [str(self.codeuri_path)] + + overrides = self.get_override( + runtime="nodejs16.x", code_uri=self.CODE_URI, architecture="x86_64", handler="main.lambdaHandler" + ) + command_list = self.get_command_list(build_in_source=build_in_source, parameter_overrides=overrides) + + run_command(command_list, self.working_dir) + + # check whether dependencies were installed in source dir + self.validate_node_modules_folder(dependencies_expected_in_source) + + # @pytest.mark.flaky(reruns=3) + def test_builds_successfully_with_local_dependency(self): + codeuri_folder = Path("Esbuild", "NodeWithLocalDependency") + + self.codeuri_path = Path(self.test_data_path, codeuri_folder) + self.CODE_URI = str(codeuri_folder) + + self.source_directories = [str(self.codeuri_path)] + + overrides = self.get_override( + runtime="nodejs16.x", code_uri=self.CODE_URI, architecture="x86_64", handler="main.lambdaHandler" + ) + command_list = self.get_command_list(build_in_source=True, parameter_overrides=overrides) + + run_command(command_list, self.working_dir) + + # check whether dependencies were installed in source dir + self.validate_node_modules_folder(True) + self.validate_node_modules_linked() \ No newline at end of file From 11eacdff3cf7fba6b342faf8963fcb50e71088d8 Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:27:19 -0700 Subject: [PATCH 2/9] make format --- tests/integration/buildcmd/test_build_in_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index 9d6d1ef942..d7cc2740ff 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -179,4 +179,4 @@ def test_builds_successfully_with_local_dependency(self): # check whether dependencies were installed in source dir self.validate_node_modules_folder(True) - self.validate_node_modules_linked() \ No newline at end of file + self.validate_node_modules_linked() From e707b585ba77bca8a4a0ec9267880b825d6db08f Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Mon, 23 Oct 2023 16:28:11 -0700 Subject: [PATCH 3/9] Enable retries --- tests/integration/buildcmd/test_build_in_source.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index d7cc2740ff..fd59a56728 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -146,7 +146,7 @@ def validate_node_modules_linked(self): (None, False), # use default for workflow (don't build in source) ] ) - # @pytest.mark.flaky(reruns=3) + @pytest.mark.flaky(reruns=3) def test_builds_successfully_without_local_dependencies(self, build_in_source, dependencies_expected_in_source): self.codeuri_path = Path(self.test_data_path, "Node") self.source_directories = [str(self.codeuri_path)] @@ -161,7 +161,7 @@ def test_builds_successfully_without_local_dependencies(self, build_in_source, d # check whether dependencies were installed in source dir self.validate_node_modules_folder(dependencies_expected_in_source) - # @pytest.mark.flaky(reruns=3) + @pytest.mark.flaky(reruns=3) def test_builds_successfully_with_local_dependency(self): codeuri_folder = Path("Esbuild", "NodeWithLocalDependency") From 0ef8a74cea0aa34700d43084158af9cca6a3b9bd Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Wed, 25 Oct 2023 11:21:37 -0700 Subject: [PATCH 4/9] Run build test inside of a scratch directory --- .../buildcmd/test_build_in_source.py | 73 +++++++++---------- 1 file changed, 34 insertions(+), 39 deletions(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index fd59a56728..9eed9b5060 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -26,6 +26,15 @@ def setUpClass(cls): cls.code_uri_path = os.path.join(cls.test_data_path, cls.code_uri) cls.file_created_from_make_command = "file-created-from-make-command.txt" + def setUp(self): + super().setUp() + + scratch_code_uri_path = Path(self.working_dir, self.code_uri) + + # copy source code into temporary directory and update code uri to that scratch dir + shutil.copytree(self.code_uri_path, scratch_code_uri_path, dirs_exist_ok=True) + self.code_uri_path = str(scratch_code_uri_path) + def tearDown(self): super().tearDown() new_file_in_codeuri_path = os.path.join(self.code_uri_path, self.file_created_from_make_command) @@ -60,13 +69,9 @@ class TestBuildCommand_BuildInSource_Esbuild(BuildIntegEsbuildBase): def setUp(self): super().setUp() - self.source_directories = [] - def tearDown(self): - super().tearDown() - # clean up dependencies installed in source directories - for source in self.source_directories: - shutil.rmtree(os.path.join(source, "node_modules"), ignore_errors=True) + source_files_path = Path(self.test_data_path, "Esbuild") + shutil.copytree(source_files_path, self.working_dir, dirs_exist_ok=True) @parameterized.expand( [ @@ -77,12 +82,11 @@ def tearDown(self): ) @pytest.mark.flaky(reruns=3) def test_builds_successfully_without_local_dependencies(self, build_in_source, dependencies_expected_in_source): - codeuri = os.path.join(self.test_data_path, "Esbuild", "Node") - self.source_directories = [codeuri] + codeuri = os.path.join(self.working_dir, "Node") self._test_with_default_package_json( build_in_source=build_in_source, - runtime="nodejs16.x", + runtime="nodejs18.x", code_uri=codeuri, handler="main.lambdaHandler", architecture="x86_64", @@ -95,9 +99,8 @@ def test_builds_successfully_without_local_dependencies(self, build_in_source, d @pytest.mark.flaky(reruns=3) def test_builds_successfully_with_local_dependency(self): - codeuri = os.path.join(self.test_data_path, "Esbuild", "NodeWithLocalDependency") - self.source_directories = [codeuri] - runtime = "nodejs16.x" + codeuri = os.path.join(self.working_dir, "NodeWithLocalDependency") + runtime = "nodejs18.x" architecture = "x86_64" self._test_with_default_package_json( @@ -120,24 +123,19 @@ class TestBuildCommand_BuildInSource_Nodejs(BuildIntegNodeBase): def setUp(self): super().setUp() - self.source_directories = [] def tearDown(self): super().tearDown() - # clean up dependencies installed in source directories - for source in self.source_directories: - shutil.rmtree(Path(source, "node_modules"), ignore_errors=True) - - def validate_node_modules_folder(self, expected_result: bool): - source_node_modules = Path(self.codeuri_path, "node_modules") - self.assertEqual(source_node_modules.is_dir(), expected_result, "node_modules not found in source folder") - - def validate_node_modules_linked(self): + def validate_node_modules(self, is_build_in_source_behaviour: bool): + # validate if node modules exist in the built artifact dir built_node_modules = Path(self.default_build_dir, "Function", "node_modules") + self.assertEqual(built_node_modules.is_dir(), True, "node_modules not found in artifact dir") + self.assertEqual(built_node_modules.is_symlink(), is_build_in_source_behaviour) - self.assertEqual(built_node_modules.is_dir(), True, "node_modules not found in artifact folder") - self.assertEqual(built_node_modules.is_symlink(), True, "node_modules not a symlink in artifact folder") + # validate that node modules are suppose to exist in the source dir + source_node_modules = Path(self.codeuri_path, "node_modules") + self.assertEqual(source_node_modules.is_dir(), is_build_in_source_behaviour) @parameterized.expand( [ @@ -147,36 +145,33 @@ def validate_node_modules_linked(self): ] ) @pytest.mark.flaky(reruns=3) - def test_builds_successfully_without_local_dependencies(self, build_in_source, dependencies_expected_in_source): - self.codeuri_path = Path(self.test_data_path, "Node") - self.source_directories = [str(self.codeuri_path)] + def test_builds_successfully_without_local_dependencies(self, build_in_source, expected_built_in_source): + code_uri = "Node" + source_path = Path(self.test_data_path, code_uri) + self.codeuri_path = Path(self.working_dir, code_uri) + + shutil.copytree(source_path, self.codeuri_path, dirs_exist_ok=True, symlinks=True) overrides = self.get_override( - runtime="nodejs16.x", code_uri=self.CODE_URI, architecture="x86_64", handler="main.lambdaHandler" + runtime="nodejs18.x", code_uri=self.codeuri_path, architecture="x86_64", handler="main.lambdaHandler" ) - command_list = self.get_command_list(build_in_source=build_in_source, parameter_overrides=overrides) + command_list = self.get_command_list(build_in_source=build_in_source, parameter_overrides=overrides, debug=True) run_command(command_list, self.working_dir) - - # check whether dependencies were installed in source dir - self.validate_node_modules_folder(dependencies_expected_in_source) + self.validate_node_modules(expected_built_in_source) @pytest.mark.flaky(reruns=3) def test_builds_successfully_with_local_dependency(self): codeuri_folder = Path("Esbuild", "NodeWithLocalDependency") + shutil.copytree(Path(self.test_data_path, "Esbuild"), self.working_dir, dirs_exist_ok=True) self.codeuri_path = Path(self.test_data_path, codeuri_folder) self.CODE_URI = str(codeuri_folder) - self.source_directories = [str(self.codeuri_path)] - overrides = self.get_override( - runtime="nodejs16.x", code_uri=self.CODE_URI, architecture="x86_64", handler="main.lambdaHandler" + runtime="nodejs18.x", code_uri=self.CODE_URI, architecture="x86_64", handler="main.lambdaHandler" ) command_list = self.get_command_list(build_in_source=True, parameter_overrides=overrides) run_command(command_list, self.working_dir) - - # check whether dependencies were installed in source dir - self.validate_node_modules_folder(True) - self.validate_node_modules_linked() + self.validate_node_modules(True) \ No newline at end of file From 5f450314577fa36623b89ad756d2edf3edab4818 Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Wed, 25 Oct 2023 11:26:09 -0700 Subject: [PATCH 5/9] make format --- tests/integration/buildcmd/test_build_in_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index 9eed9b5060..b46729218d 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -174,4 +174,4 @@ def test_builds_successfully_with_local_dependency(self): command_list = self.get_command_list(build_in_source=True, parameter_overrides=overrides) run_command(command_list, self.working_dir) - self.validate_node_modules(True) \ No newline at end of file + self.validate_node_modules(True) From 1b8a5702d3fea3eba23b59086a51ea0aacf37c9c Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:38:56 -0700 Subject: [PATCH 6/9] Updated Makefile test to properly use scratch directory and moved source copy to setup --- .../buildcmd/test_build_in_source.py | 32 +++++++------------ 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index b46729218d..6b986f6f45 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -22,24 +22,22 @@ class TestBuildCommand_BuildInSource_Makefile(BuildIntegProvidedBase): @classmethod def setUpClass(cls): super().setUpClass() - cls.code_uri = "provided_create_new_file" - cls.code_uri_path = os.path.join(cls.test_data_path, cls.code_uri) - cls.file_created_from_make_command = "file-created-from-make-command.txt" def setUp(self): super().setUp() + self.code_uri = "provided_create_new_file" + test_data_code_uri = Path(self.test_data_path, self.code_uri) + self.file_created_from_make_command = "file-created-from-make-command.txt" + scratch_code_uri_path = Path(self.working_dir, self.code_uri) + self.code_uri_path = str(scratch_code_uri_path) # copy source code into temporary directory and update code uri to that scratch dir - shutil.copytree(self.code_uri_path, scratch_code_uri_path, dirs_exist_ok=True) - self.code_uri_path = str(scratch_code_uri_path) + shutil.copytree(test_data_code_uri, scratch_code_uri_path, dirs_exist_ok=True) def tearDown(self): super().tearDown() - new_file_in_codeuri_path = os.path.join(self.code_uri_path, self.file_created_from_make_command) - if os.path.isfile(new_file_in_codeuri_path): - os.remove(new_file_in_codeuri_path) @parameterized.expand( [ @@ -54,7 +52,7 @@ def test_builds_successfully_with_makefile(self, build_in_source, new_file_shoul runtime="provided.al2", use_container=False, manifest=None, - code_uri=self.code_uri, + code_uri=self.code_uri_path, build_in_source=build_in_source, ) @@ -124,6 +122,8 @@ class TestBuildCommand_BuildInSource_Nodejs(BuildIntegNodeBase): def setUp(self): super().setUp() + shutil.copytree(Path(self.test_data_path, "Esbuild"), self.working_dir, dirs_exist_ok=True, symlinks=True) + def tearDown(self): super().tearDown() @@ -146,11 +146,7 @@ def validate_node_modules(self, is_build_in_source_behaviour: bool): ) @pytest.mark.flaky(reruns=3) def test_builds_successfully_without_local_dependencies(self, build_in_source, expected_built_in_source): - code_uri = "Node" - source_path = Path(self.test_data_path, code_uri) - self.codeuri_path = Path(self.working_dir, code_uri) - - shutil.copytree(source_path, self.codeuri_path, dirs_exist_ok=True, symlinks=True) + self.codeuri_path = Path(self.working_dir, "Node") overrides = self.get_override( runtime="nodejs18.x", code_uri=self.codeuri_path, architecture="x86_64", handler="main.lambdaHandler" @@ -162,14 +158,10 @@ def test_builds_successfully_without_local_dependencies(self, build_in_source, e @pytest.mark.flaky(reruns=3) def test_builds_successfully_with_local_dependency(self): - codeuri_folder = Path("Esbuild", "NodeWithLocalDependency") - - shutil.copytree(Path(self.test_data_path, "Esbuild"), self.working_dir, dirs_exist_ok=True) - self.codeuri_path = Path(self.test_data_path, codeuri_folder) - self.CODE_URI = str(codeuri_folder) + self.codeuri_path = Path(self.working_dir, "NodeWithLocalDependency") overrides = self.get_override( - runtime="nodejs18.x", code_uri=self.CODE_URI, architecture="x86_64", handler="main.lambdaHandler" + runtime="nodejs18.x", code_uri=self.codeuri_path, architecture="x86_64", handler="main.lambdaHandler" ) command_list = self.get_command_list(build_in_source=True, parameter_overrides=overrides) From edd438df046ef5a6583cad986d40e99197e334cf Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Mon, 30 Oct 2023 11:46:27 -0700 Subject: [PATCH 7/9] Changed copytree call to in house version --- tests/integration/buildcmd/test_build_in_source.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index 6b986f6f45..a994c0d9f1 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -1,9 +1,9 @@ import os from pathlib import Path -import shutil import pytest import logging from parameterized import parameterized +from samcli.lib.utils import osutils from tests.integration.buildcmd.build_integ_base import ( BuildIntegNodeBase, @@ -34,7 +34,7 @@ def setUp(self): self.code_uri_path = str(scratch_code_uri_path) # copy source code into temporary directory and update code uri to that scratch dir - shutil.copytree(test_data_code_uri, scratch_code_uri_path, dirs_exist_ok=True) + osutils.copytree(test_data_code_uri, scratch_code_uri_path) def tearDown(self): super().tearDown() @@ -69,7 +69,7 @@ def setUp(self): super().setUp() source_files_path = Path(self.test_data_path, "Esbuild") - shutil.copytree(source_files_path, self.working_dir, dirs_exist_ok=True) + osutils.copytree(source_files_path, self.working_dir) @parameterized.expand( [ @@ -122,7 +122,7 @@ class TestBuildCommand_BuildInSource_Nodejs(BuildIntegNodeBase): def setUp(self): super().setUp() - shutil.copytree(Path(self.test_data_path, "Esbuild"), self.working_dir, dirs_exist_ok=True, symlinks=True) + osutils.copytree(Path(self.test_data_path, "Esbuild"), self.working_dir) def tearDown(self): super().tearDown() From 0a33294bb41a62aaad248be897c4d5d7594c52d3 Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:10:01 -0700 Subject: [PATCH 8/9] Removed redundant parent method calling --- tests/integration/buildcmd/test_build_in_source.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index a994c0d9f1..7c33d5f1eb 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -18,11 +18,7 @@ class TestBuildCommand_BuildInSource_Makefile(BuildIntegProvidedBase): template = "template.yaml" is_nested_parent = False - - @classmethod - def setUpClass(cls): - super().setUpClass() - + def setUp(self): super().setUp() @@ -36,9 +32,6 @@ def setUp(self): # copy source code into temporary directory and update code uri to that scratch dir osutils.copytree(test_data_code_uri, scratch_code_uri_path) - def tearDown(self): - super().tearDown() - @parameterized.expand( [ (True, True), # build in source From 6b5915eee59052c74aa7e6d6b1ea165473bdc2cf Mon Sep 17 00:00:00 2001 From: Lucas <12496191+lucashuy@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:15:22 -0700 Subject: [PATCH 9/9] make format --- tests/integration/buildcmd/test_build_in_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/buildcmd/test_build_in_source.py b/tests/integration/buildcmd/test_build_in_source.py index 7c33d5f1eb..6bb8d4ab43 100644 --- a/tests/integration/buildcmd/test_build_in_source.py +++ b/tests/integration/buildcmd/test_build_in_source.py @@ -18,7 +18,7 @@ class TestBuildCommand_BuildInSource_Makefile(BuildIntegProvidedBase): template = "template.yaml" is_nested_parent = False - + def setUp(self): super().setUp()