diff --git a/src/main/scripts/run_e2e_test.py b/src/main/scripts/run_e2e_test.py index 0e7ebaa72..003bdc6fd 100644 --- a/src/main/scripts/run_e2e_test.py +++ b/src/main/scripts/run_e2e_test.py @@ -38,6 +38,7 @@ def run_shell_command(cmd): parser.add_argument('--framework', help='Pass this param if workflow is triggered from e2e framework repo') parser.add_argument('--mvnTestRunProfiles', help='Maven build profiles to run the e2e tests') parser.add_argument('--mvnProjectBuildProfiles', help='Maven project build profiles') +parser.add_argument('--skipPluginUpload', help='Pass this param if plugin upload needs to be skipped') args=parser.parse_args() # Start CDAP sandbox @@ -66,7 +67,7 @@ def run_shell_command(cmd): res = requests.put('http://localhost:11015/v3/preferences', headers= {'Content-Type': 'application/json'}, json={'task.executor.system.resources.memory': 4096}) assert res.ok or print(res.text) -# Upload required plugins from CDAP HUb +# Upload required plugins from CDAP Hub plugin_details_file = open(os.path.join('e2e', 'src', 'main', 'scripts', 'required_plugins.yaml')) plugin_details = yaml.load(plugin_details_file, Loader=yaml.FullLoader) @@ -79,69 +80,67 @@ def run_shell_command(cmd): if args.module: module_to_build = args.module -# Build the plugin -os.chdir("plugin") -if module_to_build: - print(f"Building plugin module: {module_to_build}") - if args.mvnProjectBuildProfiles: - run_shell_command(f"mvn clean install -pl {module_to_build} -am -DskipTests -P {args.mvnProjectBuildProfiles}") - else: - run_shell_command(f"mvn clean install -pl {module_to_build} -am -DskipTests") +if args.skipPluginUpload: + if module_to_build: + print(f"Skipping the plugin upload for module: {module_to_build}") else: - print("Building plugin") - if args.mvnProjectBuildProfiles: - run_shell_command(f"mvn clean package -DskipTests -P {args.mvnProjectBuildProfiles}") + os.chdir("plugin") + if module_to_build: + print(f"Building plugin module: {module_to_build}") + if args.mvnProjectBuildProfiles: + run_shell_command(f"mvn clean install -pl {module_to_build} -am -DskipTests -P {args.mvnProjectBuildProfiles}") + else: + run_shell_command(f"mvn clean install -pl {module_to_build} -am -DskipTests") else: - run_shell_command("mvn clean package -DskipTests") - -# Get plugin artifact name and version from pom.xml. -root = ET.parse('pom.xml').getroot() -plugin_version = root.find('{http://maven.apache.org/POM/4.0.0}version').text -if module_to_build: - os.chdir(f"{module_to_build}") - root = ET.parse('pom.xml').getroot() - plugin_name = root.find('{http://maven.apache.org/POM/4.0.0}artifactId').text -else: - plugin_name = root.find('{http://maven.apache.org/POM/4.0.0}artifactId').text - -os.chdir("target") -plugin_properties = {} -plugin_parents = [] -# Get plugin properties and parent from plugin json. -with open(f'{plugin_name}-{plugin_version}.json') as f: - obj = json.loads(f.read()) - plugin_properties = obj['properties'] - plugin_parents = obj['parents'] - -data = None -with open(f'{plugin_name}-{plugin_version}.jar', 'rb') as f: - data = f.read() - -# Install the plugin on the sandbox. -print("Installing plugin") -res=requests.post(f"http://localhost:11015/v3/namespaces/default/artifacts/{plugin_name}", headers={"Content-Type": "application/octet-stream", "Artifact-Extends": '/'.join(plugin_parents), "Artifact-Version": plugin_version}, data=data) -assert res.ok or print(res.text) -res=requests.put(f"http://localhost:11015/v3/namespaces/default/artifacts/{plugin_name}/versions/{plugin_version}/properties", json=plugin_properties) -assert res.ok or print(res.text) - -if module_to_build: - os.chdir("../../..") - driver_details_file = open(os.path.join('e2e', 'src', 'main', 'scripts', 'driver_details.yaml')) - driver_details = yaml.load(driver_details_file, Loader=yaml.FullLoader) - if module_to_build in driver_details['modules']: - url = driver_details['modules'][module_to_build]['url'] - driver_prop = str(driver_details['modules'][module_to_build]['driver_prop']) - artifact_name = driver_details['modules'][module_to_build]['artifact_name'] - artifact_version = driver_details['modules'][module_to_build]['artifact_version'] - get_driver_jar = requests.get(url) - assert get_driver_jar.ok or print(get_driver_jar.text) - driverData = get_driver_jar.content - print(f"Installing {artifact_name} driver") - res=requests.post(f"http://localhost:11015/v3/namespaces/default/artifacts/{artifact_name}", headers={"Content-Type": "application/octet-stream", "Artifact-Version": artifact_version, "Artifact-Plugins": driver_prop }, data=driverData) + print("Building plugin") + if args.mvnProjectBuildProfiles: + run_shell_command(f"mvn clean package -DskipTests -P {args.mvnProjectBuildProfiles}") + else: + run_shell_command("mvn clean package -DskipTests") + # Get plugin artifact name and version from pom.xml. + root = ET.parse('pom.xml').getroot() + plugin_version = root.find('{http://maven.apache.org/POM/4.0.0}version').text + if module_to_build: + os.chdir(f"{module_to_build}") + root = ET.parse('pom.xml').getroot() + plugin_name = root.find('{http://maven.apache.org/POM/4.0.0}artifactId').text + else: + plugin_name = root.find('{http://maven.apache.org/POM/4.0.0}artifactId').text + os.chdir("target") + plugin_properties = {} + plugin_parents = [] + # Get plugin properties and parent from plugin json. + with open(f'{plugin_name}-{plugin_version}.json') as f: + obj = json.loads(f.read()) + plugin_properties = obj['properties'] + plugin_parents = obj['parents'] + data = None + with open(f'{plugin_name}-{plugin_version}.jar', 'rb') as f: + data = f.read() + # Install the plugin on the sandbox. + print("Installing plugin") + res=requests.post(f"http://localhost:11015/v3/namespaces/default/artifacts/{plugin_name}", headers={"Content-Type": "application/octet-stream", "Artifact-Extends": '/'.join(plugin_parents), "Artifact-Version": plugin_version}, data=data) + assert res.ok or print(res.text) + res=requests.put(f"http://localhost:11015/v3/namespaces/default/artifacts/{plugin_name}/versions/{plugin_version}/properties", json=plugin_properties) assert res.ok or print(res.text) -else: - os.chdir("../..") + if module_to_build: + os.chdir("../../..") + driver_details_file = open(os.path.join('e2e', 'src', 'main', 'scripts', 'driver_details.yaml')) + driver_details = yaml.load(driver_details_file, Loader=yaml.FullLoader) + if module_to_build in driver_details['modules']: + url = driver_details['modules'][module_to_build]['url'] + driver_prop = str(driver_details['modules'][module_to_build]['driver_prop']) + artifact_name = driver_details['modules'][module_to_build]['artifact_name'] + artifact_version = driver_details['modules'][module_to_build]['artifact_version'] + get_driver_jar = requests.get(url) + assert get_driver_jar.ok or print(get_driver_jar.text) + driverData = get_driver_jar.content + print(f"Installing {artifact_name} driver") + res=requests.post(f"http://localhost:11015/v3/namespaces/default/artifacts/{artifact_name}", headers={"Content-Type": "application/octet-stream", "Artifact-Version": artifact_version, "Artifact-Plugins": driver_prop }, data=driverData) + assert res.ok or print(res.text) + else: + os.chdir("../..") print("cwd:", os.getcwd()) print("ls:", os.listdir())