Chore/test modifications for helm (#112)

* Tests modification for Helm support

* Changes for fence clients setup

* Changes for subprocess.run

* Delete gen3-integration-tests/test_data/fence_clients/clients_creds.txt

* Delete gen3-integration-tests/test_data/fence_clients/client_rotate_creds.txt

* Changes for GAT test_env_namespace

* minor fix

* minor fix

* Changes for guppy support on helm

* Guppy data - changed folder structure

* Guppy data - Change folder structure

* Updated help_setup.sh

* guppy path fix

.secrets.baseline

@@ -136,15 +136,15 @@
         "filename": "gen3-integration-tests/tests/test_audit_service.py",
         "hashed_secret": "62bd0c4d3a6b445b13212d23500a7f0916757c3e",
         "is_verified": false,
-        "line_number": 183,
+        "line_number": 185,
         "is_secret": false
       },
       {
         "type": "Hex High Entropy String",
         "filename": "gen3-integration-tests/tests/test_audit_service.py",
         "hashed_secret": "e1fa8c0685ac2b51db97b7bd00b9afa63489d30c",
         "is_verified": false,
-        "line_number": 192,
+        "line_number": 194,
         "is_secret": false
       }
     ],
@@ -244,47 +244,47 @@
         "filename": "gen3-integration-tests/tests/test_dbgap.py",
         "hashed_secret": "62bd0c4d3a6b445b13212d23500a7f0916757c3e",
         "is_verified": false,
-        "line_number": 24,
+        "line_number": 23,
         "is_secret": false
       },
       {
         "type": "Hex High Entropy String",
         "filename": "gen3-integration-tests/tests/test_dbgap.py",
         "hashed_secret": "e1fa8c0685ac2b51db97b7bd00b9afa63489d30c",
         "is_verified": false,
-        "line_number": 34,
+        "line_number": 33,
         "is_secret": false
       },
       {
         "type": "Hex High Entropy String",
         "filename": "gen3-integration-tests/tests/test_dbgap.py",
         "hashed_secret": "faad2c1400e2071d0b52a70589e6a69e960a8f66",
         "is_verified": false,
-        "line_number": 44,
+        "line_number": 43,
         "is_secret": false
       },
       {
         "type": "Hex High Entropy String",
         "filename": "gen3-integration-tests/tests/test_dbgap.py",
         "hashed_secret": "9b985a7c923d1be953bdc172f3a598fb2a4f6f41",
         "is_verified": false,
-        "line_number": 54,
+        "line_number": 53,
         "is_secret": false
       },
       {
         "type": "Hex High Entropy String",
         "filename": "gen3-integration-tests/tests/test_dbgap.py",
         "hashed_secret": "f270519ee38273a587bfad132bba4f3fd06b335e",
         "is_verified": false,
-        "line_number": 61,
+        "line_number": 60,
         "is_secret": false
       },
       {
         "type": "Hex High Entropy String",
         "filename": "gen3-integration-tests/tests/test_dbgap.py",
         "hashed_secret": "191e56b8fb18e2008781d6953cbb73be28ce66e7",
         "is_verified": false,
-        "line_number": 75,
+        "line_number": 74,
         "is_secret": false
       }
     ],
@@ -330,7 +330,7 @@
         "filename": "gen3-integration-tests/tests/test_oidc_client.py",
         "hashed_secret": "62bd0c4d3a6b445b13212d23500a7f0916757c3e",
         "is_verified": false,
-        "line_number": 147
+        "line_number": 149
       }
     ],
     "gen3-integration-tests/tests/test_presigned_url.py": [
@@ -363,5 +363,5 @@
       }
     ]
   },
-  "generated_at": "2024-09-24T17:48:54Z"
+  "generated_at": "2024-09-27T15:33:17Z"
 }

gen3-integration-tests/services/guppy.py

@@ -42,12 +42,12 @@ def validate_guppy_query(
         queryType - Can be one of the following : mapping / aggregation / historgram / download
         endpoint - Can be one of the following : /graphql or /download
         """
-        queryFile = (
-            TEST_DATA_PATH_OBJECT / "guppy_data" / "testData" / queryFile
-        ).read_text(encoding="UTF-8")
-        responseFile = (
-            TEST_DATA_PATH_OBJECT / "guppy_data" / "testData" / responseFile
-        ).read_text(encoding="UTF-8")
+        queryFile = (TEST_DATA_PATH_OBJECT / "guppy" / queryFile).read_text(
+            encoding="UTF-8"
+        )
+        responseFile = (TEST_DATA_PATH_OBJECT / "guppy" / responseFile).read_text(
+            encoding="UTF-8"
+        )
         queryToSubmit = "".join(queryFile.split("\n"))
         logger.info(queryToSubmit)
         auth = Gen3Auth(refresh_token=pytest.api_keys[user], endpoint=pytest.root_url)

gen3-integration-tests/test_data/test_setup/clients.csv

@@ -9,4 +9,4 @@ jenkins-client-medium-expiration,test-user,client_credentials,None,4,
 jenkins-client-long-expiration,test-user,client_credentials,None,30,
 ras-test-client,UCtestuser128,basic,programs.QA-admin programs.test-admin programs.DEV-admin programs.jnkns-admin,,
 ras-test-client1,UCtestuser127,auth_code,programs.QA-admin programs.test-admin programs.DEV-admin programs.jnkns-admin,,openid user data google_credentials ga4gh_passport_v1
-ras-test-client2,UCtestuser129,auth_code,programs.QA-admin programs.test-admin programs.DEV-admin programs.jnkns-admin,,openid user data google_credentials
+ras-test-client2,UCtestuser129,auth_code,programs.QA-admin programs.test-admin programs.DEV-admin programs.jnkns-admin,,openid user data google_credentials

gen3-integration-tests/test_data/test_setup/guppy_es_indexes/es_indexes_data/configs_alias.json

@@ -0,0 +1,6 @@
+{
+    "actions" : [
+        { "remove" : { "index" : "jenkins_configs_1", "alias" : "jenkins_configs_alias" } },
+        { "add" : { "index" : "jenkins_configs_1", "alias" : "jenkins_configs_alias" } }
+    ]
+}

gen3-integration-tests/test_data/test_setup/guppy_es_indexes/es_indexes_data/file_alias.json

@@ -0,0 +1,6 @@
+{
+    "actions" : [
+        { "remove" : { "index" : "jenkins_file_1", "alias" : "jenkins_file_alias" } },
+        { "add" : { "index" : "jenkins_file_1", "alias" : "jenkins_file_alias" } }
+    ]
+}

gen3-integration-tests/test_data/test_setup/guppy_es_indexes/es_indexes_data/subject_alias.json

@@ -0,0 +1,6 @@
+{
+    "actions" : [
+        { "remove" : { "index" : "jenkins_subject_1", "alias" : "jenkins_subject_alias" } },
+        { "add" : { "index" : "jenkins_subject_1", "alias" : "jenkins_subject_alias" } }
+    ]
+}

gen3-integration-tests/test_data/test_setup/guppy_es_indexes/helm_setup.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Make sure to run the below command on a seperate window for the curl commands to work
+# kubectl port-forward service/gen3-elasticsearch-master 9200:9200
+
+# Create jenkins_subject_1 indices
+curl -iv -X PUT "localhost:9200/jenkins_subject_1" -H 'Content-Type: application/json' -H 'Accept: application/json' "-d@es_indexes_data/subject_mapping.json"
+curl -X POST localhost:9200/_aliases -H 'Content-Type: application/json' -H 'Accept: application/json' "-d@es_indexes_data/subject_alias.json"
+curl -H 'Content-Type: application/x-ndjson' "localhost:9200/jenkins_subject_1/subject/_bulk" --data-binary @es_indexes_data/subject_batch.ndjson
+
+# Create jenkins_file_1 indices
+curl -iv -X PUT "localhost:9200/jenkins_file_1" -H 'Content-Type: application/json' -H 'Accept: application/json' "-d@es_indexes_data/file_mapping.json"
+curl -X POST localhost:9200/_aliases -H 'Content-Type: application/json' -H 'Accept: application/json' "-d@es_indexes_data/file_alias.json"
+curl -H 'Content-Type: application/x-ndjson' "localhost:9200/jenkins_file_1/subject/_bulk" --data-binary @es_indexes_data/file_batch.ndjson
+
+# Create jenkins_config_1 indices
+curl -iv -X PUT "localhost:9200/jenkins_configs_1" -H 'Content-Type: application/json' -H 'Accept: application/json' "-d@es_indexes_data/array_mapping.json"
+curl -X POST localhost:9200/_aliases -H 'Content-Type: application/json' -H 'Accept: application/json' "-d@es_indexes_data/configs_alias.json"

gen3-integration-tests/test_data/guppy_data/jenkinsSetup/jenkinsSetup.sh → gen3-integration-tests/test_data/test_setup/guppy_es_indexes/jenkins_setup.sh

@@ -18,17 +18,17 @@
 
 # if you're having problems - a little toy for testing
 if false; then
-  gen3 es create toy_debug_1 canineData/toyMapping.json
-  curl -H 'Content-Type: application/x-ndjson' "${ESHOST}/toy_debug_1/thing/_bulk" --data-binary @canineData/toyBatch.ndjson
+  gen3 es create toy_debug_1 es_indexes_data/toy_mapping.json
+  curl -H 'Content-Type: application/x-ndjson' "${ESHOST}/toy_debug_1/thing/_bulk" --data-binary @es_indexes_data/toy_batch.ndjson
 fi
 
-gen3 es create jenkins_subject_1 canineData/subjectMapping.json
+gen3 es create jenkins_subject_1 es_indexes_data/subject_mapping.json
 gen3 es alias jenkins_subject_1 jenkins_subject_alias
-curl -H 'Content-Type: application/x-ndjson' "${ESHOST}/jenkins_subject_1/subject/_bulk" --data-binary @canineData/subjectBatch.ndjson
+curl -H 'Content-Type: application/x-ndjson' "${ESHOST}/jenkins_subject_1/subject/_bulk" --data-binary @es_indexes_data/subject_batch.ndjson
 
-gen3 es create jenkins_file_1 canineData/fileMapping.json
+gen3 es create jenkins_file_1 es_indexes_data/file_mapping.json
 gen3 es alias jenkins_file_1 jenkins_file_alias
-curl -H 'Content-Type: application/x-ndjson' "${ESHOST}/jenkins_file_1/file/_bulk" --data-binary @canineData/fileBatch.ndjson
+curl -H 'Content-Type: application/x-ndjson' "${ESHOST}/jenkins_file_1/file/_bulk" --data-binary @es_indexes_data/file_batch.ndjson
 
-gen3 es create jenkins_configs_1 canineData/arrayMapping.json
+gen3 es create jenkins_configs_1 es_indexes_data/array_mapping.json
 gen3 es alias jenkins_configs_1 jenkins_configs_alias

gen3-integration-tests/tests/test_audit_service.py

@@ -22,11 +22,13 @@
 class TestAuditService:
     @classmethod
     def setup_class(cls):
-        assert update_audit_service_logging(pytest.namespace, "true")
+        assert update_audit_service_logging("true", test_env_namespace=pytest.namespace)
 
     @classmethod
     def teardown_class(cls):
-        assert update_audit_service_logging(pytest.namespace, "false")
+        assert update_audit_service_logging(
+            "false", test_env_namespace=pytest.namespace
+        )
 
     def test_audit_unauthorized_log_query(self):
         """

gen3-integration-tests/tests/test_client_credentials.py

@@ -64,4 +64,6 @@ def test_client_credentials(self):
             requestor.request_delete(request_id)
 
         logger.info("Revoking arborist policy for the user ...")
-        gat.revoke_arborist_policy(pytest.namespace, username, policy)
+        gat.revoke_arborist_policy(
+            username, policy, test_env_namespace=pytest.namespace
+        )

gen3-integration-tests/tests/test_dbgap.py

@@ -9,7 +9,6 @@
 from cdislogging import get_logger
 from services.indexd import Indexd
 from services.fence import Fence
-from utils.gen3_admin_tasks import create_link_google_test_buckets, run_gen3_command
 
 logger = get_logger(__name__, log_level=os.getenv("LOG_LEVEL", "info"))
 

gen3-integration-tests/tests/test_etl.py

@@ -12,7 +12,7 @@
 class TestETL:
     @classmethod
     def setup_class(cls):
-        gat.clean_up_indices(pytest.namespace)
+        gat.clean_up_indices(test_env_namespace=pytest.namespace)
         auth = Gen3Auth(refresh_token=pytest.api_keys["main_account"])
         sd_tools = GraphDataTools(
             auth=auth, program_name="jnkns", project_code="jenkins2"
@@ -22,7 +22,7 @@ def setup_class(cls):
 
     @classmethod
     def teardown_class(cls):
-        gat.clean_up_indices(pytest.namespace)
+        gat.clean_up_indices(test_env_namespace=pytest.namespace)
 
     def test_etl(self):
         """
@@ -40,4 +40,4 @@ def test_etl(self):
         logger.info("Running etl for the second time")
         gat.run_gen3_job("etl", test_env_namespace=pytest.namespace)
 
-        gat.check_indices_after_etl(pytest.namespace)
+        gat.check_indices_after_etl(test_env_namespace=pytest.namespace)

gen3-integration-tests/tests/test_graph_submit_and_query.py

@@ -104,7 +104,7 @@ def test_submit_record_unauthenticated(self):
         """
         # Generate an expired token
         res = create_access_token(
-            pytest.namespace, "fence", "1", "[email protected]"
+            "fence", "1", "[email protected]", test_env_namespace=pytest.namespace
         )
 
         res = res.splitlines()[-1].strip()

gen3-integration-tests/tests/test_guppy_service.py

@@ -1,6 +1,7 @@
 """
 GUPPY SERVICE
 """
+
 import os
 import pytest
 
@@ -25,12 +26,12 @@ def test_guppy_test_query_1(self):
         than 30 with a past stroke in ascending order of BMI.
 
         Steps:
-            1. Call API guppy/graphql using Query in testQuery1.json
+            1. Call API guppy/graphql using Query in test_query1.json
             2. Validate API response against data in tesResponse1.json
         """
         guppy = Guppy()
-        queryFile = "testQuery1.json"
-        responseFile = "testResponse1.json"
+        queryFile = "test_query1.json"
+        responseFile = "test_response1.json"
         queryType = "data"
         assert guppy.validate_guppy_query(
             queryFile, responseFile, queryType, "main_account", 200
@@ -43,12 +44,12 @@ def test_guppy_test_query_2(self):
         filter in the scenario above.
 
         Steps:
-            1. Call API guppy/graphql using Query in testQuery2.json
+            1. Call API guppy/graphql using Query in test_query2.json
             2. Validate API response against data in tesResponse2.json
         """
         guppy = Guppy()
-        queryFile = "testQuery2.json"
-        responseFile = "testResponse2.json"
+        queryFile = "test_query2.json"
+        responseFile = "test_response2.json"
         queryType = "aggregation"
         assert guppy.validate_guppy_query(
             queryFile, responseFile, queryType, "main_account", 200
@@ -62,12 +63,12 @@ def test_guppy_test_query_3(self):
         occurrence and age groups represented.
 
         Steps:
-            1. Call API guppy/graphql using Query in testQuery3.json
+            1. Call API guppy/graphql using Query in test_query3.json
             2. Validate API response against data in tesResponse3.json
         """
         guppy = Guppy()
-        queryFile = "testQuery3.json"
-        responseFile = "testResponse3.json"
+        queryFile = "test_query3.json"
+        responseFile = "test_response3.json"
         queryType = "histogram"
         assert guppy.validate_guppy_query(
             queryFile, responseFile, queryType, "main_account", 200
@@ -79,12 +80,12 @@ def test_guppy_test_query_4(self):
         Range-stepped database check of age groups.
 
         Steps:
-            1. Call API guppy/graphql using Query in testQuery4.json
+            1. Call API guppy/graphql using Query in test_query4.json
             2. Validate API response against data in tesResponse4.json
         """
         guppy = Guppy()
-        queryFile = "testQuery4.json"
-        responseFile = "testResponse4.json"
+        queryFile = "test_query4.json"
+        responseFile = "test_response4.json"
         queryType = "histogram"
         assert guppy.validate_guppy_query(
             queryFile, responseFile, queryType, "main_account", 200
@@ -96,12 +97,12 @@ def test_guppy_test_query_5(self):
         I would like to list the fields on the subject document.
 
         Steps:
-            1. Call API guppy/graphql using Query in testQuery5.json
+            1. Call API guppy/graphql using Query in test_query5.json
             2. Validate API response against data in tesResponse5.json
         """
         guppy = Guppy()
-        queryFile = "testQuery5.json"
-        responseFile = "testResponse5.json"
+        queryFile = "test_query5.json"
+        responseFile = "test_response5.json"
         queryType = "mapping"
         assert guppy.validate_guppy_query(
             queryFile, responseFile, queryType, "main_account", 200
@@ -114,12 +115,12 @@ def test_guppy_test_query_6(self):
         summarizing data in the commons.
 
         Steps:
-            1. Call API guppy/graphql using Query in testQuery6.json
+            1. Call API guppy/graphql using Query in test_query6.json
             2. Validate API response against data in tesResponse6.json
         """
         guppy = Guppy()
-        queryFile = "testQuery6.json"
-        responseFile = "testResponse6.json"
+        queryFile = "test_query6.json"
+        responseFile = "test_response6.json"
         queryType = "histogram"
         assert guppy.validate_guppy_query(
             queryFile, responseFile, queryType, "main_account", 200
@@ -132,12 +133,12 @@ def test_guppy_test_query_7(self):
         parameter to gain an understanding of its distribution.
 
         Steps:
-            1. Call API guppy/graphql using Query in testQuery7.json
+            1. Call API guppy/graphql using Query in test_query7.json
             2. Validate API response against data in tesResponse7.json
         """
         guppy = Guppy()
-        queryFile = "testQuery7.json"
-        responseFile = "testResponse7.json"
+        queryFile = "test_query7.json"
+        responseFile = "test_response7.json"
         queryType = "histogram"
         assert guppy.validate_guppy_query(
             queryFile, responseFile, queryType, "main_account", 200
@@ -149,12 +150,12 @@ def test_guppy_test_query_8(self):
         paginating the results, or whether the result will be > 10k records.
 
         Steps:
-            1. Call API guppy/download using Query in testQuery8.json
+            1. Call API guppy/download using Query in test_query8.json
             2. Validate API response against data in tesResponse8.json
         """
         guppy = Guppy()
-        queryFile = "testQuery8.json"
-        responseFile = "testResponse8.json"
+        queryFile = "test_query8.json"
+        responseFile = "test_response8.json"
         queryType = "download"
         assert guppy.validate_guppy_query(
             queryFile,