From 6bc38d033795d96654fc456c4335b5613ce1b57f Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 15 Nov 2024 03:02:09 -0600 Subject: [PATCH 1/4] Fixes issue #1087 --- .envs/.local/.django | 12 ++++++ config/server_configs.py | 64 ++++++++++++++++++++++++++++ config_generation/api.py | 17 +------- config_generation/minimum_api.py | 15 +------ functional_tests/check_collection.py | 6 +-- scripts/ej/cmr_to_models.py | 7 +-- sde_collections/models/collection.py | 6 +-- sde_collections/sinequa_api.py | 52 +--------------------- 8 files changed, 89 insertions(+), 90 deletions(-) create mode 100644 config/server_configs.py diff --git a/.envs/.local/.django b/.envs/.local/.django index 0978166d..58304f28 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -47,3 +47,15 @@ LRM_QA_PASSWORD='' #-------------------------------------------------------------------------------- LRM_DEV_TOKEN='' XLI_TOKEN='' +TEST_SERVER_TOKEN='' + +#Server Base_URLs +#-------------------------------------------------------------------------------- +BASE_URL_DEV=http://sde-renaissance.nasa-impact.net +BASE_URL_TEST=https://sciencediscoveryengine.test.nasa.gov +BASE_URL_PROD=https://sciencediscoveryengine.nasa.gov +BASE_URL_XLI=http://sde-xli.nasa-impact.net +BASE_URL_LRM_DEV=https://sde-lrm.nasa-impact.net +BASE_URL_LRM_QA=https://sde-qa.nasa-impact.net +BASE_URL_TEST_SERVER=http://10.51.14.135 +BASE_URL_REN=http://sde-renaissance.nasa-impact.net diff --git a/config/server_configs.py b/config/server_configs.py new file mode 100644 index 00000000..42d0e82b --- /dev/null +++ b/config/server_configs.py @@ -0,0 +1,64 @@ +import os + +server_configs = { + "dev": { + "app_name": "nasa-sba-smd", + "query_name": "query-smd-primary", + "base_url": os.getenv('BASE_URL_DEV'), + "index": "sde_index", + }, + "test": { + "app_name": "nasa-sba-smd", + "query_name": "query-smd-primary", + "base_url": os.getenv('BASE_URL_TEST'), + "index": "sde_index", + }, + "production": { + "app_name": "nasa-sba-smd", + "query_name": "query-smd-primary", + "base_url": os.getenv('BASE_URL_PROD'), + "index": "sde_index", + }, + "secret_test": { + "app_name": "nasa-sba-sde", + "query_name": "query-sde-primary", + "base_url": os.getenv('BASE_URL_TEST'), + "index": "sde_index", + }, + "secret_production": { + "app_name": "nasa-sba-sde", + "query_name": "query-sde-primary", + "base_url": os.getenv('BASE_URL_PROD'), + "index": "sde_index", + }, + "xli": { + "app_name": "nasa-sba-smd", + "query_name": "query-smd-primary", + "base_url": os.getenv('BASE_URL_XLI'), + "index": "sde_index", + }, + "lrm_dev": { + "app_name": "sde-init-check", + "query_name": "query-init-check", + "base_url": os.getenv('BASE_URL_LRM_DEV'), + "index": "sde_init_check", + }, + "lrm_qa": { + "app_name": "sde-init-check", + "query_name": "query-init-check", + "base_url": os.getenv('BASE_URL_LRM_QA'), + "index": "sde_init_check", + }, + "ren_server": { + "app_name": "nasa-sba-smd", + "query_name": "query-smd-primary", + "base_url": os.getenv('BASE_URL_REN'), + "index": "sde_index", + }, + "test_server": { + "app_name": "nasa-sba-smd", + "query_name": "query-smd-primary", + "base_url": os.getenv('BASE_URL_TEST_SERVER'), + "index": "sde_index", + }, +} diff --git a/config_generation/api.py b/config_generation/api.py index 7ae4d99b..6a0ec48c 100644 --- a/config_generation/api.py +++ b/config_generation/api.py @@ -1,23 +1,8 @@ from typing import Any - +from config.server_configs import server_configs import requests - from config import tokens -server_configs: dict[str, dict[str, str]] = { - "ren_server": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "http://sde-renaissance.nasa-impact.net", - }, - "test_server": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "http://10.51.14.135", - }, -} - - class Api: def __init__(self, server_name: str) -> None: self.headers: dict[str, str] = {"Authorization": f"Bearer {tokens[server_name]}"} diff --git a/config_generation/minimum_api.py b/config_generation/minimum_api.py index ec3cff1e..85180657 100644 --- a/config_generation/minimum_api.py +++ b/config_generation/minimum_api.py @@ -1,20 +1,7 @@ from typing import Any - +from config.server_configs import server_configs import requests -server_configs: dict[str, dict[str, str]] = { - "test_server": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "https://sciencediscoveryengine.test.nasa.gov", - }, - "production_server": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "https://sciencediscoveryengine.nasa.gov", - }, -} - class Api: def __init__(self, server_name: str = "test", token: str = None) -> None: diff --git a/functional_tests/check_collection.py b/functional_tests/check_collection.py index 3d227e47..86ff6c02 100644 --- a/functional_tests/check_collection.py +++ b/functional_tests/check_collection.py @@ -2,7 +2,7 @@ import sys import time import urllib.parse - +import os from selenium import webdriver from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager @@ -16,8 +16,8 @@ def server_url(config_folder, server="test", secret=True) -> str: URLS = { - "test": "https://sciencediscoveryengine.test.nasa.gov", - "prod": "https://sciencediscoveryengine.nasa.gov", + "test": os.getenv('BASE_URL_TEST'), + "prod": os.getenv('BASE_URL_PROD'), } if secret: query = "query-sde-primary" diff --git a/scripts/ej/cmr_to_models.py b/scripts/ej/cmr_to_models.py index 130de722..4fa5e478 100644 --- a/scripts/ej/cmr_to_models.py +++ b/scripts/ej/cmr_to_models.py @@ -5,7 +5,8 @@ import json import urllib.parse - +from config.server_configs import server_configs +import os from environmental_justice.models import EnvironmentalJusticeRow @@ -22,7 +23,8 @@ def concept_id_to_sinequa_id(concept_id: str) -> str: def sinequa_id_to_url(sinequa_id: str) -> str: - base_url = "https://sciencediscoveryengine.nasa.gov/app/nasa-sba-smd/#/preview" + base_domain = os.getenv('BASE_URL_PROD') + base_url = f"{base_domain}/app/nasa-sba-smd/#/preview" query = '{"name":"query-smd-primary","scope":"All","text":""}' encoded_id = urllib.parse.quote(sinequa_id, safe="") @@ -30,7 +32,6 @@ def sinequa_id_to_url(sinequa_id: str) -> str: return f"{base_url}?id={encoded_id}&query={encoded_query}" - def categorize_processing_level(level): advanced_analysis_levels = {"0", "Level 0", "NA", "Not Provided", "Not provided"} diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index 31306b8c..a31d51a6 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -1,6 +1,6 @@ import json import urllib.parse - +import os import requests from django.contrib.auth import get_user_model from django.db import models @@ -129,7 +129,7 @@ def tree_root(self) -> str: @property def server_url_secret_prod(self) -> str: - base_url = "https://sciencediscoveryengine.nasa.gov" # noqa: E231 + base_url = os.getenv('BASE_URL_SECRET_TEST') # noqa: E231 payload = { "name": "secret-prod", "scope": "All", @@ -143,7 +143,7 @@ def server_url_secret_prod(self) -> str: @property def server_url_prod(self) -> str: - base_url = "https://sciencediscoveryengine.nasa.gov" # noqa: E231 + base_url = os.getenv('BASE_URL_PROD') # noqa: E231 payload = { "name": "query-smd-primary", "scope": "All", diff --git a/sde_collections/sinequa_api.py b/sde_collections/sinequa_api.py index b28f9784..3c399be1 100644 --- a/sde_collections/sinequa_api.py +++ b/sde_collections/sinequa_api.py @@ -1,62 +1,12 @@ import json from typing import Any - +from config.server_configs import server_configs import requests import urllib3 from django.conf import settings urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -server_configs = { - "dev": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "http://sde-renaissance.nasa-impact.net", - }, - "test": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "https://sciencediscoveryengine.test.nasa.gov", - "index": "sde_index", - }, - "production": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "https://sciencediscoveryengine.nasa.gov", - "index": "sde_index", - }, - "secret_test": { - "app_name": "nasa-sba-sde", - "query_name": "query-sde-primary", - "base_url": "https://sciencediscoveryengine.test.nasa.gov", - "index": "sde_index", - }, - "secret_production": { - "app_name": "nasa-sba-sde", - "query_name": "query-sde-primary", - "base_url": "https://sciencediscoveryengine.nasa.gov", - "index": "sde_index", - }, - "xli": { - "app_name": "nasa-sba-smd", - "query_name": "query-smd-primary", - "base_url": "http://sde-xli.nasa-impact.net", - "index": "sde_index", - }, - "lrm_dev": { - "app_name": "sde-init-check", - "query_name": "query-init-check", - "base_url": "https://sde-lrm.nasa-impact.net", - "index": "sde_init_check", - }, - "lrm_qa": { - "app_name": "sde-init-check", - "query_name": "query-init-check", - "base_url": "https://sde-qa.nasa-impact.net", - }, -} - - class Api: def __init__(self, server_name: str = None, user: str = None, password: str = None, token: str = None) -> None: self.server_name = server_name From 48bf45bae27f9a7700f361ded2d35887495fe60e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:06:19 +0000 Subject: [PATCH 2/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- config/server_configs.py | 22 +++++++++++----------- config_generation/api.py | 5 ++++- config_generation/minimum_api.py | 4 +++- functional_tests/check_collection.py | 7 ++++--- scripts/ej/cmr_to_models.py | 6 ++++-- sde_collections/models/collection.py | 7 ++++--- sde_collections/sinequa_api.py | 5 ++++- 7 files changed, 34 insertions(+), 22 deletions(-) diff --git a/config/server_configs.py b/config/server_configs.py index 42d0e82b..646b125d 100644 --- a/config/server_configs.py +++ b/config/server_configs.py @@ -4,61 +4,61 @@ "dev": { "app_name": "nasa-sba-smd", "query_name": "query-smd-primary", - "base_url": os.getenv('BASE_URL_DEV'), + "base_url": os.getenv("BASE_URL_DEV"), "index": "sde_index", }, "test": { "app_name": "nasa-sba-smd", "query_name": "query-smd-primary", - "base_url": os.getenv('BASE_URL_TEST'), + "base_url": os.getenv("BASE_URL_TEST"), "index": "sde_index", }, "production": { "app_name": "nasa-sba-smd", "query_name": "query-smd-primary", - "base_url": os.getenv('BASE_URL_PROD'), + "base_url": os.getenv("BASE_URL_PROD"), "index": "sde_index", }, "secret_test": { "app_name": "nasa-sba-sde", "query_name": "query-sde-primary", - "base_url": os.getenv('BASE_URL_TEST'), + "base_url": os.getenv("BASE_URL_TEST"), "index": "sde_index", }, "secret_production": { "app_name": "nasa-sba-sde", "query_name": "query-sde-primary", - "base_url": os.getenv('BASE_URL_PROD'), + "base_url": os.getenv("BASE_URL_PROD"), "index": "sde_index", }, "xli": { "app_name": "nasa-sba-smd", "query_name": "query-smd-primary", - "base_url": os.getenv('BASE_URL_XLI'), + "base_url": os.getenv("BASE_URL_XLI"), "index": "sde_index", }, "lrm_dev": { "app_name": "sde-init-check", "query_name": "query-init-check", - "base_url": os.getenv('BASE_URL_LRM_DEV'), + "base_url": os.getenv("BASE_URL_LRM_DEV"), "index": "sde_init_check", }, "lrm_qa": { "app_name": "sde-init-check", "query_name": "query-init-check", - "base_url": os.getenv('BASE_URL_LRM_QA'), + "base_url": os.getenv("BASE_URL_LRM_QA"), "index": "sde_init_check", }, "ren_server": { "app_name": "nasa-sba-smd", "query_name": "query-smd-primary", - "base_url": os.getenv('BASE_URL_REN'), + "base_url": os.getenv("BASE_URL_REN"), "index": "sde_index", }, - "test_server": { + "test_server": { "app_name": "nasa-sba-smd", "query_name": "query-smd-primary", - "base_url": os.getenv('BASE_URL_TEST_SERVER'), + "base_url": os.getenv("BASE_URL_TEST_SERVER"), "index": "sde_index", }, } diff --git a/config_generation/api.py b/config_generation/api.py index 6a0ec48c..02705ab3 100644 --- a/config_generation/api.py +++ b/config_generation/api.py @@ -1,7 +1,10 @@ from typing import Any -from config.server_configs import server_configs + import requests + from config import tokens +from config.server_configs import server_configs + class Api: def __init__(self, server_name: str) -> None: diff --git a/config_generation/minimum_api.py b/config_generation/minimum_api.py index 85180657..f78e821f 100644 --- a/config_generation/minimum_api.py +++ b/config_generation/minimum_api.py @@ -1,7 +1,9 @@ from typing import Any -from config.server_configs import server_configs + import requests +from config.server_configs import server_configs + class Api: def __init__(self, server_name: str = "test", token: str = None) -> None: diff --git a/functional_tests/check_collection.py b/functional_tests/check_collection.py index 86ff6c02..dec57946 100644 --- a/functional_tests/check_collection.py +++ b/functional_tests/check_collection.py @@ -1,8 +1,9 @@ import json +import os import sys import time import urllib.parse -import os + from selenium import webdriver from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager @@ -16,8 +17,8 @@ def server_url(config_folder, server="test", secret=True) -> str: URLS = { - "test": os.getenv('BASE_URL_TEST'), - "prod": os.getenv('BASE_URL_PROD'), + "test": os.getenv("BASE_URL_TEST"), + "prod": os.getenv("BASE_URL_PROD"), } if secret: query = "query-sde-primary" diff --git a/scripts/ej/cmr_to_models.py b/scripts/ej/cmr_to_models.py index 4fa5e478..c1f5cf17 100644 --- a/scripts/ej/cmr_to_models.py +++ b/scripts/ej/cmr_to_models.py @@ -4,9 +4,10 @@ """ import json +import os import urllib.parse + from config.server_configs import server_configs -import os from environmental_justice.models import EnvironmentalJusticeRow @@ -23,7 +24,7 @@ def concept_id_to_sinequa_id(concept_id: str) -> str: def sinequa_id_to_url(sinequa_id: str) -> str: - base_domain = os.getenv('BASE_URL_PROD') + base_domain = os.getenv("BASE_URL_PROD") base_url = f"{base_domain}/app/nasa-sba-smd/#/preview" query = '{"name":"query-smd-primary","scope":"All","text":""}' @@ -32,6 +33,7 @@ def sinequa_id_to_url(sinequa_id: str) -> str: return f"{base_url}?id={encoded_id}&query={encoded_query}" + def categorize_processing_level(level): advanced_analysis_levels = {"0", "Level 0", "NA", "Not Provided", "Not provided"} diff --git a/sde_collections/models/collection.py b/sde_collections/models/collection.py index a31d51a6..8538530c 100644 --- a/sde_collections/models/collection.py +++ b/sde_collections/models/collection.py @@ -1,6 +1,7 @@ import json -import urllib.parse import os +import urllib.parse + import requests from django.contrib.auth import get_user_model from django.db import models @@ -129,7 +130,7 @@ def tree_root(self) -> str: @property def server_url_secret_prod(self) -> str: - base_url = os.getenv('BASE_URL_SECRET_TEST') # noqa: E231 + base_url = os.getenv("BASE_URL_SECRET_TEST") # noqa: E231 payload = { "name": "secret-prod", "scope": "All", @@ -143,7 +144,7 @@ def server_url_secret_prod(self) -> str: @property def server_url_prod(self) -> str: - base_url = os.getenv('BASE_URL_PROD') # noqa: E231 + base_url = os.getenv("BASE_URL_PROD") # noqa: E231 payload = { "name": "query-smd-primary", "scope": "All", diff --git a/sde_collections/sinequa_api.py b/sde_collections/sinequa_api.py index 3c399be1..acfb9c9b 100644 --- a/sde_collections/sinequa_api.py +++ b/sde_collections/sinequa_api.py @@ -1,12 +1,15 @@ import json from typing import Any -from config.server_configs import server_configs + import requests import urllib3 from django.conf import settings +from config.server_configs import server_configs + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + class Api: def __init__(self, server_name: str = None, user: str = None, password: str = None, token: str = None) -> None: self.server_name = server_name From 7363e14c92e2b924957da9dee66603f165a899e4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 15 Nov 2024 03:24:09 -0600 Subject: [PATCH 3/4] Fixes Issue #1087 --- scripts/ej/cmr_to_models.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/ej/cmr_to_models.py b/scripts/ej/cmr_to_models.py index c1f5cf17..14a41806 100644 --- a/scripts/ej/cmr_to_models.py +++ b/scripts/ej/cmr_to_models.py @@ -7,7 +7,6 @@ import os import urllib.parse -from config.server_configs import server_configs from environmental_justice.models import EnvironmentalJusticeRow From e235456599795812ef16a6646ebd6869f848ca61 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 15 Nov 2024 10:25:02 -0600 Subject: [PATCH 4/4] FixesIssue #1087 --- .envs/.local/.django | 1 - 1 file changed, 1 deletion(-) diff --git a/.envs/.local/.django b/.envs/.local/.django index 58304f28..f8a5059c 100644 --- a/.envs/.local/.django +++ b/.envs/.local/.django @@ -47,7 +47,6 @@ LRM_QA_PASSWORD='' #-------------------------------------------------------------------------------- LRM_DEV_TOKEN='' XLI_TOKEN='' -TEST_SERVER_TOKEN='' #Server Base_URLs #--------------------------------------------------------------------------------