feat: dotenv support (#143)

* feat: dotenv support

* feat: Load dotenv file in offload and connect

* feat: Load dotenv file in agg_validate and programmatic calls

---------

Co-authored-by: nj1973 <[email protected]>

Makefile

@@ -17,7 +17,7 @@ SHELL := /bin/bash
 TARGET_DIR=target/offload
 
 OFFLOAD_VERSION=$(shell cat version)
-GOE_WHEEL="goe-$(shell echo $(OFFLOAD_VERSION) | tr 'A-Z-' 'a-z.')0-py3-none-any.whl"
+GOE_WHEEL=goe-$(shell echo $(OFFLOAD_VERSION) | tr 'A-Z-' 'a-z.')0-py3-none-any.whl
 
 BUILD=$(strip $(shell git rev-parse --short HEAD))
 

README.md

@@ -62,9 +62,8 @@ If using Dataproc Batches to provide Spark:
 ## Install database objects
 To install supporting database objects you need access to a database admin account that can create users, grant them system privileges and create objects in the schemas created. SYSTEM has been used in the example below but this is *not* a necessity:
 ```
-. ${OFFLOAD_HOME}/conf/offload.env
 cd ${OFFLOAD_HOME}/setup
-sqlplus system@${ORA_CONN}
+sqlplus system@yourhost:1521/yoursvc
 @install_offload
 ```
 
@@ -119,9 +118,8 @@ make install-dev-extras
 ```
 
 # Running commands
-Source the correct environment:
+Activate the GOE Python virtual environment:
 ```
-. ${OFFLOAD_HOME}/conf/offload.env
 source ./.venv/bin/activate
 ```
 

bin/connect

@@ -15,15 +15,10 @@
 # limitations under the License.
 
 
-import os
-import sys
-
-if not os.environ.get("OFFLOAD_HOME"):
-    print("OFFLOAD_HOME environment variable missing")
-    print("You should source environment variables first, eg: . ../conf/offload.env")
-    sys.exit(1)
-
+from goe.config.config_file import check_config_path
 from goe.connect.connect import connect
 
+
 if __name__ == "__main__":
+    check_config_path()
     connect()

bin/offload

@@ -14,10 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 import sys
 
-from goe.config.config_checks import check_cli_path
+from goe.config import config_file
 from goe.goe import (
     get_options,
     OFFLOAD_OP_NAME,
@@ -26,12 +25,11 @@ from goe.offload.offload import OffloadOptionError, get_offload_options
 from goe.orchestration.cli_entry_points import offload_by_cli
 
 
-check_cli_path()
-
-
 if __name__ == "__main__":
+    config_file.check_config_path()
     options = None
     try:
+        config_file.load_env()
         opt = get_options(operation_name=OFFLOAD_OP_NAME)
         get_offload_options(opt)
         options, _ = opt.parse_args()

pyproject.toml

@@ -59,6 +59,9 @@ dependencies = [
     "google-cloud-bigquery",
     "google-cloud-kms",
 
+    # Env loading
+    "python-dotenv",
+
     # GOE Listener packages
     "fastapi==0.77.0",
     "uvicorn==0.17.6",

src/goe/config/config_file.py

@@ -0,0 +1,53 @@
+# Copyright 2024 The GOE Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import sys
+from typing import Optional
+
+from dotenv import load_dotenv
+
+
+CONFIG_FILE_NAME = "offload.env"
+KEY_VALUE_PATTERN = re.compile(r"#?[ ]*([A-Z_0-9]+)=(.*)")
+
+
+def check_config_path():
+    """Check OFFLOAD_HOME in top level command wrappers"""
+    if not os.environ.get("OFFLOAD_HOME"):
+        print("OFFLOAD_HOME environment variable missing")
+        sys.exit(1)
+
+
+def get_environment_file_path():
+    return os.path.join(os.environ.get("OFFLOAD_HOME"), "conf", CONFIG_FILE_NAME)
+
+
+def load_env(path: str = None):
+    """Load GOE environment from a configuration file.
+
+    By default this is a fixed location: $OFFLOAD_HOME/conf/offload.env.
+    In time this will become a parameter and support cloud storage locations.
+    """
+    if not path:
+        path = get_environment_file_path()
+
+    load_dotenv(path)
+
+
+def env_key_value_pair(line_from_file: str) -> Optional[tuple]:
+    """Used by connect to get the key names from a configuration file"""
+    m = KEY_VALUE_PATTERN.match(line_from_file)
+    return m.groups() if m else None

src/goe/config/orchestration_config.py

@@ -22,7 +22,7 @@
 import logging
 from typing import Optional
 
-from goe.config import orchestration_defaults
+from goe.config import config_file, orchestration_defaults
 from goe.config.config_validation_functions import (
     OrchestrationConfigException,
     check_offload_fs_scheme_supported_in_backend,
@@ -349,12 +349,15 @@ def as_defaults(do_not_connect=False):
         return OrchestrationConfig.from_dict({}, do_not_connect=do_not_connect)
 
     @staticmethod
-    def from_dict(config_dict, do_not_connect=False):
+    def from_dict(config_dict: dict, do_not_connect=False):
         assert isinstance(config_dict, dict)
         unexpected_keys = [k for k in config_dict if k not in EXPECTED_CONFIG_ARGS]
         assert not unexpected_keys, (
             "Unexpected OrchestrationConfig keys: %s" % unexpected_keys
         )
+        # Load environment for defaults.
+        config_file.load_env()
+        # Build config from config_dict.
         return OrchestrationConfig(
             do_not_connect=do_not_connect,
             ansi=config_dict.get("ansi", orchestration_defaults.ansi_default()),

src/goe/connect/connect.py

@@ -15,28 +15,15 @@
 from datetime import datetime
 from optparse import SUPPRESS_HELP
 import os
-import re
 import subprocess
 import sys
 import traceback
 from typing import Optional
 
 from getpass import getuser
-from goe.goe import (
-    get_common_options,
-    get_log_fh,
-    get_log_fh_name,
-    init,
-    init_log,
-    log_command_line,
-    log_timestamp,
-    version,
-    OptionValueError,
-    verbose,
-    CONFIG_FILE_NAME,
-)
+
 from goe.config.orchestration_config import OrchestrationConfig
-from goe.config import orchestration_defaults
+from goe.config import config_file, orchestration_defaults
 from goe.connect.connect_backend import (
     is_hadoop_environment,
     run_backend_tests,
@@ -63,6 +50,18 @@
     test_credential_api_alias,
 )
 from goe.filesystem.goe_dfs_factory import get_dfs_from_options
+from goe.goe import (
+    get_common_options,
+    get_log_fh,
+    get_log_fh_name,
+    init,
+    init_log,
+    log_command_line,
+    log_timestamp,
+    version,
+    OptionValueError,
+    verbose,
+)
 from goe.offload.offload_messages import OffloadMessages
 from goe.offload.offload_transport_functions import ssh_cmd_prefix
 from goe.orchestration import orchestration_constants
@@ -206,11 +205,9 @@ def get_environment_file_name(orchestration_config):
         backend_id = "hadoop"
     else:
         backend_id = orchestration_config.target.lower()
-    return "-".join([frontend_id, backend_id, CONFIG_FILE_NAME + ".template"])
-
-
-def get_environment_file_path():
-    return os.path.join(os.environ.get("OFFLOAD_HOME"), "conf", CONFIG_FILE_NAME)
+    return "-".join(
+        [frontend_id, backend_id, config_file.CONFIG_FILE_NAME + ".template"]
+    )
 
 
 def get_template_file_path(orchestration_config):
@@ -222,7 +219,7 @@ def test_conf_perms():
     test_name = "Configuration file permissions"
     test_header(test_name)
     hint = "Expected permissions are 640"
-    environment_file = get_environment_file_path()
+    environment_file = config_file.get_environment_file_path()
     perms = oct(os.stat(environment_file).st_mode & 0o777)
     # Removing oct prefix deemed safe for display only.
     detail("%s has permissions: %s" % (environment_file, perms[2:]))
@@ -310,8 +307,9 @@ def dict_from_environment_file(environment_file):
     d = {}
     with open(environment_file) as f:
         for line in f:
-            if re.match("^(#.*e|e)xport(.*)", line):
-                (k, v) = re.sub("^(#.*e|e)xport ", "", line).split("=", 1)[0], line
+            kv = config_file.env_key_value_pair(line)
+            if kv:
+                k, v = kv
                 d[k] = v
     return d
 
@@ -396,7 +394,8 @@ def check_environment(options, orchestration_config):
     section_header("Configuration")
 
     check_offload_env(
-        get_environment_file_path(), get_template_file_path(orchestration_config)
+        config_file.get_environment_file_path(),
+        get_template_file_path(orchestration_config),
     )
     test_conf_perms()
 
@@ -515,6 +514,7 @@ def get_connect_opts():
 def connect():
     options = None
     try:
+        config_file.load_env()
         opt = get_connect_opts()
         options, args = opt.parse_args()
 
@@ -540,7 +540,7 @@ def connect():
 
         if options.upgrade_environment_file:
             upgrade_environment_file(
-                get_environment_file_path(),
+                config_file.get_environment_file_path(),
                 get_template_file_path(orchestration_config),
             )
         else:

src/goe/goe.py

@@ -178,8 +178,6 @@
 
 OFFLOAD_OP_NAME = "offload"
 
-CONFIG_FILE_NAME = "offload.env"
-
 # Used in test to identify specific warnings
 HYBRID_SCHEMA_STEPS_DUE_TO_HWM_CHANGE_MESSAGE_TEXT = (
     "Including post transport steps due to HWM change"
@@ -3016,8 +3014,6 @@ class GOEOptionTypes(Option):
 def get_common_options(usage=None):
     opt = OptionParser(usage=usage, option_class=GOEOptionTypes)
 
-    opt.add_option("-c", type="posint", help=SUPPRESS_HELP)
-
     opt.add_option(
         "--version",
         dest="version",

src/goe/listener/prestart.py

@@ -21,9 +21,9 @@ def validate_config():
     """"""
     try:
         # GOE
-        from goe.config.config_checks import check_cli_path  # noqa: WPS433 F401
+        from goe.config.config_file import check_config_path  # noqa: WPS433 F401
 
-        check_cli_path()
+        check_config_path()
     except Exception:
         print(
             "failed to validate configuration.  Please check your installation"

src/goe/scripts/agg_validate.py

@@ -24,7 +24,7 @@
 import re
 import sys
 
-from goe.config import config_descriptions, orchestration_defaults
+from goe.config import config_descriptions, config_file, orchestration_defaults
 from goe.config.orchestration_config import OrchestrationConfig
 from goe.offload.offload_validation import (
     CrossDbValidator,
@@ -39,7 +39,6 @@
     is_number,
     is_pos_int,
     parse_python_from_string,
-    check_offload_env,
 )
 from goe.util.goe_log import log_exception
 
@@ -256,7 +255,8 @@ def main():
     MAIN ROUTINE
     """
 
-    check_offload_env()
+    config_file.check_config_path()
+    config_file.load_env()
 
     args = parse_args()
     init(args)

src/goe/util/misc_functions.py

@@ -334,18 +334,6 @@ def get_option(options, name, repl=None):
     return getattr(options, name) if (options and hasattr(options, name)) else repl
 
 
-def check_offload_env():
-    """Check offload environment,
-    i.e. OFFLOAD_HOME is set correctly
-    """
-    if not os.environ.get("OFFLOAD_HOME"):
-        print("OFFLOAD_HOME environment variable missing")
-        print(
-            "You should source environment variables first, eg: . ../conf/offload.env"
-        )
-        sys.exit(1)
-
-
 def str_floatlike(maybe_float):
     """Remove unnecessary 0s from the float or 'float like string'
     Warning: Relies on str() conversion for floats, which truncates floats