chore: Add workflow

.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: ci
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Configure Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.8"
+
+      - name: Building Collection scripts
+        run: pip install -e .
+
+  test:
+    name: "test (${{ matrix.python-version }})"
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.8"]
+        # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+    uses: ./.github/workflows/test.yml
+    with:
+      python-version: ${{ matrix.python-version }}

.github/workflows/pr-title.yml

@@ -0,0 +1,20 @@
+name: Lint PR Title
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+permissions:
+  pull-requests: read
+
+jobs:
+  pr_lint:
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,45 @@
+name: Unit Tests
+
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        required: true
+        type: string
+      coverage:
+        required: false
+        type: boolean
+        default: false
+      os:
+        required: false
+        type: string
+        default: "ubuntu-latest"
+      timeout:
+        required: false
+        type: number
+        default: 5
+
+jobs:
+  test:
+    runs-on: ${{ inputs.os }}
+    timeout-minutes: ${{ inputs.timeout }}
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up python ${{ inputs.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+
+      - name: Install base libraries
+        run: pip install --quiet --upgrade pip build setuptools wheel
+
+      - name: Install GOE for testing
+        run: pip install -e .[test]
+
+      - name: Test
+        run: pytest tests/unit

src/goe/filesystem/cli_hdfs.py

@@ -85,15 +85,16 @@ def __init__(
     ):
         logger.info("Client setup: (%s, %s, %s)" % (hdfs_host, ssh_user, tmp_dir))
 
-        super(CliHdfs, self).__init__(messages, dry_run=dry_run)
+        super(CliHdfs, self).__init__(
+            messages, dry_run=dry_run, do_not_connect=do_not_connect
+        )
 
         self._db_path_suffix = db_path_suffix
         self._hdfs_data = hdfs_data
         self._hdfs_host = hdfs_host
         self._ssh_user = ssh_user
         self._tmp_dir = tmp_dir
         self._dry_run = dry_run
-        self._do_not_connect = do_not_connect
         self.dfs_mechanism = GOE_DFS_SSH
         self.backend_dfs = "HDFS"
 

src/goe/filesystem/goe_azure.py

@@ -118,7 +118,9 @@ def __init__(
 
         logger.info("Client setup: GOEAzure")
 
-        super(GOEAzure, self).__init__(messages, dry_run=dry_run)
+        super(GOEAzure, self).__init__(
+            messages, dry_run=dry_run, do_not_connect=do_not_connect
+        )
 
         if azure_account_domain and not azure_account_domain.startswith("."):
             azure_account_domain = "." + azure_account_domain

src/goe/filesystem/goe_dfs.py

@@ -236,9 +236,10 @@ def gen_load_uri_from_options(
 
 
 class GOEDfs(object, metaclass=ABCMeta):
-    def __init__(self, messages, dry_run=False):
+    def __init__(self, messages, dry_run=False, do_not_connect=False):
         self._messages = messages
         self._dry_run = dry_run
+        self._do_not_connect = do_not_connect
         if dry_run:
             logger.info("* Dry run *")
 

src/goe/filesystem/goe_dfs_factory.py

@@ -23,7 +23,9 @@
 from goe.offload.offload_constants import HADOOP_BASED_BACKEND_DISTRIBUTIONS
 
 
-def get_dfs_from_options(offload_options, messages=None, force_ssh=False, dry_run=None):
+def get_dfs_from_options(
+    offload_options, messages=None, force_ssh=False, dry_run=None, do_not_connect=False
+):
     """Helper function to get an appropriate GOEDfs object based on offload options."""
     if dry_run is None:
         dry_run = bool(not offload_options.execute)
@@ -44,6 +46,7 @@ def get_dfs_from_options(offload_options, messages=None, force_ssh=False, dry_ru
                 offload_options.webhdfs_verify_ssl,
                 dry_run=dry_run,
                 messages=messages,
+                do_not_connect=do_not_connect,
                 db_path_suffix=offload_options.hdfs_db_path_suffix,
                 hdfs_data=offload_options.hdfs_data,
             )
@@ -55,6 +58,7 @@ def get_dfs_from_options(offload_options, messages=None, force_ssh=False, dry_ru
                 offload_options.hadoop_ssh_user,
                 dry_run=dry_run,
                 messages=messages,
+                do_not_connect=do_not_connect,
                 db_path_suffix=offload_options.hdfs_db_path_suffix,
                 hdfs_data=offload_options.hdfs_data,
             )
@@ -64,6 +68,7 @@ def get_dfs_from_options(offload_options, messages=None, force_ssh=False, dry_ru
         return GOEGcs(
             messages,
             dry_run=dry_run,
+            do_not_connect=do_not_connect,
             db_path_suffix=offload_options.hdfs_db_path_suffix,
         )
     elif offload_options.offload_fs_scheme in (
@@ -75,6 +80,7 @@ def get_dfs_from_options(offload_options, messages=None, force_ssh=False, dry_ru
         return GOES3(
             messages,
             dry_run=dry_run,
+            do_not_connect=do_not_connect,
             db_path_suffix=offload_options.hdfs_db_path_suffix,
         )
     elif offload_options.offload_fs_scheme in AZURE_OFFLOAD_FS_SCHEMES:
@@ -86,6 +92,7 @@ def get_dfs_from_options(offload_options, messages=None, force_ssh=False, dry_ru
             offload_options.offload_fs_azure_account_domain,
             messages,
             dry_run=dry_run,
+            do_not_connect=do_not_connect,
             db_path_suffix=offload_options.hdfs_db_path_suffix,
         )
     else:

src/goe/filesystem/goe_gcs.py

@@ -72,7 +72,9 @@ def __init__(
 
         logger.info("Client setup: GOEGcs")
 
-        super(GOEGcs, self).__init__(messages, dry_run=dry_run)
+        super(GOEGcs, self).__init__(
+            messages, dry_run=dry_run, do_not_connect=do_not_connect
+        )
 
         if do_not_connect:
             self._client = None

src/goe/filesystem/goe_s3.py

@@ -72,7 +72,9 @@ def __init__(
 
         logger.info("Client setup: GOES3")
 
-        super(GOES3, self).__init__(messages, dry_run=dry_run)
+        super(GOES3, self).__init__(
+            messages, dry_run=dry_run, do_not_connect=do_not_connect
+        )
 
         if do_not_connect:
             self._client = None

src/goe/filesystem/web_hdfs.py

@@ -83,7 +83,6 @@ def get_hdfs_session(verify=None, user=None):
 
 
 class GOEWebHdfsClient(KerberosClient):
-
     """A new client subclass for handling HTTPS connections as well as Kerberos & insecure.
     We have removed "cert" parameter. See link below for "cert" usage.
     See http://hdfscli.readthedocs.org/en/latest/advanced.html#custom-client-support for idea
@@ -144,11 +143,12 @@ def __init__(
             % (hdfs_namenode, webhdfs_port, hdfs_user, use_kerberos, verify_ssl_cert)
         )
 
-        super(WebHdfs, self).__init__(messages, dry_run=dry_run)
+        super(WebHdfs, self).__init__(
+            messages, dry_run=dry_run, do_not_connect=do_not_connect
+        )
 
         self._db_path_suffix = db_path_suffix
         self._hdfs_data = hdfs_data
-        self._do_not_connect = do_not_connect
         self._hdfs = self._hdfs_client(
             hdfs_namenode,
             webhdfs_port,

src/goe/offload/backend_table.py

@@ -155,6 +155,7 @@ def __init__(
         self._hybrid_metadata = hybrid_metadata
         self._data_gov_client = data_gov_client
         self._dry_run = dry_run
+        self._do_not_connect = do_not_connect
         # One day we may split connectivity options from other config
         self._connection_options = orchestration_options
         self._orchestration_config = orchestration_options
@@ -1027,7 +1028,9 @@ def _gen_synthetic_sql_cast(self, partition_column, source_column_cast):
     def _get_dfs_client(self):
         if self._dfs_client is None:
             self._dfs_client = get_dfs_from_options(
-                self._orchestration_config, messages=self._messages
+                self._orchestration_config,
+                messages=self._messages,
+                do_not_connect=self._do_not_connect,
             )
             self._backend_dfs = self._dfs_client.backend_dfs
         return self._dfs_client

src/goe/offload/bigquery/bigquery_backend_api.py

@@ -189,6 +189,7 @@ def __init__(
             self._google_kms_key_ring_location
             and self._google_kms_key_ring_name
             and self._google_kms_key_name
+            and not do_not_connect
         ):
             self._kms_client = self._get_kms_client()
             self._kms_key_name = self.kms_key_name()

tests/unit/offload/test_backend_api.py

@@ -109,9 +109,9 @@ def _gen_string_column(self, name):
         return self.api.gen_column_object(
             name,
             data_type=string_type,
-            data_length=10
-            if self.test_api.data_type_accepts_length(string_type)
-            else None,
+            data_length=(
+                10 if self.test_api.data_type_accepts_length(string_type) else None
+            ),
         )
 
     def _get_udf_db(self):
@@ -826,7 +826,7 @@ def _test_get_view_ddl(self):
                 pass
 
     def _test_google_kms_key_ring_unit(self):
-        if self.connect_to_backend:
+        if not self.connect_to_backend:
             return
 
         # Unit test config defines a key.