Merge pull request #25 from opengamedata/issue/3-refactor-interface-c…

…lasses

Refactor interface classes: phase 1

.github/actions/test_config/action.yml

@@ -9,14 +9,18 @@ inputs:
     description: "Whether to output extra debugging lines from tests."
     required: false
     default: "False"
-  with_schemas:
-    description: "Whether to run the Schema tests."
+  with_configs:
+    description: "Whether to run the Config tests."
     required: false
     default: "False"
   with_interfaces:
     description: "Whether to run the interface tests."
     required: false
     default: "False"
+  with_schemas:
+    description: "Whether to run the Schema tests."
+    required: false
+    default: "False"
   with_utils:
     description: "Whether to run the utils tests."
     required: false
@@ -38,12 +42,15 @@ runs:
     - name: Set Output Verbosity
       run: sed -i 's@"VERBOSE"\s*:\s*False@"VERBOSE":${{ inputs.verbose_output }}@g'       ${{ inputs.config_path }}/t_config.py
       shell: bash
-    - name: Set Schema test(s) to run or not
-      run: sed -i 's@"SCHEMAS"\s*:\s*True@"SCHEMAS":${{ inputs.with_schemas }}@g'          ${{ inputs.config_path }}/t_config.py
+    - name: Set Config test(s) to run or not
+      run: sed -i 's@"CONFIGS"\s*:\s*True@"CONFIGS":${{ inputs.with_configs }}@g'          ${{ inputs.config_path }}/t_config.py
       shell: bash
     - name: Set interface test(s) to run or not
       run: sed -i 's@"INTERFACES"\s*:\s*True@"INTERFACES":${{ inputs.with_interfaces }}@g' ${{ inputs.config_path }}/t_config.py
       shell: bash
+    - name: Set Schema test(s) to run or not
+      run: sed -i 's@"SCHEMAS"\s*:\s*True@"SCHEMAS":${{ inputs.with_schemas }}@g'          ${{ inputs.config_path }}/t_config.py
+      shell: bash
     - name: Set utils test(s) to run or not
       run: sed -i 's@"UTILS"\s*:\s*True@"UTILS":${{ inputs.with_utils }}@g'                ${{ inputs.config_path }}/t_config.py
       shell: bash

.github/workflows/CI_common.yml

@@ -20,24 +20,31 @@ jobs:
     with:
       with_caching: false
 
-  # Run testbeds in schema module
+  # Run testbeds in configs module
+
+  testbed_configs:
+    name: Config Testbeds
+    needs: build
+    uses: ./.github/workflows/TEST_Configs.yml
+
+  testbed_game_configs:
+    name: Game Config Testbeds
+    needs: build
+    uses: ./.github/workflows/TEST_GameConfigs.yml
+
+  # Run testbeds in schemas module
 
   testbed_schema:
     name: Schema base class Testbed
     needs: build
     uses: ./.github/workflows/TEST_Schema.yml
 
-  testbed_config_schemas:
-    name: Config Schema Testbeds
-    needs: build
-    uses: ./.github/workflows/TEST_ConfigSchemas.yml
-
   testbed_game_schemas:
     name: Game Schema Testbeds
     needs: build
     uses: ./.github/workflows/TEST_GameSchemas.yml
 
-  # Run testbeds in schema module
+  # Run testbeds in utils module
 
   testbed_fileio:
     name: FileIO Testbed

.github/workflows/TEST_ConfigSchemas.yml → .github/workflows/TEST_Configs.yml

@@ -6,11 +6,13 @@ on:
   workflow_call:
   push:
     paths:
-    - '.github/workflows/TEST_ConfigSchemas.yml'
+    # repo-wide dependencies
     - '.github/actions/test_config/**'
-    - 'tests/cases/schemas/config/*.py'
     - 'tests/config/**'
     - 'requirements.txt'
+    # specific dependencies
+    - '.github/workflows/TEST_Configs.yml'
+    - 'tests/cases/configs/*.py'
 
 concurrency:
   group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}-ConfigSchemas
@@ -25,8 +27,8 @@ jobs:
       matrix:
         testbed: [
           t_GameSourceSchema,
-          t_IndexingSchema,
-          t_TestConfigSchema,
+          t_IndexingConfig,
+          t_TestConfig,
         ]
       fail-fast: false # we don't want to cancel just because one testbed fails.
       max-parallel: 20
@@ -53,7 +55,7 @@ jobs:
     - name: Execute ${{ matrix.testbed }} testbed
       uses: opengamedata/[email protected]
       with:
-        directory: "tests/cases/schemas/config"
+        directory: "tests/cases/configs"
         test_file: "${{ matrix.testbed }}.py"
         python_version: ${{ vars.OGD_PYTHON_VERSION }}
 

.github/workflows/TEST_GameConfigs.yml

@@ -0,0 +1,66 @@
+# Workflow to test the schemas from the `config` subfolder
+name: Testbed - Game Configs
+run-name: ${{ format('{0} - {1}', github.workflow, github.event_name == 'push' && github.event.head_commit.message || 'Manual Run') }}
+on:
+  workflow_dispatch:
+  workflow_call:
+  push:
+    paths:
+    # repo-wide dependencies
+    - '.github/actions/test_config/**'
+    - 'tests/config/**'
+    - 'requirements.txt'
+    # specific dependencies
+    - '.github/workflows/TEST_GameConfigs.yml'
+    - 'tests/cases/configs/games/**'
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}-GameConfigs
+  cancel-in-progress: true
+
+jobs:
+
+  run_testbed_schema:
+    name: Run Game Config Testbeds
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        testbed: [
+          t_AggregateConfig,
+          t_DetectorMapConfig,
+          t_DetectorConfig,
+          t_FeatureMapConfig,
+          t_FeatureConfig,
+          t_GeneratorConfig,
+          t_PerCountConfig
+        ]
+      fail-fast: false # we don't want to cancel just because one testbed fails.
+      max-parallel: 20
+
+    steps:
+  # 1. Local checkout 
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Get Dependencies
+      uses: opengamedata/[email protected]
+      with:
+        python_version: ${{ vars.OGD_PYTHON_VERSION }}
+    - name: Local self-install
+      run: python -m pip install -e .
+    - name: Set up Config File
+      uses: ./.github/actions/test_config
+      with:
+          verbose_output: "True"
+          with_configs: "True"
+
+  # 2. Build & configure remote environments
+
+  # 3. Perform export
+    - name: Execute ${{ matrix.testbed }} testbed
+      uses: opengamedata/[email protected]
+      with:
+        directory: "tests/cases/configs/games"
+        test_file: "${{ matrix.testbed }}.py"
+        python_version: ${{ vars.OGD_PYTHON_VERSION }}
+
+  # 4. Cleanup & complete

.github/workflows/TEST_GameSchemas.yml

@@ -6,11 +6,13 @@ on:
   workflow_call:
   push:
     paths:
-    - '.github/workflows/TEST_GameSchemas.yml'
+    # repo-wide dependencies
     - '.github/actions/test_config/**'
-    - 'tests/cases/schemas/games/**'
     - 'tests/config/**'
     - 'requirements.txt'
+    # specific dependencies
+    - '.github/workflows/TEST_GameSchemas.yml'
+    - 'tests/cases/schemas/games/**'
 
 concurrency:
   group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}-GameSchemas
@@ -24,17 +26,10 @@ jobs:
     strategy:
       matrix:
         testbed: [
-          t_AggregateSchema,
           t_DataElementSchema,
-          t_DetectorMapSchema,
-          t_DetectorSchema,
           t_EventSchema,
-          t_FeatureMapSchema,
-          t_FeatureSchema,
           t_GameSchema,
-          t_GameStateSchema,
-          t_GeneratorSchema,
-          t_PerCountSchema,
+          t_GameStateSchema
         ]
       fail-fast: false # we don't want to cancel just because one testbed fails.
       max-parallel: 20

src/ogd/common/configs/Config.py

@@ -0,0 +1,8 @@
+"""Config Class Module
+"""
+## import standard libraries
+from typing import TypeAlias
+# import local files
+from ogd.common.schemas.Schema import Schema
+
+Config : TypeAlias = Schema

src/ogd/common/schemas/configs/GameSourceSchema.py → src/ogd/common/configs/GameSourceSchema.py

@@ -1,15 +1,17 @@
 # import standard libraries
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Optional
 # import local files
-from ogd.common.schemas.configs.data_sources.DataSourceSchema import DataSourceSchema
-from ogd.common.schemas.configs.data_sources.BigQuerySourceSchema import BigQuerySchema
+from ogd.common.configs.storage.DataStoreConfig import DataStoreConfig
+from ogd.common.configs.storage.BigQueryConfig import BigQueryConfig
 from ogd.common.schemas.Schema import Schema
+from ogd.common.schemas.tables.TableSchema import TableSchema
 from ogd.common.utils.Logger import Logger
 from ogd.common.utils.typing import Map
 
 class GameSourceSchema(Schema):
 
+    _DEFAULT_GAME_ID       = "UNKNOWN GAME"
     _DEFAULT_SOURCE_NAME   = "OPENGAMEDATA_BQ"
     _DEFAULT_DB_NAME       = "UNKNOWN GAME"
     _DEFAULT_TABLE_NAME    = "_daily"
@@ -31,25 +33,40 @@ class GameSourceSchema(Schema):
     - `TableName` : The neame of the specific table within the database holding the given game's data
     - `TableSchema` : A schema indicating the structure of the table containing the given game's data.
 
+    TODO : Implement and use a smart Load(...) function of TableSchema to load schema from given name, rather than FromFile.
+
     :param Schema: _description_
     :type Schema: _type_
     """
-    def __init__(self, name:str,  source_name:str, source_schema:Optional[DataSourceSchema],
+    def __init__(self, name:str,  game_id:Optional[str],
+                 source_name:str, source_schema:Optional[DataStoreConfig],
                  db_name:str,     table_name:str,  table_schema:str,
-                 other_elements:Optional[Map]=None):
-        self._source_name   : str                        = source_name
-        self._source_schema : Optional[DataSourceSchema] = source_schema
-        self._db_name       : str                        = db_name
-        self._table_name    : str                        = table_name
-        self._table_schema  : str                        = table_schema
+                 other_elements:Dict[str, Any]):
+        self._game_id           : str
+        self._source_name       : str                        = source_name
+        self._source_schema     : Optional[DataStoreConfig] = source_schema
+        self._db_name           : str                        = db_name
+        self._table_name        : str                        = table_name
+        self._table_schema_name : str                        = table_schema
+        self._table_schema      : TableSchema = TableSchema.FromFile(schema_name=self._table_schema_name)
+
+        if game_id is not None:
+            self._game_id = game_id
+        else:
+            Logger.Log(f"GameSourceSchema did not receive a game_id, defaulting to {name}")
+            self._game_id = name
         super().__init__(name=name, other_elements=other_elements)
 
+    @property
+    def GameID(self) -> str:
+        return self._game_id
+
     @property
     def SourceName(self) -> str:
         return self._source_name
 
     @property
-    def Source(self) -> Optional[DataSourceSchema]:
+    def Source(self) -> Optional[DataStoreConfig]:
         return self._source_schema
 
     @property
@@ -61,32 +78,37 @@ def TableName(self) -> str:
         return self._table_name
 
     @property
-    def TableSchema(self) -> str:
+    def TableSchema(self) -> TableSchema:
         return self._table_schema
 
+    @property
+    def TableSchemaName(self) -> str:
+        return self._table_schema_name
+
     # *** IMPLEMENT ABSTRACT FUNCTIONS ***
 
     @property
     def AsMarkdown(self) -> str:
         ret_val : str
 
-        ret_val = f"{self.Name}: _{self.TableSchema}_ format, source {self.Source.Name if self.Source else 'None'} : {self.DatabaseName}.{self.TableName}"
+        ret_val = f"{self.Name}: _{self.TableSchemaName}_ format, source {self.Source.Name if self.Source else 'None'} : {self.DatabaseName}.{self.TableName}"
         return ret_val
 
     @classmethod
     def Default(cls) -> "GameSourceSchema":
         return GameSourceSchema(
             name="DefaultGameSourceSchema",
+            game_id=cls._DEFAULT_GAME_ID,
             source_name=cls._DEFAULT_SOURCE_NAME,
-            source_schema=BigQuerySchema.Default(),
+            source_schema=BigQueryConfig.Default(),
             db_name=cls._DEFAULT_DB_NAME,
             table_name=cls._DEFAULT_TABLE_NAME,
             table_schema=cls._DEFAULT_TABLE_SCHEMA,
             other_elements={}
         )
 
     @classmethod
-    def FromDict(cls, name:str, all_elements:Dict[str, Any], logger:Optional[logging.Logger], data_sources:Dict[str, DataSourceSchema]) -> "GameSourceSchema":
+    def FromDict(cls, name:str, all_elements:Dict[str, Any], logger:Optional[logging.Logger], data_sources:Dict[str, DataStoreConfig]) -> "GameSourceSchema":
         """Create a GameSourceSchema from a given dictionary
 
         :param name: _description_
@@ -96,12 +118,12 @@ def FromDict(cls, name:str, all_elements:Dict[str, Any], logger:Optional[logging
         :param logger: _description_
         :type logger: Optional[logging.Logger]
         :param data_sources: _description_
-        :type data_sources: Dict[str, DataSourceSchema]
+        :type data_sources: Dict[str, DataStoreConfig]
         :return: _description_
         :rtype: GameSourceSchema
         """
         _source_name   : str
-        _source_schema : Optional[DataSourceSchema]
+        _source_schema : Optional[DataStoreConfig]
         _db_name       : str
         _table_schema  : str
         _table_name    : str
@@ -113,6 +135,11 @@ def FromDict(cls, name:str, all_elements:Dict[str, Any], logger:Optional[logging
                 logger.warning(_msg)
             else:
                 Logger.Log(_msg, logging.WARN)
+        _game_id = cls.ElementFromDict(all_elements=all_elements, logger=logger,
+            element_names=["game", "game_id"],
+            parser_function=cls._parseGameID,
+            default_value=name
+        )
         _source_name = cls.ElementFromDict(all_elements=all_elements, logger=logger,
             element_names=["source"],
             parser_function=cls._parseSource,
@@ -142,15 +169,15 @@ def FromDict(cls, name:str, all_elements:Dict[str, Any], logger:Optional[logging
 
         _used = {"source", "database", "table", "schema"}
         _leftovers = { key : val for key,val in all_elements.items() if key not in _used }
-        return GameSourceSchema(name=name, source_name=_source_name, source_schema=_source_schema,
+        return GameSourceSchema(name=name, game_id=_game_id, source_name=_source_name, source_schema=_source_schema,
                                 db_name=_db_name, table_name=_table_name, table_schema=_table_schema,
                                 other_elements=_leftovers)
 
     # *** PUBLIC STATICS ***
 
     @staticmethod
     def EmptySchema() -> "GameSourceSchema":
-        return GameSourceSchema(name="NOT FOUND", source_name="NOT FOUND", source_schema=None, db_name="NOT FOUND",
+        return GameSourceSchema(name="NOT FOUND", game_id="NOT FOUND", source_name="NOT FOUND", source_schema=None, db_name="NOT FOUND",
                                 table_name="NOT FOUND", table_schema="NOT FOUND", other_elements={})
 
     # *** PUBLIC METHODS ***
@@ -167,6 +194,16 @@ def _parseSource(source) -> str:
             Logger.Log(f"Game Source source name was unexpected type {type(source)}, defaulting to str(source)={ret_val}.", logging.WARN)
         return ret_val
 
+    @staticmethod
+    def _parseGameID(game_id) -> str:
+        ret_val : str
+        if isinstance(game_id, str):
+            ret_val = game_id
+        else:
+            ret_val = str(game_id)
+            Logger.Log(f"Game Source app ID was unexpected type {type(game_id)}, defaulting to str(game_id)={ret_val}.", logging.WARN)
+        return ret_val
+
     @staticmethod
     def _parseDBName(db_name) -> str:
         ret_val : str