fix(stats, tests): Major reworking of test suite; addition of github …

…workflow for test running; fixing bugs in stats event creation and aggregation

.github/workflows/test.yml

@@ -0,0 +1,34 @@
+name: Test
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    services:
+      docker:
+        image: docker:24.0.5
+        options: --privileged
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pipenv
+        pipenv install --dev
+
+    # Test runner includes taking down the docker compose stack
+    - name: Run tests
+      run: |
+        pipenv run ./run-tests.sh
+
+    # - name: Stop Docker Compose
+    #   if: always()
+    #   run: docker-compose down

.gitignore

@@ -1,7 +1,8 @@
 # Byte-compiled / optimized / DLL files
-__pycache__/
+**/__pycache__/
 *.py[cod]
 *$py.class
+**.pyc
 
 # C extensions
 *.so
@@ -24,7 +25,6 @@ share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
-MANIFEST
 
 .DS_Store
 

MANIFEST.in

@@ -0,0 +1,12 @@
+include *.lock
+include Pipfile
+include *.sh
+recursive-include dist *.gz
+recursive-include dist *.whl
+recursive-include invenio_record_importer *.DS_Store
+recursive-include invenio_record_importer *.gitkeep
+recursive-include tests *.ipynb
+recursive-include tests *.json
+recursive-include tests *.jsonl
+recursive-include tests *.py
+

Pipfile

@@ -4,7 +4,7 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-invenio-app-rdm = {version = "~=12.0.0b3.dev3", extras = ["opensearch2"]}
+invenio-app-rdm = {version = "==12.0.3", extras = ["opensearch2"]}
 timefhuman = "*"
 halo = "*"
 jsonlines = "*"
@@ -19,8 +19,13 @@ pytz = "*"
 python-dateutil = "*"
 invenio-saml = "*"
 invenio-record-importer = {file = ".", editable = true}
+invenio-rdm-records = {file = "../invenio-rdm-records", editable = true}
+invenio-records-resources = {file = "../invenio-records-resources", editable = true}
+invenio-communities = {file = "../invenio-communities", editable = true}
 invenio-remote-user-data = {file = "../invenio-remote-user-data", editable = true}
 invenio-group-collections = {file = "../invenio-group-collections", editable = true}
+unidecode = "*"
+kcworks = {file = "../knowledge-commons-works/site", editable = true}
 
 [dev-packages]
 docker-services-cli = "*"

invenio_record_importer/ext.py

@@ -45,10 +45,14 @@ def init_config(self, app) -> None:
             if k.startswith("RECORD_IMPORTER_"):
                 app.config.setdefault(k, getattr(self.config, k))
 
-        app.logger.handlers[0].setFormatter(
-            logging.Formatter(
-                "[%(asctime)s] %(levelname)s - %(message)s "
-                "{%(filename)s:%(lineno)d}",
-                "%m-%d %H:%M:%S",
-            )
-        )
+        # print("init_config")
+        # print(app.logger)
+        # print(app.logger.handlers)
+        # print(app.logger.handlers[0])
+        # app.logger.handlers[0].setFormatter(
+        #     logging.Formatter(
+        #         "[%(asctime)s] %(levelname)s - %(message)s "
+        #         "{%(filename)s:%(lineno)d}",
+        #         "%m-%d %H:%M:%S",
+        #     )
+        # )

invenio_record_importer/legacy/fedora_fetcher.py

@@ -7,7 +7,7 @@
 # modify it under the terms of the MIT License; see LICENSE file for more
 # details.
 
-from ..libs.fedoraapi import FedoraApi
+from invenio_record_importer.libs.fedoraapi import FedoraApi
 import requests
 from typing import Optional
 import os