From 591c84f5dc3feb34c30c4c4a241a1903e170af7c Mon Sep 17 00:00:00 2001 From: Stijn Peeters Date: Thu, 19 Sep 2024 15:23:18 +0200 Subject: [PATCH] Clean up references to all_modules --- backend/lib/preset.py | 2 +- backend/lib/processor.py | 3 ++- backend/lib/worker.py | 4 ++-- backend/workers/datasource_metrics.py | 4 ++-- common/lib/dataset.py | 3 --- webtool/__init__.py | 1 + webtool/views/api_tool.py | 4 ++-- webtool/views/views_admin.py | 4 ++-- 8 files changed, 12 insertions(+), 13 deletions(-) diff --git a/backend/lib/preset.py b/backend/lib/preset.py index 7b3bb170d..db62acac9 100644 --- a/backend/lib/preset.py +++ b/backend/lib/preset.py @@ -35,7 +35,7 @@ def process(self): pipeline[-1]["parameters"]["next"] = [last] analysis_pipeline = DataSet(parameters=pipeline[0]["parameters"], type=pipeline[0]["type"], db=self.db, - parent=self.dataset.key) + parent=self.dataset.key, modules=self.modules) # this starts the pipeline self.queue.add_job(pipeline[0]["type"], remote_id=analysis_pipeline.key) diff --git a/backend/lib/processor.py b/backend/lib/processor.py index 46afbe6a1..e9e4d85a4 100644 --- a/backend/lib/processor.py +++ b/backend/lib/processor.py @@ -259,7 +259,8 @@ def after_process(self): parent=self.dataset.key, extension=available_processors[next_type].extension, is_private=self.dataset.is_private, - owner=self.dataset.creator + owner=self.dataset.creator, + modules=self.modules ) self.queue.add_job(next_type, remote_id=next_analysis.key) else: diff --git a/backend/lib/worker.py b/backend/lib/worker.py index b4f2e7a46..17b1b935c 100644 --- a/backend/lib/worker.py +++ b/backend/lib/worker.py @@ -88,9 +88,9 @@ def __init__(self, logger, job, queue=None, manager=None, modules=None): self.init_time = int(time.time()) self.config = ConfigDummy() - # all_modules cannot be easily imported into a worker because all_modules itself + # ModuleCollector cannot be easily imported into a worker because it itself # imports all workers, so you get a recursive import that Python (rightly) blocks - # so for workers, all_modules' content is passed as a constructor argument + # so for workers, modules data is passed as a constructor argument self.modules = modules database_appname = "%s-%s" % (self.type, self.job.data["id"]) diff --git a/backend/workers/datasource_metrics.py b/backend/workers/datasource_metrics.py index 7340c35a6..dfc5579d8 100644 --- a/backend/workers/datasource_metrics.py +++ b/backend/workers/datasource_metrics.py @@ -97,11 +97,11 @@ def data_stats(self): added_datasources = [row["datasource"] for row in self.db.fetchall("SELECT DISTINCT(datasource) FROM metrics")] enabled_datasources = config.get("datasources.enabled", {}) - for datasource_id in self.all_modules.datasources: + for datasource_id in self.modules.datasources: if datasource_id not in enabled_datasources: continue - datasource = self.all_modules.workers.get(datasource_id + "-search") + datasource = self.modules.workers.get(datasource_id + "-search") if not datasource: continue diff --git a/common/lib/dataset.py b/common/lib/dataset.py index 76f526de9..b092d2a4e 100644 --- a/common/lib/dataset.py +++ b/common/lib/dataset.py @@ -104,20 +104,17 @@ def __init__(self, parameters=None, key=None, job=None, data=None, db=None, pare if not current: raise DataSetNotFoundException("DataSet() requires a valid dataset key for its 'key' argument, \"%s\" given" % key) - query = current["query"] elif job is not None: current = self.db.fetchone("SELECT * FROM datasets WHERE parameters::json->>'job' = %s", (job,)) if not current: raise DataSetNotFoundException("DataSet() requires a valid job ID for its 'job' argument") - query = current["query"] self.key = current["key"] elif data is not None: current = data if "query" not in data or "key" not in data or "parameters" not in data or "key_parent" not in data: raise DataSetException("DataSet() requires a complete dataset record for its 'data' argument") - query = current["query"] self.key = current["key"] else: if parameters is None: diff --git a/webtool/__init__.py b/webtool/__init__.py index e0419df3f..4c243d7a7 100644 --- a/webtool/__init__.py +++ b/webtool/__init__.py @@ -36,6 +36,7 @@ # initialize global objects for interacting with all the things login_manager = LoginManager() app = Flask(__name__) +fourcat_modules = ModuleCollector() # this ensures that HTTPS is properly applied to built URLs even if the app # is running behind a proxy diff --git a/webtool/views/api_tool.py b/webtool/views/api_tool.py index be84814fd..5b47c030d 100644 --- a/webtool/views/api_tool.py +++ b/webtool/views/api_tool.py @@ -209,7 +209,7 @@ def import_dataset(): .replace("5", "five").replace("6", "six").replace("7", "seven").replace("8", "eight") \ .replace("9", "nine") - if not platform or platform not in backend.all_modules.datasources or platform not in config.get('datasources.enabled'): + if not platform or platform not in fourcat_modules.datasources or platform not in config.get('datasources.enabled'): return error(404, message=f"Unknown platform or source format '{platform}'") worker_types = (f"{platform}-import", f"{platform}-search") @@ -928,7 +928,7 @@ def check_search_queue(): unfinished_jobs = db.fetchall("SELECT jobtype, COUNT(*)count FROM jobs WHERE jobtype LIKE '%-search' GROUP BY jobtype ORDER BY count DESC;") for i, job in enumerate(unfinished_jobs): - processor = backend.all_modules.processors.get(job["jobtype"]) + processor = fourcat_modules.processors.get(job["jobtype"]) if processor: unfinished_jobs[i]["processor_name"] = processor.title else: diff --git a/webtool/views/views_admin.py b/webtool/views/views_admin.py index d9b66ff77..9e09c9f06 100644 --- a/webtool/views/views_admin.py +++ b/webtool/views/views_admin.py @@ -143,7 +143,7 @@ def get_worker_status(): "dataset": None if not worker["dataset_key"] else DataSet(key=worker["dataset_key"], db=db) } for worker in call_api("worker-status")["response"]["running"] ] - return render_template("controlpanel/worker-status.html", workers=workers, worker_types=backend.all_modules.workers, + return render_template("controlpanel/worker-status.html", workers=workers, worker_types=fourcat_modules.workers, now=time.time()) @@ -588,7 +588,7 @@ def manipulate_settings(): submenu = "core" elif option_owner.endswith("-search"): submenu = "datasources" - elif option_owner in backend.all_modules.processors: + elif option_owner in fourcat_modules.processors: submenu = "processors" tabname = config_definition.categories.get(option_owner)