Clean up references to all_modules

backend/lib/preset.py

@@ -35,7 +35,7 @@ def process(self):
 			pipeline[-1]["parameters"]["next"] = [last]
 
 		analysis_pipeline = DataSet(parameters=pipeline[0]["parameters"], type=pipeline[0]["type"], db=self.db,
-								 parent=self.dataset.key)
+								 parent=self.dataset.key, modules=self.modules)
 
 		# this starts the pipeline
 		self.queue.add_job(pipeline[0]["type"], remote_id=analysis_pipeline.key)

backend/lib/processor.py

@@ -259,7 +259,8 @@ def after_process(self):
 					parent=self.dataset.key,
 					extension=available_processors[next_type].extension,
 					is_private=self.dataset.is_private,
-					owner=self.dataset.creator
+					owner=self.dataset.creator,
+					modules=self.modules
 				)
 				self.queue.add_job(next_type, remote_id=next_analysis.key)
 			else:

backend/lib/worker.py

@@ -88,9 +88,9 @@ def __init__(self, logger, job, queue=None, manager=None, modules=None):
 		self.init_time = int(time.time())
 		self.config = ConfigDummy()
 
-		# all_modules cannot be easily imported into a worker because all_modules itself
+		# ModuleCollector cannot be easily imported into a worker because it itself
 		# imports all workers, so you get a recursive import that Python (rightly) blocks
-		# so for workers, all_modules' content is passed as a constructor argument
+		# so for workers, modules data is passed as a constructor argument
 		self.modules = modules
 
 		database_appname = "%s-%s" % (self.type, self.job.data["id"])

backend/workers/datasource_metrics.py

@@ -97,11 +97,11 @@ def data_stats(self):
         added_datasources = [row["datasource"] for row in self.db.fetchall("SELECT DISTINCT(datasource) FROM metrics")]
         enabled_datasources = config.get("datasources.enabled", {})
 
-        for datasource_id in self.all_modules.datasources:
+        for datasource_id in self.modules.datasources:
             if datasource_id not in enabled_datasources:
                 continue
 
-            datasource = self.all_modules.workers.get(datasource_id + "-search")
+            datasource = self.modules.workers.get(datasource_id + "-search")
             if not datasource:
                 continue
 

common/lib/dataset.py

@@ -104,20 +104,17 @@ def __init__(self, parameters=None, key=None, job=None, data=None, db=None, pare
 			if not current:
 				raise DataSetNotFoundException("DataSet() requires a valid dataset key for its 'key' argument, \"%s\" given" % key)
 
-			query = current["query"]
 		elif job is not None:
 			current = self.db.fetchone("SELECT * FROM datasets WHERE parameters::json->>'job' = %s", (job,))
 			if not current:
 				raise DataSetNotFoundException("DataSet() requires a valid job ID for its 'job' argument")
 
-			query = current["query"]
 			self.key = current["key"]
 		elif data is not None:
 			current = data
 			if "query" not in data or "key" not in data or "parameters" not in data or "key_parent" not in data:
 				raise DataSetException("DataSet() requires a complete dataset record for its 'data' argument")
 
-			query = current["query"]
 			self.key = current["key"]
 		else:
 			if parameters is None:

webtool/__init__.py

@@ -36,6 +36,7 @@
 # initialize global objects for interacting with all the things
 login_manager = LoginManager()
 app = Flask(__name__)
+fourcat_modules = ModuleCollector()
 
 # this ensures that HTTPS is properly applied to built URLs even if the app
 # is running behind a proxy

webtool/views/api_tool.py

@@ -209,7 +209,7 @@ def import_dataset():
 		.replace("5", "five").replace("6", "six").replace("7", "seven").replace("8", "eight") \
 		.replace("9", "nine")
 
-	if not platform or platform not in backend.all_modules.datasources or platform not in config.get('datasources.enabled'):
+	if not platform or platform not in fourcat_modules.datasources or platform not in config.get('datasources.enabled'):
 		return error(404, message=f"Unknown platform or source format '{platform}'")
 
 	worker_types = (f"{platform}-import", f"{platform}-search")
@@ -928,7 +928,7 @@ def check_search_queue():
 	unfinished_jobs = db.fetchall("SELECT jobtype, COUNT(*)count FROM jobs WHERE jobtype LIKE '%-search' GROUP BY jobtype ORDER BY count DESC;")
 
 	for i, job in enumerate(unfinished_jobs):
-		processor = backend.all_modules.processors.get(job["jobtype"])
+		processor = fourcat_modules.processors.get(job["jobtype"])
 		if processor:
 			unfinished_jobs[i]["processor_name"] = processor.title
 		else:

webtool/views/views_admin.py

@@ -143,7 +143,7 @@ def get_worker_status():
             "dataset": None if not worker["dataset_key"] else DataSet(key=worker["dataset_key"], db=db)
         } for worker in call_api("worker-status")["response"]["running"]
     ]
-    return render_template("controlpanel/worker-status.html", workers=workers, worker_types=backend.all_modules.workers,
+    return render_template("controlpanel/worker-status.html", workers=workers, worker_types=fourcat_modules.workers,
                            now=time.time())
 
 
@@ -588,7 +588,7 @@ def manipulate_settings():
             submenu = "core"
         elif option_owner.endswith("-search"):
             submenu = "datasources"
-        elif option_owner in backend.all_modules.processors:
+        elif option_owner in fourcat_modules.processors:
             submenu = "processors"
 
         tabname = config_definition.categories.get(option_owner)