Fix: Counters saved to database are always 0

Statistics were saved to database before the actual check was finished.
This commit is adding wait for every thread executed in the pool.

Fixes #795

Signed-off-by: Michal Konečný <[email protected]>

anitya/check_service.py

@@ -26,7 +26,7 @@
 from typing import List
 from datetime import datetime
 from time import sleep
-from concurrent.futures import ThreadPoolExecutor
+from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
 
 import sqlalchemy as sa
 import arrow
@@ -166,7 +166,10 @@ def run(self):
         )
         pool_size = config.get("CRON_POOL", 10)
         pool = ThreadPoolExecutor(pool_size)
-        pool.map(self.update_project, queue)
+        futures = [pool.submit(self.update_project, project) for project in queue]
+
+        # Wait till every project is checked
+        wait(futures, return_when=ALL_COMPLETED)
 
         # 3. Finalize
         _log.info(

anitya/tests/test_check_service.py

@@ -167,10 +167,9 @@ def test_blacklist_project(self):
         self.assertEqual(self.checker.error_counter, 0)
         self.assertEqual(self.checker.ratelimit_queue["GitHub"][0], project.id)
 
-    @mock.patch("anitya.check_service.Checker.update_project")
-    def test_run(self, mock_update_project):
+    def test_run(self):
         """
-        Assert that `db.Run` is created at the end of run.
+        Assert that `db.Run` is created at the end of run with success.
         """
         project = models.Project(
             name="Foobar",
@@ -181,6 +180,11 @@ def test_run(self, mock_update_project):
         self.session.add(project)
         self.session.commit()
 
+        def increment(project_id):
+            self.checker.success_counter = self.checker.success_counter + 1
+
+        self.checker.update_project = increment
+
         self.checker.run()
 
         run_objects = models.Run.query.all()
@@ -189,8 +193,7 @@ def test_run(self, mock_update_project):
         self.assertEqual(run_objects[0].total_count, 1)
         self.assertEqual(run_objects[0].error_count, 0)
         self.assertEqual(run_objects[0].ratelimit_count, 0)
-        self.assertEqual(run_objects[0].success_count, 0)
-        mock_update_project.called_once_with(project.id)
+        self.assertEqual(run_objects[0].success_count, 1)
 
     @mock.patch("anitya.lib.utilities.check_project_release")
     def test_run_nothing_to_check(self, mock_check_project_release):

news/795.bug

@@ -0,0 +1 @@
+Check service: Counters saved to database are always 0