From 9c3e9afc0d8ff6d1cd68256a0611c97b07271feb Mon Sep 17 00:00:00 2001 From: Harold Wanyama Date: Mon, 31 Jul 2023 17:45:20 +0300 Subject: [PATCH] Bug/Concurrency support - Changed tools for handling github multiprocessing Signed-off-by: Harold Wanyama --- cla-backend/cla/models/github_models.py | 29 ++++++++++++++++--------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/cla-backend/cla/models/github_models.py b/cla-backend/cla/models/github_models.py index 8788bd7e0..dbeba9fde 100644 --- a/cla-backend/cla/models/github_models.py +++ b/cla-backend/cla/models/github_models.py @@ -8,7 +8,7 @@ import os import time import uuid -import multiprocessing +import concurrent.futures from typing import List, Union, Optional import falcon @@ -1367,17 +1367,26 @@ def get_pull_request_commit_authors(pull_request) -> List[UserCommitSummary]: commit_authors = [] - try: - num_processes = multiprocessing.cpu_count() - cla.log.debug(f'{fn} - Number of CPUs: {num_processes}') + # try: + # num_processes = multiprocessing.cpu_count() + # cla.log.debug(f'{fn} - Number of CPUs: {num_processes}') - with multiprocessing.Pool(processes=num_processes) as pool: - commit_authors = pool.starmap(get_author_summary, [(commit, pull_request.number) for commit in pull_request.get_commits()]) + # with multiprocessing.Pool(processes=num_processes) as pool: + # commit_authors = pool.starmap(get_author_summary, [(commit, pull_request.number) for commit in pull_request.get_commits()]) - except Exception as e: - cla.log.warning(f'{fn} - Exception while querying pull request commits for author information: {e}') - raise e - + # except Exception as e: + # cla.log.warning(f'{fn} - Exception while querying pull request commits for author information: {e}') + # raise e + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + future_to_commit = {executor.submit(get_author_summary, commit, pull_request.number): commit for commit in pull_request.get_commits()} + for future in concurrent.futures.as_completed(future_to_commit): + future_to_commit[future] + try: + commit_authors.append(future.result()) + except Exception as exc: + cla.log.warning(f'{fn} - PR: {pull_request.number}, get_author_summary generated an exception: {exc}') + raise exc + return commit_authors