diff --git a/api/neurolibre_celery_tasks.py b/api/neurolibre_celery_tasks.py index 162e586..8b0dce9 100644 --- a/api/neurolibre_celery_tasks.py +++ b/api/neurolibre_celery_tasks.py @@ -89,12 +89,12 @@ def rsync_data_task(self, comment_id, issue_id, project_name, reviewRepository): #logging.info(output) except subprocess.CalledProcessError as e: gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"{e.output}") - self.update_state(state=states.FAILURE, meta={'message': e.output}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': e.output}) # Performing a final check if os.path.exists(os.path.join("/DATA", project_name)): if len(os.listdir(os.path.join("/DATA", project_name))) == 0: # Directory exists but empty - self.update_state(state=states.FAILURE, meta={'message': f"Directory exists but empty {project_name}"}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Directory exists but empty {project_name}"}) gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"Directory exists but empty: {project_name}") else: # Directory exists and not empty @@ -102,7 +102,7 @@ def rsync_data_task(self, comment_id, issue_id, project_name, reviewRepository): self.update_state(state=states.SUCCESS, meta={'message': f"Data sync has been completed for {project_name}"}) else: # Directory does not exist - self.update_state(state=states.FAILURE, meta={'message': f"Directory does not exist {project_name}"}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Directory does not exist {project_name}"}) gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"Directory does not exist: {project_name}") @celery_app.task(bind=True) @@ -123,7 +123,7 @@ def rsync_book_task(self, repo_url, commit_hash, comment_id, issue_id, reviewRep [owner,repo,provider] = get_owner_repo_provider(repo_url,provider_full_name=True) if owner != "roboneurolibre": gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"Repository is not under roboneurolibre organization!") - self.update_state(state=states.FAILURE, meta={'message': f"FAILURE: Repository {owner}/{repo} has no roboneurolibre fork."}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"FAILURE: Repository {owner}/{repo} has no roboneurolibre fork."}) return commit_hash = format_commit_hash(repo_url,commit_hash) logging.info(f"{owner}{provider}{repo}{commit_hash}") @@ -144,12 +144,12 @@ def rsync_book_task(self, repo_url, commit_hash, comment_id, issue_id, reviewRep except subprocess.CalledProcessError as e: #logging.info("Subprocess exception") gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"{e.output}") - self.update_state(state=states.FAILURE, meta={'message': e.output}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': e.output}) # Check if GET works for the complicated address results = book_get_by_params(commit_hash=commit_hash) if not results: gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"Cannot retrieve book at {commit_hash}") - self.update_state(state=states.FAILURE, meta={'message': f"Cannot retrieve book at {commit_hash}"}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Cannot retrieve book at {commit_hash}"}) else: # Symlink production book to attain a proper URL book_target_tail = get_book_target_tail(results[0]['book_url'],commit_hash) @@ -179,7 +179,7 @@ def rsync_book_task(self, repo_url, commit_hash, comment_id, issue_id, reviewRep self.update_state(state=states.SUCCESS, meta={'message': message}) else: gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, output) - self.update_state(state=states.FAILURE, meta={'message': f"Cannot sync book at {commit_hash}"}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Cannot sync book at {commit_hash}"}) @celery_app.task(bind=True) def fork_configure_repository_task(self, payload): @@ -199,7 +199,7 @@ def fork_configure_repository_task(self, payload): if not book_tested_check['status']: msg = f"\n > [!WARNING] \n > A book build could not be found at commit `{payload['commit_hash']}` at {payload['repository_url']}. Production process cannot be started." gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg, collapsable=False) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return else: # Create a record for this. @@ -244,7 +244,7 @@ def fork_configure_repository_task(self, payload): if not forked_repo and retry_count == max_retries: msg = f"Forked repository is still not available after {max_retries*15} seconds! Please check if the repository is available under roboneurolibre organization, then try again." gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return else: logging.info(f"Fork already exists {payload['repository_url']}, moving on with configurations.") @@ -257,7 +257,7 @@ def fork_configure_repository_task(self, payload): if not jb_config or not jb_toc: msg = f"Could not load _config.yml or _toc.yml under the content directory of {forked_name}" gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return if 'launch_buttons' not in jb_config: @@ -279,7 +279,7 @@ def fork_configure_repository_task(self, payload): if not response['status']: msg = f"Could not update _config.yml for {forked_name}: \n {response['message']}" gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return jb_toc_new = jb_toc @@ -312,7 +312,7 @@ def fork_configure_repository_task(self, payload): if not response['status']: msg = f"Could not update toc.yml for {forked_name}: \n {response['message']}" gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': f"Could not update _toc.yml for {forked_name}"}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Could not update _toc.yml for {forked_name}"}) return msg = f"Please confirm that the forked repository is available and (_toc.yml and _config.ymlk) properly configured." @@ -437,7 +437,7 @@ def zenodo_create_buckets_task(self, payload): if os.path.exists(local_file): msg = f"Zenodo records already exist for this submission on NeuroLibre servers: {fname}. Please proceed with data uploads if the records are valid. Flush the existing records otherwise." gh_template_respond(github_client,"exists",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'],msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return data = payload['paper_data'] @@ -694,7 +694,7 @@ def zenodo_upload_repository_task(self, payload): resp = os.system(f"wget -O {zenodo_file} {download_url}") if resp != 0: gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"Cannot download: {download_url}") - self.update_state(state=states.FAILURE, meta={'message': f"Cannot download {download_url}"}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Cannot download {download_url}"}) return else: response = zenodo_upload_item(zenodo_file,payload['bucket_url'],payload['issue_id'],commit_fork,"repository") @@ -765,7 +765,7 @@ def zenodo_upload_docker_task(self, payload): # Terminate ERROR msg = f"Looks like there's not a successful book build record for {fork_url}" gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return msg = f"Found docker image: \n {lut}" @@ -777,7 +777,7 @@ def zenodo_upload_docker_task(self, payload): if not r['status']: msg = f"Cannot login to NeuroLibre private docker registry. \n {r['message']}" gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return msg = f"Pulling docker image: \n {lut['docker_image']}" @@ -788,7 +788,7 @@ def zenodo_upload_docker_task(self, payload): if not r['status']: msg = f"Cannot pull the docker image \n {r['message']}" gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return msg = f"Exporting docker image: \n {lut['docker_image']}" @@ -798,7 +798,7 @@ def zenodo_upload_docker_task(self, payload): if not r[0]['status']: msg = f"Cannot save the docker image \n {r[0]['message']}" gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return tar_file = r[1] @@ -862,7 +862,7 @@ def zenodo_publish_task(self, payload): if response == "no-record-found": msg = "
:neutral_face: I could not find any Zenodo-related records on NeuroLibre servers. Maybe start with roboneuro zenodo create buckets?" gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) return else: # Confirm that all items are published. @@ -882,7 +882,7 @@ def zenodo_publish_task(self, payload): response.append(f"\n Looks like there's a problem. {publish_status[1]} reproducibility assets are archived.") msg = "\n".join(response) gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg, False) - self.update_state(state=states.FAILURE, meta={'message': msg}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg}) ### DUPLICATION FOR NOW, SAVING THE DAY. @@ -973,7 +973,7 @@ def generate(): tmp_log = write_html_to_temp_directory(payload['commit_hash'], issue_comment) body = "

🥀 We ran into a problem building your book. Please download the log file attached and open in your web browser.

" send_email_with_html_attachment_celery(payload['email'], payload['mail_subject'], body, tmp_log) - self.update_state(state=states.FAILURE, meta={'message': f"FAILURE: Build for {owner}/{repo} at {payload['commit_hash']} has failed"}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"FAILURE: Build for {owner}/{repo} at {payload['commit_hash']} has failed"}) else: #gh_template_respond(github_client,"success","Successfully built", payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"The next comment will forward the logs") #issue_comment = [] @@ -1068,7 +1068,7 @@ def preprint_build_pdf_draft(self, payload): gh_clone_repository(payload['repository_url'], target_path, depth=1) except Exception as e: gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], str(e)) - self.update_state(state=states.FAILURE, meta={'message': str(e)}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': str(e)}) return # Crawl notebooks for the citation text and append it to paper.md, update bib. res = create_extended_pdf_sources(target_path, payload['issue_id'],payload['repository_url']) @@ -1099,8 +1099,8 @@ def preprint_build_pdf_draft(self, payload): gh_create_comment(github_client, payload['review_repository'],payload['issue_id'],comment) except subprocess.CalledProcessError as e: gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"{e.output}") - self.update_state(state=states.FAILURE, meta={'message': e.output}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': e.output}) else: gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"{res['message']}") - self.update_state(state=states.FAILURE, meta={'message': res['message']}) + self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': res['message']}) diff --git a/api/preprint.py b/api/preprint.py index 0e0d79d..cd70fbf 100644 --- a/api/preprint.py +++ b/api/preprint.py @@ -522,11 +522,12 @@ def substitute_cite_commands(input_folder="content"): # Embed citations in the cell's source code for match in matches: # Split the citations by comma and format them accordingly - citations = match.split(',') - try: - formatted_citations = '; '.join([f'@{citation.strip()}' for citation in citations]) - except: - pass + if match: + citations = match.split(',') + try: + formatted_citations = '; '.join([f'@{citation.strip()}' for citation in citations]) + except: + pass # Replace the original pattern with the formatted citations cell['source'] = re.sub(r'\{cite:p\}`([^`]*)`', f'[{formatted_citations}]', cell['source'], count=1) @@ -535,14 +536,15 @@ def substitute_cite_commands(input_folder="content"): if matches: # Embed citations in the cell's source code for match in matches: + if match: # Split the citations by comma and format them accordingly - citations = match.split(',') - try: - formatted_citations = '; '.join([f'@{citation.strip()}' for citation in citations]) - except: - pass - # Replace the original pattern with the formatted citations - cell['source'] = re.sub(rf'\{{cite:t\}}`{match}`', f'{formatted_citations}', cell['source'], count=1) + citations = match.split(',') + try: + formatted_citations = '; '.join([f'@{citation.strip()}' for citation in citations]) + except: + pass + # Replace the original pattern with the formatted citations + cell['source'] = re.sub(rf'\{{cite:t\}}`{match}`', f'{formatted_citations}', cell['source'], count=1) # Export the notebook as Markdown filtered_paragraphs = extract_paragraphs_with_citations(notebook)