Exception handle

api/neurolibre_celery_tasks.py

@@ -89,20 +89,20 @@ def rsync_data_task(self, comment_id, issue_id, project_name, reviewRepository):
         #logging.info(output)
     except subprocess.CalledProcessError as e:
         gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"{e.output}")
-        self.update_state(state=states.FAILURE, meta={'message': e.output})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': e.output})
     # Performing a final check
     if os.path.exists(os.path.join("/DATA", project_name)):
         if len(os.listdir(os.path.join("/DATA", project_name))) == 0:
             # Directory exists but empty
-            self.update_state(state=states.FAILURE, meta={'message': f"Directory exists but empty {project_name}"})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Directory exists but empty {project_name}"})
             gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"Directory exists but empty: {project_name}")
         else:
             # Directory exists and not empty
             gh_template_respond(github_client,"success",task_title,reviewRepository,issue_id,task_id,comment_id, "Success.")
             self.update_state(state=states.SUCCESS, meta={'message': f"Data sync has been completed for {project_name}"})
     else:
         # Directory does not exist
-        self.update_state(state=states.FAILURE, meta={'message': f"Directory does not exist {project_name}"})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Directory does not exist {project_name}"})
         gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"Directory does not exist: {project_name}")
 
 @celery_app.task(bind=True)
@@ -123,7 +123,7 @@ def rsync_book_task(self, repo_url, commit_hash, comment_id, issue_id, reviewRep
     [owner,repo,provider] = get_owner_repo_provider(repo_url,provider_full_name=True)
     if owner != "roboneurolibre": 
         gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"Repository is not under roboneurolibre organization!")
-        self.update_state(state=states.FAILURE, meta={'message': f"FAILURE: Repository {owner}/{repo} has no roboneurolibre fork."})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"FAILURE: Repository {owner}/{repo} has no roboneurolibre fork."})
         return
     commit_hash = format_commit_hash(repo_url,commit_hash)
     logging.info(f"{owner}{provider}{repo}{commit_hash}")
@@ -144,12 +144,12 @@ def rsync_book_task(self, repo_url, commit_hash, comment_id, issue_id, reviewRep
     except subprocess.CalledProcessError as e:
         #logging.info("Subprocess exception")
         gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"{e.output}")
-        self.update_state(state=states.FAILURE, meta={'message': e.output})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': e.output})
     # Check if GET works for the complicated address
     results = book_get_by_params(commit_hash=commit_hash)
     if not results:
         gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, f"Cannot retrieve book at {commit_hash}")
-        self.update_state(state=states.FAILURE, meta={'message': f"Cannot retrieve book at {commit_hash}"})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Cannot retrieve book at {commit_hash}"})
     else:
         # Symlink production book to attain a proper URL
         book_target_tail = get_book_target_tail(results[0]['book_url'],commit_hash)
@@ -179,7 +179,7 @@ def rsync_book_task(self, repo_url, commit_hash, comment_id, issue_id, reviewRep
             self.update_state(state=states.SUCCESS, meta={'message': message})
         else:
             gh_template_respond(github_client,"failure",task_title,reviewRepository,issue_id,task_id,comment_id, output)
-            self.update_state(state=states.FAILURE, meta={'message': f"Cannot sync book at {commit_hash}"})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Cannot sync book at {commit_hash}"})
 
 @celery_app.task(bind=True)
 def fork_configure_repository_task(self, payload):
@@ -199,7 +199,7 @@ def fork_configure_repository_task(self, payload):
     if not book_tested_check['status']:
         msg = f"\n > [!WARNING] \n > A book build could not be found at commit `{payload['commit_hash']}` at {payload['repository_url']}. Production process cannot be started."
         gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg, collapsable=False)
-        self.update_state(state=states.FAILURE, meta={'message': msg})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
         return
     else:
         # Create a record for this.
@@ -244,7 +244,7 @@ def fork_configure_repository_task(self, payload):
         if not forked_repo and retry_count == max_retries:
             msg = f"Forked repository is still not available after {max_retries*15} seconds! Please check if the repository is available under roboneurolibre organization, then try again."
             gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-            self.update_state(state=states.FAILURE, meta={'message': msg})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
             return
     else:
         logging.info(f"Fork already exists {payload['repository_url']}, moving on with configurations.")
@@ -257,7 +257,7 @@ def fork_configure_repository_task(self, payload):
     if not jb_config or not jb_toc:
         msg = f"Could not load _config.yml or _toc.yml under the content directory of {forked_name}"
         gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-        self.update_state(state=states.FAILURE, meta={'message': msg})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
         return
 
     if 'launch_buttons' not in jb_config:
@@ -279,7 +279,7 @@ def fork_configure_repository_task(self, payload):
     if not response['status']:
         msg = f"Could not update _config.yml for {forked_name}: \n {response['message']}"
         gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-        self.update_state(state=states.FAILURE, meta={'message': msg})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
         return
 
     jb_toc_new = jb_toc
@@ -312,7 +312,7 @@ def fork_configure_repository_task(self, payload):
     if not response['status']:
         msg = f"Could not update toc.yml for {forked_name}: \n {response['message']}"
         gh_template_respond(github_client,"failure",task_title,payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-        self.update_state(state=states.FAILURE, meta={'message': f"Could not update _toc.yml for {forked_name}"})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Could not update _toc.yml for {forked_name}"})
         return
 
     msg = f"Please confirm that the <a href=\"https://github.com/{forked_name}\">forked repository</a> is available and (<code>_toc.yml</code> and <code>_config.ymlk</code>) properly configured."
@@ -437,7 +437,7 @@ def zenodo_create_buckets_task(self, payload):
     if os.path.exists(local_file):
         msg = f"Zenodo records already exist for this submission on NeuroLibre servers: {fname}. Please proceed with data uploads if the records are valid. Flush the existing records otherwise."
         gh_template_respond(github_client,"exists",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'],msg)
-        self.update_state(state=states.FAILURE, meta={'message': msg})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
         return
 
     data = payload['paper_data']
@@ -694,7 +694,7 @@ def zenodo_upload_repository_task(self, payload):
     resp = os.system(f"wget -O {zenodo_file} {download_url}")
     if resp != 0:
         gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"Cannot download: {download_url}")
-        self.update_state(state=states.FAILURE, meta={'message': f"Cannot download {download_url}"})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"Cannot download {download_url}"})
         return
     else:
         response = zenodo_upload_item(zenodo_file,payload['bucket_url'],payload['issue_id'],commit_fork,"repository")
@@ -765,7 +765,7 @@ def zenodo_upload_docker_task(self, payload):
             # Terminate ERROR
             msg = f"Looks like there's not a successful book build record for {fork_url}"
             gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-            self.update_state(state=states.FAILURE, meta={'message': msg})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
             return
 
         msg = f"Found docker image: \n {lut}"
@@ -777,7 +777,7 @@ def zenodo_upload_docker_task(self, payload):
         if not r['status']:
             msg = f"Cannot login to NeuroLibre private docker registry. \n {r['message']}"
             gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-            self.update_state(state=states.FAILURE, meta={'message': msg})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
             return
 
         msg = f"Pulling docker image: \n {lut['docker_image']}"
@@ -788,7 +788,7 @@ def zenodo_upload_docker_task(self, payload):
         if not r['status']:
             msg = f"Cannot pull the docker image \n {r['message']}"
             gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-            self.update_state(state=states.FAILURE, meta={'message': msg})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
             return
 
         msg = f"Exporting docker image: \n {lut['docker_image']}"
@@ -798,7 +798,7 @@ def zenodo_upload_docker_task(self, payload):
         if not r[0]['status']:
             msg = f"Cannot save the docker image \n {r[0]['message']}"
             gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-            self.update_state(state=states.FAILURE, meta={'message': msg})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
             return
 
         tar_file = r[1]
@@ -862,7 +862,7 @@ def zenodo_publish_task(self, payload):
     if response == "no-record-found":
         msg = "<br> :neutral_face: I could not find any Zenodo-related records on NeuroLibre servers. Maybe start with <code>roboneuro zenodo create buckets</code>?"
         gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg)
-        self.update_state(state=states.FAILURE, meta={'message': msg})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
         return
     else:
         # Confirm that all items are published.
@@ -882,7 +882,7 @@ def zenodo_publish_task(self, payload):
             response.append(f"\n Looks like there's a problem. {publish_status[1]} reproducibility assets are archived.")
             msg = "\n".join(response)
             gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], msg, False)
-            self.update_state(state=states.FAILURE, meta={'message': msg})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': msg})
 
 ### DUPLICATION FOR NOW, SAVING THE DAY.
 
@@ -973,7 +973,7 @@ def generate():
         tmp_log = write_html_to_temp_directory(payload['commit_hash'], issue_comment)
         body = "<p>&#129344; We ran into a problem building your book. Please download the log file attached and open in your web browser.</p>"
         send_email_with_html_attachment_celery(payload['email'], payload['mail_subject'], body, tmp_log)
-        self.update_state(state=states.FAILURE, meta={'message': f"FAILURE: Build for {owner}/{repo} at {payload['commit_hash']} has failed"})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': f"FAILURE: Build for {owner}/{repo} at {payload['commit_hash']} has failed"})
     else:
         #gh_template_respond(github_client,"success","Successfully built", payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"The next comment will forward the logs")
         #issue_comment = []
@@ -1068,7 +1068,7 @@ def preprint_build_pdf_draft(self, payload):
         gh_clone_repository(payload['repository_url'], target_path, depth=1)
     except Exception as e: 
         gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], str(e))
-        self.update_state(state=states.FAILURE, meta={'message': str(e)})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': str(e)})
         return
     # Crawl notebooks for the citation text and append it to paper.md, update bib.
     res = create_extended_pdf_sources(target_path, payload['issue_id'],payload['repository_url'])
@@ -1099,8 +1099,8 @@ def preprint_build_pdf_draft(self, payload):
             gh_create_comment(github_client, payload['review_repository'],payload['issue_id'],comment)
         except subprocess.CalledProcessError as e:
             gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"{e.output}")
-            self.update_state(state=states.FAILURE, meta={'message': e.output})
+            self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': e.output})
     else: 
         gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"{res['message']}")
-        self.update_state(state=states.FAILURE, meta={'message': res['message']})
+        self.update_state(state=states.FAILURE, meta={'exc_type':"NeuroLibre celery exception",'exc_message': "Custom",'message': res['message']})
 

api/preprint.py

@@ -522,11 +522,12 @@ def substitute_cite_commands(input_folder="content"):
                         # Embed citations in the cell's source code
                         for match in matches:
                             # Split the citations by comma and format them accordingly
-                            citations = match.split(',')
-                            try:
-                                formatted_citations = '; '.join([f'@{citation.strip()}' for citation in citations])
-                            except:
-                                pass
+                            if match:
+                                citations = match.split(',')
+                                try:
+                                    formatted_citations = '; '.join([f'@{citation.strip()}' for citation in citations])
+                                except:
+                                    pass
 
                             # Replace the original pattern with the formatted citations
                             cell['source'] = re.sub(r'\{cite:p\}`([^`]*)`', f'[{formatted_citations}]', cell['source'], count=1)
@@ -535,14 +536,15 @@ def substitute_cite_commands(input_folder="content"):
                     if matches:
                         # Embed citations in the cell's source code
                         for match in matches:
+                            if match:
                             # Split the citations by comma and format them accordingly
-                            citations = match.split(',')
-                            try:
-                                formatted_citations = '; '.join([f'@{citation.strip()}' for citation in citations])
-                            except:
-                                pass
-                            # Replace the original pattern with the formatted citations
-                            cell['source'] = re.sub(rf'\{{cite:t\}}`{match}`', f'{formatted_citations}', cell['source'], count=1)
+                                citations = match.split(',')
+                                try:
+                                    formatted_citations = '; '.join([f'@{citation.strip()}' for citation in citations])
+                                except:
+                                    pass
+                                # Replace the original pattern with the formatted citations
+                                cell['source'] = re.sub(rf'\{{cite:t\}}`{match}`', f'{formatted_citations}', cell['source'], count=1)
 
             # Export the notebook as Markdown
             filtered_paragraphs = extract_paragraphs_with_citations(notebook)