Add new endpoint

api/config/common.yaml

@@ -55,6 +55,16 @@ DATA_ROOT_PATH: "/DATA"
 # this is expected to be under the DATA_ROOT_PATH
 JB_ROOT_FOLDER: "book-artifacts"
 
-# Name of the folder that will contain MyST websites
+# Name of the folder that will contain published MyST websites
 # this is expected to be under the DATA_ROOT_PATH
-MYST_ROOT_FOLDER: "myst"
+MYST_PUBLISH_FOLDER: "myst"
+
+# Name of the folder that will contain MyST websites
+# source code. This is expected to be under the DATA_ROOT_PATH
+MYST_SOURCE_FOLDER: "myst_sources"
+
+# Container path where the MyST sources will be mounted
+CONTAINER_MYST_SOURCE_PATH: "/home/jovyan"
+
+# Container path where the data will be mounted (read-only)
+CONTAINER_MYST_DATA_PATH: "/home/jovyan/data"

api/neurolibre_celery_tasks.py

@@ -20,17 +20,20 @@
 import tempfile
 from celery.exceptions import Ignore
 from repo2data.repo2data import Repo2Data
-# from myst_libre.tools import JupyterHubLocalSpawner, MystMD
-# from myst_libre.rees import REES
-# from myst_libre.builders import MystBuilder
+from myst_libre.tools import JupyterHubLocalSpawner, MystMD
+from myst_libre.rees import REES
+from myst_libre.builders import MystBuilder
+from celery.schedules import crontab
+
 
 preview_config = load_yaml('config/preview.yaml')
 preprint_config = load_yaml('config/preprint.yaml')
 common_config  = load_yaml('config/common.yaml')
 
 config_keys = [
     'DOI_PREFIX', 'DOI_SUFFIX', 'JOURNAL_NAME', 'PAPERS_PATH', 'BINDER_REGISTRY',
-    'DATA_ROOT_PATH', 'JB_ROOT_FOLDER', 'GH_ORGANIZATION']
+    'DATA_ROOT_PATH', 'JB_ROOT_FOLDER', 'GH_ORGANIZATION', 'MYST_SOURCE_FOLDER', 'MYST_PUBLISH_FOLDER',
+    'CONTAINER_MYST_SOURCE_PATH', 'CONTAINER_MYST_DATA_PATH']
 globals().update({key: common_config[key] for key in config_keys})
 
 JB_INTERFACE_OVERRIDE = preprint_config['JB_INTERFACE_OVERRIDE']
@@ -115,13 +118,22 @@ def update_state(self, state, meta):
 
     def get_commit_hash(self):
         return format_commit_hash(self.payload['repo_url'], self.payload.get('commit_hash', 'HEAD'))
+
+    def get_dotenv_path(self):
+        return self.path_join(os.environ.get('HOME'),'full-stack-server','api')
 
     def path_join(self, *args):
         return os.path.join(*args)
 
     def join_data_root_path(self, *args):
         return self.path_join(DATA_ROOT_PATH, *args)
 
+    def join_myst_source_path(self, *args):
+        return self.path_join(DATA_ROOT_PATH, MYST_SOURCE_FOLDER, *args)
+
+    def join_myst_publish_path(self, *args):
+        return self.path_join(DATA_ROOT_PATH, MYST_PUBLISH_FOLDER, *args)
+
     def get_deposit_dir(self, *args):
         return self.path_join(get_deposit_dir(self.payload['issue_id']), *args)
 
@@ -133,6 +145,21 @@ def get_archive_dir(self, *args):
 Celery tasks START
 """
 
+# TODO:
+# @celery_app.on_after_configure.connect
+# def setup_periodic_tasks(sender, **kwargs):
+#     sender.add_periodic_task(
+#         crontab(hour=0, minute=0),  # Daily at midnight
+#         update_github_file.s('your_username/your_repo', 'path/to/file.txt', 'Daily update', 'New content')
+#     )
+
+# @celery_app.task
+# def update_github_file(repo_name, file_path, commit_message, content):
+#     github_client = Github(os.getenv('GH_BOT'))
+#     repo = github_client.get_repo(repo_name)
+#     contents = repo.get_contents(file_path)
+#     repo.update_file(contents.path, commit_message, content, contents.sha)
+
 @celery_app.task(bind=True)
 def sleep_task(self, seconds):
     """
@@ -1237,4 +1264,41 @@ def preprint_build_pdf_draft(self, payload):
             self.update_state(state=states.FAILURE, meta={'exc_type':f"{JOURNAL_NAME} celery exception",'exc_message': "Custom",'message': e.output})
     else:
         gh_template_respond(github_client,"failure",payload['task_title'], payload['review_repository'],payload['issue_id'],task_id,payload['comment_id'], f"{res['message']}")
-        self.update_state(state=states.FAILURE, meta={'exc_type':f"{JOURNAL_NAME} celery exception",'exc_message': "Custom",'message': res['message']})
+        self.update_state(state=states.FAILURE, meta={'exc_type':f"{JOURNAL_NAME} celery exception",'exc_message': "Custom",'message': res['message']})
+
+
+@celery_app.task(bind=True)
+def preview_build_myst_task(self, screening_dict):
+    task = BaseNeuroLibreTask(self, screening_dict)
+    task.start("Started MyST build.")
+    task.screening.commit_hash = task.screening.commit_hash or format_commit_hash(task.screening.repository_url,"HEAD")
+    task.screening.binder_hash = task.screening.binder_hash or task.screening.commit_hash
+
+    rees_resources = REES(dict(
+                  registry_url=BINDER_REGISTRY,
+                  gh_user_repo_name = f"{task.owner_name}/{task.repo_name}",
+                  gh_repo_commit_hash = task.screening.commit_hash,
+                  binder_image_tag = task.screening.binder_hash,
+                  dotenv = task.get_dotenv_path()))
+
+    hub = JupyterHubLocalSpawner(rees_resources,
+                             host_build_source_parent_dir = task.get_myst_source_path(),
+                             container_build_source_mount_dir = CONTAINER_MYST_SOURCE_PATH, #default
+                             host_data_parent_dir = DATA_ROOT_PATH, #optional
+                             container_data_mount_dir = CONTAINER_MYST_DATA_PATH)
+    # Spawn the JupyterHub
+    hub.spawn_jupyter_hub()
+    # hub.rees.
+
+    builder = MystBuilder(hub)
+    expected_publish_path = task.join_myst_publish_path(task.owner_name,task.repo_name,task.screening.commit_hash)
+    builder.setenv('BASE_URL',expected_publish_path)
+    builder.build()
+
+    expected_build_path = task.join_myst_source_path(task.owner_name,task.repo_name,task.screening.commit_hash,"_build")
+    if os.path.exists(expected_build_path):
+        # Copy myst artifacts to the expected path.
+        shutil.copytree(expected_build_path,task.get_myst_publish_path())
+        task.succeed(f"MyST build succeeded: https://{PREVIEW_SERVER}/myst/{task.owner_name}/{task.repo_name}/{task.screening.commit_hash}/_build/html/index.html")
+    else:
+        raise FileNotFoundError(f"Expected build path not found: {expected_build_path}")

api/neurolibre_preview_api.py

@@ -9,7 +9,7 @@
 from apispec import APISpec
 from apispec.ext.marshmallow import MarshmallowPlugin
 from github_client import *
-from neurolibre_celery_tasks import celery_app, sleep_task, preview_build_book_task, preview_build_book_test_task, preview_download_data
+from neurolibre_celery_tasks import celery_app, sleep_task, preview_build_book_task, preview_build_book_test_task, preview_download_data,preview_build_myst_task
 from celery.events.state import State
 from github import Github, UnknownObjectException
 from screening_client import ScreeningClient
@@ -204,31 +204,21 @@ def get_task_status_test(user,task_id):
 
 docs.register(get_task_status_test)
 
-# @app.route('/api/myst/build', methods=['POST'])
-# @preview_api.auth_required
-# @marshal_with(None,code=422,description="Cannot validate the payload, missing or invalid entries.")
-# @marshal_with(None,code=200,description="Accept text/eventstream for BinderHub build logs. Keepalive 30s.")
-# @doc(description='Endpoint for building MyST Markdown formatted articles.', tags=['MyST'])
-# @use_kwargs(MystBuildSchema())
-# def api_myst_build(user, id, repo_url, commit_hash, binder_hash):
-#     GH_BOT=os.getenv('GH_BOT')
-#     github_client = Github(GH_BOT)
-#     issue_id = id
-
-#     task_title = "MyST Build (Preview)"
-#     comment_id = gh_template_respond(github_client,"pending",task_title,REVIEW_REPOSITORY,issue_id)
-
-#     celery_payload = dict(repo_url=repo_url,
-#                           commit_hash=commit_hash,
-#                           binder_hash = binder_hash,
-#                           rate_limit=RATE_LIMIT,
-#                           binder_name=BINDER_NAME,
-#                           domain_name = BINDER_DOMAIN,
-#                           comment_id=comment_id,
-#                           issue_id=issue_id,
-#                           review_repository=REVIEW_REPOSITORY,
-#                           task_title=task_title)
-
-#     task_result = preview_build_myst_task.apply_async(args=[celery_payload])
-
-# docs.register(api_myst_build)
+@app.route('/api/myst/build', methods=['POST'])
+@preview_api.auth_required
+@marshal_with(None,code=422,description="Cannot validate the payload, missing or invalid entries.")
+@use_kwargs(MystBuildSchema())
+@doc(description='Endpoint for building myst formatted articles.', tags=['Myst'])
+def api_myst_build(user, id, repository_url, commit_hash=None, binder_hash=None):
+    """
+    This endpoint is to download data from GitHub (technical screening) requests.
+    """
+    extra_payload = dict(commit_hash=commit_hash, binder_hash=binder_hash)
+    screening = ScreeningClient(task_name="Build MyST article", 
+                                issue_id=id, 
+                                target_repo_url=repository_url,
+                                **extra_payload)
+    response = screening.start_celery_task(preview_build_myst_task)
+    return response
+
+docs.register(api_myst_build)

api/schema.py

@@ -43,7 +43,7 @@ class MystBuildSchema(Schema):
     Defines payload types and requirements for book build request.
     """
     id = fields.Integer(required=True,description="Issue number of the technical screening of this preprint.")
-    repo_url = fields.Str(required=True,description="Full URL of a NeuroLibre compatible repository to be used for building the book.")
+    repository_url = fields.Str(required=True,description="Full URL of a NeuroLibre compatible repository to be used for building the book.")
     commit_hash = fields.String(required=True,dump_default="HEAD",description="Commit SHA to be checked out for building the book. Defaults to HEAD.")
     binder_hash = fields.String(required=False,dump_default="HEAD",description="Commit SHA at which a binder image was built successfully.")
 

systemd/doc.md

@@ -175,11 +175,14 @@ sudo apt-get install apache2-utils
 htpasswd -c ~/.htpasswd <username>
 ```
 
-### Add to the docs
+### Important missing from docs
 
 sudo apt install python3.8-distutils
 
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
 export NVM_DIR="$([ -z "${XDG_CONFIG_HOME-}" ] && printf %s "${HOME}/.nvm" || printf %s "${XDG_CONFIG_HOME}/nvm")"
 [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
 
+sudo mkdir -p /var/run/neurolibre
+sudo chown ubuntu:www-data /var/run/neurolibre
+sudo chmod 770 /var/run/neurolibre