From 844151e64c32e80635f699ce00f19b70b16120c5 Mon Sep 17 00:00:00 2001 From: Liam Keegan Date: Fri, 25 Oct 2024 15:23:09 +0200 Subject: [PATCH] Implement upload of results - runner creates three folders where the script runs, and uploads their contents as zipfiles: - `/user_results`: provided to user - `/trusted_user_results`: provided to user if they have "full results" enabled by an admin - `/admin_results`: only visible to admins - admin results are always uploaded, included for failed runs (for debugging) - user results are only provided if the run was successful - add `PREDICTCR_RUNNER_SCRIPT_DIR` env var to docker compose to specify where script & model can be found - add /admin/result endpoint to download admin results zipfile - switch from micromamba to miniconda docker image - reticulate doesn't seem to support micromamba, at least within docker - re-enable runner docker image CI - update runner README.md - resolves #6 - resolves #43 Also refactor admin views using tabs --- .github/workflows/ci.yml | 40 +++++------ backend/src/predicTCR_server/app.py | 47 +++++++++++-- backend/src/predicTCR_server/model.py | 44 +++++++----- backend/tests/test_app.py | 4 +- frontend/src/components/SamplesTable.vue | 16 ++++- frontend/src/utils/api-client.ts | 8 +++ frontend/src/views/AdminView.vue | 85 ++++++++++++++---------- runner/Dockerfile | 19 +++--- runner/README.md | 28 +++++--- runner/docker-compose.yml | 4 +- runner/env.yaml | 9 +-- runner/script/script.sh | 28 ++++++++ runner/scripts/script.sh | 17 ----- runner/src/predicTCR_runner/main.py | 8 +-- runner/src/predicTCR_runner/runner.py | 74 ++++++++++++++++----- 15 files changed, 287 insertions(+), 144 deletions(-) create mode 100644 runner/script/script.sh delete mode 100644 runner/scripts/script.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb5712a..17b4fcd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -110,23 +110,23 @@ jobs: steps: - uses: actions/checkout@v4 - run: docker compose build - # - uses: docker/login-action@v3 - # with: - # registry: ghcr.io - # username: ${{ github.actor }} - # password: ${{ secrets.GITHUB_TOKEN }} - # if: github.event_name == 'push' && github.ref == 'refs/heads/main' - # - run: | - # echo $PREDICTCR_DOCKER_IMAGE_TAG - # docker compose build - # docker compose push - # if: github.event_name == 'push' && github.ref == 'refs/heads/main' - # env: - # PREDICTCR_DOCKER_IMAGE_TAG: ${{ github.sha }} - # - run: | - # echo $PREDICTCR_DOCKER_IMAGE_TAG - # docker compose build - # docker compose push - # if: github.event_name == 'push' && github.ref == 'refs/heads/main' - # env: - # PREDICTCR_DOCKER_IMAGE_TAG: "latest" + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + - run: | + echo $PREDICTCR_DOCKER_IMAGE_TAG + docker compose build + docker compose push + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + PREDICTCR_DOCKER_IMAGE_TAG: ${{ github.sha }} + - run: | + echo $PREDICTCR_DOCKER_IMAGE_TAG + docker compose build + docker compose push + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + env: + PREDICTCR_DOCKER_IMAGE_TAG: "latest" diff --git a/backend/src/predicTCR_server/app.py b/backend/src/predicTCR_server/app.py index 047ef6b..be4c679 100644 --- a/backend/src/predicTCR_server/app.py +++ b/backend/src/predicTCR_server/app.py @@ -211,7 +211,10 @@ def result(): if not user_sample.has_results_zip: logger.info(f" -> sample {sample_id} found but no results available") return jsonify(message="No results available"), 400 - requested_file = user_sample.result_file_path() + if current_user.full_results: + requested_file = user_sample.trusted_user_result_file_path() + else: + requested_file = user_sample.user_result_file_path() if not requested_file.is_file(): logger.info(f" -> file {requested_file} not found") return jsonify(message="Results file not found"), 400 @@ -243,6 +246,26 @@ def add_sample(): return jsonify(sample=new_sample) return jsonify(message=error_message), 400 + @app.route("/api/admin/result", methods=["POST"]) + @jwt_required() + def admin_result(): + if not current_user.is_admin: + return jsonify(message="Admin account required"), 400 + sample_id = request.json.get("sample_id", None) + logger.info( + f"User {current_user.email} requesting admin results for sample {sample_id}" + ) + user_sample = db.session.get(Sample, sample_id) + if user_sample is None: + logger.info(f" -> sample {sample_id} not found") + return jsonify(message="Sample not found"), 400 + requested_file = user_sample.admin_result_file_path() + if not requested_file.is_file(): + logger.info(f" -> file {requested_file} not found") + return jsonify(message="Results file not found"), 400 + logger.info(f"Returning file {requested_file}") + return flask.send_file(requested_file, as_attachment=True) + @app.route("/api/admin/samples", methods=["GET"]) @jwt_required() def admin_all_samples(): @@ -258,7 +281,9 @@ def admin_resubmit_sample(sample_id: int): sample = db.session.get(Sample, sample_id) if sample is None: return jsonify(message="Sample not found"), 404 - sample.result_file_path().unlink(missing_ok=True) + sample.user_result_file_path().unlink(missing_ok=True) + sample.trusted_user_result_file_path().unlink(missing_ok=True) + sample.admin_result_file_path().unlink(missing_ok=True) sample.has_results_zip = False sample.status = Status.QUEUED db.session.commit() @@ -381,10 +406,14 @@ def runner_result(): logger.info(" -> missing success key") return jsonify(message="Missing key: success=True/False"), 400 success = success.lower() == "true" - zipfile = request.files.to_dict().get("file", None) - if success is True and zipfile is None: + user_zipfile = request.files.to_dict().get("user_results", None) + trusted_user_zipfile = request.files.to_dict().get("trusted_user_results", None) + admin_zipfile = request.files.to_dict().get("admin_results", None) + if success is True and (user_zipfile is None or admin_zipfile is None): logger.info(" -> missing zipfile") - return jsonify(message="Result has success=True but no file"), 400 + return jsonify( + message="Result has success=True but a result zipfile is missing" + ), 400 runner_hostname = form_as_dict.get("runner_hostname", "") logger.info( f"Job '{job_id}' uploaded result for '{sample_id}' from runner {current_user.email} / {runner_hostname}" @@ -393,7 +422,13 @@ def runner_result(): if error_message != "": logger.info(f" -> error message: {error_message}") message, code = process_result( - int(job_id), int(sample_id), success, error_message, zipfile + int(job_id), + int(sample_id), + success, + error_message, + user_zipfile, + trusted_user_zipfile, + admin_zipfile, ) return jsonify(message=message), code diff --git a/backend/src/predicTCR_server/model.py b/backend/src/predicTCR_server/model.py index 612f7d0..8f7e06c 100644 --- a/backend/src/predicTCR_server/model.py +++ b/backend/src/predicTCR_server/model.py @@ -95,8 +95,14 @@ def input_h5_file_path(self) -> pathlib.Path: def input_csv_file_path(self) -> pathlib.Path: return self.base_path() / "input.csv" - def result_file_path(self) -> pathlib.Path: - return self.base_path() / "result.zip" + def user_result_file_path(self) -> pathlib.Path: + return self.base_path() / "user_results.zip" + + def trusted_user_result_file_path(self) -> pathlib.Path: + return self.base_path() / "trusted_user_results.zip" + + def admin_result_file_path(self) -> pathlib.Path: + return self.base_path() / "admin_results.zip" @dataclass @@ -191,37 +197,43 @@ def process_result( sample_id: int, success: bool, error_message: str, - result_zip_file: FileStorage | None, + user_result_zip_file: FileStorage | None, + trusted_user_result_zip_file: FileStorage | None, + admin_result_zip_file: FileStorage | None, ) -> tuple[str, int]: sample = db.session.get(Sample, sample_id) if sample is None: logger.warning(f" --> Unknown sample id {sample_id}") return f"Unknown sample id {sample_id}", 400 + sample.base_path().mkdir(parents=True, exist_ok=True) job = db.session.get(Job, job_id) if job is None: logger.warning(f" --> Unknown job id {job_id}") return f"Unknown job id {job_id}", 400 job.timestamp_end = timestamp_now() - if success is False: - sample.has_results_zip = False - sample.status = Status.FAILED + if success: + job.status = Status.COMPLETED + else: job.status = Status.FAILED job.error_message = error_message - db.session.commit() - return "Result processed", 200 + db.session.commit() if sample.has_results_zip: logger.warning(f" --> Sample {sample_id} already has results") - job.status = Status.COMPLETED - db.session.commit() return f"Sample {sample_id} already has results", 400 - if result_zip_file is None: - logger.warning(" --> No zipfile") - return "Zip file missing", 400 - sample.result_file_path().parent.mkdir(parents=True, exist_ok=True) - result_zip_file.save(sample.result_file_path()) + if admin_result_zip_file is not None: + admin_result_zip_file.save(sample.admin_result_file_path()) + if success is False: + sample.has_results_zip = False + sample.status = Status.FAILED + db.session.commit() + return "Result processed", 200 + if user_result_zip_file is None or trusted_user_result_zip_file is None: + logger.warning(" --> Missing user result zipfile") + return "User result zip file missing", 400 + user_result_zip_file.save(sample.user_result_file_path()) + trusted_user_result_zip_file.save(sample.trusted_user_result_file_path()) sample.has_results_zip = True sample.status = Status.COMPLETED - job.status = Status.COMPLETED db.session.commit() return "Result processed", 200 diff --git a/backend/tests/test_app.py b/backend/tests/test_app.py index 39d2676..7279674 100644 --- a/backend/tests/test_app.py +++ b/backend/tests/test_app.py @@ -221,7 +221,9 @@ def _upload_result(client, result_zipfile: pathlib.Path, job_id: int, sample_id: "job_id": job_id, "sample_id": sample_id, "success": True, - "file": (io.BytesIO(f.read()), result_zipfile.name), + "user_results": (io.BytesIO(f.read()), result_zipfile.name), + "trusted_user_results": (io.BytesIO(f.read()), result_zipfile.name), + "admin_results": (io.BytesIO(f.read()), result_zipfile.name), }, headers=headers, ) diff --git a/frontend/src/components/SamplesTable.vue b/frontend/src/components/SamplesTable.vue index 50f6840..78e9ab6 100644 --- a/frontend/src/components/SamplesTable.vue +++ b/frontend/src/components/SamplesTable.vue @@ -16,6 +16,7 @@ import { download_input_csv_file, download_input_h5_file, download_result, + download_admin_result, logout, } from "@/utils/api-client"; import type { Sample } from "@/utils/types"; @@ -82,7 +83,11 @@ function delete_current_sample() { Actions - + {{ sample["id"] }} {{ new Date(sample["timestamp"] * 1000).toLocaleDateString("de-DE") @@ -108,7 +113,14 @@ function delete_current_sample() { -