Merge branch 'MHubAI:main' into m-gc-node21-baseline

.github/schemas/meta.schema.json

@@ -235,7 +235,7 @@
                         "training": {
                             "type": "string",
                             "description": "The training method of the model.",
-                            "enum": ["supervised", "semi-supervised", "unsupervised", "other"]
+                            "enum": ["supervised", "semi-supervised", "unsupervised", "other", "algorithmic"]
                         },
                         "cmpapproach": {
                             "type": "string",
@@ -298,6 +298,7 @@
                 },
                 "version": {
                     "type": "string",
+                    "pattern": "^\\d+(\\.\\d+)*$",
                     "description": "The version of the model."
                 },
                 "devteam": {
@@ -312,9 +313,17 @@
                     "type": "object",
                     "properties": {
                         "weights": {
-                            "type": "string",
                             "description": "The date of model weights (yyy-mm-dd).",
-                            "format": "date"
+                            "anyOf": [
+                                {
+                                  "type": "string",
+                                  "format": "date"
+                                },
+                                {
+                                  "type": "string",
+                                  "maxLength": 0
+                                }
+                              ]
                         },
                         "code": {
                             "type": "string",

.github/schemas/testmodel.schema.json

@@ -0,0 +1,77 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "sample": {
+      "type": "object",
+      "properties": {
+        "idc_version": {
+          "type": ["string", "number"]
+        },
+        "data": {
+          "type": "array",
+          "minItems": 1,
+          "items": {
+            "type": "object",
+            "properties": {
+              "SeriesInstanceUID": {
+                "type": "string",
+                "pattern": "^[\\d\\.]+$"
+              },
+              "aws_url": {
+                "type": "string",
+                "pattern": "^s3://[\\w\\-/]+/\\*$"
+              },
+              "url": {
+                "type": "string",
+                "format": "uri"
+              },
+              "description": {
+                "type": "string"
+              },
+              "path": {
+                "type": "string",
+                "pattern": "^[^\\/][\\w\\-\\.\\/]+$"
+              }
+            },
+            "oneOf": [
+              {
+                "required": [
+                  "SeriesInstanceUID",
+                  "aws_url",
+                  "path"
+                ]
+              },
+              {
+                "required": [
+                  "url",
+                  "path"
+                ]
+              }
+            ]
+          }
+        }
+      },
+      "required": [
+        "idc_version",
+        "data"
+      ]
+    },
+    "reference": {
+      "type": "object",
+      "properties": {
+        "url": {
+          "type": "string",
+          "format": "uri"
+        }
+      },
+      "required": [
+        "url"
+      ]
+    }
+  },
+  "required": [
+    "sample",
+    "reference"
+  ]
+}

.github/scripts/comment_check.py

@@ -0,0 +1,81 @@
+import sys, os, yaml, json, jsonschema
+
+YAML_TEST_DEFINITION_SCHEMA_FILE = ".github/schemas/testmodel.schema.json"
+
+def extract_yaml_test_definition(comment: str):
+
+  # find a code block starting with ```yaml and ending with ```
+  start = comment.find("```yaml")
+  end = comment.find("```", start + 1)
+  if start == -1 or end == -1:
+    raise Exception("No YAML code block found in comment")
+
+  # extract the code block
+  yaml_code = comment[start:end]
+
+  # remove the code block markers
+  yaml_code = yaml_code.replace("```yaml", "").strip()
+
+  return yaml_code
+
+def validate_yaml_test_definition(yaml_str: str):
+
+  # load yaml into dict
+  test_definition = yaml.safe_load(yaml_str)
+
+  # load schema
+  with open(YAML_TEST_DEFINITION_SCHEMA_FILE, "r") as f:
+    schema = json.load(f)
+
+  # validate
+  jsonschema.validate(test_definition, schema)
+
+
+def set_action_output(output_name, value) :
+    """Sets the GitHub Action output.
+
+    Keyword arguments:
+    output_name - The name of the output
+    value - The value of the output
+    """
+    if "GITHUB_OUTPUT" in os.environ :
+        with open(os.environ["GITHUB_OUTPUT"], "a") as f :
+            print("{0}={1}".format(output_name, value), file=f)
+
+
+if __name__ == "__main__":
+
+  try:
+    # get comment body from first argument
+    comment = sys.argv[1]
+
+    # print comment
+    print(f"Comment ----------------------")
+    print(comment)
+    print()
+
+    # extract yaml test definition
+    yaml_str = extract_yaml_test_definition(comment)
+
+    # validate yaml test definition
+    validate_yaml_test_definition(yaml_str)
+
+    # print yaml
+    print(f"Test Definition --------------")
+    print(yaml_str)
+    print()
+
+    # print success message
+    print("YAML test definition is valid")
+
+    # set environment variable for following steps
+    set_action_output("test_report", "passed")
+
+  except Exception as e:
+    # set environment variable for following steps
+    set_action_output("test_report", "failed")
+
+    # print error message
+    print("YAML test definition is invalid")
+    print(e)
+

.github/scripts/mhub_check.py

@@ -34,9 +34,12 @@
     # check folder structure
     utils.validateModelFolder(base='models', model_name=model_name)
 
-    # check meta.json
+    # check meta.json (schema)
     utils.validateModelMetaJson(model_meta_json_file=os.path.join('models', model_name, 'meta.json'))
 
+    # check additional requirements for meta.json
+    utils.validateModelMetaJson_modelName(model_meta_json_file=os.path.join('models', model_name, 'meta.json'), model_name=model_name)
+
     # validate dockerfile
     utils.validateDockerfile(base='models', model_name=model_name)
 
@@ -52,6 +55,7 @@
     print()
     print("---------------- CHECK FAILED ----------------")
     print("An unexpected error occured during compliance checks.")
+    print(str(e))
     print()
     sys.exit(1)
 
@@ -60,4 +64,4 @@
 print("---------------- CHECK PASSED ----------------")
 print("All compliance checks passed.")
 print("Note: compliance checks are a beta feature. Passing all automated compliance checks does not guarantee that your model is compliant with the MHub standard. We will now perform a manual review of your model. Testing your model on a public dataset is obligatory.")
-print()
+print()

.github/scripts/utils.py

@@ -110,6 +110,16 @@ def validateModelMetaJson(model_meta_json_file: str):
     except jsonschema.ValidationError as e:
         raise MHubComplianceError(f"Model meta json is not compliant with the schema: {e.message}", DocuRef.MODEL_META_JSON)
 
+def validateModelMetaJson_modelName(model_meta_json_file: str, model_name: str):
+
+    # load model meta json
+    with open(model_meta_json_file, "r") as f:
+        model_meta_json = json.load(f)
+
+    # check that the model name is correct
+    if model_meta_json["name"] != model_name:
+        raise MHubComplianceError(f"Model name in meta.json does not match model name in folder structure: {model_meta_json['name']} != {model_name}", DocuRef.MODEL_META_JSON)
+
 def validateDockerfile(base: str, model_name: str):
 
     # get dockerfile path

.github/workflows/submission_review.yml

@@ -0,0 +1,77 @@
+name: MHub Contribution Magic Keywords
+
+on:
+  issue_comment:                                     
+    types: [created, edited]
+
+permissions: 
+  pull-requests: write
+
+jobs:
+  request_review:
+    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/review')}}
+    name: Request Review 
+    runs-on: [ubuntu-latest]
+
+    steps:
+      - name: Add Request Review Label
+        uses: actions-ecosystem/action-add-labels@v1
+        with:
+          labels: REQUEST REVIEW
+
+  request_test:
+    if: ${{ github.event.issue.pull_request && startsWith(github.event.comment.body, '/test') }}
+    name: Request Test 
+    runs-on: [ubuntu-latest]
+
+    steps:
+
+      # Checkout the latest code from the repo
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      # Setup which version of Python to use
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+
+      # install python dependencies
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install jsonschema PyYAML 
+
+      # Display the Python version being used
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+
+      - name: Run check comment script
+        id: check_comment
+        env:
+          COMMENT: ${{ github.event.comment.body }}
+        run: python .github/scripts/comment_check.py "$COMMENT"
+
+      - name: Add TEST REQUESTED Label
+        uses: actions-ecosystem/action-add-labels@v1
+        if: ${{ steps.check_comment.outputs.test_report == 'passed' }}
+        with:
+          labels: TEST REQUESTED
+
+      - name: Remove INVALID TEST REQUEST Label
+        uses: actions-ecosystem/action-remove-labels@v1
+        if: ${{ steps.check_comment.outputs.test_report == 'passed' }}
+        with:
+          labels: INVALID TEST REQUEST
+
+      - name: Remove TEST REQUESTED Label
+        uses: actions-ecosystem/action-remove-labels@v1
+        if: ${{ steps.check_comment.outputs.test_report == 'failed' }}
+        with:
+          labels: TEST REQUESTED
+
+      - name: Add INVALID TEST REQUEST Label
+        uses: actions-ecosystem/action-add-labels@v1
+        if: ${{ steps.check_comment.outputs.test_report == 'failed' }}
+        with:
+          labels: INVALID TEST REQUEST

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 MHub.ai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

base/bin/mhub.version

@@ -15,7 +15,7 @@ echo -e "segdb==${SEGDB}"
 if [ -d "$MODEL_FOLDER" ]; then
 
     # model repo commit
-    MODEL=$(git -C $MODEL_FOLDER show -s | grep commit | cut -d" " -f 2)
+    { MODEL=$(< /app/buildutils/model_commit_hash.txt); } 2> /dev/null
 
     echo -e "model==${MODEL}"
     echo -e "+++"
@@ -24,8 +24,9 @@ if [ -d "$MODEL_FOLDER" ]; then
     for mdir in /app/models/* ; do
         MVERSION=$(jq -r '.details.version' ${mdir}/meta.json)
         echo -e "mhub.$(basename ${mdir})==${MVERSION}"
-        echo -e "+++"
     done
+
+    echo -e "+++"
 else
     echo -e "+++"
     echo -e " "
@@ -36,4 +37,21 @@ fi
 # pip freeze without segdb and mhubio (already on top of the lists,
 # since for now they are commits). Ideally, this should return only pip versions
 # (although some package might be installed from git by contributors)
-pip freeze | grep -v "segdb" | grep -v "mhubio"
+pip freeze | grep -v "segdb" | grep -v "mhubio"
+
+# collect additional information on installed system dependencies.
+#  to allow contributors to include additional dependencies, we should use a environment variable or a file instead.
+
+# versions of python, pip, plastimatch, jq, git, libopenslide-dev, libvips-dev, dcm2niix, ffmpeg, libsm6, libxext6
+# echo -e "+++"
+# echo -e "python==$(python3 --version 2>&1)"
+# echo -e "pip==$(pip --version 2>&1)"
+# echo -e "plastimatch==$(plastimatch --version 2>&1)"
+# echo -e "jq==$(jq --version 2>&1)"
+# echo -e "git==$(git --version 2>&1)"
+# echo -e "libopenslide-dev==$(dpkg -s libopenslide-dev | grep Version)"
+# echo -e "libvips-dev==$(dpkg -s libvips-dev | grep Version)"
+# echo -e "dcm2niix==$(dcm2niix -h | grep "dcm2niiX version" | cut -d"v" -f3)"
+# echo -e "ffmpeg==$(ffmpeg -version 2>&1 | grep ffmpeg | cut -d" " -f3)"
+# echo -e "libsm6==$(dpkg -s libsm6 | grep Version)"
+# echo -e "libxext6==$(dpkg -s libxext6 | grep Version)"