Feat/json schema workflow (#399)

* Draft new workflow * Add utility script * Update Makefile * Update casing to be consistent * Update Makefile * Revise workflow file * Make workflow consistent with changes * Also update if workflow updates * Lint * Combine and rename main build and test workflow * Update * Add cli tool install to step * Revise cond and test * Rebuild NF.jsonld, json * Update workflow to propagate changes to Synapse schema registry as well * Update token ref * Lint * Add term * Rebuild NF.jsonld, json --------- Co-authored-by: nf-osi[bot] <[email protected]>
nf-osi · Mar 7, 2024 · 30291f1 · 30291f1
1 parent bb6dca2
commit 30291f1
Show file tree

Hide file tree

Showing 14 changed files with 492 additions and 323 deletions.
diff --git a/.github/workflows/schematic-ci.yml → .github/workflows/main-ci.yml b/.github/workflows/schematic-ci.yml → .github/workflows/main-ci.yml
@@ -1,4 +1,4 @@
-name: schematic-ci
+name: main-ci
 
 on:
 
@@ -13,7 +13,6 @@ on:
 jobs:
   build:
     runs-on: ubuntu-20.04
-    if: ${{ !contains(github.event.head_commit.message, '[skip schematic]') }}
 
     steps:
       - uses: actions/checkout@v4
@@ -28,20 +27,30 @@ jobs:
           bash < <(curl -s https://raw.githubusercontent.com/babashka/babashka/master/install)
           git clone --depth 1 https://github.com/anngvu/retold.git
           make
+  
+      - name: Install tools and regenerate registered jsons for Superdataset, PortalDataset, etc
+        run: |
+          pip install linkml
+          npm install -g json-dereference-cli
+          make Dataset
+          make Superdataset
+          make PortalDataset
+          make PortalStudy
     
       - name: Commit files
         run: |
           git config --local user.email "[email protected]"
           git config --local user.name "nf-osi[bot]"
-          git add "NF.jsonld"
-          git commit -m "Build jsonld"
+          git add .
+          git commit -m "Rebuild NF.jsonld, json"
     
       - name: Push changes
         uses: ad-m/github-push-action@master
         with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          branch: ${{ github.event.pull_request.head.ref }}
 
+
   # Additionally test PRs
   test:
     name: Test with schematic

diff --git a/.github/workflows/propagate.yml b/.github/workflows/propagate.yml
@@ -0,0 +1,58 @@
+# When updates are merged into main, propagate data model updates to various places:
+# 1. dcc-site uses assay, etc. for drop-down options; triggers a workflow in dcc-site that creates a PR to pull in new terms.
+# 2. The Synapse schema registry has schemas for the NF entities such as portal studies and datasets. Push updated JSON schemas to the Synapse registry.
+# 
+# Highly related but to be handled by separate workflow:
+# Update data model version in DCA-prod for official releases (instead of every merge into main)
+
+name: Propagate data model updates
+
+on:
+  push:
+    branches: 
+      - main
+    paths:
+      - NF.jsonld
+
+env:
+  REPO: nf-osi/dcc-site
+  REF: refs/heads/main
+  SYNAPSE_AUTH_TOKEN: ${{ secrets.DATA_MODEL_SERVICE }}
+
+jobs:
+  trigger-sync:
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Dispatch update workflow for dcc-site
+        uses: benc-uk/workflow-dispatch@v1
+        with:
+          workflow: Handle data model update
+          repo: ${{ env.REPO }}
+          ref: ${{ env.REF }}
+          token: ${{ secrets.SERVICE_TOKEN }}
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - name: Push to Synapse schema registry only JSON schemas that changed
+        run: |
+          CHANGED=$(git diff --name-only HEAD HEAD~1 registered-json-schemas)
+          for FILE in $CHANGED
+          do
+            REQUEST_BODY=$(jq '. + { concreteType: "org.sagebionetworks.repo.model.schema.CreateSchemaRequest", dryRun: false }' $FILE)
+            RESPONSE=$(curl -X POST https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/start \
+            -H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN" \
+            -H "Content-Type: application/json" \
+            -d "$REQUEST_BODY")
+            
+            # Will error if token not within response
+            TOKEN=$( echo $RESPONSE | jq -e -r '.token')
+            sleep 1
+            STATUS=$(curl "https://repo-prod.prod.sagebase.org/repo/v1/schema/type/create/async/get/$TOKEN" \
+            -H "Authorization: Bearer $SYNAPSE_AUTH_TOKEN")
+
+            echo $STATUS | jq -e '.newVersionInfo'
+          done
+
diff --git a/.github/workflows/trigger-sync-dcc-site.yml b/.github/workflows/trigger-sync-dcc-site.yml
diff --git a/Makefile b/Makefile
@@ -29,7 +29,6 @@ PortalDataset:
 	@echo "--- Saved registered-json-schemas/PortalDataset.json ---"
 
 
-
 # yq '.slots |= with_entries(select(.value.in_subset[] == "portal"))' modules/props.yaml > relevant_props.yaml
 PortalStudy:
 	yq eval-all '. as $$item ireduce ({}; . * $$item )' modules/Data/Data.yaml modules/DCC/Portal.yaml modules/Other/Organization.yaml > relevant_enums.yaml
@@ -41,3 +40,10 @@ PortalStudy:
 	rm tmp.json
 	@echo "--- Saved registered-json-schemas/PortalStudy.json ---"
 
+Dataset:
+	enum=$$(./utils/enumerate.sh assay); jq --argjson enum "$$enum" '.properties.assay.enum = $$enum' registered-json-schemas/Dataset.json > temp.json && mv temp.json registered-json-schemas/Dataset.json
+
+
+Superdataset:
+	enum=$$(./utils/enumerate.sh assay); jq --argjson enum "$$enum" '.properties.assay.enum = $$enum' registered-json-schemas/Superdataset.json > temp.json && mv temp.json registered-json-schemas/Superdataset.json
+
diff --git a/NF.jsonld b/NF.jsonld
@@ -4890,6 +4890,8 @@
       "@id" : "bts:ChildBehaviorChecklistforAges1.5-5"
     }, {
       "@id" : "bts:ChildBehaviorChecklistforAges6-18"
+    }, {
+      "@id" : "bts:CODEX"
     }, {
       "@id" : "bts:Corsiblocks"
     }, {
@@ -5241,6 +5243,8 @@
       "@id" : "bts:ChildBehaviorChecklistforAges1.5-5"
     }, {
       "@id" : "bts:ChildBehaviorChecklistforAges6-18"
+    }, {
+      "@id" : "bts:CODEX"
     }, {
       "@id" : "bts:Corsiblocks"
     }, {
@@ -5784,6 +5788,8 @@
       "@id" : "bts:MGIT-series"
     }, {
       "@id" : "bts:VarioskanLUX"
+    }, {
+      "@id" : "bts:Panorama1.0T"
     }, {
       "@id" : "bts:NotApplicable"
     } ],
@@ -18086,6 +18092,17 @@
     },
     "sms:displayName" : "6",
     "sms:required" : "sms:false"
+  }, {
+    "@id" : "bts:CODEX",
+    "@type" : "rdfs:Class",
+    "rdfs:comment" : "TBD",
+    "rdfs:label" : "CODEX",
+    "rdfs:subClassOf" : [ ],
+    "schema:isPartOf" : {
+      "@id" : "http://schema.biothings.io/"
+    },
+    "sms:displayName" : "CODEX",
+    "sms:required" : "sms:false"
   }, {
     "@id" : "bts:MissouriUniversityofScienceandTechnology",
     "@type" : "rdfs:Class",
@@ -21298,6 +21315,17 @@
     },
     "sms:displayName" : "University of Texas Health Science Center, Tyler",
     "sms:required" : "sms:false"
+  }, {
+    "@id" : "bts:Panorama1.0T",
+    "@type" : "rdfs:Class",
+    "rdfs:comment" : "TBD",
+    "rdfs:label" : "Panorama1.0T",
+    "rdfs:subClassOf" : [ ],
+    "schema:isPartOf" : {
+      "@id" : "http://schema.biothings.io/"
+    },
+    "sms:displayName" : "Panorama 1.0T",
+    "sms:required" : "sms:false"
   }, {
     "@id" : "bts:LI-COROdysseyCLx",
     "@type" : "rdfs:Class",

diff --git a/modules/Assay/Assay.yaml b/modules/Assay/Assay.yaml
@@ -50,6 +50,8 @@ enums:
           "A standardized measure designed to detect behavioral and emotional problems in children and adolescents. 
           It is completed by the parent/caretaker who spends the most time with the child and provides ratings for 20 competence and 120 problem items. [ NCI ]"
         meaning: http://purl.obolibrary.org/obo/NCIT_C165711
+      CODEX:
+        description: CODEX imaging.
       Corsi blocks:
         description: >
           A visuospatial counterpart to the verbal-memory span task (Milner, 1971). Over the years, it has frequently been used to assess visuospatial short-term memory performance in adults 

diff --git a/modules/Assay/Platform.yaml b/modules/Assay/Platform.yaml
@@ -214,5 +214,7 @@ enums:
       Varioskan LUX: 
         description: Microplate reader for ELISAs, etc.
         source: https://www.thermofisher.com/us/en/home/life-science/lab-equipment/microplate-instruments/plate-readers/models/varioskan.html
+      Panorama 1.0T:
+        description: A conventional MRI scanner from Philips.
       Not Applicable:
         description: ''