Merge branch 'release-16.4.0'

.github/workflows/build_docs.yml

@@ -0,0 +1,36 @@
+name: Build Github Pages Docs
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Cache dependences
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          # Unique cache key based on requirements.txt in docs directory
+          key: ${{ runner.os }}-pip-${{ hashFiles('docs/requirements.txt') }}-mkdocs-deps 
+          # Restore partial cache if exact match is not found
+          restore-keys: |
+            ${{ runner.os }}-pip-  
+
+      - name: Install mkdocs and dependencies
+        run: |
+            pip install mkdocs mkdocs-material mkdocstrings mkdocstrings-python mkdocs-open-in-new-tab
+
+      - name: Build and deploy mkdocs site to GitHub Pages
+        env:
+            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: mkdocs gh-deploy --force

.github/workflows/ci.yml

@@ -5,15 +5,18 @@ name: build
 
 on:
   push:
-    branches:
-      - develop
+    branches: [main, develop, 'GEN*', 'gen*']
 
   pull_request:
 
   release:
     types:
       - created
 
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
 
   test:
@@ -22,9 +25,9 @@ jobs:
       matrix:
         python-version: [3.8, 3.9]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -44,7 +47,7 @@ jobs:
       run: |
         pytest tests/ --cov=genie --cov=genie_registry --cov-report=html
     - name: Upload pytest test results
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: pytest-results-${{ matrix.python-version }}
         path: htmlcov
@@ -54,7 +57,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
 
   deploy:
@@ -64,9 +67,9 @@ jobs:
     permissions:
       id-token: write
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies
@@ -77,3 +80,51 @@ jobs:
       run: python -m build
     - name: Publish to pypi
       uses: pypa/gh-action-pypi-publish@release/v1
+
+
+  build-container:
+    needs: [test, lint]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Format tags as registry refs
+        id: registry_refs
+        env:
+          TAGS: ${{ steps.meta.outputs.json }}
+        run: |
+          echo tags=$(echo $TAGS | jq '.tags[] | "type=registry,ref=" + . + "_cache"| @text') >> $GITHUB_OUTPUT
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v5
+        if: github.event_name != 'pull_request'
+        with:
+          context: .
+          push: true
+          provenance: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: ${{ steps.registry_refs.outputs.tags }},mode=max
+          cache-to: ${{ steps.registry_refs.outputs.tags }},mode=max

.gitignore

@@ -26,3 +26,6 @@ data_guide.out
 Genie.Rproj
 build/
 dist/
+
+# MkDocs output
+docs_site/

Dockerfile

@@ -23,8 +23,8 @@ RUN apt-get update && apt-get install -y --allow-unauthenticated --no-install-re
 		python3-pip \
 		python3-dev \
 		git \
-		r-base \
-		r-base-dev \
+		r-base-core=4.3.3-1.2004.0 \
+		r-base-dev=4.3.3-1.2004.0 \
 		cmake \
 		curl \
 		# synapser client dependencies

README.md

@@ -6,11 +6,14 @@
 [![Docker Automated](https://img.shields.io/docker/automated/sagebionetworks/genie.svg?style=for-the-badge&logo=docker)](https://hub.docker.com/r/sagebionetworks/genie)
 [![GitHub CI](https://img.shields.io/github/actions/workflow/status/Sage-Bionetworks/Genie/ci.yml?branch=develop&style=for-the-badge&logo=github)](https://github.com/Sage-Bionetworks/Genie)
 
-
 ## Introduction
 
 This repository documents code used to gather, QC, standardize, and analyze data uploaded by institutes participating in AACR's Project GENIE (Genomics, Evidence, Neoplasia, Information, Exchange).
 
+## Documentation
+
+For more information about the AACR genie repository, [visit the GitHub Pages site.](https://sage-bionetworks.github.io/Genie/)
+
 ## Dependencies
 
 This package contains both R, Python and cli tools.  These are tools or packages you will need, to be able to reproduce these results:

docs/getting_started.md

@@ -0,0 +1,74 @@
+# Getting Started
+
+## Dependencies
+
+This package contains both R, Python and cli tools.  These are tools or packages you will need, to be able to reproduce these results:
+
+- Python >=3.8 or <3.10
+    - `pip install -r requirements.txt`
+- [bedtools](https://bedtools.readthedocs.io/en/latest/content/installation.html)
+- R 4.2.2
+    - `renv::install()`
+    - Follow instructions [here](https://r-docs.synapse.org/#note-for-windows-and-mac-users) to install synapser
+- [Java > 8](https://www.java.com/en/download/)
+    - For mac users, it seems to work better to run `brew install java`
+- [wget](https://www.gnu.org/software/wget/)
+    - For mac users, have to run `brew install wget`
+
+## Installation
+
+### PyPi
+
+The [aacrgenie](https://pypi.org/project/aacrgenie/) package is available from PyPI. It can be installed or upgraded with pip.
+
+```
+pip install aacrgenie
+```
+
+### Local
+
+Source code and development versions are available on Github. Installing from source:
+
+```
+git clone https://github.com/Sage-Bionetworks/Genie.git
+cd Genie
+```
+
+Install the packages locally.
+
+```
+pip install -e .
+pip install -r requirements.txt
+pip install -r requirements-dev.txt
+```
+
+You can stay on the main branch to get the latest stable release or check out the develop branch or a tagged revision:
+
+```
+git checkout <branch or tag>
+```
+
+
+## Configuration
+
+Configure the Synapse client to authenticate to Synapse.
+
+1. Create a Synapse [Personal Access token (PAT)](https://help.synapse.org/docs/Managing-Your-Account.2055405596.html#ManagingYourAccount-PersonalAccessTokens).
+1. Add a `~/.synapseConfig` file with the following information:
+
+    ```
+    [authentication]
+    authtoken = <PAT here>
+    ```
+
+1. OR set an environmental variable
+
+    ```
+    export SYNAPSE_AUTH_TOKEN=<PAT here>
+    ```
+
+1. Confirm you can log in to synapse in your terminal.
+
+    ```shell
+    synapse login
+    ```

docs/index.md

@@ -0,0 +1,15 @@
+![genie banner](https://raw.githubusercontent.com/Sage-Bionetworks/Genie/master/genie_banner.png)
+
+# AACR Project GENIE
+
+[![PyPi](https://img.shields.io/pypi/v/aacrgenie.svg?style=for-the-badge&label=PyPi&logo=PyPi)](https://pypi.org/project/aacrgenie)
+[![Docker Automated](https://img.shields.io/docker/automated/sagebionetworks/genie.svg?style=for-the-badge&logo=docker)](https://hub.docker.com/r/sagebionetworks/genie)
+[![GitHub CI](https://img.shields.io/github/actions/workflow/status/Sage-Bionetworks/Genie/ci.yml?branch=develop&style=for-the-badge&logo=github)](https://github.com/Sage-Bionetworks/Genie)
+
+## Introduction
+
+This repository documents code used to gather, QC, standardize, and analyze data uploaded by institutes participating in AACR's Project GENIE (Genomics, Evidence, Neoplasia, Information, Exchange).
+
+## Additional Documentation
+
+Visit the [Project Genie Data Portal](https://genie.synapse.org/) to learn more about our data

docs/reference/fileformats/assay.md

@@ -0,0 +1 @@
+::: genie_registry.assay

docs/reference/fileformats/bed.md

@@ -0,0 +1 @@
+::: genie_registry.bed

docs/reference/fileformats/clinical.md

@@ -0,0 +1 @@
+::: genie_registry.clinical

docs/reference/fileformats/cna.md

@@ -0,0 +1 @@
+::: genie_registry.cna

docs/reference/fileformats/fileformat.md

@@ -0,0 +1 @@
+::: genie.example_filetype_format

docs/reference/fileformats/maf.md

@@ -0,0 +1 @@
+::: genie_registry.maf

docs/reference/fileformats/mutations_in_cis.md

@@ -0,0 +1 @@
+::: genie_registry.mutationsInCis

docs/reference/fileformats/patient_retraction.md

@@ -0,0 +1 @@
+::: genie_registry.patientRetraction

docs/reference/fileformats/sample_retraction.md

@@ -0,0 +1 @@
+::: genie_registry.sampleRetraction

docs/reference/fileformats/seg.md

@@ -0,0 +1 @@
+::: genie_registry.seg

docs/reference/fileformats/structural_variant.md

@@ -0,0 +1 @@
+::: genie_registry.structural_variant

docs/reference/fileformats/vcf.md

@@ -0,0 +1 @@
+::: genie_registry.vcf

docs/reference/fileformats/workflow.md

@@ -0,0 +1 @@
+::: genie_registry.workflow

docs/reference/helper_modules/create_case_lists.md

@@ -0,0 +1 @@
+::: genie.create_case_lists

docs/reference/helper_modules/dashboard_table_updater.md

@@ -0,0 +1 @@
+::: genie.dashboard_table_updater

docs/reference/helper_modules/extract.md

@@ -0,0 +1 @@
+::: genie.extract

docs/reference/helper_modules/load.md

@@ -0,0 +1 @@
+::: genie.load

docs/reference/helper_modules/process_functions.md

@@ -0,0 +1 @@
+::: genie.process_functions

docs/reference/helper_modules/transform.md

@@ -0,0 +1 @@
+::: genie.transform

docs/reference/helper_modules/validate.md

@@ -0,0 +1 @@
+::: genie.validate

docs/reference/main_pipeline_commands/consortium_to_public.md

@@ -0,0 +1 @@
+::: bin.consortium_to_public

docs/reference/main_pipeline_commands/database_to_staging.md

@@ -0,0 +1 @@
+::: bin.database_to_staging

docs/reference/main_pipeline_commands/input_to_database.md

@@ -0,0 +1 @@
+::: bin.input_to_database

docs/reference/main_pipeline_steps/consortium_to_public.md

@@ -0,0 +1 @@
+::: genie.consortium_to_public

docs/reference/main_pipeline_steps/database_to_staging.md

@@ -0,0 +1 @@
+::: genie.database_to_staging

docs/reference/main_pipeline_steps/input_to_database/input_to_database.md

@@ -0,0 +1 @@
+::: genie.input_to_database

docs/reference/main_pipeline_steps/input_to_database/process_mutation.md

@@ -0,0 +1 @@
+::: genie.process_mutation

docs/reference/main_pipeline_steps/input_to_database/to_retract.md

@@ -0,0 +1 @@
+::: genie.toRetract

docs/reference/main_pipeline_steps/input_to_database/write_invalid_reasons.md

@@ -0,0 +1 @@
+::: genie.write_invalid_reasons

docs/requirements.txt

@@ -0,0 +1,5 @@
+mkdocs<=1.6.0
+mkdocs-material<=9.5.23
+mkdocs-open-in-new-tab<=1.0.3
+mkdocstrings<=0.25.1
+mkdocstrings-python<=1.10.2

docs/tutorials/local_file_validation.md

@@ -0,0 +1,20 @@
+# Local File Validation
+
+One of the features of the `aacrgenie` package is that is provides a local validation tool that GENIE data contributors and install and use to validate their files locally prior to uploading to Synapse.
+
+```
+pip install aacrgenie
+genie -v
+```
+
+This will install all the necessary components for you to run the validator locally on all of your files, including the Synapse client.  Please view the help to see how to run to validator.
+
+```
+genie validate -h
+```
+
+Validate a file
+
+```
+genie validate data_clinical_supp_SAGE.txt SAGE
+```