Merge main02

.gitattributes

@@ -0,0 +1,6 @@
+#
+# https://help.github.com/articles/dealing-with-line-endings/
+#
+# These are explicitly windows files and should use crlf
+*.bat           text eol=crlf
+

.github/ISSUE_TEMPLATE/bug_report_template.yaml

@@ -7,7 +7,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/linkedin/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information.
+        Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/feathr-ai/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information.
         #### Please fill in this bug report template to ensure a timely and thorough response.
   - type: dropdown
     id: contribution

.github/ISSUE_TEMPLATE/doc_improvements.yaml

@@ -7,7 +7,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/linkedin/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information.
+        Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/feathr-ai/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information.
         #### Please fill in this non-technical template to ensure a timely and thorough response.
   - type: dropdown
     id: contribution

.github/ISSUE_TEMPLATE/feature_request_template.yaml

@@ -7,7 +7,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/linkedin/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information.
+        Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/feathr-ai/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information.
         #### Please fill in this bug report template to ensure a timely and thorough response.
   - type: dropdown
     id: contribution

.github/ISSUE_TEMPLATE/non_technical_request_template.yaml

@@ -7,7 +7,7 @@ body:
   - type: markdown
     attributes:
       value: |
-        Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/linkedin/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information.
+        Thank you for submitting an issue. Please refer to our [contribution guide](https://github.com/feathr-ai/feathr/blob/main/docs/dev_guide/new_contributor_guide.md) for additional information.
         #### Please fill in this non-technical template to ensure a timely and thorough response.
   - type: dropdown
     id: contribution

.github/workflows/.coveragerc_db

@@ -0,0 +1,9 @@
+[run]
+omit = feathr_project/feathr/registry/_feature_registry_purview.py
+       feathr_project/feathr/registry/registry_utils.py
+       feathr_project/feathr/spark_provider/_synapse_submission.py
+       feathr_project/feathr/spark_provider/_localspark_submission.py
+[report]
+exclude_lines =
+    pragma: no cover
+    @abstract

.github/workflows/.coveragerc_local

@@ -0,0 +1,9 @@
+[run]
+omit = feathr_project/feathr/registry/_feature_registry_purview.py
+       feathr_project/feathr/registry/registry_utils.py
+       feathr_project/feathr/spark_provider/_databricks_submission.py
+       feathr_project/feathr/spark_provider/_synapse_submission.py
+[report]
+exclude_lines =
+    pragma: no cover
+    @abstract

.github/workflows/.coveragerc_sy

@@ -0,0 +1,9 @@
+[run]
+omit = feathr_project/feathr/registry/_feature_registry_purview.py
+       feathr_project/feathr/registry/registry_utils.py
+       feathr_project/feathr/spark_provider/_databricks_submission.py
+       feathr_project/feathr/spark_provider/_localspark_submission.py
+[report]
+exclude_lines =
+    pragma: no cover
+    @abstract

.github/workflows/docker-publish.yml

@@ -1,5 +1,5 @@
 # This workflow builds the docker container and publishes to dockerhub with appropriate tag
-# It has two triggers, 
+# It has two triggers,
 #   1. daily i.e. runs everyday at specific time.
 #   2. Anytime a new branch is created under releases
 
@@ -15,27 +15,26 @@ on:
     branches:
       - 'releases/**'
 
-
 jobs:
   build_and_push_image_to_registry:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
-      
+
       - name: Log in to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v4
         with:
           images: feathrfeaturestore/feathr-registry
-      
+
       - name: Build and push Docker image
         uses: docker/build-push-action@v3
         with:
@@ -45,27 +44,32 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
 
-  # Deploy the docker container to the three test environments for feathr
+  # Trigger Azure Web App webhooks to pull the latest nightly image
   deploy:
     runs-on: ubuntu-latest
     needs: build_and_push_image_to_registry
-
-
+
     steps:
-      - name: Deploy to Feathr Purview Registry Azure Web App
-        id: deploy-to-purview-webapp
+      - name: Deploy to Azure Web App feathr-registry-purview
+        id: deploy-to-feathr-registry-purview
         uses: distributhor/[email protected]
         env:
-          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_PURVIEW_REGISTRY_WEBHOOK }}
+          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_PURVIEW_WEBHOOK }}
 
-      - name: Deploy to Feathr RBAC Registry Azure Web App
-        id: deploy-to-rbac-webapp
+      - name: Deploy to Azure Web App feathr-registry-purview-rbac
+        id: deploy-to-feathr-registry-purview-rbac
         uses: distributhor/[email protected]
         env:
-          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_RBAC_REGISTRY_WEBHOOK }}
-
-      - name: Deploy to Feathr SQL Registry Azure Web App
-        id: deploy-to-sql-webapp
+          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_PURVIEW_RBAC_WEBHOOK }}
+
+      - name: Deploy to Azure Web App feathr-registry-sql
+        id: deploy-to-feathr-registry-sql
+        uses: distributhor/[email protected]
+        env:
+          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_SQL_WEBHOOK }}
+
+      - name: Deploy to Azure Web App feathr-registry-sql-rbac
+        id: deploy-to-feathr-registry-sql-rbac
         uses: distributhor/[email protected]
         env:
-          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_SQL_REGISTRY_WEBHOOK }}
+          webhook_url: ${{ secrets.AZURE_WEBAPP_FEATHR_REGISTRY_SQL_RBAC_WEBHOOK }}

.github/workflows/publish-to-maven.yml

@@ -1,18 +1,18 @@
 name: Publish package to the Maven Central Repository
-on: 
+on:
   push:
     # This pipeline will get triggered everytime there is a new tag created.
-    # It is required 
+    # It is required
     tags: ["*"]
 
 jobs:
   publish-to-maven:
     runs-on: ubuntu-latest
-    
+
     steps:
       - name: Checkout source
         uses: actions/checkout@v2
-      
+
     # Setting up JDK 8, this is required to build Feathr
       - name: Set up JDK 8
         uses: actions/setup-java@v2
@@ -27,12 +27,29 @@ jobs:
 
     # CI release command defaults to publishSigned
     # Sonatype release command defaults to sonaTypeBundleRelease
-    # https://github.com/sbt/sbt-ci-release
-      - name: Sbt ci release
-        run: |
-          sbt ci-release
+      - name: Gradle publish
+        if: startsWith(github.head_ref, 'releases/v')
+        run: gradle clean publish
         env:
           PGP_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
           PGP_SECRET: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+
+    # Publish Released Fat Jar to Blob Storage 
+      - name: Gradle build
+        run: |
+          ./gradlew build
+          # remote folder for CI upload
+          echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_release" >> $GITHUB_ENV
+          # get local jar name without path
+          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV
+
+      - name: Azure Blob Storage Upload (Overwrite)
+        uses: fixpoint/azblob-upload-artifact@v4
+        with:
+          connection-string: ${{secrets.SPARK_JAR_BLOB_CONNECTION_STRING}}
+          name: ${{ env.CI_SPARK_REMOTE_JAR_FOLDER}}
+          path: ${{ env.FEATHR_LOCAL_JAR_FULL_NAME_PATH}}
+          container: ${{secrets.SPARK_JAR_BLOB_CONTAINER}}
+          cleanup: "true"

.github/workflows/pull_request_push_test.yml

@@ -22,13 +22,13 @@ on:
       - "docs/**"
       - "ui/**"
       - "**/README.md"
-  
+
   schedule:
     # Runs daily at 1 PM UTC (9 PM CST), will send notification to TEAMS_WEBHOOK
     - cron: '00 13 * * *'
 
 jobs:
-  sbt_test:
+  gradle_test:
     runs-on: ubuntu-latest
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'safe to test'))
     steps:
@@ -41,7 +41,7 @@ jobs:
           java-version: "8"
           distribution: "temurin"
       - name: Run tests
-        run: sbt clean && sbt test
+        run: ./gradlew clean && ./gradlew test
 
   python_lint:
     runs-on: ubuntu-latest
@@ -75,15 +75,15 @@ jobs:
         with:
           java-version: "8"
           distribution: "temurin"
-      - name: Build JAR
+      - name: Gradle build
         run: |
-          sbt assembly
+          ./gradlew build
           # remote folder for CI upload
           echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV
           # get local jar name without paths so version change won't affect it
-          echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
           # get local jar name without path
-          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
@@ -129,9 +129,8 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
           AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
         run: |
-          # run only test with databricks. run in 4 parallel jobs
-          pytest -n 6 feathr_project/test/
-
+          # run only test with databricks. run in 6 parallel jobs
+          pytest -n 6 --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_db
   azure_synapse_test:
     # might be a bit duplication to setup both the azure_synapse test and databricks test, but for now we will keep those to accelerate the test speed
     runs-on: ubuntu-latest
@@ -145,15 +144,16 @@ jobs:
         with:
           java-version: "8"
           distribution: "temurin"
-      - name: Build JAR
+
+      - name: Gradle build
         run: |
-          sbt assembly
+          ./gradlew build
           # remote folder for CI upload
           echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV
           # get local jar name without paths so version change won't affect it
-          echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
           # get local jar name without path
-          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
@@ -200,8 +200,8 @@ jobs:
           AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}
         run: |
           # skip databricks related test as we just ran the test; also seperate databricks and synapse test to make sure there's no write conflict
-          # run in 4 parallel jobs to make the time shorter
-          pytest -n 6 feathr_project/test/
+          # run in 6 parallel jobs to make the time shorter
+          pytest -n 6 -m "not databricks" --cov-report term-missing --cov=feathr_project/feathr feathr_project/test --cov-config=.github/workflows/.coveragerc_sy
 
   local_spark_test:
     runs-on: ubuntu-latest
@@ -215,15 +215,16 @@ jobs:
         with:
           java-version: "8"
           distribution: "temurin"
-      - name: Build JAR
+
+      - name: Gradle build
         run: |
-          sbt assembly
+          ./gradlew build
           # remote folder for CI upload
           echo "CI_SPARK_REMOTE_JAR_FOLDER=feathr_jar_github_action_$(date +"%H_%M_%S")" >> $GITHUB_ENV
           # get local jar name without paths so version change won't affect it
-          echo "FEATHR_LOCAL_JAR_NAME=$(ls target/scala-2.12/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_NAME=$(ls build/libs/*.jar|  xargs -n 1 basename)" >> $GITHUB_ENV
           # get local jar name without path
-          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls target/scala-2.12/*.jar)" >> $GITHUB_ENV
+          echo "FEATHR_LOCAL_JAR_FULL_NAME_PATH=$(ls build/libs/*.jar)" >> $GITHUB_ENV
       - name: Set up Python 3.8
         uses: actions/setup-python@v2
         with:
@@ -259,11 +260,44 @@ jobs:
           SQL1_PASSWORD: ${{secrets.SQL1_PASSWORD}}
         run: |
           # skip cloud related tests
-          pytest feathr_project/test/test_local_spark_e2e.py
+          pytest --cov-report term-missing --cov=feathr_project/feathr/spark_provider feathr_project/test/test_local_spark_e2e.py --cov-config=.github/workflows/.coveragerc_local
+
+  registry_test:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'pull_request' || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'registry test'))
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Set up JDK 8
+        uses: actions/setup-java@v2
+        with:
+          java-version: "8"
+          distribution: "temurin"
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Feathr Package
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f ./registry/test/requirements.txt ]; then pip install -r ./registry/test/requirements.txt; fi
+          if [ -f ./registry/purview-registry/requirements.txt ]; then pip install -r ./registry/purview-registry/requirements.txt; fi
+          if [ -f ./registry/sql-registry/requirements.txt ]; then pip install -r ./registry/sql-registry/requirements.txt; fi
+      - name: Run Registry Test Cases
+        env:
+          AZURE_CLIENT_ID: ${{secrets.AZURE_CLIENT_ID}}
+          AZURE_TENANT_ID: ${{secrets.AZURE_TENANT_ID}}
+          AZURE_CLIENT_SECRET: ${{secrets.AZURE_CLIENT_SECRET}}
+          PURVIEW_NAME: "feathrazuretest3-purview1"
+          CONNECTION_STR: ${{secrets.CONNECTION_STR}}
+        run: |
+          pytest --cov-report term-missing --cov=registry/sql-registry/registry --cov-config=registry/test/.coveragerc registry/test/test_sql_registry.py
+          pytest --cov-report term-missing --cov=registry/purview-registry/registry --cov-config=registry/test/.coveragerc registry/test/test_purview_registry.py
 
   failure_notification:
     # If any failure, warning message will be sent
-    needs: [sbt_test, python_lint, databricks_test, azure_synapse_test, local_spark_test]
+    needs: [gradle_test, python_lint, databricks_test, azure_synapse_test, local_spark_test]
     runs-on: ubuntu-latest
     if: failure() && github.event_name == 'schedule'
     steps:
@@ -273,12 +307,12 @@ jobs:
 
   notification:
     # Final Daily Report with all job status
-    needs: [sbt_test, python_lint, databricks_test, azure_synapse_test, local_spark_test]
+    needs: [gradle_test, python_lint, databricks_test, azure_synapse_test, local_spark_test]
     runs-on: ubuntu-latest
     if: always() && github.event_name == 'schedule'
     steps:
       - name: Get Date
         run: echo "NOW=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
       - name: Notification
         run: |
-          curl -H 'Content-Type: application/json' -d '{"text": "${{env.NOW}} Daily Report: 1. SBT Test ${{needs.sbt_test.result}}, 2. Python Lint Test ${{needs.python_lint.result}}, 3. Databricks Test ${{needs.databricks_test.result}}, 4. Synapse Test ${{needs.azure_synapse_test.result}} , 5. LOCAL SPARK TEST ${{needs.local_spark_test.result}}. Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' ${{ secrets.TEAMS_WEBHOOK }}
+          curl -H 'Content-Type: application/json' -d '{"text": "${{env.NOW}} Daily Report: 1. Gradle Test ${{needs.gradle_test.result}}, 2. Python Lint Test ${{needs.python_lint.result}}, 3. Databricks Test ${{needs.databricks_test.result}}, 4. Synapse Test ${{needs.azure_synapse_test.result}} , 5. LOCAL SPARK TEST ${{needs.local_spark_test.result}}. Link: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' ${{ secrets.TEAMS_WEBHOOK }}