feat(FSADT1-752): fuzzy search on oracle (#441)

* feat: adding proxy timeout config to legacy * fix: adding error handler to orgbook Some requests can return an error due to the amount of requests being done to orgbook. * fix: updating timeout for legacy * fix: reworking report to be automatically generated * chore: fixing tests * feat(fe): - Added new indicator to find out if a client is 100% match - Added section just for the demo * feat(BE): adding legacy search loading * feat(FE): adding handler for match error response * feat(BE): Modifying error message * Beautified code * Removed unneeded field and renamed another one * chore: fixing merge issues * chore: fixing merge issues * feat: adding index config * fix: fixing docker image * fix: reducing the PVC size * feat: adding index backup * fix: changing elastic to single-node * fix: elastic probe and pvc * fix: elastic probe and pvc * chore: removing index * feat(BE:FSADT1-752): implementing initial matcher processing * feat(BE:FSADT1-752): updating code to match processor Updating code to be a match processor, that will process fuzzy search and matches on oracle * test(BE:FSADT1-752): updating tests to increase coverage * fix(security): fixing CVE-2022-1471 * fix: fixing processor deployment * feat(BE:FSADT1-752): adding support for matchers * feat(BE:FSADT1-752): adding support for matchers * chore: unifying backend deployment * fix: fixing volume claim * chore: removing init_pg.sql * chore: updating testcontainer updating testcontainer class to auto-generate postgres init file for processor tests * feat(BE:FSADT1-773): adding default data and updating backend - added default data for submission type code - added default value for submission type code when submitted - added default values for the list submission request - updated tests * chore: updating processor * fix: fixing unit test * chore: fixing workflow * chore: fixing workflow * chore: fixing matrices * chore: updating ci * fix: updating ci * ci: changing sonar for backend * ci: fixing workflow by removing matrix * chore: adding init again * chore: updating pg init auto-updating * chore: adding ignore paths * ci: updating sonar config * ci: updating sonar config * chore: fixing mockito issue --------- Co-authored-by: Maria Martinez <[email protected]>
bcgov · May 15, 2023 · a0b45a5 · a0b45a5
1 parent 8ae2da9
commit a0b45a5
Show file tree

Hide file tree

Showing 75 changed files with 11,113 additions and 88 deletions.
diff --git a/.github/workflows/merge-main.yml b/.github/workflows/merge-main.yml
@@ -14,39 +14,50 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  tests-backend:
-    name: Backend Unit Tests
+  tests-java:
+    name: Backend Integrated Tests
+    if: github.event_name != 'pull_request' || !github.event.pull_request.draft
     runs-on: ubuntu-22.04
     steps:
       - uses: bcgov-nr/[email protected]
+        name: Backend Coverage
         with:
           commands: |
             mvn -B verify -P all-tests checkstyle:checkstyle -Dcheckstyle.skip=false --file pom.xml
           dir: backend
           java-cache: maven
           java-distribution: temurin
-          java-version: '17'
+          java-version: "17"
           sonar_args: >
             -Dsonar.organization=bcgov-sonarcloud
             -Dsonar.projectKey=forest-client-backend
           sonar_project_token: ${{ secrets.SONAR_TOKEN_BACKEND }}
-
-  tests-legacy:
-    name: Legacy Unit Tests
-    runs-on: ubuntu-22.04
-    steps:
       - uses: bcgov-nr/[email protected]
+        name: Legacy Coverage
         with:
           commands: |
             mvn -B verify -P all-tests checkstyle:checkstyle -Dcheckstyle.skip=false --file pom.xml
           dir: legacy
           java-cache: maven
           java-distribution: temurin
-          java-version: '17'
+          java-version: "17"
           sonar_args: >
             -Dsonar.organization=bcgov-sonarcloud
             -Dsonar.projectKey=forest-client-legacy
           sonar_project_token: ${{ secrets.SONAR_TOKEN_LEGACY }}
+      - uses: bcgov-nr/[email protected]
+        name: Processor Coverage
+        with:
+          commands: |
+            mvn -B verify -P all-tests checkstyle:checkstyle -Dcheckstyle.skip=false --file pom.xml
+          dir: processor
+          java-cache: maven
+          java-distribution: temurin
+          java-version: "17"
+          sonar_args: >
+            -Dsonar.organization=bcgov-sonarcloud
+            -Dsonar.projectKey=nr-forest-client_processor
+          sonar_project_token: ${{ secrets.SONAR_TOKEN_PROCESSOR }}
 
   tests-frontend:
     name: Frontend Unit Tests
@@ -69,8 +80,7 @@ jobs:
     name: Semantic Code Analysis
     runs-on: ubuntu-22.04
     needs:
-      - tests-backend
-      - tests-legacy
+      - tests-java
       - tests-frontend
     permissions:
       actions: read
@@ -259,14 +269,28 @@ jobs:
             -p VITE_KEYCLOAK_URL=https://test.loginproxy.gov.bc.ca/auth
             -p VITE_KEYCLOAK_CLIENT_ID=${{ secrets.KEYCLOAK_CLIENT_ID }}
 
+      - name: Deploy Processor
+        uses: bcgov-nr/[email protected]
+        with:
+          file: processor/openshift.deploy.yml
+          oc_namespace: ${{ secrets.OC_NAMESPACE }}
+          oc_server: ${{ secrets.OC_SERVER }}
+          oc_token: ${{ secrets.OC_TOKEN }}
+          overwrite: true
+          penetration_test: false
+          verification_path: health
+          parameters:
+            -p ZONE=${{ env.ZONE }} -p NAME=${{ github.event.repository.name }}
+            -p PROMOTE=${{ github.repository }}/processor:${{ env.ZONE }}
+
   image-promotions:
     name: Promote images to PROD
     needs:
       - test-deploy
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        component: [backend, frontend, legacy, database, common]
+        component: [backend, frontend, legacy, database, common, processor]
     steps:
       - uses: shrink/actions-docker-registry-tag@v3
         with:
@@ -329,7 +353,7 @@ jobs:
             -p CHES_CLIENT_SECRET=${{ secrets.CHES_CLIENT_SECRET }}
 
   prod-deploy:
-    name: PROD Database
+    name: PROD Deployment
     needs:
       - prod-init
     env:
@@ -385,6 +409,20 @@ jobs:
             -p ZONE=${{ env.ZONE }} -p NAME=${{ github.event.repository.name }}
             -p PROMOTE=${{ github.repository }}/legacy:${{ env.PREV }}
 
+      - name: Deploy Processor
+        uses: bcgov-nr/[email protected]
+        with:
+          file: processor/openshift.deploy.yml
+          oc_namespace: ${{ secrets.OC_NAMESPACE }}
+          oc_server: ${{ secrets.OC_SERVER }}
+          oc_token: ${{ secrets.OC_TOKEN }}
+          overwrite: true
+          penetration_test: false
+          verification_path: health
+          parameters:
+            -p ZONE=${{ env.ZONE }} -p NAME=${{ github.event.repository.name }}
+            -p PROMOTE=${{ github.repository }}/processor:${{ env.PREV }}
+
       - name: Deploy Frontend
         uses: bcgov-nr/[email protected]
         with:

diff --git a/.github/workflows/pr-close.yml b/.github/workflows/pr-close.yml
@@ -32,7 +32,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        package: [backend, database, frontend, legacy, common]
+        package: [backend, database, frontend, legacy, common, processor]
     steps:
       - uses: shrink/actions-docker-registry-tag@v3
         with:

diff --git a/.github/workflows/pr-open.yml b/.github/workflows/pr-open.yml
@@ -124,7 +124,7 @@ jobs:
       packages: write
     strategy:
       matrix:
-        package: [backend, legacy]
+        package: [backend, legacy, processor]
 
     name: Builds (${{ matrix.package }})
     runs-on: ubuntu-latest
@@ -245,7 +245,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
-      - name: Deploys
+      - name: Deploy Backend
         uses: bcgov-nr/[email protected]
         with:
           file: backend/openshift.deploy.yml
@@ -260,21 +260,19 @@ jobs:
             -p CHES_TOKEN_URL='https://dev.loginproxy.gov.bc.ca/auth/realms/comsvcauth/protocol/openid-connect/token'
             -p CHES_API_URL='https://ches-dev.api.gov.bc.ca/api/v1'
             -p BCREGISTRY_URI='https://bcregistry-prod.apigee.net'
-
-  deploy-legacy:
-    name: Deploy Legacy
-    needs:
-      - deploy-init
-    environment: dev
-    env:
-      DOMAIN: apps.silver.devops.gov.bc.ca
-      BROKER_URL: https://nr-broker.apps.silver.devops.gov.bc.ca
-      VAULT_ADDR: https://vault-iit.apps.silver.devops.gov.bc.ca
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Deploys
+      - name: Deploy Processor
+        uses: bcgov-nr/[email protected]
+        with:
+          file: processor/openshift.deploy.yml
+          oc_namespace: ${{ secrets.OC_NAMESPACE }}
+          oc_server: ${{ secrets.OC_SERVER }}
+          oc_token: ${{ secrets.OC_TOKEN }}
+          overwrite: true
+          verification_path: health
+          parameters:
+            -p ZONE=${{ github.event.number }} -p NAME=${{ github.event.repository.name }}
+            -p PROMOTE=${{ github.repository }}/processor:${{ github.event.number }}
+      - name: Deploy Legacy
         uses: bcgov-nr/[email protected]
         with:
           file: legacy/openshift.deploy.yml

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -19,12 +19,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  tests-backend:
-    name: Backend Unit Tests
+  tests-java:
+    name: ${{ matrix.package }} Integrated Tests
     if: github.event_name != 'pull_request' || !github.event.pull_request.draft
     runs-on: ubuntu-22.04
     steps:
       - uses: bcgov-nr/[email protected]
+        name: Backend Coverage
         with:
           commands: |
             mvn -B verify -P all-tests checkstyle:checkstyle -Dcheckstyle.skip=false --file pom.xml
@@ -36,47 +37,8 @@ jobs:
             -Dsonar.organization=bcgov-sonarcloud
             -Dsonar.projectKey=forest-client-backend
           sonar_project_token: ${{ secrets.SONAR_TOKEN_BACKEND }}
-      - name: Archive CycloneDX
-        continue-on-error: true
-        uses: actions/upload-artifact@v3
-        with:
-          name: cyclone-backend
-          path: target/bom.json
-          retention-days: 5
-
-      - name: Use Checkstyle report
-        continue-on-error: true
-        uses: jwgmeligmeyling/checkstyle-github-action@master
-        with:
-          path: 'backend/**/checkstyle-result.xml'
-
-      - name: Publish Test Report
-        uses: mikepenz/action-junit-report@v3
-        continue-on-error: true
-        if: success() || failure() # always run even if the previous step fails
-        with:
-          report_paths: 'backend/target/**/TEST-*.xml'
-          commit: ${{ github.event.pull_request.head.sha }}
-          summary: Pull Request Tests Backend
-          detailed_summary: true
-          job_name: Backend Tests
-
-      - name: Add coverage to PR
-        id: jacoco
-        continue-on-error: true
-        uses: madrapps/[email protected]
-        with:
-          paths: backend/target/coverage-reports/merged-test-report/jacoco.xml
-          token: ${{ secrets.GITHUB_TOKEN }}
-          min-coverage-overall: 75
-          min-coverage-changed-files: 75
-
-  tests-legacy:
-    name: Legacy Unit Tests
-    if: github.event_name != 'pull_request' || !github.event.pull_request.draft
-    runs-on: ubuntu-22.04
-    steps:
       - uses: bcgov-nr/[email protected]
+        name: Legacy Coverage
         with:
           commands: |
             mvn -B verify -P all-tests checkstyle:checkstyle -Dcheckstyle.skip=false --file pom.xml
@@ -88,37 +50,54 @@ jobs:
             -Dsonar.organization=bcgov-sonarcloud
             -Dsonar.projectKey=forest-client-legacy
           sonar_project_token: ${{ secrets.SONAR_TOKEN_LEGACY }}
+      - uses: bcgov-nr/[email protected]
+        name: Processor Coverage
+        with:
+          commands: |
+            mvn -B verify -P all-tests checkstyle:checkstyle -Dcheckstyle.skip=false --file pom.xml
+          dir: processor
+          java-cache: maven
+          java-distribution: temurin
+          java-version: "17"
+          sonar_args: >
+            -Dsonar.organization=bcgov-sonarcloud
+            -Dsonar.projectKey=nr-forest-client_processor
+          sonar_project_token: ${{ secrets.SONAR_TOKEN_PROCESSOR }}
+
       - name: Archive CycloneDX
         continue-on-error: true
         uses: actions/upload-artifact@v3
         with:
-          name: cyclone-legacy
-          path: target/bom.json
+          name: cyclone-backend
+          path: |
+            backend/target/bom.json
+            legacy/target/bom.json
+            processor/target/bom.json
           retention-days: 5
-
-      - name: Use Checkstyle report
+                   
+      - name: Checkstyle report
         continue-on-error: true
         uses: jwgmeligmeyling/checkstyle-github-action@master
         with:
-          path: 'legacy/**/checkstyle-result.xml'
-
+          path: (backend|legacy|processor)/**/checkstyle-result.xml
+      
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v3
         continue-on-error: true
         if: success() || failure() # always run even if the previous step fails
         with:
-          report_paths: 'legacy/target/**/TEST-*.xml'
+          report_paths: (backend|legacy|processor)/target/**/TEST-*.xml
           commit: ${{ github.event.pull_request.head.sha }}
-          summary: Pull Request Tests Legacy
+          summary: Pull Request Tests for Backend
           detailed_summary: true
-          job_name: Legacy Tests
+          job_name: Backend Tests
 
       - name: Add coverage to PR
         id: jacoco
         continue-on-error: true
         uses: madrapps/[email protected]
         with:
-          paths: legacy/target/coverage-reports/merged-test-report/jacoco.xml
+          paths: (backend|legacy|processor)/target/coverage-reports/merged-test-report/jacoco.xml
           token: ${{ secrets.GITHUB_TOKEN }}
           min-coverage-overall: 75
           min-coverage-changed-files: 75

diff --git a/backend/pom.xml b/backend/pom.xml
@@ -53,6 +53,7 @@
             **/exception/**,
             **/dto/**,
             **/entity/**,
+            **/models/**,
             **/repository/**,
             **/*$*Builder*,
             **/BootApplication*
@@ -199,6 +200,11 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+			<dependency>
+                <groupId>org.yaml</groupId>
+                <artifactId>snakeyaml</artifactId>
+                <version>2.0</version>
+			</dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -294,6 +300,7 @@
                         <exclude>**/exception/**</exclude>
                         <exclude>**/dto/**</exclude>
                         <exclude>**/entity/**</exclude>
+                        <exclude>**/models/**</exclude>
                         <exclude>**/repository/**</exclude>
                         <exclude>**/*$*Builder*</exclude>
                         <exclude>**/BootApplication*</exclude>

diff --git a/backend/src/main/java/ca/bc/gov/app/controller/client/ClientSubmissionController.java b/backend/src/main/java/ca/bc/gov/app/controller/client/ClientSubmissionController.java
@@ -56,7 +56,7 @@ public Flux<ClientListSubmissionDto> listSubmissions(
       int page,
       @RequestParam(required = false, defaultValue = "10")
       int size,
-      @RequestParam(required = false)
+      @RequestParam(required = false,defaultValue = "RNC,AAC")
       String[] requestType,
       @RequestParam(required = false)
       SubmissionStatusEnum[] requestStatus,

diff --git a/backend/src/main/java/ca/bc/gov/app/entity/client/SubmissionEntity.java b/backend/src/main/java/ca/bc/gov/app/entity/client/SubmissionEntity.java
@@ -2,6 +2,7 @@
 
 import ca.bc.gov.app.ApplicationConstant;
 import ca.bc.gov.app.models.client.SubmissionStatusEnum;
+import ca.bc.gov.app.models.client.SubmissionTypeCodeEnum;
 import java.time.LocalDateTime;
 import lombok.AllArgsConstructor;
 import lombok.Data;
@@ -30,6 +31,9 @@ public class SubmissionEntity extends BaseEntity {
   @Column("submission_status_code")
   private SubmissionStatusEnum submissionStatus;
 
+  @Column("submission_type_code")
+  private SubmissionTypeCodeEnum submissionType;
+
   @Column("submission_date")
   private LocalDateTime submissionDate;
 }