Release v1.1.1

.github/workflows/workflow.yml

@@ -0,0 +1,166 @@
+
+name: Main workflow
+on:
+
+  ## No sense in a dev building every time they push and no one
+  ## should be working on the common branches.
+  push:
+    branches:
+      - master
+      - main
+      - develop
+      - 'hotfix/**'
+      - 'release/**'
+      - 'feature/**'
+  ## Any pull request. Yes the syntax looks weird
+  pull_request:
+
+jobs:
+
+
+  test:
+    name: Test the loader on ${{matrix.operating-system}}
+    runs-on: ${{ matrix.operating-system }}
+    strategy:
+      matrix:
+        operating-system: [ubuntu-latest, windows-latest, macOS-latest]
+
+    steps:
+      - name: Get the code
+        uses: actions/checkout@v3
+
+      - name: Set up node
+        uses:  actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+          registry-url: https://npm.pkg.github.com
+          scope: '@nciocpl'
+
+      - name: Install packages
+        run:  |
+          npm ci
+
+      - name: Run tests
+        run:  |
+          npm test
+        env:
+          CI: true
+
+      - name: Archive test artifacts
+        uses: actions/upload-artifact@v1
+        with:
+          name: test-results
+          path: coverage
+
+
+  integration_tests:
+    name: Run Integration Tests (Linux)
+    runs-on: ubuntu-latest
+    needs: test
+
+    services:
+      elasticsearch:
+        image: elasticsearch:7.17.5
+        env:
+            discovery.type: single-node
+            ES_JAVA_OPTS: -Xms750m -Xmx750m
+        ports:
+          ## NOTE: This will be exposed as a random port referenced below by job.services.elasticsearch.ports[9200]
+          - 9200/tcp
+        options: --health-cmd="curl http://localhost:9200/_cluster/health" --health-interval=10s --health-timeout=5s --health-retries=10
+
+    steps:
+
+      - name: Get the code
+        uses: actions/checkout@v3
+
+      - name: Set up node
+        uses:  actions/setup-node@v3
+        with:
+          node-version-file: '.nvmrc'
+          cache: 'npm'
+          registry-url: https://npm.pkg.github.com
+          scope: '@nciocpl'
+
+      - name: Install packages
+        run:  |
+          npm ci
+
+      - name: Run the loader to put data in Elasticsearch.
+        # It would be awesome if we could set up the config override in the job's first step
+        # in order to make it more obvious it was there and may need to be adjusted for
+        # whatever you're doing. Unfortunately, GitHub's magic for passing environment
+        # variables via $GITHUB_ENV doesn't handle newlines very well, and this one is
+        # sufficiently complex, the newlines are vital to readability.
+        run:  |
+          export NODE_CONFIG="
+            {
+              \"pipeline\": {
+                  \"source\": {
+                      \"module\": \"./lib/sources/github-bestbets-source\",
+                      \"config\": {
+                          \"repoUrl\": \"https://github.com/nciocpl/bestbets-content\",
+                          \"branchName\": \"integration-testing\",
+                          \"resourcesPath\": \"/content\",
+                          \"authentication\": {
+                              \"token\": \"${GITHUB_TOKEN}\"
+                          }
+                      }
+                  },
+                  \"transformers\": [
+                      {
+                          \"module\": \"./lib/transformers/category-to-match-transformer\",
+                          \"config\": {
+                            \"eshosts\": [ \"http://localhost:${{ job.services.elasticsearch.ports[9200] }}\" ],
+                            \"settingsPath\": \"es-mappings/settings.json\",
+                            \"analyzer\": \"nostem\",
+                            \"concurrencyLimit\": 14
+                          }
+                      }
+                  ],
+                  \"loader\": {
+                      \"module\": \"./lib/loaders/elastic-bestbets-loader\",
+                      \"config\": {
+                        \"eshosts\": [ \"http://localhost:${{ job.services.elasticsearch.ports[9200] }}\" ],
+                        \"daysToKeep\": 10,
+                        \"aliasName\": \"bestbets_v1\",
+                        \"mappingPath\": \"es-mappings/mappings.json\",
+                        \"settingsPath\": \"es-mappings/settings.json\"
+                      }
+                  }
+              }
+            }
+          "
+          node index.js
+
+      - name: Run Integration Test
+        ## Normally bash runs with -e which exits the shell upon hitting
+        ## an error which breaks our capturing of those errors.
+        shell: bash --noprofile --norc -o pipefail {0}
+        run: |
+              ## Run Karate
+              ## Variable is picked up by karate-config.js
+              export KARATE_ESHOST="http://localhost:${{ job.services.elasticsearch.ports[9200] }}"
+              cd integration-tests && ./bin/karate ./features
+              ## Store the exit code off so we can pass this step and
+              ## capture the test output in the next step, but still
+              ## fail the entire job
+              echo "TEST_EXIT_CODE=$?" >> $GITHUB_ENV
+              exit 0
+
+      - name: Upload Integration test results
+        uses: actions/upload-artifact@v1
+        with:
+          name: integration-test-results
+          path: integration-tests/target
+
+      - name: Fail build on bad tests
+        run: |
+              ## Check if we had errors on the test step, and if so, fail the job
+              if [ $TEST_EXIT_CODE -ne 0 ]; then
+                echo "Tests Failed -- See Run Integration Test step or integration-test-results artifact for more information"
+                exit $TEST_EXIT_CODE
+              else
+                echo "Tests passed"
+              fi

.gitignore

@@ -1,5 +1,7 @@
 config/local.json
 
+.vscode
+
 # Logs
 logs
 *.log
@@ -61,3 +63,12 @@ typings/
 
 # next.js build output
 .next
+
+
+## Custom
+
+.vscode
+.DS_Store
+
+config/local.json
+integration-tests/target

.npmrc

@@ -0,0 +1 @@
+engine-strict=true

.nvmrc

@@ -0,0 +1 @@
+lts/gallium

config/default.json

@@ -20,7 +20,9 @@
                 "config": {
                     "eshosts": [ "http://localhost:9200" ],
                     "settingsPath": "es-mappings/settings.json",
-                    "analyzer": "nostem"
+                    "analyzer": "nostem",
+                    // A good value for this might be 14 times the number of ES nodes.
+                    "concurrencyLimit": 14
                 }
                 //"socketLimit": 80
             }

es-mappings/mappings.json

@@ -1,14 +1,20 @@
 {
   "mappings": {
-    "synonyms": {
       "properties": {
         "category": {
           "type": "text",
           "analyzer": "nostem"
         },
+        "name": {
+          "type": "text",
+          "index": "false"
+        },
         "contentid": {
           "type": "keyword"
         },
+        "record_type": {
+          "type": "keyword"
+        },
         "is_exact": {
           "type": "boolean"
         },
@@ -24,17 +30,6 @@
         },
         "tokencount": {
           "type": "integer"
-        }
-      }
-    },
-    "categorydisplay": {
-      "properties": {
-        "contentid": {
-          "type": "keyword"
-        },
-        "name": {
-          "type": "text",
-          "index": "false"
         },
         "weight": {
           "type": "integer",
@@ -44,7 +39,6 @@
           "type": "text",
           "index": "false"
         }
-      }
     }
   }
 }

integration-tests/README.md

@@ -0,0 +1,22 @@
+
+## Steps to Run Locally
+1. Have docker installed
+1. Have [nvm](https://github.com/nvm-sh/nvm) installed.
+1. Open a command prompt
+1. `cd <REPO_ROOT>/integration-tests/docker-r4r-loader`
+1. `docker-compose up --force-recreate`
+   * This is being run without the detached (-d) option so that it can be easier to stop. You can choose however you want to run it.
+1. `cd <REPO_ROOT>`
+1. `nvm use`
+1. `npm ci`
+6. `node index.js` -- this loads the test data
+5. `cd <REPO_ROOT>/integration-tests`
+7. `./bin/karate ./features` -- This runs the tests
+   * `./bin/karate -w ./features` will watch the feature files and rerun when they are changed. So good for devving tests
+
+## Notes
+* [Docs for understanding how to run Karate standalone](https://github.com/intuit/karate/blob/6de466bdcf105d72450a40cf31b8adb5c043037d/karate-netty/README.md#standalone-jar)
+   * Specifically this has to do with the magic naming of the logging config which is really why I am posting this here!
+* We have docker for dev testing because ES will no longer run on higher Java versions, this is the easiest way to get it up and running.
+* You need to use the `--force-recreate` option to `docker-compose up` or run `docker-compose rm` after shutting down the cluster. If the elasticsearch container is not removed, it keeps its data, and any restarts will leave the cluster in a bad state.
+

integration-tests/bin/README.md

@@ -0,0 +1,2 @@
+# Karate tool
+This is Karate 1.2.0 downloaded from https://github.com/karatelabs/karate.

integration-tests/bin/karate

@@ -0,0 +1,3 @@
+#!/bin/bash
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+java -Dlogback.configurationFile=logback.xml -cp $DIR:$DIR/karate.jar:. com.intuit.karate.Main $*

integration-tests/bin/karate-config.js

@@ -0,0 +1,9 @@
+function fn() {
+    var config = {
+        esHost: 'http://localhost:9200'
+    };
+    if (java.lang.System.getenv('KARATE_ESHOST')) {
+        config.esHost = java.lang.System.getenv('KARATE_ESHOST');
+    }
+    return config;
+}

integration-tests/bin/karate.bat

@@ -0,0 +1 @@
+java -cp .\karate.jar;. com.intuit.karate.Main %*