Fix nightly check (deephaven#641)

devinrsmith · May 27, 2021 · 45f4809 · 45f4809
1 parent f4a9301
commit 45f4809
Show file tree

Hide file tree

Showing 50 changed files with 384 additions and 272 deletions.
diff --git a/.github/env/Linux/gradle.properties b/.github/env/Linux/gradle.properties
@@ -1,6 +1,13 @@
+# TODO (deephaven-core#639): Customize gradle settings per-CI job
+#
 # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
-org.gradle.parallel=true
-org.gradle.workers.max=2
+# GitHub hosted linux runner environment has 2 CPUs and 7G RAM.
+#
+# testParallel seems to run poorly on GH Actions, and is unable to take advantage it seems of 2
+# workers. Let's bump up max heap, and target it as the lowest-common denominator.
+
+org.gradle.parallel=false
+org.gradle.workers.max=1
 org.gradle.jvmargs=-Xmx6g
 
 # Our CI JDKs should be pre-provisioned and invoked correctly,

diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml
@@ -68,9 +68,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
       - name: Create Dockerfile and context
         uses: burrunan/gradle-cache-action@v1
@@ -134,9 +133,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
       - name: Create Dockerfile and context
         uses: burrunan/gradle-cache-action@v1
@@ -194,9 +192,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
       - name: Create Dockerfile and context
         uses: burrunan/gradle-cache-action@v1
@@ -260,9 +257,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
       - name: Create Dockerfile and context
         uses: burrunan/gradle-cache-action@v1

diff --git a/.github/workflows/check-ci.yml b/.github/workflows/check-ci.yml
@@ -23,9 +23,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
       - name: Check
         uses: burrunan/gradle-cache-action@v1

diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml
@@ -28,9 +28,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java-8.outputs.path }},${{ steps.setup-java-11.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java-8.outputs.path }},${{ steps.setup-java-11.outputs.path }}" >> gradle.properties
 
       - name: All Javadoc
         uses: burrunan/gradle-cache-action@v1
@@ -71,9 +70,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
       - name: Generate Python Docs
         uses: burrunan/gradle-cache-action@v1

diff --git a/.github/workflows/long-check-ci.yml b/.github/workflows/long-check-ci.yml
@@ -23,9 +23,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
       - name: Check
         uses: burrunan/gradle-cache-action@v1

diff --git a/.github/workflows/nightly-benchmarks.yml b/.github/workflows/nightly-benchmarks.yml
@@ -22,9 +22,8 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
       - name: Build
         uses: burrunan/gradle-cache-action@v1

diff --git a/.github/workflows/nightly-check-ci.yml b/.github/workflows/nightly-check-ci.yml
@@ -6,11 +6,12 @@ on:
   schedule:
     # 2AM EST == 6AM UTC
     - cron:  '0 6 * * *'
+  push:
+    branches: [ 'nightly/**' ]
 
 jobs:
-  nightly:
+  check:
     runs-on: ubuntu-20.04
-    timeout-minutes: 720
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -24,24 +25,183 @@ jobs:
 
       - name: Setup gradle properties
         run: |
-          mkdir -p $HOME/.gradle
-          cp .github/env/${{ runner.os }}/gradle.properties $HOME/.gradle/gradle.properties
-          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> $HOME/.gradle/gradle.properties
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
 
-      - name: Check
+      - name: Run gradle
         uses: burrunan/gradle-cache-action@v1
-        timeout-minutes: 720
         with:
-          job-id: nightly
-          arguments: --scan --continue nightly
+          job-id: check
+          arguments: --scan --continue --rerun-tasks check
           gradle-version: wrapper
 
+      - name: Upload Test Results
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: nightly-check-ci-results
+          path: '**/build/test-results/*/TEST-*.xml'
+
+      - name: Upload Test Reports
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: nightly-check-ci-reports
+          path: '**/build/reports/tests/**'
+
       - name: Publish Test Results
         uses: scacap/action-surefire-report@v1
+        if: ${{ github.repository_owner == 'deephaven' }}
+        env:
+          NODE_OPTIONS: '--max_old_space_size=4096'
+        with:
+          check_name: check report
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: '**/build/test-results/*/TEST-*.xml'
+
+  testSerial:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # note: this uses Zulu and not AdoptOpenJDK or other. should make sure we build and test on the same one...
+      - name: Setup JDK
+        id: setup-java
+        uses: actions/setup-java@v1
+        with:
+          java-version: '8.0.292'
+
+      - name: Setup gradle properties
+        run: |
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
+
+      - name: Run gradle
+        uses: burrunan/gradle-cache-action@v1
+        with:
+          job-id: testSerial
+          arguments: --scan --continue --rerun-tasks testSerial
+          gradle-version: wrapper
+
+      - name: Upload Test Results
+        uses: actions/upload-artifact@v2
         if: always()
+        with:
+          name: nightly-testSerial-ci-results
+          path: '**/build/test-results/*/TEST-*.xml'
+
+      - name: Upload Test Reports
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: nightly-testSerial-ci-reports
+          path: '**/build/reports/tests/**'
+
+      - name: Publish Test Results
+        uses: scacap/action-surefire-report@v1
+        if: ${{ github.repository_owner == 'deephaven' }}
         env:
           NODE_OPTIONS: '--max_old_space_size=4096'
         with:
-          check_name: nightly-check-ci test report
+          check_name: testSerial report
           github_token: ${{ secrets.GITHUB_TOKEN }}
           report_paths: '**/build/test-results/*/TEST-*.xml'
+
+  testParallel:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # note: this uses Zulu and not AdoptOpenJDK or other. should make sure we build and test on the same one...
+      - name: Setup JDK
+        id: setup-java
+        uses: actions/setup-java@v1
+        with:
+          java-version: '8.0.292'
+
+      - name: Setup gradle properties
+        run: |
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
+
+      - name: Run gradle
+        uses: burrunan/gradle-cache-action@v1
+        with:
+          job-id: testParallel
+          arguments: --scan --continue --rerun-tasks testParallel
+          gradle-version: wrapper
+
+      - name: Upload Test Results
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: nightly-testParallel-ci-results
+          path: '**/build/test-results/*/TEST-*.xml'
+
+      - name: Upload Test Reports
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: nightly-testParallel-ci-reports
+          path: '**/build/reports/tests/**'
+
+      - name: Publish Test Results
+        uses: scacap/action-surefire-report@v1
+        if: ${{ github.repository_owner == 'deephaven' }}
+        env:
+          NODE_OPTIONS: '--max_old_space_size=4096'
+        with:
+          check_name: testParallel report
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: '**/build/test-results/*/TEST-*.xml'
+
+  testOutOfBand:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      # note: this uses Zulu and not AdoptOpenJDK or other. should make sure we build and test on the same one...
+      - name: Setup JDK
+        id: setup-java
+        uses: actions/setup-java@v1
+        with:
+          java-version: '8.0.292'
+
+      - name: Setup gradle properties
+        run: |
+          cat .github/env/${{ runner.os }}/gradle.properties >> gradle.properties
+          echo "org.gradle.java.installations.paths=${{ steps.setup-java.outputs.path }}" >> gradle.properties
+
+      - name: Run gradle
+        uses: burrunan/gradle-cache-action@v1
+        with:
+          job-id: testOutOfBand
+          arguments: --scan --continue --rerun-tasks testOutOfBand
+          gradle-version: wrapper
+
+      - name: Upload Test Results
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: nightly-testOutOfBand-ci-results
+          path: '**/build/test-results/*/TEST-*.xml'
+
+      - name: Upload Test Reports
+        uses: actions/upload-artifact@v2
+        if: always()
+        with:
+          name: nightly-testOutOfBand-ci-reports
+          path: '**/build/reports/tests/**'
+
+      - name: Publish Test Results
+        uses: scacap/action-surefire-report@v1
+        if: ${{ github.repository_owner == 'deephaven' }}
+        env:
+          NODE_OPTIONS: '--max_old_space_size=4096'
+        with:
+          check_name: testOutOfBand report
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          report_paths: '**/build/test-results/*/TEST-*.xml'
diff --git a/ClientSupport/src/main/java/io/deephaven/treetable/AbstractTreeSnapshotImpl.java b/ClientSupport/src/main/java/io/deephaven/treetable/AbstractTreeSnapshotImpl.java
@@ -3,6 +3,7 @@
 import io.deephaven.base.Pair;
 import io.deephaven.base.verify.Assert;
 import io.deephaven.configuration.Configuration;
+import io.deephaven.internal.log.LoggerFactory;
 import io.deephaven.io.logger.Logger;
 import io.deephaven.db.tables.Table;
 import io.deephaven.db.tables.live.NotificationQueue;
@@ -49,13 +50,12 @@ public abstract class AbstractTreeSnapshotImpl<INFO_TYPE extends HierarchicalTab
     private final TreeTableClientTableManager.ClientState clientState;
     private final TreeTableClientTableManager.TreeState treeState;
 
-    protected final Logger log;
+    private static final Logger log = LoggerFactory.getLogger(AbstractTreeSnapshotImpl.class);
 
     /**
      * Construct a new query that will create a flat snapshot of the tree table using a flat viewport beginning at the specified rows
      * and columns, applying the specified sorts and filters if required to fetch tables
-     *
-     * @param baseTable   The ID of the base table.  This will be used as a key to maintain this client state.
+     *  @param baseTable   The ID of the base table.  This will be used as a key to maintain this client state.
      * @param tablesByKey The tables within the tree for which viewports are being tracked, separated by table key.
      * @param firstRow     The first row of the flat viewport
      * @param lastRow     The last row of the flat viewport
@@ -65,8 +65,7 @@ public abstract class AbstractTreeSnapshotImpl<INFO_TYPE extends HierarchicalTab
      * @param client      The CLIENT_TYPE instance
      * @param includedOps The set of operations the client has performed before submitting this TSQ.
      */
-    AbstractTreeSnapshotImpl(Logger log,
-                             int baseTableId,
+    AbstractTreeSnapshotImpl(int baseTableId,
                              HierarchicalTable baseTable,
                              Map<Object, TableDetails> tablesByKey,
                              long firstRow, long lastRow,
@@ -81,7 +80,6 @@ public abstract class AbstractTreeSnapshotImpl<INFO_TYPE extends HierarchicalTab
         this.client = client;
         this.baseTable = baseTable;
         this.baseTableId = baseTableId;
-        this.log = log;
         this.includedOps = includedOps;
 
         firstViewportRow = firstRow;