diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9702251..4935faa 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -10,6 +10,11 @@ on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
 
+# Only run the latest job
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
 
 jobs:
   build:
@@ -40,6 +45,7 @@ jobs:
         shell: bash
         working-directory: wildfly-glow
       - name: Run cli tests using bash
+        if: ${{ matrix.java != '11' }}
         run: bash tests/run-cli-tests.sh
         shell: bash
         working-directory: wildfly-glow
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
new file mode 100644
index 0000000..382d3c3
--- /dev/null
+++ b/.github/workflows/nightly-ci.yml
@@ -0,0 +1,73 @@
+# This workflow will build a Java project with Maven
+# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven
+
+name: WildFly Glow - Nightly CI
+
+on:
+  push:
+    branches:
+      - 'main'
+  pull_request:
+    branches:
+      - '**'
+    paths:
+      - '.github/workflows/nightly-ci.yml'
+
+  schedule:
+    - cron: '0 3 * * *' # Every day at 03:00 UTC
+
+# Only run the latest job
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+
+
+jobs:
+  nightly-build:
+    name: ${{ matrix.os }}-jdk${{ matrix.java.version }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        java: [{version: '17', release: '17'} , {version: '21', release: '21'}, {version: '23', release: '23'}, {version: '24-ea', release: '24'}]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+            path: wildfly-glow
+      - uses: actions/checkout@v4
+        with:
+            repository: wildfly/wildfly-maven-plugin
+            path: wildfly-maven-plugin
+      - name: Set up JDK ${{ matrix.java.version }}
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java.version }}
+          cache: 'maven'
+          distribution: 'temurin'
+      - name: Build and Test on ${{ matrix.java.version }} with tests compiled to ${{ matrix.java.version }}
+        run: mvn clean install '-Dtest.maven.compiler.release=${{ matrix.java.release }}'
+        working-directory: wildfly-glow
+      - name: Run cli tests using bash
+        run: bash tests/run-cli-tests.sh
+        shell: bash
+        working-directory: wildfly-glow
+      - name: Retrieve WildFly Glow version
+        shell: bash
+        run: |
+          echo "WILDFLY_GLOW_VERSION=$(mvn -B help:evaluate -Dexpression=project.version -DforceStdout -q)" >> $GITHUB_ENV
+        working-directory: wildfly-glow
+      - name: Build and Test WildFly Maven Plugin on ${{ matrix.java.version }}
+        # Currently WildFly 35 is the only version that works with Java 24. Changes should be made to the wildfly-maven-plugin
+        # to allow this to work by default. For now, we will override the WildFly version.
+        run: mvn clean install -Dversion.org.wildfly.glow=${{ env.WILDFLY_GLOW_VERSION }}
+        shell: bash
+        working-directory: wildfly-maven-plugin
+      - uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: surefire-reports-${{ matrix.os }}-${{ matrix.java.version }}
+          path: | 
+           !**/target/**
+           **/surefire-reports/*.txt
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 8bb8bdd..74cfa07 100644
--- a/pom.xml
+++ b/pom.xml
@@ -13,8 +13,11 @@
     <packaging>pom</packaging>
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <maven.compiler.source>11</maven.compiler.source>
-        <maven.compiler.target>11</maven.compiler.target>
+        <maven.compiler.release>11</maven.compiler.release>
+        <maven.compiler.source>${maven.compiler.release}</maven.compiler.source>
+        <maven.compiler.target>${maven.compiler.release}</maven.compiler.target>
+        <!-- Used to override the release bytecode for compiling tests. This allows testing on newer bytecode. -->
+        <test.maven.compiler.release>${maven.compiler.release}</test.maven.compiler.release>
         <wildfly.glow.galleon.feature-packs.url>https://raw.githubusercontent.com/wildfly/wildfly-galleon-feature-packs/release/</wildfly.glow.galleon.feature-packs.url>
 
         <!-- versions -->
@@ -139,6 +142,17 @@
                         </execution>
                     </executions>
                 </plugin>
+                <plugin>
+                    <artifactId>maven-compiler-plugin</artifactId>
+                    <executions>
+                        <execution>
+                            <id>default-testCompile</id>
+                            <configuration>
+                                <release>${test.maven.compiler.release}</release>
+                            </configuration>
+                        </execution>
+                    </executions>
+                </plugin>
                 <plugin>
                     <groupId>org.apache.maven.plugins</groupId>
                     <artifactId>maven-surefire-plugin</artifactId>