diff --git a/.github/workflows/create-merge-release-into-dev-pr.yml b/.github/workflows/create-merge-release-into-dev-pr.yml
index d11a316ac767..72b22b195eab 100644
--- a/.github/workflows/create-merge-release-into-dev-pr.yml
+++ b/.github/workflows/create-merge-release-into-dev-pr.yml
@@ -2,6 +2,9 @@ name: create-merge-release-into-dev-pr
 on:
   workflow_dispatch:
 
+env:
+  BASE_BRANCH: test-dev
+
 permissions: {}
 jobs:
   setup:
@@ -25,9 +28,13 @@ jobs:
 
         echo "branch=${BRANCH}" >> $GITHUB_OUTPUT
 
-  create-pr:
+  merge-or-create-pr:
+    env:
+      RELEASE_BRANCH: ${{ needs.setup.outputs.latest_release_branch }}
+
     permissions:
-      pull-requests: write # for git push
+      contents: write # for git push
+      pull-requests: write # for creating pull request
 
     runs-on: ubuntu-latest
     needs: setup
@@ -35,14 +42,50 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Create pull request
-        run: >
-          gh pr create
-          --base "$BASE_BRANCH"
-          --head "$RELEASE_BRANCH"
-          --title "Merge $RELEASE_BRANCH into $BASE_BRANCH"
-          --body 'Created by GitHub action'
+        with:
+          ref: ${{ env.BASE_BRANCH }} # TODO: DELETE ME
+
+      - name: Fetch dev branch upto release branch
+        run: git fetch --shallow-exclude "$RELEASE_BRANCH" origin "$BASE_BRANCH"
+
+      - name: Fetch release branch upto dev branch
+        run: git fetch --shallow-exclude "$BASE_BRANCH" origin "$RELEASE_BRANCH"
+
+      - name: Resolve bug in git https://stackoverflow.com/a/63879454/96823
+        run: git repack -d
+
+      - name: Get one commit more to connect the history of dev and release branches
+        run: git fetch --deepen 1 origin "$BASE_BRANCH" "$RELEASE_BRANCH"
+
+      - name: Create branch without checkout just to have shorter automatic commit message
+        run: git branch "$RELEASE_BRANCH" origin/"$RELEASE_BRANCH"
+
+      - name: Required git config
+        run: |
+          git config user.name "OpenProject Actions CI"
+          git config user.email "operations+ci@openproject.com"
+
+      - name: Try merging or create a PR
+        run: |
+          if git diff --exit-code --quiet ..."$RELEASE_BRANCH"; then
+            echo "Nothing to merge from $RELEASE_BRANCH into $BASE_BRANCH"
+          else
+            if git merge --no-edit --no-ff "$RELEASE_BRANCH" && git pull --rebase && git push origin "$BASE_BRANCH"; then
+              echo "Successfully merged $RELEASE_BRANCH into $BASE_BRANCH and pushed"
+            else
+              # Close all previous PRs with label
+              for pr_number in $(gh pr list --label create-merge-release-into-dev-pr --json number --jq='.[].number'); do
+                echo gh pr close "$pr_number"
+              done
+
+              gh pr create \
+                --base "$BASE_BRANCH" \
+                --head "$RELEASE_BRANCH" \
+                --title "Merge $RELEASE_BRANCH into $BASE_BRANCH" \
+                --body 'Created by GitHub action' \
+                --label create-merge-release-into-dev-pr
+              echo "Created a PR to merge $RELEASE_BRANCH into $BASE_BRANCH"
+            fi
+          fi
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BASE_BRANCH: dev
-          RELEASE_BRANCH: ${{ needs.setup.outputs.latest_release_branch }}