Refactor github ci scripts

.github/workflows/build.yml

@@ -0,0 +1,43 @@
+name: Build
+
+on: # yamllint disable-line rule:truthy
+  workflow_call:
+    inputs:
+      build_type:
+        required: true
+        type: string
+      sanitize:
+        required: false
+        type: string
+
+jobs:
+  run:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Prepare environment
+        run: bash ${{github.workspace}}/.github/workflows/prepare-env.sh 14
+
+      - name: Generate Makefile
+        run: |
+          export CC=`which clang` CXX=`which clang++`
+          cmake -G Ninja -B ./build -DCMAKE_BUILD_TYPE=${{ inputs.build_type }} ${{ inputs.sanitize && format(' -DSANITIZE={0}', inputs.sanitize) || '' }}
+
+      - name: Build
+        working-directory: ${{ github.workspace }}/build
+        run: ninja -j 10
+
+      - name: Upload RaftKeeper binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: raftkeeper-binary${{ inputs.sanitize && format('-{0}', inputs.sanitize) || '' }}
+          path: build/programs/raftkeeper
+
+      - name: Upload unit test binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: unit-test-binary${{ inputs.sanitize && format('-{0}', inputs.sanitize) || '' }}
+          path: build/src/rk_unit_tests

.github/workflows/check-style.yml

@@ -0,0 +1,20 @@
+name: Check Style
+
+on: # yamllint disable-line rule:truthy
+  workflow_call:
+
+env:
+  REPORT_DIR: /tmp/reports
+
+jobs:
+  check-style:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install tools
+        run: sudo apt-get update && sudo apt install -y shellcheck libxml2-utils git python3-pip pylint yamllint && pip3 install codespell
+
+      - name: Check style
+        working-directory: ${{ github.workspace }}/tests/ci
+        run: python3 code_style_check.py | tee style-report.log

.github/workflows/comment.py

@@ -0,0 +1,81 @@
+import os
+import sys
+import json
+
+from github.Auth import AppAuth
+from github import Github
+
+
+def comment_on_pr(content):
+    app_id = "995720"
+    installation_id = 54780428
+    private_key = """-----BEGIN RSA PRIVATE KEY-----
+    MIIEowIBAAKCAQEAyN+CJniZ5AXyAdcHmfwq0ltaaSLBahjsndlJxJY4zxS3ywxC
+    ornAzDEKp4VtBf9EWIraMsCk1Ak9Wp0yICNngJ+Ch13173xk2nkRNBRYAnq+Ln+x
+    3VetS7hNMkHrSRHZezapzMCki9C0E7a9ks2F3sMwj8rKKhCvspMWGDBTMan5pKkc
+    YLlOhA27MF66SOIdDAqbi9m0pKeoNvaFaD5hRykYE+bDjnfTXSRWzPragEk5pbKl
+    sROzSDHqNYDH1d3QK4D1Uk7uT5ZQnkQuWFfliLbSaqPk4NhRwQx4jCqbkQZ4HvPm
+    8q8I5fql1gDwOxPr1eTLOU00C5x3wcwnFT+MXwIDAQABAoIBAC38FNRvyXMM9WD8
+    c+4Jb1gmt6TX4wVB3XEpXBzX8vtdF9Iw5VRRR9S26WR+Q/4aeO/4IYl61oD/+H60
+    +9Olpz0nxv7sQK/pf0EQdCLDAX7X0I/ehb5RIwfxkiKOOqnIn0v4sJiCBWlIhuD4
+    dZ+U0y+y6XwRhYRpu38a8vToozL7+WX/Wu656FK0H0huH53EazrxxD6JLpoIFErK
+    ojU5vzPJUgTdWnMZAdlEOjcO0Qg/XwDlzmQFga5duIzkt9CcmlOwRoWsVBph6YP8
+    Qzz2sjHslzoLo0Nmc+HVZLE+uyKjeI5UFfPRu5c+rZgbwR27fDJY/O/JsFbV8wZm
+    5u504TkCgYEA849asWpIiWWg4U00txtM1etNe450x9bo6LgQ2QtcJTk5OPGuxUt3
+    sX51upYHat8VbEuEiEe70QEUySwqOiK6ofLn8zr8tPmg0smy0zt1DJy/tPHQ4ubX
+    A9DKMqYxHJPS4iRi5R7sOIjNFBfA2iUPEesQRPXAIMkIyevv6LBfg9UCgYEA0yIB
+    bftBTJc1WDbaOU339GYm8rLjiDHyK9DAOC548m85Bm1b45EtMoB5e+0VY/C10eLy
+    A2A7t7CpA9kLZRVlx4voQ21bTjEnO470GN6SfmhcLZ65WzvbZvLHj+zGjkV8+Lot
+    Av02h6hnwfwQhvN9XsZPyNk4IdcXK7045DnGzWMCgYAcF5HHWtHo/w7STbxhzkVL
+    eythr+mqTxBoHyraTeQf6vy9o6qb2PuCPmrHzZwnaHmpFwC/Uz7HeY9zMKPiNrU+
+    Dq1QMaKKISy6g0cb9ASpIr892JJWSXfNWdyogOCzQh2VtcquUKXAU48L3T2CK7oU
+    P/+NZKb3YRihaZQvS4CIzQKBgQC6UsxILu/Vk6u0CdRTtgcYW/4LOOurichZ+oNo
+    ETsTWCxPC7uH/NqSMucDAptZ81fBvjIt4INS/Ehr6OMxdcy4aTO0LZHiU2Z4HRQ1
+    zlYh0B9o8yZI6W4aUC7lSOOBMrmzFzoZ5TR2S5wliTlcnw0I0qIecfQjiRods4O9
+    hW94WQKBgHMxir/eMLR8z9hj1jpJh+eN3Uo6UM+5dzgqNUn4CJC2Tt6tIZPoxdhX
+    MimMH9yrRcCe068VlvqN+AsTTpkpceJT52dV5iXuEm1DoYgvhvD19MMlW/QmOY8w
+    5GHXzNf9wt2HkMZZcpCasMTfWW3LvtEomcA3USQbZt5NgX7QjAVo
+    -----END RSA PRIVATE KEY-----"""
+
+    content = sys.argv[1]
+
+    app_auth = AppAuth(app_id, private_key)
+    g = Github(auth=app_auth.get_installation_auth(installation_id))
+
+    # Get the pull request number and repository details
+    repo_name = os.environ['GITHUB_REPOSITORY']
+    print("repo_name:", repo_name)
+
+    # Read the event payload
+    with open(os.environ['GITHUB_EVENT_PATH'], 'r') as f:
+        event_payload = json.load(f)
+
+    # Extract the pull request number
+    pull_request_number = event_payload['pull_request']['number']
+    print("pull_request_number:", pull_request_number)
+
+    repo = g.get_repo(repo_name)
+    pull_request = repo.get_pull(int(pull_request_number))
+
+    print("content:", content)
+
+    # Get all comments on the pull request
+    comments = pull_request.get_issue_comments()
+
+    # Check if a comment by raftkeepeer-robot[bot] exists
+    comment_found = False
+    for comment in comments:
+        print("find comment:", comment)
+        if comment.user.login == "raftkeepeer-robot[bot]":
+            # Update the existing comment
+            comment.edit(content)
+            comment_found = True
+            break
+
+    # If not, create a new comment
+    if not comment_found:
+        pull_request.create_issue_comment(content)
+
+
+if __name__ == "__main__":
+    comment_on_pr(sys.argv[1])

.github/workflows/comment.yml

@@ -0,0 +1,24 @@
+name: Comment on PR
+
+on: # yamllint disable-line rule:truthy
+  workflow_call:
+    inputs:
+      content:
+        required: true
+        type: string
+
+jobs:
+  comment:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: pip install PyGithub cryptography
+
+      - name: Comment on PR
+        run: python ./.github/workflows/comment.py ${{ inputs.content }}

.github/workflows/generate_test_report.py

@@ -0,0 +1,48 @@
+import xml.etree.ElementTree as ET
+
+def generate_report(report_file):
+    # Parse the XML data
+    with open(report_file, 'r') as file:
+        xml_data = file.read()
+    root = ET.fromstring(xml_data)
+
+    # Initialize lists to hold table rows
+    successful_tests = []
+    failed_tests = []
+
+    # Iterate through each testcase and categorize them
+    for testsuite in root.findall('testsuite'):
+        for testcase in testsuite.findall('testcase'):
+            classname = testcase.get('classname')
+            name = testcase.get('name')
+            time = testcase.get('time')
+
+            # Check for failure element
+            failure = testcase.find('failure')
+            if failure is not None:
+                error_message = failure.get('message')
+                failed_tests.append(f"| {classname} | {name} | {time} | No | {error_message} |")
+            else:
+                successful_tests.append(f"| {classname} | {name} | {time} | Yes | |")
+
+    # Construct Markdown tables
+    header = "| Classname      | Name                                   | Time  | Successful | Error Message |\n"
+    header += "|----------------|----------------------------------------|-------|------------|---------------|\n"
+
+    successful_table = header + "\n".join(successful_tests) if successful_tests else ""
+    failed_table = header + "\n".join(failed_tests) if failed_tests else ""
+
+    # Wrap the failed table in a collapsible section
+    collapsible_successful_table = (
+            "<details>\n"
+            "<summary>Successful Test Cases</summary>\n\n"
+            + successful_table +
+            "\n</details>"
+    )
+    return failed_table + "\n\n" + collapsible_successful_table
+
+def generate_unit_test_report(report_file):
+    return generate_report(report_file)
+
+def generate_integration_test_report(report_file):
+    return generate_report(report_file)

.github/workflows/integration-test.yml

@@ -0,0 +1,47 @@
+name: Test
+
+on: # yamllint disable-line rule:truthy
+  workflow_call:
+    inputs:
+      sanitize:
+        required: false
+        type: string
+
+jobs:
+  run:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Download binary
+        uses: actions/download-artifact@v4
+        with:
+          name: raftkeeper-binary${{ inputs.sanitize && format('-{0}', inputs.sanitize) || '' }}
+          path: build/programs/
+
+      - name: Add executable privileges
+        run: sudo chmod 755 ${{ github.workspace }}/build/programs/raftkeeper
+
+      - name: Run integration Tests
+        working-directory: ${{ github.workspace }}/tests/integration
+        run: bash ${{ github.workspace }}/.github/workflows/run-integration-test.sh ${{ github.workspace }}/tests/integration
+
+      - name: Add privileges to test report
+        if: always()
+        run: sudo chown $USER:$USER ${{ github.workspace }}/tests/integration/integration-test-report${{ inputs.sanitize && format('-{0}', inputs.sanitize) || '' }}.xml
+
+      - name: Upload test report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: integration-test-report${{ inputs.sanitize && format('-{0}', inputs.sanitize) || '' }}
+          path: ${{ github.workspace }}/tests/integration/integration-test-report${{ inputs.sanitize && format('-{0}', inputs.sanitize) || '' }}.xml
+
+      - name: Install dependencies
+        if: always()
+        run: pip install PyGithub cryptography
+
+      - name: Comment on PR
+        if: always()
+        run: if [ -f report.xml ]; then python3 .github/workflows/report.py ${{ github.workspace }}/tests/integration/integration-test-report${{ inputs.sanitize && format('-{0}', inputs.sanitize) || '' }}.xml; fi

.github/workflows/macos.yml

@@ -0,0 +1,26 @@
+name: Build and Test macOS
+
+on: # yamllint disable-line rule:truthy
+  workflow_call:
+
+jobs:
+  run:
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Install tools
+        run: brew install ninja ccache cmake llvm@13
+
+      - name: Generate Makefile
+        run: export CC=$(brew --prefix llvm@13)/bin/clang CXX=$(brew --prefix llvm@13)/bin/clang++ && cmake -G Ninja -B ./build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+
+      - name: Build
+        working-directory: ${{github.workspace}}/build
+        run: ninja -j 10
+
+      - name: Run unit tests
+        working-directory: ${{github.workspace}}/build
+        run: ./src/rk_unit_tests --gtest_color=yes