Merge pull request #1707 from skalenetwork/beta

Release 3.17.0

.github/workflows/custom_build.yml

@@ -56,24 +56,6 @@ jobs:
           which gcov-tool
           gcov-tool --version
 
-      - name: Get newest lcov
-        run: |
-          echo "Removing previous lcov version..."
-          sudo apt-get remove lcov || true
-          echo "Installing newest lcov version..."
-          rm -rf newer_lcov || true
-          mkdir newer_lcov
-          cd newer_lcov
-          git clone https://github.com/linux-test-project/lcov --recursive --recurse-submodules
-          cd lcov
-          sudo make install
-          cd ..
-          cd ..
-          echo "Checking installed lcov version..."
-          which lcov
-          lcov --version
-
-
       - name: Extract repo name
         run: echo ::set-env name=REPOSITORY_NAME::$(echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}')
         shell: bash
@@ -113,18 +95,18 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           cd deps
           ./clean.sh
           rm -f ./libwebsockets-from-git.tar.gz
-          ./build.sh PARALLEL_COUNT=$(nproc)
+          ./build.sh PARALLEL_COUNT=$(nproc) DEBUG=1
           cd ..
       - name: Configure all
         run: |
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           mkdir -p build
           cd build
           cmake -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE ${{ github.event.inputs.cmake_options }} ..
@@ -134,11 +116,11 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=Debug
           cd build
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
-          strip skaled/skaled
+          #echo "Ensure release mode skaled does not have any debug markers"
+          #strip skaled/skaled
           cd ..
 
       - name: Build and publish container
@@ -149,8 +131,8 @@ jobs:
           echo "Version $VERSION"
           export RELEASE=true
           bash ./scripts/build_and_publish.sh
-          
-      - name: Upload skaled binary as artifact 
+
+      - name: Upload skaled binary as artifact
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:

.github/workflows/functional-tests.yml

@@ -0,0 +1,96 @@
+ name: Run functional tests
+ on:
+  workflow_call:
+    inputs:
+      version:
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      version:
+        required: true
+        type: string
+
+ jobs:
+  functional-tests:
+    name: Functional tests
+    runs-on: ubuntu-20.04
+    env:
+      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
+      SKALED_RELEASE: ${{ inputs.version }}
+      NO_ULIMIT_CHECK: 1
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+        repository: skalenetwork/skale-ci-integration_tests
+        submodules: recursive
+    - name: Set up Node 
+      uses: actions/[email protected]
+      with:
+        node-version: 16
+    - name: Install packages
+      run: |
+        sudo apt-get update
+        sudo apt-get install python3-pip python3-venv jq btrfs-progs nodejs npm
+        sudo npm install -g truffle
+        sudo npm install -g yarn
+        sudo chown -R runner:runner ~/.config   # HACK
+        
+    - name: Prepare SGX keys/certs
+      run: |
+        echo "${{ secrets.sgx_key }}" > sgx_certs/sgx.key
+        chmod 600 sgx_certs/sgx.key
+        sudo mkdir /skale_node_data
+        sudo cp -r sgx_certs /skale_node_data
+        sudo chown -R runner:runner /skale_node_data/sgx_certs/*
+        wc /skale_node_data/sgx_certs/sgx.key
+    - name: Update Environment
+      run: |
+        ./update_environment.sh skaled+internals
+        ./update_environment.sh skaled+load_python
+        ./update_environment.sh skaled+load_js
+        ./update_environment.sh skaled+contractsRunningTest
+        ./update_environment.sh skaled+filestorage
+        ./update_environment.sh skaled+api
+    - name: skaled+internals+pytest
+      run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+internals+pytest
+
+    - name: skaled+filestorage+all
+      run: SKALED_PROVIDER=skaled_providers/endpoint_by_container ./run_tests.sh skaled+filestorage+all
+
+    - name: skaled+contractsRunningTest+all
+      run: SKALED_PROVIDER=skaled_providers/endpoint_by_container ./run_tests.sh skaled+contractsRunningTest+all
+
+    - name: skaled+load_python+all
+      run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+load_python+all
+
+    # - name: skaled+load_js+run_angry_cats
+    #   run: SKALED_PROVIDER=skaled_providers/endpoint_by_container ./run_tests.sh skaled+load_js+run_angry_cats
+
+    - name: skaled+internals+test_snapshot_api
+      run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+internals+test_snapshot_api
+
+    - name: skaled+internals+test_node_rotation
+      run: SKALED_PROVIDER=skaled_providers/binary_from_container ./run_tests.sh skaled+internals+test_node_rotation
+
+    - name: Fix access rights
+      run: |
+        sudo chown -R runner:runner /tmp/tmp* || true
+        sudo find ./integration_tests/skaled/internals/third_party/skale-node-tests/btrfs -type d -exec btrfs property set {} ro false \; || true
+        sudo chown -R runner:runner . || true
+        for C in $(docker ps -aq); do docker logs $C>$C.log; done || true
+      if: ${{ always() }}
+
+    - uses: actions/upload-artifact@v2
+      if: ${{ always() }}
+      continue-on-error: true
+      with:
+        name: debug
+        path: |
+          skaled_providers
+          !skaled_providers/**/skaled
+          /tmp/tmp*
+          *.log
+          ./integration_tests/skaled/internals/third_party/skale-node-tests/btrfs
+          !**/.env

.github/workflows/publish.yml

@@ -90,7 +90,7 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
+          export CMAKE_BUILD_TYPE=RelWithDebInfo
           cd deps
           ./clean.sh
           rm -f ./libwebsockets-from-git.tar.gz
@@ -101,8 +101,7 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
-          export CODE_COVERAGE=ON
+          export CMAKE_BUILD_TYPE=RelWithDebInfo
           mkdir -p build
           cd build
           # -DCMAKE_C_FLAGS=-O3 -DCMAKE_CXX_FLAGS=-O3
@@ -113,20 +112,19 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
-          export CODE_COVERAGE=ON
+          export CMAKE_BUILD_TYPE=RelWithDebInfo
           cd build
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
+          #echo "Ensure release mode skaled does not have any debug markers"
+          cp skaled/skaled skaled/skaled-debug
           strip skaled/skaled
           cd ..
       - name: Configure historic state build
         run: |
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
-          export CODE_COVERAGE=ON
+          export CMAKE_BUILD_TYPE=RelWithDebInfo
           mkdir -p build-historic
           cd build-historic
           # -DCMAKE_C_FLAGS=-O3 -DCMAKE_CXX_FLAGS=-O3
@@ -137,11 +135,11 @@ jobs:
           export CC=gcc-9
           export CXX=g++-9
           export TARGET=all
-          export CMAKE_BUILD_TYPE=Release
-          export CODE_COVERAGE=ON
+          export CMAKE_BUILD_TYPE=RelWithDebInfo
           cd build-historic
           make skaled -j$(nproc)
-          echo "Ensure release mode skaled does not have any debug markers"
+          #echo "Ensure release mode skaled does not have any debug markers"
+          cp skaled/skaled skaled/skaled-debug
           strip skaled/skaled
           cd ..
       - name: Build and publish container
@@ -177,6 +175,15 @@ jobs:
           asset_path: ./build/skaled/skaled
           asset_name: skaled
           asset_content_type: application/octet-stream
+      - name: Upload debug binary to Release
+        uses: actions/upload-release-asset@latest
+        env: 
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./build/skaled/skaled-debug
+          asset_name: skaled-debug
+          asset_content_type: application/octet-stream
       - name: Build and publish historic-state container
         run: |
           cp build-historic/skaled/skaled scripts/skale_build/executable/
@@ -197,3 +204,22 @@ jobs:
           asset_path: ./build-historic/skaled/skaled
           asset_name: skaled-historic
           asset_content_type: application/octet-stream
+      - name: Upload historic-state debug binary to Release
+        uses: actions/upload-release-asset@latest
+        env: 
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./build-historic/skaled/skaled-debug
+          asset_name: skaled-debug-historic
+          asset_content_type: application/octet-stream
+    outputs:
+      version: ${{ env.VERSION }}
+
+  functional-tests:
+    uses: ./.github/workflows/functional-tests.yml
+    name: Functional testing for build
+    needs: [build]
+    with:
+      version: ${{ needs.build.outputs.version }}
+    secrets: inherit

.github/workflows/test.yml

@@ -9,6 +9,14 @@ defaults:
   run:
     shell: bash
 jobs:
+  cancel-runs:
+    name: Cancel Previous Runs
+    runs-on: ubuntu-latest
+    steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ github.token }}
   build:
     runs-on: self-hosted
     env:
@@ -162,7 +170,7 @@ jobs:
           cd build/test
           export NO_NTP_CHECK=1
           export NO_ULIMIT_CHECK=1          
-          # we specifically run each test for easier log review
+          # we specifically run each test for easier log review          
           ./testeth -t BlockchainTests -- --express && touch /tmp/BlockchainTestsPassed
           ./testeth -t TransitionTests  -- --express && touch /tmp/TransitionTestsPassed
           ./testeth -t TransactionTests -- --express && touch /tmp/TransactionTestsPassed
@@ -259,6 +267,22 @@ jobs:
           ls /tmp/HashSnapshotTestSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t HashSnapshotTestSuite -- --all --verbosity 4
           ls /tmp/ClientSnapshotsSuitePassed || sudo NO_ULIMIT_CHECK=1 NO_NTP_CHECK=1 ./testeth -t ClientSnapshotsSuite -- --all --verbosity 4
           cd ..
+          
+      - name: Create lcov report
+        run: |
+          lcov --capture --directory . --output-file coverage.info
+          lcov --remove coverage.info '/usr/*' --output-file coverage.info # filter system-files
+          lcov --remove coverage.info 'deps/*' --output-file coverage.info # filter dependency files
+          lcov --remove coverage.info 'libconsensus/deps/*' --output-file coverage.info # filter dependency files
+          lcov --remove coverage.info 'libconsensus/libBLS/deps/*' --output-file coverage.info # filter dependency files
+          lcov --remove coverage.info '.hunter/*' --output-file coverage.info # filter dependency files
+           
+      - name: Upload to Codecov
+        uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ./coverage.info
+
       - name: Configure all as historic
         run: |
           export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"

CMakeLists.txt

@@ -96,8 +96,8 @@ endif()
 option( HUNTER_RUN_UPLOAD "Upload binaries to the cache server" ${run_upload} )
 
 include( HunterGate )
-HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
-#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) new leveldb
+#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
+HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) #leveldb 1.22
 
 set( CMAKE_CXX_STANDARD 17 )
 

CODEOWNERS

@@ -1,2 +1,2 @@
-* @skalenetwork/codeowners
+* @DmytroNazarenko @kladkogex
 *.md @skalenetwork/docowners

README.md

@@ -58,11 +58,15 @@ If you have already cloned the repo and forgot to pass `--recurse-submodules`, e
 
 ```
 sudo apt update
-sudo apt install autoconf build-essential cmake libprocps-dev libtool texinfo wget yasm flex bison btrfs-progs 
-sudo apt install make build-essential cmake pkg-config libgnutls28-dev libssl-dev unzip zlib1g-dev libgcrypt20-dev docker.io gcc-9 g++-9 gperf clang-format-11
+sudo apt install autoconf build-essential cmake libprocps-dev libtool texinfo wget yasm flex bison btrfs-progs python3 python3-pip gawk git vim doxygen 
+sudo apt install make build-essential cmake pkg-config libgnutls28-dev libssl-dev unzip zlib1g-dev libgcrypt20-dev docker.io gcc-9 g++-9 gperf clang-format-11 gnutls-dev
+sudo apt install nettle-dev libhiredis-dev redis-server google-perftools libgoogle-perftools-dev lcov
 ```
 
-NB cmake needs to be of version >=3.31, git of version >=2.18
+
+
+
+NB cmake needs to be of version >=3.21, git of version >=2.18
 
 ### (for Ubuntu 20.10 or later) Set  gcc-9 as default compiler
 ```
@@ -74,14 +78,15 @@ sudo update-alternatives --install /usr/bin/gcov-tool gcov-tool /usr/bin/gcov-to
 gcc --version
 ```
 
-### Build dependencies
+# Install latest cmake 
 
 ```
-cd deps
-./build.sh
+sudo apt-get purge cmake
+sudo snap install cmake --classic
 ```
 
-or, if you want to build debug version of skaled
+
+### Build dependencies
 
 ```
 cd deps
@@ -107,8 +112,6 @@ cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Debug
 cmake --build build -- -j$(nproc)
 ```
 
-Note: Currently only Debug build is supported.
-
 
 ## Testing
 

VERSION

@@ -1 +1 @@
-3.16.1
+3.17.0