From 65acacf0ca448e11650ea41ea8da6bd8bfc12bd7 Mon Sep 17 00:00:00 2001 From: Morgan Rockett Date: Thu, 13 Jun 2024 16:46:36 -0400 Subject: [PATCH] added standalone pylint.sh script; improved install-build-tools.sh over prev commit Signed-off-by: Morgan Rockett --- .github/workflows/ci.yml | 37 +++-- .github/workflows/docker-pull.yml | 1 - .gitignore | 3 +- README.md | 14 +- requirements.txt | 5 + scripts/activate-venv.sh | 28 ++++ scripts/build.sh | 2 +- scripts/install-build-tools.sh | 226 +++++++++++++++++++++++++----- scripts/pylint.sh | 43 ++++++ scripts/setup-dependencies.sh | 3 +- 10 files changed, 295 insertions(+), 67 deletions(-) create mode 100644 requirements.txt create mode 100755 scripts/activate-venv.sh create mode 100755 scripts/pylint.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 55f26ece5..f30ed91a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ env: jobs: build-release: name: Build Release Candidate - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 env: BUILD_RELEASE: 1 steps: @@ -28,9 +28,9 @@ jobs: - name: Setup Build Env run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies - run: ./scripts/setup-dependencies.sh + run: sudo ./scripts/setup-dependencies.sh - name: Build - run: scripts/build.sh + run: ./scripts/build.sh lint: name: Lint runs-on: ubuntu-20.04 @@ -41,16 +41,16 @@ jobs: - name: Setup Build Env run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies - run: ./scripts/setup-dependencies.sh + run: sudo ./scripts/setup-dependencies.sh - name: Build - run: scripts/build.sh + run: ./scripts/build.sh - name: Lint - run: scripts/lint.sh + run: ./scripts/lint.sh pylint: name: Pylint - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 continue-on-error: true - timeout-minutes: 5 + timeout-minutes: 10 strategy: matrix: python-version: ["3.10", "3.11", "3.12"] @@ -62,18 +62,15 @@ jobs: uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python3 -m pip install --upgrade pip - if [ -f requirements_py.txt ]; then pip install -r requirements_py.txt; fi - pip install pylint + - name: Setup Build Env + run: sudo ./scripts/install-build-tools.sh - name: Lint with Pylint run: | - # In the future we should have (minimum score of 8.0/10.0 or 9.0/10.0) - pylint --rcfile=.pylintrc $(git ls-files '*.py') --fail-under=5.0 + MIN_CODE_QUALITY=5.0 + ./scripts/pylint.sh $MIN_CODE_QUALITY unit-and-integration-test: name: Unit and Integration Tests - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 timeout-minutes: 30 steps: - uses: actions/checkout@v2 @@ -82,11 +79,11 @@ jobs: - name: Setup Build Env run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies - run: ./scripts/setup-dependencies.sh + run: sudo ./scripts/setup-dependencies.sh - name: Build - run: scripts/build.sh + run: ./scripts/build.sh - name: Run Unit Tests - run: scripts/test.sh + run: ./scripts/test.sh - name: Shorten SHA id: vars run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" @@ -101,7 +98,7 @@ jobs: retention-days: 7 doxygen: name: doxygen - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/docker-pull.yml b/.github/workflows/docker-pull.yml index b2dd161c6..df696b42a 100644 --- a/.github/workflows/docker-pull.yml +++ b/.github/workflows/docker-pull.yml @@ -26,7 +26,6 @@ jobs: with: files: | Dockerfile - **/configure.sh ######################## # Build Base # diff --git a/.gitignore b/.gitignore index 71f3f5701..ebb163ae7 100644 --- a/.gitignore +++ b/.gitignore @@ -47,7 +47,6 @@ ethash*/ lua-*/ benchmark-results/ CMakeFiles/ -Python-*/ plots/ .deps/ .libs/ @@ -74,3 +73,5 @@ build/tests # E2E Test results testruns/ +# Virtualenv +.py_venv/ diff --git a/README.md b/README.md index 36104729f..e0887afb8 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ There are two UHS-based architectures as follows: - Maximum demonstrated throughput ~1.7M transactions per second. - Geo-replicated latency <1 second. -Read the [2PC & Atomizer architecture guide](docs/uhs-architectures.md) for a detailed description of the system components and implementation of each architecture. +Read the [2PC & Atomizer architecture guide](docs/uhs-architectures.md) for a detailed description of the system components and implementation of each architecture. ## Parallel Architecture for Scalably Executing smart Contracts ("PArSEC") @@ -87,8 +87,13 @@ If you just want to run the system, see "Run the Code" below. Note that this script is just a convenience to install system-wide dependencies we expect. As a result, it uses the system package manager, requires `sudo`, and should only be run **once**. ```console + sudo ./scripts/install-build-tools.sh + ``` + Note: Running Homebrew as root on mac via shell script is not supported, so run without sudo and when prompted, enter the root password. + ```console ./scripts/install-build-tools.sh ``` + 2. Setup project dependencies This script builds and installs a local copy of several build-dependencies which are not widely packaged. Because it installs to a local, configurable prefix (defaulting to `./prefix`), it does not need root permissions to run. @@ -116,7 +121,7 @@ See the [live deployment](https://mit-dci.github.io/opencbdc-tx-pages/) to brows ## UHS-based Architectures (2PC & Atomizer) -See the [2PC & Atomizer User Guide](docs/2pc_atomizer_user_guide.md) +See the [2PC & Atomizer User Guide](docs/2pc_atomizer_user_guide.md) ## PArSEC Architecture @@ -159,7 +164,7 @@ Review results and logs at `testruns//` ## Linting ### General -This script checks for newlines at the end of files. +This script checks for newlines at the end of all tracked git files except images. Then it runs clang-format and clang-tidy on `.cpp` files in the following directories: `src`, `tests`, `cmake-tests`, `tools`. ```console @@ -168,6 +173,7 @@ Then it runs clang-format and clang-tidy on `.cpp` files in the following direct ### Python Lint all python files according to ruleset defined in `.pylintrc`. +Optional code quality value >= 5.0 and <= 10.0 can be entered as a threshold of failure. ```console -pylint --rcfile=.pylintrc $(git ls-files '*.py') --fail-under=8.0 +./scripts/pylint.sh 8.0 ``` diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000..5e065199b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +setuptools +eth-hash +matplotlib +numpy +pylint diff --git a/scripts/activate-venv.sh b/scripts/activate-venv.sh new file mode 100755 index 000000000..cd2d04ea3 --- /dev/null +++ b/scripts/activate-venv.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# run with 'source scripts/activate-venv.sh' +set -e + +ROOT="$(cd "$(dirname "$0")"/.. && pwd)" +ENV_NAME=".py_venv" + +# if venv does not exist, send error message and exit +if [ ! -d "${ROOT}/${ENV_NAME}" ]; then + echo "Virtual environment '${ENV_NAME}' not found." + if [[ $OSTYPE == "linux-gnu"* ]]; then + echo "run 'sudo ./scripts/install-build-tools.sh' to create it." + elif [[ $OSTYPE == "darwin"* ]]; then + echo "run './scripts/install-build-tools.sh' to create it." + fi + exit 1 +fi + +# activate virtual environment +if source "${ROOT}/${ENV_NAME}/bin/activate"; then + echo "Virtual environment '${ENV_NAME}' activated." + echo "Run 'deactivate' to exit the virtual environment."s +else + echo "Failed to activate virtual environment '${ENV_NAME}'." + exit 1 +fi + diff --git a/scripts/build.sh b/scripts/build.sh index 879f346e5..90d07991d 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -33,7 +33,7 @@ fi echo "Building..." -# see PREFIX in ./scripts/configure.sh +# see PREFIX in ./scripts/setup-dependencies.sh PREFIX="$(cd "$(dirname "$0")"/.. && pwd)/prefix" if [ -z ${BUILD_DIR+x} ]; then diff --git a/scripts/install-build-tools.sh b/scripts/install-build-tools.sh index 258e7611a..c0a8d160a 100755 --- a/scripts/install-build-tools.sh +++ b/scripts/install-build-tools.sh @@ -6,6 +6,7 @@ green="\033[0;32m" cyan="\033[0;36m" end="\033[0m" +# for debugging, 'set -x' can be added to trace the commands set -e SUDO='' @@ -14,73 +15,219 @@ if (( $EUID != 0 )); then SUDO='sudo' fi -# Function to check if a version of Python >= 3.10 is installed -check_python_version() { - if command -v python3 &>/dev/null; then - # Get the version of Python - PYTHON_VERSION=$(python3 --version | awk '{print $2}') - IFS='.' read -r -a version_parts <<< "$PYTHON_VERSION" - echo "Python version: ${version_parts[0]}.${version_parts[1]}" - # >= 3.10 - if (( ${version_parts[0]} >= 3 && ${version_parts[1]} >= 10 )); then - return 0 +# install in a custom prefix rather than /usr/local. by default, this +# chooses "prefix" directory alongside "scripts" directory. +ROOT=$(cd "$(dirname "$0")"/.. && pwd) +PREFIX="$(cd "$(dirname "$0")"/.. && pwd)/prefix" +echo "Will install local dependencies in the following prefix: $PREFIX" +mkdir -p "$PREFIX"/{bin,lib,include} + +# Supporting these versions for buildflow +PYTHON_VERSIONS=("3.12" "3.11" "3.10") +ENV_NAME=".py_venv" +PY_INSTALLED='' +echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}" + +# make a virtual environement to install python packages +create_venv_install_python() { + PY_LOC=$1 + if [ -z "$PY_LOC" ]; then + echo "Python path not provided" + exit 1 + fi + PY_VERSION=$2 + if [ -z "$PY_VERSION" ]; then + echo "python version not provided" + exit 1 + fi + # remove existing virtual environment if it exists - global var + ENV_LOCATION="${ROOT}/${ENV_NAME}" + if [ -d "${ENV_LOCATION}" ]; then + if rm -rf "${ENV_LOCATION}"; then + echo "Removed existing virtual environment" + else + echo "Failed to remove existing virtual environment"; exit 1 + fi + fi + # install pip for linux + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + if ! $SUDO apt install -y python3-pip; then + echo "Failed to install python3-pip" + fi + if ! $SUDO python${PY_VERSION} -m pip install --upgrade pip; then + echo "Failed to upgrade pip" + fi + fi + # install python3 venv module for linux + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + if ! $SUDO apt install -y "python${PY_VERSION}-venv"; then + echo "Failed to install python${PY_VERSION}-venv"; exit 1 + else + echo "Installed python${PY_VERSION}-venv" fi fi - return 1 + # create virtual environment with specific python version + if "${PY_LOC}" -m venv "${ENV_NAME}"; then + echo "Virtual environment '${ENV_NAME}' created" + else + echo "Virtual environment creation failed"; exit 1 + fi + # activate virtual environment + if source "${ENV_NAME}/bin/activate"; then + echo "Virtual environment '${ENV_NAME}' activated" + else + echo "Virtual environment activation failed"; exit 1 + fi + # install python packages + if "${ENV_NAME}/bin/pip" install -r requirements.txt; then + echo "Success installing python packages" + else + echo "Failure installing python packages" + deactivate + exit 1 + fi + deactivate } + +echo "OS Type: $OSTYPE" +# macOS install with homebrew if [[ "$OSTYPE" == "darwin"* ]]; then + + # macOS does not support running shell scripts as root with homebrew + if [ $EUID -eq 0 ]; then + echo -e "Mac users should run this script without 'sudo'. Exiting..." + exit 1 + fi + CPUS=$(sysctl -n hw.ncpu) # ensure development environment is set correctly for clang $SUDO xcode-select -switch /Library/Developer/CommandLineTools - # see if homebrew is installed and install if not - if ! [[ -f /opt/homebrew/bin/brew ]]; then - echo -e "${green}Installing Homebrew...${end}" - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + + if ! brew --version &>/dev/null; then + echo -e "${cyan}Homebrew is required to install dependencies.${end}" + exit 1 fi - export PATH="/opt/homebrew/bin:$PATH" - brew install llvm@14 googletest google-benchmark lcov make wget cmake curl bash python3 pylint python-matplotlib + brew install llvm@14 googletest google-benchmark lcov make wget cmake bash bc brew upgrade bash - # Add Python 3 to PATH - echo "export PATH=\"/opt/homebrew/opt/python@3/bin:\$PATH\"" >> ~/.bash_profile - # Make python3 default - echo "alias python=python3" >> ~/.bash_profile - . ~/.bash_profile + BREW_ROOT=$(brew --prefix) CLANG_TIDY=/usr/local/bin/clang-tidy if [ ! -L "$CLANG_TIDY" ]; then - $SUDO ln -s $(brew --prefix)/opt/llvm@14/bin/clang-tidy /usr/local/bin/clang-tidy + $SUDO ln -s "${BREW_ROOT}/opt/llvm@14/bin/clang-tidy" /usr/local/bin/clang-tidy fi GMAKE=/usr/local/bin/gmake if [ ! -L "$GMAKE" ]; then $SUDO ln -s $(xcode-select -p)/usr/bin/gnumake /usr/local/bin/gmake fi + PYTHON_PATH='' + # check if supported version of python3 is already installed, and save the version + for PY_VERS in "${PYTHON_VERSIONS[@]}"; do + PYTHON_PATH="${BREW_ROOT}/bin/python${PY_VERS}" + # save the installed version if found + if [ -e "${PYTHON_PATH}" ]; then + PY_INSTALLED=${PY_VERS} + break + fi + done + + # install valid python version if not installed yet + if [ -z "$PY_INSTALLED" ]; then + for PY_VERS in "${PYTHON_VERSIONS[@]}"; do + FULL_PY="python${PY_VERS}" + PYTHON_PATH="${BREW_ROOT}/bin/${FULL_PY}" + # try to install python version from homebrew and verify installation + if brew install "${FULL_PY}"; then + echo "${FULL_PY} installed successfully" + PY_INSTALLED=${PY_VERS} + break + else + # don't exit as we can try on another version if not the last one + echo "${FULL_PY} installation failed" + fi + done + fi + + # leave if no valid python version is installed + if [ -z "$PY_INSTALLED" ]; then + echo "Python3 install with brew failed, attempted on these versions: ${PYTHON_VERSIONS[@]}" + exit 1 + fi + + # create virtual environment and install python packages for the valid python version + create_venv_install_python "${PYTHON_PATH}" ${PY_INSTALLED} + + echo "To activate the virtual env to run python, run 'source .py_venv/bin/activate'" + +# Linux install with apt elif [[ "$OSTYPE" == "linux-gnu"* ]]; then - $SUDO apt update - $SUDO apt install -y build-essential wget cmake libgtest-dev libbenchmark-dev lcov git software-properties-common rsync unzip + # avoids getting stuck on interactive prompts which is essential for CI/CD + export DEBIAN_FRONTEND=noninteractive + $SUDO apt update -y + $SUDO apt upgrade -y + $SUDO apt install -y build-essential wget cmake libgtest-dev libbenchmark-dev \ + lcov git software-properties-common rsync unzip bc + + # Add LLVM GPG key (apt-key is deprecated in Ubuntu 21.04+ so using gpg) + wget -qO - https://apt.llvm.org/llvm-snapshot.gpg.key | \ + gpg --dearmor -o /usr/share/keyrings/llvm-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" | \ + $SUDO tee /etc/apt/sources.list.d/llvm.list - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | $SUDO apt-key add - - $SUDO add-apt-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" + $SUDO apt update -y $SUDO apt install -y clang-format-14 clang-tidy-14 - $SUDO ln -s -f $(which clang-format-14) /usr/local/bin/clang-format - $SUDO ln -s -f $(which clang-tidy-14) /usr/local/bin/clang-tidy + $SUDO ln -sf $(which clang-format-14) /usr/local/bin/clang-format + $SUDO ln -sf $(which clang-tidy-14) /usr/local/bin/clang-tidy - # Python 3.10+ not installed so get the latest version - if ! check_python_version; then - $SUDO add-apt-repository ppa:deadsnakes/ppa - $SUDO apt update + # check if supported version of python3 is already installed, and save the version + PYTHON_PATH='' + for PY_VERS in "${PYTHON_VERSIONS[@]}"; do + PYTHON_PATH="/usr/bin/python${PY_VERS}" + # save the installed version if found + if [ -e "${PYTHON_PATH}" ]; then + PY_INSTALLED=${PY_VERS} + break + fi + done + + # install valid python version if not installed yet + if [ -z "$PY_INSTALLED" ]; then + for PY_VERS in "${PYTHON_VERSIONS[@]}"; do + FULL_PY="python${PY_VERS}" + PYTHON_PATH="/usr/bin/${FULL_PY}" - latest_python_version=$(apt-cache search python3. | grep -o 'python3\.[0-9]*' | sort -V | tail -n 1) - $SUDO apt install -y $latest_python_version + # try to install python version from apt and verify installation + $SUDO apt install -y software-properties-common + $SUDO add-apt-repository -y ppa:deadsnakes/ppa + $SUDO apt update -y + + # install python3 valid version and venv module + if $SUDO apt install -y ${FULL_PY} "${FULL_PY}-pip"; then + echo "${FULL_PY} installed successfully" + PY_INSTALLED=${PY_VERS} + break + else + # don't exit as we can try on another version if not the last one + echo "${FULL_PY} installation failed" + fi + done + fi - $SUDO update-alternatives --install /usr/bin/python3 python3 /usr/bin/$(ls /usr/bin/ | grep -E '^python3\.[0-9]+$' | sort -V | tail -n 1) 1 - $SUDO update-alternatives --config python3 + # leave if no valid python version is installed + if [ -z "$PY_INSTALLED" ]; then + echo "Python3 install with apt failed, attempted on these versions: ${PYTHON_VERSIONS[@]}" + exit 1 fi + + # create virtual environment and install python packages for the valid python version + create_venv_install_python "${PYTHON_PATH}" ${PY_INSTALLED} + + echo "To activate the virtual env to run python, run 'source .py_venv/bin/activate'" + fi -python3 --version PYTHON_TIDY=/usr/local/bin/run-clang-tidy.py if [ ! -f "${PYTHON_TIDY}" ]; then @@ -88,3 +235,6 @@ if [ ! -f "${PYTHON_TIDY}" ]; then wget https://raw.githubusercontent.com/llvm/llvm-project/e837ce2a32369b2e9e8e5d60270c072c7dd63827/clang-tools-extra/clang-tidy/tool/run-clang-tidy.py $SUDO mv run-clang-tidy.py /usr/local/bin fi + +echo "Build environment setup complete." +echo "Next run './scripts/setup-dependencies.sh'." diff --git a/scripts/pylint.sh b/scripts/pylint.sh new file mode 100755 index 000000000..c332fc75a --- /dev/null +++ b/scripts/pylint.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +PREFIX="$(cd "$(dirname "$0")"/.. && pwd)/prefix" +MIN_CODE_QUALITY=8.0 + +get_code_score() { + if [ -n "$1" ]; then + # set minimum quality to user input (int/float) if provided and (5.0 <= input <= 10.0) + if [[ $1 =~ ^[0-9]*([\.][0-9])?$ ]]; then + if (( $(echo "$1 >= 5.0" | bc -l) )) && (( $(echo "$1 <= 10.0" | bc -l) )); then + MIN_CODE_QUALITY=$1 + else + # In the future, we want code quality to be at minimum 8.0/10.0 + echo "Code quality score must be between 5.0 and 10.0, inclusive." + echo "Recommended code quality score is >= 8.0." + exit 1 + fi + else + echo "Code quality score must be an integer or floating point number." + exit 1 + fi + fi + echo "Linting Python code with minimum quality of $MIN_CODE_QUALITY/10.0..." +} + +check_pylint() { + if ! command -v pylint &>/dev/null; then + echo "pylint is not installed." + echo "Run 'sudo ./scripts/install-build-tools.sh' to install pylint." + exit 1 + fi +} + +get_code_score $1 +source .py_env/bin/activate +check_pylint +if ! pylint --rcfile=.pylintrc --fail-under=$MIN_CODE_QUALITY $(git ls-files '*.py'); then + echo "Linting failed, please fix the issues and rerun." + exit 1 +else + echo "Linting passed." +fi +deactivate diff --git a/scripts/setup-dependencies.sh b/scripts/setup-dependencies.sh index 620eae75c..6dbb97a4d 100755 --- a/scripts/setup-dependencies.sh +++ b/scripts/setup-dependencies.sh @@ -10,10 +10,9 @@ set -e # install in a custom prefix rather than /usr/local. by default, this # chooses "prefix" directory alongside "scripts" directory. - PREFIX="$(cd "$(dirname "$0")"/.. && pwd)/prefix" echo "Will install local dependencies in the following prefix: $PREFIX" -mkdir -p $PREFIX $PREFIX/lib $PREFIX/include +mkdir -p "$PREFIX"/{bin,lib,include} CMAKE_BUILD_TYPE="Debug" if [[ "$BUILD_RELEASE" == "1" ]]; then