Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Jul 26, 2024
1 parent dca8a90 commit 32baacc
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 92 deletions.
174 changes: 86 additions & 88 deletions .github/workflows/examples.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

name: Test Jupyter Notebooks

on:
Expand All @@ -12,7 +11,7 @@ on:
pull_request:
branches:
- main
- test-examples-ci
- test-examples-ci

jobs:
test-notebooks:
Expand All @@ -24,101 +23,100 @@ jobs:
- examples/computer_vision/fashion_product_images

steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Checkout repository
uses: actions/checkout@v4

# Set up environment for the { matrix.dir }
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'

- name: Check for examples
shell: bash
run: |
echo "Checking for Jupyter Notebooks and Python scripts..."
if ls ${{ matrix.dir }}/*.ipynb 1> /dev/null 2>&1; then
echo "IPYNB_EXAMPLES_FOUND=true" >> $GITHUB_ENV
else
echo "IPYNB_EXAMPLES_FOUND=false" >> $GITHUB_ENV
fi
- name: Check for examples
shell: bash
run: |
echo "Checking for Jupyter Notebooks and Python scripts..."
if ls ${{ matrix.dir }}/scripts/*.py 1> /dev/null 2>&1; then
echo "PY_EXAMPLES_FOUND=true" >> $GITHUB_ENV
else
echo "PY_EXAMPLES_FOUND=false" >> $GITHUB_ENV
fi
if ls ${{ matrix.dir }}/*.ipynb 1> /dev/null 2>&1; then
echo "IPYNB_EXAMPLES_FOUND=true" >> $GITHUB_ENV
else
echo "IPYNB_EXAMPLES_FOUND=false" >> $GITHUB_ENV
fi
if ls ${{ matrix.dir }}/scripts/*.py 1> /dev/null 2>&1; then
echo "PY_EXAMPLES_FOUND=true" >> $GITHUB_ENV
else
echo "PY_EXAMPLES_FOUND=false" >> $GITHUB_ENV
fi
- name: Cache Python dependencies
if: env.IPYNB_EXAMPLES_FOUND == 'true' || env.PY_EXAMPLES_FOUND == 'true'
uses: actions/cache@v3
with:
path: |
~/.cache/pip
${{ matrix.dir }}/.venv
key: ${{ runner.os }}-pip-${{ hashFiles('${{ matrix.dir }}/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Set up virtual environment
if: env.IPYNB_EXAMPLES_FOUND == 'true' || env.PY_EXAMPLES_FOUND == 'true'
run: |
python -m venv ${{ matrix.dir }}/.venv
source ${{ matrix.dir }}/.venv/bin/activate
python -m pip install --upgrade pip
pip install notebook papermill
pip install git+https://github.com/iterative/datachain.git
if [ -f "${{ matrix.dir }}/requirements.txt" ]; then
pip install -r ${{ matrix.dir }}/requirements.txt
fi
shell: bash
- name: Cache Python dependencies
if: env.IPYNB_EXAMPLES_FOUND == 'true' || env.PY_EXAMPLES_FOUND == 'true'
uses: actions/cache@v3
with:
path: |
~/.cache/pip
${{ matrix.dir }}/.venv
key: ${{ runner.os }}-pip-${{ hashFiles('${{ matrix.dir }}/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Set up virtual environment
if: env.IPYNB_EXAMPLES_FOUND == 'true' || env.PY_EXAMPLES_FOUND == 'true'
run: |
python -m venv ${{ matrix.dir }}/.venv
source ${{ matrix.dir }}/.venv/bin/activate
python -m pip install --upgrade pip
pip install notebook papermill
pip install git+https://github.com/iterative/datachain.git
if [ -f "${{ matrix.dir }}/requirements.txt" ]; then
pip install -r ${{ matrix.dir }}/requirements.txt
fi
shell: bash

# Test Jupyter Notebooks
- name: Test notebooks
if: env.IPYNB_EXAMPLES_FOUND == 'true'
run: |
source ${{ matrix.dir }}/.venv/bin/activate
chmod +x ./tests/examples/test_notebooks.sh
./tests/examples/test_notebooks.sh -d ${{ matrix.dir }}
shell: bash
- name: Test notebooks
if: env.IPYNB_EXAMPLES_FOUND == 'true'
run: |
source ${{ matrix.dir }}/.venv/bin/activate
chmod +x ./tests/examples/test_notebooks.sh
./tests/examples/test_notebooks.sh -d ${{ matrix.dir }}
shell: bash

- name: Commit & push changes
run: |
git config --global user.name 'Test Examples CI'
git config --global user.email '[email protected]'
git add -A
git commit -m "Update Jupyter Notebooks with test results" || echo "No changes to commit"
git push
- name: Commit & push changes
run: |
git config --global user.name 'Test Examples CI'
git config --global user.email '[email protected]'
git add -A
git commit -m "Update Jupyter Notebooks with test results" || echo "No changes to commit"
git push
# Test .py scripts designed for Studio (in scripts/ dir)
- name: Test .py scripts
if: env.PY_EXAMPLES_FOUND == 'true'
run: |
set -e
source ${{ matrix.dir }}/.venv/bin/activate
cd ${{ matrix.dir }}
find scripts -type f -name '*.py' -print | sort | while read script; do
echo "Testing use case in $script"
datachain query $script
done
shell: bash

# Save artifacts
- name: Set sanitized directory name
if: always()
run: |
sanitized_dir=$(echo "${{ matrix.dir }}" | tr '/' '-')
echo "SANITIZED_DIR=$sanitized_dir" >> $GITHUB_ENV
- name: Test .py scripts
if: env.PY_EXAMPLES_FOUND == 'true'
run: |
set -e
source ${{ matrix.dir }}/.venv/bin/activate
cd ${{ matrix.dir }}
find scripts -type f -name '*.py' -print | sort | while read script; do
echo "Testing use case in $script"
datachain query $script
done
shell: bash

- name: Upload failed notebook outputs as artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: test-notebooks-${{ env.SANITIZED_DIR }}
path: |
output-*.ipynb
log-test-notebooks-*.log
# Save artifacts
- name: Set sanitized directory name
if: always()
run: |
sanitized_dir=$(echo "${{ matrix.dir }}" | tr '/' '-')
echo "SANITIZED_DIR=$sanitized_dir" >> $GITHUB_ENV
- name: Upload failed notebook outputs as artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: test-notebooks-${{ env.SANITIZED_DIR }}
path: |
output-*.ipynb
log-test-notebooks-*.log
Original file line number Diff line number Diff line change
Expand Up @@ -3105,4 +3105,4 @@
},
"nbformat": 4,
"nbformat_minor": 5
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5309,4 +5309,4 @@
},
"nbformat": 4,
"nbformat_minor": 5
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6323,4 +6323,4 @@
},
"nbformat": 4,
"nbformat_minor": 5
}
}
2 changes: 1 addition & 1 deletion tests/examples/test_notebooks.sh
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ for notebook in $(ls *.ipynb | sort); do
# Check if the notebook ran successfully
if [ $status -eq 0 ]; then
echo -e "${notebook} - ${GREEN}SUCCESS${NC}"

else
echo -e "${notebook} - ${RED}FAIL${NC}" | tee -a "$LOG_FILE"
overall_status=1
Expand Down

0 comments on commit 32baacc

Please sign in to comment.