From 64c78644ee59d34781523ae2bd38bf3892ce3fe7 Mon Sep 17 00:00:00 2001
From: F-G Fernandez <76527547+fg-mindee@users.noreply.github.com>
Date: Fri, 3 Sep 2021 14:38:30 +0200
Subject: [PATCH] chore: Added CI jobs to check evaluate, collect_env & demo
 (#456)

* chore: Added CI job to check whether the demo runs

* chore: Added CI jobs to check remaining scripts

* feat: Added options to run evaluation on limited samples

* test: Speeded up evaluation script testing
---
 .github/workflows/demo.yml    | 49 +++++++++++++++++++++++++++++++++
 .github/workflows/scripts.yml | 52 +++++++++++++++++++++++++++++++++++
 scripts/evaluate.py           |  9 ++++++
 3 files changed, 110 insertions(+)
 create mode 100644 .github/workflows/demo.yml

diff --git a/.github/workflows/demo.yml b/.github/workflows/demo.yml
new file mode 100644
index 0000000000..6baac72f40
--- /dev/null
+++ b/.github/workflows/demo.yml
@@ -0,0 +1,49 @@
+name: demo
+
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: main
+
+jobs:
+  test-demo:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python: [3.7, 3.8]
+    steps:
+      - if: matrix.os == 'macos-latest'
+        name: Install MacOS prerequisites
+        run: brew install cairo pango gdk-pixbuf libffi
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+      - name: Cache python modules
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('demo/requirements.txt') }}-${{ hashFiles('**/*.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('demo/requirements.txt') }}-
+            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-
+            ${{ runner.os }}-pkg-deps-
+            ${{ runner.os }}-
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[tf] --upgrade
+          pip install -r demo/requirements.txt
+
+      - name: Run demo
+        run: |
+          streamlit --version
+          screen -dm streamlit run demo/app.py
+          sleep 5
+          curl http://localhost:8501
\ No newline at end of file
diff --git a/.github/workflows/scripts.yml b/.github/workflows/scripts.yml
index ceee89dd19..8eaadc1206 100644
--- a/.github/workflows/scripts.yml
+++ b/.github/workflows/scripts.yml
@@ -43,3 +43,55 @@ jobs:
         run: |
           wget https://github.com/mindee/doctr/releases/download/v0.1.0/sample.pdf
           python scripts/analyze.py sample.pdf --noblock
+
+  test-evaluate:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+        python: [3.7, 3.8]
+    steps:
+      - if: matrix.os == 'macos-latest'
+        name: Install MacOS prerequisites
+        run: brew install cairo pango gdk-pixbuf libffi
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+      - name: Cache python modules
+        uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('**/*.py') }}
+          restore-keys: |
+            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-${{ hashFiles('requirements.txt') }}-
+            ${{ runner.os }}-pkg-deps-${{ matrix.python }}-
+            ${{ runner.os }}-pkg-deps-
+            ${{ runner.os }}-
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[tf] --upgrade
+      - name: Run evaluation script
+        run: |
+          python scripts/evaluate.py db_resnet50 crnn_vgg16_bn --samples 10
+
+  test-collectenv:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python: [3.7, 3.8]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: x64
+      - name: Run environment collection script
+        run: python scripts/collect_env.py
diff --git a/scripts/evaluate.py b/scripts/evaluate.py
index 77141691f1..1373021e7b 100644
--- a/scripts/evaluate.py
+++ b/scripts/evaluate.py
@@ -52,6 +52,7 @@ def main(args):
         det_metric = LocalizationConfusion(iou_thresh=args.iou, rotated_bbox=args.rotation)
         e2e_metric = OCRMetric(iou_thresh=args.iou, rotated_bbox=args.rotation)
 
+    sample_idx = 0
     for dataset in sets:
         for page, target in tqdm(dataset):
             # GT
@@ -99,6 +100,13 @@ def main(args):
             reco_metric.update(gt_labels, reco_words)
             e2e_metric.update(gt_boxes, np.asarray(pred_boxes), gt_labels, pred_labels)
 
+            # Loop break
+            sample_idx += 1
+            if isinstance(args.samples, int) and args.samples == sample_idx:
+                break
+        if isinstance(args.samples, int) and args.samples == sample_idx:
+            break
+
     # Unpack aggregated metrics
     print(f"Model Evaluation (model= {args.detection} + {args.recognition}, "
           f"dataset={'OCRDataset' if args.img_folder else args.dataset})")
@@ -125,6 +133,7 @@ def parse_args():
     parser.add_argument('--rotation', dest='rotation', action='store_true', help='evaluate with rotated bbox')
     parser.add_argument('-b', '--batch_size', type=int, default=32, help='batch size for recognition')
     parser.add_argument('--mask_shape', type=int, default=None, help='mask shape for mask iou (only for rotation)')
+    parser.add_argument('--samples', type=int, default=None, help='evaluate only on the N first samples')
     args = parser.parse_args()
 
     return args