Add new benchmarks

.github/workflows/publish.yml

@@ -0,0 +1,29 @@
+name: Python package
+on:
+  push:
+    tags:
+      - "v*.*.*"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - name: Install python dependencies
+        run: |
+          pip install poetry
+          poetry install
+          poetry remove torch
+          poetry run pip install torch --index-url https://download.pytorch.org/whl/cpu
+      - name: Build package
+        run: |
+          poetry build
+      - name: Publish package
+        env:
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          poetry config pypi-token.pypi "$PYPI_TOKEN"
+          poetry publish

.github/workflows/tests.yml

@@ -0,0 +1,35 @@
+name: Integration test
+
+on: [push]
+
+env:
+  TORCH_DEVICE: "cpu"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - name: Install python dependencies
+        run: |
+          pip install poetry
+          poetry install
+          poetry remove torch
+          poetry run pip install torch --index-url https://download.pytorch.org/whl/cpu
+      - name: Download benchmark data
+        run: |
+          wget -O benchmark_data.zip "https://drive.google.com/uc?export=download&id=1dbY0kBq2SUa885gmbLPUWSRzy5K7O5XJ"
+          unzip benchmark_data.zip
+          mkdir data
+          mv bench_data.json data/bench_data.json
+      - name: Run benchmark test
+        run: |
+          poetry run texify_benchmark --max 16
+          poetry run python scripts/verify_benchmark_scores.py data/bench_results.json
+        
+          
+

README.md

@@ -1,70 +1,80 @@
 # Texify
 
-Texify converts equations and surrounding text into markdown with LaTeX math that can be rendered by MathJax ($$ and $ are delimiters).  It will work with images or pdfs, and can run on CPU, GPU, or MPS.
+Texify converts equations and surrounding text into markdown and LaTeX that can be rendered by MathJax ($$ and $ are delimiters).  It will work with images or pdfs, and can run on CPU, GPU, or MPS.
 
-https://github.com/VikParuchuri/texify/assets/913340/39b1f139-872f-4ae8-9f31-39e396953bd9
+https://github.com/VikParuchuri/texify/assets/913340/882022a6-020d-4796-af02-67cb77bc084c
 
-> **Example**
-> 
-> ![image](data/examples/0.png)
-> 
-> **Detected Text** The potential $V_{i}$ of cell $\mathcal{C}_{j}$ centred at position $\mathbf{r}_{i}$ is related to the surface charge densities $\sigma_{j}$ of cells $\mathcal{E}_{j}$ $j\in[1,N]$ through the superposition principle as:
-> 
-> $$V_{i}\,=\,\sum_{j=0}^{N}\,\frac{\sigma_{j}}{4\pi\varepsilon_{0}}\,\int_{\mathcal{E}_{j}}\frac{1}{\left|\mathbf{r}_{i}-\mathbf{r}^{\prime}\right|}\,\mathrm{d}^{2}\mathbf{r}^{\prime}\,=\,\sum_{j=0}^{N}\,Q_{ij}\,\sigma_{j},$$
-> 
-> where the integral over the surface of cell $\mathcal{C}_{j}$ only depends on $ \mathcal{C}_{j} $ shape and on the relative position of the target point $\mathbf{r}_{i}$ with respect to $\mathcal{C}_{j}$ location, as $\sigma_{j}$ is assumed constant over the whole surface of cell $\mathcal{C}_{j}$.
+The closest open source comparisons to texify are [pix2tex](https://github.com/lukas-blecher/LaTeX-OCR) and [nougat](https://github.com/facebookresearch/nougat), although they're designed for different purposes:
 
-The closest open source comparisons to texify are pix2tex and nougat, although they're designed for different purposes:
+- Pix2tex is designed for block LaTeX equations, and hallucinates more on text.  Texify can work with inline equations and text.
+- Nougat is designed to OCR entire pages, and hallucinates more on small images. Texify is optimized for equations and small page regions.
 
-- Compared to [pix2tex](https://github.com/lukas-blecher/LaTeX-OCR), texify can detect text and inline equations. Pix2tex is designed for block LaTeX equations, and hallucinates more on text.
-- Compared to [nougat](https://github.com/facebookresearch/nougat), texify is optimized for equations and small page regions.  Nougat is designed to OCR entire pages, and hallucinates more on small images.
-
-I created texify to render equations in [marker](https://github.com/VikParuchuri/marker), but realized it could also be valuable on its own.
+Pix2tex is trained on im2latex, and nougat is trained on arxiv.  Texify is trained on a broader set of web data, and works on a range of images.
 
 See more details in the [benchmarks](#benchmarks) section.
 
 ## Community
 
 [Discord](https://discord.gg//KuZwXNGnfH) is where we discuss future development.
 
+## Examples
+
+![Example 0](data/examples/0.png)
+
+> **Detected Text** The potential $V_{i}$ of cell $\mathcal{C}_{j}$ centred at position $\mathbf{r}_{i}$ is related to the surface charge densities $\sigma_{j}$ of cells $\mathcal{E}_{j}$ $j\in[1,N]$ through the superposition principle as:
+> 
+> $$V_{i}\,=\,\sum_{j=0}^{N}\,\frac{\sigma_{j}}{4\pi\varepsilon_{0}}\,\int_{\mathcal{E}_{j}}\frac{1}{\left|\mathbf{r}_{i}-\mathbf{r}^{\prime}\right|}\,\mathrm{d}^{2}\mathbf{r}^{\prime}\,=\,\sum_{j=0}^{N}\,Q_{ij}\,\sigma_{j},$$
+> 
+> where the integral over the surface of cell $\mathcal{C}_{j}$ only depends on $ \mathcal{C}_{j} $ shape and on the relative position of the target point $\mathbf{r}_{i}$ with respect to $\mathcal{C}_{j}$ location, as $\sigma_{j}$ is assumed constant over the whole surface of cell $\mathcal{C}_{j}$.
+
+| Image                      | OCR Markdown              |
+|----------------------------|---------------------------|
+| [1](data/examples/100.png) | [1](data/examples/100.md) |
+| [2](data/examples/300.png) | [2](data/examples/300.md) |
+| [3](data/examples/400.png) | [3](data/examples/400.md) |
+
 # Installation
 
-This has been tested on Mac and Linux (Ubuntu and Debian).  You'll need python 3.10+ and [poetry](https://python-poetry.org/docs/#installing-with-the-official-installer).
+This has been tested on Mac and Linux (Ubuntu and Debian).  You'll need python 3.10+ and PyTorch. You may need to install the CPU version of torch first if you're not using a Mac or a GPU machine.  See [here](https://pytorch.org/get-started/locally/) for more details.
 
-- `git clone https://github.com/VikParuchuri/texify.git`
-- `cd texify`
-- `poetry install --without dev` # This skips the dev dependencies
+`pip install texify`
 
 Model weights will automatically download the first time you run it.
 
 # Usage
 
-First, some configuration:
-
-- Inspect the settings in `texify/settings.py`.  You can override any settings in a `local.env` file, or by setting environment variables.
+- Inspect the settings in `texify/settings.py`.  You can override any settings with environment variables.
 - Your torch device will be automatically detected, but you can override this.  For example, `TORCH_DEVICE=cuda` or `TORCH_DEVICE=mps`.
 
 ## App for interactive conversion
 
-I've included a streamlit app that lets you interactively select and convert equations from images or PDF files.  To run it, do this:
+I've included a streamlit app that lets you interactively select and convert equations from images or PDF files.  Run it with:
 
 ```
-streamlit run ocr_app.py
+texify_gui
 ```
 
 The app will allow you to select the specific equations you want to convert on each page, then render the results with KaTeX and enable easy copying.
 
-## Convert an image or directory of images
+## Convert images
 
-Run `ocr_image.py`, like this:
+You can OCR a single image or a folder of images with:
 
 ```
-python ocr_image.py /path/to/folder_or_file --max 8 --json_path results.json
+texify /path/to/folder_or_file --max 8 --json_path results.json
 ```
 
 - `--max` is how many images in the folder to convert at most.  Omit this to convert all images in the folder.
 - `--json_path` is an optional path to a json file where the results will be saved.  If you omit this, the results will be saved to `data/results.json`.
 
+# Manual install
+
+If you want to develop texify, you can install it manually:
+
+- `git clone https://github.com/VikParuchuri/texify.git`
+- `cd texify`
+- `poetry install` # This skips the dev dependencies
+
 # Limitations
 
 OCR is complicated, and texify is not perfect.  Here are some known limitations:
@@ -76,30 +86,39 @@ OCR is complicated, and texify is not perfect.  Here are some known limitations:
 
 # Benchmarks
 
-Benchmarking OCR quality is hard - you ideally need a parallel corpus that models haven't been trained on.  I've sampled some images from across a range of sources (web, arxiv, im2latex) to create a representative benchmark set.
+Benchmarking OCR quality is hard - you ideally need a parallel corpus that models haven't been trained on.  I sampled from arxiv and im2latex to create the benchmark set.
+
+![Benchmark results](data/images/texify_bench.png)
+
+Each model is trained on one of the benchmark tasks:
+
+- Nougat was trained on arxiv, possibly the images in the benchmark.
+- Pix2tex was trained on im2latex.
+- Texify was trained on im2latex. It was trained on arxiv, but not the images in the benchmark.
 
-Of these, here is what is known about the training data:
+Although this makes the benchmark results biased, it does seem like a good compromise, since nougat and pix2tex don't work as well out of domain.  Note that neither pix2tex or nougat is really designed for this task (OCR inline equations and text), so this is not a perfect comparison.
 
-- Nougat was trained on arxiv.
-- Pix2tex was trained on im2latex and web images.
-- Texify was trained on im2latex and web images.
+| Model   | BLEU ⬆       | METEOR ⬆     | Edit Distance ⬇ |
+|---------|--------------|--------------|-----------------|
+| pix2tex | 0.382659     | 0.543363     | 0.352533        |
+| nougat  | 0.697667     | 0.668331     | 0.288159        |
+| texify  | **0.837895** | **0.865492** | **0.0842209**   |
 
 ## Running your own benchmarks
 
 You can benchmark the performance of texify on your machine.  
 
-- Clone the repo if you haven't already (see above for manual installation instructions)
-- Install dev dependencies with `poetry install`
-  - If you want to use pix2tex, run `pip install pix2tex`
-  - If you want to use nougat, run `pip install nougat-ocr`
-- Download the benchmark data [here]() and put it in the `data` folder.
+- Follow the manual install instructions above.
+- If you want to use pix2tex, run `pip install pix2tex`
+- If you want to use nougat, run `pip install nougat-ocr`
+- Download the benchmark data [here](https://drive.google.com/file/d/1dbY0kBq2SUa885gmbLPUWSRzy5K7O5XJ/view?usp=sharing) and put it in the `data` folder.
 - Run `benchmark.py` like this:
 
 ```
 python benchmark.py --max 100 --pix2tex --nougat --data_path data/bench_data.json --result_path data/bench_results.json
 ```
 
-This will benchmark marker against Latex-OCR.  It will do batch inference with texify, but not with Latex-OCR, since I couldn't find an option for batching.
+This will benchmark marker against pix2tex and nougat.  It will do batch inference with texify and nougat, but not with pix2tex, since I couldn't find an option for batching.
 
 - `--max` is how many benchmark images to convert at most.
 - `--data_path` is the path to the benchmark data.  If you omit this, it will use the default path.
@@ -109,17 +128,17 @@ This will benchmark marker against Latex-OCR.  It will do batch inference with t
 
 # Training
 
-Texify was trained on latex images and paired equations from across the web.  It includes the [im2latex](https://github.com/guillaumegenthial/im2latex) dataset.  Training happened on 4x A6000 GPUs for 3 days.
+Texify was trained on latex images and paired equations from across the web.  It includes the [im2latex](https://github.com/guillaumegenthial/im2latex) dataset.  Training happened on 4x A6000s for 2 days (~6 epochs).
 
 # Commercial usage
 
-This model is trained on top of the openly licensed [Donut](https://huggingface.co/naver-clova-ix/donut-base) model, and thus can be used for commercial purposes.
+This model is trained on top of the openly licensed [Donut](https://huggingface.co/naver-clova-ix/donut-base) model, and thus can be used for commercial purposes.  Model weights are licensed under the [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) license.
 
 # Thanks
 
-This work would not have been possible without lots of amazing open source work.  I particularly want to acknowledge Lukas Blecher, whose work on Nougat and Latex-OCR was key for this project.  I learned a lot from his code, and used parts of it for texify.
+This work would not have been possible without lots of amazing open source work.  I particularly want to acknowledge [Lukas Blecher](https://github.com/lukas-blecher), whose work on Nougat and pix2tex was key for this project.  I learned a lot from his code, and used parts of it for texify.
 
 - [im2latex](https://github.com/guillaumegenthial/im2latex) - one of the datasets used for training
 - [Donut](https://huggingface.co/naver-clova-ix/donut-base) from Naver, the base model for texify
-- [Nougat](https://github.com/facebookresearch/nougat) - I used the tokenized from Nougat
+- [Nougat](https://github.com/facebookresearch/nougat) - I used the tokenizer from Nougat
 - [Latex-OCR](https://github.com/lukas-blecher/LaTeX-OCR) - The original open source Latex OCR project

benchmark.py

@@ -146,6 +146,7 @@ def main():
 
     source_path = os.path.abspath(args.data_path)
     result_path = os.path.abspath(args.result_path)
+    os.makedirs(os.path.dirname(result_path), exist_ok=True)
     model = load_model()
     processor = load_processor()
 

ocr_app.py

@@ -9,6 +9,7 @@
 from texify.inference import batch_inference
 from texify.model.model import load_model
 from texify.model.processor import load_processor
+from texify.settings import settings
 import subprocess
 import re
 from PIL import Image
@@ -30,9 +31,9 @@ def load_processor_cached():
 
 
 @st.cache_data()
-def infer_image(pil_image, bbox):
+def infer_image(pil_image, bbox, temperature):
     input_img = pil_image.crop(bbox)
-    model_output = batch_inference([input_img], model, processor)
+    model_output = batch_inference([input_img], model, processor, temperature=temperature)
     return model_output[0]
 
 
@@ -85,7 +86,9 @@ def get_image_size(pil_image):
 
 top_message = """### Texify
 
-After the model loads, upload an image or a pdf, then draw a box around the equation or text you want to OCR by clicking and dragging. Texify will convert it to Markdown with LaTeX math on the right.  If you have already cropped your image, select "OCR image" in the sidebar instead.
+After the model loads, upload an image or a pdf, then draw a box around the equation or text you want to OCR by clicking and dragging. Texify will convert it to Markdown with LaTeX math on the right. If you don't get good results, try selecting a slightly different box, or changing the temperature value.
+
+If you have already cropped your image, select "OCR image" in the sidebar instead.
 """
 
 st.markdown(top_message)
@@ -109,6 +112,8 @@ def get_image_size(pil_image):
     pil_image = get_uploaded_image(in_file)
     whole_image = st.sidebar.button("OCR image")
 
+temperature = st.sidebar.slider("Generation temperature:", min_value=0.0, max_value=1.0, value=0.0, step=0.05)
+
 canvas_hash = get_canvas_hash(pil_image) if pil_image else "canvas"
 
 with col1:
@@ -140,7 +145,7 @@ def get_image_size(pil_image):
 
     if bbox_list:
         with col2:
-            inferences = [infer_image(pil_image, bbox) for bbox in bbox_list]
+            inferences = [infer_image(pil_image, bbox, temperature) for bbox in bbox_list]
             for idx, inference in enumerate(reversed(inferences)):
                 st.markdown(f"### {len(inferences) - idx}")
                 katex_markdown = replace_katex_invalid(inference)

ocr_image.py

@@ -14,9 +14,8 @@ def inference_single_image(image_path, json_path, model, processor):
     image = Image.open(image_path)
     text = batch_inference([image], model, processor)
     write_data = [{"image_path": image_path, "text": text[0]}]
-    with open(json_path, "w") as f:
+    with open(json_path, "w+") as f:
         json_repr = json.dumps(write_data, indent=4)
-        print(json_repr)
         f.write(json_repr)
 
 
@@ -34,9 +33,8 @@ def inference_image_dir(image_dir, json_path, model, processor, max=None):
         for image_path, t in zip(batch, text):
             write_data.append({"image_path": image_path, "text": t})
 
-    with open(json_path, "w") as f:
+    with open(json_path, "w+") as f:
         json_repr = json.dumps(write_data, indent=4)
-        print(json_repr)
         f.write(json_repr)
 
 
@@ -52,6 +50,7 @@ def main():
     processor = load_processor()
 
     json_path = os.path.abspath(args.json_path)
+    os.makedirs(os.path.dirname(json_path), exist_ok=True)
 
     if os.path.isfile(image_path):
         inference_single_image(image_path, json_path, model, processor)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "texify"
-version = "0.1.0"
+version = "0.1.2"
 description = "OCR for latex images"
 authors = ["Vik Paruchuri <[email protected]>"]
 readme = "README.md"
@@ -26,9 +26,9 @@ numpy = "^1.26.2"
 pypdfium2 = "^4.25.0"
 python-dotenv = "^1.0.0"
 watchdog = "^3.0.0"
-streamlit-drawable-canvas = {git = "https://github.com/VikParuchuri/streamlit-drawable-canvas.git", rev = "develop"}
 ftfy = "^6.1.3"
 tabulate = "^0.9.0"
+streamlit-drawable-canvas-jsretry = "^0.9.3"
 
 [tool.poetry.group.dev.dependencies]
 jupyter = "^1.0.0"