diff --git a/.ci-setup/texlive-install.sh b/.ci-setup/texlive-install.sh deleted file mode 100755 index 83b57f147..000000000 --- a/.ci-setup/texlive-install.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -set -e -APP_PATH="$(readlink -f "$(dirname "$0")")" -TEX_COMPILER=lualatex - -CTAN_REPO="https://mirror.aarnet.edu.au/pub/CTAN/systems/texlive/tlnet" - -# See if there is a cached version of TL available -# shellcheck disable=SC2155 -export PATH="/tmp/texlive/bin/$(uname -m)-linux:$PATH" -if ! command -v "$TEX_COMPILER" > /dev/null; then - echo "----------------------------------------" - echo "Downloading texlive installer archive from CTAN:" - wget "$CTAN_REPO/install-tl-unx.tar.gz" - tar -xzf install-tl-unx.tar.gz - cd install-tl-20* - - echo "----------------------------------------" - echo "Installing texlive using profile:" - cat "${APP_PATH}"/texlive.profile - echo - ./install-tl --profile="${APP_PATH}/texlive.profile" -repository "$CTAN_REPO" - - echo "----------------------------------------" - echo "Set tlmgr repository:" - tlmgr option repository "$CTAN_REPO" - - echo "----------------------------------------" - echo "Installing additional texlive packages:" - tlmgr install fontawesome luatextra luacode minted fvextra catchfile xstring framed lastpage pdfmanagement-testphase newpax tcolorbox environ pdfcol tikzfill markdown paralist csvsimple gobble upquote tagpdf - - echo "----------------------------------------" - echo "Ensuring the newpax package is sufficiently up to date:" - if NEWPAX_VERSION=$(tlmgr info --only-installed --data cat-version newpax) ; then - if [[ $(echo "$NEWPAX_VERSION < 0.53" | bc) == 1 ]]; then - echo >&2 "Package newpax version lower than 0.53 contain several bugs that are now fixed, giving up."; exit 1; - else - echo "Version $NEWPAX_VERSION found." - fi - else - echo >&2 "Package newpax not found!"; exit 1; - fi - - cd .. - - # Keep no backups (not required, simply makes cache bigger) - tlmgr option -- autobackup 0 -fi - -echo "----------------------------------------" -echo "Installation complete, verifying installation of $TEX_COMPILER." -command -v "$TEX_COMPILER" >/dev/null 2>&1 || { echo >&2 "$TEX_COMPILER is not found."; exit 1; } -# Do a test compile recommended by https://www.tug.org/texlive/quickinstall.html -"$TEX_COMPILER" small2e || { echo >&2 "Failed to process test file with $TEX_COMPILER."; exit 1; } -# remove test files -rm small2e.aux small2e.log small2e.pdf diff --git a/.ci-setup/texlive.profile b/.ci-setup/texlive.profile deleted file mode 100644 index ca5e417b6..000000000 --- a/.ci-setup/texlive.profile +++ /dev/null @@ -1,11 +0,0 @@ -selected_scheme scheme-medium -TEXDIR /tmp/texlive -TEXMFCONFIG ~/.texlive/texmf-config -TEXMFHOME ~/texmf -TEXMFLOCAL /tmp/texlive/texmf-local -TEXMFSYSCONFIG /tmp/texlive/texmf-config -TEXMFSYSVAR /tmp/texlive/texmf-var -TEXMFVAR ~/.texlive/texmf-var -option_doc 0 -option_src 0 -instopt_write18_restricted 1 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 482af4d78..72b8eca1f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -27,6 +27,8 @@ env: DF_ENCRYPTION_DETERMINISTIC_KEY: "anlmuJ6cB3bN3biXRbYvmPsC5ALPFqGG" DF_ENCRYPTION_KEY_DERIVATION_SALT: "hzPR8D4qpOnAg7VeAhkhWw6JmmzKJB10" DF_REDIS_SIDEKIQ_URL: "redis://redis:6379/0" + LATEX_CONTAINER_NAME: doubtfire-texlive + LATEX_BUILD_PATH: /texlive/shell/latex_build.sh jobs: unit-tests: @@ -49,6 +51,16 @@ jobs: uses: actions/checkout@v4 - name: Set up docker buildx uses: docker/setup-buildx-action@v3 + - name: Build TexLive image + uses: docker/build-push-action@v5 + with: + context: . + file: texlive.Dockerfile + push: false + load: true + tags: doubtfire-texlive-development:local + cache-from: type=gha,scope=texlive + cache-to: type=gha,mode=max,scope=texlive - name: Build base doubtfire-api development image uses: docker/build-push-action@v5 with: @@ -56,14 +68,36 @@ jobs: push: false load: true tags: doubtfire-api-development:local - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=doubtfire-api + cache-to: type=gha,mode=max,scope=doubtfire-api + - name: Start TexLive service + uses: addnab/docker-run-action@v3 + with: + image: doubtfire-texlive-development:local + options: > + --name ${{ env.LATEX_CONTAINER_NAME }} + -v ${{ github.workspace }}/student-work:/student-work + -v ${{ github.workspace }}/public/assets/images:/doubtfire/public/assets/images + -v ${{ github.workspace }}/test_files:/doubtfire/test_files + -v ${{ github.workspace }}/tmp/rails-latex:/workdir/texlive-latex + --detach + run: sleep infinity + - name: Test TexLive container + uses: addnab/docker-run-action@v3 + with: + image: doubtfire-api-development:local + options: > + -v ${{ github.workspace }}:/doubtfire + -v /var/run/docker.sock:/var/run/docker.sock + run: docker exec -t ${{ env.LATEX_CONTAINER_NAME }} lualatex -v - name: Populate database uses: addnab/docker-run-action@v3 with: image: doubtfire-api-development:local options: > -v ${{ github.workspace }}:/doubtfire + -v ${{ github.workspace }}/student-work:/student-work + -v /var/run/docker.sock:/var/run/docker.sock -e RAILS_ENV -e DF_STUDENT_WORK_DIR -e DF_INSTITUTION_HOST @@ -81,6 +115,8 @@ jobs: -e DF_ENCRYPTION_DETERMINISTIC_KEY -e DF_ENCRYPTION_KEY_DERIVATION_SALT -e DF_REDIS_SIDEKIQ_URL + -e LATEX_CONTAINER_NAME + -e LATEX_BUILD_PATH run: bundle exec rake db:populate - name: Run unit tests uses: addnab/docker-run-action@v3 @@ -88,6 +124,8 @@ jobs: image: doubtfire-api-development:local options: > -v ${{ github.workspace }}:/doubtfire + -v ${{ github.workspace }}/student-work:/student-work + -v /var/run/docker.sock:/var/run/docker.sock -e RAILS_ENV -e DF_STUDENT_WORK_DIR -e DF_INSTITUTION_HOST @@ -105,4 +143,8 @@ jobs: -e DF_ENCRYPTION_DETERMINISTIC_KEY -e DF_ENCRYPTION_KEY_DERIVATION_SALT -e DF_REDIS_SIDEKIQ_URL + -e LATEX_CONTAINER_NAME + -e LATEX_BUILD_PATH run: TERM=xterm bundle exec rails test + - name: Stop TexLive service + run: docker rm -f ${{ env.LATEX_CONTAINER_NAME }} diff --git a/Dockerfile b/Dockerfile index f85630367..b01c43cba 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,6 @@ RUN apt-get update \ wget \ redis \ libc6-dev \ - librsvg2-bin \ docker-ce \ docker-ce-cli \ containerd.io \ @@ -34,8 +33,6 @@ RUN apt-get update \ WORKDIR /doubtfire COPY ./.ci-setup/ /doubtfire/.ci-setup/ -RUN ./.ci-setup/texlive-install.sh -ENV PATH /tmp/texlive/bin/x86_64-linux:/tmp/texlive/bin/aarch64-linux:$PATH RUN gem install bundler -v '2.4.5' diff --git a/app/helpers/latex_helper.rb b/app/helpers/latex_helper.rb new file mode 100644 index 000000000..d4c6f0c12 --- /dev/null +++ b/app/helpers/latex_helper.rb @@ -0,0 +1,7 @@ +module LatexHelper + def generate_pdf(template:) + raise 'LATEX_CONTAINER_NAME is not set' if ENV['LATEX_CONTAINER_NAME'].nil? + raise 'LATEX_BUILD_PATH is not set' if ENV['LATEX_BUILD_PATH'].nil? + render_to_string(template: template, layout: true) + end +end diff --git a/app/models/pdf_generation/project_compile_portfolio_module.rb b/app/models/pdf_generation/project_compile_portfolio_module.rb index ff1aace2a..8d44c93e0 100644 --- a/app/models/pdf_generation/project_compile_portfolio_module.rb +++ b/app/models/pdf_generation/project_compile_portfolio_module.rb @@ -35,6 +35,8 @@ def compress_portfolio # This class scaffolds the creation of the portfolio - mapping the required data into the erb template class ProjectAppController < ApplicationController + include LatexHelper + attr_accessor :student, :project, :base_path, @@ -66,7 +68,8 @@ def init(project, is_retry) end def make_pdf - render_to_string(template: '/portfolio/portfolio_pdf', layout: true) + logger.debug 'Running make_pdf: (portfolio)' + generate_pdf(template: '/portfolio/portfolio_pdf') end end diff --git a/app/models/task.rb b/app/models/task.rb index aa7ced377..3abf33fa4 100644 --- a/app/models/task.rb +++ b/app/models/task.rb @@ -1062,6 +1062,8 @@ def in_process_files_for_task(is_retry) end class TaskAppController < ApplicationController + include LatexHelper + attr_accessor :task attr_accessor :files attr_accessor :base_path @@ -1086,8 +1088,8 @@ def make_pdf FileHelper.qpdf(f[:path]) end end - logger.debug "Preprocessing complete, rendering file." - render_to_string(template: '/task/task_pdf', layout: true) + logger.debug 'Preprocessing complete, rendering file.' + generate_pdf(template: '/task/task_pdf') end end diff --git a/app/views/layouts/application.pdf.erbtex b/app/views/layouts/application.pdf.erbtex index 982bcda26..fc8051528 100644 --- a/app/views/layouts/application.pdf.erbtex +++ b/app/views/layouts/application.pdf.erbtex @@ -1,4 +1,17 @@ -<% @latex_config={ :recipe => [ {:command => "lualatex",:runs => 2} ], :supporting => Rails.root.join('app', 'views', 'layouts', 'jupynotex.py') } %> +<% +@latex_config = { + :supporting => [ + Rails.root.join('app', 'views', 'layouts', 'jupynotex.py'), + Rails.root.join('lib', 'shell', 'latex_run.sh') + ], + :recipe => [ + { :command => './latex_run.sh', :runs => 1 } + ], + :preservework => false, + :workdir => -> { "#{Process.pid}-#{Thread.current.object_id}-#{Time.now.to_i}" } +} +%> + \DocumentMetadata{uncompress} \documentclass[11pt,a4paper]{article} \usepackage[T1]{fontenc} diff --git a/deployAppSvr.Dockerfile b/deployAppSvr.Dockerfile index aa224e236..cee52099c 100644 --- a/deployAppSvr.Dockerfile +++ b/deployAppSvr.Dockerfile @@ -27,20 +27,14 @@ RUN apt-get update \ cron \ msmtp-mta bsd-mailx \ redis \ - librsvg2-bin \ docker-ce \ docker-ce-cli \ containerd.io \ - librsvg2-bin \ && apt-get clean # Setup the folder where we will deploy the code WORKDIR /doubtfire -# Install LaTex -COPY ./.ci-setup /doubtfire/.ci-setup -RUN /doubtfire/.ci-setup/texlive-install.sh - # Install bundler RUN gem install bundler -v '2.4.5' RUN bundle config set --global without development test staging @@ -49,9 +43,6 @@ RUN bundle config set --global without development test staging COPY ./Gemfile ./Gemfile.lock /doubtfire/ RUN bundle install -# Setup path -ENV PATH /tmp/texlive/bin/x86_64-linux:/tmp/texlive/bin/aarch64-linux:$PATH - # Copy doubtfire-api source COPY . /doubtfire/ diff --git a/docker-compose.yml b/docker-compose.yml index 22520ccaa..3987c71a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,7 @@ services: - ./:/doubtfire - ../data/tmp:/doubtfire/tmp - ../data/student-work:/student-work + - /var/run/docker.sock:/var/run/docker.sock depends_on: - dev-db environment: @@ -61,6 +62,10 @@ services: # RABBITMQ_USERNAME: secure_credentials # RABBITMQ_PASSWORD: secure_credentials + # Latex details + LATEX_CONTAINER_NAME: doubtfire-texlive + LATEX_BUILD_PATH: /texlive/shell/latex_build.sh + dev-db: container_name: doubtfire-dev-db image: mariadb @@ -71,3 +76,17 @@ services: MYSQL_PASSWORD: pwd volumes: - ../data/database:/var/lib/mysql + + texlive: + container_name: doubtfire-texlive + build: + context: . + dockerfile: texlive.Dockerfile + volumes: + - ../data/student-work:/student-work + - ./public/assets/images:/doubtfire/public/assets/images + - ./test_files:/doubtfire/test_files + - ./tmp/rails-latex:/workdir/texlive-latex + depends_on: + - df-api + command: sleep infinity diff --git a/lib/shell/latex_build.sh b/lib/shell/latex_build.sh new file mode 100644 index 000000000..1ba8c8de3 --- /dev/null +++ b/lib/shell/latex_build.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# This script is copied into the TeX Live container and remotely executed by latex_run.sh + +OUTPUT_DIR=$1 + +cd /workdir/texlive-latex/${OUTPUT_DIR} + +# Initialise work subfolder +mkdir -p work +cp *.tex *.py work/ +cd work + +# Compile PDF +lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex +echo "Running lualatex a second time to remove temporary last page..." +lualatex -shell-escape -interaction=batchmode -halt-on-error input.tex + +# Copy PDF to parent directory and cleanup +cp *.log ../ +cp *.pdf ../ +cd .. +rm -rf work diff --git a/lib/shell/latex_run.sh b/lib/shell/latex_run.sh new file mode 100755 index 000000000..d9fbbac07 --- /dev/null +++ b/lib/shell/latex_run.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# This script is copied into tmp/rails-latex/$WORK_DIR/ and executed by rails-latex + +WORK_DIR=$(basename "$PWD") +docker exec -t $LATEX_CONTAINER_NAME $LATEX_BUILD_PATH $WORK_DIR diff --git a/test/models/task_test.rb b/test/models/task_test.rb index 212e75508..117219fd6 100644 --- a/test/models/task_test.rb +++ b/test/models/task_test.rb @@ -369,7 +369,12 @@ def test_ipynb_to_pdf # Test if latex math was rendered properly reader = PDF::Reader.new(task.final_pdf_path) + + # PDF-reader incorrectly parses "weight (kg) / height (m)^2" as "weight (2g) / height (m)", misplacing the ^2 + # Detecting "height" and "weight" confirms correct LaTeX rendering assert reader.pages.last.text.include?("BMI: bmi ="), reader.pages.last.text + assert reader.pages.last.text.include?("weight") + assert reader.pages.last.text.include?("height (m)") # ensure the notice is not included when the notebook doesn't have long lines source code cells # and no errors diff --git a/test_files/submissions/vectorial_graph.ipynb b/test_files/submissions/vectorial_graph.ipynb index 610505cce..fcec0aa6d 100644 --- a/test_files/submissions/vectorial_graph.ipynb +++ b/test_files/submissions/vectorial_graph.ipynb @@ -101,7 +101,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -951,7 +951,7 @@ "source": [ "Formula for calculating\n", "\n", - "BMI: $\\text{bmi}=\\frac{\\text{weight}}{\\text{height}^2}$" + "BMI: $\\text{bmi}=\\frac{\\text{weig}\\text{ht (kg)}}{\\text{hei} \\text{ght (m)}^2}$" ] } ], diff --git a/texlive.Dockerfile b/texlive.Dockerfile new file mode 100644 index 000000000..e7a7f0e3e --- /dev/null +++ b/texlive.Dockerfile @@ -0,0 +1,14 @@ +FROM texlive/texlive:latest + +RUN apt-get update \ + && apt-get install -y \ + imagemagick \ + inkscape \ + librsvg2-bin \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +COPY ./lib/shell/latex_build.sh /texlive/shell/latex_build.sh +RUN chmod +x /texlive/shell/latex_build.sh + +CMD ["sh", "-c", "sleep infinity"]