improve pre-commit setup

.github/workflows/pr-checks.yml

@@ -1,12 +1,10 @@
 name: PR Checks
-
 on:
   push:
     branches: [main, dev]
   pull_request:
     branches: [main, dev]
   workflow_dispatch:
-
 jobs:
   unit_tests:
     name: PR Checks
@@ -21,15 +19,14 @@ jobs:
       - name: Code style
         run: ruff check || true
         if: success() || failure()
-
-  #code_style:
-  #  name: Code Style
-  #  runs-on: ubuntu-latest
-  #  steps:
-  #    - uses: actions/checkout@v4
-  #    - uses: ./.github/actions/setup-depiction
-  #      with:
-  #        python-version: 3.11
-  #    - name: Check code with ruff
-  #      run:
-  #        ruff check || true
+#code_style:
+#  name: Code Style
+#  runs-on: ubuntu-latest
+#  steps:
+#    - uses: actions/checkout@v4
+#    - uses: ./.github/actions/setup-depiction
+#      with:
+#        python-version: 3.11
+#    - name: Check code with ruff
+#      run:
+#        ruff check || true

.pre-commit-config.yaml

@@ -1,26 +1,60 @@
 ci:
   autoupdate_commit_msg: "chore: update pre-commit hooks"
   autofix_commit_msg: "style: pre-commit fixes"
-
 repos:
   - repo: https://github.com/psf/black
     rev: "24.4.2"
     hooks:
       - id: black
-
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v3.1.0"
-    hooks:
-      - id: prettier
-        types_or: [yaml, markdown, html, css, scss, javascript, json]
-        args: [--prose-wrap=always]
-
+  - repo: https://github.com/adamchainz/blacken-docs
+    rev: "1.16.0"
+    hooks:
+      - id: blacken-docs
+        additional_dependencies: [black==24.*]
+  # TODO add later, when improved
+  #- repo: https://github.com/astral-sh/ruff-pre-commit
+  #  rev: "v0.4.1"
+  #  hooks:
+  #    - id: ruff
+  #      args: ["--fix", "--show-fixes"]
+  #    - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: "v4.6.0"
+    hooks:
+      - id: check-added-large-files
+      - id: check-case-conflict
+      - id: check-merge-conflict
+      - id: check-symlinks
+      - id: check-yaml
+      - id: debug-statements
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+      - id: pretty-format-json
+        args:
+          - "--autofix"
+          - "--indent=4"
+          - "--no-sort-keys"
+  - repo: https://github.com/google/yamlfmt
+    rev: "v0.13.0"
+    hooks:
+      - id: yamlfmt
+  - repo: https://github.com/snakemake/snakefmt
+    rev: "v0.10.2"
+    hooks:
+      - id: snakefmt
+  - repo: https://github.com/executablebooks/mdformat/
+    rev: "0.7.17"
+    hooks:
+      - id: mdformat
   - repo: https://github.com/abravalheri/validate-pyproject
     rev: "v0.16"
     hooks:
       - id: validate-pyproject
         additional_dependencies: ["validate-pyproject-schema-store[all]"]
-
   - repo: https://github.com/python-jsonschema/check-jsonschema
     rev: "0.28.2"
     hooks:

src/depiction_targeted_preproc/workflow/Snakefile

@@ -1,5 +1,6 @@
 version: "3"
 
+
 include: "rules/rules_proc.smk"
 include: "rules/rules_vis.smk"
 include: "rules/rules_qc.smk"

src/depiction_targeted_preproc/workflow/experimental.smk

@@ -1,28 +1,34 @@
 version: "3"
 
+
 include: "rules/rules_proc.smk"
 include: "rules/rules_vis.smk"
 include: "rules/rules_qc.smk"
 include: "rules/rules_simulate.smk"
 
+
 exp_variants = ["chem_noise", "mass_cluster", "reg_shift"]
 
+
 rule exp_compare_cluster_stats:
     input:
-        csv=expand("{{sample}}/{exp_variant}/cluster_default_stats_hdbscan.csv",exp_variant=exp_variants)
+        csv=expand("{{sample}}/{exp_variant}/cluster_default_stats_hdbscan.csv", exp_variant=exp_variants),
     output:
-        pdf="{sample}/exp_compare_cluster_stats.pdf"
+        pdf="{sample}/exp_compare_cluster_stats.pdf",
     shell:
         "python -m depiction_targeted_preproc.workflow.exp.compare_cluster_stats"
         " {input.csv}"
         " --output-pdf {output}"
 
+
 rule exp_plot_compare_peak_density:
     input:
-        tables_marker_distance=expand("{{sample}}/{exp_variant}/qc/table_marker_distances_calib.parquet",exp_variant=exp_variants),
+        tables_marker_distance=expand(
+            "{{sample}}/{exp_variant}/qc/table_marker_distances_calib.parquet", exp_variant=exp_variants
+        ),
         table_marker_distance_uncalib="{sample}/reg_shift/qc/table_marker_distances_baseline.parquet",
     output:
-        pdf="{sample}/exp_plot_compare_peak_density.pdf"
+        pdf="{sample}/exp_plot_compare_peak_density.pdf",
     shell:
         "python -m depiction_targeted_preproc.workflow.exp.plot_compare_peak_density"
         " {input.tables_marker_distance}"
@@ -33,53 +39,57 @@ rule exp_plot_compare_peak_density:
 # for the poster:
 rule qc_table_marker_distances_baseline_mini:
     input:
-        imzml=multiext("{sample}/corrected.peaks",".imzML",".ibd"),
+        imzml=multiext("{sample}/corrected.peaks", ".imzML", ".ibd"),
         mass_list="{sample}/mass_list.visualization_mini.csv",
     output:
-        table="{sample}/qc/table_marker_distances_baseline_mini.parquet"
+        table="{sample}/qc/table_marker_distances_baseline_mini.parquet",
     shell:
         "python -m depiction_targeted_preproc.workflow.qc.table_marker_distances"
         " --imzml-peaks {input.imzml[0]} --mass-list {input.mass_list}"
         " --output-table {output.table}"
 
+
 rule qc_table_marker_distances_calib_mini:
     input:
         imzml_peaks="{sample}/calibrated.imzML",
         mass_list="{sample}/mass_list.visualization_mini.csv",
     output:
-        table="{sample}/qc/table_marker_distances_calib_mini.parquet"
+        table="{sample}/qc/table_marker_distances_calib_mini.parquet",
     shell:
         "python -m depiction_targeted_preproc.workflow.qc.table_marker_distances"
         " --imzml-peaks {input.imzml_peaks} --mass-list {input.mass_list}"
         " --output-table {output.table}"
 
+
 rule qc_plot_marker_presence_mini:
     input:
         table_marker_distances_baseline="{sample}/qc/table_marker_distances_baseline_mini.parquet",
-        table_marker_distances_calib="{sample}/qc/table_marker_distances_calib_mini.parquet"
+        table_marker_distances_calib="{sample}/qc/table_marker_distances_calib_mini.parquet",
     output:
-        pdf="{sample}/qc/plot_marker_presence_mini.pdf"
+        pdf="{sample}/qc/plot_marker_presence_mini.pdf",
     shell:
         "python -m depiction_targeted_preproc.workflow.qc.plot_marker_presence"
         " --table-marker-distances-baseline {input.table_marker_distances_baseline}"
         " --table-marker-distances-calib {input.table_marker_distances_calib}"
         " --layout-vertical"
         " --output-pdf {output.pdf}"
 
+
 variants_with_map = ["mass_cluster", "reg_shift"]
 
+
 rule exp_plot_map_comparison:
     input:
-        mass_shifts=expand("{{sample}}/{exp_variant}/test_mass_shifts.hdf5",exp_variant=variants_with_map)
+        mass_shifts=expand("{{sample}}/{exp_variant}/test_mass_shifts.hdf5", exp_variant=variants_with_map),
     output:
-        pdf="{sample}/exp_plot_map_comparison.pdf"
+        pdf="{sample}/exp_plot_map_comparison.pdf",
     shell:
         "python -m depiction_targeted_preproc.workflow.exp.plot_map_comparison"
         " {input.mass_shifts}"
         " --output-pdf-path {output.pdf}"
 
 
-#rule exp_plot_map_single_for_poster:
+# rule exp_plot_map_single_for_poster:
 #    input:
 #        mass_shift="{sample}/test_mass_shifts.hdf5"
 #    output:
@@ -88,4 +98,4 @@ rule exp_plot_map_comparison:
 #        "python -m depiction_targeted_preproc.workflow.exp.plot_map_single"
 #        " --input-mass-shift-path {input.mass_shift}"
 #        " --output-pdf-path {output.pdf}"
-#
+#

src/depiction_targeted_preproc/workflow/rules/rules_proc.smk

@@ -1,9 +1,9 @@
 rule proc_correct_baseline:
     input:
-        imzml=multiext("{sample}/raw",".imzML",".ibd"),
+        imzml=multiext("{sample}/raw", ".imzML", ".ibd"),
         config="{sample}/pipeline_params.yml",
     output:
-        imzml=temp(multiext("{sample}/corrected.original",".imzML",".ibd"))
+        imzml=temp(multiext("{sample}/corrected.original", ".imzML", ".ibd")),
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.correct_baseline "
         " --input-imzml-path {input.imzml[0]} --config-path {input.config} "
@@ -12,18 +12,18 @@ rule proc_correct_baseline:
 
 rule proc_pick_peaks:
     input:
-        imzml=multiext("{sample}/corrected.original",".imzML",".ibd"),
+        imzml=multiext("{sample}/corrected.original", ".imzML", ".ibd"),
         config="{sample}/pipeline_params.yml",
     output:
-        imzml=multiext("{sample}/corrected.peaks",".imzML",".ibd"),
+        imzml=multiext("{sample}/corrected.peaks", ".imzML", ".ibd"),
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.pick_peaks "
         " --input-imzml-path {input.imzml[0]} --config-path {input.config} "
         " --output-imzml-path {output.imzml[0]}"
 
 
 # TODO currently needed for peak picked data, e.g. from timsTOF
-#rule proc_pick_peaks:
+# rule proc_pick_peaks:
 #    input:
 #        imzml=multiext("{sample}/corrected.original",".imzML",".ibd"),
 #        config="{sample}/pipeline_params.yml",
@@ -36,7 +36,7 @@ rule proc_pick_peaks:
 #
 #
 ## TODO very experimental to be removed later again
-#rule proc_filter_peaks:
+# rule proc_filter_peaks:
 #    input:
 #        imzml=multiext("{sample}/corrected.peaks_all",".imzML",".ibd"),
 #    output:
@@ -46,78 +46,85 @@ rule proc_pick_peaks:
 #        " --input-imzml-path {input.imzml[0]} "
 #        " --output-imzml-path {output.imzml[0]}"
 
+
 # TODO this should be solved more efficiently in the future, but for now it is solved by calling the script twice
 rule proc_calibrate_remove_global_shift:
     input:
-        imzml=multiext("{sample}/corrected.peaks",".imzML",".ibd"),
+        imzml=multiext("{sample}/corrected.peaks", ".imzML", ".ibd"),
         config="{sample}/pipeline_params.yml",
         #mass_list="{sample}/mass_list.standards.csv",
         mass_list="{sample}/mass_list.calibration.csv",
     output:
-        imzml=temp(multiext("{sample}/calibrated.tmp",".imzML",".ibd")),
+        imzml=temp(multiext("{sample}/calibrated.tmp", ".imzML", ".ibd")),
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.calibrate "
         " --input-imzml-path {input.imzml[0]} --config-path {input.config} --mass-list-path {input.mass_list} "
         " --use-global-constant-shift"
         " --output-imzml-path {output.imzml[0]}"
 
+
 rule proc_calibrate_actual:
     input:
-        imzml=multiext("{sample}/calibrated.tmp",".imzML",".ibd"),
+        imzml=multiext("{sample}/calibrated.tmp", ".imzML", ".ibd"),
         config="{sample}/pipeline_params.yml",
         mass_list="{sample}/mass_list.calibration.csv",
     output:
-        imzml=multiext("{sample}/calibrated",".imzML",".ibd"),
+        imzml=multiext("{sample}/calibrated", ".imzML", ".ibd"),
         calib_data="{sample}/calib_data.hdf5",
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.calibrate "
         " --input-imzml-path {input.imzml[0]} --config-path {input.config} --mass-list-path {input.mass_list} "
         " --output-imzml-path {output.imzml[0]} --output-calib-data-path {output.calib_data}"
 
+
 rule proc_export_raw_metadata:
     input:
         imzml="{sample}/raw.imzML",
     output:
-        json="{sample}/raw_metadata.json"
+        json="{sample}/raw_metadata.json",
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.export_raw_metadata "
         " --input-imzml-path {input.imzml} --output-json-path {output.json}"
 
+
 rule proc_cluster_kmeans:
     input:
-        netcdf="{sample}/images_default.hdf5"
+        netcdf="{sample}/images_default.hdf5",
     output:
-        netcdf="{sample}/cluster_default_kmeans.hdf5"
+        netcdf="{sample}/cluster_default_kmeans.hdf5",
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.cluster_kmeans "
         " --input-netcdf-path {input.netcdf} --output-netcdf-path {output.netcdf}"
 
+
 rule proc_cluster_hdbscan:
     input:
-        netcdf="{sample}/images_default.hdf5"
+        netcdf="{sample}/images_default.hdf5",
     output:
-        netcdf="{sample}/cluster_default_hdbscan.hdf5"
+        netcdf="{sample}/cluster_default_hdbscan.hdf5",
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.cluster_hdbscan "
         " --input-netcdf-path {input.netcdf} --output-netcdf-path {output.netcdf}"
 
+
 rule proc_cluster_stats:
     input:
-        netcdf="{sample}/cluster_default_{variant}.hdf5"
+        netcdf="{sample}/cluster_default_{variant}.hdf5",
     output:
-        csv="{sample}/cluster_default_stats_{variant}.csv"
+        csv="{sample}/cluster_default_stats_{variant}.csv",
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.cluster_stats"
         " --input-netcdf-path {input.netcdf} --output-csv-path {output.csv}"
 
+
 rule proc_mass_list_preparation:
     input:
-        csv="{sample}/mass_list.raw.csv"
+        csv="{sample}/mass_list.raw.csv",
     output:
         calibration_csv="{sample}/mass_list.calibration.csv",
         standards_csv="{sample}/mass_list.standards.csv",
         visualization_csv="{sample}/mass_list.visualization.csv",
-        visualization_mini_csv="{sample}/mass_list.visualization_mini.csv"
+        visualization_mini_csv="{sample}/mass_list.visualization_mini.csv",
     shell:
         "python -m depiction_targeted_preproc.workflow.proc.mass_list_preparation"
         " --input-csv-path {input.csv}"