Harmonize branch

.gitignore

@@ -23,3 +23,47 @@ data/
 
 # shiny files:
 2.scDINO_processing/temporal_shiny_app/rsconnect/
+# raw data but keep the metadata
+data/2023*
+
+# logs
+**.log
+
+# IC logs
+2.cellprofiler_ic_processing/notebooks/logs/
+2.cellprofiler_ic_processing/scripts/logs/
+
+# pycache
+**/__pycache__/
+
+# illumination corrected images
+2.cellprofiler_ic_processing/illum_directory
+
+# cellprofiler logs
+3.cellprofiler_analysis/scripts/logs
+3.cellprofiler_analysis/scripts/*.log
+3.cellprofiler_analysis/notebooks/logs
+3.cellprofiler_analysis/notebooks/*.logs
+
+# analysis results
+3.cellprofiler_analysis/analysis_output
+
+# cytomining results
+4.process_features/data/
+
+5.scDINO_analysis/1.scDINO_run/outputdir/*
+5.scDINO_analysis/1.scDINO_run/logs/*
+5.scDINO_analysis/1.scDINO_run/.snakemake/*
+
+5.scDINO_analysis/2.scDINO_processing/scripts/4.deploy_shiny_app.r
+5.scDINO_analysis/2.scDINO_processing/notebooks/4.deploy_shiny_app.ipynb
+
+6.data_harmonization/data/*
+
+
+data/models/dino_deitsmall16_pretrain_full_checkpoint.pth
+data/processed_images/crops/*
+
+
+# cytomining logs
+**runinfo

0.download_data/README.md

@@ -0,0 +1,13 @@
+# Download Data
+
+## About the data
+The data are timelapse microscopy images of HeLa cells undergoing apoptosis.
+The cells are treated with varying concentrations of staurosporine, a drug that induces apoptosis.
+For more information about the data, see the [data README](../data/README.md).
+We recieved this data through a collaboration with [Saguaro Technologies](https://www.saguarobio.com/), and [Yokogawa](https://www.yokogawa.com/us/solutions/products-and-services/life-science/).
+
+## Downloading the data
+This file is a place holder for the instructions downloading the data.
+Typically data will stores in a remote location and will be downloaded using a script.
+In the past we tend to use sites like `Zenodo`, `Figshare`, or `IDR` to store the data.
+More information will be added in the future.

1.pre_process_data/notebooks/0.fix_pathing.ipynb

@@ -0,0 +1,111 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This notebook fixes issues with data paths by replacing spaces with underscores"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pathlib"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# set data path\n",
+    "data_path = pathlib.Path(\"../../data\").resolve(strict=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[PosixPath('/home/lippincm/Documents/4TB/data/live_cell_timelapse_apoptosis/data/20231017ChromaLive_6hr_4ch_MaxIP'),\n",
+       " PosixPath('/home/lippincm/Documents/4TB/data/live_cell_timelapse_apoptosis/data/20230920ChromaLiveTL_24hr4ch_MaxIP'),\n",
+       " PosixPath('/home/lippincm/Documents/4TB/data/live_cell_timelapse_apoptosis/data/20231017ChromaLive_endpoint w AnnexinV_2ch_MaxIP')]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get all directories in data path\n",
+    "directories = [x for x in data_path.iterdir() if x.is_dir()]\n",
+    "directories"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# rename the directories to replace spaces with underscores\n",
+    "for directory in directories:\n",
+    "    new_name = directory.name.replace(\" \", \"_\")\n",
+    "    directory.rename(directory.parent / new_name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[PosixPath('/home/lippincm/Documents/4TB/data/live_cell_timelapse_apoptosis/data/20231017ChromaLive_6hr_4ch_MaxIP'),\n",
+       " PosixPath('/home/lippincm/Documents/4TB/data/live_cell_timelapse_apoptosis/data/20231017ChromaLive_endpoint_w_AnnexinV_2ch_MaxIP'),\n",
+       " PosixPath('/home/lippincm/Documents/4TB/data/live_cell_timelapse_apoptosis/data/20230920ChromaLiveTL_24hr4ch_MaxIP')]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# get all directories in data path\n",
+    "directories = [x for x in data_path.iterdir() if x.is_dir()]\n",
+    "directories"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "timelapse_env",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}