diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 3efb9f8..fbd6479 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,9 +1,8 @@ name: Ruff on: [push, pull_request] - jobs: ruff: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: chartboost/ruff-action@v1 \ No newline at end of file + - uses: astral-sh/ruff-action@v1 \ No newline at end of file diff --git a/.github/workflows/pin_requirements.yml b/.github/workflows/pin_requirements.yml index 0487066..3d8c3e8 100644 --- a/.github/workflows/pin_requirements.yml +++ b/.github/workflows/pin_requirements.yml @@ -15,16 +15,14 @@ jobs: - name: Checkout code uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - python-version: ${{ matrix.python-version }} - - - name: Install pip-tools - run: pip install pip-tools + # Install a specific version of uv. + version: "0.5.4" - name: Generate requirements file - run: pip-compile --extra web --output-file requirements-${{ matrix.os }}-${{ matrix.python-version }}.txt pyproject.toml + run: uv pip compile --all-extras pyproject.toml -o requirements-${{ matrix.os }}-${{ matrix.python-version }}.txt - name: Upload requirements file uses: actions/upload-artifact@v3 diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 4b6c68a..a7dd63c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -6,40 +6,32 @@ jobs: strategy: fail-fast: false matrix: - os: [ "ubuntu-latest", "macos-latest" , "windows-latest"] python-version: ["3.10" ] - defaults: - run: - shell: bash - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Check out repository uses: actions/checkout@v3 - - name: Download test file windows - if: runner.os == 'Windows' - run: | - C:\\msys64\\usr\\bin\\wget.exe "https://filedn.eu/loRXwzWCNnU4XoFPGbllt1y/datafile_1.ptu" -O tests/test_data/input/ds1/datafile_1.ptu - - name: Download test file other platforms - if: runner.os != 'Windows' - run: | - wget "https://filedn.eu/loRXwzWCNnU4XoFPGbllt1y/datafile_1.ptu" -O tests/test_data/input/ds1/datafile_1.ptu - - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@v4 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - python-version: ${{ matrix.python-version }} - cache: pip - cache-dependency-path: requirements/requirements-${{ matrix.os }}-${{ matrix.python-version }}.txt - - - name: Install pinned requirements + # Install a specific version of uv. + version: "0.5.4" + enable-cache: true + cache-dependency-glob: requirements/requirements-ubuntu-latest-${{ matrix.python-version }}.txt + + - name: Install dependencies run: | - python -m pip install --upgrade pip - pip install -r requirements/requirements-${{ matrix.os }}-${{ matrix.python-version }}.txt --prefer-binary + python -m pip install uv + uv venv -p ${{ matrix.python-version }} + . .venv/bin/activate + echo PATH=$PATH >> $GITHUB_ENV + uv pip install -r requirements/requirements-ubuntu-latest-${{ matrix.python-version }}.txt + uv pip install -e . - - name: Install test requirements - run: pip install .[test] + - name: Download test file + run: | + wget "https://filedn.eu/loRXwzWCNnU4XoFPGbllt1y/datafile_1.ptu" -O tests/test_data/input/ds1/datafile_1.ptu - name: Run tests run: | diff --git a/CITE.txt b/CITE.txt index 2e309ff..839d12a 100644 --- a/CITE.txt +++ b/CITE.txt @@ -14,4 +14,8 @@ https://www.ncbi.nlm.nih.gov/pmc/articles/PMC4805879/ burst search: "Data registration and selective single-molecule analysis using multi-parameter fluorescence detection" -DOI: 10.1016/S0168-1656(00)00412-0 \ No newline at end of file +DOI: 10.1016/S0168-1656(00)00412-0 + +fret/alex cde: +"Disentangling Subpopulations in Single-Molecule FRET and ALEX Experiments with Photon Distribution Analysis" +https://doi.org/10.1016/j.bpj.2011.11.4025 \ No newline at end of file diff --git a/default_testing.yaml b/default_testing.yaml index 84fc04b..d5ac65c 100644 --- a/default_testing.yaml +++ b/default_testing.yaml @@ -40,6 +40,12 @@ burst_search: M: 100 T: 500.e-6 +hooks: + alex_2cde: + tau: 50.e-6 # make sure to format as float + fret_2cde: + tau: 50.e-6 + # settings related to dont-fret's web interface web: password: null @@ -48,3 +54,5 @@ web: burst_filters: # default filters to apply to burst search filters - name: n_photons min: 150 + - name: alex_2cde + max: 100 diff --git a/dont_fret/channel_kde.py b/dont_fret/channel_kde.py new file mode 100644 index 0000000..f91d91b --- /dev/null +++ b/dont_fret/channel_kde.py @@ -0,0 +1,205 @@ +""" +Module for channel based kernel density estimation + +This is based on: +Disentangling Subpopulations in Single-Molecule FRET and ALEX Experiments with Photon Distribution Analysis +https://doi.org/10.1016/j.bpj.2011.11.4025 + +Please cite the paper when using this module. + +""" + +import warnings +from typing import Optional + +import numpy as np +import polars as pl +from numba import float64, int64, jit, types + +from dont_fret.expr import is_in_expr + + +def compute_fret_2cde( + burst_photons: pl.DataFrame, + kde_rates: pl.DataFrame, + dem_stream: str = "DD", + aem_stream: str = "DA", +) -> np.ndarray: + """ + burst_photons: Dataframe with columns: timestamps, stream, burst_index + kde_rates: Dataframe with columns: timestamps, D_em, A_em. + dem_stream: photon stream which is donor emission (default: DD) + aem_stream: photon stream which is acceptor emission (default: DA) + """ + joined_df = burst_photons.join(kde_rates, on=["timestamps"], how="left").filter( + pl.col("stream") == pl.col("stream_right") + ) + + f_DA = pl.col("stream") == aem_stream + f_DD = pl.col("stream") == dem_stream + df_DA = joined_df.filter(f_DA) + df_DD = joined_df.filter(f_DD) + + matching_indices = np.intersect1d(df_DA["burst_index"].unique(), df_DD["burst_index"].unique()) + + DA_groups = {k: v for k, v in df_DA.group_by(["burst_index"])} + DD_groups = {k: v for k, v in df_DD.group_by(["burst_index"])} + + N: int = burst_photons["burst_index"].max() + 1 # type: ignore + output = np.full(N, fill_value=np.nan) + for j in matching_indices: + df_DA_j = DA_groups[(j,)] + df_DD_j = DD_groups[(j,)] + + kde_DA_DA = df_DA_j["A_em"].to_numpy() # select DA density - kde^DA_DA + kde_DA_DD = df_DD_j["A_em"].to_numpy() # kde^DA_DD (in the paper called kde^A_D) + kde_DD_DD = df_DD_j["D_em"].to_numpy() # kde^DD_DD + kde_DD_DA = df_DA_j["D_em"].to_numpy() # kde^DD_DA + + try: + nbkde_DA_DA = (1 + 2 / len(kde_DA_DA)) * (kde_DA_DA - 1) + nbkde_DD_DD = (1 + 2 / len(kde_DD_DD)) * (kde_DD_DD - 1) + + # when denom is zero, it doesnt count towards number of photons + # see "Such cases are removed by the computer algorithm", in Tomov et al. + denom = kde_DA_DD + nbkde_DD_DD + ED = (kde_DA_DD / denom).sum() / np.count_nonzero(denom) # when + + denom = kde_DD_DA + nbkde_DA_DA + EA = (kde_DD_DA / denom).sum() / np.count_nonzero(denom) # = (1 - E)_A + + fret_cde = 110 - 100 * (ED + EA) + output[j] = fret_cde + except ZeroDivisionError: + output[j] = np.nan + return output + + +# refactor to indices / loop +def compute_alex_2cde( + burst_photons: pl.DataFrame, + kde_rates: pl.DataFrame, + dex_streams: Optional[list[str]] = None, + aex_streams: Optional[list[str]] = None, +) -> pl.Series: + """ + burst_photons: Dataframe with columns: timestamps, stream, burst_index + kde_rates: Dataframe with columns: timestamps, D_ex, A_ex. + dex_streams: list of photon streams which are donor excitation (default: DD, DA) + aex_streams: list of photon streams which are acceptor excitation (default: AA) + """ + dex_streams = dex_streams if dex_streams else ["DD", "DA"] + aex_streams = aex_streams if aex_streams else ["AA"] + + f_dex = is_in_expr("stream", dex_streams) + f_aex = is_in_expr("stream", aex_streams) + + # equivalent to (but faster): + # joined_df = burst_photons.join(kde_rates, on=['timestamps', 'stream'], how='inner') + # still, this is the slowest step. would be nice if we can improve since we know the indices + + joined_df = burst_photons.join(kde_rates, on=["timestamps"], how="left").filter( + pl.col("stream") == pl.col("stream_right") + ) + + b_df = joined_df.select( + [ + pl.col("burst_index"), + pl.col("stream"), + (pl.col("A_ex") / pl.col("D_ex")).alias("ratio_AD"), + ] + ) + + # tomov et al eqn 10 and 11 + df_f_dex = b_df.filter(f_dex) + agg_dex = df_f_dex.group_by("burst_index", maintain_order=True).agg( + [pl.col("ratio_AD").sum(), pl.len().alias("N_dex")] + ) + + df_f_aex = b_df.filter(f_aex) + agg_dax = df_f_aex.group_by("burst_index", maintain_order=True).agg( + [(1 / pl.col("ratio_AD")).sum().alias("ratio_DA"), pl.len().alias("N_aex")] + ) + + combined = pl.concat([agg_dex, agg_dax], how="align") + + # tomov et al eqn 12 + # this is an addition in the paper + # in fretbursts its subtracted + ax_2cde_bracket = (1 / pl.col("N_aex")) * pl.col("ratio_AD") + (1 / pl.col("N_dex")) * pl.col( + "ratio_DA" + ) + + ax_2cde_norm = pl.lit(100) - pl.lit(50) * ax_2cde_bracket + + alex_2cde = combined.select(ax_2cde_norm.alias("alex_2cde")).to_series() + + return alex_2cde + + +def make_kernel( + tau: float, timestamps_unit: float, domain_size: int = 10, kernel="laplace" +) -> np.ndarray: + window_size = domain_size * (tau / timestamps_unit) + window_size_even_int = 2 * round(window_size / 2) + + # check that rounding error isnt too large + rel_dev = (window_size - window_size_even_int) / window_size + if np.abs(rel_dev) > 0.01: + warnings.warn( + "Kernel window size deviation from rounding larger than 1 percent. Choose a smaller `tau` with respect to `timestamps_unit'" + ) + + t_eval = np.linspace(-domain_size / 2, domain_size / 2, window_size_even_int + 1, endpoint=True) + kernel = np.exp(-np.abs(t_eval)) + + return kernel + + +def convolve_stream(data: pl.DataFrame, streams: list[str], kernel: np.ndarray) -> np.ndarray: + f_expr = is_in_expr("stream", streams) + + df = data.filter(f_expr) + # TODO warn on copy + # dataframes read from .pq cannot be converted zero-copy + event_times = df["timestamps"].to_numpy(allow_copy=True) + eval_times = data["timestamps"].to_numpy(allow_copy=True) + return async_convolve(event_times, eval_times, kernel) + + +@jit( + float64[:]( + types.Array(int64, 1, "C", readonly=True), + types.Array(int64, 1, "C", readonly=True), + types.Array(float64, 1, "C", readonly=True), + ), + nopython=True, + nogil=True, +) +def async_convolve(event_times, eval_times, kernel): + """convolve integer timestamps with a kernel""" + + i_lower = 0 + i_upper = 0 + + window_half_size = len(kernel) // 2 + result = np.zeros_like(eval_times, dtype=np.float64) + + for i, time in enumerate(eval_times): + while event_times[i_upper] < time + window_half_size: + i_upper += 1 + if i_upper == len(event_times): + i_upper -= 1 + break + + while event_times[i_lower] < time - window_half_size: + i_lower += 1 + if i_lower == len(event_times): + i_lower -= 1 + break + + for j in range(i_lower, i_upper): + idx = event_times[j] - time + window_half_size + result[i] += kernel[idx] + + return result diff --git a/dont_fret/config/config.py b/dont_fret/config/config.py index 93dc0ce..fbbca5c 100644 --- a/dont_fret/config/config.py +++ b/dont_fret/config/config.py @@ -3,7 +3,7 @@ import os from dataclasses import asdict, dataclass, field from pathlib import Path -from typing import Optional, Union +from typing import Any, Optional, Union import polars as pl import yaml @@ -43,11 +43,15 @@ def as_expr(self) -> list[pl.Expr]: class Web: """settings related to web application""" - default_dir: Path + default_dir: Path = field(default_factory=Path) protect_filebrowser: bool = True burst_filters: list[BurstFilterItem] = field(default_factory=list) password: Optional[str] = None + # todo configurable settings + fret_2cde: bool = True # calculate fret_2cde after burst search with default settings + alex_2cde: bool = True + @dataclass class BurstColor: @@ -62,7 +66,8 @@ class DontFRETConfig: channels: dict[str, Channel] streams: dict[str, list[str]] burst_search: dict[str, list[BurstColor]] - web: Web + hooks: dict[str, dict[str, Any]] = field(default_factory=dict) + web: Web = field(default_factory=Web) @classmethod def from_dict(cls, data: Data): diff --git a/dont_fret/config/default.yaml b/dont_fret/config/default.yaml index f3dc249..39ca05e 100644 --- a/dont_fret/config/default.yaml +++ b/dont_fret/config/default.yaml @@ -34,6 +34,15 @@ burst_search: M: 100 T: 500.e-6 +# post-burst search hooks to apply +# hooks are of the form my_hook(burst_data, photon_data, **kwargs) +# kwargs are as specified here +hooks: + alex_2cde: + tau: 150.e-6 # make sure to format as float + fret_2cde: + tau: 50.e-6 + # settings related to dont-fret's web interface web: password: null # set to null to disable password protection diff --git a/dont_fret/config/hooks.py b/dont_fret/config/hooks.py new file mode 100644 index 0000000..ad61f35 --- /dev/null +++ b/dont_fret/config/hooks.py @@ -0,0 +1,17 @@ +from dont_fret.models import Bursts, PhotonData + + +def fret_2cde( + bursts: Bursts, + photon_data: PhotonData, + tau: float = 50e-6, + dem_stream: str = "DD", + aem_stream: str = "DA", +) -> Bursts: + return bursts.fret_2cde(photon_data, tau=tau, dem_stream=dem_stream, aem_stream=aem_stream) + + +def alex_2cde( + bursts: Bursts, photon_data: PhotonData, tau: float = 50e-6, dex_streams=None, aex_streams=None +) -> Bursts: + return bursts.alex_2cde(photon_data, tau=tau, dex_streams=dex_streams, aex_streams=aex_streams) diff --git a/dont_fret/expr.py b/dont_fret/expr.py index 123dc07..a175424 100644 --- a/dont_fret/expr.py +++ b/dont_fret/expr.py @@ -32,6 +32,17 @@ def reduce_or(exprs: list[pl.Expr]) -> pl.Expr: return reduce(or_, exprs) +def is_in_expr(field: str, values: list) -> pl.Expr: + """generate an polars expression equivalent to pl.col(field).is_in(values) by chaining equal + and or operations together + + this is (sometimes?) faster to execute + """ + + exprs = [pl.col(field) == value for value in values] + return reduce_or(exprs) + + def parse_yaml_expressions(yaml_content: str) -> Dict[str, pl.Expr]: yaml_data = yaml.safe_load(yaml_content) return {key: parse_expression(value).alias(key) for key, value in yaml_data.items()} diff --git a/dont_fret/models.py b/dont_fret/models.py index 418d1d4..06cdcf1 100644 --- a/dont_fret/models.py +++ b/dont_fret/models.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +from dataclasses import dataclass from functools import cached_property, reduce from pathlib import Path from typing import TYPE_CHECKING, Optional, Union @@ -9,7 +10,9 @@ import polars as pl from dont_fret.burst_search import bs_eggeling, return_intersections -from dont_fret.config.config import BurstColor, DontFRETConfig, cfg +from dont_fret.channel_kde import compute_alex_2cde, compute_fret_2cde, convolve_stream, make_kernel +from dont_fret.config import cfg as global_cfg +from dont_fret.config.config import BurstColor, DontFRETConfig from dont_fret.support import get_binned from dont_fret.utils import clean_types @@ -32,7 +35,7 @@ class PhotonData: """ def __init__( - self, data: pl.DataFrame, metadata: Optional[dict] = None, cfg: DontFRETConfig = cfg + self, data: pl.DataFrame, metadata: Optional[dict] = None, cfg: DontFRETConfig = global_cfg ): self.data = data self.metadata = metadata or {} @@ -222,7 +225,7 @@ def burst_search(self, colors: Union[str, list[BurstColor]]) -> Bursts: """ if isinstance(colors, str): - burst_colors = cfg.burst_search[colors] + burst_colors = global_cfg.burst_search[colors] elif isinstance(colors, list): burst_colors = colors else: @@ -250,28 +253,33 @@ def burst_search(self, colors: Union[str, list[BurstColor]]) -> Bursts: # Check if any of the times _items is empty, if so, bursts is empty if any(len(t) == 0 for t in times_list): burst_photons = pl.DataFrame({k: [] for k in self.data.columns + ["burst_index"]}) + indices = pl.DataFrame({"imin": [], "imax": []}) else: # Take the intersection of the time intervals found by the multi-color burst search final_times = reduce(return_intersections, times_list) if len(final_times) == 0: # No overlap found burst_photons = pl.DataFrame({k: [] for k in self.data.columns + ["burst_index"]}) + indices = pl.DataFrame({"imin": [], "imax": []}) else: tmin, tmax = np.array(final_times).T # Convert back to indices imin = np.searchsorted(self.timestamps, tmin) imax = np.searchsorted(self.timestamps, tmax) + indices = pl.DataFrame({"imin": imin, "imax": imax}) # take all photons (up to and including? edges need to be checked!) b_num = int(2 ** np.ceil(np.log2((np.log2(len(imin)))))) - dtype = getattr(pl, f"UInt{b_num}", pl.Int32) + index_dtype = getattr(pl, f"UInt{b_num}", pl.Int32) bursts = [ - self.data[i1 : i2 + 1].with_columns(pl.lit(bi).alias("burst_index").cast(dtype)) + self.data[i1 : i2 + 1].with_columns( + pl.lit(bi).alias("burst_index").cast(index_dtype) + ) for bi, (i1, i2) in enumerate(zip(imin, imax)) ] burst_photons = pl.concat(bursts) - bs = Bursts(burst_photons, metadata=self.metadata) + bs = Bursts.from_photons(burst_photons, metadata=self.metadata) return bs @@ -335,7 +343,7 @@ def time(self) -> np.ndarray: # float array return self.time @property - def photon_times(self) -> np.ndarray: + def photon_times(self) -> pl.Series: """Photon arrival times in this trace, subject to `bounds`""" b_arr = np.array(self.bounds) / self.photon_data.timestamps_unit bounds_int = np.array([np.floor(b_arr[0]), np.ceil(b_arr[1])]).astype("uint64") @@ -353,22 +361,29 @@ def __len__(self) -> int: return len(self.time) -class Bursts(object): +@dataclass +class Bursts: """ Class which holds a set of bursts. attrs: - bursts np.array with Burst Objects + burst_data Dataframe with per-burst aggregated data + photon_data Dataframe with per-photon data """ - # todo add metadata support - # bursts: numpy.typing.ArrayLike[Bursts] ? - def __init__( - self, photon_data: pl.DataFrame, metadata: Optional[dict] = None, cfg: DontFRETConfig = cfg - ): - self.photon_data = photon_data - self.metadata: dict = metadata or {} - self.cfg = cfg + burst_data: pl.DataFrame + photon_data: pl.DataFrame + metadata: Optional[dict] = None + cfg: Optional[DontFRETConfig] = None + + @classmethod + def from_photons( + cls, + photon_data: pl.DataFrame, + metadata: Optional[dict] = None, + cfg: DontFRETConfig = global_cfg, + ) -> Bursts: + # todo move to classmethod # number of photons per stream per burst agg = [(pl.col("stream") == stream).sum().alias(f"n_{stream}") for stream in cfg.streams] @@ -400,7 +415,7 @@ def __init__( # TODO These should move somewhere else; possibly some second step conversion # from raw stamps to times - t_unit = self.metadata.get("timestamps_unit", None) + t_unit = metadata.get("timestamps_unit", None) if t_unit is not None: columns.extend( [ @@ -410,7 +425,7 @@ def __init__( ), ] ) - nanotimes_unit = self.metadata.get("nanotimes_unit", None) + nanotimes_unit = metadata.get("nanotimes_unit", None) if nanotimes_unit is not None: columns.extend( [ @@ -419,27 +434,88 @@ def __init__( ] ) - self.burst_data = ( - self.photon_data.group_by("burst_index", maintain_order=True) - .agg(agg) - .with_columns(columns) + burst_data = ( + photon_data.group_by("burst_index", maintain_order=True).agg(agg).with_columns(columns) ) + return Bursts(burst_data, photon_data, metadata=metadata, cfg=cfg) + @classmethod def load(cls, directory: Path) -> Bursts: - data = pl.read_parquet(directory / "data.pq") + burst_data = pl.read_parquet(directory / "burst_data.pq") + photon_data = pl.read_parquet(directory / "photon_data.pq") with open(directory / "metadata.json", "r") as f: metadata = json.load(f) - cfg = DontFRETConfig.from_yaml(directory / "config.yaml") - return Bursts(data, metadata, cfg) + try: + cfg = DontFRETConfig.from_yaml(directory / "config.yaml") + except FileNotFoundError: + cfg = None + return Bursts(burst_data, photon_data, metadata, cfg) def save(self, directory: Path) -> None: directory.mkdir(parents=True, exist_ok=True) - self.photon_data.write_parquet(directory / "data.pq") + self.burst_data.write_parquet(directory / "burst_data.pq") + self.photon_data.write_parquet(directory / "photon_data.pq") + with open(directory / "metadata.json", "w") as f: json.dump(self.metadata, f) - self.cfg.to_yaml(directory / "config.yaml") + if self.cfg is not None: + self.cfg.to_yaml(directory / "config.yaml") + + def fret_2cde( + self, + photons: PhotonData, + tau: float = 50e-6, + dem_stream: str = "DD", + aem_stream: str = "DA", + ) -> Bursts: + assert photons.timestamps_unit + kernel = make_kernel(tau, photons.timestamps_unit) + acceptor_em_rates = convolve_stream(photons.data, [dem_stream], kernel) + donor_em_rates = convolve_stream(photons.data, [aem_stream], kernel) + kde_data = photons.data.select( + [ + pl.col("timestamps"), + pl.col("stream"), + pl.lit(acceptor_em_rates).alias("A_em"), + pl.lit(donor_em_rates).alias("D_em"), + ] + ) + + fret_2cde = compute_fret_2cde(self.photon_data, kde_data) + burst_data = self.burst_data.with_columns(pl.lit(fret_2cde).alias("fret_2cde")) + + return Bursts(burst_data, self.photon_data, self.metadata, self.cfg) + + def alex_2cde( + self, + photons: PhotonData, + tau: float = 50e-6, + dex_streams: Optional[list[str]] = None, + aex_streams: Optional[list[str]] = None, + ) -> Bursts: + dex_streams = dex_streams if dex_streams else ["DD", "DA"] + aex_streams = aex_streams if aex_streams else ["AA"] + + assert photons.timestamps_unit + kernel = make_kernel(tau, photons.timestamps_unit) + donor_ex_rates = convolve_stream(photons.data, dex_streams, kernel) + acceptor_ex_rates = convolve_stream(photons.data, aex_streams, kernel) + + kde_data = photons.data.select( + [ + pl.col("timestamps"), + pl.col("stream"), + pl.lit(donor_ex_rates).alias("D_ex"), + pl.lit(acceptor_ex_rates).alias("A_ex"), + ] + ) + + alex_2cde = compute_alex_2cde(self.photon_data, kde_data) + burst_data = self.burst_data.with_columns(pl.lit(alex_2cde).alias("alex_2cde")) + + return Bursts(burst_data, self.photon_data, self.metadata, self.cfg) def __len__(self) -> int: """Number of bursts""" @@ -451,6 +527,8 @@ def __iter__(self): @property def timestamps_unit(self) -> Optional[float]: """Multiplication factor to covert timestamps integers to seconds""" + if self.metadata is None: + return None try: return self.metadata["timestamps_unit"] except KeyError: diff --git a/dont_fret/process.py b/dont_fret/process.py index 421fe29..2d2ed1a 100644 --- a/dont_fret/process.py +++ b/dont_fret/process.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path from typing import Literal, Optional @@ -8,8 +9,9 @@ from tqdm.auto import tqdm from dont_fret.config import cfg +from dont_fret.config.config import BurstColor from dont_fret.fileIO import PhotonFile -from dont_fret.models import PhotonData +from dont_fret.models import Bursts, PhotonData def search_and_save( @@ -69,3 +71,23 @@ def batch_search_and_save( for f in tqdm(as_completed(futures), total=len(futures)): f.result() + + +def process_photon_data( + photon_data: PhotonData, burst_colors: list[BurstColor], hooks: dict = cfg.hooks +) -> Bursts: + """search and apply hooks""" + bursts = photon_data.burst_search(burst_colors) + + for hook_name, hook_params in hooks.items(): + try: + hook = getattr(importlib.import_module("dont_fret.config.hooks"), hook_name) + except AttributeError: + try: + hook = getattr(importlib.import_module("hooks"), hook_name) + except (ImportError, AttributeError): + raise ValueError(f"Hook '{hook_name}' not found") + + bursts = hook(bursts, photon_data, **hook_params) + + return bursts diff --git a/dont_fret/web/bursts/components.py b/dont_fret/web/bursts/components.py index 2c46444..714396c 100644 --- a/dont_fret/web/bursts/components.py +++ b/dont_fret/web/bursts/components.py @@ -39,31 +39,6 @@ WRATIO = 3.5 DEFAULT_FIELD = "n_photons" -# TODO this is hardcoded but it depends on cfg settings what fields burst item dataframes have -# -> deduce from selection -FIELD_OPTIONS = [ - "burst_index", - "n_DD", - "n_DA", - "n_AA", - "n_AD", - "nanotimes_DD", - "nanotimes_DA", - "nanotimes_AA", - "nanotimes_AD", - "timestamps_mean", - "timestamps_min", - "timestamps_max", - "E_app", - "S_app", - "n_photons", - "time_mean", - "time_length", - "tau_DD", - "tau_DA", - "tau_AA", - "tau_AD", -] def fd_bin_width(data: pl.Series) -> float: @@ -209,6 +184,9 @@ def FilterEditDialog(): ) levels = list(selectors) burst_node = get_bursts(state.fret_nodes.items, burst_node_choice.items) + field_options = [ + col for col, dtype in zip(burst_node.df.columns, burst_node.df.dtypes) if dtype.is_numeric() + ] def make_chart(): new_chart = make_overlay_chart(burst_node.df, field.value, state.filters.items) @@ -262,7 +240,7 @@ def on_field(value): field.set(value) solara.Select( - label="Field", value=field.value, values=FIELD_OPTIONS, on_value=on_field + label="Field", value=field.value, values=field_options, on_value=on_field ) with solara.Tooltip("Add or edit filter"): solara.IconButton("mdi-filter-plus", on_click=filter_from_selection) diff --git a/dont_fret/web/datamanager.py b/dont_fret/web/datamanager.py index 34d44a1..cfd90a8 100644 --- a/dont_fret/web/datamanager.py +++ b/dont_fret/web/datamanager.py @@ -11,22 +11,21 @@ Optional, ) -import numpy as np -import polars as pl - -from dont_fret.config.config import BurstColor +from dont_fret.config.config import BurstColor, DontFRETConfig, cfg from dont_fret.fileIO import PhotonFile from dont_fret.models import Bursts, PhotonData +from dont_fret.process import process_photon_data from dont_fret.web.methods import get_duration, get_info, make_burst_dataframe from dont_fret.web.models import BurstNode, PhotonNode class ThreadedDataManager: - def __init__(self) -> None: + def __init__(self, cfg: DontFRETConfig = cfg) -> None: self.photon_cache: Dict[uuid.UUID, asyncio.Future[PhotonData]] = {} self.burst_cache: Dict[tuple[uuid.UUID, str], asyncio.Future[Bursts]] = {} self.executor = ThreadPoolExecutor(max_workers=4) # todo config wokers self.running_jobs = {} + self.cfg = cfg # todo allow passing loop to init @property @@ -78,7 +77,11 @@ async def get_bursts( self.burst_cache[key] = future try: - bursts = await self.search(photon_node, burst_colors) + photon_data = await self.get_photons(photon_node) + bursts = await self.run( + process_photon_data, photon_data, burst_colors, self.cfg.hooks + ) + future.set_result(bursts) except Exception as e: self.burst_cache.pop(key) @@ -115,29 +118,15 @@ async def get_bursts_batch( return results - async def get_dataframe( - self, - photon_nodes: list[PhotonNode], - burst_colors: list[BurstColor], - on_progress: Optional[Callable[[float | bool], None]] = None, - ) -> pl.DataFrame: - on_progress = on_progress or (lambda _: None) - on_progress(True) - raise DeprecationWarning("USe get burst node instead") - results = await self.get_bursts_batch(photon_nodes, burst_colors, on_progress) - on_progress(True) - - names = [ph_node.name for ph_node in photon_nodes] - lens = [len(burst) for burst in results] - - dtype = pl.Enum(categories=names) - filenames = pl.Series(name="filename", values=np.repeat(names, lens), dtype=dtype) - - df = pl.concat([b.burst_data for b in results], how="vertical_relaxed").with_columns( - filenames - ) + async def alex_2cde(self, photon_node: PhotonNode, bursts: Bursts) -> Bursts: + photons = await self.get_photons(photon_node) + new_bursts = self.run(bursts.alex_2cde, photons) + return await new_bursts - return df + async def fret_2cde(self, photon_node: PhotonNode, bursts: Bursts) -> Bursts: + photons = await self.get_photons(photon_node) + new_bursts = self.run(bursts.fret_2cde, photons) + return await new_bursts async def get_burst_node( self, @@ -148,7 +137,6 @@ async def get_burst_node( ) -> BurstNode: bursts = await self.get_bursts_batch(photon_nodes, burst_colors, on_progress=on_progress) bursts_df = make_burst_dataframe(bursts, names=[ph_node.name for ph_node in photon_nodes]) - # burst_df = await self.get_dataframe(photon_nodes, burst_colors, on_progress=on_progress) info_list = [await self.get_info(node) for node in photon_nodes] duration = get_duration(info_list) diff --git a/dont_fret/web/home/info_cards.py b/dont_fret/web/home/info_cards.py index e2d9d2d..8b610f6 100644 --- a/dont_fret/web/home/info_cards.py +++ b/dont_fret/web/home/info_cards.py @@ -302,7 +302,7 @@ async def load_info(): info = await state.data_manager.get_info(photon_node) return info - result = solara.lab.use_task(load_info, dependencies=[photon_node], prefer_threaded=False) # type: ignore + result = solara.lab.use_task(load_info, dependencies=[photon_node], prefer_threaded=False) # type: ignore # noqa: SH101 with solara.Card(f"{node_name} / Photon file / {photon_node.name}"): solara.ProgressLinear(result.pending and not loaded) diff --git a/dont_fret/web/methods.py b/dont_fret/web/methods.py index 86bd229..0cb95c2 100644 --- a/dont_fret/web/methods.py +++ b/dont_fret/web/methods.py @@ -4,7 +4,7 @@ import math from functools import reduce from operator import and_ -from typing import Literal, Optional, TypedDict, Union +from typing import Any, Literal, Optional, TypedDict, Union import altair as alt import numpy as np @@ -15,6 +15,7 @@ from dont_fret.config.config import BurstColor from dont_fret.fileIO import PhotonFile from dont_fret.models import Bursts, PhotonData +from dont_fret.process import process_photon_data from dont_fret.web.models import BurstFilterItem, BurstNode, FRETNode, PhotonNode @@ -45,14 +46,25 @@ def make_burst_dataframe( return concat +# hooks? def make_burst_nodes( - photon_nodes: list[PhotonNode], burst_settings: dict[str, list[BurstColor]] + photon_nodes: list[PhotonNode], + burst_settings: dict[str, list[BurstColor]], + hooks: Optional[dict[str, dict[str, Any]]] = None, ) -> list[BurstNode]: photons = [PhotonData.from_file(PhotonFile(node.file_path)) for node in photon_nodes] burst_nodes = [] # todo tqdm? + + hooks = hooks or {} for name, burst_colors in burst_settings.items(): - bursts = [photons.burst_search(burst_colors) for photons in photons] + bursts = [process_photon_data(photon_data, burst_colors, hooks) for photon_data in photons] + # bursts = [photons.burst_search(burst_colors) for photons in photons] + # if alex_2cde: + # bursts = [b.alex_2cde(photons) for b, photons in zip(bursts, photons)] + # if fret_2cde: + # bursts = [b.fret_2cde(photons) for b, photons in zip(bursts, photons)] + infos = [get_info(photons) for photons in photons] duration = get_duration(infos) df = make_burst_dataframe(bursts, names=[node.name for node in photon_nodes]) diff --git a/dont_fret/web/trace/page.py b/dont_fret/web/trace/page.py index f7cfc78..4376b97 100644 --- a/dont_fret/web/trace/page.py +++ b/dont_fret/web/trace/page.py @@ -106,7 +106,7 @@ async def redraw(): return fig - figure_task = solara.lab.use_task(redraw, dependencies=[photon_node, settings, dark_effective]) # type: ignore + figure_task = solara.lab.use_task(redraw, dependencies=[photon_node, settings, dark_effective]) # type: ignore # noqa: SH101 FigureFromTask(figure_task) diff --git a/requirements/requirements-macOS-latest-3.10.txt b/requirements/requirements-macOS-latest-3.10.txt index 5102108..d2ed391 100644 --- a/requirements/requirements-macOS-latest-3.10.txt +++ b/requirements/requirements-macOS-latest-3.10.txt @@ -1,116 +1,117 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --extra=web --output-file=requirements-macOS-latest-3.10.txt pyproject.toml -# -anyio==4.3.0 +# This file was autogenerated by uv via the following command: +# uv pip compile --all-extras pyproject.toml -o requirements-macOS-latest-3.10.txt +altair==5.5.0 + # via dont-fret (pyproject.toml) +anyio==4.6.2.post1 # via # starlette # watchfiles +anywidget==0.9.13 + # via dont-fret (pyproject.toml) appnope==0.1.4 # via ipykernel +arro3-core==0.2.1 + # via vegafusion asttokens==2.4.1 # via stack-data -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing -cachetools==5.3.3 +babel==2.16.0 + # via mkdocs-material +blosc2==2.7.1 + # via tables +cachetools==5.5.0 # via solara-ui -certifi==2024.2.2 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via - # dask - # distributed + # dont-fret (pyproject.toml) + # mkdocs + # mkdocstrings # rich-click # solara-server # uvicorn -cloudpickle==3.0.0 +colorama==0.4.6 # via - # dask - # distributed + # griffe + # mkdocs-material comm==0.2.2 # via # ipykernel # ipywidgets -contourpy==1.2.1 +contourpy==1.3.1 # via matplotlib cycler==0.12.1 # via matplotlib dacite==1.8.1 # via dont-fret (pyproject.toml) -dask==2024.5.0 - # via distributed -debugpy==1.8.1 +debugpy==1.8.9 # via ipykernel decorator==5.1.1 # via ipython -distributed==2024.5.0 - # via dont-fret (pyproject.toml) -exceptiongroup==1.2.1 - # via - # anyio - # ipython -executing==2.0.1 +executing==2.1.0 # via stack-data -fastjsonschema==2.19.1 +fastjsonschema==2.20.0 # via nbformat -filelock==3.14.0 +filelock==3.16.1 # via solara-server -fonttools==4.51.0 +fonttools==4.55.0 # via matplotlib -fsspec==2024.3.1 - # via dask +ghp-import==2.1.0 + # via mkdocs +greenlet==3.1.1 + # via playwright +griffe==1.5.1 + # via mkdocstrings-python h11==0.14.0 # via uvicorn -h5py==3.11.0 - # via dont-fret (pyproject.toml) -humanize==4.9.0 +humanize==4.11.0 # via solara-ui -idna==3.7 +idna==3.10 # via # anyio # requests -importlib-metadata==7.1.0 - # via dask -ipykernel==6.29.4 +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.5 # via solara-server -ipympl==0.9.4 - # via dont-fret (pyproject.toml) -ipython==8.24.0 +ipython==8.29.0 # via # ipykernel - # ipympl # ipywidgets -ipython-genutils==0.2.0 - # via ipympl -ipyvue==1.11.1 +ipyvue==1.11.2 # via # ipyvuetify # solara-ui -ipyvuetify==1.9.4 +ipyvuetify==1.10.0 # via solara-ui -ipywidgets==8.1.2 +ipywidgets==8.1.5 # via - # ipympl + # anywidget # ipyvue # reacton # solara-ui -jedi==0.19.1 +jedi==0.19.2 # via ipython jinja2==3.1.4 # via - # distributed + # altair + # mkdocs + # mkdocs-material + # mkdocstrings # solara-server -jsonschema==4.22.0 - # via nbformat -jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 + # via + # altair + # nbformat +jsonschema-specifications==2024.10.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.3 # via # ipykernel # solara-server @@ -119,113 +120,213 @@ jupyter-core==5.7.2 # ipykernel # jupyter-client # nbformat -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.13 # via ipywidgets -kdepy==1.1.9 +kdepy==1.1.11 # via dont-fret (pyproject.toml) -kiwisolver==1.4.5 +kiwisolver==1.4.7 # via matplotlib -llvmlite==0.42.0 +llvmlite==0.43.0 # via numba -locket==1.0.0 - # via - # distributed - # partd -markdown==3.6 +markdown==3.7 # via + # mkdocs + # mkdocs-autorefs + # mkdocs-material + # mkdocstrings # pymdown-extensions # solara-ui markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 - # via jinja2 -matplotlib==3.8.4 - # via ipympl +markupsafe==3.0.2 + # via + # jinja2 + # mkdocs + # mkdocs-autorefs + # mkdocstrings +matplotlib==3.9.2 + # via dont-fret (pyproject.toml) matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -msgpack==1.0.8 - # via distributed +mergedeep==1.3.4 + # via + # mkdocs + # mkdocs-get-deps +mkdocs==1.6.1 + # via + # dont-fret (pyproject.toml) + # mkdocs-autorefs + # mkdocs-gen-files + # mkdocs-literate-nav + # mkdocs-material + # mkdocstrings +mkdocs-autorefs==1.2.0 + # via + # mkdocstrings + # mkdocstrings-python +mkdocs-gen-files==0.5.0 + # via dont-fret (pyproject.toml) +mkdocs-get-deps==0.2.0 + # via mkdocs +mkdocs-literate-nav==0.6.1 + # via dont-fret (pyproject.toml) +mkdocs-material==9.5.46 + # via dont-fret (pyproject.toml) +mkdocs-material-extensions==1.3.1 + # via mkdocs-material +mkdocstrings==0.27.0 + # via + # dont-fret (pyproject.toml) + # mkdocstrings-python +mkdocstrings-python==1.12.2 + # via mkdocstrings +msgpack==1.1.0 + # via blosc2 +narwhals==1.14.2 + # via + # altair + # vegafusion nbformat==5.10.4 # via solara-server +ndindex==1.9.2 + # via blosc2 nest-asyncio==1.6.0 # via ipykernel -numba==0.59.1 +numba==0.60.0 # via dont-fret (pyproject.toml) -numpy==1.26.4 +numexpr==2.10.2 + # via + # blosc2 + # tables +numpy==2.0.2 # via - # contourpy # dont-fret (pyproject.toml) - # h5py - # ipympl + # blosc2 + # contourpy # kdepy # matplotlib # numba + # numexpr # pandas + # phconvert # scipy # solara-ui -packaging==24.0 + # tables +packaging==24.2 # via - # dask - # distributed + # altair # ipykernel # matplotlib + # mkdocs # plotly -pandas==2.2.2 + # pytest + # tables + # vegafusion +paginate==0.5.7 + # via mkdocs-material +pandas==2.2.3 # via dont-fret (pyproject.toml) parso==0.8.4 # via jedi -partd==1.4.2 - # via dask +pathspec==0.12.1 + # via mkdocs pexpect==4.9.0 # via ipython -pillow==10.3.0 +phconvert==0.9.1 + # via dont-fret (pyproject.toml) +pillow==11.0.0 # via - # ipympl # matplotlib + # pytest-ipywidgets # solara-ui -platformdirs==4.2.1 - # via jupyter-core -plotly==5.22.0 +pixelmatch==0.3.0 + # via pytest-ipywidgets +platformdirs==4.3.6 + # via + # jupyter-core + # mkdocs-get-deps + # mkdocstrings +playwright==1.49.0 + # via + # pytest-ipywidgets + # pytest-playwright +plotly==5.24.1 # via dont-fret (pyproject.toml) -polars==0.20.25 +pluggy==1.5.0 + # via pytest +polars==1.15.0 # via dont-fret (pyproject.toml) -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.48 # via ipython -psutil==5.9.8 - # via - # distributed - # ipykernel +psutil==6.1.0 + # via ipykernel +psygnal==0.11.1 + # via anywidget ptyprocess==0.7.0 # via pexpect -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data +py-cpuinfo==9.0.0 + # via + # blosc2 + # tables +pyee==12.0.0 + # via playwright pygments==2.18.0 # via + # dont-fret (pyproject.toml) # ipython + # mkdocs-material # rich # solara-ui -pymdown-extensions==10.8.1 - # via solara-ui -pyparsing==3.1.2 +pymdown-extensions==10.12 + # via + # mkdocs-material + # mkdocstrings + # solara-ui +pyparsing==3.2.0 # via matplotlib +pytest==8.3.3 + # via + # dont-fret (pyproject.toml) + # pytest-asyncio + # pytest-base-url + # pytest-ipywidgets + # pytest-playwright +pytest-asyncio==0.24.0 + # via dont-fret (pyproject.toml) +pytest-base-url==2.1.0 + # via pytest-playwright +pytest-ipywidgets==1.41.0 + # via dont-fret (pyproject.toml) +pytest-playwright==0.6.1 + # via + # dont-fret (pyproject.toml) + # pytest-ipywidgets python-dateutil==2.9.0.post0 # via + # ghp-import # jupyter-client # matplotlib # pandas -pytz==2024.1 +python-slugify==8.0.4 + # via pytest-playwright +pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via - # dask - # distributed # dont-fret (pyproject.toml) + # mkdocs + # mkdocs-get-deps # pymdown-extensions -pyzmq==26.0.3 + # pyyaml-env-tag +pyyaml-env-tag==0.1 + # via mkdocs +pyzmq==26.2.0 # via # ipykernel # jupyter-client @@ -235,94 +336,92 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 - # via solara-ui -rich==13.7.1 +regex==2024.11.6 + # via mkdocs-material +requests==2.32.3 + # via + # mkdocs-material + # pytest-base-url + # solara-ui +rich==13.9.4 # via rich-click -rich-click==1.8.1 +rich-click==1.8.4 # via solara-server -rpds-py==0.18.1 +rpds-py==0.21.0 # via # jsonschema # referencing -scipy==1.13.0 - # via - # dont-fret (pyproject.toml) - # kdepy +scipy==1.14.1 + # via kdepy six==1.16.0 # via # asttokens # python-dateutil sniffio==1.3.1 # via anyio -solara==1.32.1 +solara==1.41.0 # via dont-fret (pyproject.toml) -solara-server[dev,starlette]==1.32.1 - # via solara -solara-ui[all,cache,extra,markdown]==1.32.1 +solara-server==1.41.0 + # via + # pytest-ipywidgets + # solara +solara-ui==1.41.0 # via + # pytest-ipywidgets # solara # solara-server -sortedcontainers==2.4.0 - # via distributed stack-data==0.6.3 # via ipython -starlette==0.37.2 +starlette==0.41.3 # via solara-server -tblib==3.0.0 - # via distributed -tenacity==8.3.0 +tables==3.10.1 + # via phconvert +tenacity==9.0.0 # via plotly -toolz==0.12.1 +text-unidecode==1.3 + # via python-slugify +tornado==6.4.2 # via - # dask - # distributed - # partd -tornado==6.4 - # via - # distributed # ipykernel # jupyter-client -tqdm==4.66.4 +tqdm==4.67.1 # via dont-fret (pyproject.toml) traitlets==5.14.3 # via # comm # ipykernel - # ipympl # ipython # ipywidgets # jupyter-client # jupyter-core # matplotlib-inline # nbformat -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via - # anyio # dont-fret (pyproject.toml) - # ipython + # altair + # anywidget + # pyee # reacton # rich-click - # uvicorn -tzdata==2024.1 + # tables +tzdata==2024.2 # via pandas -urllib3==2.2.1 - # via - # distributed - # requests -uvicorn==0.29.0 - # via solara-server -watchdog==4.0.0 +urllib3==2.2.3 + # via requests +uvicorn==0.32.1 # via solara-server -watchfiles==0.21.0 +vegafusion==2.0.1 + # via dont-fret (pyproject.toml) +watchdog==6.0.0 + # via + # mkdocs + # solara-server +watchfiles==1.0.0 # via solara-server wcwidth==0.2.13 # via prompt-toolkit -websockets==12.0 +websockets==14.1 # via solara-server -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.13 # via ipywidgets -zict==3.0.0 - # via distributed -zipp==3.18.1 - # via importlib-metadata diff --git a/requirements/requirements-macOS-latest-3.9.txt b/requirements/requirements-macOS-latest-3.9.txt deleted file mode 100644 index 0372f16..0000000 --- a/requirements/requirements-macOS-latest-3.9.txt +++ /dev/null @@ -1,336 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --extra=web --output-file=requirements-macOS-latest-3.9.txt pyproject.toml -# -anyio==4.3.0 - # via - # starlette - # watchfiles -appnope==0.1.4 - # via ipykernel -asttokens==2.4.1 - # via stack-data -attrs==23.2.0 - # via - # jsonschema - # referencing -cachetools==5.3.3 - # via solara-ui -certifi==2024.2.2 - # via requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # dask - # distributed - # rich-click - # solara-server - # uvicorn -cloudpickle==3.0.0 - # via - # dask - # distributed -comm==0.2.2 - # via - # ipykernel - # ipywidgets -contourpy==1.2.1 - # via matplotlib -cycler==0.12.1 - # via matplotlib -dacite==1.8.1 - # via dont-fret (pyproject.toml) -dask==2024.5.0 - # via distributed -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -distributed==2024.5.0 - # via dont-fret (pyproject.toml) -exceptiongroup==1.2.1 - # via - # anyio - # ipython -executing==2.0.1 - # via stack-data -fastjsonschema==2.19.1 - # via nbformat -filelock==3.14.0 - # via solara-server -fonttools==4.51.0 - # via matplotlib -fsspec==2024.3.1 - # via dask -h11==0.14.0 - # via uvicorn -h5py==3.11.0 - # via dont-fret (pyproject.toml) -humanize==4.9.0 - # via solara-ui -idna==3.7 - # via - # anyio - # requests -importlib-metadata==7.1.0 - # via - # dask - # jupyter-client - # markdown -importlib-resources==6.4.0 - # via matplotlib -ipykernel==6.29.4 - # via solara-server -ipympl==0.9.4 - # via dont-fret (pyproject.toml) -ipython==8.18.1 - # via - # ipykernel - # ipympl - # ipywidgets -ipython-genutils==0.2.0 - # via ipympl -ipyvue==1.11.1 - # via - # ipyvuetify - # solara-ui -ipyvuetify==1.9.4 - # via solara-ui -ipywidgets==8.1.2 - # via - # ipympl - # ipyvue - # reacton - # solara-ui -jedi==0.19.1 - # via ipython -jinja2==3.1.4 - # via - # distributed - # solara-server -jsonschema==4.22.0 - # via nbformat -jsonschema-specifications==2023.12.1 - # via jsonschema -jupyter-client==8.6.1 - # via - # ipykernel - # solara-server -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # nbformat -jupyterlab-widgets==3.0.10 - # via ipywidgets -kdepy==1.1.9 - # via dont-fret (pyproject.toml) -kiwisolver==1.4.5 - # via matplotlib -llvmlite==0.42.0 - # via numba -locket==1.0.0 - # via - # distributed - # partd -markdown==3.6 - # via - # pymdown-extensions - # solara-ui -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.5 - # via jinja2 -matplotlib==3.8.4 - # via ipympl -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython -mdurl==0.1.2 - # via markdown-it-py -msgpack==1.0.8 - # via distributed -nbformat==5.10.4 - # via solara-server -nest-asyncio==1.6.0 - # via ipykernel -numba==0.59.1 - # via dont-fret (pyproject.toml) -numpy==1.26.4 - # via - # contourpy - # dont-fret (pyproject.toml) - # h5py - # ipympl - # kdepy - # matplotlib - # numba - # pandas - # scipy - # solara-ui -packaging==24.0 - # via - # dask - # distributed - # ipykernel - # matplotlib - # plotly -pandas==2.2.2 - # via dont-fret (pyproject.toml) -parso==0.8.4 - # via jedi -partd==1.4.2 - # via dask -pexpect==4.9.0 - # via ipython -pillow==10.3.0 - # via - # ipympl - # matplotlib - # solara-ui -platformdirs==4.2.1 - # via jupyter-core -plotly==5.22.0 - # via dont-fret (pyproject.toml) -polars==0.20.25 - # via dont-fret (pyproject.toml) -prompt-toolkit==3.0.43 - # via ipython -psutil==5.9.8 - # via - # distributed - # ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pygments==2.18.0 - # via - # ipython - # rich - # solara-ui -pymdown-extensions==10.8.1 - # via solara-ui -pyparsing==3.1.2 - # via matplotlib -python-dateutil==2.9.0.post0 - # via - # jupyter-client - # matplotlib - # pandas -pytz==2024.1 - # via pandas -pyyaml==6.0.1 - # via - # dask - # distributed - # dont-fret (pyproject.toml) - # pymdown-extensions -pyzmq==26.0.3 - # via - # ipykernel - # jupyter-client -reacton==1.8.3 - # via solara-ui -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via solara-ui -rich==13.7.1 - # via rich-click -rich-click==1.8.1 - # via solara-server -rpds-py==0.18.1 - # via - # jsonschema - # referencing -scipy==1.13.0 - # via - # dont-fret (pyproject.toml) - # kdepy -six==1.16.0 - # via - # asttokens - # python-dateutil -sniffio==1.3.1 - # via anyio -solara==1.32.1 - # via dont-fret (pyproject.toml) -solara-server[dev,starlette]==1.32.1 - # via solara -solara-ui[all,cache,extra,markdown]==1.32.1 - # via - # solara - # solara-server -sortedcontainers==2.4.0 - # via distributed -stack-data==0.6.3 - # via ipython -starlette==0.37.2 - # via solara-server -tblib==3.0.0 - # via distributed -tenacity==8.3.0 - # via plotly -toolz==0.12.1 - # via - # dask - # distributed - # partd -tornado==6.4 - # via - # distributed - # ipykernel - # jupyter-client -tqdm==4.66.4 - # via dont-fret (pyproject.toml) -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipympl - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbformat -typing-extensions==4.11.0 - # via - # anyio - # dont-fret (pyproject.toml) - # ipython - # reacton - # rich-click - # starlette - # uvicorn -tzdata==2024.1 - # via pandas -urllib3==2.2.1 - # via - # distributed - # requests -uvicorn==0.29.0 - # via solara-server -watchdog==4.0.0 - # via solara-server -watchfiles==0.21.0 - # via solara-server -wcwidth==0.2.13 - # via prompt-toolkit -websockets==12.0 - # via solara-server -widgetsnbextension==4.0.10 - # via ipywidgets -zict==3.0.0 - # via distributed -zipp==3.18.1 - # via - # importlib-metadata - # importlib-resources diff --git a/requirements/requirements-ubuntu-latest-3.10.txt b/requirements/requirements-ubuntu-latest-3.10.txt index 85a14f4..850c086 100644 --- a/requirements/requirements-ubuntu-latest-3.10.txt +++ b/requirements/requirements-ubuntu-latest-3.10.txt @@ -1,114 +1,122 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --extra=web --output-file=requirements-ubuntu-latest-3.10.txt pyproject.toml -# -anyio==4.3.0 +# This file was autogenerated by uv via the following command: +# uv pip compile --all-extras pyproject.toml -o requirements-ubuntu-latest-3.10.txt +altair==5.5.0 + # via dont-fret (pyproject.toml) +anyio==4.6.2.post1 # via # starlette # watchfiles +anywidget==0.9.13 + # via dont-fret (pyproject.toml) +arro3-core==0.4.2 + # via vegafusion asttokens==2.4.1 # via stack-data -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing -cachetools==5.3.3 +babel==2.16.0 + # via mkdocs-material +blosc2==2.7.1 + # via tables +cachetools==5.5.0 # via solara-ui -certifi==2024.2.2 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via - # dask - # distributed + # dont-fret (pyproject.toml) + # mkdocs + # mkdocstrings # rich-click # solara-server # uvicorn -cloudpickle==3.0.0 +colorama==0.4.6 # via - # dask - # distributed + # griffe + # mkdocs-material comm==0.2.2 # via # ipykernel # ipywidgets -contourpy==1.2.1 +contourpy==1.3.1 # via matplotlib cycler==0.12.1 # via matplotlib dacite==1.8.1 # via dont-fret (pyproject.toml) -dask==2024.5.0 - # via distributed -debugpy==1.8.1 +debugpy==1.8.9 # via ipykernel decorator==5.1.1 # via ipython -distributed==2024.5.0 - # via dont-fret (pyproject.toml) -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via # anyio # ipython -executing==2.0.1 + # pytest +executing==2.1.0 # via stack-data -fastjsonschema==2.19.1 +fastjsonschema==2.20.0 # via nbformat -filelock==3.14.0 +filelock==3.16.1 # via solara-server -fonttools==4.51.0 +fonttools==4.55.0 # via matplotlib -fsspec==2024.3.1 - # via dask +ghp-import==2.1.0 + # via mkdocs +greenlet==3.1.1 + # via playwright +griffe==1.5.1 + # via mkdocstrings-python h11==0.14.0 # via uvicorn -h5py==3.11.0 - # via dont-fret (pyproject.toml) -humanize==4.9.0 +humanize==4.11.0 # via solara-ui -idna==3.7 +idna==3.10 # via # anyio # requests -importlib-metadata==7.1.0 - # via dask -ipykernel==6.29.4 +importlib-resources==6.4.5 + # via phconvert +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.5 # via solara-server -ipympl==0.9.4 - # via dont-fret (pyproject.toml) -ipython==8.24.0 +ipython==8.29.0 # via # ipykernel - # ipympl # ipywidgets -ipython-genutils==0.2.0 - # via ipympl -ipyvue==1.11.1 +ipyvue==1.11.2 # via # ipyvuetify # solara-ui -ipyvuetify==1.9.4 +ipyvuetify==1.10.0 # via solara-ui -ipywidgets==8.1.2 +ipywidgets==8.1.5 # via - # ipympl + # anywidget # ipyvue # reacton # solara-ui -jedi==0.19.1 +jedi==0.19.2 # via ipython jinja2==3.1.4 # via - # distributed + # altair + # mkdocs + # mkdocs-material + # mkdocstrings # solara-server -jsonschema==4.22.0 - # via nbformat -jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 + # via + # altair + # nbformat +jsonschema-specifications==2024.10.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.3 # via # ipykernel # solara-server @@ -117,113 +125,213 @@ jupyter-core==5.7.2 # ipykernel # jupyter-client # nbformat -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.13 # via ipywidgets -kdepy==1.1.9 +kdepy==1.1.11 # via dont-fret (pyproject.toml) -kiwisolver==1.4.5 +kiwisolver==1.4.7 # via matplotlib -llvmlite==0.42.0 +llvmlite==0.43.0 # via numba -locket==1.0.0 - # via - # distributed - # partd -markdown==3.6 +markdown==3.7 # via + # mkdocs + # mkdocs-autorefs + # mkdocs-material + # mkdocstrings # pymdown-extensions # solara-ui markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 - # via jinja2 -matplotlib==3.8.4 - # via ipympl +markupsafe==3.0.2 + # via + # jinja2 + # mkdocs + # mkdocs-autorefs + # mkdocstrings +matplotlib==3.9.2 + # via dont-fret (pyproject.toml) matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -msgpack==1.0.8 - # via distributed +mergedeep==1.3.4 + # via + # mkdocs + # mkdocs-get-deps +mkdocs==1.6.1 + # via + # dont-fret (pyproject.toml) + # mkdocs-autorefs + # mkdocs-gen-files + # mkdocs-literate-nav + # mkdocs-material + # mkdocstrings +mkdocs-autorefs==1.2.0 + # via + # mkdocstrings + # mkdocstrings-python +mkdocs-gen-files==0.5.0 + # via dont-fret (pyproject.toml) +mkdocs-get-deps==0.2.0 + # via mkdocs +mkdocs-literate-nav==0.6.1 + # via dont-fret (pyproject.toml) +mkdocs-material==9.5.46 + # via dont-fret (pyproject.toml) +mkdocs-material-extensions==1.3.1 + # via mkdocs-material +mkdocstrings==0.27.0 + # via + # dont-fret (pyproject.toml) + # mkdocstrings-python +mkdocstrings-python==1.12.2 + # via mkdocstrings +msgpack==1.1.0 + # via blosc2 +narwhals==1.14.2 + # via + # altair + # vegafusion nbformat==5.10.4 # via solara-server +ndindex==1.9.2 + # via blosc2 nest-asyncio==1.6.0 # via ipykernel -numba==0.59.1 +numba==0.60.0 # via dont-fret (pyproject.toml) -numpy==1.26.4 +numexpr==2.10.2 + # via + # blosc2 + # tables +numpy==2.0.2 # via - # contourpy # dont-fret (pyproject.toml) - # h5py - # ipympl + # blosc2 + # contourpy # kdepy # matplotlib # numba + # numexpr # pandas + # phconvert # scipy # solara-ui -packaging==24.0 + # tables +packaging==24.2 # via - # dask - # distributed + # altair # ipykernel # matplotlib + # mkdocs # plotly -pandas==2.2.2 + # pytest + # tables + # vegafusion +paginate==0.5.7 + # via mkdocs-material +pandas==2.2.3 # via dont-fret (pyproject.toml) parso==0.8.4 # via jedi -partd==1.4.2 - # via dask +pathspec==0.12.1 + # via mkdocs pexpect==4.9.0 # via ipython -pillow==10.3.0 +phconvert==0.9.1 + # via dont-fret (pyproject.toml) +pillow==11.0.0 # via - # ipympl # matplotlib + # pytest-ipywidgets # solara-ui -platformdirs==4.2.1 - # via jupyter-core -plotly==5.22.0 +pixelmatch==0.3.0 + # via pytest-ipywidgets +platformdirs==4.3.6 + # via + # jupyter-core + # mkdocs-get-deps + # mkdocstrings +playwright==1.49.0 + # via + # pytest-ipywidgets + # pytest-playwright +plotly==5.24.1 # via dont-fret (pyproject.toml) -polars==0.20.25 +pluggy==1.5.0 + # via pytest +polars==1.15.0 # via dont-fret (pyproject.toml) -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.48 # via ipython -psutil==5.9.8 - # via - # distributed - # ipykernel +psutil==6.1.0 + # via ipykernel +psygnal==0.11.1 + # via anywidget ptyprocess==0.7.0 # via pexpect -pure-eval==0.2.2 +pure-eval==0.2.3 # via stack-data +py-cpuinfo==9.0.0 + # via + # blosc2 + # tables +pyee==12.0.0 + # via playwright pygments==2.18.0 # via + # dont-fret (pyproject.toml) # ipython + # mkdocs-material # rich # solara-ui -pymdown-extensions==10.8.1 - # via solara-ui -pyparsing==3.1.2 +pymdown-extensions==10.12 + # via + # mkdocs-material + # mkdocstrings + # solara-ui +pyparsing==3.2.0 # via matplotlib +pytest==8.3.3 + # via + # dont-fret (pyproject.toml) + # pytest-asyncio + # pytest-base-url + # pytest-ipywidgets + # pytest-playwright +pytest-asyncio==0.24.0 + # via dont-fret (pyproject.toml) +pytest-base-url==2.1.0 + # via pytest-playwright +pytest-ipywidgets==1.41.0 + # via dont-fret (pyproject.toml) +pytest-playwright==0.6.1 + # via + # dont-fret (pyproject.toml) + # pytest-ipywidgets python-dateutil==2.9.0.post0 # via + # ghp-import # jupyter-client # matplotlib # pandas -pytz==2024.1 +python-slugify==8.0.4 + # via pytest-playwright +pytz==2024.2 # via pandas -pyyaml==6.0.1 +pyyaml==6.0.2 # via - # dask - # distributed # dont-fret (pyproject.toml) + # mkdocs + # mkdocs-get-deps # pymdown-extensions -pyzmq==26.0.3 + # pyyaml-env-tag +pyyaml-env-tag==0.1 + # via mkdocs +pyzmq==26.2.0 # via # ipykernel # jupyter-client @@ -233,94 +341,98 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 - # via solara-ui -rich==13.7.1 +regex==2024.11.6 + # via mkdocs-material +requests==2.32.3 + # via + # mkdocs-material + # pytest-base-url + # solara-ui +rich==13.9.4 # via rich-click -rich-click==1.8.1 +rich-click==1.8.4 # via solara-server -rpds-py==0.18.1 +rpds-py==0.21.0 # via # jsonschema # referencing -scipy==1.13.0 - # via - # dont-fret (pyproject.toml) - # kdepy +scipy==1.14.1 + # via kdepy six==1.16.0 # via # asttokens # python-dateutil sniffio==1.3.1 # via anyio -solara==1.32.1 +solara==1.41.0 # via dont-fret (pyproject.toml) -solara-server[dev,starlette]==1.32.1 - # via solara -solara-ui[all,cache,extra,markdown]==1.32.1 +solara-server==1.41.0 + # via + # pytest-ipywidgets + # solara +solara-ui==1.41.0 # via + # pytest-ipywidgets # solara # solara-server -sortedcontainers==2.4.0 - # via distributed stack-data==0.6.3 # via ipython -starlette==0.37.2 +starlette==0.41.3 # via solara-server -tblib==3.0.0 - # via distributed -tenacity==8.3.0 +tables==3.10.1 + # via phconvert +tenacity==9.0.0 # via plotly -toolz==0.12.1 +text-unidecode==1.3 + # via python-slugify +tomli==2.1.0 + # via pytest +tornado==6.4.2 # via - # dask - # distributed - # partd -tornado==6.4 - # via - # distributed # ipykernel # jupyter-client -tqdm==4.66.4 +tqdm==4.67.1 # via dont-fret (pyproject.toml) traitlets==5.14.3 # via # comm # ipykernel - # ipympl # ipython # ipywidgets # jupyter-client # jupyter-core # matplotlib-inline # nbformat -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via - # anyio # dont-fret (pyproject.toml) + # altair + # anyio + # anywidget # ipython + # pyee # reacton + # rich # rich-click + # tables # uvicorn -tzdata==2024.1 +tzdata==2024.2 # via pandas -urllib3==2.2.1 - # via - # distributed - # requests -uvicorn==0.29.0 - # via solara-server -watchdog==4.0.0 +urllib3==2.2.3 + # via requests +uvicorn==0.32.1 # via solara-server -watchfiles==0.21.0 +vegafusion==2.0.1 + # via dont-fret (pyproject.toml) +watchdog==6.0.0 + # via + # mkdocs + # solara-server +watchfiles==1.0.0 # via solara-server wcwidth==0.2.13 # via prompt-toolkit -websockets==12.0 +websockets==14.1 # via solara-server -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.13 # via ipywidgets -zict==3.0.0 - # via distributed -zipp==3.18.1 - # via importlib-metadata diff --git a/requirements/requirements-ubuntu-latest-3.9.txt b/requirements/requirements-ubuntu-latest-3.9.txt deleted file mode 100644 index 72d66e2..0000000 --- a/requirements/requirements-ubuntu-latest-3.9.txt +++ /dev/null @@ -1,334 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --extra=web --output-file=requirements-ubuntu-latest-3.9.txt pyproject.toml -# -anyio==4.3.0 - # via - # starlette - # watchfiles -asttokens==2.4.1 - # via stack-data -attrs==23.2.0 - # via - # jsonschema - # referencing -cachetools==5.3.3 - # via solara-ui -certifi==2024.2.2 - # via requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # dask - # distributed - # rich-click - # solara-server - # uvicorn -cloudpickle==3.0.0 - # via - # dask - # distributed -comm==0.2.2 - # via - # ipykernel - # ipywidgets -contourpy==1.2.1 - # via matplotlib -cycler==0.12.1 - # via matplotlib -dacite==1.8.1 - # via dont-fret (pyproject.toml) -dask==2024.5.0 - # via distributed -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -distributed==2024.5.0 - # via dont-fret (pyproject.toml) -exceptiongroup==1.2.1 - # via - # anyio - # ipython -executing==2.0.1 - # via stack-data -fastjsonschema==2.19.1 - # via nbformat -filelock==3.14.0 - # via solara-server -fonttools==4.51.0 - # via matplotlib -fsspec==2024.3.1 - # via dask -h11==0.14.0 - # via uvicorn -h5py==3.11.0 - # via dont-fret (pyproject.toml) -humanize==4.9.0 - # via solara-ui -idna==3.7 - # via - # anyio - # requests -importlib-metadata==7.1.0 - # via - # dask - # jupyter-client - # markdown -importlib-resources==6.4.0 - # via matplotlib -ipykernel==6.29.4 - # via solara-server -ipympl==0.9.4 - # via dont-fret (pyproject.toml) -ipython==8.18.1 - # via - # ipykernel - # ipympl - # ipywidgets -ipython-genutils==0.2.0 - # via ipympl -ipyvue==1.11.1 - # via - # ipyvuetify - # solara-ui -ipyvuetify==1.9.4 - # via solara-ui -ipywidgets==8.1.2 - # via - # ipympl - # ipyvue - # reacton - # solara-ui -jedi==0.19.1 - # via ipython -jinja2==3.1.4 - # via - # distributed - # solara-server -jsonschema==4.22.0 - # via nbformat -jsonschema-specifications==2023.12.1 - # via jsonschema -jupyter-client==8.6.1 - # via - # ipykernel - # solara-server -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # nbformat -jupyterlab-widgets==3.0.10 - # via ipywidgets -kdepy==1.1.9 - # via dont-fret (pyproject.toml) -kiwisolver==1.4.5 - # via matplotlib -llvmlite==0.42.0 - # via numba -locket==1.0.0 - # via - # distributed - # partd -markdown==3.6 - # via - # pymdown-extensions - # solara-ui -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.5 - # via jinja2 -matplotlib==3.8.4 - # via ipympl -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython -mdurl==0.1.2 - # via markdown-it-py -msgpack==1.0.8 - # via distributed -nbformat==5.10.4 - # via solara-server -nest-asyncio==1.6.0 - # via ipykernel -numba==0.59.1 - # via dont-fret (pyproject.toml) -numpy==1.26.4 - # via - # contourpy - # dont-fret (pyproject.toml) - # h5py - # ipympl - # kdepy - # matplotlib - # numba - # pandas - # scipy - # solara-ui -packaging==24.0 - # via - # dask - # distributed - # ipykernel - # matplotlib - # plotly -pandas==2.2.2 - # via dont-fret (pyproject.toml) -parso==0.8.4 - # via jedi -partd==1.4.2 - # via dask -pexpect==4.9.0 - # via ipython -pillow==10.3.0 - # via - # ipympl - # matplotlib - # solara-ui -platformdirs==4.2.1 - # via jupyter-core -plotly==5.22.0 - # via dont-fret (pyproject.toml) -polars==0.20.25 - # via dont-fret (pyproject.toml) -prompt-toolkit==3.0.43 - # via ipython -psutil==5.9.8 - # via - # distributed - # ipykernel -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pygments==2.18.0 - # via - # ipython - # rich - # solara-ui -pymdown-extensions==10.8.1 - # via solara-ui -pyparsing==3.1.2 - # via matplotlib -python-dateutil==2.9.0.post0 - # via - # jupyter-client - # matplotlib - # pandas -pytz==2024.1 - # via pandas -pyyaml==6.0.1 - # via - # dask - # distributed - # dont-fret (pyproject.toml) - # pymdown-extensions -pyzmq==26.0.3 - # via - # ipykernel - # jupyter-client -reacton==1.8.3 - # via solara-ui -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via solara-ui -rich==13.7.1 - # via rich-click -rich-click==1.8.1 - # via solara-server -rpds-py==0.18.1 - # via - # jsonschema - # referencing -scipy==1.13.0 - # via - # dont-fret (pyproject.toml) - # kdepy -six==1.16.0 - # via - # asttokens - # python-dateutil -sniffio==1.3.1 - # via anyio -solara==1.32.1 - # via dont-fret (pyproject.toml) -solara-server[dev,starlette]==1.32.1 - # via solara -solara-ui[all,cache,extra,markdown]==1.32.1 - # via - # solara - # solara-server -sortedcontainers==2.4.0 - # via distributed -stack-data==0.6.3 - # via ipython -starlette==0.37.2 - # via solara-server -tblib==3.0.0 - # via distributed -tenacity==8.3.0 - # via plotly -toolz==0.12.1 - # via - # dask - # distributed - # partd -tornado==6.4 - # via - # distributed - # ipykernel - # jupyter-client -tqdm==4.66.4 - # via dont-fret (pyproject.toml) -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipympl - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbformat -typing-extensions==4.11.0 - # via - # anyio - # dont-fret (pyproject.toml) - # ipython - # reacton - # rich-click - # starlette - # uvicorn -tzdata==2024.1 - # via pandas -urllib3==2.2.1 - # via - # distributed - # requests -uvicorn==0.29.0 - # via solara-server -watchdog==4.0.0 - # via solara-server -watchfiles==0.21.0 - # via solara-server -wcwidth==0.2.13 - # via prompt-toolkit -websockets==12.0 - # via solara-server -widgetsnbextension==4.0.10 - # via ipywidgets -zict==3.0.0 - # via distributed -zipp==3.18.1 - # via - # importlib-metadata - # importlib-resources diff --git a/requirements/requirements-windows-latest-3.10.txt b/requirements/requirements-windows-latest-3.10.txt index a939e4a..dcf5d99 100644 --- a/requirements/requirements-windows-latest-3.10.txt +++ b/requirements/requirements-windows-latest-3.10.txt @@ -1,119 +1,136 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --extra=web --output-file=requirements-windows-latest-3.10.txt pyproject.toml -# -anyio==4.3.0 +# This file was autogenerated by uv via the following command: +# uv pip compile --all-extras pyproject.toml -o requirements-windows-latest-3.10.txt +altair==5.5.0 + # via dont-fret (pyproject.toml) +anyio==4.6.2.post1 # via # starlette # watchfiles +anywidget==0.9.13 + # via dont-fret (pyproject.toml) +arro3-core==0.4.2 + # via vegafusion asttokens==2.4.1 # via stack-data -attrs==23.2.0 +attrs==24.2.0 # via # jsonschema # referencing -cachetools==5.3.3 +babel==2.16.0 + # via mkdocs-material +blosc2==2.5.1 + # via tables +cachetools==5.5.0 # via solara-ui -certifi==2024.2.2 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via - # dask - # distributed + # dont-fret (pyproject.toml) + # mkdocs + # mkdocstrings # rich-click # solara-server # uvicorn -cloudpickle==3.0.0 - # via - # dask - # distributed colorama==0.4.6 # via # click + # griffe # ipython + # mkdocs + # mkdocs-material + # pytest # tqdm comm==0.2.2 # via # ipykernel # ipywidgets -contourpy==1.2.1 +contourpy==1.3.0 # via matplotlib cycler==0.12.1 # via matplotlib dacite==1.8.1 # via dont-fret (pyproject.toml) -dask==2024.5.0 - # via distributed -debugpy==1.8.1 +debugpy==1.8.9 # via ipykernel decorator==5.1.1 # via ipython -distributed==2024.5.0 - # via dont-fret (pyproject.toml) -exceptiongroup==1.2.1 +exceptiongroup==1.2.2 # via # anyio # ipython -executing==2.0.1 + # pytest +executing==2.1.0 # via stack-data -fastjsonschema==2.19.1 +fastjsonschema==2.20.0 # via nbformat -filelock==3.14.0 +filelock==3.16.1 # via solara-server -fonttools==4.51.0 +fonttools==4.55.0 # via matplotlib -fsspec==2024.3.1 - # via dask +ghp-import==2.1.0 + # via mkdocs +greenlet==3.1.1 + # via playwright +griffe==1.5.1 + # via mkdocstrings-python h11==0.14.0 # via uvicorn -h5py==3.11.0 - # via dont-fret (pyproject.toml) -humanize==4.9.0 +humanize==4.11.0 # via solara-ui -idna==3.7 +idna==3.10 # via # anyio # requests -importlib-metadata==7.1.0 - # via dask -ipykernel==6.29.4 +importlib-metadata==8.5.0 + # via + # jupyter-client + # markdown + # mkdocs + # mkdocs-get-deps + # mkdocstrings +importlib-resources==6.4.5 + # via + # matplotlib + # phconvert +iniconfig==2.0.0 + # via pytest +ipykernel==6.29.5 # via solara-server -ipympl==0.9.4 - # via dont-fret (pyproject.toml) -ipython==8.24.0 +ipython==8.18.1 # via # ipykernel - # ipympl # ipywidgets -ipython-genutils==0.2.0 - # via ipympl -ipyvue==1.11.1 +ipyvue==1.11.2 # via # ipyvuetify # solara-ui -ipyvuetify==1.9.4 +ipyvuetify==1.10.0 # via solara-ui -ipywidgets==8.1.2 +ipywidgets==8.1.5 # via - # ipympl + # anywidget # ipyvue # reacton # solara-ui -jedi==0.19.1 +jedi==0.19.2 # via ipython jinja2==3.1.4 # via - # distributed + # altair + # mkdocs + # mkdocs-material + # mkdocstrings # solara-server -jsonschema==4.22.0 - # via nbformat -jsonschema-specifications==2023.12.1 +jsonschema==4.23.0 + # via + # altair + # nbformat +jsonschema-specifications==2024.10.1 # via jsonschema -jupyter-client==8.6.1 +jupyter-client==8.6.3 # via # ipykernel # solara-server @@ -122,111 +139,209 @@ jupyter-core==5.7.2 # ipykernel # jupyter-client # nbformat -jupyterlab-widgets==3.0.10 +jupyterlab-widgets==3.0.13 # via ipywidgets -kdepy==1.1.9 +kdepy==1.1.11 # via dont-fret (pyproject.toml) -kiwisolver==1.4.5 +kiwisolver==1.4.7 # via matplotlib -llvmlite==0.42.0 +llvmlite==0.43.0 # via numba -locket==1.0.0 - # via - # distributed - # partd -markdown==3.6 +markdown==3.7 # via + # mkdocs + # mkdocs-autorefs + # mkdocs-material + # mkdocstrings # pymdown-extensions # solara-ui markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 - # via jinja2 -matplotlib==3.8.4 - # via ipympl +markupsafe==3.0.2 + # via + # jinja2 + # mkdocs + # mkdocs-autorefs + # mkdocstrings +matplotlib==3.9.2 + # via dont-fret (pyproject.toml) matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -msgpack==1.0.8 - # via distributed +mergedeep==1.3.4 + # via + # mkdocs + # mkdocs-get-deps +mkdocs==1.6.1 + # via + # dont-fret (pyproject.toml) + # mkdocs-autorefs + # mkdocs-gen-files + # mkdocs-literate-nav + # mkdocs-material + # mkdocstrings +mkdocs-autorefs==1.2.0 + # via + # mkdocstrings + # mkdocstrings-python +mkdocs-gen-files==0.5.0 + # via dont-fret (pyproject.toml) +mkdocs-get-deps==0.2.0 + # via mkdocs +mkdocs-literate-nav==0.6.1 + # via dont-fret (pyproject.toml) +mkdocs-material==9.5.46 + # via dont-fret (pyproject.toml) +mkdocs-material-extensions==1.3.1 + # via mkdocs-material +mkdocstrings==0.27.0 + # via + # dont-fret (pyproject.toml) + # mkdocstrings-python +mkdocstrings-python==1.12.2 + # via mkdocstrings +msgpack==1.1.0 + # via blosc2 +narwhals==1.14.2 + # via + # altair + # vegafusion nbformat==5.10.4 # via solara-server +ndindex==1.9.2 + # via blosc2 nest-asyncio==1.6.0 # via ipykernel -numba==0.59.1 +numba==0.60.0 # via dont-fret (pyproject.toml) -numpy==1.26.4 +numexpr==2.10.2 + # via tables +numpy==2.0.2 # via - # contourpy # dont-fret (pyproject.toml) - # h5py - # ipympl + # blosc2 + # contourpy # kdepy # matplotlib # numba + # numexpr # pandas + # phconvert # scipy # solara-ui -packaging==24.0 + # tables +packaging==24.2 # via - # dask - # distributed + # altair # ipykernel # matplotlib + # mkdocs # plotly -pandas==2.2.2 + # pytest + # tables + # vegafusion +paginate==0.5.7 + # via mkdocs-material +pandas==2.2.3 # via dont-fret (pyproject.toml) parso==0.8.4 # via jedi -partd==1.4.2 - # via dask -pillow==10.3.0 +pathspec==0.12.1 + # via mkdocs +phconvert==0.9.1 + # via dont-fret (pyproject.toml) +pillow==11.0.0 # via - # ipympl # matplotlib + # pytest-ipywidgets # solara-ui -platformdirs==4.2.1 - # via jupyter-core -plotly==5.22.0 +pixelmatch==0.3.0 + # via pytest-ipywidgets +platformdirs==4.3.6 + # via + # jupyter-core + # mkdocs-get-deps + # mkdocstrings +playwright==1.49.0 + # via + # pytest-ipywidgets + # pytest-playwright +plotly==5.24.1 # via dont-fret (pyproject.toml) -polars==0.20.25 +pluggy==1.5.0 + # via pytest +polars==1.15.0 # via dont-fret (pyproject.toml) -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.48 # via ipython -psutil==5.9.8 - # via - # distributed - # ipykernel -pure-eval==0.2.2 +psutil==6.1.0 + # via ipykernel +psygnal==0.11.1 + # via anywidget +pure-eval==0.2.3 # via stack-data +py-cpuinfo==9.0.0 + # via + # blosc2 + # tables +pyee==12.0.0 + # via playwright pygments==2.18.0 # via + # dont-fret (pyproject.toml) # ipython + # mkdocs-material # rich # solara-ui -pymdown-extensions==10.8.1 - # via solara-ui -pyparsing==3.1.2 +pymdown-extensions==10.12 + # via + # mkdocs-material + # mkdocstrings + # solara-ui +pyparsing==3.2.0 # via matplotlib +pytest==8.3.3 + # via + # dont-fret (pyproject.toml) + # pytest-asyncio + # pytest-base-url + # pytest-ipywidgets + # pytest-playwright +pytest-asyncio==0.24.0 + # via dont-fret (pyproject.toml) +pytest-base-url==2.1.0 + # via pytest-playwright +pytest-ipywidgets==1.41.0 + # via dont-fret (pyproject.toml) +pytest-playwright==0.6.1 + # via + # dont-fret (pyproject.toml) + # pytest-ipywidgets python-dateutil==2.9.0.post0 # via + # ghp-import # jupyter-client # matplotlib # pandas -pytz==2024.1 +python-slugify==8.0.4 + # via pytest-playwright +pytz==2024.2 # via pandas -pywin32==306 +pywin32==308 # via jupyter-core -pyyaml==6.0.1 +pyyaml==6.0.2 # via - # dask - # distributed # dont-fret (pyproject.toml) + # mkdocs + # mkdocs-get-deps # pymdown-extensions -pyzmq==26.0.3 + # pyyaml-env-tag +pyyaml-env-tag==0.1 + # via mkdocs +pyzmq==26.2.0 # via # ipykernel # jupyter-client @@ -236,94 +351,103 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 - # via solara-ui -rich==13.7.1 +regex==2024.11.6 + # via mkdocs-material +requests==2.32.3 + # via + # mkdocs-material + # pytest-base-url + # solara-ui +rich==13.9.4 # via rich-click -rich-click==1.8.1 +rich-click==1.8.4 # via solara-server -rpds-py==0.18.1 +rpds-py==0.21.0 # via # jsonschema # referencing -scipy==1.13.0 - # via - # dont-fret (pyproject.toml) - # kdepy +scipy==1.13.1 + # via kdepy six==1.16.0 # via # asttokens # python-dateutil sniffio==1.3.1 # via anyio -solara==1.32.1 +solara==1.41.0 # via dont-fret (pyproject.toml) -solara-server[dev,starlette]==1.32.1 - # via solara -solara-ui[all,cache,extra,markdown]==1.32.1 +solara-server==1.41.0 + # via + # pytest-ipywidgets + # solara +solara-ui==1.41.0 # via + # pytest-ipywidgets # solara # solara-server -sortedcontainers==2.4.0 - # via distributed stack-data==0.6.3 # via ipython -starlette==0.37.2 +starlette==0.41.3 # via solara-server -tblib==3.0.0 - # via distributed -tenacity==8.3.0 +tables==3.9.2 + # via phconvert +tenacity==9.0.0 # via plotly -toolz==0.12.1 - # via - # dask - # distributed - # partd -tornado==6.4 +text-unidecode==1.3 + # via python-slugify +tomli==2.1.0 + # via pytest +tornado==6.4.2 # via - # distributed # ipykernel # jupyter-client -tqdm==4.66.4 +tqdm==4.67.1 # via dont-fret (pyproject.toml) traitlets==5.14.3 # via # comm # ipykernel - # ipympl # ipython # ipywidgets # jupyter-client # jupyter-core # matplotlib-inline # nbformat -typing-extensions==4.11.0 +typing-extensions==4.12.2 # via - # anyio # dont-fret (pyproject.toml) + # altair + # anyio + # anywidget # ipython + # mkdocstrings + # pyee # reacton + # rich # rich-click + # starlette # uvicorn -tzdata==2024.1 +tzdata==2024.2 # via pandas -urllib3==2.2.1 - # via - # distributed - # requests -uvicorn==0.29.0 - # via solara-server -watchdog==4.0.0 +urllib3==2.2.3 + # via requests +uvicorn==0.32.1 # via solara-server -watchfiles==0.21.0 +vegafusion==2.0.1 + # via dont-fret (pyproject.toml) +watchdog==6.0.0 + # via + # mkdocs + # solara-server +watchfiles==1.0.0 # via solara-server wcwidth==0.2.13 # via prompt-toolkit -websockets==12.0 +websockets==14.1 # via solara-server -widgetsnbextension==4.0.10 +widgetsnbextension==4.0.13 # via ipywidgets -zict==3.0.0 - # via distributed -zipp==3.18.1 - # via importlib-metadata +zipp==3.21.0 + # via + # importlib-metadata + # importlib-resources diff --git a/requirements/requirements-windows-latest-3.9.txt b/requirements/requirements-windows-latest-3.9.txt deleted file mode 100644 index bbe8c9d..0000000 --- a/requirements/requirements-windows-latest-3.9.txt +++ /dev/null @@ -1,337 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --extra=web --output-file=requirements-windows-latest-3.9.txt pyproject.toml -# -anyio==4.3.0 - # via - # starlette - # watchfiles -asttokens==2.4.1 - # via stack-data -attrs==23.2.0 - # via - # jsonschema - # referencing -cachetools==5.3.3 - # via solara-ui -certifi==2024.2.2 - # via requests -charset-normalizer==3.3.2 - # via requests -click==8.1.7 - # via - # dask - # distributed - # rich-click - # solara-server - # uvicorn -cloudpickle==3.0.0 - # via - # dask - # distributed -colorama==0.4.6 - # via - # click - # ipython - # tqdm -comm==0.2.2 - # via - # ipykernel - # ipywidgets -contourpy==1.2.1 - # via matplotlib -cycler==0.12.1 - # via matplotlib -dacite==1.8.1 - # via dont-fret (pyproject.toml) -dask==2024.5.0 - # via distributed -debugpy==1.8.1 - # via ipykernel -decorator==5.1.1 - # via ipython -distributed==2024.5.0 - # via dont-fret (pyproject.toml) -exceptiongroup==1.2.1 - # via - # anyio - # ipython -executing==2.0.1 - # via stack-data -fastjsonschema==2.19.1 - # via nbformat -filelock==3.14.0 - # via solara-server -fonttools==4.51.0 - # via matplotlib -fsspec==2024.3.1 - # via dask -h11==0.14.0 - # via uvicorn -h5py==3.11.0 - # via dont-fret (pyproject.toml) -humanize==4.9.0 - # via solara-ui -idna==3.7 - # via - # anyio - # requests -importlib-metadata==7.1.0 - # via - # dask - # jupyter-client - # markdown -importlib-resources==6.4.0 - # via matplotlib -ipykernel==6.29.4 - # via solara-server -ipympl==0.9.4 - # via dont-fret (pyproject.toml) -ipython==8.18.1 - # via - # ipykernel - # ipympl - # ipywidgets -ipython-genutils==0.2.0 - # via ipympl -ipyvue==1.11.1 - # via - # ipyvuetify - # solara-ui -ipyvuetify==1.9.4 - # via solara-ui -ipywidgets==8.1.2 - # via - # ipympl - # ipyvue - # reacton - # solara-ui -jedi==0.19.1 - # via ipython -jinja2==3.1.4 - # via - # distributed - # solara-server -jsonschema==4.22.0 - # via nbformat -jsonschema-specifications==2023.12.1 - # via jsonschema -jupyter-client==8.6.1 - # via - # ipykernel - # solara-server -jupyter-core==5.7.2 - # via - # ipykernel - # jupyter-client - # nbformat -jupyterlab-widgets==3.0.10 - # via ipywidgets -kdepy==1.1.9 - # via dont-fret (pyproject.toml) -kiwisolver==1.4.5 - # via matplotlib -llvmlite==0.42.0 - # via numba -locket==1.0.0 - # via - # distributed - # partd -markdown==3.6 - # via - # pymdown-extensions - # solara-ui -markdown-it-py==3.0.0 - # via rich -markupsafe==2.1.5 - # via jinja2 -matplotlib==3.8.4 - # via ipympl -matplotlib-inline==0.1.7 - # via - # ipykernel - # ipython -mdurl==0.1.2 - # via markdown-it-py -msgpack==1.0.8 - # via distributed -nbformat==5.10.4 - # via solara-server -nest-asyncio==1.6.0 - # via ipykernel -numba==0.59.1 - # via dont-fret (pyproject.toml) -numpy==1.26.4 - # via - # contourpy - # dont-fret (pyproject.toml) - # h5py - # ipympl - # kdepy - # matplotlib - # numba - # pandas - # scipy - # solara-ui -packaging==24.0 - # via - # dask - # distributed - # ipykernel - # matplotlib - # plotly -pandas==2.2.2 - # via dont-fret (pyproject.toml) -parso==0.8.4 - # via jedi -partd==1.4.2 - # via dask -pillow==10.3.0 - # via - # ipympl - # matplotlib - # solara-ui -platformdirs==4.2.1 - # via jupyter-core -plotly==5.22.0 - # via dont-fret (pyproject.toml) -polars==0.20.25 - # via dont-fret (pyproject.toml) -prompt-toolkit==3.0.43 - # via ipython -psutil==5.9.8 - # via - # distributed - # ipykernel -pure-eval==0.2.2 - # via stack-data -pygments==2.18.0 - # via - # ipython - # rich - # solara-ui -pymdown-extensions==10.8.1 - # via solara-ui -pyparsing==3.1.2 - # via matplotlib -python-dateutil==2.9.0.post0 - # via - # jupyter-client - # matplotlib - # pandas -pytz==2024.1 - # via pandas -pywin32==306 - # via jupyter-core -pyyaml==6.0.1 - # via - # dask - # distributed - # dont-fret (pyproject.toml) - # pymdown-extensions -pyzmq==26.0.3 - # via - # ipykernel - # jupyter-client -reacton==1.8.3 - # via solara-ui -referencing==0.35.1 - # via - # jsonschema - # jsonschema-specifications -requests==2.31.0 - # via solara-ui -rich==13.7.1 - # via rich-click -rich-click==1.8.1 - # via solara-server -rpds-py==0.18.1 - # via - # jsonschema - # referencing -scipy==1.13.0 - # via - # dont-fret (pyproject.toml) - # kdepy -six==1.16.0 - # via - # asttokens - # python-dateutil -sniffio==1.3.1 - # via anyio -solara==1.32.1 - # via dont-fret (pyproject.toml) -solara-server[dev,starlette]==1.32.1 - # via solara -solara-ui[all,cache,extra,markdown]==1.32.1 - # via - # solara - # solara-server -sortedcontainers==2.4.0 - # via distributed -stack-data==0.6.3 - # via ipython -starlette==0.37.2 - # via solara-server -tblib==3.0.0 - # via distributed -tenacity==8.3.0 - # via plotly -toolz==0.12.1 - # via - # dask - # distributed - # partd -tornado==6.4 - # via - # distributed - # ipykernel - # jupyter-client -tqdm==4.66.4 - # via dont-fret (pyproject.toml) -traitlets==5.14.3 - # via - # comm - # ipykernel - # ipympl - # ipython - # ipywidgets - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbformat -typing-extensions==4.11.0 - # via - # anyio - # dont-fret (pyproject.toml) - # ipython - # reacton - # rich-click - # starlette - # uvicorn -tzdata==2024.1 - # via pandas -urllib3==2.2.1 - # via - # distributed - # requests -uvicorn==0.29.0 - # via solara-server -watchdog==4.0.0 - # via solara-server -watchfiles==0.21.0 - # via solara-server -wcwidth==0.2.13 - # via prompt-toolkit -websockets==12.0 - # via solara-server -widgetsnbextension==4.0.10 - # via ipywidgets -zict==3.0.0 - # via distributed -zipp==3.18.1 - # via - # importlib-metadata - # importlib-resources diff --git a/templates/01_load_datafile_1.py b/templates/01_load_datafile_1.py index fd9f9bc..60b291b 100644 --- a/templates/01_load_datafile_1.py +++ b/templates/01_load_datafile_1.py @@ -12,10 +12,15 @@ # %% photons = PhotonData.from_file(PhotonFile(test_data_dir / ptu_file)) - # %% bursts = photons.burst_search("DCBS") bursts.burst_data # %% + +bursts = bursts.alex_2cde(photons).fret_2cde(photons) + +bursts.burst_data + +# %% diff --git a/templates/07_c_kde.py b/templates/07_c_kde.py new file mode 100644 index 0000000..77bfc38 --- /dev/null +++ b/templates/07_c_kde.py @@ -0,0 +1,64 @@ +# %% +from pathlib import Path + +import matplotlib.pyplot as plt +import polars as pl + +from dont_fret.channel_kde import compute_alex_2cde, compute_fret_2cde, convolve_stream, make_kernel +from dont_fret.fileIO import PhotonFile +from dont_fret.models import PhotonData + +# %% +cwd = Path(__file__).parent +test_data_dir = cwd.parent / "tests" / "test_data" / "input" / "ds3" +ptu_file = "datafile_1.ptu" +ptu_file = "210122_sFRET_MBP5K_apo_15.ptu" + +# %% +photons = PhotonData.from_file(PhotonFile(test_data_dir / ptu_file)) +bursts = photons.burst_search("APBS") + +tau = 50e-6 +assert photons.timestamps_unit +kernel = make_kernel(tau, photons.timestamps_unit) + +fig, ax = plt.subplots() +ax.plot(kernel) + +# %% +# tomov et al eqn 9 +D_ex = convolve_stream(photons.data, ["DD", "DA"], kernel) +A_ex = convolve_stream(photons.data, ["AA"], kernel) # crashed the kernel (sometimes) + +# %% + +kde_data = photons.data.select( + [pl.col("timestamps"), pl.col("stream"), pl.lit(D_ex).alias("D_ex"), pl.lit(A_ex).alias("A_ex")] +) + +# %% +alex_2cde = compute_alex_2cde(bursts.photon_data, kde_data) + +# %% +DA = convolve_stream(photons.data, ["DA"], kernel) +DD = convolve_stream(photons.data, ["DD"], kernel) +kde_data = photons.data.select( + [pl.col("timestamps"), pl.col("stream"), pl.lit(DA).alias("DA"), pl.lit(DD).alias("DD")] +) + +# 500 ms +fret_2cde = compute_fret_2cde(bursts.photon_data, kde_data) + +# modify bursts inplace +bursts.burst_data = bursts.burst_data.with_columns(pl.lit(fret_2cde).alias("fret_2cde")) + +# %% + + +# %% + +fig, axes = plt.subplots(ncols=2) +h = axes[0].hist(alex_2cde, bins="fd") +h = axes[1].hist(fret_2cde, bins="fd") +# axes[1].axvline(10, color='r') +# %% diff --git a/tests/generate_fretbursts_kde.py b/tests/generate_fretbursts_kde.py new file mode 100644 index 0000000..059711e --- /dev/null +++ b/tests/generate_fretbursts_kde.py @@ -0,0 +1,194 @@ +""" +generates test data for comparing fretbursts / dontfret channel kde + +ran with fretbursts '0.8.3' + +adapted from https://github.com/OpenSMFS/FRETBursts/blob/master/notebooks/Example%20-%202CDE%20Method.ipynb + +""" +# %% + +import polars as pl +from fretbursts import * +from fretbursts.phtools import phrates + +# %% + +url = "http://files.figshare.com/2182601/0023uLRpitc_NTP_20dT_0.5GndCl.hdf5" +download_file(url, save_dir="./data") + +filename = "data/0023uLRpitc_NTP_20dT_0.5GndCl.hdf5" + +d = loader.photon_hdf5(filename) +loader.alex_apply_period(d) +d.calc_bg(fun=bg.exp_fit, time_s=20, tail_min_us="auto", F_bg=1.7) +d.burst_search() + +ds1 = d.select_bursts(select_bursts.size, th1=30) +ds = ds1.select_bursts(select_bursts.naa, th1=30) + +# %% +ph = d.ph_times_m[0] + +tau_s = 50e-6 # in seconds +tau = int(tau_s / d.clk_p) # in raw timestamp units +tau + +ph = d.get_ph_times(ph_sel=Ph_sel("all")) +mask_d = d.get_ph_mask(ph_sel=Ph_sel(Dex="Dem")) +mask_a = d.get_ph_mask(ph_sel=Ph_sel(Dex="Aem")) + +# bursts = ds.mburst[0] +bursts = ds1.mburst[0] +# %% + + +def calc_fret_2cde(tau, ph, mask_d, mask_a, bursts): + """ + Compute FRET-2CDE for each burst. + + FRET-2CDE is a quantity that tends to be around 10 for bursts which have no + dynamics, while it has larger values (e.g. 30..100) for bursts with + millisecond dynamics. + + References: + Tomov et al. BJ (2012) doi:10.1016/j.bpj.2011.11.4025 + + Arguments: + tau (scalar): time-constant of the exponential KDE + ph (1D array): array of all-photons timestamps. + mask_d (bool array): mask for DexDem photons + mask_a (bool array): mask for DexAem photons + bursts (Bursts object): object containing burst data + (start-stop indexes are relative to `ph`). + + Returns: + FRET_2CDE (1D array): array of FRET_2CDE quantities, one element + per burst. This array contains NaN in correspondence of bursts + containing to few photons to compute FRET-2CDE. + """ + # Computing KDE burst-by-burst would cause inaccuracies at the burst edges. + # Therefore, we first compute KDE on the full timestamps array and then + # we take slices for each burst. + # These KDEs are evaluated on all-photons array `ph` (hence the Ti suffix) + # using D or A photons during D-excitation (argument ph[mask_d] or ph[mask_a]). + KDE_DTi = phrates.kde_laplace(ph[mask_d], tau, time_axis=ph) + KDE_ATi = phrates.kde_laplace(ph[mask_a], tau, time_axis=ph) + + FRET_2CDE = [] + for ib, burst in enumerate(bursts): + burst_slice = slice(int(burst.istart), int(burst.istop) + 1) + if ~mask_d[burst_slice].any() or ~mask_a[burst_slice].any(): + # Either D or A photon stream has no photons in current burst, + # thus FRET_2CDE cannot be computed. Fill position with NaN. + print("nan") + FRET_2CDE.append(np.nan) + continue + + # Take slices of KDEs for current burst + kde_adi = KDE_ATi[burst_slice][mask_d[burst_slice]] + kde_ddi = KDE_DTi[burst_slice][mask_d[burst_slice]] + kde_dai = KDE_DTi[burst_slice][mask_a[burst_slice]] + kde_aai = KDE_ATi[burst_slice][mask_a[burst_slice]] + + # nbKDE does not include the "center" timestamp which contributes 1. + # We thus subtract 1 from the precomputed KDEs. + # The N_CHD (N_CHA) value in the correction factor is the number of + # timestamps in DexDem (DexAem) stream falling within the current burst. + N_CHD = mask_d[burst_slice].sum() + N_CHA = mask_a[burst_slice].sum() + nbkde_ddi = (1 + 2 / N_CHD) * (kde_ddi - 1) + nbkde_aai = (1 + 2 / N_CHA) * (kde_aai - 1) + + # N_CHD (N_CHA) in eq. 6 (eq. 7) of (Tomov 2012) is the number of photons + # in DexDem (DexAem) in current burst. Thus the sum is a mean. + ED = np.mean(kde_adi / (kde_adi + nbkde_ddi)) # (E)_D + if np.isnan(ED): + print(kde_adi, nbkde_ddi) + print(N_CHD) + print("______") + EA = np.mean(kde_dai / (kde_dai + nbkde_aai)) # (1 - E)_A + + # Compute fret_2cde for current burst + fret_2cde = 110 - 100 * (ED + EA) + FRET_2CDE.append(fret_2cde) + return np.array(FRET_2CDE) + + +fret_2cde = calc_fret_2cde(tau, ph, mask_d, mask_a, bursts) +fret_2cde +# %% + +# %% + +# %% +bursts = ds1.mburst[0] + +ph_dex = d.get_ph_times(ph_sel=Ph_sel(Dex="DAem")) +ph_aex = d.get_ph_times(ph_sel=Ph_sel(Aex="Aem")) + +mask_dex = d.get_ph_mask(ph_sel=Ph_sel(Dex="DAem")) +mask_aex = d.get_ph_mask(ph_sel=Ph_sel(Aex="Aem")) + +KDE_DexTi = phrates.kde_laplace(ph_dex, tau, time_axis=ph) +KDE_AexTi = phrates.kde_laplace(ph_aex, tau, time_axis=ph) + +# %% + +ALEX_2CDE = [] +BRDex, BRAex = [], [] +for ib, burst in enumerate(bursts): + burst_slice = slice(int(burst.istart), int(burst.istop) + 1) + if ~mask_dex[burst_slice].any() or ~mask_aex[burst_slice].any(): + # Either D or A photon stream has no photons in current burst, + # thus ALEX_2CDE cannot be computed. + ALEX_2CDE.append(np.nan) + continue + + kde_dexdex = KDE_DexTi[burst_slice][mask_dex[burst_slice]] + kde_aexdex = KDE_AexTi[burst_slice][mask_dex[burst_slice]] + N_chaex = mask_aex[burst_slice].sum() + BRDex.append(np.sum(kde_aexdex / kde_dexdex) / N_chaex) + + kde_aexaex = KDE_AexTi[burst_slice][mask_aex[burst_slice]] + kde_dexaex = KDE_DexTi[burst_slice][mask_aex[burst_slice]] + N_chdex = mask_dex[burst_slice].sum() + BRAex.append(np.sum(kde_dexaex / kde_aexaex) / N_chdex) + + # !! changed to + compared to fretbrusts code + alex_2cde = 100 - 50 * (BRDex[-1] + BRAex[-1]) + ALEX_2CDE.append(alex_2cde) +ALEX_2CDE = np.array(ALEX_2CDE) + + +# %% +timestamps = ph +streams = { + "DD": {"Dex": "Dem"}, + "DA": {"Dex": "Aem"}, + "AA": {"Aex": "Aem"}, + "AD": {"Aex": "Dem"}, +} + +stream = np.empty_like(timestamps, dtype="U2") +for stream_label, kwargs in streams.items(): + mask = d.get_ph_mask(ph_sel=Ph_sel(**kwargs)) + stream[mask] = stream_label + +# %% + + +photons_export = pl.DataFrame({"timestamps": timestamps, "stream": stream}) +photons_export.write_parquet("photon_data.pq") + +# %% + + +burst_export = pl.DataFrame([{"istart": b.istart, "istop": b.istop} for b in bursts]) +burst_export = burst_export.with_columns( + [ + pl.lit(ALEX_2CDE).alias("alex_2cde"), + pl.lit(fret_2cde).alias("fret_2cde"), + ] +) +burst_export.write_parquet("burst_data.pq") diff --git a/tests/test_c_kde.py b/tests/test_c_kde.py new file mode 100644 index 0000000..515aa1b --- /dev/null +++ b/tests/test_c_kde.py @@ -0,0 +1,71 @@ +# %% + +from pathlib import Path + +import numpy as np +import polars as pl +import polars.testing as pl_test +import pytest + +from dont_fret.channel_kde import compute_alex_2cde, compute_fret_2cde, convolve_stream, make_kernel + +cwd = Path(__file__).parent +input_data_dir = cwd / "test_data" / "input" +output_data_dir = cwd / "test_data" / "output" + +tau = 50e-6 +TIMESTAMPS_UNIT = 1.25e-08 + + +photon_data = pl.read_parquet(output_data_dir / "kde" / "photon_data.pq") +bursts_ref = pl.read_parquet(output_data_dir / "kde" / "burst_data.pq") + + +@pytest.fixture +def burst_data() -> pl.DataFrame: + burst_dfs = [] + for i in range(len(bursts_ref)): + istart = bursts_ref[i]["istart"].item() + istop = bursts_ref[i]["istop"].item() + b_df = photon_data[istart : istop + 1].with_columns(pl.lit(i).alias("burst_index")) + burst_dfs.append(b_df) + + burst_data = pl.concat(burst_dfs, how="vertical_relaxed") + + return burst_data + + +def test_compare_fret_cde_fretbursts(burst_data): + kernel = make_kernel(tau, TIMESTAMPS_UNIT) + A_em = convolve_stream(photon_data, ["DA"], kernel) + D_em = convolve_stream(photon_data, ["DD"], kernel) + kde_data = photon_data.select( + [ + pl.col("timestamps"), + pl.col("stream"), + pl.lit(A_em).alias("A_em"), + pl.lit(D_em).alias("D_em"), + ] + ) + + fret_2cde = compute_fret_2cde(burst_data, kde_data) + pl_test.assert_series_equal( + pl.Series(name="fret_2cde", values=fret_2cde), bursts_ref["fret_2cde"] + ) + + +def test_compare_alex_cde_fretbursts(burst_data): + kernel = make_kernel(tau, TIMESTAMPS_UNIT) + D_ex = convolve_stream(photon_data, ["DD", "DA"], kernel) + A_ex = convolve_stream(photon_data, ["AA"], kernel) + kde_data = photon_data.select( + [ + pl.col("timestamps"), + pl.col("stream"), + pl.lit(D_ex).alias("D_ex"), + pl.lit(A_ex).alias("A_ex"), + ] + ) + + alex_2cde_vals = compute_alex_2cde(burst_data, kde_data) + pl_test.assert_series_equal(alex_2cde_vals.fill_null(np.nan), bursts_ref["alex_2cde"]) diff --git a/tests/test_data/output/kde/burst_data.pq b/tests/test_data/output/kde/burst_data.pq new file mode 100644 index 0000000..cd9b57d Binary files /dev/null and b/tests/test_data/output/kde/burst_data.pq differ diff --git a/tests/test_data/output/kde/photon_data.pq b/tests/test_data/output/kde/photon_data.pq new file mode 100644 index 0000000..139f93b Binary files /dev/null and b/tests/test_data/output/kde/photon_data.pq differ diff --git a/tests/test_models.py b/tests/test_models.py index 5f35f20..f951e9e 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -8,6 +8,7 @@ from dont_fret.config.config import BurstColor from dont_fret.fileIO import PhotonFile from dont_fret.models import BinnedPhotonData, Bursts, PhotonData +from dont_fret.process import process_photon_data cwd = Path(__file__).parent input_data_dir = cwd / "test_data" / "input" @@ -80,6 +81,8 @@ def test_load_save_bursts(dcbs_bursts: Bursts, tmp_path: Path): assert dcbs_bursts.burst_data.equals(bursts_load.burst_data) assert dcbs_bursts.photon_data.equals(bursts_load.photon_data) assert dcbs_bursts.cfg == bursts_load.cfg + assert dcbs_bursts.cfg + assert bursts_load.cfg assert asdict(dcbs_bursts.cfg) == asdict(bursts_load.cfg) assert dcbs_bursts.metadata == bursts_load.metadata @@ -109,6 +112,17 @@ def test_burst_search(ph_ds1: PhotonData): assert (df_ref["time_max"] == df_test["timestamps_max"] * ph_ds1.timestamps_unit).all() +def test_process_photon_data(ph_ds1: PhotonData): + hooks = { + "alex_2cde": {}, + "fret_2cde": {}, + } + + bursts = process_photon_data(ph_ds1, APBS_TEST, hooks=hooks) + assert "alex_2cde" in bursts.burst_data.columns + assert "fret_2cde" in bursts.burst_data.columns + + def test_binning(ph_ds1): trace = BinnedPhotonData(ph_ds1, bounds=(0, ph_ds1.tmax)) assert len(trace) == 81439 diff --git a/tests/test_web.py b/tests/test_web.py index 318c03a..44abba9 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -72,7 +72,7 @@ async def test_burst_search(): assert new_node.bursts burst_item = new_node.bursts[0] assert burst_item.name == "DCBS" - assert burst_item.df.shape == (72, 22) + assert burst_item.df.shape == (72, 24) assert burst_item.df["filename"].unique()[0] == "datafile_1.ptu" await asyncio.sleep(0)