Merge pull request #66 from neuro-ml/develop

v0.1.2

.github/workflows/release.yml

@@ -0,0 +1,38 @@
+name: Release
+
+on:
+  release:
+    types: [ released ]
+
+env:
+  MODULE_NAME: deep-pipe
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - id: get_version
+        name: Get the release version
+        uses: battila7/get-version-action@v2
+
+      - name: Check the version and build the package
+        run: |
+          RELEASE=${{ steps.get_version.outputs.version-without-v }}
+          VERSION=$(python -c "from pathlib import Path; import runpy; folder, = {d.parent for d in Path().resolve().glob('*/__init__.py')}; print(runpy.run_path(folder / '__init__.py')['__version__'])")
+          MATCH=$(pip index versions $MODULE_NAME | grep "Available versions:" | grep $VERSION) || echo
+          echo $MATCH
+          if [ "$GITHUB_BASE_REF" = "master" ] && [ "$MATCH" != "" ]; then echo "Version $VERSION already present" && exit 1; fi
+          if [ "$VERSION" != "$RELEASE" ]; then echo "$VERSION vs $RELEASE" && exit 1; fi
+          python setup.py sdist
+
+      - name: Publish to PyPi
+        uses: pypa/gh-action-pypi-publish@master
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

dpipe/__init__.py

@@ -1 +1 @@
-__version__ = '0.1.1'
+__version__ = '0.1.2'

dpipe/commands.py

@@ -10,7 +10,6 @@
 from tqdm import tqdm
 
 from .io import save_json, PathLike, load as _load, save as _save
-from dpipe.itertools import collect
 
 
 def populate(path: PathLike, func: Callable, *args, **kwargs):
@@ -78,7 +77,6 @@ def transform(input_path, output_path, transform_fn):
         np.save(os.path.join(output_path, f), transform_fn(np.load(os.path.join(input_path, f))))
 
 
-@collect
 def load_from_folder(path: PathLike, loader=_load, ext='.npy'):
     """Yields (id, object) pairs loaded from ``path``."""
     for file in sorted(Path(path).iterdir()):

dpipe/im/grid.py

@@ -2,7 +2,7 @@
 Function for working with patches from tensors.
 See the :doc:`tutorials/patches` tutorial for more details.
 """
-from typing import Iterable, Type, Tuple
+from typing import Iterable, Type, Tuple, Callable
 
 import numpy as np
 
@@ -49,7 +49,7 @@ def get_boxes(shape: AxesLike, box_size: AxesLike, stride: AxesLike, axis: AxesL
 
 
 def divide(x: np.ndarray, patch_size: AxesLike, stride: AxesLike, axis: AxesLike = None,
-           valid: bool = False) -> Iterable[np.ndarray]:
+           valid: bool = False, get_boxes: Callable = get_boxes) -> Iterable[np.ndarray]:
     """
     A convolution-like approach to generating patches from a tensor.
 
@@ -63,6 +63,8 @@ def divide(x: np.ndarray, patch_size: AxesLike, stride: AxesLike, axis: AxesLike
         the stride (step-size) of the slice.
     valid
         whether patches of size smaller than ``patch_size`` should be left out.
+    get_boxes
+        function that yields boxes, for signature see ``get_boxes``
 
     References
     ----------
@@ -101,7 +103,8 @@ def build(self):
 
 
 def combine(patches: Iterable[np.ndarray], output_shape: AxesLike, stride: AxesLike,
-            axis: AxesLike = None, valid: bool = False, combiner: Type[PatchCombiner] = Average) -> np.ndarray:
+            axis: AxesLike = None, valid: bool = False,
+            combiner: Type[PatchCombiner] = Average, get_boxes: Callable = get_boxes) -> np.ndarray:
     """
     Build a tensor of shape ``output_shape`` from ``patches`` obtained in a convolution-like approach
     with corresponding parameters.

dpipe/im/metrics.py

@@ -52,7 +52,9 @@ def sensitivity(y_true, y_pred):
 @add_check_bool
 @add_check_shapes
 def specificity(y_true, y_pred):
-    return fraction(np.sum(y_pred & y_true), np.sum(y_pred), empty_val=0)
+    tn = np.sum((~y_true) & (~y_pred))
+    fp = np.sum(y_pred & (~y_true))
+    return fraction(tn, tn + fp, empty_val=0)
 
 
 @add_check_bool

dpipe/layout/scripts.py

@@ -1,7 +1,7 @@
 import argparse
 
 import lazycon
-
+from pathlib import Path
 
 def build():
     parser = argparse.ArgumentParser('Build an experiment layout from the provided config.', add_help=False)
@@ -20,10 +20,12 @@ def run():
     parser = argparse.ArgumentParser('Run an experiment based on the provided config.', add_help=False)
     parser.add_argument('config')
     args = parser.parse_known_args()[0]
+    config_path = Path(args.config).absolute()
 
-    layout = lazycon.load(args.config).layout
+    layout = lazycon.load(config_path).layout
     layout.run_parser(parser)
     parser.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='Show this message and exit')
 
     args = parser.parse_args()
-    layout.run(**vars(args))
+    folds = args.folds
+    layout.run(config=config_path, folds=folds)

dpipe/predict/shape.py

@@ -4,7 +4,7 @@
 import numpy as np
 
 from ..im.axes import broadcast_to_axis, AxesLike, AxesParams, axis_from_dim, resolve_deprecation
-from ..im.grid import divide, combine, PatchCombiner, Average
+from ..im.grid import divide, combine, get_boxes, PatchCombiner, Average
 from ..itertools import extract, pmap
 from ..im.shape_ops import pad_to_shape, crop_to_shape, pad_to_divisible
 from ..im.shape_utils import prepend_dims, extract_dims
@@ -81,7 +81,7 @@ def wrapper(x, *args, **kwargs):
 
 def patches_grid(patch_size: AxesLike, stride: AxesLike, axis: AxesLike = None,
                  padding_values: Union[AxesParams, Callable] = 0, ratio: AxesParams = 0.5,
-                 combiner: Type[PatchCombiner] = Average):
+                 combiner: Type[PatchCombiner] = Average, get_boxes: Callable = get_boxes):
     """
     Divide an incoming array into patches of corresponding ``patch_size`` and ``stride`` and then combine
     the predicted patches by aggregating the overlapping regions using the ``combiner`` - Average by default.
@@ -107,8 +107,15 @@ def wrapper(x, *args, **kwargs):
                 new_shape = padded_shape + (local_stride - padded_shape + local_size) % local_stride
                 x = pad_to_shape(x, new_shape, input_axis, padding_values, ratio)
 
-            patches = pmap(predict, divide(x, local_size, local_stride, input_axis), *args, **kwargs)
-            prediction = combine(patches, extract(x.shape, input_axis), local_stride, axis, combiner=combiner)
+            patches = pmap(
+                predict,
+                divide(x, local_size, local_stride, input_axis, get_boxes=get_boxes),
+                *args, **kwargs
+            )
+            prediction = combine(
+                patches, extract(x.shape, input_axis), local_stride, axis,
+                combiner=combiner, get_boxes=get_boxes
+            )
 
             if valid:
                 prediction = crop_to_shape(prediction, shape, axis, ratio)

dpipe/py.typed

@@ -0,0 +1,4 @@
+This marker file declares that the package supports type checking.
+For details, you can refer to:
+- PEP561: https://www.python.org/dev/peps/pep-0561/
+- mypy docs: https://mypy.readthedocs.io/en/stable/installed_packages.html

dpipe/tests/test_gradient_accumulation.py

@@ -0,0 +1,76 @@
+import pytest
+
+import torch
+import numpy as np
+from torch import nn
+from dpipe.torch import train_step
+from dpipe.train import train
+
+
+@pytest.mark.parametrize('batch_size', [4, 16, 64])
+def test_train(batch_size):
+    net1 = nn.Sequential(
+        nn.Conv2d(3, 4, kernel_size=3, padding=1),
+        nn.LayerNorm([28, 28]),
+        nn.GELU(),
+        nn.Conv2d(4, 8, kernel_size=3, padding=1),
+        nn.LayerNorm([28, 28]),
+        nn.GELU(),
+        nn.Conv2d(8, 16, kernel_size=3, padding=1),
+        nn.LayerNorm([28, 28]),
+        nn.GELU(),
+    )
+    net2 = nn.Sequential(
+        nn.Conv2d(3, 4, kernel_size=3, padding=1),
+        nn.LayerNorm([28, 28]),
+        nn.GELU(),
+        nn.Conv2d(4, 8, kernel_size=3, padding=1),
+        nn.LayerNorm([28, 28]),
+        nn.GELU(),
+        nn.Conv2d(8, 16, kernel_size=3, padding=1),
+        nn.LayerNorm([28, 28]),
+        nn.GELU(),
+    )
+    net2.load_state_dict(net1.state_dict())
+
+    opt1 = torch.optim.SGD(net1.parameters(), lr=3e-4)
+    opt2 = torch.optim.SGD(net2.parameters(), lr=3e-4)
+
+    n_epochs = 10
+    n_batches = 10
+
+    data = np.random.randn(n_batches, batch_size, 3, 28, 28).astype(np.float32)
+
+    def batch_iter1():
+        for x in data:
+            yield x, 0
+
+    def batch_iter2():
+        for x in data:
+            for batch_el in x:
+                yield batch_el[None], 0
+
+    def criterion(x, y):
+        return x.mean()
+
+    train(
+        train_step,
+        batch_iter1,
+        n_epochs=n_epochs,
+        architecture=net1,
+        optimizer=opt1,
+        criterion=criterion,
+    )
+
+    train(
+        train_step,
+        batch_iter2,
+        n_epochs=n_epochs,
+        architecture=net2,
+        optimizer=opt2,
+        criterion=criterion,
+        gradient_accumulation_steps=batch_size,
+    )
+
+    for param1, param2 in zip(net1.parameters(), net2.parameters()):
+        assert torch.allclose(param1, param2)

dpipe/torch/functional.py

@@ -1,3 +1,4 @@
+import warnings
 from typing import Union, Callable
 
 import numpy as np
@@ -8,6 +9,8 @@
 
 __all__ = [
     'focal_loss_with_logits', 'linear_focal_loss_with_logits', 'weighted_cross_entropy_with_logits',
+    'tversky_loss', 'focal_tversky_loss', 'tversky_loss_with_logits', 'focal_tversky_loss_with_logits',
+    'dice_loss', 'dice_loss_with_logits',
     'masked_loss', 'moveaxis', 'softmax',
 ]
 
@@ -141,7 +144,7 @@ def weighted_cross_entropy_with_logits(logit: torch.Tensor, target: torch.Tensor
     return loss
 
 
-def dice_loss(pred: torch.Tensor, target: torch.Tensor):
+def dice_loss(pred: torch.Tensor, target: torch.Tensor, epsilon=1e-7):
     """
     References
     ----------
@@ -152,22 +155,72 @@ def dice_loss(pred: torch.Tensor, target: torch.Tensor):
 
     sum_dims = list(range(1, target.dim()))
 
-    dice = 2 * torch.sum(pred * target, dim=sum_dims) / torch.sum(pred ** 2 + target ** 2, dim=sum_dims)
+    dice = 2 * torch.sum(pred * target, dim=sum_dims) / (torch.sum(pred ** 2 + target ** 2, dim=sum_dims) + epsilon)
     loss = 1 - dice
 
     return loss.mean()
 
 
-def dice_loss_with_logits(logit: torch.Tensor, target: torch.Tensor):
+def tversky_loss(pred: torch.Tensor, target: torch.Tensor, alpha=0.5, epsilon=1e-7,
+                 reduce: Union[Callable, None] = torch.mean):
+    """
+    References
+    ----------
+    `Tversky Loss https://arxiv.org/abs/1706.05721`_
+    """
+    if not (target.size() == pred.size()):
+        raise ValueError("Target size ({}) must be the same as logit size ({})".format(target.size(), pred.size()))
+
+    if alpha < 0 or alpha > 1:
+        raise ValueError("Invalid alpha value, expected to be in (0, 1) interval")
+
+    sum_dims = list(range(1, target.dim()))
+    beta = 1 - alpha
+
+    intersection = pred*target
+    fps, fns = pred*(1-target), (1-pred)*target
+
+    numerator = torch.sum(intersection, dim=sum_dims)
+    denumenator = torch.sum(intersection, dim=sum_dims) + alpha*torch.sum(fps, dim=sum_dims) + beta*torch.sum(fns, dim=sum_dims)
+    tversky = numerator / (denumenator + epsilon)
+    loss = 1 - tversky
+
+    if reduce is not None:
+        loss = reduce(loss)
+    return loss
+
+
+def focal_tversky_loss(pred: torch.Tensor, target: torch.Tensor, gamma=4/3, alpha=0.5, epsilon=1e-7):
+    """
+    References
+    ----------
+    `Focal Tversky Loss https://arxiv.org/abs/1810.07842`_
     """
-        References
-        ----------
-        `Dice Loss <https://arxiv.org/abs/1606.04797>`_
-        """
+    if gamma <= 1:
+        warnings.warn("Gamma is <=1, to focus on less accurate predictions choose gamma > 1.")
+    tl = tversky_loss(pred, target, alpha, epsilon, reduce=None)
+
+    return torch.pow(tl, 1/gamma).mean()
+
+
+def loss_with_logits(criterion: Callable, logit: torch.Tensor, target: torch.Tensor, **kwargs):
     if not (target.size() == logit.size()):
         raise ValueError("Target size ({}) must be the same as logit size ({})".format(target.size(), logit.size()))
     pred = torch.sigmoid(logit)
-    return dice_loss(pred, target)
+
+    return criterion(pred, target, **kwargs)
+
+
+def dice_loss_with_logits(logit: torch.Tensor, target: torch.Tensor):
+    return loss_with_logits(dice_loss, logit, target)
+
+
+def tversky_loss_with_logits(logit: torch.Tensor, target: torch.Tensor, alpha=0.5):
+    return loss_with_logits(tversky_loss, logit, target, alpha=alpha)
+
+
+def focal_tversky_loss_with_logits(logit: torch.Tensor, target: torch.Tensor, gamma, alpha=0.5):
+    return loss_with_logits(focal_tversky_loss, logit, target, gamma=gamma, alpha=alpha)
 
 
 def masked_loss(mask: torch.Tensor, criterion: Callable, prediction: torch.Tensor, target: torch.Tensor, **kwargs):