Merge pull request #88 from will-moore/ome_2021_workshop_features

ome 2021 workshop

maintenance/preparation/idr0079-data-prep.md

@@ -0,0 +1,107 @@
+
+Workshop data preparation (idr0079)
+===================================
+
+This document details the steps to prepare data from idr0079 for use in an OMERO.parade
+and OMERO.figure workshop. 
+
+
+IDR data import
+===============
+
+For OME team, steps for in-place import of IDR data can be found at
+https://docs.google.com/document/d/18cLSvUKVn8jEp7KSW7e48NvuxHT_mp3tyh98loTTOYY/edit
+
+For other users, you will need to have Docker installed.
+This container uses Aspera to download the data from EBI:
+
+	$ docker run --rm -v /tmp:/data imagedata/download idr0079 . /data/
+
+Clone https://github.com/IDR/idr0079-hartmann-lateralline and edit
+```experimentA/idr0079-experimentA-filePaths.tsv```
+to point ALL paths at the location of the data downloaded above.
+
+If you don't want to use in-place import, comment out this line in
+```experimentA/idr0079-experimentA-bulk.yml```:
+
+	transfer: "ln_s"
+
+
+Do the bulk import:
+
+	$ cd experimentA/
+	$ omero import --bulk idr0079-experimentA-bulk.yml
+
+
+Add Map Annotations from IDR
+============================
+
+Use the script [idr_get_map_annotations.py](../scripts/idr_get_map_annotations.py) with the ID of
+the 'idr0079-hartmann-lateralline/experimentA' Project created above and the corresponding
+Project on IDR (1102):
+
+	$ python idr_get_map_annotations.py username password Project:1102 Project:1301 --server localhost
+
+This will get map annotations from all Images in `idr0079-hartmann-lateralline/experimentA` and
+create identical map annotations on the corresponding Images.
+
+
+Rename Channels from Map Annotations
+====================================
+
+We can now use the map annotations to rename channels on all images.
+Run the [channel_names_from_maps.py](../scripts/channel_names_from_maps.py)
+script on the local data, using the local Project ID.
+We are using the `stain` to name channels, e.g. `NLStdTomato`, and also adding
+map annotations for individual channels to use for label creation in OMERO.figure:
+
+    $ python channel_names_from_maps.py username password 1301 --server localhost --use_stain --add_map_anns
+
+
+Manual Annotations and Rendering Settings
+=========================================
+
+Add 5* rating to 1 image each from membranes_actin, membranes_cisgolgi, membranes_nuclei, membranes_recendo.
+
+Set rendering settings: Channel-1: Green, Channel-2: Red. Maybe adjust levels too.
+
+Set Pixel Sizes
+==========================================
+
+See the commands used for IDR at [idr0079_voxel_sizes.sh]
+(https://github.com/IDR/idr0079-hartmann-lateralline/blob/master/scripts/idr0079_voxel_sizes.sh)
+and run these commands on the local server, using the appropriate Dataset IDs, at least
+for the Datasets you wish to use.
+
+Copy Masks to Polygons
+======================
+
+The idr0079 images in IDR have Masks, but we want to use Polygons in OMERO.figure.
+Use the [copy_masks_2_polygons.py](../scripts/copy_masks_2_polygons.py) script to
+convert. NB: requires `skimage`. We can process an Image or a Dataset at a time:
+
+    # login to IDR
+    $ omero login
+
+    # copy from IDR e.g Dataset:1 to local server TARGET e.g. Dataset:2
+    $ python copy_masks_2_polygons.py username password server Dataset:1 Dataset:2
+
+Create OMERO.tables
+===================
+
+We use the tsv files in https://github.com/IDR/idr0079-hartmann-lateralline to create
+an OMERO.table on the Project (for use with OMERO.parade), with one row per Image, summarising the stats for all the
+ROIs in that Image. This uses [idr0079_csv_to_table_on_project.py](../scripts/idr0079_csv_to_table_on_project.py)
+
+Clone the `idr0079-hartmann-lateralline` github repo, then:
+
+    $ cd idr0079-hartmann-lateralline
+    $ python /path/to/training-scripts/maintenance/scripts/csv_to_table_on_project.py
+
+We then create an OMERO.table on each Image that has ROIs added above, using
+the `_other_measurements.tsv` table for each Image.
+Use the optional `--name NAME` to run on a single named Image:
+
+    $ cd idr0079-hartmann-lateralline
+    # process ALL raw images (use --name NAME to process 1 image)
+    $ python scripts/csv_to_roi_table.py _other_measurements.tsv

maintenance/scripts/channel_names_from_maps.py

@@ -20,14 +20,31 @@
 # Script uses map annotations on each Image to rename channels
 
 import argparse
-from omero.gateway import BlitzGateway
+from omero.gateway import BlitzGateway, MapAnnotationWrapper
 
 
 NAMESPACE = "openmicroscopy.org/omero/bulk_annotations"
 MAP_KEY = "Channels"
 
 
-def run(username, password, project_id, host, port):
+def create_map_ann(conn, obj, key_value_data):
+    map_ann = MapAnnotationWrapper(conn)
+    map_ann.setValue(key_value_data)
+    map_ann.setNs('from.channels.keyvaluepair')
+    map_ann.save()
+    obj.linkAnnotation(map_ann)
+
+
+def run(args):
+    username = args.username
+    password = args.password
+    project_id = args.project_id
+    host = args.server
+    port = args.port
+    use_stain = args.use_stain
+    add_map_anns = args.add_map_anns
+
+    token_index = 0 if use_stain else 1
 
     conn = BlitzGateway(username, password, host=host, port=port)
     try:
@@ -50,10 +67,23 @@ def run(username, password, project_id, host, port):
                 channels = values[0].split("; ")
                 print("Channels", channels)
                 name_dict = {}
+                key_value_pairs = []
                 for c, ch_name in enumerate(channels):
-                    name_dict[c + 1] = ch_name.split(":")[1]
+                    tokens = ch_name.split(":")
+                    if add_map_anns and len(tokens) > 1:
+                        key_value_pairs.extend(
+                            [["Ch%s_Stain" % c, tokens[0]],
+                             ["Ch%s_Label" % c, tokens[1]]]
+                        )
+                    if len(tokens) > token_index:
+                        label = tokens[token_index]
+                    else:
+                        label = ch_name
+                    name_dict[c + 1] = label
                 conn.setChannelNames("Image", [image.id], name_dict,
                                      channelCount=None)
+                if len(key_value_pairs) > 0:
+                    create_map_ann(conn, image, key_value_pairs)
     except Exception as exc:
         print("Error while changing names: %s" % str(exc))
     finally:
@@ -65,11 +95,19 @@ def main(args):
     parser.add_argument('username')
     parser.add_argument('password')
     parser.add_argument('project_id')
+    parser.add_argument(
+        '--use_stain', action='store_true',
+        help="""Map Ann Channels are in the form stain:label, e.g. DAPI:DNA.
+If use_stain, channels will be named with the stain instead of the label""")
+    parser.add_argument(
+        '--add_map_anns', action='store_true',
+        help="""Create new Map Anns of the form
+Ch1_Stain:DAPI, Ch1_Label:DNA etc using the Channels Key-Value Pair""")
     parser.add_argument('--server', default="workshop.openmicroscopy.org",
                         help="OMERO server hostname")
     parser.add_argument('--port', default=4064, help="OMERO server port")
     args = parser.parse_args(args)
-    run(args.username, args.password, args.project_id, args.server, args.port)
+    run(args)
 
 
 if __name__ == '__main__':

maintenance/scripts/copy_masks_2_polygons.py

@@ -0,0 +1,211 @@
+
+from skimage import measure
+
+import argparse
+import sys
+import numpy as np
+
+import omero
+import omero.clients
+from omero.gateway import BlitzGateway
+from omero.rtypes import unwrap, rint, rstring
+from omero.cli import cli_login
+from omero.api import RoiOptions
+
+PAGE_SIZE = 10
+
+def mask_to_binim_yx(mask):
+    """
+    # code from omero-cli-zarr
+    :param mask MaskI: An OMERO mask
+
+    :return: tuple of
+            - Binary mask
+            - (T, C, Z, Y, X, w, h) tuple of mask settings (T, C, Z may be
+            None)
+    """
+
+    t = unwrap(mask.theT)
+    c = unwrap(mask.theC)
+    z = unwrap(mask.theZ)
+
+    x = int(mask.x.val)
+    y = int(mask.y.val)
+    w = int(mask.width.val)
+    h = int(mask.height.val)
+
+    mask_packed = mask.getBytes()
+    # convert bytearray into something we can use
+    intarray = np.fromstring(mask_packed, dtype=np.uint8)
+    binarray = np.unpackbits(intarray)
+    # truncate and reshape
+    binarray = np.reshape(binarray[: (w * h)], (h, w))
+
+    return binarray, (t, c, z, y, x, h, w)
+
+
+def rgba_to_int(red, green, blue, alpha=255):
+    """ Return the color as an Integer in RGBA encoding """
+    r = red << 24
+    g = green << 16
+    b = blue << 8
+    a = alpha
+    rgba_int = r+g+b+a
+    if (rgba_int > (2**31-1)):       # convert to signed 32-bit int
+        rgba_int = rgba_int - 2**32
+    return rgba_int
+
+
+def get_longest_contour(contours):
+    contour = contours[0]
+    for c in contours:
+        if len(c) > len(contour):
+            contour = c
+    return c
+
+
+def image_ids_by_name(dataset):
+    ids_by_name = {}
+    for image in dataset.listChildren():
+        ids_by_name[image.name] = image.id
+    return ids_by_name
+
+
+def add_polygon(roi, contour, x_offset=0, y_offset=0, z=None, t=None):
+    """ points is 2D list of [[x, y], [x, y]...]"""
+
+    stride = 4
+    coords = []
+    # points in contour are adjacent pixels, which is too verbose
+    # take every nth point
+    for count, xy in enumerate(contour):
+        if count % stride == 0:
+            coords.append(xy)
+    if len(coords) < 2:
+        return
+    points = ["%s,%s" % (xy[1] + x_offset, xy[0] + y_offset) for xy in coords]
+    points = ", ".join(points)
+
+    polygon = omero.model.PolygonI()
+    if z is not None:
+        polygon.theZ = rint(z)
+    if t is not None:
+        polygon.theT = rint(t)
+    polygon.strokeColor = rint(rgba_to_int(255, 255, 255))
+    # points = "10,20, 50,150, 200,200, 250,75"
+    polygon.points = rstring(points)
+    roi.addShape(polygon)
+
+
+def process_image(conn, conn2, image, to_image_id):
+    roi_service = conn.getRoiService()
+    update_service = conn2.getUpdateService()
+
+    print("Processing...", image.name, image.id)
+
+    old = conn2.getRoiService().findByImage(to_image_id, None, conn2.SERVICE_OPTS)
+    if len(old.rois) > 0:
+        print("Image", to_image_id, "already has ROIs. Ignoring...")
+        return
+
+    opts = RoiOptions()
+    offset = 0
+    opts.offset = rint(offset)
+    opts.limit = rint(PAGE_SIZE)
+
+    size_x = image.getSizeX()
+    size_y = image.getSizeY()
+
+    # NB: we repeat this query below for each 'page' of ROIs
+    result = roi_service.findByImage(image.id, opts, conn.SERVICE_OPTS)
+
+    while len(result.rois) > 0:
+        print("offset", offset)
+        print("Found ROIs:", len(result.rois))
+        for roi in result.rois:
+
+            new_roi = omero.model.RoiI()
+            new_roi.setImage(omero.model.ImageI(to_image_id, False))
+            shapes_added = False
+
+            for shape in roi.copyShapes():
+
+                if not isinstance(shape, omero.model.MaskI):
+                    continue
+                # assume shape is a Mask
+                np_mask, dims = mask_to_binim_yx(shape)
+                t, c, z, y, x, h, w = dims
+                plane = np.zeros((size_y, size_x))
+                plane[y:y+h, x:x+w] = np_mask
+                contours = measure.find_contours(plane, 0.5)
+                print('Found contours:', len(contours))
+                if len(contours) > 0:
+                    contour = get_longest_contour(contours)
+                    # Only add 1 Polygon per Mask Shape.
+                    # First is usually the longest
+                    add_polygon(new_roi, contour, 0, 0, z, t)
+                shapes_added = True
+
+            if shapes_added:
+                update_service.saveObject(new_roi)
+
+        offset += PAGE_SIZE
+        opts.offset = rint(offset)
+        result = roi_service.findByImage(image.id, opts, conn.SERVICE_OPTS)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('username2', help='Target server Username')
+    parser.add_argument('password2', help='Target server Password')
+    parser.add_argument('server2', help='Target server')
+    parser.add_argument('source', help=(
+        'Copy ROIs FROM this: Image:ID or Dataset:ID'))
+    parser.add_argument('target', help=(
+        'Copy ROIs TO this: Image:ID or Dataset:ID'))
+    args = parser.parse_args(argv)
+
+
+    with cli_login() as cli:
+        conn = BlitzGateway(client_obj=cli._client)
+        conn.SERVICE_OPTS.setOmeroGroup(-1)
+
+        conn2 = BlitzGateway(args.username2, args.password2,
+                             port=4064, host=args.server2)
+        conn2.connect()
+
+        source_images = []
+        target_image_ids = []
+
+        source = args.source
+        source_id = int(source.split(":")[1])
+        target = args.target
+        target_id = int(target.split(":")[1])
+
+        if source.startswith('Image:'):
+            source_images.append(conn.getObject('Image', source_id))
+            target_image_ids.append(target_id)
+        elif source.startswith('Dataset:'):
+            dataset = conn.getObject('Dataset', source_id)
+            target_dataset = conn2.getObject('Dataset', target_id)
+            ids_by_name = image_ids_by_name(target_dataset)
+            for image in dataset.listChildren():
+                if image.name in ids_by_name:
+                    source_images.append(image)
+                    target_image_ids.append(ids_by_name[image.name])
+        else:
+            print("Source needs to be Image:ID or Dataset:ID")
+
+        print("Processing", source_images)
+        print("...to target images:", target_image_ids)
+        for image, to_target_id in zip(source_images, target_image_ids):
+            process_image(conn, conn2, image, to_target_id)
+
+        conn2.close()
+
+
+if __name__ == '__main__':
+    # First, login to source OMERO $ omero login
+    # Copy to target server, with source and target. e.g. Image:123 Image:456
+    # $ python copy_masks_2_polygons.py user pass server FROM_TARGET TO_TARGET
+    main(sys.argv[1:])