Mercurial > repos > iuc > idr_download_by_ids
changeset 0:57aa9597cd31 draft
"planemo upload for repository https://github.com/galaxyproject/tools-iuc/tree/master/tools/idr_download commit 253efabdfea3a1fecc4c0f2c54c0e97a7d7960ab"
| author | iuc |
|---|---|
| date | Sat, 08 Feb 2020 13:24:39 -0500 |
| parents | |
| children | 9340cbc7796c |
| files | idr_download_by_ids.py idr_download_by_ids.xml test-data/ids.txt test-data/test1.tiff |
| diffstat | 4 files changed, 529 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/idr_download_by_ids.py Sat Feb 08 13:24:39 2020 -0500 @@ -0,0 +1,271 @@ +import argparse +import os +import sys + +from matplotlib import pyplot as plt +from omero.gateway import BlitzGateway # noqa +from omero.constants.namespaces import NSBULKANNOTATIONS # noqa + + +def warn(message, image_identifier): + print( + 'ImageSpecWarning for {0}: {1}' + .format(image_identifier, message), + file=sys.stderr + ) + + +def find_channel_index(image, channel_name): + channel_name = channel_name.lower() + for n, channel in enumerate(image.getChannels()): + if channel_name == channel.getLabel().lower(): + return n + # Check map annotation for information (this is necessary for some images) + for ann in image.listAnnotations(NSBULKANNOTATIONS): + pairs = ann.getValue() + for p in pairs: + if p[0] == "Channels": + channels = p[1].replace(" ", "").split(";") + for n, c in enumerate(channels): + for value in c.split(':'): + if channel_name == value.lower(): + return n + return -1 + + +def get_clipping_region(image, x, y, w, h): + # If the (x, y) coordinate falls outside the image boundaries, we + # cannot just shift it because that would render the meaning of + # w and h undefined (should width and height be decreased or the whole + # region be shifted to keep them fixed?). + # It may be better to abort in this situation. + if x < 0 or y < 0: + raise ValueError( + 'Too small upper left coordinate ({0}, {1}) for clipping region.' + .format(x, y) + ) + size_x = image.getSizeX() + size_y = image.getSizeY() + if x >= size_x or y >= size_y: + raise ValueError( + 'Upper left coordinate ({0}, {1}) of clipping region lies ' + 'outside of image.' + .format(x, y) + ) + # adjust width and height to the image dimensions + if w <= 0 or x + w > size_x: + w = size_x - x + if h <= 0 or y + h > size_y: + h = size_y - y + return [x, y, w, h] + + +def confine_plane(image, z): + if z < 0: + z = 0 + else: + max_z = image.getSizeZ() - 1 + if z > max_z: + z = max_z + return z + + +def confine_frame(image, t): + if t < 0: + t = 0 + else: + max_t = image.getSizeT() - 1 + if t > max_t: + t = max_t + return t + + +def download_plane_as_tiff(image, tile, z, c, t, fname): + pixels = image.getPrimaryPixels() + selection = pixels.getTile(theZ=z, theT=t, theC=c, tile=tile) + + if fname[-5:] != '.tiff': + fname += '.tiff' + plt.imsave(fname, selection) + + +def download_image_data( + image_ids, + channel=None, z_stack=0, frame=0, + coord=(0, 0), width=0, height=0, region_spec='rectangle', + skip_failed=False +): + + # connect to idr + conn = BlitzGateway('public', 'public', + host='idr.openmicroscopy.org', + secure=True) + conn.connect() + + try: + prefix = 'image-' + for image_id in image_ids: + if image_id[:len(prefix)] == prefix: + image_id = image_id[len(prefix):] + image_id = int(image_id) + image = conn.getObject("Image", image_id) + + if image is None: + image_warning_id = 'Image-ID: {0}'.format(image_id) + if skip_failed: + warn( + 'Unable to find an image with this ID in the ' + 'database. Skipping download!', + image_warning_id + ) + continue + raise ValueError( + '{0}: Unable to find an image with this ID in the ' + 'database. Aborting!' + .format(image_warning_id) + ) + + image_name = os.path.splitext(image.getName())[0] + image_warning_id = '{0} (ID: {1})'.format( + image_name, image_id + ) + + if region_spec == 'rectangle': + tile = get_clipping_region(image, *coord, width, height) + elif region_spec == 'center': + tile = get_clipping_region( + image, + *_center_to_ul(*coord, width, height) + ) + else: + raise ValueError( + 'Got unknown value "{0}" as region_spec argument' + .format(region_spec) + ) + if tile[2] < width or tile[3] < height: + # The downloaded image region will have smaller dimensions + # than the specified width x height. + warn( + 'Downloaded image dimensions ({0} x {1}) will be smaller ' + 'than the specified width and height ({2} x {3}).' + .format(tile[2], tile[3], width, height), + image_warning_id + ) + + ori_z, z_stack = z_stack, confine_plane(image, z_stack) + if z_stack != ori_z: + warn( + 'Specified image plane ({0}) is out of bounds. Using {1} ' + 'instead.' + .format(ori_z, z_stack), + image_warning_id + ) + + ori_frame, frame = frame, confine_frame(image, frame) + if frame != ori_frame: + warn( + 'Specified image frame ({0}) is out of bounds. Using ' + 'frame {1} instead.' + .format(ori_frame, frame), + image_warning_id + ) + # Get the channel index. If the index is not valid, skip the image + if channel is None: + channel_index = 0 + num_channels = image.getSizeC() + if num_channels > 1: + warn( + 'No specific channel selected for multi-channel ' + 'image. Using first of {0} channels.' + .format(num_channels), + image_warning_id + ) + else: + channel_index = find_channel_index(image, channel) + if channel_index == -1: + raise ValueError( + '"{0}" is not a known channel name for image {1}' + .format(channel, image.getName()) + ) + + # download and save the region as TIFF + fname = '_'.join( + [image_name, str(image_id)] + [str(x) for x in tile] + ) + download_plane_as_tiff(image, tile, z_stack, channel_index, frame, fname) + finally: + # Close the connection + conn.close() + + +def _center_to_ul(center_x, center_y, width, height): + if width > 0: + ext_x = (width - 1) // 2 + ul_x = max([center_x - ext_x, 0]) + width = center_x + ext_x + 1 - ul_x + else: + ul_x = 0 + if height > 0: + ext_y = (height - 1) // 2 + ul_y = max([center_y - ext_y, 0]) + height = center_y + ext_y + 1 - ul_y + else: + ul_y = 0 + return ul_x, ul_y, width, height + + +if __name__ == "__main__": + p = argparse.ArgumentParser() + p.add_argument( + 'image_ids', nargs='*', default=[], + help='one or more IDR image ids for which to retrieve data (default: ' + 'read ids from stdin).' + ) + p.add_argument( + '-c', '--channel', + help='name of the channel to retrieve data for ' + '(note: the first channel of each image will be downloaded if ' + 'left unspecified)' + ) + region = p.add_mutually_exclusive_group() + region.add_argument( + '--rectangle', nargs=4, type=int, default=argparse.SUPPRESS, + help='specify a clipping region for the image as x y width height, ' + 'where x and y give the upper left coordinate of the rectangle ' + 'to clip to. Set width and height to 0 to extend the rectangle ' + 'to the actual size of the image.' + ) + region.add_argument( + '--center', nargs=4, type=int, default=argparse.SUPPRESS, + help='specify a clipping region for the image as x y width height, ' + 'where x and y define the center of a width x height rectangle. ' + 'Set either width or height to 0 to extend the region to the ' + 'actual size of the image along the x- or y-axis.\n' + 'Note: Even values for width and height will be rounded down to ' + 'the nearest odd number.' + ) + p.add_argument( + '-f', '--frame', type=int, default=0 + ) + p.add_argument( + '-z', '--z-stack', type=int, default=0 + ) + p.add_argument( + '--skip-failed', action='store_true' + ) + args = p.parse_args() + if not args.image_ids: + args.image_ids = sys.stdin.read().split() + if 'center' in args: + args.coord, args.width, args.height = ( + args.center[:2], args.center[2], args.center[3] + ) + args.region_spec = 'center' + del args.center + elif 'rectangle' in args: + args.coord, args.width, args.height = ( + args.rectangle[:2], args.rectangle[2], args.rectangle[3] + ) + args.region_spec = 'rectangle' + del args.rectangle + download_image_data(**vars(args))
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/idr_download_by_ids.xml Sat Feb 08 13:24:39 2020 -0500 @@ -0,0 +1,257 @@ +<?xml version="1.0"?> +<tool id="idr_download_by_ids" name="IDR Download" version="0.9" profile="18.09"> + <description>- download images from the Image Data Resource using image IDs</description> + <macros> + <xml name="region_spec" token_pos="upper-left corner"> + <param name="x_coord" type="integer" value="0" min="0" + label="x-coordinate of region @POS@" /> + <param name="y_coord" type="integer" value="0" min="0" + label="y-coordinate of region @POS@" /> + <param name="width" type="integer" value="0" min="0" + label="Region width" + help="Set to zero to extend the rectangle maximally along the x-axis of the image" /> + <param name="height" type="integer" value="0" min="0" + label="Region height" + help="Set to zero to extend the rectangle maximally along the y-axis of the image" /> + </xml> + </macros> + <requirements> + <requirement type="package" version="5.6.0">python-omero</requirement> + <requirement type="package" version="3.1.3">matplotlib</requirement> + </requirements> + <command detect_errors="exit_code"><![CDATA[ + mkdir downloads && cd downloads && + #if str($image_ids.source) == 'link': + python -c 'print("${image_ids.id_spec}".replace(",", "|").split("?show=")[-1].replace("|", "\n"))' + ## https://idr.openmicroscopy.org/webclient/?show=image-3426274|image-3426275|image-3426276|image-3426277 + #else: + cat '${image_ids.id_spec}' + #end if + | python '$__tool_directory__/idr_download_by_ids.py' + #set $channel = str($channel).strip() + #if $channel: + -c '$channel' + #end if + -f $frame + -z $z_section + #if str($clip_image.select): + ${clip_image.select} ${clip_image.x_coord} ${clip_image.y_coord} ${clip_image.width} ${clip_image.height} + #end if + $skip_failed + + 2> >(tee -a $out_log >&2) + ]]></command> +<inputs> + <conditional name="image_ids"> + <param name="source" type="select" label="How would you like to specify the IDs of images to download?"> + <option value="link">As text (comma-separated list of IDs or a valid IDR link)</option> + <option value="dataset">As a dataset (one image ID per line)</option> + </param> + <when value="link"> + <param name="id_spec" type="text" + label="Image IDs to download" + help="You can enter a single image-id, or a comma (or '|')-separated list of IDs. Alternatively, you can paste here a link to an image selection obtained through the IDR webclient."> + <sanitizer> + <valid><add value="|" /></valid> + </sanitizer> + </param> + </when> + <when value="dataset"> + <param name="id_spec" type="data" format="txt" + label="Select a dataset with image IDs (one per line)" /> + </when> + </conditional> + <param name="channel" type="text" + label="Name of the channel to download" + help="For all image IDs only the specified channel will be downloaded. If left empty, the first channel (whatever this is) will be downloaded by default." /> + <param name="z_section" type="integer" value="0" min="0" + label="z-plane of images to download" /> + <param name="frame" type="integer" value="0" min="0" + label="Image frame to download" /> + <conditional name="clip_image"> + <param name="select" type="select" + label="Limit the download to a selected region of the image?"> + <option value="">No, download the entire image plane</option> + <option value="--rectangle">Specify a region using its upper-left corner</option> + <option value="--center">Specify a width x height region around a central point</option> + </param> + <when value="" /> + <when value="--rectangle"> + <expand macro="region_spec" /> + </when> + <when value="--center"> + <expand macro="region_spec" pos="center" /> + </when> + </conditional> + <param name="skip_failed" type="boolean" checked="false" truevalue="--skip-failed" falsevalue="" + label="Skip failed retrievals?" + help="By default the tool will fail with an error on the first non-retrievable image ID. Here, you can choose to skip non-retrievable image IDs and continue downloading the available ones instead. The error log will contain warnings about failed IDs in this case." /> + </inputs> + <outputs> + <data name="out_log" format="txt" label="${tool.name} error log" /> + <collection name="output_file" type="list"> + <discover_datasets pattern="__name_and_ext__" directory="downloads" /> + </collection> + </outputs> + <tests> + <test> + <param name="source" value="dataset" /> + <param name="id_spec" value="ids.txt" /> + <param name="channel" value="615.0" /> + <conditional name="clip_image"> + <param name="select" value="--rectangle" /> + <param name="x_coord" value="3" /> + <param name="y_coord" value="3" /> + <param name="width" value="5" /> + <param name="height" value="5" /> + </conditional> + <param name="frame" value="2" /> + <output_collection name="output_file" type="list"> + <element name="Centrin_PCNT_Cep215_20110506_Fri-1545_0_SIR_PRJ_1884807_3_3_5_5" file="test1.tiff"/> + </output_collection> + </test> + <test> + <param name="source" value="link" /> + <param name="id_spec" value="1884807" /> + <param name="channel" value="615.0" /> + <conditional name="clip_image"> + <param name="select" value="--rectangle" /> + <param name="x_coord" value="3" /> + <param name="y_coord" value="3" /> + <param name="width" value="5" /> + <param name="height" value="5" /> + </conditional> + <param name="frame" value="2" /> + <output_collection name="output_file" type="list"> + <element name="Centrin_PCNT_Cep215_20110506_Fri-1545_0_SIR_PRJ_1884807_3_3_5_5" file="test1.tiff"/> + </output_collection> + </test> + <test> + <param name="source" value="link" /> + <param name="id_spec" value="1884807" /> + <param name="channel" value="615.0" /> + <conditional name="clip_image"> + <param name="select" value="--center" /> + <param name="x_coord" value="5" /> + <param name="y_coord" value="5" /> + <param name="width" value="5" /> + <param name="height" value="5" /> + </conditional> + <param name="frame" value="2" /> + <output_collection name="output_file" type="list"> + <element name="Centrin_PCNT_Cep215_20110506_Fri-1545_0_SIR_PRJ_1884807_3_3_5_5" file="test1.tiff"/> + </output_collection> + </test> + <test> + <param name="source" value="link" /> + <param name="id_spec" value="https://idr.openmicroscopy.org/webclient/?show=image-9036708|image-9036710|image-9036711" /> + <param name="channel" value="Spo20(51-91)" /> + <param name="z_section" value="4" /> + <conditional name="clip_image"> + <param name="select" value="" /> + </conditional> + <param name="frame" value="20" /> + <output_collection name="output_file" type="list" count="3"> + <element name="171101_LeadingEdgeDeletionPSMMovies01_15_R3D_9036711_0_0_1024_1024"> + <assert_contents> + <has_size value="4194510" /> + </assert_contents> + </element> + </output_collection> + <output name="out_log"> + <assert_contents> + <has_text text="(ID: 9036710): Specified image plane (4) is out of bounds"/> + <has_n_lines n="1"/> + </assert_contents> + </output> + </test> + <test expect_failure="true"> + <!-- Test behavior with non-existing image-ID 9036708999 --> + <param name="source" value="link" /> + <param name="id_spec" value="https://idr.openmicroscopy.org/webclient/?show=image-9036708999|image-9036710|image-9036711" /> + <param name="channel" value="Spo20(51-91)" /> + <param name="z_section" value="4" /> + <conditional name="clip_image"> + <param name="select" value="" /> + </conditional> + <param name="frame" value="20" /> + </test> + <test> + <!-- Repeat test with non-existing image-ID 9036708999, + but use skip-failed option --> + <param name="source" value="link" /> + <param name="id_spec" value="https://idr.openmicroscopy.org/webclient/?show=image-9036708999|image-9036710|image-9036711" /> + <param name="channel" value="Spo20(51-91)" /> + <param name="z_section" value="4" /> + <conditional name="clip_image"> + <param name="select" value="" /> + </conditional> + <param name="frame" value="20" /> + <param name="skip_failed" value="true" /> + <output_collection name="output_file" type="list" count="2"> + <element name="171101_LeadingEdgeDeletionPSMMovies01_15_R3D_9036711_0_0_1024_1024"> + <assert_contents> + <has_size value="4194510" /> + </assert_contents> + </element> + </output_collection> + <output name="out_log"> + <assert_contents> + <has_text text="9036708999: Unable to find an image with this ID in the database. Skipping download!" /> + <has_text text="(ID: 9036710): Specified image plane (4) is out of bounds" /> + <has_n_lines n="2"/> + </assert_contents> + </output> + </test> + </tests> + <help><![CDATA[ +Download image data from the IDR_ (Image Data Resource) - a public repository +of reference image datasets from published scientific studies. + +.. _IDR: https://idr.openmicroscopy.org/about/ + +----- + +.. class:: infomark + +**Input** + +A set of image IDs as can be obtained from the IDR webclient_ like this: + +1. select the images you want to download +2. click on the 'Link' button on the top right of the page and copy the provided + URL +3. paste the URL into the input field of this tool, for example: :: + + https://idr.openmicroscopy.org/webclient/?show=image-9036708|image-9036710|image-9036711 + +Alternatively, you can simply provide a list of known image IDs in the text +input field (comma or '|'-separated ), or as an input file (each ID on a +separate line). + +Most images in the IDR have more than two dimensions. Thus, there are +parameters available which allow you to select a particular recording channel, +z-plane or time frame to download. +You can also select a region which will be used to crop the images, rather +than downloading the entire file. + +.. _webclient: https://idr.openmicroscopy.org/ + + +----- + +.. class:: infomark + +**Output** + +Downloaded images will be saved in TIFF format. If you selected multiple image +IDs to download, these will be saved as elements of a collection. + +The accompanying error log dataset may hold valuable information about requests +that could not be fulfilled exactly as specified, or that were found to be +ambiguous. + ]]></help> + <citations> + <citation type="doi">10.1038/nmeth.4326</citation> + </citations> +</tool>