| Previous changeset 0:99308601eaa6 (2021-05-19) Next changeset 2:224e0cf4aaeb (2022-09-01) |
|
Commit message:
"planemo upload commit fec9dc76b3dd17b14b02c2f04be9d30f71eba1ae" |
|
modified:
UNet2DtCycifTRAINCoreograph.py UNetCoreograph.py coreograph.xml macros.xml |
|
added:
Dockerfile LICENSE README.md images/coreographbanner.png images/coreographbannerv2.png images/coreographbannerv3.png images/coreographbannerv4.png images/coreographbannerv5.png |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e Dockerfile --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Dockerfile Fri Mar 11 23:40:51 2022 +0000 |
| b |
| @@ -0,0 +1,12 @@ +FROM tensorflow/tensorflow:1.15.0-py3 + +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata +RUN apt-get install -y python3-opencv +RUN apt-get install -y libtiff5-dev git + +RUN pip install cython scikit-image==0.14.2 matplotlib tifffile==2020.2.16 scipy==1.1.0 opencv-python==4.3.0.36 + +RUN pip install git+https://github.com/FZJ-INM1-BDA/pytiff.git@0701f28e5862c26024e8daa34201005b16db4c8f + +COPY . /app |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e LICENSE --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/LICENSE Fri Mar 11 23:40:51 2022 +0000 |
| b |
| @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 HMS-IDAC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e README.md --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.md Fri Mar 11 23:40:51 2022 +0000 |
| [ |
| @@ -0,0 +1,26 @@ + + +*Great*....yet **another** TMA dearray program. What does *this* one do? + +Coreograph uses UNet, a deep learning model, to identify complete/incomplete tissue cores on a tissue microarray. It has been trained on 9 TMA slides of different sizes and tissue types. + +<img src="/images/raw.jpg" width="425" height="315" /> <img src="/images/probmap.jpg" width="425" height="315" /> + +Training sets were acquired at 0.2micron/pixel resolution and downsampled 1/32 times to speed up performance. Once the center of each core has been identifed, active contours is used to generate a tissue mask of each core that can aid downstream single cell segmentation. A GPU is not required but will reduce computation time. + +*Coreograph exports these files:** +1. individual cores as tiff stacks with user-selectable channel ranges +2. binary tissue masks (saved in the 'mask' subfolder) +3. a TMA map showing the labels and outlines of each core for quality control purposes + + + +*Instructions for use:** +`python UNetCoreograph.py` +1. `--imagePath` : the path to the image file. Should be tif or ome.tif +2. `--outputPath` : the path to save the above-mentioned files +3. `--downsampleFactor` : how many times to downsample the raw image file. Default is 5 times to match the training data. +4. `--channel` : which is the channel to feed into UNet and generate probabiltiy maps from. This is usually a DAPI channel +5. `--buffer` : the extra space around a core before cropping it. A value of 2 means there is twice the width of the core added as buffer around it. 2 is default +6. `--outputChan` : a range of channels to be exported. -1 is default and will export all channels (takes awhile). Select a single channel or a continuous range. --outputChan 0 10 will export channel 0 up to (and including) channel 10 + |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e UNet2DtCycifTRAINCoreograph.py --- a/UNet2DtCycifTRAINCoreograph.py Wed May 19 21:34:38 2021 +0000 +++ b/UNet2DtCycifTRAINCoreograph.py Fri Mar 11 23:40:51 2022 +0000 |
| [ |
| @@ -524,63 +524,6 @@ UNet2D.train(imPath, logPath, modelPath, pmPath, 2053, 513 , 641, True, 10, 1, 1) UNet2D.deploy(imPath,100,modelPath,pmPath,1,1) - # I = im2double(tifread('/home/mc457/files/CellBiology/IDAC/Marcelo/Etc/UNetTestSets/SinemSaka_NucleiSegmentation_SingleImageInferenceTest3.tif')) - # UNet2D.singleImageInferenceSetup(modelPath,0) - # J = UNet2D.singleImageInference(I,'accumulate',0) - # UNet2D.singleImageInferenceCleanup() - # # imshowlist([I,J]) - # # sys.exit(0) - # # tifwrite(np.uint8(255*I),'/home/mc457/Workspace/I1.tif') - # # tifwrite(np.uint8(255*J),'/home/mc457/Workspace/I2.tif') - # K = np.zeros((2,I.shape[0],I.shape[1])) - # K[0,:,:] = I - # K[1,:,:] = J - # tifwrite(np.uint8(255*K),'/home/mc457/Workspace/Sinem_NucSeg.tif') - - # UNet2D.singleImageInferenceSetup(modelPath,0) - # imagePath = 'Y://sorger//data//RareCyte//Connor//Topacio_P2_AF//ashlar//C0078' - # - # fileList = glob.glob(imagePath + '//registration//C0078.ome.tif') - # print(fileList) - # for iFile in fileList: - # fileName = os.path.basename(iFile) - # fileNamePrefix = fileName.split(os.extsep, 1) - # I = im2double(tifffile.imread(iFile, key=0)) - # hsize = int((float(I.shape[0])*float(0.75))) - # vsize = int((float(I.shape[1])*float(0.75))) - # I = resize(I,(hsize,vsize)) - # J = UNet2D.singleImageInference(I,'accumulate',1) - # K = np.zeros((3,I.shape[0],I.shape[1])) - # K[2,:,:] = I - # K[0,:,:] = J - # J = UNet2D.singleImageInference(I, 'accumulate', 2) - # K[1, :, :] = J - # outputPath = imagePath + '//prob_maps' - # if not os.path.exists(outputPath): - # os.makedirs(outputPath) - # tifwrite(np.uint8(255*K),outputPath + '//' + fileNamePrefix[0] +'_NucSeg.tif') - # UNet2D.singleImageInferenceCleanup() - # ----- test 2 ----- - # imPath = '/home/mc457/files/CellBiology/IDAC/Marcelo/Etc/UNetTestSets/ClarenceYapp_NucleiSegmentation' - # UNet2D.setup(128,1,2,8,2,2,3,1,0.1,3,4) - # UNet2D.train(imPath,logPath,modelPath,pmPath,800,100,100,False,10,1) - # UNet2D.deploy(imPath,100,modelPath,pmPath,1) - - - # ----- test 3 ----- - - # imPath = '/home/mc457/files/CellBiology/IDAC/Marcelo/Etc/UNetTestSets/CarmanLi_CellTypeSegmentation' - # # UNet2D.setup(256,1,2,8,2,2,3,1,0.1,3,4) - # # UNet2D.train(imPath,logPath,modelPath,pmPath,1400,100,164,False,10000,1) - # UNet2D.deploy(imPath,164,modelPath,pmPath,1) - - - # ----- test 4 ----- - - # imPath = '/home/cicconet/Downloads/TrainSet1' - # UNet2D.setup(64,1,2,8,2,2,3,1,0.1,3,4) - # UNet2D.train(imPath,logPath,modelPath,pmPath,200,8,8,False,2000,1,0) - # # UNet2D.deploy(imPath,164,modelPath,pmPath,1) \ No newline at end of file |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e UNetCoreograph.py --- a/UNetCoreograph.py Wed May 19 21:34:38 2021 +0000 +++ b/UNetCoreograph.py Fri Mar 11 23:40:51 2022 +0000 |
| [ |
| b'@@ -3,6 +3,9 @@\n import shutil\r\n import scipy.io as sio\r\n import os\r\n+os.environ[\'TF_CPP_MIN_LOG_LEVEL\'] = \'2\'\r\n+import logging\r\n+logging.getLogger(\'tensorflow\').setLevel(logging.FATAL)\r\n import skimage.exposure as sk\r\n import cv2\r\n import argparse\r\n@@ -14,10 +17,11 @@\n from skimage.segmentation import chan_vese, find_boundaries, morphological_chan_vese\r\n from skimage.measure import regionprops,label, find_contours\r\n from skimage.transform import resize\r\n-from skimage.filters import gaussian\r\n+from skimage.filters import gaussian, threshold_otsu\r\n from skimage.feature import peak_local_max,blob_log\r\n-from skimage.color import label2rgb\r\n+from skimage.color import gray2rgb as gray2rgb\r\n import skimage.io as skio\r\n+from scipy.ndimage.morphology import binary_fill_holes\r\n from skimage import img_as_bool\r\n from skimage.draw import circle_perimeter\r\n from scipy.ndimage.filters import uniform_filter\r\n@@ -525,27 +529,27 @@\n \r\n \r\n def identifyNumChan(path):\r\n- tiff = tifffile.TiffFile(path)\r\n- shape = tiff.pages[0].shape\r\n- numChan=None\r\n- for i, page in enumerate(tiff.pages):\r\n- if page.shape != shape:\r\n- numChan = i\r\n- return numChan\r\n- break\r\n-# else:\r\n-# raise Exception("Did not find any pyramid subresolutions") \r\n \r\n- if not numChan:\r\n- numChan = len(tiff.pages)\r\n- return numChan\r\n+\ts = tifffile.TiffFile(path).series[0]\r\n+\treturn s.shape[0] if len(s.shape) > 2 else 1\r\n+ # shape = tiff.pages[0].shape\r\n+ # tiff = tifffile.TiffFile(path)\r\n+ # for i, page in enumerate(tiff.pages):\r\n+\t# print(page.shape)\r\n+\t# if page.shape != shape:\r\n+\t# \t numChan = i\r\n+\t# \t return numChan\r\n+\t# \t break\r\n+#\t else:\r\n+#\t\t raise Exception("Did not find any pyramid subresolutions") \r\n+\r\n \r\n def getProbMaps(I,dsFactor,modelPath):\r\n hsize = int((float(I.shape[0]) * float(0.5)))\r\n vsize = int((float(I.shape[1]) * float(0.5)))\r\n imagesub = cv2.resize(I,(vsize,hsize),cv2.INTER_NEAREST)\r\n \r\n- UNet2D.singleImageInferenceSetup(modelPath, 1)\r\n+ UNet2D.singleImageInferenceSetup(modelPath, 0)\r\n \r\n for iSize in range(dsFactor):\r\n \t hsize = int((float(I.shape[0]) * float(0.5)))\r\n@@ -557,42 +561,22 @@\n UNet2D.singleImageInferenceCleanup()\r\n return probMaps \r\n \r\n-def coreSegmenterOutput(I,probMap,initialmask,preBlur,findCenter):\r\n+def coreSegmenterOutput(I,initialmask,findCenter):\r\n \thsize = int((float(I.shape[0]) * float(0.1)))\r\n \tvsize = int((float(I.shape[1]) * float(0.1)))\r\n \tnucGF = cv2.resize(I,(vsize,hsize),cv2.INTER_CUBIC)\r\n-#\tIrs = cv2.resize(I,(vsize,hsize),cv2.INTER_CUBIC)\r\n-#\tI=I.astype(np.float)\r\n-#\tr,c = I.shape\r\n-#\tI+=np.random.rand(r,c)*1e-6\r\n-#\tc1 = uniform_filter(I, 3, mode=\'reflect\')\r\n-#\tc2 = uniform_filter(I*I, 3, mode=\'reflect\')\r\n-#\tnucGF = np.sqrt(c2 - c1*c1)*np.sqrt(9./8)\r\n-#\tnucGF[np.isnan(nucGF)]=0\r\n \t#active contours\r\n \thsize = int(float(nucGF.shape[0]))\r\n \tvsize = int(float(nucGF.shape[1]))\r\n \tinitialmask = cv2.resize(initialmask,(vsize,hsize),cv2.INTER_NEAREST)\r\n \tinitialmask = dilation(initialmask,disk(15)) >0\r\n-\t\t\r\n-#\tinit=np.argwhere(eroded>0)\r\n+\r\n \tnucGF = gaussian(nucGF,0.7)\r\n \tnucGF=nucGF/np.amax(nucGF)\r\n \t\r\n- \r\n-#\tinitialmask = nucGF>0\r\n \tnuclearMask = morphological_chan_vese(nucGF, 100, init_level_set=initialmask, smoothing=10,lambda1=1.001, lambda2=1)\r\n \t\r\n-#\tnuclearMask = chan_vese(nucGF, mu=1.5, lambda1=6, lambda2=1, tol=0.0005, max_iter=2000, dt=15, init_level_set=initialmask, extended_output=True)\t\r\n-#\tnuclearMask = nuclearMask[0]\r\n- \r\n-\t\r\n \tTMAmask = nuclearMask\r\n-#\tnMaskDist =distance_transform_edt(nuclearMask)\r\n-#\tfgm = peak_local_max(h_maxima(nMaskDist, 2*preBlur),indices =False)\r\n-#\tmarkers= np.logical_or(erosion(1-nuclearMask,disk(3)),fgm)\r\n-#\tTMAmask=watershed(-nMaskDist,label(markers),watershed_line=True)\r\n-#\tTMAmask = nuclearMask*(TMAmask>0)\r\n \tTMAmask = remove_small_objects(TMAmask>0,round(TMAmask.shape[0])*round(TMAmask.shape[1])*0.005)\r\n \tTMAlabel = label(TMAmask)\r\n # find object closest to center\r\n@@ -632,7 '..b'31 @@\n \t\t\ty[iCore]=1\r\n \r\n \t\tbbox[iCore] = [round(x[iCore]), round(y[iCore]), round(xLim[iCore]), round(yLim[iCore])]\r\n-\t\t\r\n+\t\tcoreStack = np.zeros((outputChan[1]-outputChan[0]+1,np.int(round(yLim[iCore])-round(y[iCore])-1),np.int(round(xLim[iCore])-round(x[iCore])-1)),dtype=\'uint16\')\r\n+\r\n \t\tfor iChan in range(outputChan[0],outputChan[1]+1):\r\n \t\t\twith pytiff.Tiff(imagePath, "r", encoding=\'utf-8\') as handle:\r\n \t\t\t\thandle.set_page(iChan)\r\n-\t\t\t\tcoreStack= handle[np.uint32(bbox[iCore][1]):np.uint32(bbox[iCore][3]-1), np.uint32(bbox[iCore][0]):np.uint32(bbox[iCore][2]-1)]\r\n-\t\t\tskio.imsave(outputPath + os.path.sep + str(iCore+1) + \'.tif\',coreStack,append=True)\t\r\n+\t\t\t\tcoreStack[iChan,:,:] =handle[np.uint32(bbox[iCore][1]):np.uint32(bbox[iCore][3]-1), np.uint32(bbox[iCore][0]):np.uint32(bbox[iCore][2]-1)]\r\n \r\n+\t\tskio.imsave(outputPath + os.path.sep + str(iCore+1) + \'.tif\',np.uint16(coreStack),imagej=True,bigtiff=True)\r\n \t\twith pytiff.Tiff(imagePath, "r", encoding=\'utf-8\') as handle:\r\n \t\t\thandle.set_page(args.channel)\r\n \t\t\tcoreSlice= handle[np.uint32(bbox[iCore][1]):np.uint32(bbox[iCore][3]-1), np.uint32(bbox[iCore][0]):np.uint32(bbox[iCore][2]-1)]\r\n \r\n \t\tcore = (coreLabel ==(iCore+1))\r\n-\t\tinitialmask = core[np.uint32(y[iCore]*dsFactor):np.uint32(yLim[iCore]*dsFactor),np.uint32(x[iCore]*dsFactor):np.uint32(xLim[iCore]*dsFactor)]\r\n-\t\tinitialmask = resize(initialmask,size(coreSlice),cv2.INTER_NEAREST)\r\n+\t\tinitialmask = core[np.uint32(y[iCore] * dsFactor):np.uint32(yLim[iCore] * dsFactor),\r\n+\t\t\t\t\t np.uint32(x[iCore] * dsFactor):np.uint32(xLim[iCore] * dsFactor)]\r\n+\t\tif not args.tissue:\r\n+\t\t\tinitialmask = resize(initialmask,size(coreSlice),cv2.INTER_NEAREST)\r\n \r\n-\t\tsingleProbMap = classProbs[np.uint32(y[iCore]*dsFactor):np.uint32(yLim[iCore]*dsFactor),np.uint32(x[iCore]*dsFactor):np.uint32(xLim[iCore]*dsFactor)]\r\n-\t\tsingleProbMap = resize(np.uint8(255*singleProbMap),size(coreSlice),cv2.INTER_NEAREST)\r\n-\t\tTMAmask = coreSegmenterOutput(coreSlice,singleProbMap,initialmask,coreRad/20,False) \r\n+\t\t\tsingleProbMap = classProbs[np.uint32(y[iCore]*dsFactor):np.uint32(yLim[iCore]*dsFactor),np.uint32(x[iCore]*dsFactor):np.uint32(xLim[iCore]*dsFactor)]\r\n+\t\t\tsingleProbMap = resize(np.uint8(255*singleProbMap),size(coreSlice),cv2.INTER_NEAREST)\r\n+\t\t\tTMAmask = coreSegmenterOutput(coreSlice,initialmask,False)\r\n+\t\telse:\r\n+\t\t\tIrs = resize(coreSlice,(int((float(coreSlice.shape[0]) * 0.25)),int((float(coreSlice.shape[1]) * 0.25))))\r\n+\t\t\tTMAmask = coreSegmenterOutput(Irs, np.uint8(initialmask), False)\r\n+\r\n \t\tif np.sum(TMAmask)==0:\r\n \t\t\tTMAmask = np.ones(TMAmask.shape)\r\n \t\tvsize = int(float(coreSlice.shape[0]))\r\n@@ -786,17 +785,16 @@\n \t\tmasksub = resize(resize(TMAmask,(vsize,hsize),cv2.INTER_NEAREST),(int((float(coreSlice.shape[0])*dsFactor)),int((float(coreSlice.shape[1])*dsFactor))),cv2.INTER_NEAREST)\r\n \t\tsingleMaskTMA[int(y[iCore]*dsFactor):int(y[iCore]*dsFactor)+masksub.shape[0],int(x[iCore]*dsFactor):int(x[iCore]*dsFactor)+masksub.shape[1]]=masksub\r\n \t\tmaskTMA = maskTMA + resize(singleMaskTMA,maskTMA.shape,cv2.INTER_NEAREST)\r\n-\t\tcv2.putText(imagesub, str(iCore+1), (int(P[iCore].centroid[1]),int(P[iCore].centroid[0])), 0, 0.5, (np.amax(imagesub), np.amax(imagesub), np.amax(imagesub)), 1, cv2.LINE_AA)\r\n+\r\n+\t\tcv2.putText(imagesub, str(iCore+1), (int(P[iCore].centroid[1]),int(P[iCore].centroid[0])), 0, 0.5, (0,255,0), 1, cv2.LINE_AA)\r\n \t\t\r\n \t\tskio.imsave(maskOutputPath + os.path.sep + str(iCore+1) + \'_mask.tif\',np.uint8(TMAmask))\r\n-\t\tprint(\'Segmented core \' + str(iCore+1))\t\r\n+\t\tprint(\'Segmented core/tissue \' + str(iCore+1))\r\n \t\t\r\n \tboundaries = find_boundaries(maskTMA)\r\n-\timagesub = imagesub/np.percentile(imagesub,99.9)\r\n-\timagesub[boundaries==1] = 1\r\n-\tskio.imsave(outputPath + os.path.sep + \'TMA_MAP.tif\' ,np.uint8(imagesub*255))\r\n-\tprint(\'Segmented all cores!\')\r\n-\t\r\n+\timagesub[boundaries==1] = 255\r\n+\tskio.imsave(outputPath + os.path.sep + \'TMA_MAP.tif\' ,imagesub)\r\n+\tprint(\'Segmented all cores/tissues!\')\r\n \r\n #restore GPU to 0\r\n \t#image load using tifffile\r\n' |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e coreograph.xml --- a/coreograph.xml Wed May 19 21:34:38 2021 +0000 +++ b/coreograph.xml Fri Mar 11 23:40:51 2022 +0000 |
| b |
| @@ -12,6 +12,8 @@ ln -s $source_image `basename $type_corrected`; @CMD_BEGIN@ + + python \$UNET_PATH --imagePath `basename $type_corrected` --downsampleFactor $downsamplefactor --channel $channel @@ -26,6 +28,10 @@ --cluster #end if + #if $tissue + --tissue + #end if + --outputPath .; ]]></command> @@ -39,6 +45,7 @@ <param name="sensitivity" type="float" value="0.3" label="Sensitivity"/> <!--<param name="usegrid" type="boolean" label="Use Grid"/>--> <param name="cluster" type="boolean" checked="false" label="Cluster"/> + <param name="tissue" type="boolean" checked="false" label="Tissue"/> </inputs> <outputs> |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e images/coreographbanner.png |
| b |
| Binary file images/coreographbanner.png has changed |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e images/coreographbannerv2.png |
| b |
| Binary file images/coreographbannerv2.png has changed |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e images/coreographbannerv3.png |
| b |
| Binary file images/coreographbannerv3.png has changed |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e images/coreographbannerv4.png |
| b |
| Binary file images/coreographbannerv4.png has changed |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e images/coreographbannerv5.png |
| b |
| Binary file images/coreographbannerv5.png has changed |
| b |
| diff -r 99308601eaa6 -r 57f1260ca94e macros.xml --- a/macros.xml Wed May 19 21:34:38 2021 +0000 +++ b/macros.xml Fri Mar 11 23:40:51 2022 +0000 |
| [ |
| @@ -24,6 +24,13 @@ </citations> </xml> - <token name="@VERSION@">2.2.0</token> - <token name="@CMD_BEGIN@">python ${__tool_directory__}/UNetCoreograph.py</token> + <token name="@VERSION@">2.2.8</token> + <token name="@CMD_BEGIN@"><![CDATA[ + UNET_PATH=""; + if [ -f "/app/UNetCoreograph.py" ]; then + export UNET_PATH="/app/UNetCoreograph.py"; + else + export UNET_PATH="${__tool_directory__}/UNetCoreograph.py"; + fi; + ]]></token> </macros> |
