| Previous changeset 3:c496306c1cba (2022-10-08) Next changeset 5:ac55e2f4d9e3 (2023-11-13) |
|
Commit message:
planemo upload for repository https://github.com/BMCV/galaxy-image-analysis/tools/segmetrics/ commit 075271cee9cb9c2625c04dbefd903cdea6e74724 |
|
modified:
run-segmetrics.py segmetrics.xml test-data/results1.tsv test-data/results2.tsv |
|
removed:
test-data/results1.csv test-data/results2.csv |
| b |
| diff -r c496306c1cba -r 7989264b5780 run-segmetrics.py --- a/run-segmetrics.py Sat Oct 08 21:54:40 2022 +0000 +++ b/run-segmetrics.py Tue Jun 20 21:40:31 2023 +0000 |
| [ |
| @@ -1,5 +1,5 @@ """ -Copyright 2022 Leonid Kostrykin, Biomedical Computer Vision Group, Heidelberg University. +Copyright 2022-2023 Leonid Kostrykin, Biomedical Computer Vision Group, Heidelberg University. Distributed under the MIT license. See file LICENSE for detail or copy at https://opensource.org/licenses/MIT @@ -7,127 +7,50 @@ """ import argparse -import csv -import itertools import pathlib +import subprocess import tempfile import zipfile -import numpy as np -import segmetrics as sm -import skimage.io +import pandas as pd -measures = [ - ('dice', 'Dice', sm.regional.Dice()), - ('seg', 'SEG', sm.regional.ISBIScore()), - ('jc', 'Jaccard coefficient', sm.regional.JaccardSimilarityIndex()), - ('ji', 'Jaccard index', sm.regional.JaccardIndex()), - ('ri', 'Rand index', sm.regional.RandIndex()), - ('ari', 'Adjusted Rand index', sm.regional.AdjustedRandIndex()), - ('hsd_sym', 'HSD (sym)', sm.boundary.Hausdorff('sym')), - ('hsd_e2a', 'HSD (e2a)', sm.boundary.Hausdorff('e2a')), - ('hsd_a2e', 'HSD (a2e)', sm.boundary.Hausdorff('a2e')), - ('nsd', 'NSD', sm.boundary.NSD()), - ('o_hsd_sym', 'Ob. HSD (sym)', sm.boundary.ObjectBasedDistance(sm.boundary.Hausdorff('sym'))), - ('o_hsd_e2a', 'Ob. HSD (e2a)', sm.boundary.ObjectBasedDistance(sm.boundary.Hausdorff('e2a'))), - ('o_hsd_a2e', 'Ob. HSD (a2e)', sm.boundary.ObjectBasedDistance(sm.boundary.Hausdorff('a2e'))), - ('o_nsd', 'Ob. NSD', sm.boundary.ObjectBasedDistance(sm.boundary.NSD())), - ('fs', 'Split', sm.detection.FalseSplit()), - ('fm', 'Merge', sm.detection.FalseMerge()), - ('fp', 'Spurious', sm.detection.FalsePositive()), - ('fn', 'Missing', sm.detection.FalseNegative()), -] - - -def process_batch(study, gt_filelist, seg_filelist, namelist, gt_is_unique, seg_is_unique): - for gt_filename, seg_filename, name in zip(gt_filelist, seg_filelist, namelist): - img_ref = skimage.io.imread(gt_filename) - img_seg = skimage.io.imread(seg_filename) - study.set_expected(img_ref, unique=gt_is_unique) - study.process(img_seg, unique=seg_is_unique, chunk_id=name) - - -def aggregate(measure, values): - fnc = np.sum if measure.ACCUMULATIVE else np.mean - return fnc(values) - - -def is_zip_filepath(filepath): - return filepath.lower().endswith('.zip') - - -def is_image_filepath(filepath): - suffixes = ['png', 'tif', 'tiff'] - return any((filepath.lower().endswith(f'.{suffix}') for suffix in suffixes)) +def process_batch(seg_dir, seg_file, gt_file, tsv_output_file, recursive, gt_unique, seg_unique, measures): + with tempfile.NamedTemporaryFile() as csv_output_file: + cmd = ['python', '-m', 'segmetrics.cli', str(seg_dir), str(seg_file), str(gt_file), str(csv_output_file.name), '--semicolon'] + if recursive: + cmd.append('--recursive') + if gt_unique: + cmd.append('--gt-unique') + if seg_unique: + cmd.append('--seg-unique') + cmd += measures + subprocess.run(cmd, check=True) + df = pd.read_csv(csv_output_file.name, sep=';') + df.to_csv(str(tsv_output_file), sep='\t', index=False) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Image segmentation and object detection performance measures for 2-D image data') parser.add_argument('input_seg', help='Path to the segmented image or image archive (ZIP)') parser.add_argument('input_gt', help='Path to the ground truth image or image archive (ZIP)') - parser.add_argument('results', help='Path to the results file (CSV)') + parser.add_argument('results', help='Path to the results file (TSV)') parser.add_argument('-unzip', action='store_true') parser.add_argument('-seg_unique', action='store_true') parser.add_argument('-gt_unique', action='store_true') - for measure in measures: - parser.add_argument(f'-measure-{measure[0]}', action='store_true', help=f'Include {measure[1]}') - + parser.add_argument('measures', nargs='+', type=str, help='list of performance measures') args = parser.parse_args() - study = sm.study.Study() - - used_measures = [] - for measure in measures: - if getattr(args, f'measure_{measure[0]}'): - used_measures.append(measure) - study.add_measure(measure[2], measure[1]) if args.unzip: zipfile_seg = zipfile.ZipFile(args.input_seg) zipfile_gt = zipfile.ZipFile(args.input_gt) - namelist = [filepath for filepath in zipfile_seg.namelist() if is_image_filepath(filepath) and filepath in zipfile_gt.namelist()] - print('namelist:', namelist) with tempfile.TemporaryDirectory() as tmpdir: basepath = pathlib.Path(tmpdir) gt_path, seg_path = basepath / 'gt', basepath / 'seg' zipfile_seg.extractall(str(seg_path)) zipfile_gt.extractall(str(gt_path)) - gt_filelist, seg_filelist = list(), list() - for filepath in namelist: - seg_filelist.append(str(seg_path / filepath)) - gt_filelist.append(str(gt_path / filepath)) - process_batch(study, gt_filelist, seg_filelist, namelist, args.gt_unique, args.seg_unique) + process_batch(seg_dir=seg_path, seg_file=rf'^{seg_path}/(.+\.(?:png|PNG|tif|TIF|tiff|TIFF))$', gt_file=gt_path / r'\1', tsv_output_file=args.results, recursive=True, gt_unique=args.gt_unique, seg_unique=args.seg_unique, measures=args.measures) else: - namelist = [''] - process_batch(study, [args.input_gt], [args.input_seg], namelist, args.gt_unique, args.seg_unique) - - # define header - rows = [[''] + [measure[1] for measure in used_measures]] - - # define rows - if len(namelist) > 1: - for chunk_id in namelist: - row = [chunk_id] - for measure in used_measures: - measure_name = measure[1] - measure = study.measures[measure_name] - chunks = study.results[measure_name] - row += [aggregate(measure, chunks[chunk_id])] - rows.append(row) - - # define footer - rows.append(['']) - for measure in used_measures: - measure_name = measure[1] - measure = study.measures[measure_name] - chunks = study.results[measure_name] - values = list(itertools.chain(*[chunks[chunk_id] for chunk_id in chunks])) - val = aggregate(measure, values) - rows[-1].append(val) - - # write results - with open(args.results, 'w', newline='') as fout: - csv_writer = csv.writer(fout, delimiter='\t', quotechar='"', quoting=csv.QUOTE_MINIMAL) - for row in rows: - csv_writer.writerow(row) + seg_path = pathlib.Path(args.input_seg) + process_batch(seg_dir=seg_path.parent, seg_file=seg_path, gt_file=args.input_gt, tsv_output_file=args.results, recursive=False, gt_unique=args.gt_unique, seg_unique=args.seg_unique, measures=args.measures) |
| b |
| diff -r c496306c1cba -r 7989264b5780 segmetrics.xml --- a/segmetrics.xml Sat Oct 08 21:54:40 2022 +0000 +++ b/segmetrics.xml Tue Jun 20 21:40:31 2023 +0000 |
| [ |
| b'@@ -1,8 +1,8 @@\n-<tool id="ip_segmetrics" name="SegMetrics" version="0.11.3-2" profile="20.05">\r\n+<tool id="ip_segmetrics" name="SegMetrics" version="1.4.0-1" profile="20.05">\r\n <description>image segmentation and object detection performance measures</description>\r\n <requirements> \r\n- <requirement type="package" version="0.11.3">segmetrics</requirement>\r\n- <requirement type="package" version="0.18.1">scikit-image</requirement>\r\n+ <requirement type="package" version="1.4">segmetrics</requirement>\r\n+ <requirement type="package" version="0.23.4">pandas</requirement>\r\n </requirements>\r\n <command detect_errors="aggressive">\r\n <![CDATA[\r\n@@ -13,24 +13,28 @@\n $unzip\r\n $is_seg_unique\r\n $is_gt_unique\r\n- $measures.dice\r\n- $measures.seg\r\n- $measures.jc\r\n- $measures.ji\r\n- $measures.ri\r\n- $measures.ari\r\n- $measures.hsd_sym\r\n- $measures.hsd_e2a\r\n- $measures.hsd_a2e\r\n- $measures.nsd\r\n- $measures.o_hsd_sym\r\n- $measures.o_hsd_e2a\r\n- $measures.o_hsd_a2e\r\n- $measures.o_nsd\r\n- $measures.fs\r\n- $measures.fm\r\n- $measures.fp\r\n- $measures.fn\r\n+ #for $m in $measures\r\n+ #set $kwargs = \'\'\r\n+ #set $suffix = \'\'\r\n+ #set $is_distance = False\r\n+ #if str($m.measure_type.measure_type_selector) == \'ISBIScore\':\r\n+ #set $kwargs = \'min_ref_size=\' + str($m.measure_type.min_ref_size) + \', \'\r\n+ #elif str($m.measure_type.measure_type_selector) == \'Hausdorff\':\r\n+ #set $kwargs = \'mode="\' + str($m.measure_type.mode) + \'", quantile=\' + str($m.measure_type.quantile) + \', \'\r\n+ #set $is_distance = True\r\n+ #elif str($m.measure_type.measure_type_selector) == \'NSD\':\r\n+ #set $is_distance = True\r\n+ #end if\r\n+ #if $is_distance:\r\n+ #if $m.measure_type.object_based:\r\n+ #set $suffix = \'.object_based()\'\r\n+ #end if\r\n+ #end if\r\n+ #if str($m.measure_type.aggregation) != \'\':\r\n+ #set $kwargs = \'aggregation="\' + str($m.measure_type.aggregation) + \'", \' + $kwargs\r\n+ #end if\r\n+ \'sm.${m.measure_type.measure_type_selector}(${kwargs})${suffix}\'\r\n+ #end for\r\n ]]>\r\n </command>\r\n <inputs>\r\n@@ -43,46 +47,260 @@\n <param name="is_seg_unique" type="boolean" checked="false" truevalue="-seg_unique" falsevalue="" label="Segmentation is uniquely labeled" />\r\n <param name="is_gt_unique" type="boolean" checked="false" truevalue="-gt_unique" falsevalue="" label="Ground truth is uniquely labeled" />\r\n \r\n- <section name="measures" title="Performance measures" >\r\n- <param name="dice" type="boolean" checked="true" truevalue="-measure-dice" falsevalue="" label="Dice" />\r\n- <param name="seg" type="boolean" checked="true" truevalue="-measure-seg" falsevalue="" label="SEG" />\r\n- <param name="jc" type="boolean" checked="false" truevalue="-measure-jc" falsevalue="" label="Jaccard coefficient" />\r\n- <param name="ji" type="boolean" checked="true" truevalue="-measure-ji" falsevalue="" label="Jaccard index" />\r\n- <param name="ri" type="boolean" checked="false" truevalue="-measure-ri" falsevalue="" label="Rand index" />\r\n- <param name="ari" type="boolean" checked="false" truevalue="-measure-ari" falsevalue="" label="Adjusted Rand index" />\r\n- <param name="hsd_sym" type="boolean" checked="false" truevalue="-measure-hsd_sym" falsevalue="" label="Hausdorff distance (symmetric)" />\r\n- <param name="hsd_e2a" type="boolean" checked="false" truevalue="-measure-hsd_e2a" falsevalue="" label="Hausdorff distance (ground truth to segmented)" />\r\n- <param name="hsd_a2e" type="boolean" checked="false" truevalue="-measure-hsd_a2e" falsevalue="" label="Hausdorff distance (segmented to ground truth)" />\r\n- <param name="nsd" type="boolean" checked="false" truevalue="-measure-nsd" falsevalue="" label="Normalized sum of distances" /'..b'unzip" value="true"/>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="Dice" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="ISBIScore" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="JaccardCoefficient" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="JaccardIndex" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="RandIndex" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="AdjustedRandIndex" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="Hausdorff" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="Hausdorff" />\r\n+ <param name="quantile" value="0.9" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="Hausdorff" />\r\n+ <param name="object_based" value="true" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="NSD" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="NSD" />\r\n+ <param name="object_based" value="true" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="FalseSplit" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="FalseMerge" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="FalsePositive" />\r\n+ </conditional>\r\n+ </repeat>\r\n+ <repeat name="measures">\r\n+ <conditional name="measure_type">\r\n+ <param name="measure_type_selector" value="FalseNegative" />\r\n+ </conditional>\r\n+ </repeat>\r\n </test>\r\n </tests>\r\n <help>\r\n@@ -96,5 +314,6 @@\n <citation type="doi">10.1093/bioinformatics/btu080</citation>\r\n <citation type="doi">10.1109/ISBI.2009.5193098</citation>\r\n <citation type="doi">10.1109/ICIP.2003.1246871</citation>\r\n+ <citation type="doi">10.1023/A:1007975324482</citation>\r\n </citations>\r\n </tool>\r\n' |
| b |
| diff -r c496306c1cba -r 7989264b5780 test-data/results1.csv --- a/test-data/results1.csv Sat Oct 08 21:54:40 2022 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,2 +0,0 @@ -,Dice,SEG,Jaccard index,Ob. HSD (sym),Ob. NSD,Split,Merge,Spurious,Missing -,0.5473684210526316,0.291005291005291,1.6238374627624792,141.68949443548658,0.8175405481022531,0,1,2,0 |
| b |
| diff -r c496306c1cba -r 7989264b5780 test-data/results1.tsv --- a/test-data/results1.tsv Sat Oct 08 21:54:40 2022 +0000 +++ b/test-data/results1.tsv Tue Jun 20 21:40:31 2023 +0000 |
| b |
| @@ -1,2 +1,2 @@ - Dice SEG Jaccard index Ob. HSD (sym) Ob. NSD Split Merge Spurious Missing - 0.5473684210526316 0.291005291005291 1.6238374627624792 141.68949443548658 0.8175405481022531 0 1 2 0 +Sample Dice SEG Jaccard coef. Jaccard index Rand ARI HSD (sym) HSD (sym, Q=0.9) Ob. HSD (sym) NSD Ob. NSD Split Merge Spurious Missing + 0.5473684210526316 0.14285714285714285 0.37681159420289856 1.6238374627624792 0.7151668606674426 0.3776254329131262 179.0 94.0 142.42099468121046 0.5911414982164089 0.7765050343914003 1.0 0.0 0.0 2.0 |
| b |
| diff -r c496306c1cba -r 7989264b5780 test-data/results2.csv --- a/test-data/results2.csv Sat Oct 08 21:54:40 2022 +0000 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 |
| b |
| @@ -1,4 +0,0 @@ -,Dice,SEG,Jaccard index,Ob. HSD (sym),Ob. NSD,Split,Merge,Spurious,Missing -directory/img1.png,0.5473684210526316,0.14285714285714285,1.6238374627624792,142.42099468121046,0.7765050343914003,1,0,0,2 -directory/img2.png,0.5473684210526316,0.291005291005291,1.6238374627624792,141.68949443548658,0.8175405481022531,0,1,2,0 -,0.5473684210526316,0.2169312169312169,1.6238374627624792,142.0552445583485,0.7970227912468267,1,1,2,2 |
| b |
| diff -r c496306c1cba -r 7989264b5780 test-data/results2.tsv --- a/test-data/results2.tsv Sat Oct 08 21:54:40 2022 +0000 +++ b/test-data/results2.tsv Tue Jun 20 21:40:31 2023 +0000 |
| b |
| @@ -1,4 +1,4 @@ - Dice SEG Jaccard index Ob. HSD (sym) Ob. NSD Split Merge Spurious Missing -directory/img1.png 0.5473684210526316 0.14285714285714285 1.6238374627624792 142.42099468121046 0.7765050343914003 1 0 0 2 -directory/img2.png 0.5473684210526316 0.291005291005291 1.6238374627624792 141.68949443548658 0.8175405481022531 0 1 2 0 - 0.5473684210526316 0.2169312169312169 1.6238374627624792 142.0552445583485 0.7970227912468267 1 1 2 2 +Sample Dice SEG Jaccard coef. Jaccard index Rand ARI HSD (sym) HSD (sym, Q=0.9) Ob. HSD (sym) NSD Ob. NSD Split Merge Spurious Missing +directory/img1.png 0.5473684210526316 0.14285714285714285 0.37681159420289856 1.6238374627624792 0.7151668606674426 0.3776254329131262 179.0 94.0 142.42099468121046 0.5911414982164089 0.7765050343914003 1.0 0.0 0.0 2.0 +directory/img2.png 0.5473684210526316 0.291005291005291 0.37681159420289856 1.6238374627624792 0.7151668606674426 0.3776254329131262 179.0 94.0 141.6894944354866 0.8107992461737683 0.8175405481022531 0.0 1.0 2.0 0.0 + 0.5473684210526316 0.2169312169312169 0.37681159420289856 1.6238374627624792 0.7151668606674426 0.3776254329131262 179.0 94.0 142.0552445583485 0.7009703721950886 0.7970227912468267 0.5 0.5 1.0 1.0 |